Skip to content

GitHub Issue #1037: Calculated columns on assays don't get round tripped through folder archive #7959

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cnathe wants to merge 5 commits into
base: develop
Choose a base branch
from
Open

GitHub Issue #1037: Calculated columns on assays don't get round tripped through folder archive #7959

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
48e58df
GitHub Issue #1037: QueryImporter to track deferredMetadataFiles and …
cnathe Aug 20, 2026
1d46384
Claude code review - cleanup some warning log entries
cnathe Aug 20, 2026
9eb09f9
Restore CRLF line endings in QueryImporter.java
cnathe Aug 20, 2026
a7c495d
Merge remote-tracking branch 'origin/develop' into fb_assayCalcField
cnathe Aug 21, 2026
7e42938
QueryImporter refactor to share importMetadataOverrides() method
cnathe Aug 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 95 additions & 37 deletions query/src/org/labkey/query/QueryImporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,10 @@ private static class QueryImportContext extends FolderImportContext
// Map of metadata xml files that didn't resolve to a built-in table during import.
// The map of files will be validated during postProcess
Map<String, QueryDocument> unresolvedMetadataFiles = new LinkedHashMap<>();

// GitHub Issue #1037: Map of metadata xml files whose schema didn't exist yet during import.
// The map of files will be imported during postProcess
Map<String, QueryDocument> deferredMetadataFiles = new LinkedHashMap<>();
}

@Override
Expand Down Expand Up @@ -155,53 +159,102 @@ else if (fileName.endsWith(QueryWriter.META_FILE_EXTENSION))
ctx.addContext(QueryImportContext.class, qic);

// import any remaining metadata xml overrides
for (Map.Entry<String, QueryDocument> entry : metaFilesMap.entrySet())
{
String metaFileName = entry.getKey();
importMetadataOverrides(ctx, qic, metaFilesMap, createdQueries, changedQueries, true);

QueryDocument queryDoc = entry.getValue();
QueryType queryXml = queryDoc.getQuery();
fireQueryEvents(ctx, createdQueries, changedQueries);

String queryName = queryXml.getName();
String schemaName = queryXml.getSchemaName();
SchemaKey schemaKey = SchemaKey.fromString(schemaName);
UserSchema schema = QueryService.get().getUserSchema(ctx.getUser(), ctx.getContainer(), schemaKey);
if (schema == null)
{
ctx.getLogger().warn("Skipping import: {}", queryImportMessage(schemaName, queryName, null, metaFileName, "schema doesn't exist."));
continue;
}
// Only FolderImportTask calls postProcess(); subfolder import and create-folder-from-template drop these
if (!qic.deferredMetadataFiles.isEmpty())
ctx.getLogger().warn("{} metadata xml override(s) deferred to post-processing; they are skipped by imports that don't post-process", qic.deferredMetadataFiles.size());

if (!schema.getTableNames().contains(queryName))
{
// warn if the table doesn't exist -- it may be created later during the import (e.g., a SampleType may be created as a part of the import process)
ctx.getLogger().warn("Importing: {}", queryImportMessage(schemaName, queryName, null, metaFileName, "Creating metadata xml override for table that doesn't exist"));
qic.unresolvedMetadataFiles.put(metaFileName, queryDoc);//
}
ctx.getLogger().info("{} quer{} imported", sqlFileNames.size(), 1 == sqlFileNames.size() ? "y" : "ies");
ctx.getLogger().info("Done importing {}", getDescription());
}
}

createQueryDef(ctx, createdQueries, changedQueries, metaFileName, queryDoc, null, null);
}
private void fireQueryEvents(FolderImportContext ctx,
Map<SchemaKey, List<String>> createdQueries,
Map<Pair<SchemaKey, QueryProperty>, List<QueryPropertyChange<?>>> changedQueries)
{
// fire query created events (one set of changes per container/schema)
for (Map.Entry<SchemaKey, List<String>> entry : createdQueries.entrySet())
{
SchemaKey schemaKey = entry.getKey();
List<String> queries = entry.getValue();
QueryService.get().fireQueryCreated(ctx.getUser(), ctx.getContainer(), null, schemaKey, queries);
}

// fire query changed events (one set of changes per container/schema/queryproperty)
for (Map.Entry<Pair<SchemaKey, QueryProperty>, List<QueryPropertyChange<?>>> entry : changedQueries.entrySet())
{
SchemaKey schemaKey = entry.getKey().first;
QueryProperty property = entry.getKey().second;
List<QueryPropertyChange<?>> changes = entry.getValue();
QueryService.get().fireQueryChanged(ctx.getUser(), ctx.getContainer(), null, schemaKey, property, changes);
}
}

// fire query created events (one set of changes per container/schema)
for (Map.Entry<SchemaKey, List<String>> entry : createdQueries.entrySet())
// Import the metadata xml overrides whose schema didn't exist when process() ran
private void importDeferredMetadataFiles(FolderImportContext ctx, QueryImportContext qic) throws SQLException
{
Map<SchemaKey, List<String>> createdQueries = new LinkedHashMap<>();
Map<Pair<SchemaKey, QueryProperty>, List<QueryPropertyChange<?>>> changedQueries = new LinkedHashMap<>();

Map<String, QueryDocument> deferred = new LinkedHashMap<>(qic.deferredMetadataFiles);
qic.deferredMetadataFiles.clear();

int importedCount = importMetadataOverrides(ctx, qic, deferred, createdQueries, changedQueries, false);
fireQueryEvents(ctx, createdQueries, changedQueries);

ctx.getLogger().info("{} deferred metadata quer{} imported", importedCount, 1 == importedCount ? "y" : "ies");
}

private int importMetadataOverrides(FolderImportContext ctx, QueryImportContext qic,
Map<String, QueryDocument> metaFiles,
Map<SchemaKey, List<String>> createdQueries,
Map<Pair<SchemaKey, QueryProperty>, List<QueryPropertyChange<?>>> changedQueries,
boolean canDefer)
throws SQLException
{
int importedCount = 0;

for (Map.Entry<String, QueryDocument> entry : metaFiles.entrySet())
{
String metaFileName = entry.getKey();

@XingY XingY Aug 20, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems mostly duplicate code inside process. Can those be refactored?

@cnathe cnathe Aug 21, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, done

QueryDocument queryDoc = entry.getValue();
QueryType queryXml = queryDoc.getQuery();

String queryName = queryXml.getName();
String schemaName = queryXml.getSchemaName();
SchemaKey schemaKey = SchemaKey.fromString(schemaName);
UserSchema schema = QueryService.get().getUserSchema(ctx.getUser(), ctx.getContainer(), schemaKey);
if (schema == null)
{
SchemaKey schemaKey = entry.getKey();
List<String> queries = entry.getValue();
QueryService.get().fireQueryCreated(ctx.getUser(), ctx.getContainer(), null, schemaKey, queries);
if (canDefer)
{
// Schemas whose existence depends on data imported by a later importer, such as an assay protocol
// schema created by the XAR importer, can only be resolved once every importer has run
ctx.getLogger().info("Deferring import: {}", queryImportMessage(schemaName, queryName, null, metaFileName, "schema doesn't exist yet."));
qic.deferredMetadataFiles.put(metaFileName, queryDoc);
}
else
ctx.getLogger().warn("Skipping import: {}", queryImportMessage(schemaName, queryName, null, metaFileName, "schema doesn't exist."));
continue;
}

// fire query changed events (one set of changes per container/schema/queryproperty)
for (Map.Entry<Pair<SchemaKey, QueryProperty>, List<QueryPropertyChange<?>>> entry : changedQueries.entrySet())
if (!schema.getTableNames().contains(queryName))
{
SchemaKey schemaKey = entry.getKey().first;
QueryProperty property = entry.getKey().second;
List<QueryPropertyChange<?>> changes = entry.getValue();
QueryService.get().fireQueryChanged(ctx.getUser(), ctx.getContainer(), null, schemaKey, property, changes);
// warn if the table doesn't exist -- it may be created later during the import (e.g., a SampleType may be created as a part of the import process)
ctx.getLogger().warn("Importing: {}", queryImportMessage(schemaName, queryName, null, metaFileName, "Creating metadata xml override for table that doesn't exist"));
if (canDefer)
qic.unresolvedMetadataFiles.put(metaFileName, queryDoc);
}

ctx.getLogger().info("{} quer{} imported", sqlFileNames.size(), 1 == sqlFileNames.size() ? "y" : "ies");
ctx.getLogger().info("Done importing {}", getDescription());
createQueryDef(ctx, createdQueries, changedQueries, metaFileName, queryDoc, null, null);
importedCount++;
}

return importedCount;
}

private void createQueryDef(FolderImportContext ctx,
Expand Down Expand Up @@ -317,21 +370,26 @@ String queryImportMessage(String schemaName, String queryName, @Nullable String

@Override
@NotNull
public Collection<PipelineJobWarning> postProcess(FolderImportContext ctx, VirtualFile root)
public Collection<PipelineJobWarning> postProcess(FolderImportContext ctx, VirtualFile root) throws SQLException
{
List<PipelineJobWarning> warnings = new ArrayList<>();

//validate all queries in all schemas in the container
ctx.getLogger().info("Post-processing {}", getDescription());

QueryImportContext qic = ctx.getContext(QueryImportContext.class);

// import deferred metadata regardless of isSkipQueryValidation() -- skipping validation must not skip an import
if (qic != null && !qic.deferredMetadataFiles.isEmpty())
importDeferredMetadataFiles(ctx, qic);

if (ctx.isSkipQueryValidation())
{
ctx.getLogger().info("Skipping query validation.");
}
else
{
// check that each meta xml file was applied to a built-in table after import of all data structures has completed
QueryImportContext qic = ctx.getContext(QueryImportContext.class);
if (qic != null)
{
for (Map.Entry<String, QueryDocument> entry : qic.unresolvedMetadataFiles.entrySet())
Expand Down