diff --git a/query/src/org/labkey/query/QueryImporter.java b/query/src/org/labkey/query/QueryImporter.java index 17f551fb716..1be83f09d41 100644 --- a/query/src/org/labkey/query/QueryImporter.java +++ b/query/src/org/labkey/query/QueryImporter.java @@ -79,6 +79,10 @@ private static class QueryImportContext extends FolderImportContext // Map of metadata xml files that didn't resolve to a built-in table during import. // The map of files will be validated during postProcess Map unresolvedMetadataFiles = new LinkedHashMap<>(); + + // GitHub Issue #1037: Map of metadata xml files whose schema didn't exist yet during import. + // The map of files will be imported during postProcess + Map deferredMetadataFiles = new LinkedHashMap<>(); } @Override @@ -155,53 +159,102 @@ else if (fileName.endsWith(QueryWriter.META_FILE_EXTENSION)) ctx.addContext(QueryImportContext.class, qic); // import any remaining metadata xml overrides - for (Map.Entry entry : metaFilesMap.entrySet()) - { - String metaFileName = entry.getKey(); + importMetadataOverrides(ctx, qic, metaFilesMap, createdQueries, changedQueries, true); - QueryDocument queryDoc = entry.getValue(); - QueryType queryXml = queryDoc.getQuery(); + fireQueryEvents(ctx, createdQueries, changedQueries); - String queryName = queryXml.getName(); - String schemaName = queryXml.getSchemaName(); - SchemaKey schemaKey = SchemaKey.fromString(schemaName); - UserSchema schema = QueryService.get().getUserSchema(ctx.getUser(), ctx.getContainer(), schemaKey); - if (schema == null) - { - ctx.getLogger().warn("Skipping import: {}", queryImportMessage(schemaName, queryName, null, metaFileName, "schema doesn't exist.")); - continue; - } + // Only FolderImportTask calls postProcess(); subfolder import and create-folder-from-template drop these + if (!qic.deferredMetadataFiles.isEmpty()) + ctx.getLogger().warn("{} metadata xml override(s) deferred to post-processing; they are skipped by imports that don't post-process", qic.deferredMetadataFiles.size()); - if (!schema.getTableNames().contains(queryName)) - { - // warn if the table doesn't exist -- it may be created later during the import (e.g., a SampleType may be created as a part of the import process) - ctx.getLogger().warn("Importing: {}", queryImportMessage(schemaName, queryName, null, metaFileName, "Creating metadata xml override for table that doesn't exist")); - qic.unresolvedMetadataFiles.put(metaFileName, queryDoc);// - } + ctx.getLogger().info("{} quer{} imported", sqlFileNames.size(), 1 == sqlFileNames.size() ? "y" : "ies"); + ctx.getLogger().info("Done importing {}", getDescription()); + } + } - createQueryDef(ctx, createdQueries, changedQueries, metaFileName, queryDoc, null, null); - } + private void fireQueryEvents(FolderImportContext ctx, + Map> createdQueries, + Map, List>> changedQueries) + { + // fire query created events (one set of changes per container/schema) + for (Map.Entry> entry : createdQueries.entrySet()) + { + SchemaKey schemaKey = entry.getKey(); + List queries = entry.getValue(); + QueryService.get().fireQueryCreated(ctx.getUser(), ctx.getContainer(), null, schemaKey, queries); + } + + // fire query changed events (one set of changes per container/schema/queryproperty) + for (Map.Entry, List>> entry : changedQueries.entrySet()) + { + SchemaKey schemaKey = entry.getKey().first; + QueryProperty property = entry.getKey().second; + List> changes = entry.getValue(); + QueryService.get().fireQueryChanged(ctx.getUser(), ctx.getContainer(), null, schemaKey, property, changes); + } + } - // fire query created events (one set of changes per container/schema) - for (Map.Entry> entry : createdQueries.entrySet()) + // Import the metadata xml overrides whose schema didn't exist when process() ran + private void importDeferredMetadataFiles(FolderImportContext ctx, QueryImportContext qic) throws SQLException + { + Map> createdQueries = new LinkedHashMap<>(); + Map, List>> changedQueries = new LinkedHashMap<>(); + + Map deferred = new LinkedHashMap<>(qic.deferredMetadataFiles); + qic.deferredMetadataFiles.clear(); + + int importedCount = importMetadataOverrides(ctx, qic, deferred, createdQueries, changedQueries, false); + fireQueryEvents(ctx, createdQueries, changedQueries); + + ctx.getLogger().info("{} deferred metadata quer{} imported", importedCount, 1 == importedCount ? "y" : "ies"); + } + + private int importMetadataOverrides(FolderImportContext ctx, QueryImportContext qic, + Map metaFiles, + Map> createdQueries, + Map, List>> changedQueries, + boolean canDefer) + throws SQLException + { + int importedCount = 0; + + for (Map.Entry entry : metaFiles.entrySet()) + { + String metaFileName = entry.getKey(); + QueryDocument queryDoc = entry.getValue(); + QueryType queryXml = queryDoc.getQuery(); + + String queryName = queryXml.getName(); + String schemaName = queryXml.getSchemaName(); + SchemaKey schemaKey = SchemaKey.fromString(schemaName); + UserSchema schema = QueryService.get().getUserSchema(ctx.getUser(), ctx.getContainer(), schemaKey); + if (schema == null) { - SchemaKey schemaKey = entry.getKey(); - List queries = entry.getValue(); - QueryService.get().fireQueryCreated(ctx.getUser(), ctx.getContainer(), null, schemaKey, queries); + if (canDefer) + { + // Schemas whose existence depends on data imported by a later importer, such as an assay protocol + // schema created by the XAR importer, can only be resolved once every importer has run + ctx.getLogger().info("Deferring import: {}", queryImportMessage(schemaName, queryName, null, metaFileName, "schema doesn't exist yet.")); + qic.deferredMetadataFiles.put(metaFileName, queryDoc); + } + else + ctx.getLogger().warn("Skipping import: {}", queryImportMessage(schemaName, queryName, null, metaFileName, "schema doesn't exist.")); + continue; } - // fire query changed events (one set of changes per container/schema/queryproperty) - for (Map.Entry, List>> entry : changedQueries.entrySet()) + if (!schema.getTableNames().contains(queryName)) { - SchemaKey schemaKey = entry.getKey().first; - QueryProperty property = entry.getKey().second; - List> changes = entry.getValue(); - QueryService.get().fireQueryChanged(ctx.getUser(), ctx.getContainer(), null, schemaKey, property, changes); + // warn if the table doesn't exist -- it may be created later during the import (e.g., a SampleType may be created as a part of the import process) + ctx.getLogger().warn("Importing: {}", queryImportMessage(schemaName, queryName, null, metaFileName, "Creating metadata xml override for table that doesn't exist")); + if (canDefer) + qic.unresolvedMetadataFiles.put(metaFileName, queryDoc); } - ctx.getLogger().info("{} quer{} imported", sqlFileNames.size(), 1 == sqlFileNames.size() ? "y" : "ies"); - ctx.getLogger().info("Done importing {}", getDescription()); + createQueryDef(ctx, createdQueries, changedQueries, metaFileName, queryDoc, null, null); + importedCount++; } + + return importedCount; } private void createQueryDef(FolderImportContext ctx, @@ -317,13 +370,19 @@ String queryImportMessage(String schemaName, String queryName, @Nullable String @Override @NotNull - public Collection postProcess(FolderImportContext ctx, VirtualFile root) + public Collection postProcess(FolderImportContext ctx, VirtualFile root) throws SQLException { List warnings = new ArrayList<>(); //validate all queries in all schemas in the container ctx.getLogger().info("Post-processing {}", getDescription()); + QueryImportContext qic = ctx.getContext(QueryImportContext.class); + + // import deferred metadata regardless of isSkipQueryValidation() -- skipping validation must not skip an import + if (qic != null && !qic.deferredMetadataFiles.isEmpty()) + importDeferredMetadataFiles(ctx, qic); + if (ctx.isSkipQueryValidation()) { ctx.getLogger().info("Skipping query validation."); @@ -331,7 +390,6 @@ public Collection postProcess(FolderImportContext ctx, Virtu else { // check that each meta xml file was applied to a built-in table after import of all data structures has completed - QueryImportContext qic = ctx.getContext(QueryImportContext.class); if (qic != null) { for (Map.Entry entry : qic.unresolvedMetadataFiles.entrySet())