feat(admin): split matching stats and filter search updates

backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java

@@ -137,21 +137,25 @@ final void updateFilterSearch() {
 	abstract void registerColumnValuesInSearch(Set<Column> columns);
 
 	/**
-	 * Hook for actions that are best done after all data has been imported and is in a consistent state.
-	 * Such actions are for example search initialization and collection of matching statistics.
-	 *
-	 * @implNote This intentionally submits a SlowJob so that it will be queued after all jobs that are already in the queue (usually import jobs).
+	 * Schedules the collection of matching statistics after all jobs that are already queued (usually import jobs).
 	 */
-	public void postprocessData() {
+	public void scheduleMatchingStatsUpdate() {
+		getJobManager().addSlowJob(new SimpleJob(
+				"Initiate Update Matching Stats",
+				this::updateMatchingStats
+		));
+	}
 
+	/**
+	 * Schedules the regeneration of the filter search after all jobs that are already queued (usually import jobs).
+	 */
+	public void scheduleFilterSearchUpdate() {
 		getJobManager().addSlowJob(new SimpleJob(
-				"Initiate Update Matching Stats and InternalFilterSearch",
+				"Initiate Update InternalFilterSearch",
 				() -> {
 					updateInternToExternMappings();
-					updateMatchingStats();
 					updateFilterSearch();
 				}
 		));
-
 	}
 }

backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java

@@ -337,12 +337,21 @@ public synchronized void deleteImport(ImportId imp) {
 	}
 
 	/**
-	 * Issues a postprocessing of the imported data for initializing certain internal modules that are either expensive or need the whole data present.
+	 * Schedules collection of matching statistics for the dataset.
 	 */
-	public void postprocessNamespace(DatasetId dataset) {
+	public void updateMatchingStats(DatasetId dataset) {
 		final Namespace ns = getDatasetRegistry().get(dataset);
 
-		ns.postprocessData();
+		ns.scheduleMatchingStatsUpdate();
+	}
+
+	/**
+	 * Schedules regeneration of the filter search for the dataset.
+	 */
+	public void updateFilterSearch(DatasetId dataset) {
+		final Namespace ns = getDatasetRegistry().get(dataset);
+
+		ns.scheduleFilterSearchUpdate();
 	}
 
 	public EntityIdMap getIdMapping(Namespace namespace) {

backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetResource.java

@@ -235,15 +235,18 @@ public void delete() {
 		processor.deleteDataset(dataset);
 	}
 
-	/**
-	 * @param dataset the namespace to postprocess
-	 * @implNote The path mapping is historical named. Renaming the path requires some coordination.
-	 */
 	@POST
-	@Path("/update-matching-stats")
+	@Path("update-matching-stats")
 	@Consumes(MediaType.WILDCARD)
-	public void postprocessNamespace(@PathParam(DATASET) DatasetId dataset) {
-		processor.postprocessNamespace(dataset);
+	public void updateMatchingStats(@PathParam(DATASET) DatasetId dataset) {
+		processor.updateMatchingStats(dataset);
+	}
+
+	@POST
+	@Path("update-filter-search")
+	@Consumes(MediaType.WILDCARD)
+	public void updateFilterSearch(@PathParam(DATASET) DatasetId dataset) {
+		processor.updateFilterSearch(dataset);
 	}
 
 	@POST

backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/dataset.html.ftl

@@ -81,6 +81,14 @@
       >
         Update Matching Stats
       </button>
+      <button
+        type="button"
+        class="btn btn-secondary"
+        onclick="rest('/admin/datasets/${c.ds.id}/update-filter-search',{method: 'post'})"
+        data-cy="update-filter-search"
+      >
+        Update Filter Search
+      </button>
       <button
         type="button"
         class="btn btn-danger"

backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java

@@ -417,7 +417,7 @@ private static void uploadSearchIndex(@NonNull StandaloneSupport support, @NonNu
 
 	public static void updateMatchingStats(@NonNull StandaloneSupport support) {
 		final URI matchingStatsUri = HierarchyHelper.hierarchicalPath(support.defaultAdminURIBuilder()
-															, AdminDatasetResource.class, "postprocessNamespace"
+													, AdminDatasetResource.class, "updateMatchingStats"
 													)
 													.buildFromMap(Map.of(DATASET, support.getDataset()));
 
@@ -427,6 +427,18 @@ public static void updateMatchingStats(@NonNull StandaloneSupport support) {
 		post.close();
 	}
 
+	public static void updateFilterSearch(@NonNull StandaloneSupport support) {
+		final URI filterSearchUri = HierarchyHelper.hierarchicalPath(support.defaultAdminURIBuilder()
+													, AdminDatasetResource.class, "updateFilterSearch"
+											)
+											.buildFromMap(Map.of(DATASET, support.getDataset()));
+
+		final Response post = support.getClient().target(filterSearchUri)
+									 .request(MediaType.APPLICATION_JSON_TYPE)
+									 .post(null);
+		post.close();
+	}
+
 	@SneakyThrows
 	public static InputStream openResource(String path) {
 		return IOUtils.resourceToURL(path).openStream();

backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptResolutionTest.java

@@ -20,7 +20,6 @@
 import com.bakdata.conquery.models.exceptions.ValidatorHelper;
 import com.bakdata.conquery.models.identifiable.ids.Id;
 import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId;
-import com.bakdata.conquery.resources.admin.rest.AdminDatasetResource;
 import com.bakdata.conquery.resources.api.ConceptResource;
 import com.bakdata.conquery.resources.api.ConceptsProcessor.ResolvedConceptsResult;
 import com.bakdata.conquery.resources.hierarchies.HierarchyHelper;
@@ -42,13 +41,7 @@ public void execute(StandaloneSupport conquery) throws Exception {
 
 		test.importRequiredData(conquery);
 
-		final URI matchingStatsUri = HierarchyHelper.hierarchicalPath(conquery.defaultAdminURIBuilder()
-															, AdminDatasetResource.class, "postprocessNamespace")
-													.buildFromMap(Map.of(DATASET, conquery.getDataset()));
-
-		conquery.getClient().target(matchingStatsUri)
-				.request(MediaType.APPLICATION_JSON_TYPE)
-				.post(null);
+		LoadingUtil.updateFilterSearch(conquery);
 
 		conquery.waitUntilWorkDone();
 

backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java

@@ -32,7 +32,6 @@
 import com.bakdata.conquery.models.exceptions.ValidatorHelper;
 import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId;
 import com.bakdata.conquery.models.index.IndexService;
-import com.bakdata.conquery.resources.admin.rest.AdminDatasetResource;
 import com.bakdata.conquery.resources.api.ConceptsProcessor;
 import com.bakdata.conquery.resources.api.FilterResource;
 import com.bakdata.conquery.resources.hierarchies.HierarchyHelper;
@@ -186,19 +185,11 @@ private static SelectFilter<?> setupSearch(StandaloneSupport conquery) throws Ex
 		filterTemplate.setName("test");
 		filter.setTemplate(filterTemplate.getId());
 
-		// We need to persist the modification before we submit the update matching stats request
+		// We need to persist the modification before we submit the filter search update request
 		namespaceStorage.addSearchIndex(filterTemplate);
 		namespaceStorage.updateConcept(concept);
 
-		final URI matchingStatsUri = HierarchyHelper.hierarchicalPath(conquery.defaultAdminURIBuilder()
-															, AdminDatasetResource.class, "postprocessNamespace"
-													)
-													.buildFromMap(Map.of(DATASET, conquery.getDataset()));
-
-		conquery.getClient().target(matchingStatsUri)
-				.request(MediaType.APPLICATION_JSON_TYPE)
-				.post(null)
-				.close();
+		LoadingUtil.updateFilterSearch(conquery);
 
 		conquery.waitUntilWorkDone();
 

backend/src/test/java/com/bakdata/conquery/integration/tests/FilterResolutionTest.java

@@ -27,7 +27,6 @@
 import com.bakdata.conquery.models.exceptions.ValidatorHelper;
 import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId;
 import com.bakdata.conquery.models.index.IndexService;
-import com.bakdata.conquery.resources.admin.rest.AdminDatasetResource;
 import com.bakdata.conquery.resources.api.ConceptsProcessor;
 import com.bakdata.conquery.resources.api.FilterResource;
 import com.bakdata.conquery.resources.hierarchies.HierarchyHelper;
@@ -90,19 +89,11 @@ public void execute(StandaloneSupport conquery) throws Exception {
 
 		filter.setTemplate(filterTemplate.getId());
 
-		// We need to persist the modification before we submit the update matching stats request
+		// We need to persist the modification before we submit the filter search update request
 		namespaceStorage.addSearchIndex(filterTemplate);
 		namespaceStorage.updateConcept(concept);
 
-		final URI matchingStatsUri = HierarchyHelper.hierarchicalPath(conquery.defaultAdminURIBuilder()
-															, AdminDatasetResource.class, "postprocessNamespace"
-													)
-													.buildFromMap(Map.of(DATASET, conquery.getDataset()));
-
-		final Response post = conquery.getClient().target(matchingStatsUri)
-									  .request(MediaType.APPLICATION_JSON_TYPE)
-									  .post(null);
-		post.close();
+		LoadingUtil.updateFilterSearch(conquery);
 
 		conquery.waitUntilWorkDone();
 

backend/src/test/resources/tests/endpoints/adminEndpointInfo.json

@@ -169,6 +169,11 @@
     "path": "/datasets/{dataset}/update-matching-stats",
     "clazz": "AdminDatasetResource"
   },
+  {
+    "method": "POST",
+    "path": "/datasets/{dataset}/update-filter-search",
+    "clazz": "AdminDatasetResource"
+  },
   {
     "method": "POST",
     "path": "/datasets/{dataset}/weight",
@@ -324,4 +329,4 @@
     "path": "/auth-overview/csv/group/{groupId}",
     "clazz": "AuthOverviewResource"
   }
-]
+]

cypress/e2e/backend-admin-ui/test_2_dataset.cy.js

@@ -198,6 +198,19 @@ context("Admin UI Single Dataset", () => {
       cy.get('#origin_Manager > .card-header > .row > .accordion-infotext > div > .jobsAmount').eq(0);
     })
   });
+  describe("Perform Update Filter Search", () => {
+
+    it("Start Update Filter Search", () => {
+      visitAdminUI("datasets");
+      cy.get('[data-cy="datasets"]').contains(testDSID).click();
+      cy.get('[data-cy="update-filter-search"]').click();
+    })
+
+    it("Wait for Update Filter Search to finish", () => {
+      visitAdminUI("jobs");
+      cy.get('#origin_Manager > .card-header > .row > .accordion-infotext > div > .jobsAmount').eq(0);
+    })
+  });
   describe("Visit Index Service Page", () => {
 
     it("Check Statistics", () => {

scripts/load_e2e_data.sh

@@ -61,5 +61,6 @@ sleep 3
 
 echo "Init Matching Stats and Search"
 curl --fail -X POST  "$admin_api/datasets/dataset1/update-matching-stats" -H "$h_ct" -H "$h_auth"
+curl --fail -X POST  "$admin_api/datasets/dataset1/update-filter-search" -H "$h_ct" -H "$h_auth"
 
 echo "Done loading data"

tutorial/mimic_iii_demo/preprocess_and_upload.ipynb

@@ -159,6 +159,9 @@
    "source": [
     "print(f'Updating matching stats ... ', end='')\n",
     "res = r.post(f\"http://localhost:8081/admin/datasets/{datasetId}/update-matching-stats\", headers={\"content-type\":\"application/json\"})\n",
+    "print(process_respose(res))\n",
+    "print(f'Updating filter search ... ', end='')\n",
+    "res = r.post(f\"http://localhost:8081/admin/datasets/{datasetId}/update-filter-search\", headers={\"content-type\":\"application/json\"})\n",
     "print(process_respose(res))"
    ]
   },