feat: extract sql-module

.github/workflows/test_backend.yml

@@ -62,4 +62,4 @@ jobs:
           java-version: '21'
           overwrite-settings: false
       - name: ${{ matrix.test-group.name }}
-        run: mvn test -T 1C -pl backend ${{ matrix.test-group.flags }}
+        run: mvn test -T 1C -pl backend -am ${{ matrix.test-group.flags }}

backend/pom.xml

@@ -112,6 +112,16 @@
     </build>
 
     <dependencies>
+        <dependency>
+            <groupId>com.bakdata.conquery</groupId>
+            <artifactId>dataset-model</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.bakdata.conquery</groupId>
+            <artifactId>sql-connector</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>
             <artifactId>log4j-to-slf4j</artifactId>
@@ -346,7 +356,7 @@
         <dependency>
             <groupId>org.jooq</groupId>
             <artifactId>jooq</artifactId>
-            <version>3.20.2</version>
+            <version>${jooq.version}</version>
         </dependency>
         <dependency>
             <groupId>com.zaxxer</groupId>

backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java

@@ -53,9 +53,9 @@
 import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId;
 import com.bakdata.conquery.models.identifiable.mapping.IdPrinter;
 import com.bakdata.conquery.models.query.*;
+import com.bakdata.conquery.models.query.DateAggregationAction;
 import com.bakdata.conquery.models.query.preview.EntityPreviewExecution;
 import com.bakdata.conquery.models.query.preview.EntityPreviewForm;
-import com.bakdata.conquery.models.query.queryplan.DateAggregationAction;
 import com.bakdata.conquery.models.query.resultinfo.ResultInfo;
 import com.bakdata.conquery.models.query.resultinfo.UniqueNamer;
 import com.bakdata.conquery.models.query.resultinfo.printers.JavaResultPrinters;

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java

@@ -12,19 +12,20 @@
 import com.bakdata.conquery.models.common.Range.LongRange;
 import com.bakdata.conquery.models.common.Range.MoneyRange;
 import com.bakdata.conquery.models.config.ConqueryConfig;
-import com.bakdata.conquery.models.datasets.concepts.filters.Filter;
 import com.bakdata.conquery.models.identifiable.ids.specific.FilterId;
 import com.bakdata.conquery.models.query.QueryResolveContext;
-import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext;
-import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables;
-import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext;
-import com.bakdata.conquery.sql.conversion.model.SqlIdColumns;
-import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters;
 import com.bakdata.conquery.util.validation.ResolvableId;
-import com.fasterxml.jackson.annotation.*;
-import lombok.*;
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.fasterxml.jackson.annotation.OptBoolean;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.Setter;
+import lombok.ToString;
 import org.jetbrains.annotations.TestOnly;
-import org.jooq.Condition;
 
 /**
  * @implNote The {@link JsonCreator} annos are necessary. Otherwise, Jackson will deserilaize all values as generic objects.
@@ -52,19 +53,6 @@ public VALUE readValue() {
 		return ((VALUE) value);
 	}
 
-	public SqlFilters convertToSqlFilter(SqlIdColumns ids, ConversionContext context, ConnectorSqlTables tables) {
-		FilterContext<VALUE> filterContext = FilterContext.forConceptConversion(ids, readValue(), context, tables);
-		final Filter<VALUE> resolve = (Filter<VALUE>) filter.resolve();
-		SqlFilters sqlFilters = resolve.createConverter().convertToSqlFilter(resolve, filterContext);
-		return sqlFilters;
-	}
-
-	public Condition convertForTableExport(SqlIdColumns ids, ConversionContext context) {
-		FilterContext<VALUE> filterContext = FilterContext.forTableExport(ids, readValue(), context);
-		final Filter<VALUE> resolve = (Filter<VALUE>) filter.resolve();
-		return resolve.createConverter().convertForTableExport(resolve, filterContext);
-	}
-
 	@CPSType(id = FrontendFilterType.Fields.MULTI_SELECT, base = FilterValue.class)
 	public static class CQMultiSelectFilter extends FilterValue<Set<String>> {
 		@JsonCreator

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java

@@ -17,14 +17,14 @@
 import com.bakdata.conquery.io.jackson.View;
 import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId;
 import com.bakdata.conquery.models.query.C10nCache;
+import com.bakdata.conquery.models.query.DateAggregationAction;
 import com.bakdata.conquery.models.query.PrintSettings;
 import com.bakdata.conquery.models.query.QueryExecutionContext;
 import com.bakdata.conquery.models.query.QueryPlanContext;
 import com.bakdata.conquery.models.query.QueryResolveContext;
 import com.bakdata.conquery.models.query.RequiredEntities;
 import com.bakdata.conquery.models.query.Visitable;
 import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan;
-import com.bakdata.conquery.models.query.queryplan.DateAggregationAction;
 import com.bakdata.conquery.models.query.queryplan.QPNode;
 import com.bakdata.conquery.models.query.queryplan.aggregators.specific.ExistsAggregator;
 import com.bakdata.conquery.models.query.queryplan.specific.AndNode;

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java

@@ -22,8 +22,8 @@
 import com.bakdata.conquery.models.identifiable.NamespacedIdentifiable;
 import com.bakdata.conquery.models.identifiable.ids.specific.*;
 import com.bakdata.conquery.models.query.*;
+import com.bakdata.conquery.models.query.DateAggregationAction;
 import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan;
-import com.bakdata.conquery.models.query.queryplan.DateAggregationAction;
 import com.bakdata.conquery.models.query.queryplan.QPNode;
 import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator;
 import com.bakdata.conquery.models.query.queryplan.aggregators.specific.EventDateUnionAggregator;

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java

@@ -10,12 +10,12 @@
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.io.jackson.View;
 import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId;
+import com.bakdata.conquery.models.query.DateAggregationAction;
 import com.bakdata.conquery.models.query.DateAggregationMode;
 import com.bakdata.conquery.models.query.QueryPlanContext;
 import com.bakdata.conquery.models.query.QueryResolveContext;
 import com.bakdata.conquery.models.query.Visitable;
 import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan;
-import com.bakdata.conquery.models.query.queryplan.DateAggregationAction;
 import com.bakdata.conquery.models.query.queryplan.QPNode;
 import com.bakdata.conquery.models.query.queryplan.specific.NegatingNode;
 import com.bakdata.conquery.models.query.resultinfo.ResultInfo;

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java

@@ -17,14 +17,14 @@
 import com.bakdata.conquery.io.jackson.View;
 import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId;
 import com.bakdata.conquery.models.query.C10nCache;
+import com.bakdata.conquery.models.query.DateAggregationAction;
 import com.bakdata.conquery.models.query.PrintSettings;
 import com.bakdata.conquery.models.query.QueryExecutionContext;
 import com.bakdata.conquery.models.query.QueryPlanContext;
 import com.bakdata.conquery.models.query.QueryResolveContext;
 import com.bakdata.conquery.models.query.RequiredEntities;
 import com.bakdata.conquery.models.query.Visitable;
 import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan;
-import com.bakdata.conquery.models.query.queryplan.DateAggregationAction;
 import com.bakdata.conquery.models.query.queryplan.QPNode;
 import com.bakdata.conquery.models.query.queryplan.aggregators.specific.ExistsAggregator;
 import com.bakdata.conquery.models.query.queryplan.specific.OrNode;

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java

@@ -8,9 +8,9 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 import jakarta.validation.constraints.NotEmpty;
 
 import com.bakdata.conquery.apiv1.query.CQElement;
@@ -33,7 +33,6 @@
 import com.bakdata.conquery.models.types.ResultType;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonView;
-import com.google.common.collect.Streams;
 import io.dropwizard.validation.ValidationMethod;
 import lombok.AccessLevel;
 import lombok.Getter;
@@ -106,13 +105,7 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) {
 			throw new IllegalStateException("CQExternal needs to be resolved before creating a plan");
 		}
 
-		final String[] extraHeaders = Streams.zip(
-													 Arrays.stream(headers),
-													 format.stream(),
-													 (header, format) -> format.equals(EntityResolverUtil.FORMAT_EXTRA) ? header : null
-											 )
-											 .filter(Objects::nonNull)
-											 .toArray(String[]::new);
+		final String[] extraHeaders = getExtraHeaders().toArray(String[]::new);
 
 		if (onlySingles) {
 			return createExternalNodeOnlySingle(context, plan, extraHeaders);
@@ -218,19 +211,24 @@ public List<ResultInfo> getResultInfos() {
 		if (extra == null) {
 			return Collections.emptyList();
 		}
-		List<ResultInfo> resultInfos = new ArrayList<>();
-		for (int col = 0; col < format.size(); col++) {
-			if (!format.get(col).equals(EntityResolverUtil.FORMAT_EXTRA)) {
-				continue;
-			}
-
-			final String column = headers[col];
+		final ResultType type = onlySingles
+				? ResultType.Primitive.STRING
+				: new ResultType.ListT<>(ResultType.Primitive.STRING);
+		return getExtraHeaders().stream()
+				.<ResultInfo>map(column -> new ExternalResultInfo(column, type))
+				.toList();
+	}
 
-			final ResultType type = onlySingles ? ResultType.Primitive.STRING : new ResultType.ListT<>(ResultType.Primitive.STRING);
-			resultInfos.add(new ExternalResultInfo(column, type));
+	/** Ordered names of the resolved columns that are returned as external result values. */
+	@JsonIgnore
+	public List<String> getExtraHeaders() {
+		if (headers == null) {
+			return List.of();
 		}
-
-		return resultInfos;
+		return IntStream.range(0, format.size())
+				.filter(index -> EntityResolverUtil.FORMAT_EXTRA.equals(format.get(index)))
+				.mapToObj(index -> headers[index])
+				.toList();
 	}
 
 
@@ -268,6 +266,9 @@ public boolean isWithExtras() {
 	}
 
 	public List<Map.Entry<String, List<String>>> getExtrasForId(String id) {
+		if (extra == null) {
+			return List.of();
+		}
 		Map<String, List<String>> extras = extra.getOrDefault(id, Collections.emptyMap());
 		// we need to bring the extras in the correct order
 		List<Map.Entry<String, List<String>>> inOrder = new ArrayList<>();

backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java

@@ -12,8 +12,16 @@
 import com.bakdata.conquery.models.query.ExecutionManager;
 import com.bakdata.conquery.models.worker.DatasetRegistry;
 import com.bakdata.conquery.models.worker.LocalNamespace;
+import com.bakdata.conquery.sql.translate.DefaultSqlTranslator;
+import com.bakdata.conquery.sql.translate.FormSqlTranslator;
+import com.bakdata.conquery.sql.translate.TableExportSqlTranslator;
 import com.bakdata.conquery.sql.conquery.SqlExecutionManager;
-import com.bakdata.conquery.sql.conversion.NodeConversions;
+import com.bakdata.conquery.sql.conversion.ResolvedFormAdapter;
+import com.bakdata.conquery.sql.conversion.ResolvedFormMapper;
+import com.bakdata.conquery.sql.conversion.ResolvedQueryAdapter;
+import com.bakdata.conquery.sql.conversion.ResolvedQueryMapper;
+import com.bakdata.conquery.sql.conversion.ResolvedTableExportAdapter;
+import com.bakdata.conquery.sql.conversion.ResolvedTableExportMapper;
 import com.bakdata.conquery.sql.conversion.SqlConverter;
 import com.bakdata.conquery.sql.conversion.dialect.DialectBundle;
 import com.bakdata.conquery.sql.execution.ResultSetProcessor;
@@ -49,8 +57,18 @@ public LocalNamespace createNamespace(
 			ResultSetProcessor resultSetProcessor = dialectBundle.getResultSetProcessor(config);
 			SqlExecutionService sqlExecutionService = new SqlExecutionService(dslContext, resultSetProcessor);
 
-			NodeConversions nodeConversions = new NodeConversions(config.getIdColumns(), dialectBundle, dslContext, sqlExecutionService, clock, connection.getConnection().getPrimaryColumn());
-			SqlConverter sqlConverter = new SqlConverter(nodeConversions, config);
+			ResolvedQueryMapper resolvedQueryMapper = new ResolvedQueryMapper(
+					config.getIdColumns(), dialectBundle.getFunctionProvider(), clock,
+					connection.getConnection().getPrimaryColumn());
+			ResolvedQueryAdapter resolvedQueryAdapter = new ResolvedQueryAdapter(
+					environment.getValidator(), new DefaultSqlTranslator(dslContext), resolvedQueryMapper);
+			ResolvedTableExportAdapter tableExportAdapter = new ResolvedTableExportAdapter(
+					new ResolvedTableExportMapper(resolvedQueryMapper, clock, connection.getConnection().getPrimaryColumn()),
+					new TableExportSqlTranslator(dslContext));
+			ResolvedFormAdapter formAdapter = new ResolvedFormAdapter(
+					new ResolvedFormMapper(resolvedQueryMapper), new FormSqlTranslator(dslContext));
+			SqlConverter sqlConverter = new SqlConverter(
+					resolvedQueryAdapter, tableExportAdapter, formAdapter, dialectBundle.getTranslatorDialect());
 			ExecutionManager executionManager = new SqlExecutionManager(sqlConverter, sqlExecutionService, metaStorage, datasetRegistry, config);
 			SqlStorageHandler sqlStorageHandler = new SqlStorageHandler(sqlExecutionService);
 			SqlEntityResolver sqlEntityResolver = new SqlEntityResolver(config.getIdColumns(), dslContext, dialectBundle, sqlExecutionService);

backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java

@@ -17,7 +17,7 @@
 import com.bakdata.conquery.models.config.IdColumnConfig;
 import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
 import com.bakdata.conquery.models.identifiable.mapping.ExternalId;
-import com.bakdata.conquery.sql.conversion.SharedAliases;
+import com.bakdata.conquery.sql.translate.ir.SharedAliases;
 import com.bakdata.conquery.sql.conversion.dialect.DialectBundle;
 import com.bakdata.conquery.sql.execution.SqlExecutionService;
 import com.bakdata.conquery.util.DateReader;
@@ -152,7 +152,7 @@ private Map<Integer, IdResolveInfo> resolveIds(String[][] values, List<Function<
 		SelectConditionStep<Record3<Integer, String, Boolean>> resolveIdsQuery =
 				context.with(unresolvedCte)
 					   .select(rowIndex, externalPrimaryColumn, isResolved)
-					   .from(dialect.getFunctionProvider().innerJoin(allIdsTable, unresolvedCte, List.of(externalPrimaryColumn.eq(innerPrimaryColumn))))
+					   .from(allIdsTable.innerJoin(unresolvedCte).on(externalPrimaryColumn.eq(innerPrimaryColumn)))
 					   .where(externalPrimaryColumn.eq(innerPrimaryColumn));
 
 		return executionService.fetchStream(resolveIdsQuery)

backend/src/main/java/com/bakdata/conquery/models/common/IRange.java

@@ -1,13 +1,25 @@
 package com.bakdata.conquery.models.common;
 
+import java.util.Optional;
+
 import lombok.NonNull;
 
-public interface IRange<VALUE extends Comparable, CLASS extends IRange<VALUE, CLASS>> {
+public interface IRange<VALUE extends Comparable<? super VALUE>, CLASS extends IRange<VALUE, CLASS>> extends InclusiveRange<VALUE> {
 
 	VALUE getMin();
 
 	VALUE getMax();
 
+	@Override
+	default Optional<VALUE> minimum() {
+		return Optional.ofNullable(getMin());
+	}
+
+	@Override
+	default Optional<VALUE> maximum() {
+		return Optional.ofNullable(getMax());
+	}
+
 	boolean contains(VALUE value);
 
 	boolean contains(CLASS other);

backend/src/main/java/com/bakdata/conquery/models/common/Range.java

@@ -16,7 +16,7 @@
 @With
 @Getter
 @EqualsAndHashCode
-public class Range<T extends Comparable> implements IRange<T, Range<T>> {
+public class Range<T extends Comparable<? super T>> implements IRange<T, Range<T>> {
 
 	private final T min;
 	private final T max;
@@ -42,24 +42,24 @@ public boolean isOpen() {
 		return max == null || min == null;
 	}
 
-	public static <T extends Comparable<?>> Range<T> exactly(T exactly) {
+	public static <T extends Comparable<? super T>> Range<T> exactly(T exactly) {
 		return new Range<>(exactly, exactly);
 	}
 
 	@JsonCreator
-	public static <T extends Comparable<?>> Range<T> of(@JsonProperty("min") T min, @JsonProperty("max") T max) {
+	public static <T extends Comparable<? super T>> Range<T> of(@JsonProperty("min") T min, @JsonProperty("max") T max) {
 		return new Range<>(min, max);
 	}
 
-	public static <T extends Comparable<?>> Range<T> atMost(T bound) {
+	public static <T extends Comparable<? super T>> Range<T> atMost(T bound) {
 		return new Range<>(null, bound);
 	}
 
-	public static <T extends Comparable<?>> Range<T> atLeast(T bound) {
+	public static <T extends Comparable<? super T>> Range<T> atLeast(T bound) {
 		return new Range<>(bound, null);
 	}
 
-	public static <T extends Comparable<?>> Range<T> all() {
+	public static <T extends Comparable<? super T>> Range<T> all() {
 		return new Range<>(null, null);
 	}
 

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/AndCondition.java

@@ -9,8 +9,8 @@
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.models.datasets.concepts.ConceptElement;
 import com.bakdata.conquery.models.exceptions.ConceptConfigurationException;
+import com.bakdata.conquery.sql.translate.ir.condition.WhereCondition;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext;
-import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition;
 import com.bakdata.conquery.util.CalculatedValue;
 import lombok.Getter;
 import lombok.Setter;

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/CTCondition.java

@@ -14,8 +14,8 @@
 import com.bakdata.conquery.io.cps.CPSBase;
 import com.bakdata.conquery.models.datasets.concepts.ConceptElement;
 import com.bakdata.conquery.models.exceptions.ConceptConfigurationException;
+import com.bakdata.conquery.sql.translate.ir.condition.WhereCondition;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext;
-import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition;
 import com.bakdata.conquery.util.CalculatedValue;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.google.common.collect.Sets;

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java

@@ -12,9 +12,9 @@
 
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.models.datasets.concepts.ConceptElement;
+import com.bakdata.conquery.sql.translate.ir.condition.StringValuesCondition;
+import com.bakdata.conquery.sql.translate.ir.condition.WhereCondition;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext;
-import com.bakdata.conquery.sql.conversion.model.filter.MultiSelectCondition;
-import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition;
 import com.bakdata.conquery.util.CalculatedValue;
 import com.bakdata.conquery.util.CollectionsUtil;
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -58,7 +58,7 @@ public boolean matches(String value, CalculatedValue<Map<String, Object>> rowMap
 	@Override
 	public WhereCondition convertToSqlCondition(CTConditionContext context) {
 		Field<String> field = field(name(column), String.class);
-		return new MultiSelectCondition(field, values.toArray(String[]::new), context.getFunctionProvider());
+		return new StringValuesCondition(field, values.toArray(String[]::new));
 	}
 
 	private int fieldLength() {