diff --git a/.github/workflows/test_backend.yml b/.github/workflows/test_backend.yml index 6bf14d5827..a1800986a7 100644 --- a/.github/workflows/test_backend.yml +++ b/.github/workflows/test_backend.yml @@ -62,4 +62,4 @@ jobs: java-version: '21' overwrite-settings: false - name: ${{ matrix.test-group.name }} - run: mvn test -T 1C -pl backend ${{ matrix.test-group.flags }} + run: mvn test -T 1C -pl backend -am ${{ matrix.test-group.flags }} diff --git a/backend/pom.xml b/backend/pom.xml index 27a433b345..740ba1ea49 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -112,6 +112,16 @@ + + com.bakdata.conquery + dataset-model + ${project.version} + + + com.bakdata.conquery + sql-connector + ${project.version} + org.apache.logging.log4j log4j-to-slf4j @@ -346,7 +356,7 @@ org.jooq jooq - 3.20.2 + ${jooq.version} com.zaxxer diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index 268a6c85a4..fa7c662cd5 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -53,9 +53,9 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.*; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.preview.EntityPreviewExecution; import com.bakdata.conquery.models.query.preview.EntityPreviewForm; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; import com.bakdata.conquery.models.query.resultinfo.printers.JavaResultPrinters; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java index db38fdff61..baeb9a0baf 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java @@ -12,19 +12,20 @@ import com.bakdata.conquery.models.common.Range.LongRange; import com.bakdata.conquery.models.common.Range.MoneyRange; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.identifiable.ids.specific.FilterId; import com.bakdata.conquery.models.query.QueryResolveContext; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; -import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.util.validation.ResolvableId; -import com.fasterxml.jackson.annotation.*; -import lombok.*; +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.OptBoolean; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Setter; +import lombok.ToString; import org.jetbrains.annotations.TestOnly; -import org.jooq.Condition; /** * @implNote The {@link JsonCreator} annos are necessary. Otherwise, Jackson will deserilaize all values as generic objects. @@ -52,19 +53,6 @@ public VALUE readValue() { return ((VALUE) value); } - public SqlFilters convertToSqlFilter(SqlIdColumns ids, ConversionContext context, ConnectorSqlTables tables) { - FilterContext filterContext = FilterContext.forConceptConversion(ids, readValue(), context, tables); - final Filter resolve = (Filter) filter.resolve(); - SqlFilters sqlFilters = resolve.createConverter().convertToSqlFilter(resolve, filterContext); - return sqlFilters; - } - - public Condition convertForTableExport(SqlIdColumns ids, ConversionContext context) { - FilterContext filterContext = FilterContext.forTableExport(ids, readValue(), context); - final Filter resolve = (Filter) filter.resolve(); - return resolve.createConverter().convertForTableExport(resolve, filterContext); - } - @CPSType(id = FrontendFilterType.Fields.MULTI_SELECT, base = FilterValue.class) public static class CQMultiSelectFilter extends FilterValue> { @JsonCreator diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java index cf9577ff15..6eb7d8ce98 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java @@ -17,6 +17,7 @@ import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.C10nCache; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; @@ -24,7 +25,6 @@ import com.bakdata.conquery.models.query.RequiredEntities; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.ExistsAggregator; import com.bakdata.conquery.models.query.queryplan.specific.AndNode; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java index 8288ff72cb..a5f365f86d 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java @@ -22,8 +22,8 @@ import com.bakdata.conquery.models.identifiable.NamespacedIdentifiable; import com.bakdata.conquery.models.identifiable.ids.specific.*; import com.bakdata.conquery.models.query.*; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.EventDateUnionAggregator; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java index 190b021d50..7c5e5e4f82 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java @@ -10,12 +10,12 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.DateAggregationMode; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.specific.NegatingNode; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java index cb019ee307..bbd5ae1afe 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java @@ -17,6 +17,7 @@ import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.C10nCache; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; @@ -24,7 +25,6 @@ import com.bakdata.conquery.models.query.RequiredEntities; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.ExistsAggregator; import com.bakdata.conquery.models.query.queryplan.specific.OrNode; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index eae417caad..ac88e87009 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -8,9 +8,9 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.IntStream; import jakarta.validation.constraints.NotEmpty; import com.bakdata.conquery.apiv1.query.CQElement; @@ -33,7 +33,6 @@ import com.bakdata.conquery.models.types.ResultType; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonView; -import com.google.common.collect.Streams; import io.dropwizard.validation.ValidationMethod; import lombok.AccessLevel; import lombok.Getter; @@ -106,13 +105,7 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { throw new IllegalStateException("CQExternal needs to be resolved before creating a plan"); } - final String[] extraHeaders = Streams.zip( - Arrays.stream(headers), - format.stream(), - (header, format) -> format.equals(EntityResolverUtil.FORMAT_EXTRA) ? header : null - ) - .filter(Objects::nonNull) - .toArray(String[]::new); + final String[] extraHeaders = getExtraHeaders().toArray(String[]::new); if (onlySingles) { return createExternalNodeOnlySingle(context, plan, extraHeaders); @@ -218,19 +211,24 @@ public List getResultInfos() { if (extra == null) { return Collections.emptyList(); } - List resultInfos = new ArrayList<>(); - for (int col = 0; col < format.size(); col++) { - if (!format.get(col).equals(EntityResolverUtil.FORMAT_EXTRA)) { - continue; - } - - final String column = headers[col]; + final ResultType type = onlySingles + ? ResultType.Primitive.STRING + : new ResultType.ListT<>(ResultType.Primitive.STRING); + return getExtraHeaders().stream() + .map(column -> new ExternalResultInfo(column, type)) + .toList(); + } - final ResultType type = onlySingles ? ResultType.Primitive.STRING : new ResultType.ListT<>(ResultType.Primitive.STRING); - resultInfos.add(new ExternalResultInfo(column, type)); + /** Ordered names of the resolved columns that are returned as external result values. */ + @JsonIgnore + public List getExtraHeaders() { + if (headers == null) { + return List.of(); } - - return resultInfos; + return IntStream.range(0, format.size()) + .filter(index -> EntityResolverUtil.FORMAT_EXTRA.equals(format.get(index))) + .mapToObj(index -> headers[index]) + .toList(); } @@ -268,6 +266,9 @@ public boolean isWithExtras() { } public List>> getExtrasForId(String id) { + if (extra == null) { + return List.of(); + } Map> extras = extra.getOrDefault(id, Collections.emptyMap()); // we need to bring the extras in the correct order List>> inOrder = new ArrayList<>(); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java index 5c4eb9333c..2d1a4fb3fd 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java @@ -12,8 +12,16 @@ import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.LocalNamespace; +import com.bakdata.conquery.sql.translate.DefaultSqlTranslator; +import com.bakdata.conquery.sql.translate.FormSqlTranslator; +import com.bakdata.conquery.sql.translate.TableExportSqlTranslator; import com.bakdata.conquery.sql.conquery.SqlExecutionManager; -import com.bakdata.conquery.sql.conversion.NodeConversions; +import com.bakdata.conquery.sql.conversion.ResolvedFormAdapter; +import com.bakdata.conquery.sql.conversion.ResolvedFormMapper; +import com.bakdata.conquery.sql.conversion.ResolvedQueryAdapter; +import com.bakdata.conquery.sql.conversion.ResolvedQueryMapper; +import com.bakdata.conquery.sql.conversion.ResolvedTableExportAdapter; +import com.bakdata.conquery.sql.conversion.ResolvedTableExportMapper; import com.bakdata.conquery.sql.conversion.SqlConverter; import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; import com.bakdata.conquery.sql.execution.ResultSetProcessor; @@ -49,8 +57,18 @@ public LocalNamespace createNamespace( ResultSetProcessor resultSetProcessor = dialectBundle.getResultSetProcessor(config); SqlExecutionService sqlExecutionService = new SqlExecutionService(dslContext, resultSetProcessor); - NodeConversions nodeConversions = new NodeConversions(config.getIdColumns(), dialectBundle, dslContext, sqlExecutionService, clock, connection.getConnection().getPrimaryColumn()); - SqlConverter sqlConverter = new SqlConverter(nodeConversions, config); + ResolvedQueryMapper resolvedQueryMapper = new ResolvedQueryMapper( + config.getIdColumns(), dialectBundle.getFunctionProvider(), clock, + connection.getConnection().getPrimaryColumn()); + ResolvedQueryAdapter resolvedQueryAdapter = new ResolvedQueryAdapter( + environment.getValidator(), new DefaultSqlTranslator(dslContext), resolvedQueryMapper); + ResolvedTableExportAdapter tableExportAdapter = new ResolvedTableExportAdapter( + new ResolvedTableExportMapper(resolvedQueryMapper, clock, connection.getConnection().getPrimaryColumn()), + new TableExportSqlTranslator(dslContext)); + ResolvedFormAdapter formAdapter = new ResolvedFormAdapter( + new ResolvedFormMapper(resolvedQueryMapper), new FormSqlTranslator(dslContext)); + SqlConverter sqlConverter = new SqlConverter( + resolvedQueryAdapter, tableExportAdapter, formAdapter, dialectBundle.getTranslatorDialect()); ExecutionManager executionManager = new SqlExecutionManager(sqlConverter, sqlExecutionService, metaStorage, datasetRegistry, config); SqlStorageHandler sqlStorageHandler = new SqlStorageHandler(sqlExecutionService); SqlEntityResolver sqlEntityResolver = new SqlEntityResolver(config.getIdColumns(), dslContext, dialectBundle, sqlExecutionService); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java index 36ffacd184..c25423a538 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java @@ -17,7 +17,7 @@ import com.bakdata.conquery.models.config.IdColumnConfig; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.identifiable.mapping.ExternalId; -import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.translate.ir.SharedAliases; import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; import com.bakdata.conquery.sql.execution.SqlExecutionService; import com.bakdata.conquery.util.DateReader; @@ -152,7 +152,7 @@ private Map resolveIds(String[][] values, List> resolveIdsQuery = context.with(unresolvedCte) .select(rowIndex, externalPrimaryColumn, isResolved) - .from(dialect.getFunctionProvider().innerJoin(allIdsTable, unresolvedCte, List.of(externalPrimaryColumn.eq(innerPrimaryColumn)))) + .from(allIdsTable.innerJoin(unresolvedCte).on(externalPrimaryColumn.eq(innerPrimaryColumn))) .where(externalPrimaryColumn.eq(innerPrimaryColumn)); return executionService.fetchStream(resolveIdsQuery) diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/IRange.java b/backend/src/main/java/com/bakdata/conquery/models/common/IRange.java index 073da17389..a52999f9b5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/common/IRange.java +++ b/backend/src/main/java/com/bakdata/conquery/models/common/IRange.java @@ -1,13 +1,25 @@ package com.bakdata.conquery.models.common; +import java.util.Optional; + import lombok.NonNull; -public interface IRange> { +public interface IRange, CLASS extends IRange> extends InclusiveRange { VALUE getMin(); VALUE getMax(); + @Override + default Optional minimum() { + return Optional.ofNullable(getMin()); + } + + @Override + default Optional maximum() { + return Optional.ofNullable(getMax()); + } + boolean contains(VALUE value); boolean contains(CLASS other); diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/Range.java b/backend/src/main/java/com/bakdata/conquery/models/common/Range.java index c085c4aec2..6481b4fb8e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/common/Range.java +++ b/backend/src/main/java/com/bakdata/conquery/models/common/Range.java @@ -16,7 +16,7 @@ @With @Getter @EqualsAndHashCode -public class Range implements IRange> { +public class Range> implements IRange> { private final T min; private final T max; @@ -42,24 +42,24 @@ public boolean isOpen() { return max == null || min == null; } - public static > Range exactly(T exactly) { + public static > Range exactly(T exactly) { return new Range<>(exactly, exactly); } @JsonCreator - public static > Range of(@JsonProperty("min") T min, @JsonProperty("max") T max) { + public static > Range of(@JsonProperty("min") T min, @JsonProperty("max") T max) { return new Range<>(min, max); } - public static > Range atMost(T bound) { + public static > Range atMost(T bound) { return new Range<>(null, bound); } - public static > Range atLeast(T bound) { + public static > Range atLeast(T bound) { return new Range<>(bound, null); } - public static > Range all() { + public static > Range all() { return new Range<>(null, null); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/AndCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/AndCondition.java index 6e06ea5118..b452614569 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/AndCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/AndCondition.java @@ -9,8 +9,8 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.models.exceptions.ConceptConfigurationException; +import com.bakdata.conquery.sql.translate.ir.condition.WhereCondition; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import lombok.Getter; import lombok.Setter; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/CTCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/CTCondition.java index 241031bf34..fa20519cf5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/CTCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/CTCondition.java @@ -14,8 +14,8 @@ import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.models.exceptions.ConceptConfigurationException; +import com.bakdata.conquery.sql.translate.ir.condition.WhereCondition; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.collect.Sets; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java index 9b599268ea..ce18572e9a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java @@ -12,9 +12,9 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; +import com.bakdata.conquery.sql.translate.ir.condition.StringValuesCondition; +import com.bakdata.conquery.sql.translate.ir.condition.WhereCondition; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; -import com.bakdata.conquery.sql.conversion.model.filter.MultiSelectCondition; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import com.bakdata.conquery.util.CollectionsUtil; import com.fasterxml.jackson.annotation.JsonCreator; @@ -58,7 +58,7 @@ public boolean matches(String value, CalculatedValue> rowMap @Override public WhereCondition convertToSqlCondition(CTConditionContext context) { Field field = field(name(column), String.class); - return new MultiSelectCondition(field, values.toArray(String[]::new), context.getFunctionProvider()); + return new StringValuesCondition(field, values.toArray(String[]::new)); } private int fieldLength() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java index 5afdfae99c..0a33d2ff52 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java @@ -11,9 +11,9 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; +import com.bakdata.conquery.sql.translate.ir.condition.StringValuesCondition; +import com.bakdata.conquery.sql.translate.ir.condition.WhereCondition; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; -import com.bakdata.conquery.sql.conversion.model.filter.MultiSelectCondition; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import com.bakdata.conquery.util.CollectionsUtil; import com.fasterxml.jackson.annotation.JsonCreator; @@ -46,7 +46,7 @@ public boolean matches(String value, CalculatedValue> rowMap @Override public WhereCondition convertToSqlCondition(CTConditionContext context) { - return new MultiSelectCondition(field(context.getConnectorColumn(), VARCHAR), values.toArray(String[]::new), context.getFunctionProvider()); + return new StringValuesCondition(field(context.getConnectorColumn(), VARCHAR), values.toArray(String[]::new)); } private int fieldLength() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/GroovyCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/GroovyCondition.java index 13bbb04a45..f39a2be8b1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/GroovyCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/GroovyCondition.java @@ -9,8 +9,8 @@ import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.models.exceptions.ConceptConfigurationException; +import com.bakdata.conquery.sql.translate.ir.condition.WhereCondition; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import com.fasterxml.jackson.annotation.JsonIgnore; import groovy.lang.GroovyShell; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsEmptyCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsEmptyCondition.java index f8589fbdc8..295acbb65c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsEmptyCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsEmptyCondition.java @@ -8,9 +8,9 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; +import com.bakdata.conquery.sql.translate.ir.condition.ConditionWrappingWhereCondition; +import com.bakdata.conquery.sql.translate.ir.condition.WhereCondition; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; -import com.bakdata.conquery.sql.conversion.model.filter.ConditionWrappingWhereCondition; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import lombok.Getter; import lombok.NonNull; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsPresentCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsPresentCondition.java index bb36d46238..5e8eaefd37 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsPresentCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsPresentCondition.java @@ -8,9 +8,9 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; +import com.bakdata.conquery.sql.translate.ir.condition.ConditionWrappingWhereCondition; +import com.bakdata.conquery.sql.translate.ir.condition.WhereCondition; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; -import com.bakdata.conquery.sql.conversion.model.filter.ConditionWrappingWhereCondition; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import lombok.Getter; import lombok.NonNull; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/NotCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/NotCondition.java index 3dfb4d902e..38d75d7ab1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/NotCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/NotCondition.java @@ -6,8 +6,8 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.models.exceptions.ConceptConfigurationException; +import com.bakdata.conquery.sql.translate.ir.condition.WhereCondition; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import lombok.Getter; import lombok.Setter; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixCondition.java index cf73ef60af..3ea0945921 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixCondition.java @@ -10,9 +10,9 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; +import com.bakdata.conquery.sql.translate.ir.condition.ConditionWrappingWhereCondition; +import com.bakdata.conquery.sql.translate.ir.condition.WhereCondition; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; -import com.bakdata.conquery.sql.conversion.model.filter.ConditionWrappingWhereCondition; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import lombok.Getter; import lombok.Setter; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixRangeCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixRangeCondition.java index 7b033fa92a..7960163765 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixRangeCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixRangeCondition.java @@ -8,10 +8,10 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; +import com.bakdata.conquery.sql.translate.ir.condition.ConditionWrappingWhereCondition; +import com.bakdata.conquery.sql.translate.ir.condition.WhereCondition; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.filter.ConditionWrappingWhereCondition; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/Filter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/Filter.java index 5f9ee9f52e..030a917a5d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/Filter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/Filter.java @@ -13,7 +13,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.FilterId; import com.bakdata.conquery.models.identifiable.ids.specific.TableId; import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonBackReference; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -96,11 +95,6 @@ public boolean isForConnectorsTable() { @JsonIgnore public abstract List getRequiredColumns(); - @JsonIgnore - public > FilterConverter createConverter() { - throw new UnsupportedOperationException("No converter implemented for Filter %s".formatted(getClass())); - } - @Override public FilterId createId() { return new FilterId(connector.getId(), getName()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java index d5328474a7..cdd834bb31 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java @@ -8,8 +8,6 @@ import com.bakdata.conquery.models.query.filter.event.MultiSelectFilterNode; import com.bakdata.conquery.models.query.filter.event.SubstringMultiSelectFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.EventFilterNode; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; -import com.bakdata.conquery.sql.conversion.model.filter.MultiSelectFilterConverter; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.Setter; @@ -40,9 +38,4 @@ public EventFilterNode> createFilterNode(Set value) { return new MultiSelectFilterNode(getColumn().resolve(), value); } - @Override - public FilterConverter> createConverter() { - //TODO (FK) Converter for Substring - return new MultiSelectFilterConverter(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java index d73f3b9e01..981b00acad 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java @@ -17,8 +17,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.DistinctValuesWrapperAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.CountAggregator; import com.bakdata.conquery.models.query.queryplan.filter.AggregationFilterNode; -import com.bakdata.conquery.sql.conversion.model.aggregator.CountSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; @@ -69,8 +67,4 @@ public List getRequiredColumns() { return out; } - @Override - public FilterConverter createConverter() { - return new CountSqlAggregator(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountQuartersFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountQuartersFilter.java index 269aba930f..e4abedd655 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountQuartersFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountQuartersFilter.java @@ -17,8 +17,6 @@ import com.bakdata.conquery.models.query.filter.RangeFilterNode; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.CountQuartersOfDatesAggregator; import com.bakdata.conquery.models.query.queryplan.filter.AggregationFilterNode; -import com.bakdata.conquery.sql.conversion.model.aggregator.CountQuartersSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import lombok.Getter; import lombok.Setter; @@ -55,8 +53,4 @@ public List getRequiredColumns() { return new RangeFilterNode(value, new CountQuartersOfDatesAggregator(column)); } - @Override - public FilterConverter createConverter() { - return new CountQuartersSqlAggregator(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java index 0b07bd0f24..b7bffe6d5c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java @@ -17,8 +17,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; import com.bakdata.conquery.models.query.filter.event.DateDistanceFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.EventFilterNode; -import com.bakdata.conquery.sql.conversion.model.aggregator.DateDistanceSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.bakdata.conquery.util.validation.ResolvableId; import com.bakdata.conquery.util.validation.SupportedColumnTypes; import lombok.Getter; @@ -63,8 +61,4 @@ public EventFilterNode createFilterNode(Range.LongRange value) { return new DateDistanceFilterNode(getColumn().resolve(), timeUnit, value); } - @Override - public FilterConverter createConverter() { - return new DateDistanceSqlAggregator(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DurationSumFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DurationSumFilter.java index 5afd76ed6c..e37be15435 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DurationSumFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DurationSumFilter.java @@ -21,8 +21,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.specific.DurationSumAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.TwoColumnDurationSumAggregator; import com.bakdata.conquery.models.query.queryplan.filter.AggregationFilterNode; -import com.bakdata.conquery.sql.conversion.model.aggregator.DurationSumSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.bakdata.conquery.util.validation.ResolvableId; import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -88,10 +86,4 @@ private boolean hasDistinct() { return new RangeFilterNode(value, aggregator); } - @Override - public FilterConverter createConverter() { - return new DurationSumSqlAggregator(); - } - - } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java index bd4f78f1d8..1636008935 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java @@ -1,9 +1,6 @@ package com.bakdata.conquery.models.datasets.concepts.filters.specific; import java.util.*; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration; import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; @@ -19,8 +16,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; import com.bakdata.conquery.models.query.filter.event.FlagColumnsFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.EventFilterNode; -import com.bakdata.conquery.sql.conversion.model.aggregator.FlagSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.bakdata.conquery.util.validation.ResolvableId; import com.bakdata.conquery.util.validation.SupportedColumnTypes; import com.fasterxml.jackson.annotation.JsonCreator; @@ -42,9 +37,6 @@ @ToString public class FlagFilter extends EventFilter> { - @Valid - @NotEmpty - @NotNull private final Map flags; @Override @@ -90,9 +82,4 @@ public boolean isAllColumnsOfSameTable() { return flags.values().stream().distinct().count() == flags.size(); } - - @Override - public FilterConverter> createConverter() { - return new FlagSqlAggregator(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/MultiSelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/MultiSelectFilter.java index 8bd5e85a7e..6be0672de2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/MultiSelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/MultiSelectFilter.java @@ -8,8 +8,6 @@ import com.bakdata.conquery.models.query.filter.event.MultiSelectFilterNode; import com.bakdata.conquery.models.query.filter.event.SubstringMultiSelectFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.EventFilterNode; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; -import com.bakdata.conquery.sql.conversion.model.filter.MultiSelectFilterConverter; import com.fasterxml.jackson.annotation.JsonIgnore; /** @@ -39,8 +37,4 @@ public EventFilterNode> createFilterNode(Set value) { return new MultiSelectFilterNode(getColumn().resolve(), value); } - @Override - public FilterConverter> createConverter() { - return new MultiSelectFilterConverter(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java index 29f3aa1fa5..18f0933fa3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java @@ -22,8 +22,6 @@ import com.bakdata.conquery.models.query.filter.event.number.MoneyFilterNode; import com.bakdata.conquery.models.query.filter.event.number.RealFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.EventFilterNode; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; -import com.bakdata.conquery.sql.conversion.model.filter.NumberFilterConverter; import com.bakdata.conquery.util.validation.ResolvableId; import com.bakdata.conquery.util.validation.SupportedColumnTypes; import com.fasterxml.jackson.annotation.JacksonInject; @@ -88,9 +86,4 @@ public EventFilterNode createFilterNode(RANGE value) { }; } - @Override - public FilterConverter, RANGE> createConverter() { - return new NumberFilterConverter<>(); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SingleSelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SingleSelectFilter.java index cb19b38917..7c97ce71c0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SingleSelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SingleSelectFilter.java @@ -8,8 +8,6 @@ import com.bakdata.conquery.models.query.filter.event.MultiSelectFilterNode; import com.bakdata.conquery.models.query.filter.event.SubstringMultiSelectFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.EventFilterNode; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; -import com.bakdata.conquery.sql.conversion.model.filter.SingleSelectFilterConverter; import net.minidev.json.annotate.JsonIgnore; /** @@ -35,8 +33,4 @@ public String getFilterType() { return FrontendFilterType.Fields.SELECT; } - @Override - public FilterConverter createConverter() { - return new SingleSelectFilterConverter(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java index c9d04bc10c..52fbaf4d48 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java @@ -30,8 +30,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.specific.sum.MoneySumAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.sum.RealSumAggregator; import com.bakdata.conquery.models.query.queryplan.filter.AggregationFilterNode; -import com.bakdata.conquery.sql.conversion.model.aggregator.SumSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.bakdata.conquery.util.validation.ResolvableId; import com.bakdata.conquery.util.validation.SupportedColumnTypes; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -112,11 +110,6 @@ public List getRequiredColumns() { return new RangeFilterNode(range, getAggregator()); } - @Override - public FilterConverter, RANGE> createConverter() { - return new SumSqlAggregator<>(); - } - @JsonIgnore private ColumnAggregator getAggregator() { final Column resolvedColumn = getColumn().resolve(); @@ -141,4 +134,5 @@ private ColumnAggregator getAggregator() { default -> throw new IllegalStateException("No Sum Filter for type " + typeId.name()); }; } + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java index 4a1828699e..2d6ddab496 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java @@ -20,7 +20,6 @@ import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonBackReference; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -108,11 +107,6 @@ public boolean isForConnectorsTable() { @JsonIgnore public abstract List getRequiredColumns(); - @JsonIgnore - public SelectConverter createConverter() { - throw new UnsupportedOperationException("No converter implemented for Select %s".formatted(getClass())); - } - @JsonIgnore public boolean isEventDateSelect() { return false; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java index fe4d87818c..7f710b0959 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java @@ -17,8 +17,6 @@ import com.bakdata.conquery.models.query.resultinfo.printers.common.ConceptIdPrinter; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; -import com.bakdata.conquery.sql.conversion.model.select.ConceptColumnSelectConverter; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; @@ -75,12 +73,6 @@ public ResultType getResultType() { return new ResultType.ListT<>(ResultType.Primitive.STRING); } - @Override - public SelectConverter createConverter() { - //TODO bind Select to converter here - return new ConceptColumnSelectConverter(); - } - @Override public ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor) { return isAsIds() ? processor::getIntegerList : processor::getStringList; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java index 5199c9ee76..9b5a6316bc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java @@ -11,8 +11,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.EventDateUnionAggregator; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.select.EventDateUnionSelectConverter; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; /** @@ -41,11 +39,6 @@ public boolean isEventDateSelect() { return true; } - @Override - public SelectConverter createConverter() { - return new EventDateUnionSelectConverter(); - } - @Override public ResultType getResultType() { return new ResultType.ListT<>(ResultType.Primitive.DATE_RANGE); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java index 8710858e1a..fe21607e25 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java @@ -7,8 +7,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.EventDurationSumAggregator; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.select.EventDurationSumSelectConverter; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.google.common.base.Preconditions; import lombok.AccessLevel; @@ -42,11 +40,6 @@ public ResultSetProcessor.Reader createResultSetReader(ResultSetProcess return processor::getInteger; } - @Override - public SelectConverter createConverter() { - return new EventDurationSumSelectConverter(); - } - @Override public ResultType getResultType() { return ResultType.Primitive.INTEGER; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java index 53f7e6208d..fa9264a4fb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java @@ -10,8 +10,6 @@ import com.bakdata.conquery.models.datasets.concepts.select.concept.UniversalSelect; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.ExistsAggregator; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.select.ExistsSelectConverter; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; @@ -33,11 +31,6 @@ public ResultSetProcessor.Reader createResultSetReader(ResultSetProcess return processor::getBoolean; } - @Override - public SelectConverter createConverter() { - return new ExistsSelectConverter(); - } - @Override public ResultType getResultType() { return ResultType.Primitive.BOOLEAN; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java index a014283044..fb5e792758 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java @@ -17,8 +17,6 @@ import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.query.resultinfo.printers.common.OneToManyMappingPrinter; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.select.DistinctSelectConverter; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; @@ -35,11 +33,6 @@ public Aggregator createAggregator() { return new AllValuesAggregator(getColumn().resolve(), getSubstringRange()); } - @Override - public SelectConverter createConverter() { - return new DistinctSelectConverter(); - } - @Override public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { if (getMapping() == null) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java index 6f33e792c5..021c94953d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java @@ -9,8 +9,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.InternToExternMapperId; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.FirstValueAggregator; -import com.bakdata.conquery.sql.conversion.model.select.FirstValueSelectConverter; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.fasterxml.jackson.annotation.JsonCreator; @CPSType(id = "FIRST", base = Select.class) @@ -32,9 +30,4 @@ public Aggregator createAggregator() { - @Override - public SelectConverter createConverter() { - return new FirstValueSelectConverter(); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/LastValueSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/LastValueSelect.java index e5c1fd01a4..8e97baf806 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/LastValueSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/LastValueSelect.java @@ -8,8 +8,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.InternToExternMapperId; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.LastValueAggregator; -import com.bakdata.conquery.sql.conversion.model.select.LastValueSelectConverter; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.fasterxml.jackson.annotation.JsonCreator; @CPSType(id = "LAST", base = Select.class) @@ -28,8 +26,4 @@ public Aggregator createAggregator() { return new LastValueAggregator<>(getColumn().resolve(), getSubstringRange()); } - @Override - public SelectConverter createConverter() { - return new LastValueSelectConverter(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/RandomValueSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/RandomValueSelect.java index 1bc92c9d7a..631b491daf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/RandomValueSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/RandomValueSelect.java @@ -8,8 +8,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.InternToExternMapperId; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.RandomValueAggregator; -import com.bakdata.conquery.sql.conversion.model.select.RandomValueSelectConverter; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.fasterxml.jackson.annotation.JsonCreator; @CPSType(id = "RANDOM", base = Select.class) @@ -27,8 +25,4 @@ public Aggregator createAggregator() { return new RandomValueAggregator<>(getColumn().resolve(), getSubstringRange()); } - @Override - public SelectConverter createConverter() { - return new RandomValueSelectConverter(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountQuartersSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountQuartersSelect.java index 8938a8e9f2..0a9f4ff0da 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountQuartersSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountQuartersSelect.java @@ -11,8 +11,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.CountQuartersOfDatesAggregator; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.aggregator.CountQuartersSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; import lombok.Getter; @@ -61,8 +59,4 @@ public ResultSetProcessor.Reader createResultSetReader(ResultSetProcess return processor::getInteger; } - @Override - public SelectConverter createConverter() { - return new CountQuartersSqlAggregator(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountSelect.java index 2c38b91782..0fe97ef847 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountSelect.java @@ -13,8 +13,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.DistinctValuesWrapperAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.CountAggregator; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.aggregator.CountSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.bakdata.conquery.util.validation.ResolvableId; import lombok.Data; @@ -63,11 +61,6 @@ public List getRequiredColumns() { return out; } - @Override - public SelectConverter createConverter() { - return new CountSqlAggregator(); - } - @Override public ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor) { return processor::getInteger; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateDistanceSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateDistanceSelect.java index 37ac089745..c9dd730883 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateDistanceSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateDistanceSelect.java @@ -12,8 +12,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.DateDistanceAggregator; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.aggregator.DateDistanceSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; import lombok.Getter; @@ -42,11 +40,6 @@ public Aggregator createAggregator() { return new DateDistanceAggregator(getColumn().resolve(), getTimeUnit()); } - @Override - public SelectConverter createConverter() { - return new DateDistanceSqlAggregator(); - } - @Override public ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor) { return processor::getInteger; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java index 400ecbc6ec..0a14b7f628 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java @@ -10,8 +10,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.DateUnionAggregator; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.select.DateUnionSelectConverter; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -57,8 +55,4 @@ public ResultSetProcessor.Reader>> createResultSetReader(Resu return processor::getDateRangeList; } - @Override - public SelectConverter createConverter() { - return new DateUnionSelectConverter(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java index 1780c0e32b..fa1b9942ad 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java @@ -14,8 +14,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.specific.DurationSumAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.TwoColumnDurationSumAggregator; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.aggregator.DurationSumSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.bakdata.conquery.util.validation.ResolvableId; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -80,9 +78,4 @@ public ResultType getResultType() { return ResultType.Primitive.INTEGER; } - @Override - public SelectConverter createConverter() { - //TODO apply distinctBy (though needs to be done once other branches are merged) - return new DurationSumSqlAggregator(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java index 61386bc5e1..382bdb543b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java @@ -14,8 +14,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.FlagsAggregator; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.aggregator.FlagSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -68,11 +66,6 @@ public boolean isAllColumnsBoolean() { return flags.values().stream().map(ColumnId::get).filter(Objects::nonNull).map(Column::getType).allMatch(MajorTypeId.BOOLEAN::equals); } - @Override - public SelectConverter createConverter() { - return new FlagSqlAggregator(); - } - @Override public ResultType getResultType() { return new ResultType.ListT<>(ResultType.Primitive.STRING); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java index 865485eeb8..cf703f0086 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java @@ -2,7 +2,6 @@ import static com.bakdata.conquery.models.types.ResultType.Primitive.STRING; import static com.bakdata.conquery.models.types.ResultType.resolveResultType; -import static org.jooq.impl.DSL.*; import java.util.Collections; import java.util.Set; @@ -25,15 +24,11 @@ import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; -import com.bakdata.conquery.sql.conversion.model.select.SelectContext; -import com.bakdata.conquery.sql.conversion.model.select.SingleColumnSqlSelect; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; import lombok.Getter; -import org.jooq.Field; + @Getter public abstract class MappableSingleColumnSelect extends SingleColumnSelect { @@ -57,31 +52,6 @@ public MappableSingleColumnSelect(ColumnId column, @Nullable InternToExternMappe this.substringRange = substringRange; } - public static SingleColumnSqlSelect getSubstringSelect( - Column column, Range.IntegerRange substringRange, SelectContext selectContext, - String alias) { - - Field field = field(name(selectContext.getTables().getRootTable(), column.getName()), String.class); - - if (substringRange != null && !substringRange.isAll()) { - if (substringRange.isAtLeast()) { - field = substring(field, 1 + substringRange.getMin()); - } - else if (substringRange.isAtMost()) { - field = substring(field, 1, substringRange.getMax()); - } - else { - field = substring(field, 1 + substringRange.getMin(), substringRange.getMax() - substringRange.getMin()); - } - } - - if (alias != null) { - field = field.as(name(alias)); - } - - return new FieldWrapper<>(field, column.getName()); - } - @Override public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { if (mapping == null) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java index 49e38d8b48..02f192d66a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java @@ -21,8 +21,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.specific.sum.MoneySumAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.sum.RealSumAggregator; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.aggregator.SumSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.bakdata.conquery.util.validation.ResolvableId; import com.bakdata.conquery.util.validation.SupportedColumnTypes; @@ -117,11 +115,6 @@ public ResultSetProcessor.Reader createResultSetReader(ResultSetProcess return processor::getInteger; } - @Override - public SelectConverter createConverter() { - return new SumSqlAggregator<>(); - } - @Override public ResultType getResultType() { return ResultType.resolveResultType(getColumn().resolve().getType()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java index a1da6bcc8a..a3fdb38592 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java @@ -14,10 +14,10 @@ import com.bakdata.conquery.models.forms.util.CalendarUnit; import com.bakdata.conquery.models.forms.util.DateContext; import com.bakdata.conquery.models.forms.util.ResultModifier; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.ArrayConceptQueryPlan; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.models.query.queryplan.DateAggregator; import com.bakdata.conquery.models.query.queryplan.QueryPlan; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/DateAggregationMode.java b/backend/src/main/java/com/bakdata/conquery/models/query/DateAggregationMode.java index d54bf4242f..3ec42e5cb0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/DateAggregationMode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/DateAggregationMode.java @@ -4,7 +4,7 @@ /** * Determines how the dates of hit events are aggregated for a representation as a column the resulting table. - * This mode causes a specific {@link com.bakdata.conquery.models.query.queryplan.DateAggregationAction} to be set + * This mode causes a specific {@link DateAggregationAction} to be set * in certain query nodes that influence the date aggregation (see {@link CQOr} and {@link com.bakdata.conquery.models.query.queryplan.specific.OrNode} and others). */ public enum DateAggregationMode { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/RangeFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/RangeFilterNode.java index d011e6bf4d..cb0ecaa85a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/RangeFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/RangeFilterNode.java @@ -9,7 +9,7 @@ * Entity is included, when the result of the aggregator is contained in the range. */ @ToString(callSuper = true) -public class RangeFilterNode> extends AggregationFilterNode, IRange> { +public class RangeFilterNode> extends AggregationFilterNode, IRange> { public RangeFilterNode(IRange filterValue, Aggregator aggregator) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java index e59852310a..60e43f71cd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.forms.util.ResultModifier; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.entity.Entity; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java index 806f068b91..133f0c5a0e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.EmptyBucket; import com.bakdata.conquery.models.identifiable.ids.specific.BucketId; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregationAction.java deleted file mode 100644 index 98fea55885..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregationAction.java +++ /dev/null @@ -1,73 +0,0 @@ -package com.bakdata.conquery.models.query.queryplan; - -import java.util.Iterator; -import java.util.Set; - -import com.bakdata.conquery.models.common.CDateSet; - -/** - * The action that is used in certain nodes of the query plan to combine {@link CDateSet}s collected from lower - * levels of the query plan. - */ -public enum DateAggregationAction { - /** - * Not propagate dates from lower levels to higher. - */ - BLOCK() { - @Override - public CDateSet aggregate(Set all) { - return null; - } - }, - /** - * Merge all collected dates from a lower level into a union. - */ - MERGE() { - @Override - public CDateSet aggregate(Set all) { - CDateSet combined = CDateSet.createEmpty(); - all.forEach(combined::addAll); - return combined; - } - }, - /** - * Intersect all collected date sets from lower level with each other. - */ - INTERSECT() { - @Override - public CDateSet aggregate(Set all) { - if (all.size() < 1) { - return CDateSet.createEmpty(); - } - - Iterator it = all.iterator(); - CDateSet intersection = it.next(); - - if (all.size() == 1) { - return intersection; - } - // Use the first range as mask and subtract all other ranges from it - - - // Intersect - while (it.hasNext()) { - final CDateSet next = it.next(); - intersection.retainAll(next); - } - return intersection; - } - }, - /** - * Build the negative of the the union of all collected dates - */ - NEGATE() { - @Override - public CDateSet aggregate(Set all) { - CDateSet negative = CDateSet.createFull(); - all.forEach(negative::removeAll); - return negative; - } - }; - - public abstract CDateSet aggregate(Set all); -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java index 46dac03a6a..9bd1216401 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java @@ -2,10 +2,12 @@ import java.util.Collection; import java.util.HashSet; +import java.util.Iterator; import java.util.Set; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.events.Bucket; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; @@ -61,10 +63,44 @@ public CDateSet createAggregationResult() { } }); - // Repackage to get the results sorted. Might need some optimization. - return action.aggregate(all); + // Repackage to get the results sorted. Might need some optimization. + return aggregate(action, all); } + static CDateSet aggregate(DateAggregationAction action, Set dateSets) { + return switch (action) { + case BLOCK -> null; + case MERGE -> merge(dateSets); + case INTERSECT -> intersect(dateSets); + case NEGATE -> negate(dateSets); + }; + } + + private static CDateSet merge(Set dateSets) { + CDateSet combined = CDateSet.createEmpty(); + dateSets.forEach(combined::addAll); + return combined; + } + + private static CDateSet intersect(Set dateSets) { + if (dateSets.isEmpty()) { + return CDateSet.createEmpty(); + } + + Iterator iterator = dateSets.iterator(); + CDateSet intersection = iterator.next(); + while (iterator.hasNext()) { + intersection.retainAll(iterator.next()); + } + return intersection; + } + + private static CDateSet negate(Set dateSets) { + CDateSet negative = CDateSet.createFull(); + dateSets.forEach(negative::removeAll); + return negative; + } + public boolean hasChildren() { return !children.isEmpty(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPParentNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPParentNode.java index 529e86860e..33e4cf476d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPParentNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPParentNode.java @@ -8,6 +8,7 @@ import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.events.Bucket; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java index 31b8431500..8674effe33 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java @@ -18,6 +18,7 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.identifiable.ids.specific.BucketId; import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.entity.Entity; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/AndNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/AndNode.java index 1066deefef..5a3258b601 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/AndNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/AndNode.java @@ -4,7 +4,7 @@ import java.util.Collection; import java.util.List; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.QPParentNode; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/NegatingNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/NegatingNode.java index 7604e1d948..10d8131512 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/NegatingNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/NegatingNode.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.events.Bucket; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.queryplan.DateAggregator; import com.bakdata.conquery.models.query.queryplan.QPChainNode; import com.bakdata.conquery.models.query.queryplan.QPNode; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/OrNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/OrNode.java index f708ccdb9b..9d7460b64e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/OrNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/OrNode.java @@ -5,7 +5,7 @@ import java.util.List; import com.bakdata.conquery.models.datasets.Table; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.queryplan.DateAggregator; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.QPParentNode; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java index 6a92f8c30c..f280c49979 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java @@ -17,6 +17,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptIdMapping; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.mapping.ConceptIdMappingTableManager; import com.bakdata.conquery.util.TablePrimaryColumnUtil; import com.google.common.base.Stopwatch; import lombok.Data; @@ -86,31 +87,7 @@ private static Select unionSelects(List> fields = createConceptIdsTable(tableName, mapping.tableFields()); - - insertConceptIdMappings(tableName, fields, mapping.getRows(), dslContext); - createConceptIdIndexes(mapping); - } - - private void createConceptIdIndexes(ConceptIdMapping mapping) { - if (dslContext.dialect().equals(SQLDialect.CLICKHOUSE)) { - //TODO test if skip indices would still do something. - log.trace("Skipping secondary indexes for ClickHouse concept id table {}", mapping.getTableName()); - return; - } - - String indexToken = Integer.toUnsignedString(mapping.getTableName().hashCode(), 16); - if (!mapping.getKeyFields().isEmpty()) { - dslContext.createIndex(name("cq_map_%s_keys".formatted(indexToken))) - .on(mapping.table(), mapping.getKeyFields().stream().map(Field::sortDefault).toList()) - .execute(); - } - dslContext.createIndex(name("cq_map_%s_id".formatted(indexToken))) - .on(mapping.table(), mapping.resolvedId().sortDefault()) - .execute(); + new ConceptIdMappingTableManager(dslContext).recreate(mapping.mappingTable()); } @NotNull @@ -183,38 +160,6 @@ record -> { return matchingStats; } - private void insertConceptIdMappings(Name tableName, List> fieldNames, List rows, DSLContext dsl) { - log.info("BEGIN inserting {} rows into {}", rows.size(), tableName); - Stopwatch stopwatch = Stopwatch.createStarted(); - - // We're using batching here because some DBMS don't allow mass inserts. - // There's a chance, we rework this to use a prepared statement with lots of bindings under the hood. But that needs to rework the entire stream of rows. - List> inserts = new ArrayList<>(rows.size()); - - for (RowN row : rows) { - inserts.add(dsl.insertInto(table(tableName)).columns(fieldNames).values(row)); - } - - dsl.batch(inserts).execute(); - - log.debug("DONE inserting into {} within {}", tableName, stopwatch); - } - - /** - * Create a table and its fields. Assumes the table has been dropped already. - */ - private List> createConceptIdsTable(Name tableName, List> fields) { - log.debug("Creating table {} with fields {}", tableName, fields); - - CreateTableElementListStep createTable = - dslContext.createTable(tableName) - .columns(fields); - - createTable.execute(); - - return fields; - } - public void collectMatchingStatsForConcept(TreeConcept concept, int tries) { int remainingTries = tries; while (true) { @@ -291,13 +236,7 @@ private SelectJoinStep> createMatchin public void deleteConceptIdJoinTable(ConceptId concept) { Name tableName = ConceptIdMapping.tableName(concept); log.debug("Trying to delete id-table {}", tableName); - - try { - dslContext.dropTable(tableName).execute(); - } catch (DataAccessException exception) { - // Likely it doesn't exist. Some DBMS just don't support drop-IfExists so this is the next best thing :^) - log.trace("Failed to drop table {}", tableName, exception); - } + new ConceptIdMappingTableManager(dslContext).delete(tableName); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Context.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/Context.java deleted file mode 100644 index df64b1e9be..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Context.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.bakdata.conquery.sql.conversion; - -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.NameGenerator; - -/** - * Marker for a conversion context. - */ -public interface Context { - - ConversionContext getConversionContext(); - - DialectBundle getDialectBundle(); - - default NameGenerator getNameGenerator() { - return getConversionContext().getNameGenerator(); - } - - default SqlFunctionProvider getFunctionProvider(){ - return getConversionContext().getDialectBundle().getFunctionProvider(); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Conversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/Conversions.java deleted file mode 100644 index 50116dea6e..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Conversions.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.bakdata.conquery.sql.conversion; - -import java.util.List; -import java.util.Optional; - -import lombok.Getter; - -/** - * Converts an input to a result with an applicable converter. - * - * @param type that can be converted - * @param type of the result - * @param context of the convertible - * @see Converter - */ -public abstract class Conversions { - - @Getter - private final List> converters; - - protected Conversions(List> converters) { - this.converters = converters; - } - - public R convert(C node, X context) { - R converted = null; - for (Converter converter : converters) { - Optional maybeConverted = converter.tryConvert(node, context); - if (maybeConverted.isPresent()) { - if (converted == null) { - converted = maybeConverted.get(); - } - else { - throw new IllegalStateException("Multiple Converters for %s".formatted(node)); - } - } - } - - if (converted == null) { - throw new IllegalStateException("No converter found for %s".formatted(node)); - } - - return converted; - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Converter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/Converter.java deleted file mode 100644 index cd34b0af27..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Converter.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.bakdata.conquery.sql.conversion; - -import java.util.Optional; - -/** - * A converter converts an input into a result object if the input matches the conversion class. - * - * @param type that can be converted - * @param type of the result - * @param context of the convertible - */ -public interface Converter { - - default Optional tryConvert(I input, X context) { - if (getConversionClass().isInstance(input)) { - return Optional.ofNullable(convert(getConversionClass().cast(input), context)); - } - return Optional.empty(); - } - - Class getConversionClass(); - - R convert(final C convert, final X context); - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java deleted file mode 100644 index 21c48cd420..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java +++ /dev/null @@ -1,63 +0,0 @@ -package com.bakdata.conquery.sql.conversion; - -import java.time.Clock; -import java.util.Locale; - -import com.bakdata.conquery.apiv1.query.QueryDescription; -import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.config.IdColumnConfig; -import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.query.Visitable; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; -import com.bakdata.conquery.sql.conversion.model.NameGenerator; -import com.bakdata.conquery.sql.execution.SqlExecutionService; -import lombok.NonNull; -import org.jooq.DSLContext; - -/** - * Entry point for converting {@link QueryDescription} to an SQL query. - */ -public class NodeConversions extends Conversions { - - private final IdColumnConfig idColumns; - private final DialectBundle dialect; - private final NameGenerator nameGenerator; - private final SqlExecutionService executionService; - private final Clock clock; - @NonNull - private final String defaultPrimaryColumn; - - public NodeConversions( - IdColumnConfig idColumns, - DialectBundle dialectBundle, - DSLContext dslContext, - SqlExecutionService executionService, - Clock clock, - String defaultPrimaryColumn - ) { - super(dialectBundle.getNodeConverters(dslContext)); - this.idColumns = idColumns; - this.dialect = dialectBundle; - this.nameGenerator = new NameGenerator(dialectBundle.getNameMaxLength()); - this.executionService = executionService; - this.clock = clock; - this.defaultPrimaryColumn = defaultPrimaryColumn; - } - - public ConversionContext convert(QueryDescription queryDescription, ConqueryConfig conqueryConfig) { - ConversionContext initialCtx = ConversionContext.builder() - .idColumns(idColumns) - .sqlPrintSettings(new PrintSettings(false, Locale.ROOT, conqueryConfig, null, null)) - .nameGenerator(nameGenerator) - .nodeConversions(this) - .clock(clock) - .defaultPrimaryColumn(this.defaultPrimaryColumn) - .stratificationFunctions(dialect.getStratificationFunctions()) - .dialectBundle(dialect) - .executionService(executionService) - .build(); - return convert(queryDescription, initialCtx); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverter.java deleted file mode 100644 index 8b49b696fb..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverter.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.bakdata.conquery.sql.conversion; - -import com.bakdata.conquery.models.query.Visitable; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; - -/** - * Interface for converters that implement the translation of a ConQuery query to an SQL query. - * - *

- * A ConQuery is a graph that has a {@link com.bakdata.conquery.apiv1.query.QueryDescription} as its root. - * The children of the root are of type {@link com.bakdata.conquery.apiv1.query.CQElement}. - * - * @param type of the node to convert - */ -public interface NodeConverter extends Converter { - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedFormAdapter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedFormAdapter.java new file mode 100644 index 0000000000..971cf0cb83 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedFormAdapter.java @@ -0,0 +1,37 @@ +package com.bakdata.conquery.sql.conversion; + +import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery; +import com.bakdata.conquery.models.forms.managed.EntityDateQuery; +import com.bakdata.conquery.models.forms.managed.RelativeFormQuery; +import com.bakdata.conquery.sql.translate.TranslatedQuery; +import com.bakdata.conquery.sql.translate.FormSqlTranslator; +import com.bakdata.conquery.sql.translate.dialect.TranslatorDialect; +import com.bakdata.conquery.sql.conversion.model.SqlQuery; + +/** Backend boundary for resolved form compilation. */ +public final class ResolvedFormAdapter { + + private final ResolvedFormMapper mapper; + private final FormSqlTranslator translator; + + public ResolvedFormAdapter(ResolvedFormMapper mapper, FormSqlTranslator translator) { + this.mapper = mapper; + this.translator = translator; + } + + public SqlQuery translate(AbsoluteFormQuery query, TranslatorDialect dialect) { + return adapt(translator.translate(mapper.map(query), dialect), query); + } + + public SqlQuery translate(RelativeFormQuery query, TranslatorDialect dialect) { + return adapt(translator.translate(mapper.map(query), dialect), query); + } + + public SqlQuery translate(EntityDateQuery query, TranslatorDialect dialect) { + return adapt(translator.translate(mapper.map(query), dialect), query); + } + + private static SqlQuery adapt(TranslatedQuery compiled, com.bakdata.conquery.apiv1.query.Query query) { + return SqlQuery.fromCompiled(compiled, query.getResultInfos()); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedFormMapper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedFormMapper.java new file mode 100644 index 0000000000..5f8a9e0690 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedFormMapper.java @@ -0,0 +1,94 @@ +package com.bakdata.conquery.sql.conversion; + +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; +import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery; +import com.bakdata.conquery.models.forms.managed.EntityDateQuery; +import com.bakdata.conquery.models.forms.managed.RelativeFormQuery; +import com.bakdata.conquery.sql.model.ResolvedQuery; +import com.bakdata.conquery.sql.model.form.FormAlignment; +import com.bakdata.conquery.sql.model.form.FormCalendarUnit; +import com.bakdata.conquery.sql.model.form.FormIndexPlacement; +import com.bakdata.conquery.sql.model.form.FormIndexSelector; +import com.bakdata.conquery.sql.model.form.FormResolution; +import com.bakdata.conquery.sql.model.form.RelativeFormSettings; +import com.bakdata.conquery.sql.model.form.ResolutionAndAlignment; +import com.bakdata.conquery.sql.model.form.ResolvedFormMode; +import com.bakdata.conquery.sql.model.form.ResolvedFormQuery; +import com.bakdata.conquery.sql.model.range.DateRange; + +/** Resolves initialized backend form DTOs into the connector-owned form model. */ +public final class ResolvedFormMapper { + + private final ResolvedQueryMapper queryMapper; + + public ResolvedFormMapper(ResolvedQueryMapper queryMapper) { + this.queryMapper = queryMapper; + } + + public ResolvedFormQuery map(AbsoluteFormQuery form) { + return create( + form.getQuery(), + form.getFeatures().getChildQueries(), + new ResolvedFormMode.Absolute( + new DateRange(Optional.ofNullable(form.getDateRange().getMin()), + Optional.ofNullable(form.getDateRange().getMax())), + resolutions(form.getResolutionsAndAlignmentMap())), + form.getResultInfos() + ); + } + + public ResolvedFormQuery map(RelativeFormQuery form) { + RelativeFormSettings settings = new RelativeFormSettings( + FormIndexSelector.valueOf(form.getIndexSelector().name()), + FormIndexPlacement.valueOf(form.getIndexPlacement().name()), + form.getTimeCountBefore(), + form.getTimeCountAfter(), + FormCalendarUnit.valueOf(form.getTimeUnit().name()), + resolutions(form.getResolutionsAndAlignmentMap()) + ); + return create(form.getQuery(), form.getFeatures().getChildQueries(), + new ResolvedFormMode.Relative(settings), form.getResultInfos()); + } + + public ResolvedFormQuery map(EntityDateQuery form) { + Optional bounds = Optional.ofNullable(form.getDateRange()).map(ResolvedFormMapper::dateRange); + return create(form.getQuery(), form.getFeatures().getChildQueries(), + new ResolvedFormMode.EntityDate(bounds, resolutions(form.getResolutionsAndAlignments())), + form.getResultInfos()); + } + + private ResolvedFormQuery create( + com.bakdata.conquery.apiv1.query.Query prerequisite, + List featureQueries, + ResolvedFormMode mode, + List resultInfos + ) { + ResolvedQuery resolvedPrerequisite = queryMapper.map(prerequisite, List.of()); + List features = featureQueries.stream() + .map(feature -> queryMapper.map(feature, Optional.empty(), feature.getResultInfos())) + .toList(); + return new ResolvedFormQuery( + resolvedPrerequisite, + features, + mode, + ResolvedOperationAdapter.resultColumns(resultInfos) + ); + } + + private static List resolutions( + List values + ) { + return values.stream().map(value -> new ResolutionAndAlignment( + FormResolution.valueOf(value.getResolution().name()), + FormAlignment.valueOf(value.getAlignment().name()))).toList(); + } + + private static DateRange dateRange(CDateRange value) { + return new DateRange(Optional.ofNullable(value.getMin()), Optional.ofNullable(value.getMax())); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedOperationAdapter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedOperationAdapter.java new file mode 100644 index 0000000000..ac8613f9d8 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedOperationAdapter.java @@ -0,0 +1,218 @@ +package com.bakdata.conquery.sql.conversion; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; +import com.bakdata.conquery.models.common.IRange; +import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.models.datasets.concepts.conditions.AndCondition; +import com.bakdata.conquery.models.datasets.concepts.conditions.CTCondition; +import com.bakdata.conquery.models.datasets.concepts.conditions.ColumnEqualCondition; +import com.bakdata.conquery.models.datasets.concepts.conditions.EqualCondition; +import com.bakdata.conquery.models.datasets.concepts.conditions.IsEmptyCondition; +import com.bakdata.conquery.models.datasets.concepts.conditions.IsPresentCondition; +import com.bakdata.conquery.models.datasets.concepts.conditions.NotCondition; +import com.bakdata.conquery.models.datasets.concepts.conditions.PrefixCondition; +import com.bakdata.conquery.models.datasets.concepts.conditions.PrefixRangeCondition; +import com.bakdata.conquery.models.datasets.concepts.filters.Filter; +import com.bakdata.conquery.models.datasets.concepts.filters.specific.CountFilter; +import com.bakdata.conquery.models.datasets.concepts.filters.specific.CountQuartersFilter; +import com.bakdata.conquery.models.datasets.concepts.filters.specific.DateDistanceFilter; +import com.bakdata.conquery.models.datasets.concepts.filters.specific.DurationSumFilter; +import com.bakdata.conquery.models.datasets.concepts.filters.specific.FlagFilter; +import com.bakdata.conquery.models.datasets.concepts.filters.specific.NumberFilter; +import com.bakdata.conquery.models.datasets.concepts.filters.specific.SelectFilter; +import com.bakdata.conquery.models.datasets.concepts.filters.specific.SumFilter; +import com.bakdata.conquery.models.datasets.concepts.select.Select; +import com.bakdata.conquery.models.datasets.concepts.select.concept.ConceptColumnSelect; +import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDateUnionSelect; +import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDurationSumSelect; +import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.ExistsSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.DistinctSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.FirstValueSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.LastValueSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.RandomValueSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.CountQuartersSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.CountSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DateDistanceSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DateUnionSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DurationSumSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.FlagSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.SumSelect; +import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.models.types.ResultType; +import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; +import com.bakdata.conquery.sql.model.operation.BuiltInAggregations; +import com.bakdata.conquery.sql.model.operation.BuiltInConditions; +import com.bakdata.conquery.sql.model.operation.BuiltInFilters; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; +import com.bakdata.conquery.sql.model.operation.ResolvedAggregation; +import com.bakdata.conquery.sql.model.operation.ResolvedCondition; +import com.bakdata.conquery.sql.model.operation.ResolvedFilter; +import com.bakdata.conquery.sql.model.operation.ResolvedSelect; +import com.bakdata.conquery.sql.model.range.NumberRange; +import com.bakdata.conquery.sql.model.range.SubstringRange; +import com.bakdata.conquery.sql.model.result.ResultColumn; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.SqlTable; +import lombok.experimental.UtilityClass; + +/** Converts already-resolved backend operations into the connector's immutable input model. */ +@UtilityClass +public class ResolvedOperationAdapter { + + public static ResolvedSelect select(Select select, List conceptColumns, LocalDate endDate) { + return switch (select) { + case ConceptColumnSelect value -> new BuiltInSelects.ConceptValues(value.getName(), conceptColumns); + case EventDateUnionSelect value -> new BuiltInSelects.EventDateUnion(value.getName()); + case EventDurationSumSelect value -> new BuiltInSelects.EventDurationSum(value.getName()); + case ExistsSelect value -> new BuiltInSelects.Exists(value.getName()); + case DistinctSelect value -> values(value, BuiltInSelects.ValueOperation.DISTINCT); + case FirstValueSelect value -> values(value, BuiltInSelects.ValueOperation.FIRST); + case LastValueSelect value -> values(value, BuiltInSelects.ValueOperation.LAST); + case RandomValueSelect value -> values(value, BuiltInSelects.ValueOperation.RANDOM); + case CountSelect value -> aggregation(value.getName(), count(value.getColumn().resolve(), value.isDistinct())); + case CountQuartersSelect value -> aggregation(value.getName(), new BuiltInAggregations.CountQuarters(EntitySchemaAdapter.from(value))); + case DurationSumSelect value -> aggregation(value.getName(), new BuiltInAggregations.DurationSum( + EntitySchemaAdapter.from(value), List.of())); + case SumSelect value -> aggregation(value.getName(), sum( + value.getColumn().resolve(), value.getSubtractColumn(), value.getDistinctByColumn())); + case FlagSelect value -> aggregation(value.getName(), flags(value.getFlags())); + case DateDistanceSelect value -> new BuiltInSelects.DateDistance( + value.getName(), EntitySchemaAdapter.from(value.getColumn().resolve()), value.getTimeUnit(), endDate); + case DateUnionSelect value -> new BuiltInSelects.DateUnion(value.getName(), EntitySchemaAdapter.from(value)); + default -> throw new UnsupportedOperationException("SQL resolved select is not implemented for " + select.getClass()); + }; + } + + public static ResolvedFilter filter(FilterValue filterValue, LocalDate endDate) { + Filter filter = filterValue.getFilter().resolve(); + Object value = filterValue.readValue(); + return switch (filter) { + case SelectFilter selectFilter -> new BuiltInFilters.StringValues( + filter.getName(), EntitySchemaAdapter.from(selectFilter.getColumn().resolve()), stringValues(value), + substring(selectFilter.getSubstringRange())); + case NumberFilter numberFilter -> new BuiltInFilters.NumericColumnRange( + filter.getName(), EntitySchemaAdapter.from(numberFilter.getColumn().resolve()), numberRange((IRange) value)); + case CountFilter countFilter -> aggregationFilter(filter.getName(), + count(countFilter.getColumn().resolve(), countFilter.isDistinct()), value); + case CountQuartersFilter quartersFilter -> aggregationFilter(filter.getName(), + new BuiltInAggregations.CountQuarters(EntitySchemaAdapter.from(quartersFilter)), value); + case DurationSumFilter durationFilter -> aggregationFilter(filter.getName(), + new BuiltInAggregations.DurationSum(EntitySchemaAdapter.from(durationFilter), List.of()), value); + case SumFilter sumFilter -> aggregationFilter(filter.getName(), + sum(sumFilter.getColumn().resolve(), sumFilter.getSubtractColumn(), sumFilter.getDistinctByColumn()), value); + case FlagFilter flagFilter -> new BuiltInFilters.Flags(filter.getName(), flags(flagFilter.getFlags()), (Set) value); + case DateDistanceFilter distanceFilter -> new BuiltInFilters.DateDistanceRange( + filter.getName(), EntitySchemaAdapter.from(distanceFilter.getColumn().resolve()), distanceFilter.getTimeUnit(), + endDate, numberRange((IRange) value)); + default -> throw new UnsupportedOperationException("SQL resolved filter is not implemented for " + filter.getClass()); + }; + } + + public static ResolvedCondition condition(CTCondition condition, SqlTable table, Optional connectorColumn) { + return switch (condition) { + case AndCondition value -> new BuiltInConditions.AllOf(value.getConditions().stream() + .map(child -> condition(child, table, connectorColumn)).toList()); + case NotCondition value -> new BuiltInConditions.Not(condition(value.getCondition(), table, connectorColumn)); + case EqualCondition value -> new BuiltInConditions.StringValues(connectorColumn.orElseThrow(), value.getValues()); + case ColumnEqualCondition value -> new BuiltInConditions.StringValues(stringColumn(table, value.getColumn()), value.getValues()); + case IsPresentCondition value -> new BuiltInConditions.Presence(stringColumn(table, value.getColumn()), true); + case IsEmptyCondition value -> new BuiltInConditions.Presence(stringColumn(table, value.getColumn()), false); + case PrefixCondition value -> new BuiltInConditions.Prefixes(connectorColumn.orElseThrow(), List.of(value.getPrefixes())); + case PrefixRangeCondition value -> new BuiltInConditions.PrefixRange( + connectorColumn.orElseThrow(), value.getMin(), value.getMax()); + default -> throw new UnsupportedOperationException("SQL resolved condition is not implemented for " + condition.getClass()); + }; + } + + public static List resultColumns(List infos) { + return java.util.stream.IntStream.range(0, infos.size()) + .mapToObj(index -> new ResultColumn("result-%d".formatted(index), resultType(infos.get(index).getType()))) + .toList(); + } + + private static BuiltInSelects.Aggregation aggregation(String name, ResolvedAggregation aggregation) { + return new BuiltInSelects.Aggregation(name, aggregation); + } + + private static BuiltInFilters.AggregationRange aggregationFilter(String name, ResolvedAggregation aggregation, Object range) { + return new BuiltInFilters.AggregationRange(name, aggregation, numberRange((IRange) range)); + } + + private static BuiltInSelects.Values values(MappableSingleColumnSelect select, BuiltInSelects.ValueOperation operation) { + return new BuiltInSelects.Values(select.getName(), EntitySchemaAdapter.from(select.getColumn().resolve()), operation, + substring(select.getSubstringRange())); + } + + private static Optional substring(Range.IntegerRange range) { + if (range == null || range.isAll()) { + return Optional.empty(); + } + return Optional.of(new SubstringRange(range.getMin() == null ? 0 : range.getMin(), Optional.ofNullable(range.getMax()))); + } + + private static BuiltInAggregations.Count count(com.bakdata.conquery.models.datasets.Column column, boolean distinct) { + ResolvedColumn resolved = EntitySchemaAdapter.from(column); + return new BuiltInAggregations.Count(resolved, distinct ? List.of(resolved) : List.of()); + } + + private static BuiltInAggregations.Sum sum( + com.bakdata.conquery.models.datasets.Column column, + com.bakdata.conquery.models.identifiable.ids.specific.ColumnId subtract, + List distinctBy + ) { + return new BuiltInAggregations.Sum(EntitySchemaAdapter.from(column), + Optional.ofNullable(subtract).map(id -> EntitySchemaAdapter.from(id.resolve())), columns(distinctBy)); + } + + private static BuiltInAggregations.Flags flags(Map flags) { + Map resolved = new LinkedHashMap<>(); + flags.forEach((name, column) -> resolved.put(name, EntitySchemaAdapter.from(column.resolve()))); + return new BuiltInAggregations.Flags(resolved); + } + + private static List columns(List columns) { + return Optional.ofNullable(columns).orElseGet(List::of).stream() + .map(id -> EntitySchemaAdapter.from(id.resolve())).toList(); + } + + private static Set stringValues(Object value) { + return value instanceof Set values + ? values.stream().map(String.class::cast).collect(java.util.stream.Collectors.toCollection(java.util.LinkedHashSet::new)) + : Set.of((String) value); + } + + private static NumberRange numberRange(IRange range) { + return new NumberRange( + Optional.ofNullable((Number) range.getMin()).map(ResolvedOperationAdapter::decimal), + Optional.ofNullable((Number) range.getMax()).map(ResolvedOperationAdapter::decimal)); + } + + private static BigDecimal decimal(Number value) { + return value instanceof BigDecimal decimal ? decimal : new BigDecimal(value.toString()); + } + + private static ResolvedColumn stringColumn(SqlTable table, String physicalName) { + return new ResolvedColumn(table.logicalId() + "." + physicalName, table, physicalName, + com.bakdata.conquery.models.datasets.ColumnType.STRING, true); + } + + private static com.bakdata.conquery.sql.model.result.ResultType resultType(ResultType type) { + if (type instanceof ResultType.ListT list) { + return new com.bakdata.conquery.sql.model.result.ResultType.ListType(primitive((ResultType.Primitive) list.getElementType())); + } + return primitive((ResultType.Primitive) type); + } + + private static com.bakdata.conquery.sql.model.result.ResultType.Primitive primitive(ResultType.Primitive type) { + return com.bakdata.conquery.sql.model.result.ResultType.Primitive.valueOf(type.name()); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryAdapter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryAdapter.java new file mode 100644 index 0000000000..d1e62834df --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryAdapter.java @@ -0,0 +1,72 @@ +package com.bakdata.conquery.sql.conversion; + +import java.util.List; +import java.util.Optional; +import jakarta.validation.Validator; + +import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; +import com.bakdata.conquery.models.datasets.SecondaryIdDescription; +import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.sql.translate.ColumnRole; +import com.bakdata.conquery.sql.translate.TranslatedQuery; +import com.bakdata.conquery.sql.translate.SqlTranslator; +import com.bakdata.conquery.sql.translate.dialect.TranslatorDialect; +import com.bakdata.conquery.sql.conversion.model.SqlQuery; +import com.bakdata.conquery.sql.model.ResolvedQuery; +import com.bakdata.conquery.sql.validation.ResolvedQueryValidation; + +/** Maps initialized backend queries, validates the resolved input, and adapts translate output to the backend contract. */ +public final class ResolvedQueryAdapter { + + private final ResolvedQueryValidation validation; + private final SqlTranslator translator; + private final ResolvedQueryMapper mapper; + + public ResolvedQueryAdapter(Validator validator, SqlTranslator translator, ResolvedQueryMapper mapper) { + this.validation = new ResolvedQueryValidation(validator); + this.translator = translator; + this.mapper = mapper; + } + + public SqlQuery translate(ConceptQuery query, TranslatorDialect dialect) { + return translate(query, Optional.empty(), query.getResultInfos(), dialect); + } + + public SqlQuery translate(SecondaryIdQuery query, TranslatorDialect dialect) { + return translate( + query.getQuery(), + Optional.of(query.getSecondaryId().resolve()), + query.getResultInfos(), + dialect + ); + } + + private SqlQuery translate( + ConceptQuery query, + Optional secondaryId, + List resultInfos, + TranslatorDialect dialect + ) { + return translate(mapper.map(query, secondaryId, resultInfos), dialect, resultInfos); + } + + public SqlQuery translate( + ResolvedQuery resolvedQuery, + TranslatorDialect dialect, + List resultInfos + ) { + validation.validate(resolvedQuery); + TranslatedQuery compiled = translator.translate(resolvedQuery, dialect); + long resultColumnCount = compiled.columns().stream() + .filter(column -> column.role() == ColumnRole.RESULT) + .count(); + if (resultColumnCount != resultInfos.size()) { + throw new IllegalStateException( + "Translator returned %d result columns for %d backend result descriptors" + .formatted(resultColumnCount, resultInfos.size()) + ); + } + return SqlQuery.fromCompiled(compiled, resultInfos); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java new file mode 100644 index 0000000000..7e9fda1f1a --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java @@ -0,0 +1,235 @@ +package com.bakdata.conquery.sql.conversion; + +import java.time.Clock; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import com.bakdata.conquery.apiv1.query.CQElement; +import com.bakdata.conquery.apiv1.query.CQYes; +import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.apiv1.query.Query; +import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; +import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; +import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import com.bakdata.conquery.apiv1.query.concept.specific.CQDateRestriction; +import com.bakdata.conquery.apiv1.query.concept.specific.CQNegation; +import com.bakdata.conquery.apiv1.query.concept.specific.CQOr; +import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; +import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.config.IdColumnConfig; +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.datasets.SecondaryIdDescription; +import com.bakdata.conquery.models.datasets.concepts.ConceptElement; +import com.bakdata.conquery.models.datasets.concepts.Connector; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; +import com.bakdata.conquery.models.datasets.concepts.select.concept.ConceptColumnSelect; +import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; +import com.bakdata.conquery.models.identifiable.ids.specific.ConceptElementId; +import com.bakdata.conquery.models.identifiable.ids.specific.SelectId; +import com.bakdata.conquery.models.query.DateAggregationAction; +import com.bakdata.conquery.models.query.DateAggregationMode; +import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptIdMapping; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; +import com.bakdata.conquery.sql.mapping.ConceptIdMappingSelection; +import com.bakdata.conquery.sql.model.ResolvedQuery; +import com.bakdata.conquery.sql.model.node.AllEntitiesNode; +import com.bakdata.conquery.sql.model.node.AndNode; +import com.bakdata.conquery.sql.model.node.ConceptNode; +import com.bakdata.conquery.sql.model.node.DateRestrictionNode; +import com.bakdata.conquery.sql.model.node.ExternalEntity; +import com.bakdata.conquery.sql.model.node.ExternalNode; +import com.bakdata.conquery.sql.model.node.NegationNode; +import com.bakdata.conquery.sql.model.node.OrNode; +import com.bakdata.conquery.sql.model.node.QueryNode; +import com.bakdata.conquery.sql.model.operation.BuiltInConditions; +import com.bakdata.conquery.sql.model.operation.ResolvedCondition; +import com.bakdata.conquery.sql.model.operation.ResolvedSelect; +import com.bakdata.conquery.sql.model.range.DateRange; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.ResolvedConnector; +import com.bakdata.conquery.sql.model.schema.ResolvedValidityDate; +import com.bakdata.conquery.sql.model.schema.SqlTable; +import com.bakdata.conquery.util.TablePrimaryColumnUtil; + +/** Resolves the initialized backend concept-query tree into the SQL connector API. */ +public final class ResolvedQueryMapper { + + private final IdColumnConfig idColumns; + private final SqlFunctionProvider functionProvider; + private final Clock clock; + private final String defaultPrimaryColumn; + + public ResolvedQueryMapper( + IdColumnConfig idColumns, + SqlFunctionProvider functionProvider, + Clock clock, + String defaultPrimaryColumn + ) { + this.idColumns = idColumns; + this.functionProvider = functionProvider; + this.clock = clock; + this.defaultPrimaryColumn = defaultPrimaryColumn; + } + + ResolvedQuery map( + ConceptQuery query, + Optional secondaryId, + List resultInfos + ) { + NodeContext context = new NodeContext(secondaryId, Optional.empty()); + return new ResolvedQuery( + EntitySchemaAdapter.from(idColumns), + mapNode(query.getRoot(), context), + query.getResolvedDateAggregationMode() != DateAggregationMode.NONE, + ResolvedOperationAdapter.resultColumns(resultInfos) + ); + } + + ResolvedQuery map(Query query, List resultInfos) { + return switch (query) { + case ConceptQuery concept -> map(concept, Optional.empty(), resultInfos); + case SecondaryIdQuery secondary -> map( + secondary.getQuery(), Optional.of(secondary.getSecondaryId().resolve()), resultInfos); + default -> throw new UnsupportedOperationException( + "Resolved SQL query mapping is not implemented for " + query.getClass()); + }; + } + + private QueryNode mapNode(CQElement node, NodeContext context) { + return switch (node) { + case CQYes ignored -> new AllEntitiesNode(); + case CQAnd and -> new AndNode(and.getChildren().stream().map(child -> mapNode(child, context)).toList(), + and.getDateAction(), and.getCreateExists().orElse(false)); + case CQOr or -> new OrNode(or.getChildren().stream().map(child -> mapNode(child, context)).toList(), + or.getDateAction(), or.getCreateExists().orElse(false)); + case CQNegation negation -> new NegationNode(mapNode(negation.getChild(), context), negation.getDateAction()); + case CQDateRestriction restriction -> { + DateRange range = new DateRange(Optional.ofNullable(restriction.getDateRange().getMin()), + Optional.ofNullable(restriction.getDateRange().getMax())); + yield new DateRestrictionNode(range, mapNode(restriction.getChild(), context.withDateRestriction(range))); + } + case CQExternal external -> external(external); + case CQReusedQuery reused -> mapNode(reused.getResolvedQuery().getReusableComponents(), + reused.isExcludeFromSecondaryId() ? context.withSecondaryId(Optional.empty()) : context); + case CQConcept concept -> concept(concept, context); + default -> throw new UnsupportedOperationException("SQL resolved query node is not implemented for " + node.getClass()); + }; + } + + private ConceptNode concept(CQConcept query, NodeContext context) { + TreeConcept concept = (TreeConcept) query.getConcept(); + List connectors = query.getTables().stream().map(table -> table.getConnector().resolve()).toList(); + List conceptColumns = connectors.stream() + .map(Connector::getColumn).filter(java.util.Objects::nonNull) + .map(id -> EntitySchemaAdapter.from(id.resolve())).toList(); + LocalDate endDate = context.dateRestriction().flatMap(DateRange::endInclusive).orElseGet(() -> LocalDate.now(clock)); + List conceptSelects = query.getSelects().stream().map(SelectId::resolve) + .map(select -> ResolvedOperationAdapter.select(select, conceptColumns, endDate)).toList(); + boolean resolveConceptIds = query.getSelects().stream().map(SelectId::resolve) + .filter(ConceptColumnSelect.class::isInstance).map(ConceptColumnSelect.class::cast) + .anyMatch(ConceptColumnSelect::isAsIds); + List> selected = query.getElements().stream().>map(ConceptElementId::resolve).toList(); + List resolvedConnectors = query.getTables().stream() + .map(table -> connector(query, table, concept, selected, resolveConceptIds, conceptColumns, endDate, context)) + .toList(); + return new ConceptNode( + query.userLabel(java.util.Locale.ROOT), + resolvedConnectors, + conceptSelects, + query.isAggregateEventDates() ? DateAggregationAction.MERGE : DateAggregationAction.BLOCK + ); + } + + private ResolvedConnector connector( + CQConcept query, + CQTable table, + TreeConcept concept, + List> selected, + boolean resolveConceptIds, + List conceptColumns, + LocalDate endDate, + NodeContext context + ) { + Connector connector = table.getConnector().resolve(); + SqlTable sqlTable = EntitySchemaAdapter.from(connector.getResolvedTable()); + org.jooq.Field primary = TablePrimaryColumnUtil.findPrimaryColumn(connector.getResolvedTable(), defaultPrimaryColumn); + ResolvedColumn primaryId = new ResolvedColumn( + sqlTable.logicalId() + ".primary-id", sqlTable, primary.getName(), + com.bakdata.conquery.models.datasets.ColumnType.STRING, false); + Optional secondaryId = Optional.empty(); + if (!query.isExcludeFromSecondaryId() && context.secondaryId().isPresent() + && table.hasSelectedSecondaryId(context.secondaryId().orElseThrow().getId())) { + Column column = connector.getResolvedTable().findSecondaryIdColumn(context.secondaryId().orElseThrow().getId()); + secondaryId = Optional.of(EntitySchemaAdapter.from(column)); + } + Optional connectorColumn = Optional.ofNullable(connector.getColumn()) + .map(id -> EntitySchemaAdapter.from(id.resolve())); + List conditions = new ArrayList<>(); + if (connector.getCondition() != null) { + conditions.add(ResolvedOperationAdapter.condition(connector.getCondition(), sqlTable, connectorColumn)); + } + connectorColumn.ifPresent(column -> conditions.add(new BuiltInConditions.Presence(column, true))); + List selects = table.getSelects().stream().map(SelectId::resolve) + .map(select -> ResolvedOperationAdapter.select(select, conceptColumns, endDate)).toList(); + ConceptIdMapping mapping = new ConceptIdMapping(concept, functionProvider); + ConceptIdMappingSelection mappingSelection = new ConceptIdMappingSelection( + mapping.source(connector), mapping.includedLocalIds(selected), mapping.includesRoot(selected), + resolveConceptIds, concept.getLocalId()); + return new ResolvedConnector( + connector.getName(), sqlTable, primaryId, secondaryId, validityDate(table.findValidityDate()), + table.getFilters().stream().map(filter -> ResolvedOperationAdapter.filter(filter, endDate)).toList(), + selects, conditions, Optional.of(mappingSelection) + ); + } + + static ResolvedValidityDate validityDate(ValidityDate validityDate) { + if (validityDate == null) { + return new ResolvedValidityDate.None(); + } + if (validityDate.isSingleColumnDaterange()) { + return new ResolvedValidityDate.Point(EntitySchemaAdapter.from(validityDate.getColumn().resolve())); + } + return new ResolvedValidityDate.Range( + EntitySchemaAdapter.from(validityDate.getStartColumn().resolve()), + EntitySchemaAdapter.from(validityDate.getEndColumn().resolve())); + } + + private static ExternalNode external(CQExternal external) { + List entities = external.getValuesResolved().entrySet().stream() + .map(entry -> externalEntity(external, entry.getKey(), entry.getValue().asRanges())).toList(); + return new ExternalNode(entities, external.getExtraHeaders()); + } + + private static ExternalEntity externalEntity( + CQExternal external, + String entityId, + Collection validityDates + ) { + Map> values = new LinkedHashMap<>(); + external.getExtrasForId(entityId).forEach(entry -> values.put(entry.getKey(), entry.getValue())); + return new ExternalEntity(entityId, validityDates.stream().map(ResolvedQueryMapper::dateRange).toList(), values); + } + + private static DateRange dateRange(CDateRange range) { + return new DateRange(Optional.ofNullable(range.getMin()), Optional.ofNullable(range.getMax())); + } + + private record NodeContext(Optional secondaryId, Optional dateRestriction) { + private NodeContext withSecondaryId(Optional value) { + return new NodeContext(value, dateRestriction); + } + + private NodeContext withDateRestriction(DateRange value) { + return new NodeContext(secondaryId, Optional.of(value)); + } + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedTableExportAdapter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedTableExportAdapter.java new file mode 100644 index 0000000000..de2d2f2387 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedTableExportAdapter.java @@ -0,0 +1,24 @@ +package com.bakdata.conquery.sql.conversion; + +import com.bakdata.conquery.apiv1.query.TableExportQuery; +import com.bakdata.conquery.sql.translate.TranslatedQuery; +import com.bakdata.conquery.sql.translate.TableExportSqlTranslator; +import com.bakdata.conquery.sql.translate.dialect.TranslatorDialect; +import com.bakdata.conquery.sql.conversion.model.SqlQuery; + +/** Backend boundary for resolved row-level table exports. */ +public final class ResolvedTableExportAdapter { + + private final ResolvedTableExportMapper mapper; + private final TableExportSqlTranslator translator; + + public ResolvedTableExportAdapter(ResolvedTableExportMapper mapper, TableExportSqlTranslator compiler) { + this.mapper = mapper; + this.translator = compiler; + } + + public SqlQuery translate(TableExportQuery query, TranslatorDialect dialect) { + TranslatedQuery compiled = translator.translate(mapper.map(query, dialect), dialect); + return SqlQuery.fromCompiled(compiled, query.getResultInfos()); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedTableExportMapper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedTableExportMapper.java new file mode 100644 index 0000000000..84dd0f887b --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedTableExportMapper.java @@ -0,0 +1,85 @@ +package com.bakdata.conquery.sql.conversion; + +import java.time.Clock; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.apiv1.query.TableExportQuery; +import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; +import com.bakdata.conquery.sql.translate.dialect.TranslatorDialect; +import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; +import com.bakdata.conquery.sql.model.export.ResolvedExportTable; +import com.bakdata.conquery.sql.model.export.ResolvedTableExportQuery; +import com.bakdata.conquery.sql.model.range.DateRange; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.SqlTable; +import com.bakdata.conquery.util.TablePrimaryColumnUtil; + +/** Resolves backend table-export DTOs into the connector-owned export model. */ +public final class ResolvedTableExportMapper { + + private final ResolvedQueryMapper queryMapper; + private final Clock clock; + private final String defaultPrimaryColumn; + + public ResolvedTableExportMapper(ResolvedQueryMapper queryMapper, Clock clock, String defaultPrimaryColumn) { + this.queryMapper = queryMapper; + this.clock = clock; + this.defaultPrimaryColumn = defaultPrimaryColumn; + } + + public ResolvedTableExportQuery map(TableExportQuery query, TranslatorDialect dialect) { + if (!(query.getQuery() instanceof ConceptQuery prerequisite)) { + throw new UnsupportedOperationException("Resolved table exports currently require a concept-query prerequisite"); + } + DateRange restriction = new DateRange(Optional.ofNullable(query.getDateRange().getMin()), + Optional.ofNullable(query.getDateRange().getMax())); + LocalDate endDate = restriction.endInclusive().orElseGet(() -> LocalDate.now(clock)); + List tables = query.getConcepts().stream() + .flatMap(concept -> concept.getTables().stream().map(table -> table( + concept, table, query.getPositions(), endDate))) + .toList(); + return new ResolvedTableExportQuery( + queryMapper.map(prerequisite, Optional.empty(), prerequisite.getResultInfos()), + restriction, + tables, + ResolvedOperationAdapter.resultColumns(query.getResultInfos()) + ); + } + + private ResolvedExportTable table(CQConcept concept, CQTable table, Map positions, + LocalDate endDate) { + com.bakdata.conquery.models.datasets.concepts.Connector connector = table.getConnector().resolve(); + SqlTable sqlTable = EntitySchemaAdapter.from(connector.getResolvedTable()); + org.jooq.Field primary = TablePrimaryColumnUtil.findPrimaryColumn( + connector.getResolvedTable(), defaultPrimaryColumn); + ResolvedColumn primaryId = new ResolvedColumn(sqlTable.logicalId() + ".primary-id", sqlTable, + primary.getName(), com.bakdata.conquery.models.datasets.ColumnType.STRING, false); + int width = TableExportQuery.calculateWidth(positions) - 1; + List> output = new ArrayList<>(java.util.Collections.nCopies(width, Optional.empty())); + for (Column column : connector.getResolvedTable().getColumns()) { + Integer position = positions.get(column.getId()); + if (position != null) { + output.set(position - 1, Optional.of(EntitySchemaAdapter.from(column))); + } + } + return new ResolvedExportTable( + concept.userLabel(Locale.ROOT), + connector.getName(), + sqlTable, + primaryId, + ResolvedQueryMapper.validityDate(table.findValidityDate()), + connector.getResolvedTable().getName(), + output, + table.getFilters().stream().map(filter -> ResolvedOperationAdapter.filter(filter, endDate)).toList() + ); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java index 3b7f22822b..6b690b85b5 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java @@ -1,22 +1,54 @@ package com.bakdata.conquery.sql.conversion; +import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.QueryDescription; -import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; +import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; +import com.bakdata.conquery.apiv1.query.TableExportQuery; +import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery; +import com.bakdata.conquery.models.forms.managed.EntityDateQuery; +import com.bakdata.conquery.models.forms.managed.RelativeFormQuery; +import com.bakdata.conquery.sql.translate.dialect.TranslatorDialect; import com.bakdata.conquery.sql.conversion.model.SqlQuery; public class SqlConverter { - private final NodeConversions nodeConversions; - private final ConqueryConfig config; + private final ResolvedQueryAdapter resolvedQueryAdapter; + private final ResolvedTableExportAdapter tableExportAdapter; + private final ResolvedFormAdapter formAdapter; + private final TranslatorDialect dialect; - public SqlConverter(NodeConversions nodeConversions, ConqueryConfig config) { - this.nodeConversions = nodeConversions; - this.config = config; + public SqlConverter( + ResolvedQueryAdapter resolvedQueryAdapter, + ResolvedTableExportAdapter tableExportAdapter, + ResolvedFormAdapter formAdapter, + TranslatorDialect dialect + ) { + this.resolvedQueryAdapter = resolvedQueryAdapter; + this.tableExportAdapter = tableExportAdapter; + this.formAdapter = formAdapter; + this.dialect = dialect; } public SqlQuery convert(QueryDescription queryDescription) { - ConversionContext converted = nodeConversions.convert(queryDescription, config); - return converted.getFinalQuery(); + if (queryDescription instanceof ConceptQuery conceptQuery) { + return resolvedQueryAdapter.translate(conceptQuery, dialect); + } + if (queryDescription instanceof SecondaryIdQuery secondaryIdQuery) { + return resolvedQueryAdapter.translate(secondaryIdQuery, dialect); + } + if (queryDescription instanceof TableExportQuery tableExportQuery) { + return tableExportAdapter.translate(tableExportQuery, dialect); + } + if (queryDescription instanceof AbsoluteFormQuery absoluteForm) { + return formAdapter.translate(absoluteForm, dialect); + } + if (queryDescription instanceof RelativeFormQuery relativeForm) { + return formAdapter.translate(relativeForm, dialect); + } + if (queryDescription instanceof EntityDateQuery entityDateForm) { + return formAdapter.translate(entityDateForm, dialect); + } + throw new UnsupportedOperationException( + "SQL compilation is not implemented for " + queryDescription.getClass()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java deleted file mode 100644 index 6c8f98ced3..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement; - -import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; -import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.model.ConqueryJoinType; -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; -import com.bakdata.conquery.sql.conversion.model.select.ExistsSqlSelect; - -public class CQAndConverter implements NodeConverter { - - @Override - public Class getConversionClass() { - return CQAnd.class; - } - - @Override - public ConversionContext convert(CQAnd andNode, ConversionContext context) { - - QueryStep joined; - if (andNode.getChildren().size() == 1) { - ConversionContext withConvertedChild = context.getNodeConversions().convert(andNode.getChildren().get(0), context); - joined = withConvertedChild.getLastConvertedStep(); - } - else { - joined = QueryStepJoiner.joinChildren( - andNode.getChildren(), - context, - ConqueryJoinType.INNER_JOIN, - andNode.getDateAction() - ); - } - - if (andNode.getCreateExists().isEmpty()) { - return context.withQueryStep(joined); - } - - String joinedNodeName = joined.getCteName(); - ExistsSqlSelect existsSqlSelect = ExistsSqlSelect.withAlias(joinedNodeName); - return context.withQueryStep(joined.addSqlSelect(existsSqlSelect)); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQDateRestrictionConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQDateRestrictionConverter.java deleted file mode 100644 index 5ce01a3634..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQDateRestrictionConverter.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement; - -import com.bakdata.conquery.apiv1.query.concept.specific.CQDateRestriction; -import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.sql.conversion.NodeConverter; - -public class CQDateRestrictionConverter implements NodeConverter { - - @Override - public ConversionContext convert(CQDateRestriction dateRestrictionNode, ConversionContext context) { - ConversionContext childContext = context.withDateRestrictionRange(CDateRange.of(dateRestrictionNode.getDateRange())); - return context.getNodeConversions().convert(dateRestrictionNode.getChild(), childContext).withDateRestrictionRange(null); - } - - @Override - public Class getConversionClass() { - return CQDateRestriction.class; - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java deleted file mode 100644 index 0d1d32d34f..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java +++ /dev/null @@ -1,180 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement; - -import static org.jooq.impl.DSL.*; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; -import com.bakdata.conquery.models.common.CDateSet; -import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.SharedAliases; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; -import com.google.common.base.Preconditions; -import org.jooq.Field; -import org.jooq.Name; -import org.jooq.Record; -import org.jooq.Table; -import org.jooq.impl.DSL; - -public class CQExternalConverter implements NodeConverter { - - private static final String CQ_EXTERNAL_IDS_CTE_NAME = "external_ids"; - private static final String CQ_EXTERNAL_EXTRAS_CTE_NAME = "external_extra"; - private static final String UNDERSCORE = "_"; - private static final String WHITESPACE = " "; - - private static QueryStep createExternalIdsCte(CQExternal external, SqlFunctionProvider functionProvider, ConversionContext context) { - List unions = new ArrayList<>(); - for (Map.Entry entry : external.getValuesResolved().entrySet()) { - List rowSelects = createRowSelects(entry, functionProvider); - unions.addAll(rowSelects); - } - Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 resolved row when converting a CQExternal"); - QueryStep allStep = QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_IDS_CTE_NAME, Collections.emptyList(), context.isNegation(), functionProvider); - - Optional maybeValidityDate = allStep.getSelects().getValidityDate(); - - if (maybeValidityDate.isEmpty() || !context.dateRestrictionActive()) { - return allStep; - } - - return QueryStep.builder() - .predecessors(List.of(allStep)) - .conditions(List.of(functionProvider.dateRestriction(functionProvider.forCDateRange(context.getDateRestrictionRange()), maybeValidityDate.get()))) - .selects(allStep.getQualifiedSelects()) - .fromTable(table(name(allStep.getCteName()))) - .cteName(CQ_EXTERNAL_IDS_CTE_NAME + "_date_restriction") - .build(); - - } - - /** - * For each entry, we need to create a SELECT statement of static values for each pid -> date set. For dialects that support date multiranges, - * 1 row per ID is sufficient. For other dialects there can be multiple rows with the same pid -> date range from the date set. - */ - private static List createRowSelects(Map.Entry entry, SqlFunctionProvider functionProvider) { - SqlIdColumns ids = createIdSelect(entry, functionProvider); - List validityDateEntries = functionProvider.forCDateSet(entry.getValue(), SharedAliases.DATES_COLUMN); - return validityDateEntries.stream() - .map(validityDateEntry -> createIdRowSelect(ids, validityDateEntry, functionProvider)) - .toList(); - } - - private static SqlIdColumns createIdSelect(Map.Entry entry, SqlFunctionProvider functionProvider) { - Field primaryColumn = functionProvider.externalId(entry.getKey()).as(SharedAliases.PRIMARY_COLUMN.getAlias()); - return new SqlIdColumns(primaryColumn); - } - - /** - * For each entry, we need to create a SELECT statement of static values for each pid -> extras. - */ - private QueryStep createRowSelects( - Map.Entry entry, - List>> extra, - SqlFunctionProvider functionProvider - ) { - SqlIdColumns ids = createIdSelect(entry); - List extraSelects = extra.stream().map(extraEntry -> createExtraColumnValue(extraEntry, functionProvider)).collect(Collectors.toList()); - return createExtraRowSelect(ids, extraSelects, functionProvider); - } - - private static SqlIdColumns createIdSelect(Map.Entry entry) { - Field primaryColumn = DSL.val(entry.getKey()).coerce(String.class).as(SharedAliases.PRIMARY_COLUMN.getAlias()); - return new SqlIdColumns(primaryColumn); - } - - private static FieldWrapper createExtraColumnValue(Map.Entry> extraEntry, SqlFunctionProvider functionProvider) { - Field extraValues = functionProvider.asArrayRepr(extraEntry.getValue()); - final Name alias = name(extraEntry.getKey().replace(WHITESPACE, UNDERSCORE)); - final Field withAlias = extraValues.as(alias); - return new FieldWrapper<>(withAlias); - } - - /** - * Creates a SELECT statement of static values for each pid -> date entry, like - * 

{@code select 1 as "pid", '[2021-01-01,2022-01-01)'::daterange as "date_range"}
- */ - private static QueryStep createIdRowSelect( - SqlIdColumns ids, - ColumnDateRange validityDate, - SqlFunctionProvider functionProvider - ) { - Selects selects = Selects.builder() - .ids(ids) - .validityDate(Optional.ofNullable(validityDate)) - .build(); - return wrapInQueryStep(selects, functionProvider); - } - - /** - * Creates a SELECT statement of static values for each pid -> extras, like - * 
{@code select 1 as "pid", '1' as "payload_1"}
- */ - private static QueryStep createExtraRowSelect( - SqlIdColumns ids, - List extraSelects, - SqlFunctionProvider functionProvider - ) { - Selects selects = Selects.builder() - .ids(ids) - .sqlSelects(extraSelects) - .build(); - return wrapInQueryStep(selects, functionProvider); - } - - private static QueryStep wrapInQueryStep(Selects selects, SqlFunctionProvider functionProvider) { - - // not all SQL dialects can create a SELECT statement without a FROM clause, - // so we ensure there is some no-op table to select the static values from - Table table = functionProvider.getNoOpTable(); - - return QueryStep.builder() - .selects(selects) - .fromTable(table) - .build(); - } - - @Override - public Class getConversionClass() { - return CQExternal.class; - } - - @Override - public ConversionContext convert(CQExternal external, ConversionContext context) { - SqlFunctionProvider functionProvider = context.getDialectBundle().getFunctionProvider(); - - QueryStep externalIdsCte = createExternalIdsCte(external, functionProvider, context); - ConversionContext withExternalIdCte = context.withQueryStep(externalIdsCte); - - if (!external.isWithExtras()) { - return withExternalIdCte; - } - - QueryStep externalExtrasCte = createExternalExtrasCte(external, functionProvider, context); - return withExternalIdCte.withExternalExtras(externalExtrasCte); - } - - private QueryStep createExternalExtrasCte(CQExternal external, SqlFunctionProvider functionProvider, ConversionContext context) { - List unions = new ArrayList<>(); - for (Map.Entry entry : external.getValuesResolved().entrySet()) { - List>> extrasForId = external.getExtrasForId(entry.getKey()); - QueryStep rowSelects = createRowSelects(entry, extrasForId, functionProvider); - unions.add(rowSelects); - } - Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 converted resolved row when converting a CQExternal"); - return QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_EXTRAS_CTE_NAME, Collections.emptyList(), context.isNegation(), functionProvider); - } - - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java deleted file mode 100644 index 2bb90a6d62..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement; - -import com.bakdata.conquery.apiv1.query.concept.specific.CQNegation; -import com.bakdata.conquery.sql.conversion.NodeConverter; - -public class CQNegationConverter implements NodeConverter { - - @Override - public Class getConversionClass() { - return CQNegation.class; - } - - - @Override - public ConversionContext convert(CQNegation negationNode, ConversionContext context) { - return context.getNodeConversions() - .convert(negationNode.getChild(), context.withNegation(true)) - .withNegation(false); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java deleted file mode 100644 index d77d52a207..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java +++ /dev/null @@ -1,44 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement; - -import com.bakdata.conquery.apiv1.query.concept.specific.CQOr; -import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.model.ConqueryJoinType; -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; -import com.bakdata.conquery.sql.conversion.model.select.ExistsSqlSelect; - -public class CQOrConverter implements NodeConverter { - - @Override - public Class getConversionClass() { - return CQOr.class; - } - - @Override - public ConversionContext convert(CQOr orNode, ConversionContext context) { - - QueryStep joined; - if (orNode.getChildren().size() == 1) { - ConversionContext withConvertedChild = context.getNodeConversions().convert(orNode.getChildren().get(0), context); - joined = withConvertedChild.getLastConvertedStep(); - } - else { - joined = QueryStepJoiner.joinChildren( - orNode.getChildren(), - context, - ConqueryJoinType.OUTER_JOIN, - orNode.getDateAction() - ); - } - - if (orNode.getCreateExists().isEmpty()) { - return context.withQueryStep(joined); - } - - String joinedNodeName = joined.getCteName(); - ExistsSqlSelect existsSqlSelect = ExistsSqlSelect.withAlias(joinedNodeName); - return context.withQueryStep(joined.addSqlSelect(existsSqlSelect)); - } - - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java deleted file mode 100644 index 4df45eeaf9..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java +++ /dev/null @@ -1,44 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement; - -import static org.jooq.impl.DSL.*; - -import java.util.Optional; - -import com.bakdata.conquery.apiv1.query.CQYes; -import com.bakdata.conquery.models.config.ColumnConfig; -import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import org.jooq.Field; -import org.jooq.Record; - -public class CQYesConverter implements NodeConverter { - - private static final String ALL_IDS_CTE = "all_ids"; - - @Override - public Class getConversionClass() { - return CQYes.class; - } - - @Override - public ConversionContext convert(CQYes cqYes, ConversionContext context) { - - ColumnConfig primaryColumnConfig = context.getIdColumns().findPrimaryIdColumn(); - Field primaryColumn = field(name(primaryColumnConfig.getField()), String.class); - SqlIdColumns ids = new SqlIdColumns(primaryColumn); - - Selects selects = Selects.builder().ids(ids) - .validityDate(Optional.of(context.getFunctionProvider().emptyColumnDateRange().asValidityDateRange(ALL_IDS_CTE))) - .build(); - org.jooq.Table fromTable = table(name(context.getIdColumns().getTable())); - - QueryStep cqYesTep = QueryStep.builder() - .cteName(ALL_IDS_CTE) - .selects(selects) - .fromTable(fromTable) - .build(); - return context.withQueryStep(cqYesTep); - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java deleted file mode 100644 index b0b696dbbc..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java +++ /dev/null @@ -1,106 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement; - -import java.time.Clock; -import java.util.List; -import javax.annotation.Nullable; - -import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; -import com.bakdata.conquery.apiv1.query.concept.specific.CQDateRestriction; -import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.models.config.IdColumnConfig; -import com.bakdata.conquery.models.datasets.SecondaryIdDescription; -import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.sql.conversion.Context; -import com.bakdata.conquery.sql.conversion.NodeConversions; -import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; -import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; -import com.bakdata.conquery.sql.conversion.model.NameGenerator; -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.SqlQuery; -import com.bakdata.conquery.sql.execution.SqlExecutionService; -import lombok.Builder; -import lombok.NonNull; -import lombok.Singular; -import lombok.Value; -import lombok.With; - -//TODO this class is less context and more state. It's also incredibly dangerous because it hides a lot of moving parts and creates indirections. -@Value -@With -@Builder(toBuilder = true) -public class ConversionContext implements Context { - - IdColumnConfig idColumns; - - Clock clock; - @NonNull - StratificationFunctions stratificationFunctions; - String defaultPrimaryColumn; - PrintSettings sqlPrintSettings; - NodeConversions nodeConversions; - DialectBundle dialectBundle; - NameGenerator nameGenerator; - SqlExecutionService executionService; - @Singular - List querySteps; - @Nullable - SqlQuery finalQuery; - @Nullable - QueryStep stratificationTable; - @Nullable - QueryStep externalExtras; - - /** - * An optional date restriction range. Is set when converting a {@link CQDateRestriction}. - */ - @Nullable - CDateRange dateRestrictionRange; - - /** - * An optional secondary id to group results by in addition to the primary id. Only set when converting {@link SecondaryIdQuery}s. - */ - @Nullable - SecondaryIdDescription secondaryIdDescription; - - boolean negation; - - boolean isGroupBy; - - public boolean dateRestrictionActive() { - return this.dateRestrictionRange != null; - } - - public boolean isWithStratification() { - return this.stratificationTable != null; - } - - - - /** - * Adds a query step to the list of {@link QueryStep} of this context. - */ - public ConversionContext withQueryStep(QueryStep queryStep) { - return this.toBuilder().queryStep(queryStep).build(); - } - - /** - * @return Creates a child context that keeps the meta information of this context, but empties the list of already converted querySteps. - */ - public ConversionContext createChildContext() { - return this.toBuilder().clearQuerySteps().build(); - } - - @Override - public ConversionContext getConversionContext() { - return this; - } - - - /** - * Get the last query {@link QueryStep} that has been added to this context query steps. - */ - public QueryStep getLastConvertedStep() { - return this.querySteps.getLast(); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java deleted file mode 100644 index 9b3fa29c8d..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java +++ /dev/null @@ -1,118 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.aggregation; - - -import java.sql.Date; -import java.util.List; - -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingContext; -import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep; -import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; -import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlTables; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; -import lombok.Data; -import org.jooq.Field; - -@Data -public class AnsiSqlDateAggregator implements SqlDateAggregator { - - private final IntervalPacker intervalPacker; - private final SqlFunctionProvider functionProvider; - - - @Override - public QueryStep apply( - QueryStep joinedStep, - List carryThroughSelects, - DateAggregationDates dateAggregationDates, - DateAggregationAction dateAggregationAction, - ConversionContext conversionContext - ) { - SqlAggregationAction aggregationAction = switch (dateAggregationAction) { - case MERGE -> new MergeAggregateAction(joinedStep); - case INTERSECT -> new IntersectAggregationAction(joinedStep); - default -> throw new IllegalStateException("Unexpected date aggregation action: %s".formatted(dateAggregationAction)); - }; - - DateAggregationContext context = - DateAggregationContext.builder() - .sqlAggregationAction(aggregationAction) - .carryThroughSelects(carryThroughSelects) - .dateAggregationDates(dateAggregationDates) - .dateAggregationTables(aggregationAction.tableNames(conversionContext.getNameGenerator())) - .ids(joinedStep.getQualifiedSelects().getIds()) - .conversionContext(conversionContext) - .build(); - - QueryStep finalDateAggregationStep = convertSteps(joinedStep, aggregationAction.dateAggregationCtes(), context); - if (!aggregationAction.requiresIntervalPackingAfterwards()) { - return finalDateAggregationStep; - } - - Selects predecessorSelects = finalDateAggregationStep.getSelects(); - SqlTables intervalPackingTables = IntervalPackingCteStep.createTables(finalDateAggregationStep, context); - - IntervalPackingContext intervalPackingContext = - IntervalPackingContext.builder() - .ids(predecessorSelects.getIds()) - .daterange(predecessorSelects.getValidityDate().get()) - .predecessor(finalDateAggregationStep) - .carryThroughSelects(carryThroughSelects) - .tables(intervalPackingTables) - .conversionContext(conversionContext) - .build(); - - return this.intervalPacker.aggregateAsValidityDate(intervalPackingContext); - } - - @Override - public ColumnDateRange getAggregatedValidityDate(DateAggregationDates dateAggregationDates, DateAggregationAction dateAggregationAction) { - //TODO(FK): i think this is only ever relevant with dateMode=Logical which i want to remove - Field rangeStart = functionProvider.least(dateAggregationDates.allStarts()); - Field rangeEnd = functionProvider.greatest(dateAggregationDates.allEnds()); - - return ColumnDateRange.of( - rangeStart.as(DateAggregationCte.RANGE_START), - rangeEnd.as(DateAggregationCte.RANGE_END) - ); - } - - @Override - public QueryStep invertAggregatedIntervals(QueryStep baseStep, ConversionContext conversionContext) { - - DateAggregationDates dateAggregationDates = DateAggregationDates.forSingleStep(baseStep); - if (dateAggregationDates.dateAggregationImpossible()) { - return baseStep; - } - - Selects baseStepQualifiedSelects = baseStep.getQualifiedSelects(); - SqlTables dateAggregationTables = DateAggregationCteStep.createInvertTables(baseStep, conversionContext.getNameGenerator()); - - DateAggregationContext context = DateAggregationContext.builder() - .sqlAggregationAction(null) // when inverting, an aggregation has already been applied - .carryThroughSelects(baseStepQualifiedSelects.getSqlSelects()) - .dateAggregationDates(dateAggregationDates) - .dateAggregationTables(dateAggregationTables) - .ids(baseStepQualifiedSelects.getIds()) - .conversionContext(conversionContext) - .build(); - - return convertSteps(baseStep, DateAggregationCteStep.createInvertCtes(), context); - } - - private QueryStep convertSteps(QueryStep baseStep, List dateAggregationCTEs, DateAggregationContext context) { - QueryStep finalDateAggregationStep = baseStep; - for (DateAggregationCte step : dateAggregationCTEs) { - finalDateAggregationStep = step.convert(context, finalDateAggregationStep); - context = context.withStep(step.getCteStep(), finalDateAggregationStep); - } - return finalDateAggregationStep; - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java deleted file mode 100644 index 721d99c84f..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.aggregation; - -import java.util.List; - -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.SqlTables; - -/** - * Base class for a CTE that is part of the date aggregation process. - */ -abstract class DateAggregationCte { - - public static final String RANGE_START = "RANGE_START"; - public static final String RANGE_END = "RANGE_END"; - - public QueryStep convert(DateAggregationContext context, QueryStep previous) { - - DateAggregationCteStep cteStep = getCteStep(); - SqlTables dateAggregationTables = context.getDateAggregationTables(); - - // this way all selects are already qualified, and we don't need to care for that in the respective steps - String predecessor = dateAggregationTables.getPredecessor(cteStep); - context = context.qualify(predecessor); - - QueryStep.QueryStepBuilder builder = this.convertStep(context, predecessor); - - if (cteStep != DateAggregationCteStep.NODE_NO_OVERLAP) { - builder = builder.cteName(dateAggregationTables.cteName(cteStep)) - .predecessors(List.of(previous)); - } - if (cteStep != DateAggregationCteStep.INVERT && cteStep != DateAggregationCteStep.NODE_NO_OVERLAP) { - builder = builder.fromTable(QueryStep.toTableLike(predecessor)); - } - - return builder.build(); - } - - protected abstract QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, String predecessor); - - public abstract DateAggregationCteStep getCteStep(); - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java deleted file mode 100644 index 514c4fb145..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.aggregation; - -import java.sql.Date; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; - -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Getter; -import org.jooq.Field; - -/** - * {@link DateAggregationDates} keep track of all validity dates of list of {@link QueryStep}s that need to be aggregated. - */ -@Getter -@AllArgsConstructor(access = AccessLevel.PRIVATE) -public class DateAggregationDates { - - private final List validityDates; - - public static DateAggregationDates forValidityDates(final List> validityDates) { - final List filtered = validityDates.stream() - .filter(Optional::isPresent) - .map(Optional::get) - .toList(); - return new DateAggregationDates(filtered); - } - - public static DateAggregationDates forSingleStep(QueryStep queryStep) { - List validityDates = queryStep.getSelects() - .getValidityDate() - .map(List::of) - .orElse(Collections.emptyList()); - return new DateAggregationDates(validityDates); - } - - public static DateAggregationDates forSteps(List querySteps) { - final List validityDates = querySteps.stream() - .flatMap(queryStep -> queryStep.getQualifiedSelects().getValidityDate().stream()) - .toList(); - return new DateAggregationDates(validityDates); - } - - public boolean dateAggregationImpossible() { - return this.validityDates.isEmpty(); - } - - public List> allStarts() { - return this.validityDates.stream().map(ColumnDateRange::getStart).toList(); - } - - public List> allEnds() { - return this.validityDates.stream().map(ColumnDateRange::getEnd).toList(); - } - - public List allStartsAndEnds() { - return this.validityDates.stream() - .flatMap(validityDate -> validityDate.toFields().stream()) - .map(FieldWrapper::new) - .collect(Collectors.toList()); - } - - public DateAggregationDates qualify(String qualifier) { - List qualified = this.validityDates.stream() - .map(validityDate -> validityDate.qualify(qualifier)) - .toList(); - // validity dates will already be numerated, no we don't need no apply a counter again - return new DateAggregationDates(qualified); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java deleted file mode 100644 index 5b98c52122..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.aggregation; - -import java.util.List; - -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.NameGenerator; -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.SqlTables; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; - -/** - * Represents a subset of {@link DateAggregationAction}. - */ -interface SqlAggregationAction { - - SqlTables tableNames(NameGenerator nameGenerator); - - List dateAggregationCtes(); - - ColumnDateRange getOverlapValidityDate(DateAggregationDates dateAggregationDates, SqlFunctionProvider functionProvider); - - List getIntermediateTableSelects(DateAggregationDates dateAggregationDates, List carryThroughSelects); - - List getNoOverlapSelects(DateAggregationContext dateAggregationContext); - - QueryStep getOverlapStep(DateAggregationContext dateAggregationContext); - - boolean requiresIntervalPackingAfterwards(); - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java deleted file mode 100644 index 85179b7043..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import java.util.List; -import java.util.stream.Collectors; - -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; -import org.jooq.Condition; - -class AggregationFilterCte extends ConnectorCte { - - @Override - public ConceptCteStep cteStep() { - return ConceptCteStep.AGGREGATION_FILTER; - } - - @Override - public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { - - Selects aggregationFilterSelects = collectSelects(tableContext); - - List aggregationFilterConditions = tableContext.getSqlFilters().stream() - .flatMap(conceptFilter -> conceptFilter.getWhereClauses().getGroupFilters().stream()) - .map(WhereCondition::condition) - .toList(); - - return QueryStep.builder() - .selects(aggregationFilterSelects) - .conditions(aggregationFilterConditions); - } - - private Selects collectSelects(CQTableContext tableContext) { - - QueryStep previous = tableContext.getPrevious(); - Selects previousSelects = previous.getQualifiedSelects(); - List forAggregationFilterStep = - tableContext.allSqlSelects().stream() - .flatMap(sqlSelects -> sqlSelects.getFinalSelects().stream()) - .map(sqlSelect -> { - // universal selects like an ExistsSelect have no predecessor in preceding CTE - if (sqlSelect.isUniversal()) { - return sqlSelect; - } - return sqlSelect.qualify(previous.getCteName()); - }) - .collect(Collectors.toList()); - - return Selects.builder() - .ids(previousSelects.getIds()) - .stratificationDate(previousSelects.getStratificationDate()) - .validityDate(previousSelects.getValidityDate()) - .sqlSelects(forAggregationFilterStep) - .build(); - - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java deleted file mode 100644 index 26ae3c6772..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java +++ /dev/null @@ -1,48 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import java.util.List; -import java.util.Optional; -import java.util.stream.Stream; - -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; -import org.jooq.Field; - -class AggregationSelectCte extends ConnectorCte { - - @Override - public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { - - List requiredInAggregationFilterStep = tableContext.allSqlSelects().stream() - .flatMap(sqlSelects -> sqlSelects.getAggregationSelects().stream()) - .toList(); - - Selects predecessorSelects = tableContext.getPrevious().getQualifiedSelects(); - SqlIdColumns ids = predecessorSelects.getIds(); - Optional stratificationDate = predecessorSelects.getStratificationDate(); - Selects aggregationSelectSelects = Selects.builder() - .ids(ids) - .stratificationDate(stratificationDate) - .sqlSelects(requiredInAggregationFilterStep) - .build(); - - List> groupByFields = Stream.concat( - ids.toFields().stream(), - stratificationDate.stream().flatMap(range -> range.toFields().stream()) - ) - .toList(); - - return QueryStep.builder() - .selects(aggregationSelectSelects) - .groupBy(groupByFields); - } - - @Override - public ConceptCteStep cteStep() { - return ConceptCteStep.AGGREGATION_SELECT; - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java deleted file mode 100644 index 026a1285bf..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ /dev/null @@ -1,314 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import static org.jooq.impl.DSL.*; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.function.Predicate; -import java.util.stream.Stream; - -import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; -import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; -import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.models.datasets.Table; -import com.bakdata.conquery.models.datasets.concepts.ConceptElement; -import com.bakdata.conquery.models.datasets.concepts.Connector; -import com.bakdata.conquery.models.datasets.concepts.ValidityDate; -import com.bakdata.conquery.models.datasets.concepts.select.Select; -import com.bakdata.conquery.models.datasets.concepts.select.concept.ConceptColumnSelect; -import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; -import com.bakdata.conquery.models.identifiable.ids.specific.ConceptElementId; -import com.bakdata.conquery.models.identifiable.ids.specific.SelectId; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; -import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.*; -import com.bakdata.conquery.sql.conversion.model.filter.ConditionUtil; -import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; -import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; -import com.bakdata.conquery.sql.conversion.model.select.ConceptSqlSelects; -import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; -import com.bakdata.conquery.sql.conversion.model.select.SelectContext; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; -import com.bakdata.conquery.util.TablePrimaryColumnUtil; -import com.google.common.base.Preconditions; -import org.jooq.Condition; -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.TableLike; - -public class CQConceptConverter implements NodeConverter { - - private final List connectorCTEs; - - public CQConceptConverter() { - this.connectorCTEs = List.of( - new PreprocessingCte(), - new AggregationSelectCte(), - new JoinBranchesCte(), - new AggregationFilterCte() - ); - } - - private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcept cqConcept, TablePath tablePath, ConversionContext context) { - - ConceptSqlTables universalTables = tablePath.createConceptTables(predecessor); - - Selects predecessorSelects = predecessor.getQualifiedSelects(); - Optional validityDate = predecessorSelects.getValidityDate(); - SqlIdColumns ids = predecessorSelects.getIds(); - - SelectContext selectContext = SelectContext.create(ids, validityDate, universalTables, context); - List converted = cqConcept.getSelects().stream() - .map(selectId -> { - Select select = selectId.resolve(); - return context.getDialectBundle().getSelectConverter(select).conceptSelect(select, selectContext); - }) - .toList(); - - List queriesToJoin = new ArrayList<>(); - queriesToJoin.add(predecessor); - converted.stream().map(ConceptSqlSelects::getAdditionalPredecessor).filter(Optional::isPresent).map(Optional::get).forEach(queriesToJoin::add); - - if (universalTables.isRequiredStep(ConceptCteStep.INTERVAL_PACKING_SELECTS)) { - QueryStep eventDateSelectsStep = IntervalPackingSelectsCte.forConcept(predecessor, universalTables, converted, context); - queriesToJoin.add(eventDateSelectsStep); - } - - // combine all universal selects and connector selects from preceding step - List allConceptSelects = Stream.concat( - converted.stream().flatMap(sqlSelects -> sqlSelects.getFinalSelects().stream()), - // aggregate special selects (e.g. Exists) - predecessor.getQualifiedSelects().getSqlSelects().stream().map(SqlSelect::connectorAggregate) - ) - .toList(); - - Selects finalSelects = Selects.builder() - .ids(ids) - .stratificationDate(predecessorSelects.getStratificationDate()) - .validityDate(validityDate) - .sqlSelects(allConceptSelects) - .build(); - // Event-date selects need the validity date internally, even when the completed concept must not contribute to time aggregation. - if (!cqConcept.isAggregateEventDates()) { - finalSelects = finalSelects.blockValidityDate(); - } - - TableLike joinedTable = QueryStepJoiner.constructJoinedTable(queriesToJoin, ConqueryJoinType.INNER_JOIN, context); - - // group by everything which is not part of an aggregation in this step - List> groupByFields = - Stream.concat( - finalSelects.nonExplicitSelects().stream(), - finalSelects.getSqlSelects().stream() - .filter(Predicate.not(SqlSelect::isUniversal)) - .flatMap(sqlSelect -> sqlSelect.toFields().stream()) - ).toList(); - - return QueryStep.builder() - .cteName(universalTables.cteName(ConceptCteStep.UNIVERSAL_SELECTS)) - .selects(finalSelects) - .fromTable(joinedTable) - .groupBy(groupByFields) - .predecessors(queriesToJoin) - .negate(context.isNegation()) - .build(); - } - - public static SqlIdColumns convertIds(CQConcept cqConcept, CQTable cqTable, ConversionContext conversionContext) { - - Table table = cqTable.getConnector().resolve().getResolvedTable(); - Field primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(table, conversionContext.getDefaultPrimaryColumn()); - - if (cqConcept.isExcludeFromSecondaryId() - || conversionContext.getSecondaryIdDescription() == null - || !cqTable.hasSelectedSecondaryId(conversionContext.getSecondaryIdDescription().getId()) - ) { - return new SqlIdColumns(primaryColumn).withAlias(); - } - - Column secondaryIdColumn = table.findSecondaryIdColumn(conversionContext.getSecondaryIdDescription().getId()); - - Preconditions.checkArgument( - secondaryIdColumn != null, - "Expecting Table %s to have a matching secondary id for %s".formatted( - table, - conversionContext.getSecondaryIdDescription() - ) - ); - - Field secondaryId = field(name(table.getName(), secondaryIdColumn.getName()), String.class); - return new SqlIdColumns(primaryColumn, secondaryId).withAlias(); - } - - private static ColumnDateRange convertValidityDate(ConversionContext context, ValidityDate validityDate) { - SqlFunctionProvider functionProvider = context.getFunctionProvider(); - ColumnDateRange sqlValidityDate; - - boolean hasValidityDate = validityDate != null; - boolean hasDateRestriction = context.getDateRestrictionRange() != null; - - if (hasValidityDate) { - if (hasDateRestriction) { - sqlValidityDate = functionProvider.forValidityDate(validityDate, context.getDateRestrictionRange()); - } else { - sqlValidityDate = functionProvider.forValidityDate(validityDate); - } - } else { - if (hasDateRestriction) { - sqlValidityDate = functionProvider.forCDateRange(context.getDateRestrictionRange()); - } else { - sqlValidityDate = functionProvider.allRange(); - } - } - - return sqlValidityDate; - } - - private static SqlFilters collectConditionFilters( - CQTable cqTable, SqlFunctionProvider functionProvider) { - List conditions = new ArrayList<>(); - convertConnectorCondition(cqTable, functionProvider).ifPresent(conditions::add); - - Connector connector = cqTable.getConnector().resolve(); - if (connector.getColumn() != null) { - conditions.add(ConditionUtil.wrap(field(CTConditionContext.forConnector(connector, functionProvider).getConnectorColumn(), String.class).isNotNull())); - } - - ValidityDate validityDate = cqTable.findValidityDate(); - Condition validityDateFilter = noCondition(); - if (validityDate != null) { - validityDateFilter = functionProvider.isNotEmptyValidityDate(validityDate); - } - - return new SqlFilters(ConnectorSqlSelects.none(), - WhereClauses.builder() - .preprocessingConditions(conditions) - .preprocessingCondition(ConditionUtil.wrap(validityDateFilter)) - .build() - ); - } - - private static Optional convertConnectorCondition(CQTable cqTable, SqlFunctionProvider functionProvider) { - final Connector connector = cqTable.getConnector().resolve(); - - return Optional.ofNullable(connector.getCondition()) - .map(condition -> condition.convertToSqlCondition(CTConditionContext.forConnector(connector, functionProvider))); - } - - private static SqlFilters dateRestrictionFilter(ConversionContext context, ColumnDateRange validityDate) { - - List conditions = new ArrayList<>(); - SqlFunctionProvider functionProvider = context.getFunctionProvider(); - - if (context.getDateRestrictionRange() != null) { - ColumnDateRange dateRestriction = functionProvider.forCDateRange(context.getDateRestrictionRange()); - conditions.add(ConditionUtil.wrap(functionProvider.dateRestriction(dateRestriction, validityDate))); - } - - return new SqlFilters( - ConnectorSqlSelects.none(), - WhereClauses.builder() - .eventFilters(conditions) - .build() - ); - } - - private static ConnectorSqlSelects createConceptColumnConnectorSqlSelects(CQConcept cqConcept, SelectContext selectContext) { - - - return cqConcept.getSelects().stream() - .map(SelectId::resolve) - .filter(select -> select instanceof ConceptColumnSelect) - .findFirst() - .map(select -> selectContext.getDialectBundle().getSelectConverter(select).connectorSelect(select, selectContext)) - .orElse(ConnectorSqlSelects.none()); - } - - @Override - public Class getConversionClass() { - return CQConcept.class; - } - - @Override - public ConversionContext convert(CQConcept cqConcept, ConversionContext context) { - - TablePath tablePath = new TablePath(cqConcept, context); - List convertedCQTables = cqConcept.getTables().stream() - .flatMap(cqTable -> convertCqTable(tablePath, cqConcept, cqTable, context).stream()) - .toList(); - - QueryStep joinedStep = QueryStepJoiner.joinSteps(convertedCQTables, ConqueryJoinType.OUTER_JOIN, DateAggregationAction.MERGE, context); - QueryStep lastConceptStep = finishConceptConversion(joinedStep, cqConcept, tablePath, context); - return context.withQueryStep(lastConceptStep); - } - - private Optional convertCqTable(TablePath tablePath, CQConcept cqConcept, CQTable cqTable, ConversionContext context) { - CQTableContext tableContext = createTableContext(tablePath, cqConcept, cqTable, context); - Optional lastQueryStep = Optional.empty(); - for (ConnectorCte queryStep : connectorCTEs) { - Optional convertedStep = queryStep.convert(tableContext, lastQueryStep); - if (convertedStep.isEmpty()) { - continue; - } - lastQueryStep = convertedStep; - tableContext = tableContext.withPrevious(lastQueryStep.get()); - } - return lastQueryStep; - } - - private CQTableContext createTableContext(TablePath tablePath, CQConcept cqConcept, CQTable cqTable, ConversionContext conversionContext) { - - SqlIdColumns ids = convertIds(cqConcept, cqTable, conversionContext); - ConnectorSqlTables connectorTables = tablePath.getConnectorTables(cqTable); - ColumnDateRange validityDateCalculation = convertValidityDate(conversionContext, cqTable.findValidityDate()); - - // convert filters - SqlFunctionProvider functionProvider = conversionContext.getFunctionProvider(); - List allSqlFiltersForTable = new ArrayList<>(); - - cqTable.getFilters().stream() - .map(filterValue -> filterValue.convertToSqlFilter(ids, conversionContext, connectorTables)) - .forEach(allSqlFiltersForTable::add); - - List> conceptElements = cqConcept.getElements().stream().>map(ConceptElementId::resolve).toList(); - TreeConcept concept = (TreeConcept) cqConcept.getConcept(); - ConceptIdMapping conceptIdMapping = new ConceptIdMapping(concept, functionProvider); - boolean resolveConceptIds = cqConcept.getSelects().stream() - .map(SelectId::resolve) - .filter(ConceptColumnSelect.class::isInstance) - .map(ConceptColumnSelect.class::cast) - .anyMatch(ConceptColumnSelect::isAsIds); - allSqlFiltersForTable.add(collectConditionFilters(cqTable, functionProvider)); - - allSqlFiltersForTable.add(dateRestrictionFilter(conversionContext, validityDateCalculation)); - - // convert selects - SelectContext selectContext = SelectContext.create(ids, Optional.of(validityDateCalculation.asValidityDateRange(connectorTables.getName())), connectorTables, conversionContext); - List allSelectsForTable = new ArrayList<>(); - ConnectorSqlSelects conceptColumnSelect = createConceptColumnConnectorSqlSelects(cqConcept, selectContext); - allSelectsForTable.add(conceptColumnSelect); - cqTable.getSelects() - .stream() - .map(SelectId::resolve) - .map(select -> selectContext.getDialectBundle().getSelectConverter(select).connectorSelect(select, selectContext)) - .forEach(allSelectsForTable::add); - - return CQTableContext.builder() - .ids(ids) - .connector(connectorTables.getName()) - .rawValidityDate(validityDateCalculation) - .sqlSelects(allSelectsForTable) - .sqlFilters(allSqlFiltersForTable) - .connectorTables(connectorTables) - .conceptIdMapping(conceptIdMapping) - .selectedConceptElements(conceptElements) - .resolveConceptIds(resolveConceptIds) - .conversionContext(conversionContext) - .build(); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java deleted file mode 100644 index cf2c51a974..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import java.util.List; -import java.util.stream.Stream; - -import com.bakdata.conquery.models.datasets.concepts.ConceptElement; -import com.bakdata.conquery.models.datasets.concepts.filters.Filter; -import com.bakdata.conquery.models.datasets.concepts.select.Select; -import com.bakdata.conquery.sql.conversion.Context; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; -import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; -import lombok.Builder; -import lombok.Value; -import lombok.With; - -@Value -@Builder -class CQTableContext implements Context { - - String connector; - SqlIdColumns ids; - /** - * Unaliased validity-date expression for predicates rendered in the preprocessing SELECT. - */ - ColumnDateRange rawValidityDate; - List sqlSelects; - List sqlFilters; - ConnectorSqlTables connectorTables; - ConceptIdMapping conceptIdMapping; - List> selectedConceptElements; - boolean resolveConceptIds; - ConversionContext conversionContext; - @With - QueryStep previous; - - - public ColumnDateRange getValidityDate() { - return rawValidityDate.asValidityDateRange(connectorTables.getName()); - } - - /** - * @return All {@link ConnectorSqlSelects} that are either required for {@link Filter}'s or {@link Select}'s. - */ - public List allSqlSelects() { - return Stream.concat(sqlSelects.stream(), sqlFilters.stream().map(SqlFilters::getSelects)).toList(); - } - - @Override - public DialectBundle getDialectBundle() { - return getConversionContext().getDialectBundle(); - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptIdMapping.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptIdMapping.java index f1b69bb30b..340c13eee1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptIdMapping.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptIdMapping.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; import static org.jooq.impl.DSL.*; -import static org.jooq.impl.SQLDataType.VARCHAR; import java.util.*; import java.util.stream.Collectors; @@ -13,130 +12,57 @@ import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.google.common.collect.Sets; -import lombok.AccessLevel; -import lombok.Getter; +import com.bakdata.conquery.sql.mapping.ConceptIdMappingResolver; +import com.bakdata.conquery.sql.mapping.ConceptIdMappingSource; +import com.bakdata.conquery.sql.mapping.ConceptIdMappingTable; +import lombok.Data; import org.jooq.*; import org.jooq.Record; -import org.jooq.impl.DSL; /** * Description of the physical lookup table that maps connector values to their most specific concept element. */ -@Getter +@Data public final class ConceptIdMapping { - public static final String RESOLVED_ID_COLUMN = "resolved_id"; + public static final String RESOLVED_ID_COLUMN = ConceptIdMappingTable.RESOLVED_ID_COLUMN; private final TreeConcept concept; private final SqlFunctionProvider functionProvider; private final Name tableName; private final List> keyFields; - @Getter(AccessLevel.NONE) - private List expressions; - private List rows; + private final ConceptIdMappingTable mappingTable; public ConceptIdMapping(TreeConcept concept, SqlFunctionProvider functionProvider) { this.concept = concept; this.functionProvider = functionProvider; - this.expressions = collectAllExpressions(concept, null, CTConditionContext.forJoinTables(functionProvider)); - this.keyFields = collectKeyFields(expressions); + CTConditionContext context = CTConditionContext.forJoinTables(functionProvider); + List expressions = collectAllExpressions(concept, null, context); this.tableName = tableName(concept.getId()); + this.mappingTable = ConceptIdMappingResolver.resolve( + concept.getId().toString(), tableName, resolveExpressions(expressions) + ); + this.keyFields = mappingTable.keyFields(); } public static Name tableName(ConceptId conceptId) { return name("%s_ids".formatted(conceptId)); } - private static List> collectKeyFields(List expressions) { - Map> fields = new TreeMap<>(); - for (CTCondition.ConceptConditions expression : expressions) { - for (Field field : expression.conditions().keySet()) { - fields.merge(field.getName(), field, ConceptIdMapping::mergeFieldType); - } - } - return List.copyOf(fields.values()); - } - - private static Field mergeFieldType(Field left, Field right) { - DataType leftType = left.getDataType(); - DataType rightType = right.getDataType(); - - if (!leftType.getType().equals(rightType.getType())) { - throw new IllegalArgumentException( - "Concept mapping field `%s` is used with incompatible types %s and %s" - .formatted(left.getName(), leftType, rightType) - ); - } - - if (leftType.isString() && rightType.isString()) { - int length = Math.max(leftType.length(), rightType.length()); - return length > 0 ? field(left.getUnqualifiedName(), VARCHAR(length)) : field(left.getUnqualifiedName(), VARCHAR); - } - - return left; - } - - private static Map> collectExtractors(List expressions) { - Map> extractors = new LinkedHashMap<>(); - for (CTCondition.ConceptConditions expression : expressions) { - for (Map.Entry, CTCondition.FieldCondition> entry : expression.conditions().entrySet()) { - extractors.putIfAbsent(entry.getKey().getName(), entry.getValue().extractor()); - } - } - return extractors; - } - - private static List expressionsToRows( - TreeConcept concept, - List expressions, - List> keyFields + private static List resolveExpressions( + List expressions ) { - Map>, ConceptElement> resolvedMappings = new HashMap<>(); - - for (CTCondition.ConceptConditions expression : expressions) { - Map conditions = expression.conditions().entrySet().stream() - .collect(Collectors.toMap(entry -> entry.getKey().getName(), Map.Entry::getValue)); - List>> valuesByField = new ArrayList<>(); - for (Field keyField : keyFields) { - CTCondition.FieldCondition fieldCondition = conditions.get(keyField.getName()); - valuesByField.add(fieldCondition != null ? fieldCondition.params() : Set.of(defaultValue(keyField))); - } - - for (List> params : Sets.cartesianProduct(valuesByField)) { - ConceptElement previous = resolvedMappings.get(params); - ConceptElement current = expression.conceptElement(); - if (previous == null || previous.getDepth() < current.getDepth()) { - resolvedMappings.put(params, current); - continue; - } - if (previous.getDepth() == current.getDepth() && !previous.equals(current)) { - throw new IllegalArgumentException( - "Concept %s has overlapping sibling mappings for %s and %s on values %s" - .formatted(concept.getId(), previous.getId(), current.getId(), params) - ); - } - } - } - - return resolvedMappings.entrySet().stream() - .map(entry -> { - List> values = new ArrayList<>(entry.getKey().size() + 1); - values.add(val(entry.getValue().getLocalId())); - values.addAll(entry.getKey()); - return row(values); - }) - .toList(); - } - - private static Param defaultValue(Field field) { - if (field.getDataType().isBoolean()) { - return inline(false); - } - if (field.getDataType().isString()) { - return inline(null, String.class); - } - throw new IllegalStateException("Fields of type %s are not expected".formatted(field.getDataType())); + return expressions.stream().map(expression -> new ConceptIdMappingResolver.MappingExpression( + expression.conceptElement().getId().toString(), + expression.conceptElement().getLocalId(), + expression.conceptElement().getDepth(), + expression.conditions().entrySet().stream().collect(Collectors.toMap( + Map.Entry::getKey, + entry -> new ConceptIdMappingResolver.FieldCondition( + entry.getValue().extractor(), entry.getValue().params() + ) + )) + )).toList(); } private static List collectAllExpressions( @@ -158,51 +84,35 @@ private static List collectAllExpressions( return expressions; } - /** - * Materialize mapping rows only while creating the physical mapping table. Query conversion only needs the table metadata. - */ - public synchronized List getRows() { - if (rows == null) { - rows = expressionsToRows(concept, expressions, keyFields); - expressions = null; - } - return rows; - } - public Table table() { - return DSL.table(tableName); + return mappingTable().table(); } public Field resolvedId() { - return field(name(tableName, name(RESOLVED_ID_COLUMN)), Integer.class); + return mappingTable().resolvedId(); } public List> tableFields() { - List> fields = new ArrayList<>(keyFields.size() + 1); - fields.add(field(name(RESOLVED_ID_COLUMN), Integer.class)); - fields.addAll(keyFields); - return fields; + return mappingTable().tableFields(); } - public Condition joinCondition(Connector connector) { + public ConceptIdMappingTable mappingTable() { + return mappingTable; + } + + public List getRows() { + return mappingTable.rows(); + } + + public ConceptIdMappingSource source(Connector connector) { CTConditionContext context = CTConditionContext.forConnector(connector, functionProvider); List expressions = collectAllExpressions(concept, null, context); - Map> extractors = collectExtractors(expressions); - - Condition condition = noCondition(); - for (Field keyField : keyFields) { - Field extractor = extractors.get(keyField.getName()); - if (extractor == null) { - throw new IllegalArgumentException( - "No connector expression for concept mapping field `%s` in connector %s" - .formatted(keyField.getName(), connector.getId()) - ); - } - Field mappingField = field(name(tableName, keyField.getUnqualifiedName()), keyField.getDataType()); - condition = condition.and(mappingField.eq((Field) extractor)); - } + Map> extractors = ConceptIdMappingResolver.collectExtractors(resolveExpressions(expressions)); + return new ConceptIdMappingSource(mappingTable(), extractors); + } - return condition.equals(noCondition()) ? functionProvider.unconditionalJoinCondition() : condition; + public Condition joinCondition(Connector connector) { + return source(connector).joinCondition(functionProvider.unconditionalJoinCondition()); } /** diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java deleted file mode 100644 index 2fda1a3295..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import java.util.List; -import java.util.Map; - -import com.bakdata.conquery.sql.conversion.model.CteStep; -import com.bakdata.conquery.sql.conversion.model.SqlTables; -import lombok.Getter; - -@Getter -public class ConceptSqlTables extends SqlTables { - - private final List connectorTables; - - public ConceptSqlTables( - String rootTable, - Map cteNameMap, - Map predecessorMap, - List connectorTables - ) { - super(rootTable, cteNameMap, predecessorMap); - this.connectorTables = connectorTables; - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java deleted file mode 100644 index 0840b875e3..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java +++ /dev/null @@ -1,44 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import java.util.List; -import java.util.Optional; - -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.SqlTables; - -abstract class ConnectorCte { - - - protected Optional convert(CQTableContext tableContext, Optional previous) { - - if (!isRequired(tableContext.getConnectorTables())) { - return Optional.empty(); - } - - String cteName = tableContext.getConnectorTables().cteName(cteStep()); - QueryStep.QueryStepBuilder queryStepBuilder = this.convertStep(tableContext).cteName(cteName); - - // only preprocessing has no predecessor - if (previous.isEmpty()) { - queryStepBuilder.predecessors(List.of()); - } - // if interval packing takes place, fromTable and predecessors of the final concept step are already set - else if (queryStepBuilder.build().getFromTables().isEmpty() && queryStepBuilder.build().getPredecessors().isEmpty()) { - queryStepBuilder.fromTable(QueryStep.toTableLike(previous.get().getCteName())) - .predecessors(List.of(previous.get())); - } - return Optional.of(queryStepBuilder.build()); - } - - /** - * @return The {@link ConceptCteStep} this instance belongs to. - */ - protected abstract ConceptCteStep cteStep(); - - protected abstract QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext); - - private boolean isRequired(SqlTables connectorTables) { - return connectorTables.isRequiredStep(cteStep()); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java deleted file mode 100644 index c38650ee67..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import java.util.Map; - -import com.bakdata.conquery.models.datasets.concepts.Connector; -import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep; -import com.bakdata.conquery.sql.conversion.model.CteStep; -import com.bakdata.conquery.sql.conversion.model.SqlTables; -import lombok.Getter; - -@Getter -public class ConnectorSqlTables extends SqlTables { - - /** - * A unique label for these tables to create unique SQL CTE/Select names. - */ - private final String name; - - /** - * True if these tables contain interval packing CTEs {@link IntervalPackingCteStep}. - */ - private final boolean withIntervalPacking; - - /** - * Corresponding {@link Connector} of these {@link SqlTables}. - */ - private final Connector connector; - - public ConnectorSqlTables( - Connector connector, - String connectorName, - String rootTable, - Map cteNameMap, - Map predecessorMap, - boolean containsIntervalPacking - ) { - super(rootTable, cteNameMap, predecessorMap); - this.connector = connector; - this.name = connectorName; - this.withIntervalPacking = containsIntervalPacking; - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java deleted file mode 100644 index 6aa498302a..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java +++ /dev/null @@ -1,51 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import javax.annotation.Nullable; - -import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; -import com.bakdata.conquery.sql.conversion.Context; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Value; - -@Value -@AllArgsConstructor(access = AccessLevel.PRIVATE) -public class FilterContext implements Context { - - SqlIdColumns ids; - - /** - * A filter value ({@link FilterValue#getValue()}) - */ - V value; - - ConversionContext conversionContext; - - /** - * Not present if this context is for table export. - */ - @Nullable - ConnectorSqlTables tables; - - public static FilterContext forConceptConversion(SqlIdColumns ids, V value, ConversionContext conversionContext, ConnectorSqlTables tables) { - return new FilterContext<>(ids, value, conversionContext, tables); - } - - public static FilterContext forTableExport(SqlIdColumns ids, V value, ConversionContext conversionContext) { - return new FilterContext<>(ids, value, conversionContext, null); - } - - @Override - public DialectBundle getDialectBundle() { - return getConversionContext().getDialectBundle(); - } - - public SqlFunctionProvider getFunctionProvider() { - return getDialectBundle().getFunctionProvider(); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java deleted file mode 100644 index 45e03332c0..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java +++ /dev/null @@ -1,112 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import java.util.List; -import java.util.Optional; - -import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDateUnionSelect; -import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDurationSumSelect; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlTables; -import com.bakdata.conquery.sql.conversion.model.select.ConceptSqlSelects; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; -import com.google.common.base.Preconditions; - -public class IntervalPackingSelectsCte { - - public static QueryStep forSelect( - QueryStep withAggregatedDaterange, - ColumnDateRange daterange, - SqlSelect select, - SqlTables tables, - DialectBundle dialectBundle - ) { - List predecessors = List.of(withAggregatedDaterange); - QueryStep directPredecessor = withAggregatedDaterange; - - // we need an additional predecessor to unnest the validity date if it is a single column range - if (dialectBundle.supportsSingleColumnRanges()) { - String unnestCteName = tables.cteName(ConceptCteStep.UNNEST_DATE); - directPredecessor = dialectBundle.getFunctionProvider().unnestDaterange(daterange, withAggregatedDaterange, unnestCteName); - predecessors = List.of(withAggregatedDaterange, directPredecessor); - } - - Selects predecessorSelects = directPredecessor.getQualifiedSelects(); - Selects selects = Selects.builder() - .ids(predecessorSelects.getIds()) - .sqlSelect(select) - .build(); - - return QueryStep.builder() - .cteName(tables.cteName(ConceptCteStep.INTERVAL_PACKING_SELECTS)) - .selects(selects) - .fromTable(QueryStep.toTableLike(directPredecessor.getCteName())) - .groupBy(predecessorSelects.getIds().toFields()) - .predecessors(predecessors) - .build(); - } - - public static QueryStep forConnector(QueryStep predecessor, CQTableContext cqTableContext) { - return create( - predecessor, - cqTableContext.getSqlSelects().stream().flatMap(selects -> selects.getEventDateSelects().stream()).toList(), - cqTableContext.getConnectorTables(), - cqTableContext.getFunctionProvider() - ); - } - - public static QueryStep forConcept( - QueryStep predecessor, - SqlTables tables, - List sqlSelects, - ConversionContext conversionContext - ) { - return create( - predecessor, - sqlSelects.stream().flatMap(selects -> selects.getEventDateSelects().stream()).toList(), - tables, - conversionContext.getFunctionProvider() - ); - } - - /** - * @param predecessor The preceding query step which must contain an aggregated validity date. - * @param intervalPackingSelects {@link SqlSelect}s which will be part of the returned {@link QueryStep}. - * @return A {@link QueryStep} containing converted interval packing selects, like {@link EventDurationSumSelect}, {@link EventDateUnionSelect}, etc. - * Returns the given predecessor as is if the given list of interval packing selects is empty. - */ - private static QueryStep create( - QueryStep predecessor, - List intervalPackingSelects, - SqlTables tables, - SqlFunctionProvider functionProvider - ) { - if (intervalPackingSelects.isEmpty()) { - return predecessor; - } - - Optional validityDate = predecessor.getQualifiedSelects().getValidityDate(); - Preconditions.checkArgument(validityDate.isPresent(), "Can't create a IntervalPackingSelectsCte without a validity date present."); - - // we need an additional predecessor to unnest the validity date if it is a single column range - - Selects predecessorSelects = predecessor.getQualifiedSelects(); - Selects selects = Selects.builder() - .ids(predecessorSelects.getIds()) - .sqlSelects(intervalPackingSelects) - .build(); - - return QueryStep.builder() - .cteName(tables.cteName(ConceptCteStep.INTERVAL_PACKING_SELECTS)) - .selects(selects) - .fromTable(QueryStep.toTableLike(predecessor.getCteName())) - .groupBy(predecessorSelects.getIds().toFields()) - .predecessors(List.of()) - .build(); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java deleted file mode 100644 index 625ad7d35c..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java +++ /dev/null @@ -1,114 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingContext; -import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.ConqueryJoinType; -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; -import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.aggregator.SumSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; -import org.jooq.Record; -import org.jooq.TableLike; - -/** - * Joins the {@link ConceptCteStep#AGGREGATION_SELECT} with the interval packing branch for the aggregated validity date and optional validity date selects - * {@link IntervalPackingSelectsCte} as well as optional additional predecessors. - *

- * Joining is optional. If interval packing is not required and there is no additional predecessor, no join will take place. See {@link SumSqlAggregator} with - * distinct-by columns for an example of additional predecessors. - * - * 

- *     {@code
- *     "join_branches" as (
- *  	  select
- *  	    coalesce("group_select"."pid", "interval_complete"."pid", "interval_packing_selects"."pid", "row_number_filtered"."pid") as "pid",
- *  	    "interval_complete"."concept_concept-1_validity_date",
- *  	    "interval_packing_selects"."event_duration_sum",
- *  	    "row_number_filtered"."sum_distinct-1"
- *  	  from "group_select"
- *  	    join "interval_complete"
- *  	      on "group_select"."pid" = "interval_complete"."pid"
- *  	    join "interval_packing_selects"
- *    	      on "group_select"."pid" = "interval_packing_selects"."pid"
- *  	    join "row_number_filtered"
- *  	      on "interval_complete"."pid" = "row_number_filtered"."pid"
- *  	)
- *     }
- *
- */ -class JoinBranchesCte extends ConnectorCte { - - @Override - protected ConceptCteStep cteStep() { - return ConceptCteStep.JOIN_BRANCHES; - } - - @Override - protected QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { - - List queriesToJoin = new ArrayList<>(); - queriesToJoin.add(tableContext.getPrevious()); - - // validity date aggregation - Optional validityDate; - if (!tableContext.getConnectorTables().isWithIntervalPacking()) { - validityDate = Optional.empty(); - } - else { - IntervalPackingContext intervalPackingContext = createIntervalPackingContext(tableContext); - IntervalPacker intervalPacker = tableContext.getConversionContext().getDialectBundle().getIntervalPacker(); - QueryStep lastIntervalPackingStep = intervalPacker.aggregateAsValidityDate(intervalPackingContext); - queriesToJoin.add(lastIntervalPackingStep); - validityDate = lastIntervalPackingStep.getQualifiedSelects().getValidityDate(); - - QueryStep intervalPackingSelectsStep = IntervalPackingSelectsCte.forConnector(lastIntervalPackingStep, tableContext); - if (intervalPackingSelectsStep != lastIntervalPackingStep) { - queriesToJoin.add(intervalPackingSelectsStep); - } - } - - // additional preceding tables - tableContext.allSqlSelects().stream() - .flatMap(sqlSelects -> sqlSelects.getAdditionalPredecessor().stream()) - .forEach(queriesToJoin::add); - - Selects selects = collectSelects(validityDate, queriesToJoin, tableContext); - TableLike fromTable = QueryStepJoiner.constructJoinedTable(queriesToJoin, ConqueryJoinType.OUTER_JOIN, tableContext.getConversionContext()); - - return QueryStep.builder() - .selects(selects) - .fromTable(fromTable) - .predecessors(queriesToJoin); - } - - private static IntervalPackingContext createIntervalPackingContext(CQTableContext tableContext) { - Selects predcessorSelects = tableContext.getPrevious().getQualifiedSelects(); - return IntervalPackingContext.builder() - .ids(predcessorSelects.getIds()) - .daterange(tableContext.getValidityDate()) - .tables(tableContext.getConnectorTables()) - .build(); - } - - private static Selects collectSelects(Optional validityDate, List queriesToJoin, CQTableContext tableContext) { - - SqlIdColumns ids = QueryStepJoiner.coalesceIds(queriesToJoin); - List mergedSqlSelects = QueryStepJoiner.mergeSelects(queriesToJoin); - Optional stratificationDate = tableContext.getPrevious().getQualifiedSelects().getStratificationDate(); - - return Selects.builder() - .ids(ids) - .stratificationDate(stratificationDate) - .validityDate(validityDate) - .sqlSelects(mergedSqlSelects) - .build(); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java deleted file mode 100644 index c31d3e6b51..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java +++ /dev/null @@ -1,136 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -import com.bakdata.conquery.models.datasets.concepts.Connector; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; -import org.jooq.Condition; -import org.jooq.Record; -import org.jooq.Table; -import org.jooq.TableLike; -import org.jooq.impl.DSL; - -class PreprocessingCte extends ConnectorCte { - - private static QueryStep.QueryStepBuilder joinWithStratificationTable( - List preprocessingSelects, - List conditions, - CQTableContext tableContext - ) { - QueryStep stratificationTableCte = tableContext.getConversionContext().getStratificationTable(); - Table stratificationTable = DSL.table(DSL.name(stratificationTableCte.getCteName())); - - Selects stratificationSelects = stratificationTableCte.getQualifiedSelects(); - SqlIdColumns stratificationIds = stratificationSelects.getIds(); - SqlIdColumns rootTableIds = tableContext.getIds().getPredecessor().orElseThrow(() -> new IllegalStateException( - "Id's should have been qualified during conversion and thus have a predecessor") - ); - List idConditions = stratificationIds.join(rootTableIds); - - // join full stratification with connector table on all ID's from prerequisite query - SqlFunctionProvider functionProvider = tableContext.getConversionContext().getFunctionProvider(); - ColumnDateRange stratificationDate = stratificationSelects.getStratificationDate().orElseThrow(() -> new IllegalStateException( - "Stratification table must provide a stratification date" - )); - // Both expressions are available from the joined source tables; do not reference aliases produced by this SELECT. - conditions.add(functionProvider.dateRestriction(stratificationDate, tableContext.getRawValidityDate())); - - Table connectorTable = createConceptJoin(tableContext); - TableLike joinedTable = functionProvider.innerJoin(connectorTable, stratificationTable, idConditions); - - Selects selects = Selects.builder() - .ids(stratificationSelects.getIds()) - .validityDate(Optional.of(tableContext.getValidityDate())) - .stratificationDate(stratificationSelects.getStratificationDate()) - .sqlSelects(preprocessingSelects) - .build(); - - return QueryStep.builder() - .selects(selects) - .fromTable(joinedTable) - .conditions(conditions); - } - - private static Table createConceptJoin(CQTableContext tableContext) { - Table connectorTable = DSL.table(DSL.name(tableContext.getConnectorTables().getPredecessor(ConceptCteStep.PREPROCESSING))); - ConceptIdMapping mapping = tableContext.getConceptIdMapping(); - - if (mapping.includesRoot(tableContext.getSelectedConceptElements()) && !tableContext.isResolveConceptIds()) { - return connectorTable; - } - - Condition joinCondition = mapping.joinCondition(mappingConnector(tableContext)); - Condition conceptFilterCondition; - - if (!tableContext.getSelectedConceptElements().isEmpty()) { - conceptFilterCondition = mapping.resolvedId().in(mapping.includedLocalIds(tableContext.getSelectedConceptElements())); - } else { - conceptFilterCondition = DSL.noCondition(); - } - - return tableContext.getConversionContext().getFunctionProvider().innerJoin(connectorTable, mapping.table(), List.of(joinCondition.and(conceptFilterCondition))); - } - - private static Connector mappingConnector(CQTableContext tableContext) { - return tableContext.getConnectorTables().getConnector(); - - - } - - @Override - public ConceptCteStep cteStep() { - return ConceptCteStep.PREPROCESSING; - } - - @Override - public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { - - List forPreprocessing = tableContext.allSqlSelects().stream() - .flatMap(sqlSelects -> sqlSelects.getPreprocessingSelects().stream()) - .toList(); - - Selects preprocessingSelects = Selects.builder() - .ids(tableContext.getIds()) - .validityDate(Optional.of(tableContext.getValidityDate())) - .sqlSelects(forPreprocessing) - .build(); - // All event-level where clauses are applied directly to the connector table while preprocessing. - List conditions = new ArrayList<>(); - - for (SqlFilters sqlFilter : tableContext.getSqlFilters()) { - for (WhereCondition whereCondition : sqlFilter.getWhereClauses().getPreprocessingConditions()) { - conditions.add(whereCondition.condition()); - } - for (WhereCondition whereCondition : sqlFilter.getWhereClauses().getEventFilters()) { - conditions.add(whereCondition.condition()); - } - } - - // The aliased secondary ID is selected in this CTE, so its root-table expression must be used in the WHERE clause. - tableContext.getIds() - .getPredecessor() - .flatMap(SqlIdColumns::getSecondaryId) - .ifPresent(secondaryId -> conditions.add(secondaryId.isNotNull())); - - QueryStep.QueryStepBuilder builder = QueryStep.builder() - .selects(preprocessingSelects) - .conditions(conditions); - - if (tableContext.getConversionContext().isWithStratification()) { - return joinWithStratificationTable(forPreprocessing, conditions, tableContext); - } - - return builder.fromTable(createConceptJoin(tableContext)); - - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java deleted file mode 100644 index 03c1b2219a..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java +++ /dev/null @@ -1,182 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import static com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep.*; -import static com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep.INTERVAL_COMPLETE; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; -import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; -import com.bakdata.conquery.models.datasets.concepts.select.Select; -import com.bakdata.conquery.models.identifiable.ids.specific.SelectId; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep; -import com.bakdata.conquery.sql.conversion.model.CteStep; -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.google.common.base.Preconditions; -import lombok.Data; -import lombok.Getter; - -/** - * Determines all table/CTE names and creates the respective required {@link ConnectorSqlTables} and {@link ConceptSqlTables} which will be created during the - * conversion of a {@link CQConcept}. - */ -class TablePath { - - private final Map connectorTableMap = new HashMap<>(); - - @Getter - private final CQConcept cqConcept; - - @Getter - private final ConversionContext context; - - public TablePath(CQConcept cqConcept, ConversionContext context) { - this.cqConcept = cqConcept; - this.context = context; - cqConcept.getTables().forEach(cqTable -> this.connectorTableMap.put(cqTable, createConnectorTables(cqConcept, cqTable, context))); - } - - public ConnectorSqlTables getConnectorTables(CQTable cqTable) { - return connectorTableMap.get(cqTable); - } - - private static ConnectorSqlTables createConnectorTables(CQConcept cqConcept, CQTable cqTable, ConversionContext context) { - - String connectorName = context.getNameGenerator().conceptConnectorName(cqConcept, cqTable.getConnector().resolve(), context.getSqlPrintSettings() - .getLocale()); - TablePathInfo tableInfo = collectConnectorTables(cqConcept, cqTable, context); - Map cteNameMap = CteStep.createCteNameMap(tableInfo.getMappings().keySet(), connectorName, context.getNameGenerator()); - - return new ConnectorSqlTables( - cqTable.getConnector().resolve(), - connectorName, - tableInfo.getRootTable(), - cteNameMap, - tableInfo.getMappings(), - tableInfo.isContainsIntervalPacking() - ); - } - - public ConceptSqlTables createConceptTables(QueryStep predecessor) { - - TablePathInfo tableInfo = collectConceptTables(predecessor); - String conceptName = context.getNameGenerator().conceptName(cqConcept, context.getSqlPrintSettings().getLocale()); - Map cteNameMap = CteStep.createCteNameMap(tableInfo.getMappings().keySet(), conceptName, context.getNameGenerator()); - List connectorSqlTables = this.connectorTableMap.values().stream().toList(); - - return new ConceptSqlTables( - tableInfo.getRootTable(), - cteNameMap, - tableInfo.getMappings(), - connectorSqlTables - ); - } - - private static TablePathInfo collectConnectorTables(CQConcept cqConcept, CQTable cqTable, ConversionContext context) { - - TablePathInfo tableInfo = new TablePathInfo(); - tableInfo.setRootTable(cqTable.getConnector().resolve().resolveTableId().getTable()); - tableInfo.addWithDefaultMapping(MANDATORY_STEPS); - - boolean connectorEventDateSelectsPresent = cqTable.getSelects().stream().map(SelectId::resolve).anyMatch(Select::isEventDateSelect); - boolean conceptEventDateSelectsPresent = cqConcept.getSelects().stream().map(SelectId::resolve).anyMatch(Select::isEventDateSelect); - // no validity date aggregation necessary - if (!cqConcept.isAggregateEventDates() && !connectorEventDateSelectsPresent && !conceptEventDateSelectsPresent) { - return tableInfo; - } - - // interval packing required - tableInfo.setContainsIntervalPacking(true); - tableInfo.addMappings(IntervalPackingCteStep.getMappings(PREPROCESSING, context.getDialectBundle())); - - if (!connectorEventDateSelectsPresent) { - return tableInfo; - } - - // interval packing selects required with optional unnest step - if (context.getDialectBundle().supportsSingleColumnRanges()) { - tableInfo.addMappings(Map.of( - UNNEST_DATE, INTERVAL_COMPLETE, - INTERVAL_PACKING_SELECTS, UNNEST_DATE - )); - } - else { - tableInfo.addMappings(Map.of( - INTERVAL_PACKING_SELECTS, INTERVAL_COMPLETE - )); - } - - return tableInfo; - } - - private TablePathInfo collectConceptTables(QueryStep predecessor) { - - TablePathInfo tableInfo = new TablePathInfo(); - tableInfo.setRootTable(predecessor.getCteName()); // last table of a single connector or merged and aggregated table of multiple connectors - tableInfo.addRootTableMapping(UNIVERSAL_SELECTS); - - // no event date selects present - if (cqConcept.getSelects().stream().map(SelectId::resolve).noneMatch(Select::isEventDateSelect)) { - return tableInfo; - } - - Preconditions.checkArgument( - predecessor.getSelects().getValidityDate().isPresent(), - "Can not convert Selects that require interval packing without a validity date present after converting (a) connector(s)" - ); - - // universal event date selects required with optional additional unnest step - if (context.getDialectBundle().supportsSingleColumnRanges()) { - tableInfo.addRootTableMapping(UNNEST_DATE); - tableInfo.addMappings(Map.of(INTERVAL_PACKING_SELECTS, UNNEST_DATE)); - } - else { - tableInfo.addRootTableMapping(INTERVAL_PACKING_SELECTS); - } - - tableInfo.addMappings(Map.of(UNIVERSAL_SELECTS, INTERVAL_PACKING_SELECTS)); - - return tableInfo; - } - - @Data - private static class TablePathInfo { - - /** - * Mapping of a CTE step to their respective preceding CTE step. - */ - private final Map mappings; - - /** - * The root table is the predecessor of all CteSteps from {@link TablePathInfo#mappings} which have a null-predecessor. - */ - private String rootTable; - - /** - * True if this path info contains CTEs from {@link IntervalPackingCteStep}. - */ - private boolean containsIntervalPacking; - - public TablePathInfo() { - this.mappings = new HashMap<>(); - } - - public void addMappings(Map mappings) { - this.mappings.putAll(mappings); - } - - public void addWithDefaultMapping(Set steps) { - this.mappings.putAll(CteStep.getDefaultPredecessorMap(steps)); - } - - public void addRootTableMapping(CteStep step) { - this.mappings.put(step, null); - } - - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java deleted file mode 100644 index fd6d0f70c4..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java +++ /dev/null @@ -1,51 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.intervalpacking; - -import java.util.Collections; -import java.util.List; -import javax.annotation.Nullable; - -import com.bakdata.conquery.sql.conversion.Context; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.SqlTables; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; -import lombok.Builder; -import lombok.Value; - -@Value -@Builder -public class IntervalPackingContext implements Context { - - SqlIdColumns ids; - - /** - * The daterange that will be aggregated. - */ - ColumnDateRange daterange; - - /** - * An optional predecessor of the first interval packing CTE. - */ - @Nullable - QueryStep predecessor; - - SqlTables tables; - - /** - * The selects you want to carry through all interval packing steps. They won't get touched besides qualifying. - */ - @Builder.Default - List carryThroughSelects = Collections.emptyList(); - - @Nullable - ConversionContext conversionContext; - - @Override - public DialectBundle getDialectBundle() { - return getConversionContext().getDialectBundle(); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingResult.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingResult.java deleted file mode 100644 index c8e71a3166..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingResult.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.intervalpacking; - -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import lombok.Value; - -@Value -public class IntervalPackingResult { - - QueryStep finalIntervalPackingStep; - ColumnDateRange aggregatedRange; - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java deleted file mode 100644 index 8fae49bdb8..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.intervalpacking; - -import java.util.Optional; - -import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import lombok.RequiredArgsConstructor; - -/** - * PostgreSql supports interval packing with a native range function. - *

- * See && range function operator - */ -@RequiredArgsConstructor -public class PostgreSqlIntervalPacker implements IntervalPacker { - - private final SqlFunctionProvider functionProvider; - - @Override - public QueryStep aggregateAsValidityDate(IntervalPackingContext context) { - return aggregateDate(context, AggregationMode.VALIDITY_DATE); - } - - @Override - public QueryStep aggregateAsArbitrarySelect(IntervalPackingContext context) { - return aggregateDate(context, AggregationMode.ARBITRARY_SELECT); - } - - private QueryStep aggregateDate(IntervalPackingContext context, AggregationMode aggregationMode) { - - String sourceTableName = context.getTables().getPredecessor(IntervalPackingCteStep.INTERVAL_COMPLETE); - SqlIdColumns ids = context.getIds().qualify(sourceTableName); - ColumnDateRange qualifiedDaterange = context.getDaterange().qualify(sourceTableName); - ColumnDateRange aggregatedDaterange = this.functionProvider.aggregated(qualifiedDaterange); - - Selects.SelectsBuilder selectsBuilder = Selects.builder() - .ids(ids) - .sqlSelects(context.getCarryThroughSelects()); - - switch (aggregationMode) { - case VALIDITY_DATE -> selectsBuilder.validityDate(Optional.of(aggregatedDaterange)); - case ARBITRARY_SELECT -> selectsBuilder.sqlSelect(aggregatedDaterange); - } - - return QueryStep.builder() - .cteName(context.getTables().cteName(IntervalPackingCteStep.INTERVAL_COMPLETE)) - .selects(selectsBuilder.build()) - .fromTable(QueryStep.toTableLike(sourceTableName)) - .groupBy(ids.toFields()) - .predecessors(Optional.ofNullable(context.getPredecessor()).stream().toList()) - .build(); - } - - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java index 5189546239..9386a431ad 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java @@ -1,119 +1,34 @@ package com.bakdata.conquery.sql.conversion.dialect; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; - import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.Dialect; -import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.events.MajorTypeId; -import com.bakdata.conquery.models.query.Visitable; -import com.bakdata.conquery.sql.conversion.Converter; -import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.cqelement.CQAndConverter; -import com.bakdata.conquery.sql.conversion.cqelement.CQDateRestrictionConverter; -import com.bakdata.conquery.sql.conversion.cqelement.CQExternalConverter; -import com.bakdata.conquery.sql.conversion.cqelement.CQNegationConverter; -import com.bakdata.conquery.sql.conversion.cqelement.CQOrConverter; -import com.bakdata.conquery.sql.conversion.cqelement.CQYesConverter; -import com.bakdata.conquery.sql.conversion.cqelement.concept.CQConceptConverter; -import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; -import com.bakdata.conquery.sql.conversion.model.QueryStepTransformer; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; -import com.bakdata.conquery.sql.conversion.query.AbsoluteFormQueryConverter; -import com.bakdata.conquery.sql.conversion.query.CQReusedQueryConverter; -import com.bakdata.conquery.sql.conversion.query.ConceptQueryConverter; -import com.bakdata.conquery.sql.conversion.query.EntityDateQueryConverter; -import com.bakdata.conquery.sql.conversion.query.FormConversionHelper; -import com.bakdata.conquery.sql.conversion.query.RelativFormQueryConverter; -import com.bakdata.conquery.sql.conversion.query.SecondaryIdQueryConverter; -import com.bakdata.conquery.sql.conversion.query.TableExportQueryConverter; +import com.bakdata.conquery.sql.translate.dialect.TranslatorDialect; import com.bakdata.conquery.sql.execution.ResultSetProcessor; -import org.jooq.DSLContext; import org.jooq.Field; import org.jooq.SQLDialect; +/** + * Aggregate used when wiring database-specific services for a backend namespace. + * + *

The translate and SQL-function capabilities are exposed as separate services. Runtime wiring may retain the + * bundle when it needs to construct translate and execution services for the same database.

+ * + *

This interface is an application composition boundary, not a translate dependency.

+ */ public interface DialectBundle { - private static > List customize(List defaults, List substitutes) { - Map, C> substituteMap = getSubstituteMap(substitutes); - return defaults.stream() - .map(converter -> substituteMap.getOrDefault(converter.getConversionClass(), converter)) - .toList(); - } + TranslatorDialect getTranslatorDialect(); - private static > Map, C> getSubstituteMap(List substitutes) { - return substitutes.stream() - .collect(Collectors.toMap( - Converter::getConversionClass, - Function.identity() - )); - } + SqlFunctionProvider getFunctionProvider(); ResultSetProcessor getResultSetProcessor(ConqueryConfig config); Dialect getDialect(); - int getNameMaxLength(); - String getConnectionTestString(); SQLDialect getJooqDialect(); - - StratificationFunctions getStratificationFunctions(); - boolean isTypeCompatible(Field field, MajorTypeId type); - - SqlFunctionProvider getFunctionProvider(); - - IntervalPacker getIntervalPacker(); - - SqlDateAggregator getDateAggregator(); - - List> getNodeConverters(DSLContext context); - - default boolean supportsSingleColumnRanges() { - return false; - } - - default List> getDefaultNodeConverters(DSLContext dslContext) { - - QueryStepTransformer queryStepTransformer = new QueryStepTransformer(dslContext); - FormConversionHelper formConversionUtil = new FormConversionHelper(queryStepTransformer); - - return List.of( - new CQDateRestrictionConverter(), - new CQAndConverter(), - new CQOrConverter(), - new CQNegationConverter(), - new CQYesConverter(), - new CQConceptConverter(), - new CQExternalConverter(), - new CQReusedQueryConverter(), - new ConceptQueryConverter(queryStepTransformer), - new SecondaryIdQueryConverter(), - new AbsoluteFormQueryConverter(formConversionUtil), - new EntityDateQueryConverter(formConversionUtil), - new RelativFormQueryConverter(formConversionUtil), - new TableExportQueryConverter(queryStepTransformer) - ); - } - - default Map, ? extends SelectConverter> getSelectConverterOverrides(){ - return Collections.emptyMap(); - } - - default SelectConverter maybeOverride = (SelectConverter