From cf4f21fb1280e10981b8b7c3f69da188035a6f55 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:32:51 +0200 Subject: [PATCH 01/81] feat(sql): introduce resolved query model Co-authored-by: Codex GPT-5 --- pom.xml | 1 + sql-connector/README.md | 22 ++++ sql-connector/pom.xml | 24 ++++ .../bakdata/conquery/sql/query/AndNode.java | 14 +++ .../conquery/sql/query/ColumnType.java | 13 +++ .../conquery/sql/query/ConceptNode.java | 21 ++++ .../sql/query/DateAggregationAction.java | 9 ++ .../bakdata/conquery/sql/query/DateRange.java | 22 ++++ .../sql/query/DateRestrictionNode.java | 12 ++ .../conquery/sql/query/EntitySchema.java | 15 +++ .../conquery/sql/query/ExecutionTarget.java | 19 +++ .../conquery/sql/query/ModelValidation.java | 26 +++++ .../conquery/sql/query/NegationNode.java | 11 ++ .../bakdata/conquery/sql/query/OrNode.java | 15 +++ .../bakdata/conquery/sql/query/QueryNode.java | 5 + .../conquery/sql/query/ResolvedColumn.java | 20 ++++ .../conquery/sql/query/ResolvedCondition.java | 5 + .../conquery/sql/query/ResolvedConnector.java | 52 +++++++++ .../conquery/sql/query/ResolvedFilter.java | 10 ++ .../conquery/sql/query/ResolvedQuery.java | 25 ++++ .../conquery/sql/query/ResolvedSelect.java | 9 ++ .../sql/query/ResolvedValidityDate.java | 33 ++++++ .../conquery/sql/query/ResultColumn.java | 24 ++++ .../conquery/sql/query/ResultType.java | 23 ++++ .../bakdata/conquery/sql/query/SqlTable.java | 19 +++ .../conquery/sql/query/package-info.java | 8 ++ .../conquery/sql/query/ResolvedQueryTest.java | 110 ++++++++++++++++++ 27 files changed, 567 insertions(+) create mode 100644 sql-connector/README.md create mode 100644 sql-connector/pom.xml create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/AndNode.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/ColumnType.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/ConceptNode.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateAggregationAction.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateRange.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateRestrictionNode.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/EntitySchema.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/ExecutionTarget.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/ModelValidation.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/NegationNode.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/OrNode.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/QueryNode.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedColumn.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedCondition.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedConnector.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedFilter.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedQuery.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedSelect.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedValidityDate.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResultColumn.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResultType.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/SqlTable.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/package-info.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/query/ResolvedQueryTest.java diff --git a/pom.xml b/pom.xml index 860dda8e2f..1fb3cca8fe 100644 --- a/pom.xml +++ b/pom.xml @@ -29,6 +29,7 @@ + sql-connector backend executable autodoc diff --git a/sql-connector/README.md b/sql-connector/README.md new file mode 100644 index 0000000000..7222983e10 --- /dev/null +++ b/sql-connector/README.md @@ -0,0 +1,22 @@ +# SQL connector + +This module contains the framework-neutral contract between query resolution and SQL compilation. + +## Resolved query model + +`ResolvedQuery` is an in-memory execution model. A producer must resolve and validate a request against one dataset +catalog snapshot before constructing it. In particular: + +- logical references are replaced with physical tables and typed columns; +- saved or reusable query references are expanded; +- defaults and date-aggregation actions are materialized; +- filter values, selects, conditions, validity dates, and secondary IDs are validated; +- result columns are ordered and their presentation metadata is finalized. + +The SQL compiler may perform dialect capability checks, but it must not access a dataset repository or resolve logical +identifiers. Connections, dialect implementations, compiler state, and execution services are intentionally not part of +the resolved model. + +`ResolvedFilter`, `ResolvedSelect`, and `ResolvedCondition` are open extension points. Implementations must be immutable +and carry resolved columns and typed values. They must not carry unresolved repository identifiers or arbitrary SQL +received from a query request. diff --git a/sql-connector/pom.xml b/sql-connector/pom.xml new file mode 100644 index 0000000000..f06f2c1c34 --- /dev/null +++ b/sql-connector/pom.xml @@ -0,0 +1,24 @@ + + + 4.0.0 + + + com.bakdata.conquery + parent + ${revision} + ../pom.xml + + + sql-connector + Conquery SQL Connector + + + + org.junit.jupiter + junit-jupiter-api + test + + + diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/AndNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/AndNode.java new file mode 100644 index 0000000000..10bd7b3f16 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/AndNode.java @@ -0,0 +1,14 @@ +package com.bakdata.conquery.sql.query; + +import java.util.List; +import java.util.Objects; + +/** Logical conjunction with its already-derived validity-date behavior. */ +public record AndNode(List children, DateAggregationAction dateAction, boolean createExists) implements QueryNode { + + public AndNode { + ModelValidation.requireNotEmpty(children, "children"); + children = List.copyOf(children); + dateAction = Objects.requireNonNull(dateAction, "dateAction"); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ColumnType.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ColumnType.java new file mode 100644 index 0000000000..6f57bc40be --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ColumnType.java @@ -0,0 +1,13 @@ +package com.bakdata.conquery.sql.query; + +/** Logical type of a resolved database column. */ +public enum ColumnType { + STRING, + INTEGER, + BOOLEAN, + REAL, + DECIMAL, + MONEY, + DATE, + DATE_RANGE +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ConceptNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ConceptNode.java new file mode 100644 index 0000000000..5b1d7baad7 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ConceptNode.java @@ -0,0 +1,21 @@ +package com.bakdata.conquery.sql.query; + +import java.util.List; +import java.util.Objects; + +/** A concept selection whose connectors, conditions, filters, and selects are fully resolved. */ +public record ConceptNode( + String logicalId, + List connectors, + List selects, + DateAggregationAction dateAction +) implements QueryNode { + + public ConceptNode { + logicalId = ModelValidation.requireNonBlank(logicalId, "logicalId"); + ModelValidation.requireNotEmpty(connectors, "connectors"); + connectors = List.copyOf(connectors); + selects = List.copyOf(Objects.requireNonNull(selects, "selects")); + dateAction = Objects.requireNonNull(dateAction, "dateAction"); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateAggregationAction.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateAggregationAction.java new file mode 100644 index 0000000000..0f5abe96ee --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateAggregationAction.java @@ -0,0 +1,9 @@ +package com.bakdata.conquery.sql.query; + +/** Defines how validity dates produced by child nodes are propagated. */ +public enum DateAggregationAction { + BLOCK, + MERGE, + INTERSECT, + NEGATE +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateRange.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateRange.java new file mode 100644 index 0000000000..3830a7d175 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateRange.java @@ -0,0 +1,22 @@ +package com.bakdata.conquery.sql.query; + +import java.time.LocalDate; +import java.util.Objects; +import java.util.Optional; + +/** Inclusive date restriction. An empty bound represents an unbounded side. */ +public record DateRange(Optional startInclusive, Optional endInclusive) { + + public DateRange { + startInclusive = Objects.requireNonNull(startInclusive, "startInclusive"); + endInclusive = Objects.requireNonNull(endInclusive, "endInclusive"); + if (startInclusive.isPresent() && endInclusive.isPresent() + && startInclusive.get().isAfter(endInclusive.get())) { + throw new IllegalArgumentException("startInclusive must not be after endInclusive"); + } + } + + public static DateRange closed(LocalDate startInclusive, LocalDate endInclusive) { + return new DateRange(Optional.of(startInclusive), Optional.of(endInclusive)); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateRestrictionNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateRestrictionNode.java new file mode 100644 index 0000000000..0c6c59e64f --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateRestrictionNode.java @@ -0,0 +1,12 @@ +package com.bakdata.conquery.sql.query; + +import java.util.Objects; + +/** Applies an inclusive date restriction to its child. */ +public record DateRestrictionNode(DateRange dateRange, QueryNode child) implements QueryNode { + + public DateRestrictionNode { + dateRange = Objects.requireNonNull(dateRange, "dateRange"); + child = Objects.requireNonNull(child, "child"); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/EntitySchema.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/EntitySchema.java new file mode 100644 index 0000000000..2cf2ee125c --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/EntitySchema.java @@ -0,0 +1,15 @@ +package com.bakdata.conquery.sql.query; + +import java.util.Objects; + +/** Physical source used to enumerate all entities, for example when compiling a root-level negation. */ +public record EntitySchema(SqlTable table, ResolvedColumn primaryId) { + + public EntitySchema { + table = Objects.requireNonNull(table, "table"); + primaryId = Objects.requireNonNull(primaryId, "primaryId"); + if (!table.equals(primaryId.table())) { + throw new IllegalArgumentException("primaryId must belong to the entity table"); + } + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ExecutionTarget.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ExecutionTarget.java new file mode 100644 index 0000000000..542b6c664d --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ExecutionTarget.java @@ -0,0 +1,19 @@ +package com.bakdata.conquery.sql.query; + +import java.util.Objects; +import java.util.Optional; + +/** Selects the dataset snapshot and configured datasource used to execute a query. */ +public record ExecutionTarget(String datasetId, String dataSource, Optional catalogRevision) { + + public ExecutionTarget { + datasetId = ModelValidation.requireNonBlank(datasetId, "datasetId"); + dataSource = ModelValidation.requireNonBlank(dataSource, "dataSource"); + catalogRevision = Objects.requireNonNull(catalogRevision, "catalogRevision") + .map(revision -> ModelValidation.requireNonBlank(revision, "catalogRevision")); + } + + public ExecutionTarget(String datasetId, String dataSource) { + this(datasetId, dataSource, Optional.empty()); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ModelValidation.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ModelValidation.java new file mode 100644 index 0000000000..8f65dc6d7e --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ModelValidation.java @@ -0,0 +1,26 @@ +package com.bakdata.conquery.sql.query; + +import java.util.Collection; +import java.util.Objects; + +final class ModelValidation { + + private ModelValidation() { + } + + static String requireNonBlank(String value, String name) { + Objects.requireNonNull(value, name); + if (value.isBlank()) { + throw new IllegalArgumentException(name + " must not be blank"); + } + return value; + } + + static Collection requireNotEmpty(Collection values, String name) { + Objects.requireNonNull(values, name); + if (values.isEmpty()) { + throw new IllegalArgumentException(name + " must not be empty"); + } + return values; + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/NegationNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/NegationNode.java new file mode 100644 index 0000000000..9562532f1b --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/NegationNode.java @@ -0,0 +1,11 @@ +package com.bakdata.conquery.sql.query; + +import java.util.Objects; + +/** Logical negation. */ +public record NegationNode(QueryNode child) implements QueryNode { + + public NegationNode { + Objects.requireNonNull(child, "child"); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/OrNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/OrNode.java new file mode 100644 index 0000000000..33d21304f4 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/OrNode.java @@ -0,0 +1,15 @@ +package com.bakdata.conquery.sql.query; + +import java.util.List; +import java.util.Objects; + +/** Logical disjunction with its already-derived validity-date behavior. */ +public record OrNode(List children, DateAggregationAction dateAction, + boolean createExists) implements QueryNode { + + public OrNode { + ModelValidation.requireNotEmpty(children, "children"); + children = List.copyOf(children); + Objects.requireNonNull(dateAction, "dateAction"); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/QueryNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/QueryNode.java new file mode 100644 index 0000000000..8813838524 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/QueryNode.java @@ -0,0 +1,5 @@ +package com.bakdata.conquery.sql.query; + +/** A normalized query-tree node accepted by SQL compilation. */ +public sealed interface QueryNode permits AndNode, OrNode, NegationNode, DateRestrictionNode, ConceptNode { +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedColumn.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedColumn.java new file mode 100644 index 0000000000..c1840a9325 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedColumn.java @@ -0,0 +1,20 @@ +package com.bakdata.conquery.sql.query; + +import java.util.Objects; + +/** A column whose owning table, physical name, and logical type are known. */ +public record ResolvedColumn( + String logicalId, + SqlTable table, + String physicalName, + ColumnType type, + boolean nullable +) { + + public ResolvedColumn { + logicalId = ModelValidation.requireNonBlank(logicalId, "logicalId"); + Objects.requireNonNull(table, "table"); + physicalName = ModelValidation.requireNonBlank(physicalName, "physicalName"); + Objects.requireNonNull(type, "type"); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedCondition.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedCondition.java new file mode 100644 index 0000000000..2e9255bc53 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedCondition.java @@ -0,0 +1,5 @@ +package com.bakdata.conquery.sql.query; + +/** A resolved connector or concept-element condition. */ +public interface ResolvedCondition { +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedConnector.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedConnector.java new file mode 100644 index 0000000000..df50742fe4 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedConnector.java @@ -0,0 +1,52 @@ +package com.bakdata.conquery.sql.query; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** A selected connector with all physical columns and operations resolved. */ +public record ResolvedConnector( + String logicalId, + SqlTable table, + ResolvedColumn primaryId, + Optional secondaryId, + ResolvedValidityDate validityDate, + List filters, + List selects, + List conditions +) { + + public ResolvedConnector { + logicalId = ModelValidation.requireNonBlank(logicalId, "logicalId"); + Objects.requireNonNull(table, "table"); + requireColumnOnTable(primaryId, table, "primaryId"); + if (Objects.requireNonNull(secondaryId, "secondaryId").isPresent()) { + secondaryId = Optional.of(requireColumnOnTable(secondaryId.get(), table, "secondaryId")); + } + Objects.requireNonNull(validityDate, "validityDate"); + validateValidityDateTable(validityDate, table); + filters = List.copyOf(Objects.requireNonNull(filters, "filters")); + selects = List.copyOf(Objects.requireNonNull(selects, "selects")); + conditions = List.copyOf(Objects.requireNonNull(conditions, "conditions")); + } + + private static ResolvedColumn requireColumnOnTable(ResolvedColumn column, SqlTable table, String name) { + Objects.requireNonNull(column, name); + if (!table.equals(column.table())) { + throw new IllegalArgumentException(name + " must belong to the connector table"); + } + return column; + } + + private static void validateValidityDateTable(ResolvedValidityDate validityDate, SqlTable table) { + switch (validityDate) { + case ResolvedValidityDate.None ignored -> { + } + case ResolvedValidityDate.Point point -> requireColumnOnTable(point.column(), table, "validityDate"); + case ResolvedValidityDate.Range range -> { + requireColumnOnTable(range.start(), table, "validityDate.start"); + requireColumnOnTable(range.end(), table, "validityDate.end"); + } + } + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedFilter.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedFilter.java new file mode 100644 index 0000000000..c6712c958b --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedFilter.java @@ -0,0 +1,10 @@ +package com.bakdata.conquery.sql.query; + +/** + * Marker for an immutable, validated filter operation. + * + *

Implementations may be supplied by extensions. They must contain resolved columns and typed values rather than + * repository identifiers.

+ */ +public interface ResolvedFilter { +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedQuery.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedQuery.java new file mode 100644 index 0000000000..c6fcba1565 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedQuery.java @@ -0,0 +1,25 @@ +package com.bakdata.conquery.sql.query; + +import java.util.List; +import java.util.Objects; + +/** + * Complete input for compiling a concept query to SQL. + * + * @param includeValidityDate whether the final result exposes an aggregated validity-date column + */ +public record ResolvedQuery( + ExecutionTarget target, + EntitySchema entitySchema, + QueryNode root, + boolean includeValidityDate, + List resultColumns +) { + + public ResolvedQuery { + Objects.requireNonNull(target, "target"); + Objects.requireNonNull(entitySchema, "entitySchema"); + Objects.requireNonNull(root, "root"); + resultColumns = List.copyOf(Objects.requireNonNull(resultColumns, "resultColumns")); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedSelect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedSelect.java new file mode 100644 index 0000000000..44cb0b2707 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedSelect.java @@ -0,0 +1,9 @@ +package com.bakdata.conquery.sql.query; + +/** + * Marker for an immutable, validated select operation. + * + *

Implementations may be supplied by extensions and are dispatched to matching SQL converters.

+ */ +public interface ResolvedSelect { +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedValidityDate.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedValidityDate.java new file mode 100644 index 0000000000..88f3e4e93e --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedValidityDate.java @@ -0,0 +1,33 @@ +package com.bakdata.conquery.sql.query; + +import java.util.Objects; + +/** Physical representation of event validity for a connector. */ +public sealed interface ResolvedValidityDate permits ResolvedValidityDate.None, ResolvedValidityDate.Point, ResolvedValidityDate.Range { + + record None() implements ResolvedValidityDate { + } + + record Point(ResolvedColumn column) implements ResolvedValidityDate { + public Point { + requireDateColumn(column, "column"); + } + } + + record Range(ResolvedColumn start, ResolvedColumn end) implements ResolvedValidityDate { + public Range { + requireDateColumn(start, "start"); + requireDateColumn(end, "end"); + if (!start.table().equals(end.table())) { + throw new IllegalArgumentException("validity-date columns must belong to the same table"); + } + } + } + + private static void requireDateColumn(ResolvedColumn column, String name) { + Objects.requireNonNull(column, name); + if (column.type() != ColumnType.DATE) { + throw new IllegalArgumentException(name + " must be a DATE column"); + } + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResultColumn.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResultColumn.java new file mode 100644 index 0000000000..e02a0e584c --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResultColumn.java @@ -0,0 +1,24 @@ +package com.bakdata.conquery.sql.query; + +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +/** Ordered output-column metadata, with presentation names already resolved for the query locale. */ +public record ResultColumn( + String label, + Optional defaultLabel, + Optional description, + ResultType type, + Set semantics +) { + + public ResultColumn { + label = ModelValidation.requireNonBlank(label, "label"); + defaultLabel = Objects.requireNonNull(defaultLabel, "defaultLabel") + .map(value -> ModelValidation.requireNonBlank(value, "defaultLabel")); + Objects.requireNonNull(description, "description"); + Objects.requireNonNull(type, "type"); + semantics = Set.copyOf(Objects.requireNonNull(semantics, "semantics")); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResultType.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResultType.java new file mode 100644 index 0000000000..929c1af930 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResultType.java @@ -0,0 +1,23 @@ +package com.bakdata.conquery.sql.query; + +import java.util.Objects; + +/** Logical type of a result column. */ +public sealed interface ResultType permits ResultType.Primitive, ResultType.ListType { + + enum Primitive implements ResultType { + BOOLEAN, + INTEGER, + NUMERIC, + DATE, + DATE_RANGE, + STRING, + MONEY + } + + record ListType(Primitive elementType) implements ResultType { + public ListType { + Objects.requireNonNull(elementType, "elementType"); + } + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/SqlTable.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/SqlTable.java new file mode 100644 index 0000000000..1909f560d6 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/SqlTable.java @@ -0,0 +1,19 @@ +package com.bakdata.conquery.sql.query; + +import java.util.Arrays; +import java.util.List; + +/** A logical table together with its unquoted physical SQL name. */ +public record SqlTable(String logicalId, List physicalName) { + + public SqlTable { + logicalId = ModelValidation.requireNonBlank(logicalId, "logicalId"); + ModelValidation.requireNotEmpty(physicalName, "physicalName"); + physicalName = List.copyOf(physicalName); + physicalName.forEach(part -> ModelValidation.requireNonBlank(part, "physicalName part")); + } + + public static SqlTable of(String logicalId, String... physicalName) { + return new SqlTable(logicalId, Arrays.asList(physicalName)); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/package-info.java new file mode 100644 index 0000000000..75e6f6214f --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/package-info.java @@ -0,0 +1,8 @@ +/** + * Framework-neutral, fully resolved input model for SQL query compilation. + * + *

Instances in this package must not require repository access or identifier resolution during compilation. They + * describe query semantics and physical database objects, but deliberately exclude connections, dialects, and SQL + * compiler state.

+ */ +package com.bakdata.conquery.sql.query; diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/query/ResolvedQueryTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/query/ResolvedQueryTest.java new file mode 100644 index 0000000000..692df371c1 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/query/ResolvedQueryTest.java @@ -0,0 +1,110 @@ +package com.bakdata.conquery.sql.query; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +class ResolvedQueryTest { + + private static final SqlTable EVENTS = SqlTable.of("dataset.events", "analytics", "events"); + private static final ResolvedColumn ENTITY_ID = new ResolvedColumn( + "dataset.events.entity_id", EVENTS, "entity_id", ColumnType.STRING, false + ); + + @Test + void shouldCreateResolvedQueryWithoutFrameworkTypes() { + ResolvedConnector connector = new ResolvedConnector( + "dataset.concept.connector", + EVENTS, + ENTITY_ID, + Optional.empty(), + new ResolvedValidityDate.Point(dateColumn("event_date")), + List.of(), + List.of(), + List.of() + ); + ConceptNode concept = new ConceptNode( + "dataset.concept", List.of(connector), List.of(), DateAggregationAction.MERGE + ); + + ResolvedQuery query = new ResolvedQuery( + new ExecutionTarget("dataset", "warehouse"), + new EntitySchema(EVENTS, ENTITY_ID), + concept, + true, + List.of(new ResultColumn( + "Entity", Optional.of("Entity"), Optional.empty(), ResultType.Primitive.STRING, Set.of("ID") + )) + ); + + assertEquals(concept, query.root()); + assertEquals("warehouse", query.target().dataSource()); + } + + @Test + void shouldDefensivelyCopyOrderedCollections() { + List children = new ArrayList<>(); + children.add(conceptNode(EVENTS)); + AndNode andNode = new AndNode(children, DateAggregationAction.INTERSECT, false); + + children.add(conceptNode(EVENTS)); + + assertEquals(1, andNode.children().size()); + assertThrows(UnsupportedOperationException.class, () -> andNode.children().clear()); + } + + @Test + void shouldRejectColumnsFromAnotherConnectorTable() { + SqlTable otherTable = SqlTable.of("dataset.other", "analytics", "other"); + ResolvedColumn otherId = new ResolvedColumn( + "dataset.other.entity_id", otherTable, "entity_id", ColumnType.STRING, false + ); + + assertThrows(IllegalArgumentException.class, () -> new ResolvedConnector( + "dataset.concept.connector", + EVENTS, + otherId, + Optional.empty(), + new ResolvedValidityDate.None(), + List.of(), + List.of(), + List.of() + )); + } + + @Test + void shouldRejectNonDateValidityColumns() { + assertThrows( + IllegalArgumentException.class, + () -> new ResolvedValidityDate.Point(ENTITY_ID) + ); + } + + @Test + void shouldRejectInvertedDateRestriction() { + assertThrows(IllegalArgumentException.class, () -> DateRange.closed( + LocalDate.of(2025, 2, 1), + LocalDate.of(2025, 1, 1) + )); + } + + private static ConceptNode conceptNode(SqlTable table) { + ResolvedColumn id = new ResolvedColumn("concept.id", table, "entity_id", ColumnType.STRING, false); + ResolvedConnector connector = new ResolvedConnector( + "concept.connector", table, id, Optional.empty(), new ResolvedValidityDate.None(), + List.of(), List.of(), List.of() + ); + return new ConceptNode("concept", List.of(connector), List.of(), DateAggregationAction.BLOCK); + } + + private static ResolvedColumn dateColumn(String name) { + return new ResolvedColumn("dataset.events." + name, EVENTS, name, ColumnType.DATE, true); + } +} From 2aad4d28a4d7a9aac5d890fca83dc36a2459efc3 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:40:25 +0200 Subject: [PATCH 02/81] refactor(sql): decouple node conversion dispatch Co-authored-by: Codex GPT-5 --- .../bakdata/conquery/sql/conversion/Conversions.java | 5 ++--- .../sql/conversion/NodeConversionDispatcher.java | 10 ++++++++++ .../conquery/sql/conversion/NodeConversions.java | 11 ++++++++--- .../sql/conversion/cqelement/ConversionContext.java | 6 ++---- 4 files changed, 22 insertions(+), 10 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversionDispatcher.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Conversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/Conversions.java index 50116dea6e..48a59e2b84 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Conversions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/Conversions.java @@ -13,7 +13,7 @@ * @param context of the convertible * @see Converter */ -public abstract class Conversions { +public class Conversions { @Getter private final List> converters; @@ -29,8 +29,7 @@ public R convert(C node, X context) { if (maybeConverted.isPresent()) { if (converted == null) { converted = maybeConverted.get(); - } - else { + } else { throw new IllegalStateException("Multiple Converters for %s".formatted(node)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversionDispatcher.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversionDispatcher.java new file mode 100644 index 0000000000..64c1749fe9 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversionDispatcher.java @@ -0,0 +1,10 @@ +package com.bakdata.conquery.sql.conversion; + +import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; + +/** Recursively dispatches a query node to its matching SQL converter. */ +@FunctionalInterface +public interface NodeConversionDispatcher { + + ConversionContext convert(Object node, ConversionContext context); +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java index 742538f3a1..c61a301d8f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java @@ -4,7 +4,6 @@ import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.IdColumnConfig; import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; import com.bakdata.conquery.sql.conversion.model.NameGenerator; @@ -18,8 +17,9 @@ /** * Entry point for converting {@link QueryDescription} to an SQL query. */ -public class NodeConversions extends Conversions { +public class NodeConversions implements NodeConversionDispatcher { + private final Conversions conversions; private final IdColumnConfig idColumns; private final DialectBundle dialect; private final NameGenerator nameGenerator; @@ -36,7 +36,7 @@ public NodeConversions( Clock clock, String defaultPrimaryColumn ) { - super(dialectBundle.getNodeConverters(dslContext)); + this.conversions = new Conversions<>(dialectBundle.getNodeConverters(dslContext)); this.idColumns = idColumns; this.dialect = dialectBundle; this.nameGenerator = new NameGenerator(dialectBundle.getNameMaxLength()); @@ -45,6 +45,11 @@ public NodeConversions( this.defaultPrimaryColumn = defaultPrimaryColumn; } + @Override + public ConversionContext convert(Object node, ConversionContext context) { + return conversions.convert(node, context); + } + public ConversionContext convert(QueryDescription queryDescription, ConqueryConfig conqueryConfig) { ConversionContext initialCtx = ConversionContext.builder() .idColumns(idColumns) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java index a2973ec90d..235ba767f9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java @@ -11,9 +11,8 @@ import com.bakdata.conquery.models.datasets.SecondaryIdDescription; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.sql.conversion.Context; -import com.bakdata.conquery.sql.conversion.NodeConversions; +import com.bakdata.conquery.sql.conversion.NodeConversionDispatcher; import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.conversion.model.QueryStep; @@ -38,7 +37,7 @@ public class ConversionContext implements Context { StratificationFunctions stratificationFunctions; String defaultPrimaryColumn; PrintSettings sqlPrintSettings; - NodeConversions nodeConversions; + NodeConversionDispatcher nodeConversions; DialectBundle dialectBundle; NameGenerator nameGenerator; SqlExecutionService executionService; @@ -76,7 +75,6 @@ public boolean isWithStratification() { } - /** * Adds a query step to the list of {@link QueryStep} of this context. */ From a2eecb4950373ccdaf3df3c5a722e576ff69d7d2 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:59:59 +0200 Subject: [PATCH 03/81] feat(sql): define validated resolved operations Co-authored-by: Codex GPT-5 --- dataset-model/README.md | 7 + dataset-model/pom.xml | 16 ++ .../models/datasets/ColumnDescriptor.java | 5 + .../conquery/models/datasets}/ColumnType.java | 4 +- .../models/datasets/package-info.java | 2 + pom.xml | 1 + sql-connector/README.md | 30 +++- sql-connector/pom.xml | 19 ++ .../bakdata/conquery/sql/query/AndNode.java | 14 -- .../conquery/sql/query/ConceptNode.java | 21 --- .../bakdata/conquery/sql/query/DateRange.java | 22 --- .../sql/query/DateRestrictionNode.java | 12 -- .../conquery/sql/query/EntitySchema.java | 15 -- .../conquery/sql/query/ExecutionTarget.java | 19 +- .../conquery/sql/query/ModelValidation.java | 26 --- .../conquery/sql/query/NegationNode.java | 11 -- .../bakdata/conquery/sql/query/OrNode.java | 15 -- .../conquery/sql/query/ResolvedColumn.java | 20 --- .../conquery/sql/query/ResolvedConnector.java | 52 ------ .../conquery/sql/query/ResolvedQuery.java | 21 ++- .../sql/query/ResolvedValidityDate.java | 33 ---- .../conquery/sql/query/ResultColumn.java | 24 --- .../bakdata/conquery/sql/query/SqlTable.java | 19 -- .../query/internal/ModelNormalization.java | 27 +++ .../conquery/sql/query/node/AndNode.java | 20 +++ .../conquery/sql/query/node/ConceptNode.java | 25 +++ .../{ => node}/DateAggregationAction.java | 2 +- .../sql/query/node/DateRestrictionNode.java | 12 ++ .../conquery/sql/query/node/NegationNode.java | 8 + .../conquery/sql/query/node/OrNode.java | 20 +++ .../sql/query/{ => node}/QueryNode.java | 2 +- .../conquery/sql/query/node/package-info.java | 2 + .../query/operation/BuiltInAggregations.java | 132 ++++++++++++++ .../query/operation/BuiltInConditions.java | 69 ++++++++ .../sql/query/operation/BuiltInFilters.java | 81 +++++++++ .../sql/query/operation/BuiltInSelects.java | 76 ++++++++ .../query/operation/ResolvedAggregation.java | 5 + .../{ => operation}/ResolvedCondition.java | 2 +- .../query/{ => operation}/ResolvedFilter.java | 2 +- .../query/{ => operation}/ResolvedSelect.java | 2 +- .../sql/query/operation/package-info.java | 2 + .../conquery/sql/query/package-info.java | 3 + .../conquery/sql/query/range/DateRange.java | 25 +++ .../conquery/sql/query/range/NumberRange.java | 43 +++++ .../sql/query/range/SubstringRange.java | 27 +++ .../sql/query/range/package-info.java | 2 + .../sql/query/result/ResultColumn.java | 29 ++++ .../sql/query/{ => result}/ResultType.java | 9 +- .../sql/query/result/package-info.java | 2 + .../sql/query/schema/DateColumns.java | 42 +++++ .../sql/query/schema/EntitySchema.java | 17 ++ .../sql/query/schema/ResolvedColumn.java | 16 ++ .../sql/query/schema/ResolvedConnector.java | 59 +++++++ .../query/schema/ResolvedValidityDate.java | 32 ++++ .../conquery/sql/query/schema/SqlTable.java | 26 +++ .../sql/query/schema/package-info.java | 2 + .../query/validation/AllowedColumnTypes.java | 26 +++ .../AllowedColumnTypesValidator.java | 23 +++ .../validation/ResolvedQueryValidation.java | 26 +++ .../sql/query/validation/package-info.java | 2 + .../sql/query/BuiltInOperationsTest.java | 164 ++++++++++++++++++ .../conquery/sql/query/ResolvedQueryTest.java | 58 ++++++- .../sql/query/ValidationTestSupport.java | 28 +++ 63 files changed, 1235 insertions(+), 323 deletions(-) create mode 100644 dataset-model/README.md create mode 100644 dataset-model/pom.xml create mode 100644 dataset-model/src/main/java/com/bakdata/conquery/models/datasets/ColumnDescriptor.java rename {sql-connector/src/main/java/com/bakdata/conquery/sql/query => dataset-model/src/main/java/com/bakdata/conquery/models/datasets}/ColumnType.java (52%) create mode 100644 dataset-model/src/main/java/com/bakdata/conquery/models/datasets/package-info.java delete mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/AndNode.java delete mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/ConceptNode.java delete mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateRange.java delete mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateRestrictionNode.java delete mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/EntitySchema.java delete mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/ModelValidation.java delete mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/NegationNode.java delete mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/OrNode.java delete mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedColumn.java delete mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedConnector.java delete mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedValidityDate.java delete mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResultColumn.java delete mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/SqlTable.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/internal/ModelNormalization.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/AndNode.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/ConceptNode.java rename sql-connector/src/main/java/com/bakdata/conquery/sql/query/{ => node}/DateAggregationAction.java (76%) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/DateRestrictionNode.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/NegationNode.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/OrNode.java rename sql-connector/src/main/java/com/bakdata/conquery/sql/query/{ => node}/QueryNode.java (79%) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/package-info.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInAggregations.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInConditions.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInFilters.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInSelects.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedAggregation.java rename sql-connector/src/main/java/com/bakdata/conquery/sql/query/{ => operation}/ResolvedCondition.java (66%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/query/{ => operation}/ResolvedFilter.java (83%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/query/{ => operation}/ResolvedSelect.java (80%) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/package-info.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/DateRange.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/NumberRange.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/SubstringRange.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/package-info.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/result/ResultColumn.java rename sql-connector/src/main/java/com/bakdata/conquery/sql/query/{ => result}/ResultType.java (54%) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/result/package-info.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/DateColumns.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/EntitySchema.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/ResolvedColumn.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/ResolvedConnector.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/ResolvedValidityDate.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/SqlTable.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/package-info.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/AllowedColumnTypes.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/AllowedColumnTypesValidator.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/ResolvedQueryValidation.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/package-info.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/query/BuiltInOperationsTest.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/query/ValidationTestSupport.java diff --git a/dataset-model/README.md b/dataset-model/README.md new file mode 100644 index 0000000000..d10f263d63 --- /dev/null +++ b/dataset-model/README.md @@ -0,0 +1,7 @@ +# Dataset model + +This module contains the dependency-free dataset vocabulary shared by query producers, plugins, and execution engines. +It must not depend on a backend framework, persistence technology, or SQL implementation. + +Keep types here only when they have the same meaning for every consumer. Backend storage records and SQL compilation +types remain in their respective modules and reference this shared vocabulary. diff --git a/dataset-model/pom.xml b/dataset-model/pom.xml new file mode 100644 index 0000000000..e567eca481 --- /dev/null +++ b/dataset-model/pom.xml @@ -0,0 +1,16 @@ + + + 4.0.0 + + + com.bakdata.conquery + parent + ${revision} + ../pom.xml + + + dataset-model + Conquery Dataset Model + diff --git a/dataset-model/src/main/java/com/bakdata/conquery/models/datasets/ColumnDescriptor.java b/dataset-model/src/main/java/com/bakdata/conquery/models/datasets/ColumnDescriptor.java new file mode 100644 index 0000000000..3f61572d6c --- /dev/null +++ b/dataset-model/src/main/java/com/bakdata/conquery/models/datasets/ColumnDescriptor.java @@ -0,0 +1,5 @@ +package com.bakdata.conquery.models.datasets; + +/** Framework-neutral description of a validated dataset column. */ +public record ColumnDescriptor(String name, ColumnType type) { +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ColumnType.java b/dataset-model/src/main/java/com/bakdata/conquery/models/datasets/ColumnType.java similarity index 52% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/ColumnType.java rename to dataset-model/src/main/java/com/bakdata/conquery/models/datasets/ColumnType.java index 6f57bc40be..d522d392d8 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ColumnType.java +++ b/dataset-model/src/main/java/com/bakdata/conquery/models/datasets/ColumnType.java @@ -1,6 +1,6 @@ -package com.bakdata.conquery.sql.query; +package com.bakdata.conquery.models.datasets; -/** Logical type of a resolved database column. */ +/** Logical type of a dataset column. */ public enum ColumnType { STRING, INTEGER, diff --git a/dataset-model/src/main/java/com/bakdata/conquery/models/datasets/package-info.java b/dataset-model/src/main/java/com/bakdata/conquery/models/datasets/package-info.java new file mode 100644 index 0000000000..4f9a0019d6 --- /dev/null +++ b/dataset-model/src/main/java/com/bakdata/conquery/models/datasets/package-info.java @@ -0,0 +1,2 @@ +/** Framework-neutral dataset types shared by query producers, plugins, and execution engines. */ +package com.bakdata.conquery.models.datasets; diff --git a/pom.xml b/pom.xml index 1fb3cca8fe..5e2cd7311e 100644 --- a/pom.xml +++ b/pom.xml @@ -29,6 +29,7 @@ + dataset-model sql-connector backend executable diff --git a/sql-connector/README.md b/sql-connector/README.md index 7222983e10..959b9358b4 100644 --- a/sql-connector/README.md +++ b/sql-connector/README.md @@ -17,6 +17,30 @@ The SQL compiler may perform dialect capability checks, but it must not access a identifiers. Connections, dialect implementations, compiler state, and execution services are intentionally not part of the resolved model. -`ResolvedFilter`, `ResolvedSelect`, and `ResolvedCondition` are open extension points. Implementations must be immutable -and carry resolved columns and typed values. They must not carry unresolved repository identifiers or arbitrary SQL -received from a query request. +The model is organized by responsibility: + +- `query.node` contains the normalized query tree; +- `query.operation` contains the open operation interfaces and standard implementations; +- `query.schema` contains resolved physical tables, columns, connectors, and entity metadata; +- `query.range` and `query.result` contain shared value objects; +- `query.validation` contains the validation boundary and reusable constraints. + +Shared dataset vocabulary such as `ColumnType` comes from the dependency-free `dataset-model` module. SQL-specific +physical metadata remains in `query.schema`. + +`ResolvedFilter`, `ResolvedSelect`, `ResolvedCondition`, and `ResolvedAggregation` are open extension points. +Implementations must be immutable and carry resolved columns and typed values. They must not carry unresolved repository +identifiers or arbitrary SQL received from a query request. + +The `BuiltInFilters`, `BuiltInSelects`, `BuiltInConditions`, and `BuiltInAggregations` types define the normalized +operations understood by the standard compiler. They intentionally describe semantics rather than the configuration +classes used by a particular query-producing application. + +## Validation + +The resolved model uses Jakarta Bean Validation annotations. The application supplies a validation provider, normally +Hibernate Validator, and calls `ResolvedQueryValidation` once before passing a query to the SQL compiler. Validation is +cascaded through the complete query graph, including extension-provided filters, selects, conditions, and aggregations. + +Compact record constructors only create immutable copies of collections. Domain constraints such as compatible column +types, ordered ranges, and same-table requirements are declarative and are reported together as constraint violations. diff --git a/sql-connector/pom.xml b/sql-connector/pom.xml index f06f2c1c34..073afc8767 100644 --- a/sql-connector/pom.xml +++ b/sql-connector/pom.xml @@ -15,10 +15,29 @@ Conquery SQL Connector + + com.bakdata.conquery + dataset-model + ${project.version} + + + jakarta.validation + jakarta.validation-api + org.junit.jupiter junit-jupiter-api test + + org.hibernate.validator + hibernate-validator + test + + + org.glassfish + jakarta.el + test + diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/AndNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/AndNode.java deleted file mode 100644 index 10bd7b3f16..0000000000 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/AndNode.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.bakdata.conquery.sql.query; - -import java.util.List; -import java.util.Objects; - -/** Logical conjunction with its already-derived validity-date behavior. */ -public record AndNode(List children, DateAggregationAction dateAction, boolean createExists) implements QueryNode { - - public AndNode { - ModelValidation.requireNotEmpty(children, "children"); - children = List.copyOf(children); - dateAction = Objects.requireNonNull(dateAction, "dateAction"); - } -} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ConceptNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ConceptNode.java deleted file mode 100644 index 5b1d7baad7..0000000000 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ConceptNode.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.bakdata.conquery.sql.query; - -import java.util.List; -import java.util.Objects; - -/** A concept selection whose connectors, conditions, filters, and selects are fully resolved. */ -public record ConceptNode( - String logicalId, - List connectors, - List selects, - DateAggregationAction dateAction -) implements QueryNode { - - public ConceptNode { - logicalId = ModelValidation.requireNonBlank(logicalId, "logicalId"); - ModelValidation.requireNotEmpty(connectors, "connectors"); - connectors = List.copyOf(connectors); - selects = List.copyOf(Objects.requireNonNull(selects, "selects")); - dateAction = Objects.requireNonNull(dateAction, "dateAction"); - } -} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateRange.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateRange.java deleted file mode 100644 index 3830a7d175..0000000000 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateRange.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.bakdata.conquery.sql.query; - -import java.time.LocalDate; -import java.util.Objects; -import java.util.Optional; - -/** Inclusive date restriction. An empty bound represents an unbounded side. */ -public record DateRange(Optional startInclusive, Optional endInclusive) { - - public DateRange { - startInclusive = Objects.requireNonNull(startInclusive, "startInclusive"); - endInclusive = Objects.requireNonNull(endInclusive, "endInclusive"); - if (startInclusive.isPresent() && endInclusive.isPresent() - && startInclusive.get().isAfter(endInclusive.get())) { - throw new IllegalArgumentException("startInclusive must not be after endInclusive"); - } - } - - public static DateRange closed(LocalDate startInclusive, LocalDate endInclusive) { - return new DateRange(Optional.of(startInclusive), Optional.of(endInclusive)); - } -} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateRestrictionNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateRestrictionNode.java deleted file mode 100644 index 0c6c59e64f..0000000000 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateRestrictionNode.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.bakdata.conquery.sql.query; - -import java.util.Objects; - -/** Applies an inclusive date restriction to its child. */ -public record DateRestrictionNode(DateRange dateRange, QueryNode child) implements QueryNode { - - public DateRestrictionNode { - dateRange = Objects.requireNonNull(dateRange, "dateRange"); - child = Objects.requireNonNull(child, "child"); - } -} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/EntitySchema.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/EntitySchema.java deleted file mode 100644 index 2cf2ee125c..0000000000 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/EntitySchema.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.bakdata.conquery.sql.query; - -import java.util.Objects; - -/** Physical source used to enumerate all entities, for example when compiling a root-level negation. */ -public record EntitySchema(SqlTable table, ResolvedColumn primaryId) { - - public EntitySchema { - table = Objects.requireNonNull(table, "table"); - primaryId = Objects.requireNonNull(primaryId, "primaryId"); - if (!table.equals(primaryId.table())) { - throw new IllegalArgumentException("primaryId must belong to the entity table"); - } - } -} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ExecutionTarget.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ExecutionTarget.java index 542b6c664d..7b38bea7b2 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ExecutionTarget.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ExecutionTarget.java @@ -1,16 +1,21 @@ package com.bakdata.conquery.sql.query; -import java.util.Objects; import java.util.Optional; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; + /** Selects the dataset snapshot and configured datasource used to execute a query. */ -public record ExecutionTarget(String datasetId, String dataSource, Optional catalogRevision) { +public record ExecutionTarget( + @NotBlank String datasetId, + @NotBlank String dataSource, + @NotNull Optional catalogRevision +) { - public ExecutionTarget { - datasetId = ModelValidation.requireNonBlank(datasetId, "datasetId"); - dataSource = ModelValidation.requireNonBlank(dataSource, "dataSource"); - catalogRevision = Objects.requireNonNull(catalogRevision, "catalogRevision") - .map(revision -> ModelValidation.requireNonBlank(revision, "catalogRevision")); + @AssertTrue(message = "catalogRevision must not be blank when present") + public boolean isCatalogRevisionValid() { + return catalogRevision == null || catalogRevision.isEmpty() || !catalogRevision.get().isBlank(); } public ExecutionTarget(String datasetId, String dataSource) { diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ModelValidation.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ModelValidation.java deleted file mode 100644 index 8f65dc6d7e..0000000000 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ModelValidation.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.bakdata.conquery.sql.query; - -import java.util.Collection; -import java.util.Objects; - -final class ModelValidation { - - private ModelValidation() { - } - - static String requireNonBlank(String value, String name) { - Objects.requireNonNull(value, name); - if (value.isBlank()) { - throw new IllegalArgumentException(name + " must not be blank"); - } - return value; - } - - static Collection requireNotEmpty(Collection values, String name) { - Objects.requireNonNull(values, name); - if (values.isEmpty()) { - throw new IllegalArgumentException(name + " must not be empty"); - } - return values; - } -} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/NegationNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/NegationNode.java deleted file mode 100644 index 9562532f1b..0000000000 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/NegationNode.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.bakdata.conquery.sql.query; - -import java.util.Objects; - -/** Logical negation. */ -public record NegationNode(QueryNode child) implements QueryNode { - - public NegationNode { - Objects.requireNonNull(child, "child"); - } -} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/OrNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/OrNode.java deleted file mode 100644 index 33d21304f4..0000000000 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/OrNode.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.bakdata.conquery.sql.query; - -import java.util.List; -import java.util.Objects; - -/** Logical disjunction with its already-derived validity-date behavior. */ -public record OrNode(List children, DateAggregationAction dateAction, - boolean createExists) implements QueryNode { - - public OrNode { - ModelValidation.requireNotEmpty(children, "children"); - children = List.copyOf(children); - Objects.requireNonNull(dateAction, "dateAction"); - } -} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedColumn.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedColumn.java deleted file mode 100644 index c1840a9325..0000000000 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedColumn.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.bakdata.conquery.sql.query; - -import java.util.Objects; - -/** A column whose owning table, physical name, and logical type are known. */ -public record ResolvedColumn( - String logicalId, - SqlTable table, - String physicalName, - ColumnType type, - boolean nullable -) { - - public ResolvedColumn { - logicalId = ModelValidation.requireNonBlank(logicalId, "logicalId"); - Objects.requireNonNull(table, "table"); - physicalName = ModelValidation.requireNonBlank(physicalName, "physicalName"); - Objects.requireNonNull(type, "type"); - } -} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedConnector.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedConnector.java deleted file mode 100644 index df50742fe4..0000000000 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedConnector.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.bakdata.conquery.sql.query; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -/** A selected connector with all physical columns and operations resolved. */ -public record ResolvedConnector( - String logicalId, - SqlTable table, - ResolvedColumn primaryId, - Optional secondaryId, - ResolvedValidityDate validityDate, - List filters, - List selects, - List conditions -) { - - public ResolvedConnector { - logicalId = ModelValidation.requireNonBlank(logicalId, "logicalId"); - Objects.requireNonNull(table, "table"); - requireColumnOnTable(primaryId, table, "primaryId"); - if (Objects.requireNonNull(secondaryId, "secondaryId").isPresent()) { - secondaryId = Optional.of(requireColumnOnTable(secondaryId.get(), table, "secondaryId")); - } - Objects.requireNonNull(validityDate, "validityDate"); - validateValidityDateTable(validityDate, table); - filters = List.copyOf(Objects.requireNonNull(filters, "filters")); - selects = List.copyOf(Objects.requireNonNull(selects, "selects")); - conditions = List.copyOf(Objects.requireNonNull(conditions, "conditions")); - } - - private static ResolvedColumn requireColumnOnTable(ResolvedColumn column, SqlTable table, String name) { - Objects.requireNonNull(column, name); - if (!table.equals(column.table())) { - throw new IllegalArgumentException(name + " must belong to the connector table"); - } - return column; - } - - private static void validateValidityDateTable(ResolvedValidityDate validityDate, SqlTable table) { - switch (validityDate) { - case ResolvedValidityDate.None ignored -> { - } - case ResolvedValidityDate.Point point -> requireColumnOnTable(point.column(), table, "validityDate"); - case ResolvedValidityDate.Range range -> { - requireColumnOnTable(range.start(), table, "validityDate.start"); - requireColumnOnTable(range.end(), table, "validityDate.end"); - } - } - } -} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedQuery.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedQuery.java index c6fcba1565..f820a6db00 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedQuery.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedQuery.java @@ -1,7 +1,13 @@ package com.bakdata.conquery.sql.query; import java.util.List; -import java.util.Objects; + +import com.bakdata.conquery.sql.query.internal.ModelNormalization; +import com.bakdata.conquery.sql.query.node.QueryNode; +import com.bakdata.conquery.sql.query.result.ResultColumn; +import com.bakdata.conquery.sql.query.schema.EntitySchema; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; /** * Complete input for compiling a concept query to SQL. @@ -9,17 +15,14 @@ * @param includeValidityDate whether the final result exposes an aggregated validity-date column */ public record ResolvedQuery( - ExecutionTarget target, - EntitySchema entitySchema, - QueryNode root, + @NotNull @Valid ExecutionTarget target, + @NotNull @Valid EntitySchema entitySchema, + @NotNull @Valid QueryNode root, boolean includeValidityDate, - List resultColumns + @NotNull List<@NotNull @Valid ResultColumn> resultColumns ) { public ResolvedQuery { - Objects.requireNonNull(target, "target"); - Objects.requireNonNull(entitySchema, "entitySchema"); - Objects.requireNonNull(root, "root"); - resultColumns = List.copyOf(Objects.requireNonNull(resultColumns, "resultColumns")); + resultColumns = ModelNormalization.immutableCopy(resultColumns); } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedValidityDate.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedValidityDate.java deleted file mode 100644 index 88f3e4e93e..0000000000 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedValidityDate.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.bakdata.conquery.sql.query; - -import java.util.Objects; - -/** Physical representation of event validity for a connector. */ -public sealed interface ResolvedValidityDate permits ResolvedValidityDate.None, ResolvedValidityDate.Point, ResolvedValidityDate.Range { - - record None() implements ResolvedValidityDate { - } - - record Point(ResolvedColumn column) implements ResolvedValidityDate { - public Point { - requireDateColumn(column, "column"); - } - } - - record Range(ResolvedColumn start, ResolvedColumn end) implements ResolvedValidityDate { - public Range { - requireDateColumn(start, "start"); - requireDateColumn(end, "end"); - if (!start.table().equals(end.table())) { - throw new IllegalArgumentException("validity-date columns must belong to the same table"); - } - } - } - - private static void requireDateColumn(ResolvedColumn column, String name) { - Objects.requireNonNull(column, name); - if (column.type() != ColumnType.DATE) { - throw new IllegalArgumentException(name + " must be a DATE column"); - } - } -} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResultColumn.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResultColumn.java deleted file mode 100644 index e02a0e584c..0000000000 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResultColumn.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.bakdata.conquery.sql.query; - -import java.util.Objects; -import java.util.Optional; -import java.util.Set; - -/** Ordered output-column metadata, with presentation names already resolved for the query locale. */ -public record ResultColumn( - String label, - Optional defaultLabel, - Optional description, - ResultType type, - Set semantics -) { - - public ResultColumn { - label = ModelValidation.requireNonBlank(label, "label"); - defaultLabel = Objects.requireNonNull(defaultLabel, "defaultLabel") - .map(value -> ModelValidation.requireNonBlank(value, "defaultLabel")); - Objects.requireNonNull(description, "description"); - Objects.requireNonNull(type, "type"); - semantics = Set.copyOf(Objects.requireNonNull(semantics, "semantics")); - } -} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/SqlTable.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/SqlTable.java deleted file mode 100644 index 1909f560d6..0000000000 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/SqlTable.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.bakdata.conquery.sql.query; - -import java.util.Arrays; -import java.util.List; - -/** A logical table together with its unquoted physical SQL name. */ -public record SqlTable(String logicalId, List physicalName) { - - public SqlTable { - logicalId = ModelValidation.requireNonBlank(logicalId, "logicalId"); - ModelValidation.requireNotEmpty(physicalName, "physicalName"); - physicalName = List.copyOf(physicalName); - physicalName.forEach(part -> ModelValidation.requireNonBlank(part, "physicalName part")); - } - - public static SqlTable of(String logicalId, String... physicalName) { - return new SqlTable(logicalId, Arrays.asList(physicalName)); - } -} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/internal/ModelNormalization.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/internal/ModelNormalization.java new file mode 100644 index 0000000000..f004d0ab79 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/internal/ModelNormalization.java @@ -0,0 +1,27 @@ +package com.bakdata.conquery.sql.query.internal; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public final class ModelNormalization { + + private ModelNormalization() { + } + + public static List immutableCopy(List values) { + return values == null ? null : Collections.unmodifiableList(new ArrayList<>(values)); + } + + public static Set immutableCopy(Set values) { + return values == null ? null : Collections.unmodifiableSet(new LinkedHashSet<>(values)); + } + + public static Map immutableCopy(Map values) { + return values == null ? null : Collections.unmodifiableMap(new LinkedHashMap<>(values)); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/AndNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/AndNode.java new file mode 100644 index 0000000000..a3b614ce53 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/AndNode.java @@ -0,0 +1,20 @@ +package com.bakdata.conquery.sql.query.node; + +import java.util.List; + +import com.bakdata.conquery.sql.query.internal.ModelNormalization; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; + +/** Logical conjunction with its already-derived validity-date behavior. */ +public record AndNode( + @NotEmpty List<@NotNull @Valid QueryNode> children, + @NotNull DateAggregationAction dateAction, + boolean createExists +) implements QueryNode { + + public AndNode { + children = ModelNormalization.immutableCopy(children); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/ConceptNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/ConceptNode.java new file mode 100644 index 0000000000..4e23392cef --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/ConceptNode.java @@ -0,0 +1,25 @@ +package com.bakdata.conquery.sql.query.node; + +import java.util.List; + +import com.bakdata.conquery.sql.query.internal.ModelNormalization; +import com.bakdata.conquery.sql.query.operation.ResolvedSelect; +import com.bakdata.conquery.sql.query.schema.ResolvedConnector; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; + +/** A concept selection whose connectors, conditions, filters, and selects are fully resolved. */ +public record ConceptNode( + @NotBlank String logicalId, + @NotEmpty List<@NotNull @Valid ResolvedConnector> connectors, + @NotNull List<@NotNull @Valid ResolvedSelect> selects, + @NotNull DateAggregationAction dateAction +) implements QueryNode { + + public ConceptNode { + connectors = ModelNormalization.immutableCopy(connectors); + selects = ModelNormalization.immutableCopy(selects); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateAggregationAction.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/DateAggregationAction.java similarity index 76% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateAggregationAction.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/DateAggregationAction.java index 0f5abe96ee..17a56d680c 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/DateAggregationAction.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/DateAggregationAction.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query; +package com.bakdata.conquery.sql.query.node; /** Defines how validity dates produced by child nodes are propagated. */ public enum DateAggregationAction { diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/DateRestrictionNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/DateRestrictionNode.java new file mode 100644 index 0000000000..8a0dbe6367 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/DateRestrictionNode.java @@ -0,0 +1,12 @@ +package com.bakdata.conquery.sql.query.node; + +import com.bakdata.conquery.sql.query.range.DateRange; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; + +/** Applies an inclusive date restriction to its child. */ +public record DateRestrictionNode( + @NotNull @Valid DateRange dateRange, + @NotNull @Valid QueryNode child +) implements QueryNode { +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/NegationNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/NegationNode.java new file mode 100644 index 0000000000..4fbe3100f8 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/NegationNode.java @@ -0,0 +1,8 @@ +package com.bakdata.conquery.sql.query.node; + +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; + +/** Logical negation. */ +public record NegationNode(@NotNull @Valid QueryNode child) implements QueryNode { +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/OrNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/OrNode.java new file mode 100644 index 0000000000..14cdb6c9b6 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/OrNode.java @@ -0,0 +1,20 @@ +package com.bakdata.conquery.sql.query.node; + +import java.util.List; + +import com.bakdata.conquery.sql.query.internal.ModelNormalization; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; + +/** Logical disjunction with its already-derived validity-date behavior. */ +public record OrNode( + @NotEmpty List<@NotNull @Valid QueryNode> children, + @NotNull DateAggregationAction dateAction, + boolean createExists +) implements QueryNode { + + public OrNode { + children = ModelNormalization.immutableCopy(children); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/QueryNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/QueryNode.java similarity index 79% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/QueryNode.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/QueryNode.java index 8813838524..23f932c988 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/QueryNode.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/QueryNode.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query; +package com.bakdata.conquery.sql.query.node; /** A normalized query-tree node accepted by SQL compilation. */ public sealed interface QueryNode permits AndNode, OrNode, NegationNode, DateRestrictionNode, ConceptNode { diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/package-info.java new file mode 100644 index 0000000000..df44c5f5ee --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/package-info.java @@ -0,0 +1,2 @@ +/** Normalized nodes of a fully resolved query tree. */ +package com.bakdata.conquery.sql.query.node; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInAggregations.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInAggregations.java new file mode 100644 index 0000000000..e4b0fb50cb --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInAggregations.java @@ -0,0 +1,132 @@ +package com.bakdata.conquery.sql.query.operation; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.sql.query.internal.ModelNormalization; +import com.bakdata.conquery.sql.query.schema.DateColumns; +import com.bakdata.conquery.sql.query.schema.ResolvedColumn; +import com.bakdata.conquery.sql.query.validation.AllowedColumnTypes; +import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; + +/** Framework-neutral aggregation operations supported by the SQL connector. */ +public final class BuiltInAggregations { + + private BuiltInAggregations() { + } + + public record Count( + @NotNull @Valid ResolvedColumn column, + @NotNull List<@NotNull @Valid ResolvedColumn> distinctBy + ) implements ResolvedAggregation { + + public Count { + distinctBy = ModelNormalization.immutableCopy(distinctBy); + } + + @AssertTrue(message = "count columns must belong to the same table") + public boolean isOnSameTable() { + return sameTable(withColumn(column, distinctBy)); + } + } + + public record CountQuarters(@NotNull @Valid DateColumns dates) implements ResolvedAggregation { + } + + public record DurationSum( + @NotNull @Valid DateColumns dates, + @NotNull List<@NotNull @Valid ResolvedColumn> distinctBy + ) implements ResolvedAggregation { + + public DurationSum { + distinctBy = ModelNormalization.immutableCopy(distinctBy); + } + + @AssertTrue(message = "duration-sum columns must belong to the same table") + public boolean isOnSameTable() { + return dates == null || sameTable(withColumns(dates.columns(), distinctBy)); + } + } + + public record Sum( + @NotNull @Valid @AllowedColumnTypes({ColumnType.INTEGER, ColumnType.REAL, ColumnType.DECIMAL, ColumnType.MONEY}) + ResolvedColumn column, + @NotNull Optional<@Valid @AllowedColumnTypes({ + ColumnType.INTEGER, ColumnType.REAL, ColumnType.DECIMAL, ColumnType.MONEY + }) ResolvedColumn> subtractColumn, + @NotNull List<@NotNull @Valid ResolvedColumn> distinctBy + ) implements ResolvedAggregation { + + public Sum { + distinctBy = ModelNormalization.immutableCopy(distinctBy); + } + + @AssertTrue(message = "column and subtractColumn must have the same type") + public boolean isMatchingColumnType() { + return column == null || subtractColumn == null || subtractColumn.isEmpty() + || column.type() == subtractColumn.get().type(); + } + + @AssertTrue(message = "sum columns must belong to the same table") + public boolean isOnSameTable() { + if (subtractColumn == null) { + return true; + } + List columns = withColumn(column, distinctBy); + subtractColumn.ifPresent(columns::add); + return sameTable(columns); + } + } + + public record Flags( + @NotEmpty Map<@NotBlank String, @NotNull @Valid @AllowedColumnTypes(ColumnType.BOOLEAN) ResolvedColumn> columns + ) implements ResolvedAggregation { + + public Flags { + columns = ModelNormalization.immutableCopy(columns); + } + + @AssertTrue(message = "flag columns must be unique") + public boolean isUnique() { + return columns == null || columns.values().stream().distinct().count() == columns.size(); + } + + @AssertTrue(message = "flag columns must belong to the same table") + public boolean isOnSameTable() { + return columns == null || sameTable(columns.values()); + } + } + + private static List withColumn(ResolvedColumn column, List other) { + List columns = new ArrayList<>(); + if (column != null) { + columns.add(column); + } + return withColumns(columns, other); + } + + private static List withColumns(List columns, List other) { + List combined = new ArrayList<>(columns); + if (other != null) { + combined.addAll(other); + } + return combined; + } + + private static boolean sameTable(Collection columns) { + return columns.stream() + .filter(java.util.Objects::nonNull) + .map(ResolvedColumn::table) + .distinct() + .limit(2) + .count() < 2; + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInConditions.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInConditions.java new file mode 100644 index 0000000000..10e7215273 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInConditions.java @@ -0,0 +1,69 @@ +package com.bakdata.conquery.sql.query.operation; + +import java.util.List; +import java.util.Set; + +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.sql.query.internal.ModelNormalization; +import com.bakdata.conquery.sql.query.schema.ResolvedColumn; +import com.bakdata.conquery.sql.query.validation.AllowedColumnTypes; +import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; + +/** Declarative connector and concept-element conditions supported by the SQL connector. */ +public final class BuiltInConditions { + + private BuiltInConditions() { + } + + public record AllOf( + @NotEmpty List<@NotNull @Valid ResolvedCondition> conditions + ) implements ResolvedCondition { + + public AllOf { + conditions = ModelNormalization.immutableCopy(conditions); + } + } + + public record Not(@NotNull @Valid ResolvedCondition condition) implements ResolvedCondition { + } + + public record StringValues( + @NotNull @Valid @AllowedColumnTypes(ColumnType.STRING) ResolvedColumn column, + @NotEmpty Set<@NotNull String> values + ) implements ResolvedCondition { + + public StringValues { + values = ModelNormalization.immutableCopy(values); + } + } + + public record Presence(@NotNull @Valid ResolvedColumn column, boolean present) implements ResolvedCondition { + } + + public record Prefixes( + @NotNull @Valid @AllowedColumnTypes(ColumnType.STRING) ResolvedColumn column, + @NotEmpty List<@NotBlank String> prefixes + ) implements ResolvedCondition { + + public Prefixes { + prefixes = ModelNormalization.immutableCopy(prefixes); + } + } + + public record PrefixRange( + @NotNull @Valid @AllowedColumnTypes(ColumnType.STRING) ResolvedColumn column, + @NotBlank String minimum, + @NotBlank String maximum + ) implements ResolvedCondition { + + @AssertTrue(message = "prefix bounds must have equal length and be ordered") + public boolean isOrdered() { + return minimum == null || maximum == null + || minimum.length() == maximum.length() && minimum.compareTo(maximum) <= 0; + } + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInFilters.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInFilters.java new file mode 100644 index 0000000000..c44cbcb4d4 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInFilters.java @@ -0,0 +1,81 @@ +package com.bakdata.conquery.sql.query.operation; + +import java.time.LocalDate; +import java.time.temporal.ChronoUnit; +import java.util.Optional; +import java.util.Set; + +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.sql.query.internal.ModelNormalization; +import com.bakdata.conquery.sql.query.range.NumberRange; +import com.bakdata.conquery.sql.query.range.SubstringRange; +import com.bakdata.conquery.sql.query.schema.ResolvedColumn; +import com.bakdata.conquery.sql.query.validation.AllowedColumnTypes; +import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; + +/** Framework-neutral filter operations supported by the SQL connector. */ +public final class BuiltInFilters { + + private BuiltInFilters() { + } + + public record StringValues( + @NotNull @Valid @AllowedColumnTypes(ColumnType.STRING) ResolvedColumn column, + @NotEmpty Set<@NotNull String> values, + @NotNull Optional<@Valid SubstringRange> substring + ) implements ResolvedFilter { + + public StringValues { + values = ModelNormalization.immutableCopy(values); + } + + public StringValues(ResolvedColumn column, Set values) { + this(column, values, Optional.empty()); + } + } + + public record NumericColumnRange( + @NotNull @Valid @AllowedColumnTypes({ColumnType.INTEGER, ColumnType.REAL, ColumnType.DECIMAL, ColumnType.MONEY}) + ResolvedColumn column, + @NotNull @Valid NumberRange range + ) implements ResolvedFilter { + } + + public record AggregationRange( + @NotNull @Valid ResolvedAggregation aggregation, + @NotNull @Valid NumberRange range + ) implements ResolvedFilter { + } + + /** Filters by distance to an end date that was frozen while resolving the query. */ + public record DateDistanceRange( + @NotNull @Valid @AllowedColumnTypes(ColumnType.DATE) ResolvedColumn column, + @NotNull ChronoUnit unit, + @NotNull LocalDate endDate, + @NotNull @Valid NumberRange range + ) implements ResolvedFilter { + } + + public record Flags( + @NotNull @Valid BuiltInAggregations.Flags availableFlags, + @NotEmpty Set<@NotNull String> selectedFlags + ) implements ResolvedFilter { + + public Flags { + selectedFlags = ModelNormalization.immutableCopy(selectedFlags); + } + + public Flags(java.util.Map availableFlags, Set selectedFlags) { + this(new BuiltInAggregations.Flags(availableFlags), selectedFlags); + } + + @AssertTrue(message = "selectedFlags must be defined by availableFlags") + public boolean isSelectionKnown() { + return availableFlags == null || availableFlags.columns() == null || selectedFlags == null + || availableFlags.columns().keySet().containsAll(selectedFlags); + } + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInSelects.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInSelects.java new file mode 100644 index 0000000000..b06d8f4b00 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInSelects.java @@ -0,0 +1,76 @@ +package com.bakdata.conquery.sql.query.operation; + +import java.time.LocalDate; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.sql.query.internal.ModelNormalization; +import com.bakdata.conquery.sql.query.range.SubstringRange; +import com.bakdata.conquery.sql.query.schema.DateColumns; +import com.bakdata.conquery.sql.query.schema.ResolvedColumn; +import com.bakdata.conquery.sql.query.validation.AllowedColumnTypes; +import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; + +/** Framework-neutral select operations supported by the SQL connector. */ +public final class BuiltInSelects { + + private BuiltInSelects() { + } + + public enum ValueOperation { + DISTINCT, + FIRST, + LAST, + RANDOM + } + + public record Aggregation(@NotNull @Valid ResolvedAggregation aggregation) implements ResolvedSelect { + } + + public record Values( + @NotNull @Valid ResolvedColumn column, + @NotNull ValueOperation operation, + @NotNull Optional<@Valid SubstringRange> substring + ) implements ResolvedSelect { + + @AssertTrue(message = "substring requires a STRING column") + public boolean isSubstringOnStringColumn() { + return column == null || substring == null || substring.isEmpty() || column.type() == ColumnType.STRING; + } + } + + public record DateUnion(@NotNull @Valid DateColumns dates) implements ResolvedSelect { + } + + /** Selects the distance to an end date that was frozen while resolving the query. */ + public record DateDistance( + @NotNull @Valid @AllowedColumnTypes({ColumnType.DATE, ColumnType.DATE_RANGE}) ResolvedColumn column, + @NotNull ChronoUnit unit, + @NotNull LocalDate endDate + ) implements ResolvedSelect { + } + + /** Aggregates the resolved concept-value columns across connector tables. */ + public record ConceptValues( + @NotEmpty List<@NotNull @Valid ResolvedColumn> columns + ) implements ResolvedSelect { + + public ConceptValues { + columns = ModelNormalization.immutableCopy(columns); + } + } + + public record EventDateUnion() implements ResolvedSelect { + } + + public record EventDurationSum() implements ResolvedSelect { + } + + public record Exists() implements ResolvedSelect { + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedAggregation.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedAggregation.java new file mode 100644 index 0000000000..b33eff1500 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedAggregation.java @@ -0,0 +1,5 @@ +package com.bakdata.conquery.sql.query.operation; + +/** A resolved aggregation that can be exposed as a result or constrained by an aggregation filter. */ +public interface ResolvedAggregation { +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedCondition.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedCondition.java similarity index 66% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedCondition.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedCondition.java index 2e9255bc53..7ce1db4d1b 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedCondition.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedCondition.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query; +package com.bakdata.conquery.sql.query.operation; /** A resolved connector or concept-element condition. */ public interface ResolvedCondition { diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedFilter.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedFilter.java similarity index 83% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedFilter.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedFilter.java index c6712c958b..5a2eef35fa 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedFilter.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedFilter.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query; +package com.bakdata.conquery.sql.query.operation; /** * Marker for an immutable, validated filter operation. diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedSelect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedSelect.java similarity index 80% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedSelect.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedSelect.java index 44cb0b2707..c9210791dd 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedSelect.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedSelect.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query; +package com.bakdata.conquery.sql.query.operation; /** * Marker for an immutable, validated select operation. diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/package-info.java new file mode 100644 index 0000000000..381f90f0fd --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/package-info.java @@ -0,0 +1,2 @@ +/** Extensible filter, select, condition, and aggregation operations accepted by SQL compilation. */ +package com.bakdata.conquery.sql.query.operation; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/package-info.java index 75e6f6214f..bd3cf6dbf2 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/package-info.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/package-info.java @@ -4,5 +4,8 @@ *

Instances in this package must not require repository access or identifier resolution during compilation. They * describe query semantics and physical database objects, but deliberately exclude connections, dialects, and SQL * compiler state.

+ * + *

The root query aggregate is complemented by dedicated packages for query nodes, operations, ranges, result + * metadata, physical schema references, and validation.

*/ package com.bakdata.conquery.sql.query; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/DateRange.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/DateRange.java new file mode 100644 index 0000000000..5251e45b95 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/DateRange.java @@ -0,0 +1,25 @@ +package com.bakdata.conquery.sql.query.range; + +import java.time.LocalDate; +import java.util.Optional; + +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.NotNull; + +/** Inclusive date restriction. An empty bound represents an unbounded side. */ +public record DateRange( + @NotNull Optional startInclusive, + @NotNull Optional endInclusive +) { + + @AssertTrue(message = "startInclusive must not be after endInclusive") + public boolean isOrdered() { + return startInclusive == null || endInclusive == null + || startInclusive.isEmpty() || endInclusive.isEmpty() + || !startInclusive.get().isAfter(endInclusive.get()); + } + + public static DateRange closed(LocalDate startInclusive, LocalDate endInclusive) { + return new DateRange(Optional.of(startInclusive), Optional.of(endInclusive)); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/NumberRange.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/NumberRange.java new file mode 100644 index 0000000000..82c91aef72 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/NumberRange.java @@ -0,0 +1,43 @@ +package com.bakdata.conquery.sql.query.range; + +import java.math.BigDecimal; +import java.util.Objects; +import java.util.Optional; + +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.NotNull; + +/** Inclusive numeric range. An empty bound represents an unbounded side. */ +public record NumberRange( + @NotNull Optional minimum, + @NotNull Optional maximum +) { + + @AssertTrue(message = "minimum must not be greater than maximum") + public boolean isOrdered() { + return minimum == null || maximum == null + || minimum.isEmpty() || maximum.isEmpty() + || minimum.get().compareTo(maximum.get()) <= 0; + } + + public static NumberRange closed(Number minimum, Number maximum) { + return new NumberRange(Optional.of(decimal(minimum)), Optional.of(decimal(maximum))); + } + + public static NumberRange atLeast(Number minimum) { + return new NumberRange(Optional.of(decimal(minimum)), Optional.empty()); + } + + public static NumberRange atMost(Number maximum) { + return new NumberRange(Optional.empty(), Optional.of(decimal(maximum))); + } + + public static NumberRange unbounded() { + return new NumberRange(Optional.empty(), Optional.empty()); + } + + private static BigDecimal decimal(Number value) { + Objects.requireNonNull(value, "value"); + return value instanceof BigDecimal decimal ? decimal : new BigDecimal(value.toString()); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/SubstringRange.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/SubstringRange.java new file mode 100644 index 0000000000..ce45606115 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/SubstringRange.java @@ -0,0 +1,27 @@ +package com.bakdata.conquery.sql.query.range; + +import java.util.Optional; + +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.PositiveOrZero; + +/** Zero-based substring bounds with an exclusive, optionally unbounded end. */ +public record SubstringRange( + @PositiveOrZero int startInclusive, + @NotNull Optional<@PositiveOrZero Integer> endExclusive +) { + + @AssertTrue(message = "endExclusive must not be smaller than startInclusive") + public boolean isOrdered() { + return endExclusive == null || endExclusive.isEmpty() || endExclusive.get() >= startInclusive; + } + + public static SubstringRange from(int startInclusive) { + return new SubstringRange(startInclusive, Optional.empty()); + } + + public static SubstringRange between(int startInclusive, int endExclusive) { + return new SubstringRange(startInclusive, Optional.of(endExclusive)); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/package-info.java new file mode 100644 index 0000000000..a9ec695bf5 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/package-info.java @@ -0,0 +1,2 @@ +/** Typed, framework-neutral range values used by resolved nodes and operations. */ +package com.bakdata.conquery.sql.query.range; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/result/ResultColumn.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/result/ResultColumn.java new file mode 100644 index 0000000000..9d7a3619d5 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/result/ResultColumn.java @@ -0,0 +1,29 @@ +package com.bakdata.conquery.sql.query.result; + +import java.util.Optional; +import java.util.Set; + +import com.bakdata.conquery.sql.query.internal.ModelNormalization; +import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; + +/** Ordered output-column metadata, with presentation names already resolved for the query locale. */ +public record ResultColumn( + @NotBlank String label, + @NotNull Optional defaultLabel, + @NotNull Optional description, + @NotNull @Valid ResultType type, + @NotNull Set<@NotBlank String> semantics +) { + + public ResultColumn { + semantics = ModelNormalization.immutableCopy(semantics); + } + + @AssertTrue(message = "defaultLabel must not be blank when present") + public boolean isDefaultLabelValid() { + return defaultLabel == null || defaultLabel.isEmpty() || !defaultLabel.get().isBlank(); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResultType.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/result/ResultType.java similarity index 54% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResultType.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/query/result/ResultType.java index 929c1af930..a1bb6826af 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResultType.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/result/ResultType.java @@ -1,6 +1,6 @@ -package com.bakdata.conquery.sql.query; +package com.bakdata.conquery.sql.query.result; -import java.util.Objects; +import jakarta.validation.constraints.NotNull; /** Logical type of a result column. */ public sealed interface ResultType permits ResultType.Primitive, ResultType.ListType { @@ -15,9 +15,6 @@ enum Primitive implements ResultType { MONEY } - record ListType(Primitive elementType) implements ResultType { - public ListType { - Objects.requireNonNull(elementType, "elementType"); - } + record ListType(@NotNull Primitive elementType) implements ResultType { } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/result/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/result/package-info.java new file mode 100644 index 0000000000..54870eda8a --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/result/package-info.java @@ -0,0 +1,2 @@ +/** Ordered result-column metadata and logical result types. */ +package com.bakdata.conquery.sql.query.result; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/DateColumns.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/DateColumns.java new file mode 100644 index 0000000000..ea566c006c --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/DateColumns.java @@ -0,0 +1,42 @@ +package com.bakdata.conquery.sql.query.schema; + +import java.util.List; +import java.util.Objects; + +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.sql.query.validation.AllowedColumnTypes; +import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.NotNull; + +/** Resolved physical representation of a date or date range used by an operation. */ +public sealed interface DateColumns permits DateColumns.Single, DateColumns.Pair { + + List columns(); + + record Single( + @NotNull @Valid @AllowedColumnTypes({ColumnType.DATE, ColumnType.DATE_RANGE}) ResolvedColumn column + ) implements DateColumns { + + @Override + public List columns() { + return column == null ? List.of() : List.of(column); + } + } + + record Pair( + @NotNull @Valid @AllowedColumnTypes(ColumnType.DATE) ResolvedColumn start, + @NotNull @Valid @AllowedColumnTypes(ColumnType.DATE) ResolvedColumn end + ) implements DateColumns { + + @Override + public List columns() { + return start == null || end == null ? List.of() : List.of(start, end); + } + + @AssertTrue(message = "date columns must belong to the same table") + public boolean isOnSameTable() { + return start == null || end == null || Objects.equals(start.table(), end.table()); + } + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/EntitySchema.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/EntitySchema.java new file mode 100644 index 0000000000..212e1ab712 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/EntitySchema.java @@ -0,0 +1,17 @@ +package com.bakdata.conquery.sql.query.schema; + +import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.NotNull; + +/** Physical source used to enumerate all entities, for example when compiling a root-level negation. */ +public record EntitySchema( + @NotNull @Valid SqlTable table, + @NotNull @Valid ResolvedColumn primaryId +) { + + @AssertTrue(message = "primaryId must belong to the entity table") + public boolean isPrimaryIdOnEntityTable() { + return table == null || primaryId == null || table.equals(primaryId.table()); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/ResolvedColumn.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/ResolvedColumn.java new file mode 100644 index 0000000000..ff285f38b9 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/ResolvedColumn.java @@ -0,0 +1,16 @@ +package com.bakdata.conquery.sql.query.schema; + +import com.bakdata.conquery.models.datasets.ColumnType; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; + +/** A column whose owning table, physical name, and logical type are known. */ +public record ResolvedColumn( + @NotBlank String logicalId, + @NotNull @Valid SqlTable table, + @NotBlank String physicalName, + @NotNull ColumnType type, + boolean nullable +) { +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/ResolvedConnector.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/ResolvedConnector.java new file mode 100644 index 0000000000..d23a9ec1af --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/ResolvedConnector.java @@ -0,0 +1,59 @@ +package com.bakdata.conquery.sql.query.schema; + +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.sql.query.internal.ModelNormalization; +import com.bakdata.conquery.sql.query.operation.ResolvedCondition; +import com.bakdata.conquery.sql.query.operation.ResolvedFilter; +import com.bakdata.conquery.sql.query.operation.ResolvedSelect; +import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; + +/** A selected connector with all physical columns and operations resolved. */ +public record ResolvedConnector( + @NotBlank String logicalId, + @NotNull @Valid SqlTable table, + @NotNull @Valid ResolvedColumn primaryId, + @NotNull Optional<@Valid ResolvedColumn> secondaryId, + @NotNull @Valid ResolvedValidityDate validityDate, + @NotNull List<@NotNull @Valid ResolvedFilter> filters, + @NotNull List<@NotNull @Valid ResolvedSelect> selects, + @NotNull List<@NotNull @Valid ResolvedCondition> conditions +) { + + public ResolvedConnector { + filters = ModelNormalization.immutableCopy(filters); + selects = ModelNormalization.immutableCopy(selects); + conditions = ModelNormalization.immutableCopy(conditions); + } + + @AssertTrue(message = "identifier columns must belong to the connector table") + public boolean isIdentifierColumnsOnConnectorTable() { + return table == null + || primaryId == null + || secondaryId == null + || table.equals(primaryId.table()) && secondaryId.stream().map(ResolvedColumn::table).allMatch(table::equals); + } + + @AssertTrue(message = "validity-date columns must belong to the connector table") + public boolean isValidityDateOnConnectorTable() { + if (validityDate == null || table == null) { + return true; + } + switch (validityDate) { + case ResolvedValidityDate.None ignored -> { + return true; + } + case ResolvedValidityDate.Point point -> { + return point.column() == null || table.equals(point.column().table()); + } + case ResolvedValidityDate.Range range -> { + return range.start() == null || range.end() == null + || table.equals(range.start().table()) && table.equals(range.end().table()); + } + } + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/ResolvedValidityDate.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/ResolvedValidityDate.java new file mode 100644 index 0000000000..73d71f6179 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/ResolvedValidityDate.java @@ -0,0 +1,32 @@ +package com.bakdata.conquery.sql.query.schema; + +import java.util.Objects; + +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.sql.query.validation.AllowedColumnTypes; +import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.NotNull; + +/** Physical representation of event validity for a connector. */ +public sealed interface ResolvedValidityDate permits ResolvedValidityDate.None, ResolvedValidityDate.Point, ResolvedValidityDate.Range { + + record None() implements ResolvedValidityDate { + } + + record Point( + @NotNull @Valid @AllowedColumnTypes(ColumnType.DATE) ResolvedColumn column + ) implements ResolvedValidityDate { + } + + record Range( + @NotNull @Valid @AllowedColumnTypes(ColumnType.DATE) ResolvedColumn start, + @NotNull @Valid @AllowedColumnTypes(ColumnType.DATE) ResolvedColumn end + ) implements ResolvedValidityDate { + + @AssertTrue(message = "validity-date columns must belong to the same table") + public boolean isOnSameTable() { + return start == null || end == null || Objects.equals(start.table(), end.table()); + } + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/SqlTable.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/SqlTable.java new file mode 100644 index 0000000000..e927aaaa57 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/SqlTable.java @@ -0,0 +1,26 @@ +package com.bakdata.conquery.sql.query.schema; + +import java.util.Arrays; +import java.util.List; + +import com.bakdata.conquery.sql.query.internal.ModelNormalization; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotEmpty; + +/** A logical table together with its unquoted physical SQL name. */ +public record SqlTable( + @NotBlank String logicalId, + @NotEmpty List<@NotBlank String> physicalName +) { + + public SqlTable { + physicalName = ModelNormalization.immutableCopy(physicalName); + } + + public static SqlTable of(String logicalId, String... physicalName) { + if (physicalName == null) { + return new SqlTable(logicalId, null); + } + return new SqlTable(logicalId, Arrays.asList(physicalName)); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/package-info.java new file mode 100644 index 0000000000..1b56e9adc2 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/package-info.java @@ -0,0 +1,2 @@ +/** Resolved physical tables, columns, connectors, and entity schema used during SQL compilation. */ +package com.bakdata.conquery.sql.query.schema; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/AllowedColumnTypes.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/AllowedColumnTypes.java new file mode 100644 index 0000000000..9f7b5beeba --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/AllowedColumnTypes.java @@ -0,0 +1,26 @@ +package com.bakdata.conquery.sql.query.validation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import com.bakdata.conquery.models.datasets.ColumnType; +import jakarta.validation.Constraint; +import jakarta.validation.Payload; + +@Documented +@Constraint(validatedBy = AllowedColumnTypesValidator.class) +@Target({ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER, ElementType.TYPE_USE, ElementType.RECORD_COMPONENT}) +@Retention(RetentionPolicy.RUNTIME) +public @interface AllowedColumnTypes { + + String message() default "column type is not supported"; + + Class[] groups() default {}; + + Class[] payload() default {}; + + ColumnType[] value(); +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/AllowedColumnTypesValidator.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/AllowedColumnTypesValidator.java new file mode 100644 index 0000000000..d79d3703be --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/AllowedColumnTypesValidator.java @@ -0,0 +1,23 @@ +package com.bakdata.conquery.sql.query.validation; + +import java.util.EnumSet; + +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.sql.query.schema.ResolvedColumn; +import jakarta.validation.ConstraintValidator; +import jakarta.validation.ConstraintValidatorContext; + +public final class AllowedColumnTypesValidator implements ConstraintValidator { + + private EnumSet allowedTypes; + + @Override + public void initialize(AllowedColumnTypes constraint) { + allowedTypes = EnumSet.copyOf(java.util.List.of(constraint.value())); + } + + @Override + public boolean isValid(ResolvedColumn column, ConstraintValidatorContext context) { + return column == null || column.type() == null || allowedTypes.contains(column.type()); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/ResolvedQueryValidation.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/ResolvedQueryValidation.java new file mode 100644 index 0000000000..5991b5e759 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/ResolvedQueryValidation.java @@ -0,0 +1,26 @@ +package com.bakdata.conquery.sql.query.validation; + +import java.util.Objects; +import java.util.Set; + +import com.bakdata.conquery.sql.query.ResolvedQuery; +import jakarta.validation.ConstraintViolation; +import jakarta.validation.ConstraintViolationException; +import jakarta.validation.Validator; + +/** Validates the complete resolved query graph before SQL compilation starts. */ +public final class ResolvedQueryValidation { + + private final Validator validator; + + public ResolvedQueryValidation(Validator validator) { + this.validator = Objects.requireNonNull(validator, "validator"); + } + + public void validate(ResolvedQuery query) { + Set> violations = validator.validate(Objects.requireNonNull(query, "query")); + if (!violations.isEmpty()) { + throw new ConstraintViolationException(violations); + } + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/package-info.java new file mode 100644 index 0000000000..06434d8420 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/package-info.java @@ -0,0 +1,2 @@ +/** Jakarta Bean Validation constraints and the validation boundary for resolved queries. */ +package com.bakdata.conquery.sql.query.validation; diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/query/BuiltInOperationsTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/query/BuiltInOperationsTest.java new file mode 100644 index 0000000000..4dcc56e333 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/query/BuiltInOperationsTest.java @@ -0,0 +1,164 @@ +package com.bakdata.conquery.sql.query; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static com.bakdata.conquery.sql.query.ValidationTestSupport.assertInvalid; +import static com.bakdata.conquery.sql.query.ValidationTestSupport.assertValid; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.sql.query.operation.BuiltInAggregations; +import com.bakdata.conquery.sql.query.operation.BuiltInConditions; +import com.bakdata.conquery.sql.query.operation.BuiltInFilters; +import com.bakdata.conquery.sql.query.operation.BuiltInSelects; +import com.bakdata.conquery.sql.query.operation.ResolvedCondition; +import com.bakdata.conquery.sql.query.operation.ResolvedFilter; +import com.bakdata.conquery.sql.query.operation.ResolvedSelect; +import com.bakdata.conquery.sql.query.range.NumberRange; +import com.bakdata.conquery.sql.query.range.SubstringRange; +import com.bakdata.conquery.sql.query.schema.DateColumns; +import com.bakdata.conquery.sql.query.schema.ResolvedColumn; +import com.bakdata.conquery.sql.query.schema.SqlTable; +import org.junit.jupiter.api.Test; + +class BuiltInOperationsTest { + + private static final SqlTable EVENTS = SqlTable.of("events", "analytics", "events"); + private static final SqlTable OTHER_EVENTS = SqlTable.of("other-events", "analytics", "other_events"); + private static final ResolvedColumn CODE = column("code", ColumnType.STRING); + private static final ResolvedColumn AMOUNT = column("amount", ColumnType.DECIMAL); + private static final ResolvedColumn START = column("start_date", ColumnType.DATE); + private static final ResolvedColumn END = column("end_date", ColumnType.DATE); + private static final ResolvedColumn FLAG_A = column("flag_a", ColumnType.BOOLEAN); + private static final ResolvedColumn FLAG_B = column("flag_b", ColumnType.BOOLEAN); + + @Test + void shouldRepresentStandardFiltersWithResolvedInputs() { + ResolvedFilter values = new BuiltInFilters.StringValues( + CODE, Set.of("A", "B"), Optional.of(SubstringRange.between(0, 3)) + ); + ResolvedFilter number = new BuiltInFilters.NumericColumnRange(AMOUNT, NumberRange.closed(1, 10)); + ResolvedFilter aggregation = new BuiltInFilters.AggregationRange( + new BuiltInAggregations.Sum(AMOUNT, Optional.empty(), List.of()), + NumberRange.atLeast(BigDecimal.TEN) + ); + ResolvedFilter distance = new BuiltInFilters.DateDistanceRange( + START, ChronoUnit.YEARS, LocalDate.of(2026, 8, 27), NumberRange.atMost(65) + ); + ResolvedFilter flags = new BuiltInFilters.Flags( + Map.of("A", FLAG_A, "B", FLAG_B), Set.of("A") + ); + + assertValid(values); + assertValid(number); + assertValid(aggregation); + assertValid(distance); + assertValid(flags); + assertInstanceOf(BuiltInFilters.StringValues.class, values); + assertInstanceOf(BuiltInFilters.NumericColumnRange.class, number); + assertInstanceOf(BuiltInFilters.AggregationRange.class, aggregation); + assertInstanceOf(BuiltInFilters.DateDistanceRange.class, distance); + assertInstanceOf(BuiltInFilters.Flags.class, flags); + } + + @Test + void shouldRepresentStandardSelectsWithResolvedInputs() { + List selects = List.of( + new BuiltInSelects.Aggregation(new BuiltInAggregations.Count(CODE, List.of(CODE))), + new BuiltInSelects.Aggregation(new BuiltInAggregations.CountQuarters(new DateColumns.Pair(START, END))), + new BuiltInSelects.Aggregation(new BuiltInAggregations.DurationSum(new DateColumns.Pair(START, END), List.of(CODE))), + new BuiltInSelects.Aggregation(new BuiltInAggregations.Flags(Map.of("A", FLAG_A, "B", FLAG_B))), + new BuiltInSelects.Values( + CODE, BuiltInSelects.ValueOperation.DISTINCT, Optional.of(SubstringRange.between(0, 3)) + ), + new BuiltInSelects.DateUnion(new DateColumns.Pair(START, END)), + new BuiltInSelects.DateDistance(START, ChronoUnit.DAYS, LocalDate.of(2026, 8, 27)), + new BuiltInSelects.ConceptValues(List.of(CODE)), + new BuiltInSelects.EventDateUnion(), + new BuiltInSelects.EventDurationSum(), + new BuiltInSelects.Exists() + ); + + selects.forEach(ValidationTestSupport::assertValid); + assertEquals(11, selects.size()); + } + + @Test + void shouldRepresentDeclarativeConditions() { + ResolvedCondition condition = new BuiltInConditions.AllOf(List.of( + new BuiltInConditions.StringValues(CODE, Set.of("A")), + new BuiltInConditions.Presence(AMOUNT, true), + new BuiltInConditions.Prefixes(CODE, List.of("AB", "CD")), + new BuiltInConditions.Not(new BuiltInConditions.PrefixRange(CODE, "10", "20")) + )); + + assertValid(condition); + assertEquals(4, assertInstanceOf(BuiltInConditions.AllOf.class, condition).conditions().size()); + } + + @Test + void shouldDefensivelyCopyOperationCollections() { + List distinctBy = new ArrayList<>(List.of(CODE)); + BuiltInAggregations.Count count = new BuiltInAggregations.Count(AMOUNT, distinctBy); + distinctBy.clear(); + + Map flagColumns = new LinkedHashMap<>(); + flagColumns.put("A", FLAG_A); + flagColumns.put("B", FLAG_B); + BuiltInAggregations.Flags flags = new BuiltInAggregations.Flags(flagColumns); + flagColumns.clear(); + + assertEquals(List.of(CODE), count.distinctBy()); + assertEquals(List.of("A", "B"), new ArrayList<>(flags.columns().keySet())); + assertThrows(UnsupportedOperationException.class, () -> flags.columns().clear()); + } + + @Test + void shouldRejectColumnsFromDifferentTablesInOneOperation() { + ResolvedColumn otherColumn = new ResolvedColumn( + "other.code", OTHER_EVENTS, "code", ColumnType.STRING, true + ); + + assertInvalid(new BuiltInAggregations.Count(CODE, List.of(otherColumn))); + assertInvalid(new DateColumns.Pair(START, new ResolvedColumn( + "other.end", OTHER_EVENTS, "end_date", ColumnType.DATE, true + ))); + } + + @Test + void shouldRejectInvalidColumnTypes() { + assertInvalid(new BuiltInAggregations.Sum(CODE, Optional.empty(), List.of())); + assertInvalid(new BuiltInFilters.DateDistanceRange( + CODE, ChronoUnit.DAYS, LocalDate.of(2026, 8, 27), NumberRange.unbounded() + )); + assertInvalid(new BuiltInSelects.Values( + AMOUNT, BuiltInSelects.ValueOperation.FIRST, Optional.of(SubstringRange.from(1)) + )); + } + + @Test + void shouldRejectUnknownSelectedFlags() { + assertInvalid(new BuiltInFilters.Flags(Map.of("A", FLAG_A), Set.of("missing"))); + } + + @Test + void shouldRejectInvalidRanges() { + assertInvalid(NumberRange.closed(2, 1)); + assertInvalid(SubstringRange.between(3, 2)); + assertInvalid(new BuiltInConditions.PrefixRange(CODE, "100", "20")); + } + + private static ResolvedColumn column(String name, ColumnType type) { + return new ResolvedColumn("events." + name, EVENTS, name, type, true); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/query/ResolvedQueryTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/query/ResolvedQueryTest.java index 692df371c1..1f4831370f 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/query/ResolvedQueryTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/query/ResolvedQueryTest.java @@ -2,6 +2,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static com.bakdata.conquery.sql.query.ValidationTestSupport.assertInvalid; +import static com.bakdata.conquery.sql.query.ValidationTestSupport.assertValid; import java.time.LocalDate; import java.util.ArrayList; @@ -9,6 +11,21 @@ import java.util.Optional; import java.util.Set; +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.sql.query.node.AndNode; +import com.bakdata.conquery.sql.query.node.ConceptNode; +import com.bakdata.conquery.sql.query.node.DateAggregationAction; +import com.bakdata.conquery.sql.query.node.QueryNode; +import com.bakdata.conquery.sql.query.range.DateRange; +import com.bakdata.conquery.sql.query.result.ResultColumn; +import com.bakdata.conquery.sql.query.result.ResultType; +import com.bakdata.conquery.sql.query.schema.EntitySchema; +import com.bakdata.conquery.sql.query.schema.ResolvedColumn; +import com.bakdata.conquery.sql.query.schema.ResolvedConnector; +import com.bakdata.conquery.sql.query.schema.ResolvedValidityDate; +import com.bakdata.conquery.sql.query.schema.SqlTable; +import com.bakdata.conquery.sql.query.validation.ResolvedQueryValidation; +import jakarta.validation.ConstraintViolationException; import org.junit.jupiter.api.Test; class ResolvedQueryTest { @@ -44,6 +61,8 @@ void shouldCreateResolvedQueryWithoutFrameworkTypes() { )) ); + assertValid(query); + new ResolvedQueryValidation(ValidationTestSupport.VALIDATOR).validate(query); assertEquals(concept, query.root()); assertEquals("warehouse", query.target().dataSource()); } @@ -67,7 +86,7 @@ void shouldRejectColumnsFromAnotherConnectorTable() { "dataset.other.entity_id", otherTable, "entity_id", ColumnType.STRING, false ); - assertThrows(IllegalArgumentException.class, () -> new ResolvedConnector( + ResolvedConnector connector = new ResolvedConnector( "dataset.concept.connector", EVENTS, otherId, @@ -76,25 +95,50 @@ void shouldRejectColumnsFromAnotherConnectorTable() { List.of(), List.of(), List.of() - )); + ); + + assertInvalid(connector); } @Test void shouldRejectNonDateValidityColumns() { - assertThrows( - IllegalArgumentException.class, - () -> new ResolvedValidityDate.Point(ENTITY_ID) - ); + assertInvalid(new ResolvedValidityDate.Point(ENTITY_ID)); } @Test void shouldRejectInvertedDateRestriction() { - assertThrows(IllegalArgumentException.class, () -> DateRange.closed( + assertInvalid(DateRange.closed( LocalDate.of(2025, 2, 1), LocalDate.of(2025, 1, 1) )); } + @Test + void shouldValidateTheCompleteQueryGraphAtTheCompilerBoundary() { + ResolvedConnector invalidConnector = new ResolvedConnector( + "concept.connector", + EVENTS, + ENTITY_ID, + Optional.empty(), + new ResolvedValidityDate.Point(ENTITY_ID), + List.of(), + List.of(), + List.of() + ); + ResolvedQuery query = new ResolvedQuery( + new ExecutionTarget("dataset", "warehouse"), + new EntitySchema(EVENTS, ENTITY_ID), + new ConceptNode("concept", List.of(invalidConnector), List.of(), DateAggregationAction.MERGE), + true, + List.of() + ); + + assertThrows( + ConstraintViolationException.class, + () -> new ResolvedQueryValidation(ValidationTestSupport.VALIDATOR).validate(query) + ); + } + private static ConceptNode conceptNode(SqlTable table) { ResolvedColumn id = new ResolvedColumn("concept.id", table, "entity_id", ColumnType.STRING, false); ResolvedConnector connector = new ResolvedConnector( diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/query/ValidationTestSupport.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/query/ValidationTestSupport.java new file mode 100644 index 0000000000..748861d6f0 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/query/ValidationTestSupport.java @@ -0,0 +1,28 @@ +package com.bakdata.conquery.sql.query; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Set; + +import jakarta.validation.ConstraintViolation; +import jakarta.validation.Validation; +import jakarta.validation.Validator; + +final class ValidationTestSupport { + + static final Validator VALIDATOR = Validation.buildDefaultValidatorFactory().getValidator(); + + private ValidationTestSupport() { + } + + static void assertValid(T value) { + Set> violations = VALIDATOR.validate(value); + assertTrue(violations.isEmpty(), () -> "Expected no constraint violations, but got: " + violations); + } + + static void assertInvalid(T value) { + Set> violations = VALIDATOR.validate(value); + assertFalse(violations.isEmpty(), "Expected at least one constraint violation"); + } +} From 9a4db7f892455f93aab4f0ca45b218c902457844 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:42:39 +0200 Subject: [PATCH 04/81] refactor(sql): separate model and validation packages Co-authored-by: Codex GPT-5 --- sql-connector/README.md | 23 +++++++++---- .../sql/{query => model}/ExecutionTarget.java | 2 +- .../sql/{query => model}/ResolvedQuery.java | 10 +++--- .../internal/ModelNormalization.java | 2 +- .../sql/{query => model}/node/AndNode.java | 4 +-- .../{query => model}/node/ConceptNode.java | 8 ++--- .../node/DateAggregationAction.java | 2 +- .../node/DateRestrictionNode.java | 4 +-- .../{query => model}/node/NegationNode.java | 2 +- .../sql/{query => model}/node/OrNode.java | 4 +-- .../sql/{query => model}/node/QueryNode.java | 2 +- .../{query => model}/node/package-info.java | 2 +- .../operation/BuiltInAggregations.java | 10 +++--- .../operation/BuiltInConditions.java | 8 ++--- .../operation/BuiltInFilters.java | 12 +++---- .../operation/BuiltInSelects.java | 12 +++---- .../operation/ResolvedAggregation.java | 2 +- .../operation/ResolvedCondition.java | 2 +- .../operation/ResolvedFilter.java | 2 +- .../operation/ResolvedSelect.java | 2 +- .../operation/package-info.java | 2 +- .../sql/{query => model}/package-info.java | 5 +-- .../sql/{query => model}/range/DateRange.java | 2 +- .../{query => model}/range/NumberRange.java | 2 +- .../range/SubstringRange.java | 2 +- .../{query => model}/range/package-info.java | 2 +- .../{query => model}/result/ResultColumn.java | 4 +-- .../{query => model}/result/ResultType.java | 2 +- .../{query => model}/result/package-info.java | 2 +- .../{query => model}/schema/DateColumns.java | 4 +-- .../{query => model}/schema/EntitySchema.java | 2 +- .../schema/ResolvedColumn.java | 2 +- .../schema/ResolvedConnector.java | 10 +++--- .../schema/ResolvedValidityDate.java | 4 +-- .../sql/{query => model}/schema/SqlTable.java | 4 +-- .../{query => model}/schema/package-info.java | 2 +- .../bakdata/conquery/sql/package-info.java | 8 +++++ .../sql/query/validation/package-info.java | 2 -- .../validation/AllowedColumnTypes.java | 2 +- .../AllowedColumnTypesValidator.java | 4 +-- .../validation/ResolvedQueryValidation.java | 4 +-- .../conquery/sql/validation/package-info.java | 6 ++++ .../BuiltInOperationsTest.java | 30 ++++++++--------- .../{query => model}/ResolvedQueryTest.java | 32 +++++++++---------- .../ValidationTestSupport.java | 2 +- 45 files changed, 139 insertions(+), 117 deletions(-) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/ExecutionTarget.java (94%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/ResolvedQuery.java (69%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/internal/ModelNormalization.java (93%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/node/AndNode.java (81%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/node/ConceptNode.java (74%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/node/DateAggregationAction.java (76%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/node/DateRestrictionNode.java (73%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/node/NegationNode.java (81%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/node/OrNode.java (81%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/node/QueryNode.java (79%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/node/package-info.java (55%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/operation/BuiltInAggregations.java (92%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/operation/BuiltInConditions.java (88%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/operation/BuiltInFilters.java (86%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/operation/BuiltInSelects.java (84%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/operation/ResolvedAggregation.java (74%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/operation/ResolvedCondition.java (66%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/operation/ResolvedFilter.java (83%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/operation/ResolvedSelect.java (80%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/operation/package-info.java (66%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/package-info.java (67%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/range/DateRange.java (94%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/range/NumberRange.java (96%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/range/SubstringRange.java (95%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/range/package-info.java (64%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/result/ResultColumn.java (88%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/result/ResultType.java (88%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/result/package-info.java (57%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/schema/DateColumns.java (91%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/schema/EntitySchema.java (92%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/schema/ResolvedColumn.java (90%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/schema/ResolvedConnector.java (86%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/schema/ResolvedValidityDate.java (89%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/schema/SqlTable.java (84%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => model}/schema/package-info.java (68%) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/package-info.java delete mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/package-info.java rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => }/validation/AllowedColumnTypes.java (93%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => }/validation/AllowedColumnTypesValidator.java (85%) rename sql-connector/src/main/java/com/bakdata/conquery/sql/{query => }/validation/ResolvedQueryValidation.java (87%) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/validation/package-info.java rename sql-connector/src/test/java/com/bakdata/conquery/sql/{query => model}/BuiltInOperationsTest.java (87%) rename sql-connector/src/test/java/com/bakdata/conquery/sql/{query => model}/ResolvedQueryTest.java (81%) rename sql-connector/src/test/java/com/bakdata/conquery/sql/{query => model}/ValidationTestSupport.java (95%) diff --git a/sql-connector/README.md b/sql-connector/README.md index 959b9358b4..c29418cf29 100644 --- a/sql-connector/README.md +++ b/sql-connector/README.md @@ -17,16 +17,21 @@ The SQL compiler may perform dialect capability checks, but it must not access a identifiers. Connections, dialect implementations, compiler state, and execution services are intentionally not part of the resolved model. -The model is organized by responsibility: +The module uses top-level packages as architectural boundaries: -- `query.node` contains the normalized query tree; -- `query.operation` contains the open operation interfaces and standard implementations; -- `query.schema` contains resolved physical tables, columns, connectors, and entity metadata; -- `query.range` and `query.result` contain shared value objects; -- `query.validation` contains the validation boundary and reusable constraints. +- `model` contains only the immutable, fully resolved compiler input; +- `validation` contains Bean Validation constraints and the explicit validation boundary; +- SQL business logic belongs in sibling packages such as `conversion`, `dialect`, and `execution`, not below `model`. + +The resolved model is organized by responsibility: + +- `model.node` contains the normalized query tree; +- `model.operation` contains the open operation interfaces and standard implementations; +- `model.schema` contains resolved physical tables, columns, connectors, and entity metadata; +- `model.range` and `model.result` contain shared value objects. Shared dataset vocabulary such as `ColumnType` comes from the dependency-free `dataset-model` module. SQL-specific -physical metadata remains in `query.schema`. +physical metadata remains in `model.schema`. `ResolvedFilter`, `ResolvedSelect`, `ResolvedCondition`, and `ResolvedAggregation` are open extension points. Implementations must be immutable and carry resolved columns and typed values. They must not carry unresolved repository @@ -42,5 +47,9 @@ The resolved model uses Jakarta Bean Validation annotations. The application sup Hibernate Validator, and calls `ResolvedQueryValidation` once before passing a query to the SQL compiler. Validation is cascaded through the complete query graph, including extension-provided filters, selects, conditions, and aggregations. +Model records may reference declarative constraints from `validation`, but validation services must never enrich or +rewrite the model. Conversion and execution code consumes a successfully validated `ResolvedQuery` and must not perform +logical identifier resolution. + Compact record constructors only create immutable copies of collections. Domain constraints such as compatible column types, ordered ranges, and same-table requirements are declarative and are reported together as constraint violations. diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ExecutionTarget.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/ExecutionTarget.java similarity index 94% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/ExecutionTarget.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/ExecutionTarget.java index 7b38bea7b2..c4eed00aa1 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ExecutionTarget.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/ExecutionTarget.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query; +package com.bakdata.conquery.sql.model; import java.util.Optional; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedQuery.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/ResolvedQuery.java similarity index 69% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedQuery.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/ResolvedQuery.java index f820a6db00..3216636ab8 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/ResolvedQuery.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/ResolvedQuery.java @@ -1,11 +1,11 @@ -package com.bakdata.conquery.sql.query; +package com.bakdata.conquery.sql.model; import java.util.List; -import com.bakdata.conquery.sql.query.internal.ModelNormalization; -import com.bakdata.conquery.sql.query.node.QueryNode; -import com.bakdata.conquery.sql.query.result.ResultColumn; -import com.bakdata.conquery.sql.query.schema.EntitySchema; +import com.bakdata.conquery.sql.model.internal.ModelNormalization; +import com.bakdata.conquery.sql.model.node.QueryNode; +import com.bakdata.conquery.sql.model.result.ResultColumn; +import com.bakdata.conquery.sql.model.schema.EntitySchema; import jakarta.validation.Valid; import jakarta.validation.constraints.NotNull; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/internal/ModelNormalization.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/internal/ModelNormalization.java similarity index 93% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/internal/ModelNormalization.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/internal/ModelNormalization.java index f004d0ab79..1f311ab6e8 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/internal/ModelNormalization.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/internal/ModelNormalization.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query.internal; +package com.bakdata.conquery.sql.model.internal; import java.util.ArrayList; import java.util.Collections; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/AndNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/AndNode.java similarity index 81% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/AndNode.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/AndNode.java index a3b614ce53..9e0de0c8dc 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/AndNode.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/AndNode.java @@ -1,8 +1,8 @@ -package com.bakdata.conquery.sql.query.node; +package com.bakdata.conquery.sql.model.node; import java.util.List; -import com.bakdata.conquery.sql.query.internal.ModelNormalization; +import com.bakdata.conquery.sql.model.internal.ModelNormalization; import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/ConceptNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/ConceptNode.java similarity index 74% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/ConceptNode.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/ConceptNode.java index 4e23392cef..9985c95874 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/ConceptNode.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/ConceptNode.java @@ -1,10 +1,10 @@ -package com.bakdata.conquery.sql.query.node; +package com.bakdata.conquery.sql.model.node; import java.util.List; -import com.bakdata.conquery.sql.query.internal.ModelNormalization; -import com.bakdata.conquery.sql.query.operation.ResolvedSelect; -import com.bakdata.conquery.sql.query.schema.ResolvedConnector; +import com.bakdata.conquery.sql.model.internal.ModelNormalization; +import com.bakdata.conquery.sql.model.operation.ResolvedSelect; +import com.bakdata.conquery.sql.model.schema.ResolvedConnector; import jakarta.validation.Valid; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotEmpty; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/DateAggregationAction.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/DateAggregationAction.java similarity index 76% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/DateAggregationAction.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/DateAggregationAction.java index 17a56d680c..688608c2de 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/DateAggregationAction.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/DateAggregationAction.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query.node; +package com.bakdata.conquery.sql.model.node; /** Defines how validity dates produced by child nodes are propagated. */ public enum DateAggregationAction { diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/DateRestrictionNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/DateRestrictionNode.java similarity index 73% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/DateRestrictionNode.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/DateRestrictionNode.java index 8a0dbe6367..168a574652 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/DateRestrictionNode.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/DateRestrictionNode.java @@ -1,6 +1,6 @@ -package com.bakdata.conquery.sql.query.node; +package com.bakdata.conquery.sql.model.node; -import com.bakdata.conquery.sql.query.range.DateRange; +import com.bakdata.conquery.sql.model.range.DateRange; import jakarta.validation.Valid; import jakarta.validation.constraints.NotNull; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/NegationNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/NegationNode.java similarity index 81% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/NegationNode.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/NegationNode.java index 4fbe3100f8..27538a6d3f 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/NegationNode.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/NegationNode.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query.node; +package com.bakdata.conquery.sql.model.node; import jakarta.validation.Valid; import jakarta.validation.constraints.NotNull; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/OrNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/OrNode.java similarity index 81% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/OrNode.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/OrNode.java index 14cdb6c9b6..9f0c98e74e 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/OrNode.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/OrNode.java @@ -1,8 +1,8 @@ -package com.bakdata.conquery.sql.query.node; +package com.bakdata.conquery.sql.model.node; import java.util.List; -import com.bakdata.conquery.sql.query.internal.ModelNormalization; +import com.bakdata.conquery.sql.model.internal.ModelNormalization; import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/QueryNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/QueryNode.java similarity index 79% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/QueryNode.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/QueryNode.java index 23f932c988..8aa8152bb0 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/QueryNode.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/QueryNode.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query.node; +package com.bakdata.conquery.sql.model.node; /** A normalized query-tree node accepted by SQL compilation. */ public sealed interface QueryNode permits AndNode, OrNode, NegationNode, DateRestrictionNode, ConceptNode { diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/package-info.java similarity index 55% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/package-info.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/package-info.java index df44c5f5ee..b24d1e8cca 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/node/package-info.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/package-info.java @@ -1,2 +1,2 @@ /** Normalized nodes of a fully resolved query tree. */ -package com.bakdata.conquery.sql.query.node; +package com.bakdata.conquery.sql.model.node; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInAggregations.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInAggregations.java similarity index 92% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInAggregations.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInAggregations.java index e4b0fb50cb..849378f558 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInAggregations.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInAggregations.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query.operation; +package com.bakdata.conquery.sql.model.operation; import java.util.ArrayList; import java.util.Collection; @@ -7,10 +7,10 @@ import java.util.Optional; import com.bakdata.conquery.models.datasets.ColumnType; -import com.bakdata.conquery.sql.query.internal.ModelNormalization; -import com.bakdata.conquery.sql.query.schema.DateColumns; -import com.bakdata.conquery.sql.query.schema.ResolvedColumn; -import com.bakdata.conquery.sql.query.validation.AllowedColumnTypes; +import com.bakdata.conquery.sql.model.internal.ModelNormalization; +import com.bakdata.conquery.sql.model.schema.DateColumns; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.validation.AllowedColumnTypes; import jakarta.validation.Valid; import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.NotBlank; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInConditions.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInConditions.java similarity index 88% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInConditions.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInConditions.java index 10e7215273..af4e34591b 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInConditions.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInConditions.java @@ -1,12 +1,12 @@ -package com.bakdata.conquery.sql.query.operation; +package com.bakdata.conquery.sql.model.operation; import java.util.List; import java.util.Set; import com.bakdata.conquery.models.datasets.ColumnType; -import com.bakdata.conquery.sql.query.internal.ModelNormalization; -import com.bakdata.conquery.sql.query.schema.ResolvedColumn; -import com.bakdata.conquery.sql.query.validation.AllowedColumnTypes; +import com.bakdata.conquery.sql.model.internal.ModelNormalization; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.validation.AllowedColumnTypes; import jakarta.validation.Valid; import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.NotBlank; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInFilters.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInFilters.java similarity index 86% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInFilters.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInFilters.java index c44cbcb4d4..6c4a3c6cea 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInFilters.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInFilters.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query.operation; +package com.bakdata.conquery.sql.model.operation; import java.time.LocalDate; import java.time.temporal.ChronoUnit; @@ -6,11 +6,11 @@ import java.util.Set; import com.bakdata.conquery.models.datasets.ColumnType; -import com.bakdata.conquery.sql.query.internal.ModelNormalization; -import com.bakdata.conquery.sql.query.range.NumberRange; -import com.bakdata.conquery.sql.query.range.SubstringRange; -import com.bakdata.conquery.sql.query.schema.ResolvedColumn; -import com.bakdata.conquery.sql.query.validation.AllowedColumnTypes; +import com.bakdata.conquery.sql.model.internal.ModelNormalization; +import com.bakdata.conquery.sql.model.range.NumberRange; +import com.bakdata.conquery.sql.model.range.SubstringRange; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.validation.AllowedColumnTypes; import jakarta.validation.Valid; import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.NotEmpty; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInSelects.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInSelects.java similarity index 84% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInSelects.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInSelects.java index b06d8f4b00..6bf44ff043 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/BuiltInSelects.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInSelects.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query.operation; +package com.bakdata.conquery.sql.model.operation; import java.time.LocalDate; import java.time.temporal.ChronoUnit; @@ -6,11 +6,11 @@ import java.util.Optional; import com.bakdata.conquery.models.datasets.ColumnType; -import com.bakdata.conquery.sql.query.internal.ModelNormalization; -import com.bakdata.conquery.sql.query.range.SubstringRange; -import com.bakdata.conquery.sql.query.schema.DateColumns; -import com.bakdata.conquery.sql.query.schema.ResolvedColumn; -import com.bakdata.conquery.sql.query.validation.AllowedColumnTypes; +import com.bakdata.conquery.sql.model.internal.ModelNormalization; +import com.bakdata.conquery.sql.model.range.SubstringRange; +import com.bakdata.conquery.sql.model.schema.DateColumns; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.validation.AllowedColumnTypes; import jakarta.validation.Valid; import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.NotEmpty; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedAggregation.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedAggregation.java similarity index 74% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedAggregation.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedAggregation.java index b33eff1500..ca477dcd11 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedAggregation.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedAggregation.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query.operation; +package com.bakdata.conquery.sql.model.operation; /** A resolved aggregation that can be exposed as a result or constrained by an aggregation filter. */ public interface ResolvedAggregation { diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedCondition.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedCondition.java similarity index 66% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedCondition.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedCondition.java index 7ce1db4d1b..8662889588 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedCondition.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedCondition.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query.operation; +package com.bakdata.conquery.sql.model.operation; /** A resolved connector or concept-element condition. */ public interface ResolvedCondition { diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedFilter.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedFilter.java similarity index 83% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedFilter.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedFilter.java index 5a2eef35fa..8b42def1ff 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedFilter.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedFilter.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query.operation; +package com.bakdata.conquery.sql.model.operation; /** * Marker for an immutable, validated filter operation. diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedSelect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedSelect.java similarity index 80% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedSelect.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedSelect.java index c9210791dd..20ff85d949 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/ResolvedSelect.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedSelect.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query.operation; +package com.bakdata.conquery.sql.model.operation; /** * Marker for an immutable, validated select operation. diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/package-info.java similarity index 66% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/package-info.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/package-info.java index 381f90f0fd..a2b0015f72 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/operation/package-info.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/package-info.java @@ -1,2 +1,2 @@ /** Extensible filter, select, condition, and aggregation operations accepted by SQL compilation. */ -package com.bakdata.conquery.sql.query.operation; +package com.bakdata.conquery.sql.model.operation; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/package-info.java similarity index 67% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/package-info.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/package-info.java index bd3cf6dbf2..9944bce082 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/package-info.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/package-info.java @@ -6,6 +6,7 @@ * compiler state.

* *

The root query aggregate is complemented by dedicated packages for query nodes, operations, ranges, result - * metadata, physical schema references, and validation.

+ * metadata, and physical schema references. Validation and SQL business logic live in sibling packages so this package + * remains the connector's immutable input boundary.

*/ -package com.bakdata.conquery.sql.query; +package com.bakdata.conquery.sql.model; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/DateRange.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/range/DateRange.java similarity index 94% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/DateRange.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/range/DateRange.java index 5251e45b95..5df5120293 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/DateRange.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/range/DateRange.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query.range; +package com.bakdata.conquery.sql.model.range; import java.time.LocalDate; import java.util.Optional; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/NumberRange.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/range/NumberRange.java similarity index 96% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/NumberRange.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/range/NumberRange.java index 82c91aef72..040be4dd72 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/NumberRange.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/range/NumberRange.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query.range; +package com.bakdata.conquery.sql.model.range; import java.math.BigDecimal; import java.util.Objects; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/SubstringRange.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/range/SubstringRange.java similarity index 95% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/SubstringRange.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/range/SubstringRange.java index ce45606115..f087350e41 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/SubstringRange.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/range/SubstringRange.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query.range; +package com.bakdata.conquery.sql.model.range; import java.util.Optional; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/range/package-info.java similarity index 64% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/package-info.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/range/package-info.java index a9ec695bf5..778528444f 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/range/package-info.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/range/package-info.java @@ -1,2 +1,2 @@ /** Typed, framework-neutral range values used by resolved nodes and operations. */ -package com.bakdata.conquery.sql.query.range; +package com.bakdata.conquery.sql.model.range; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/result/ResultColumn.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/result/ResultColumn.java similarity index 88% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/result/ResultColumn.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/result/ResultColumn.java index 9d7a3619d5..d67dff0702 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/result/ResultColumn.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/result/ResultColumn.java @@ -1,9 +1,9 @@ -package com.bakdata.conquery.sql.query.result; +package com.bakdata.conquery.sql.model.result; import java.util.Optional; import java.util.Set; -import com.bakdata.conquery.sql.query.internal.ModelNormalization; +import com.bakdata.conquery.sql.model.internal.ModelNormalization; import jakarta.validation.Valid; import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.NotBlank; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/result/ResultType.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/result/ResultType.java similarity index 88% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/result/ResultType.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/result/ResultType.java index a1bb6826af..56f10b365d 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/result/ResultType.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/result/ResultType.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query.result; +package com.bakdata.conquery.sql.model.result; import jakarta.validation.constraints.NotNull; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/result/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/result/package-info.java similarity index 57% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/result/package-info.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/result/package-info.java index 54870eda8a..f20ad2f630 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/result/package-info.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/result/package-info.java @@ -1,2 +1,2 @@ /** Ordered result-column metadata and logical result types. */ -package com.bakdata.conquery.sql.query.result; +package com.bakdata.conquery.sql.model.result; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/DateColumns.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/DateColumns.java similarity index 91% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/DateColumns.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/DateColumns.java index ea566c006c..1c6fa99a0d 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/DateColumns.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/DateColumns.java @@ -1,10 +1,10 @@ -package com.bakdata.conquery.sql.query.schema; +package com.bakdata.conquery.sql.model.schema; import java.util.List; import java.util.Objects; import com.bakdata.conquery.models.datasets.ColumnType; -import com.bakdata.conquery.sql.query.validation.AllowedColumnTypes; +import com.bakdata.conquery.sql.validation.AllowedColumnTypes; import jakarta.validation.Valid; import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.NotNull; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/EntitySchema.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/EntitySchema.java similarity index 92% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/EntitySchema.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/EntitySchema.java index 212e1ab712..4b0d51e4b6 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/EntitySchema.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/EntitySchema.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query.schema; +package com.bakdata.conquery.sql.model.schema; import jakarta.validation.Valid; import jakarta.validation.constraints.AssertTrue; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/ResolvedColumn.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/ResolvedColumn.java similarity index 90% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/ResolvedColumn.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/ResolvedColumn.java index ff285f38b9..560a9a0d70 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/ResolvedColumn.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/ResolvedColumn.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query.schema; +package com.bakdata.conquery.sql.model.schema; import com.bakdata.conquery.models.datasets.ColumnType; import jakarta.validation.Valid; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/ResolvedConnector.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/ResolvedConnector.java similarity index 86% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/ResolvedConnector.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/ResolvedConnector.java index d23a9ec1af..4201426902 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/ResolvedConnector.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/ResolvedConnector.java @@ -1,12 +1,12 @@ -package com.bakdata.conquery.sql.query.schema; +package com.bakdata.conquery.sql.model.schema; import java.util.List; import java.util.Optional; -import com.bakdata.conquery.sql.query.internal.ModelNormalization; -import com.bakdata.conquery.sql.query.operation.ResolvedCondition; -import com.bakdata.conquery.sql.query.operation.ResolvedFilter; -import com.bakdata.conquery.sql.query.operation.ResolvedSelect; +import com.bakdata.conquery.sql.model.internal.ModelNormalization; +import com.bakdata.conquery.sql.model.operation.ResolvedCondition; +import com.bakdata.conquery.sql.model.operation.ResolvedFilter; +import com.bakdata.conquery.sql.model.operation.ResolvedSelect; import jakarta.validation.Valid; import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.NotBlank; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/ResolvedValidityDate.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/ResolvedValidityDate.java similarity index 89% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/ResolvedValidityDate.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/ResolvedValidityDate.java index 73d71f6179..2fc8a30041 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/ResolvedValidityDate.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/ResolvedValidityDate.java @@ -1,9 +1,9 @@ -package com.bakdata.conquery.sql.query.schema; +package com.bakdata.conquery.sql.model.schema; import java.util.Objects; import com.bakdata.conquery.models.datasets.ColumnType; -import com.bakdata.conquery.sql.query.validation.AllowedColumnTypes; +import com.bakdata.conquery.sql.validation.AllowedColumnTypes; import jakarta.validation.Valid; import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.NotNull; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/SqlTable.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/SqlTable.java similarity index 84% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/SqlTable.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/SqlTable.java index e927aaaa57..16e59d6b4d 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/SqlTable.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/SqlTable.java @@ -1,9 +1,9 @@ -package com.bakdata.conquery.sql.query.schema; +package com.bakdata.conquery.sql.model.schema; import java.util.Arrays; import java.util.List; -import com.bakdata.conquery.sql.query.internal.ModelNormalization; +import com.bakdata.conquery.sql.model.internal.ModelNormalization; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotEmpty; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/package-info.java similarity index 68% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/package-info.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/package-info.java index 1b56e9adc2..79d98246e4 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/schema/package-info.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/package-info.java @@ -1,2 +1,2 @@ /** Resolved physical tables, columns, connectors, and entity schema used during SQL compilation. */ -package com.bakdata.conquery.sql.query.schema; +package com.bakdata.conquery.sql.model.schema; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/package-info.java new file mode 100644 index 0000000000..92d232da46 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/package-info.java @@ -0,0 +1,8 @@ +/** + * Framework-neutral SQL connector contracts and services. + * + *

{@code model} defines the fully resolved input, {@code validation} verifies that input, and sibling packages such + * as {@code conversion}, {@code dialect}, and {@code execution} contain SQL business logic. Business logic may depend on + * the model and validation boundary; model code must not depend on compiler or execution services.

+ */ +package com.bakdata.conquery.sql; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/package-info.java deleted file mode 100644 index 06434d8420..0000000000 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/package-info.java +++ /dev/null @@ -1,2 +0,0 @@ -/** Jakarta Bean Validation constraints and the validation boundary for resolved queries. */ -package com.bakdata.conquery.sql.query.validation; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/AllowedColumnTypes.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/validation/AllowedColumnTypes.java similarity index 93% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/AllowedColumnTypes.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/validation/AllowedColumnTypes.java index 9f7b5beeba..f5a9c004d0 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/AllowedColumnTypes.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/validation/AllowedColumnTypes.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query.validation; +package com.bakdata.conquery.sql.validation; import java.lang.annotation.Documented; import java.lang.annotation.ElementType; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/AllowedColumnTypesValidator.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/validation/AllowedColumnTypesValidator.java similarity index 85% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/AllowedColumnTypesValidator.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/validation/AllowedColumnTypesValidator.java index d79d3703be..6b1c9524f1 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/AllowedColumnTypesValidator.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/validation/AllowedColumnTypesValidator.java @@ -1,9 +1,9 @@ -package com.bakdata.conquery.sql.query.validation; +package com.bakdata.conquery.sql.validation; import java.util.EnumSet; import com.bakdata.conquery.models.datasets.ColumnType; -import com.bakdata.conquery.sql.query.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; import jakarta.validation.ConstraintValidator; import jakarta.validation.ConstraintValidatorContext; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/ResolvedQueryValidation.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/validation/ResolvedQueryValidation.java similarity index 87% rename from sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/ResolvedQueryValidation.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/validation/ResolvedQueryValidation.java index 5991b5e759..2032de20cf 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/query/validation/ResolvedQueryValidation.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/validation/ResolvedQueryValidation.java @@ -1,9 +1,9 @@ -package com.bakdata.conquery.sql.query.validation; +package com.bakdata.conquery.sql.validation; import java.util.Objects; import java.util.Set; -import com.bakdata.conquery.sql.query.ResolvedQuery; +import com.bakdata.conquery.sql.model.ResolvedQuery; import jakarta.validation.ConstraintViolation; import jakarta.validation.ConstraintViolationException; import jakarta.validation.Validator; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/validation/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/validation/package-info.java new file mode 100644 index 0000000000..c33adf3ff6 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/validation/package-info.java @@ -0,0 +1,6 @@ +/** + * Jakarta Bean Validation constraints and the validation boundary for resolved SQL models. + * + *

Validation reports violations but never resolves identifiers, supplies defaults, or mutates model instances.

+ */ +package com.bakdata.conquery.sql.validation; diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/query/BuiltInOperationsTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/BuiltInOperationsTest.java similarity index 87% rename from sql-connector/src/test/java/com/bakdata/conquery/sql/query/BuiltInOperationsTest.java rename to sql-connector/src/test/java/com/bakdata/conquery/sql/model/BuiltInOperationsTest.java index 4dcc56e333..5826a63192 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/query/BuiltInOperationsTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/BuiltInOperationsTest.java @@ -1,10 +1,10 @@ -package com.bakdata.conquery.sql.query; +package com.bakdata.conquery.sql.model; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; -import static com.bakdata.conquery.sql.query.ValidationTestSupport.assertInvalid; -import static com.bakdata.conquery.sql.query.ValidationTestSupport.assertValid; +import static com.bakdata.conquery.sql.model.ValidationTestSupport.assertInvalid; +import static com.bakdata.conquery.sql.model.ValidationTestSupport.assertValid; import java.math.BigDecimal; import java.time.LocalDate; @@ -17,18 +17,18 @@ import java.util.Set; import com.bakdata.conquery.models.datasets.ColumnType; -import com.bakdata.conquery.sql.query.operation.BuiltInAggregations; -import com.bakdata.conquery.sql.query.operation.BuiltInConditions; -import com.bakdata.conquery.sql.query.operation.BuiltInFilters; -import com.bakdata.conquery.sql.query.operation.BuiltInSelects; -import com.bakdata.conquery.sql.query.operation.ResolvedCondition; -import com.bakdata.conquery.sql.query.operation.ResolvedFilter; -import com.bakdata.conquery.sql.query.operation.ResolvedSelect; -import com.bakdata.conquery.sql.query.range.NumberRange; -import com.bakdata.conquery.sql.query.range.SubstringRange; -import com.bakdata.conquery.sql.query.schema.DateColumns; -import com.bakdata.conquery.sql.query.schema.ResolvedColumn; -import com.bakdata.conquery.sql.query.schema.SqlTable; +import com.bakdata.conquery.sql.model.operation.BuiltInAggregations; +import com.bakdata.conquery.sql.model.operation.BuiltInConditions; +import com.bakdata.conquery.sql.model.operation.BuiltInFilters; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; +import com.bakdata.conquery.sql.model.operation.ResolvedCondition; +import com.bakdata.conquery.sql.model.operation.ResolvedFilter; +import com.bakdata.conquery.sql.model.operation.ResolvedSelect; +import com.bakdata.conquery.sql.model.range.NumberRange; +import com.bakdata.conquery.sql.model.range.SubstringRange; +import com.bakdata.conquery.sql.model.schema.DateColumns; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.SqlTable; import org.junit.jupiter.api.Test; class BuiltInOperationsTest { diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/query/ResolvedQueryTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java similarity index 81% rename from sql-connector/src/test/java/com/bakdata/conquery/sql/query/ResolvedQueryTest.java rename to sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java index 1f4831370f..4bcb23d39e 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/query/ResolvedQueryTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java @@ -1,9 +1,9 @@ -package com.bakdata.conquery.sql.query; +package com.bakdata.conquery.sql.model; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static com.bakdata.conquery.sql.query.ValidationTestSupport.assertInvalid; -import static com.bakdata.conquery.sql.query.ValidationTestSupport.assertValid; +import static com.bakdata.conquery.sql.model.ValidationTestSupport.assertInvalid; +import static com.bakdata.conquery.sql.model.ValidationTestSupport.assertValid; import java.time.LocalDate; import java.util.ArrayList; @@ -12,19 +12,19 @@ import java.util.Set; import com.bakdata.conquery.models.datasets.ColumnType; -import com.bakdata.conquery.sql.query.node.AndNode; -import com.bakdata.conquery.sql.query.node.ConceptNode; -import com.bakdata.conquery.sql.query.node.DateAggregationAction; -import com.bakdata.conquery.sql.query.node.QueryNode; -import com.bakdata.conquery.sql.query.range.DateRange; -import com.bakdata.conquery.sql.query.result.ResultColumn; -import com.bakdata.conquery.sql.query.result.ResultType; -import com.bakdata.conquery.sql.query.schema.EntitySchema; -import com.bakdata.conquery.sql.query.schema.ResolvedColumn; -import com.bakdata.conquery.sql.query.schema.ResolvedConnector; -import com.bakdata.conquery.sql.query.schema.ResolvedValidityDate; -import com.bakdata.conquery.sql.query.schema.SqlTable; -import com.bakdata.conquery.sql.query.validation.ResolvedQueryValidation; +import com.bakdata.conquery.sql.model.node.AndNode; +import com.bakdata.conquery.sql.model.node.ConceptNode; +import com.bakdata.conquery.sql.model.node.DateAggregationAction; +import com.bakdata.conquery.sql.model.node.QueryNode; +import com.bakdata.conquery.sql.model.range.DateRange; +import com.bakdata.conquery.sql.model.result.ResultColumn; +import com.bakdata.conquery.sql.model.result.ResultType; +import com.bakdata.conquery.sql.model.schema.EntitySchema; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.ResolvedConnector; +import com.bakdata.conquery.sql.model.schema.ResolvedValidityDate; +import com.bakdata.conquery.sql.model.schema.SqlTable; +import com.bakdata.conquery.sql.validation.ResolvedQueryValidation; import jakarta.validation.ConstraintViolationException; import org.junit.jupiter.api.Test; diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/query/ValidationTestSupport.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ValidationTestSupport.java similarity index 95% rename from sql-connector/src/test/java/com/bakdata/conquery/sql/query/ValidationTestSupport.java rename to sql-connector/src/test/java/com/bakdata/conquery/sql/model/ValidationTestSupport.java index 748861d6f0..9b57aeae8a 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/query/ValidationTestSupport.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ValidationTestSupport.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.query; +package com.bakdata.conquery.sql.model; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; From 05f5488d7037c611846879cb2e18976a77af8e7d Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Fri, 28 Aug 2026 14:39:48 +0200 Subject: [PATCH 05/81] refactor(sql): separate compiler runtime dependencies Co-authored-by: Codex GPT-5 --- backend/pom.xml | 5 + .../mode/local/LocalNamespaceHandler.java | 2 +- .../conquery/sql/conversion/Context.java | 6 +- .../sql/conversion/NodeConversions.java | 19 ++- .../cqelement/CQExternalConverter.java | 2 +- .../cqelement/ConversionContext.java | 6 +- .../aggregation/DateAggregationContext.java | 6 +- .../cqelement/concept/CQConceptConverter.java | 6 +- .../cqelement/concept/CQTableContext.java | 6 +- .../cqelement/concept/FilterContext.java | 8 +- .../concept/IntervalPackingSelectsCte.java | 8 +- .../cqelement/concept/JoinBranchesCte.java | 2 +- .../cqelement/concept/TablePath.java | 6 +- .../IntervalPackingContext.java | 6 +- .../IntervalPackingCteStep.java | 8 +- .../sql/conversion/dialect/DialectBundle.java | 112 ++---------------- .../dialect/LegacyCompilerDialect.java | 86 ++++++++++++++ .../sql/conversion/model/QueryStepJoiner.java | 8 +- .../CountQuartersSqlAggregator.java | 4 +- .../model/aggregator/FlagSqlAggregator.java | 2 +- .../model/select/DaterangeSelectUtil.java | 18 +-- .../model/select/SelectContext.java | 8 +- .../query/AbsoluteFormQueryConverter.java | 2 +- .../query/ConceptQueryConverter.java | 2 +- sql-connector/README.md | 3 +- .../sql/compiler/dialect/CompilerDialect.java | 22 ++++ .../sql/compiler/dialect/package-info.java | 2 + .../conquery/sql/compiler/package-info.java | 2 + .../bakdata/conquery/sql/package-info.java | 6 +- 29 files changed, 198 insertions(+), 175 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/package-info.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/package-info.java diff --git a/backend/pom.xml b/backend/pom.xml index 27a433b345..6bfa236947 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -112,6 +112,11 @@ + + com.bakdata.conquery + sql-connector + ${project.version} + org.apache.logging.log4j log4j-to-slf4j diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java index 8a4a292b61..e458522d30 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java @@ -49,7 +49,7 @@ public LocalNamespace createNamespace( ResultSetProcessor resultSetProcessor = dialectBundle.getResultSetProcessor(config); SqlExecutionService sqlExecutionService = new SqlExecutionService(dslContext, resultSetProcessor); - NodeConversions nodeConversions = new NodeConversions(config.getIdColumns(), dialectBundle, dslContext, sqlExecutionService, clock, connection.getConnection().getPrimaryColumn()); + NodeConversions nodeConversions = new NodeConversions(config.getIdColumns(), dialectBundle, dslContext, clock, connection.getConnection().getPrimaryColumn()); SqlConverter sqlConverter = new SqlConverter(nodeConversions, config); ExecutionManager executionManager = new SqlExecutionManager(sqlConverter, sqlExecutionService, metaStorage, datasetRegistry, config); SqlStorageHandler sqlStorageHandler = new SqlStorageHandler(sqlExecutionService); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Context.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/Context.java index 4bc8338bd1..9d679eb311 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Context.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/Context.java @@ -1,7 +1,7 @@ package com.bakdata.conquery.sql.conversion; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.NameGenerator; @@ -12,14 +12,14 @@ public interface Context { ConversionContext getConversionContext(); - DialectBundle getDialectBundle(); + LegacyCompilerDialect getCompilerDialect(); default NameGenerator getNameGenerator() { return getConversionContext().getNameGenerator(); } default SqlFunctionProvider getFunctionProvider(){ - return getConversionContext().getDialectBundle().getFunctionProvider(); + return getConversionContext().getCompilerDialect().getFunctionProvider(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java index c61a301d8f..6ec5be439f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java @@ -5,9 +5,8 @@ import com.bakdata.conquery.models.config.IdColumnConfig; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.model.NameGenerator; -import com.bakdata.conquery.sql.execution.SqlExecutionService; import lombok.NonNull; import org.jooq.DSLContext; @@ -21,26 +20,23 @@ public class NodeConversions implements NodeConversionDispatcher { private final Conversions conversions; private final IdColumnConfig idColumns; - private final DialectBundle dialect; + private final LegacyCompilerDialect dialect; private final NameGenerator nameGenerator; - private final SqlExecutionService executionService; private final Clock clock; @NonNull private final String defaultPrimaryColumn; public NodeConversions( IdColumnConfig idColumns, - DialectBundle dialectBundle, + LegacyCompilerDialect compilerDialect, DSLContext dslContext, - SqlExecutionService executionService, Clock clock, String defaultPrimaryColumn ) { - this.conversions = new Conversions<>(dialectBundle.getNodeConverters(dslContext)); + this.conversions = new Conversions<>(compilerDialect.getNodeConverters(dslContext)); this.idColumns = idColumns; - this.dialect = dialectBundle; - this.nameGenerator = new NameGenerator(dialectBundle.getNameMaxLength()); - this.executionService = executionService; + this.dialect = compilerDialect; + this.nameGenerator = new NameGenerator(compilerDialect.getNameMaxLength()); this.clock = clock; this.defaultPrimaryColumn = defaultPrimaryColumn; } @@ -59,8 +55,7 @@ public ConversionContext convert(QueryDescription queryDescription, ConqueryConf .clock(clock) .defaultPrimaryColumn(this.defaultPrimaryColumn) .stratificationFunctions(dialect.getStratificationFunctions()) - .dialectBundle(dialect) - .executionService(executionService) + .compilerDialect(dialect) .build(); return convert(queryDescription, initialCtx); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java index dd9e63cc36..0b70eb2435 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java @@ -152,7 +152,7 @@ public Class getConversionClass() { @Override public ConversionContext convert(CQExternal external, ConversionContext context) { - SqlFunctionProvider functionProvider = context.getDialectBundle().getFunctionProvider(); + SqlFunctionProvider functionProvider = context.getCompilerDialect().getFunctionProvider(); QueryStep externalIdsCte = createExternalIdsCte(external, functionProvider, context); ConversionContext withExternalIdCte = context.withQueryStep(externalIdsCte); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java index 235ba767f9..49612303ef 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java @@ -12,12 +12,11 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.NodeConversionDispatcher; -import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlQuery; -import com.bakdata.conquery.sql.execution.SqlExecutionService; import lombok.Builder; import lombok.NonNull; import lombok.Singular; @@ -38,9 +37,8 @@ public class ConversionContext implements Context { String defaultPrimaryColumn; PrintSettings sqlPrintSettings; NodeConversionDispatcher nodeConversions; - DialectBundle dialectBundle; + LegacyCompilerDialect compilerDialect; NameGenerator nameGenerator; - SqlExecutionService executionService; @Singular List querySteps; @Nullable diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java index 7eae7cb528..f00fe6d120 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.model.Qualifiable; import com.bakdata.conquery.sql.conversion.model.QualifyingUtil; import com.bakdata.conquery.sql.conversion.model.QueryStep; @@ -63,8 +63,8 @@ public DateAggregationContext qualify(String qualifier) { } @Override - public DialectBundle getDialectBundle() { - return getConversionContext().getDialectBundle(); + public LegacyCompilerDialect getCompilerDialect() { + return getConversionContext().getCompilerDialect(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 1c879805a3..e43c830d81 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -65,7 +65,7 @@ private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcep List converted = cqConcept.getSelects().stream() .map(selectId -> { Select select = selectId.resolve(); - return context.getDialectBundle().getSelectConverter(select).conceptSelect(select, selectContext); + return context.getCompilerDialect().getSelectConverter(select).conceptSelect(select, selectContext); }) .toList(); @@ -220,7 +220,7 @@ private static ConnectorSqlSelects createConceptColumnConnectorSqlSelects(CQConc .map(SelectId::resolve) .filter(select -> select instanceof ConceptColumnSelect) .findFirst() - .map(select -> selectContext.getDialectBundle().getSelectConverter(select).connectorSelect(select, selectContext)) + .map(select -> selectContext.getCompilerDialect().getSelectConverter(select).connectorSelect(select, selectContext)) .orElse(ConnectorSqlSelects.none()); } @@ -290,7 +290,7 @@ private CQTableContext createTableContext(TablePath tablePath, CQConcept cqConce cqTable.getSelects() .stream() .map(SelectId::resolve) - .map(select -> selectContext.getDialectBundle().getSelectConverter(select).connectorSelect(select, selectContext)) + .map(select -> selectContext.getCompilerDialect().getSelectConverter(select).connectorSelect(select, selectContext)) .forEach(allSelectsForTable::add); return CQTableContext.builder() diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java index cf2c51a974..71199d2e0c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java @@ -8,7 +8,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; @@ -51,7 +51,7 @@ public List allSqlSelects() { } @Override - public DialectBundle getDialectBundle() { - return getConversionContext().getDialectBundle(); + public LegacyCompilerDialect getCompilerDialect() { + return getConversionContext().getCompilerDialect(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java index 6aa498302a..ac0a1eeb5d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java @@ -5,7 +5,7 @@ import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import lombok.AccessLevel; @@ -40,12 +40,12 @@ public static FilterContext forTableExport(SqlIdColumns ids, V value, Con } @Override - public DialectBundle getDialectBundle() { - return getConversionContext().getDialectBundle(); + public LegacyCompilerDialect getCompilerDialect() { + return getConversionContext().getCompilerDialect(); } public SqlFunctionProvider getFunctionProvider() { - return getDialectBundle().getFunctionProvider(); + return getCompilerDialect().getFunctionProvider(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java index 45e03332c0..77572d97b2 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDateUnionSelect; import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDurationSumSelect; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; @@ -23,15 +23,15 @@ public static QueryStep forSelect( ColumnDateRange daterange, SqlSelect select, SqlTables tables, - DialectBundle dialectBundle + LegacyCompilerDialect compilerDialect ) { List predecessors = List.of(withAggregatedDaterange); QueryStep directPredecessor = withAggregatedDaterange; // we need an additional predecessor to unnest the validity date if it is a single column range - if (dialectBundle.supportsSingleColumnRanges()) { + if (compilerDialect.supportsSingleColumnRanges()) { String unnestCteName = tables.cteName(ConceptCteStep.UNNEST_DATE); - directPredecessor = dialectBundle.getFunctionProvider().unnestDaterange(daterange, withAggregatedDaterange, unnestCteName); + directPredecessor = compilerDialect.getFunctionProvider().unnestDaterange(daterange, withAggregatedDaterange, unnestCteName); predecessors = List.of(withAggregatedDaterange, directPredecessor); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java index b2f6718dfd..8a78d1dbd1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java @@ -63,7 +63,7 @@ protected QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { } else { IntervalPackingContext intervalPackingContext = createIntervalPackingContext(tableContext); - IntervalPacker intervalPacker = tableContext.getConversionContext().getDialectBundle().getIntervalPacker(); + IntervalPacker intervalPacker = tableContext.getConversionContext().getCompilerDialect().getIntervalPacker(); QueryStep lastIntervalPackingStep = intervalPacker.aggregateAsValidityDate(intervalPackingContext); queriesToJoin.add(lastIntervalPackingStep); validityDate = lastIntervalPackingStep.getQualifiedSelects().getValidityDate(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java index 0ccbe64151..a021c2db28 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java @@ -91,7 +91,7 @@ private static TablePathInfo collectConnectorTables(CQConcept cqConcept, CQTable // interval packing requiredw tableInfo.setContainsIntervalPacking(true); - tableInfo.addMappings(IntervalPackingCteStep.getMappings(PREPROCESSING, context.getDialectBundle())); + tableInfo.addMappings(IntervalPackingCteStep.getMappings(PREPROCESSING, context.getCompilerDialect())); // validity date propagation not necessary if (!cqConcept.isAggregateEventDates()) { @@ -103,7 +103,7 @@ private static TablePathInfo collectConnectorTables(CQConcept cqConcept, CQTable } // interval packing selects required with optional unnest step - if (context.getDialectBundle().supportsSingleColumnRanges()) { + if (context.getCompilerDialect().supportsSingleColumnRanges()) { tableInfo.addMappings(Map.of( UNNEST_DATE, INTERVAL_COMPLETE, INTERVAL_PACKING_SELECTS, UNNEST_DATE @@ -135,7 +135,7 @@ private TablePathInfo collectConceptTables(QueryStep predecessor) { ); // universal event date selects required with optional additional unnest step - if (context.getDialectBundle().supportsSingleColumnRanges()) { + if (context.getCompilerDialect().supportsSingleColumnRanges()) { tableInfo.addRootTableMapping(UNNEST_DATE); tableInfo.addMappings(Map.of(INTERVAL_PACKING_SELECTS, UNNEST_DATE)); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java index d644a635e9..ee0f7e45ab 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; @@ -45,8 +45,8 @@ public class IntervalPackingContext implements Context { ConversionContext conversionContext; @Override - public DialectBundle getDialectBundle() { - return getConversionContext().getDialectBundle(); + public LegacyCompilerDialect getCompilerDialect() { + return getConversionContext().getCompilerDialect(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java index ef5d068653..640e6b789d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java @@ -5,7 +5,7 @@ import java.util.Set; import com.bakdata.conquery.sql.conversion.Context; -import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.model.CteStep; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlTables; @@ -31,7 +31,7 @@ public enum IntervalPackingCteStep implements CteStep { public static SqlTables createTables(QueryStep predecessor, Context context) { String rootTable = predecessor.getCteName(); - Set requiredSteps = context.getDialectBundle().supportsSingleColumnRanges() + Set requiredSteps = context.getCompilerDialect().supportsSingleColumnRanges() ? Set.of(INTERVAL_COMPLETE) : Set.of(values()); @@ -45,7 +45,7 @@ public static SqlTables createTables(QueryStep predecessor, Context context) { * Create predecessor mappings for these interval packing {@link CteStep}s based on a preceding root step that must contain a validity date which * shall be interval-packed. */ - public static Map getMappings(CteStep root, DialectBundle dialect) { + public static Map getMappings(CteStep root, LegacyCompilerDialect dialect) { if (dialect.supportsSingleColumnRanges()) { return Map.of(INTERVAL_COMPLETE, root); } @@ -57,7 +57,7 @@ public static Map getMappings(CteStep root, DialectBundle dial /** * Create predecessor mappings for these interval packing {@link CteStep}s based on the default mapping. */ - public static Map getMappings(DialectBundle dialect) { + public static Map getMappings(LegacyCompilerDialect dialect) { if (dialect.supportsSingleColumnRanges()) { Map mappings = new HashMap<>(); mappings.put(INTERVAL_COMPLETE, null); // final step directly mapped onto root table diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java index d0596962c3..1e38a3ab88 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java @@ -1,120 +1,32 @@ package com.bakdata.conquery.sql.conversion.dialect; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; - import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.Dialect; -import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.events.MajorTypeId; -import com.bakdata.conquery.models.query.Visitable; -import com.bakdata.conquery.sql.conversion.Converter; -import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.cqelement.CQAndConverter; -import com.bakdata.conquery.sql.conversion.cqelement.CQDateRestrictionConverter; -import com.bakdata.conquery.sql.conversion.cqelement.CQExternalConverter; -import com.bakdata.conquery.sql.conversion.cqelement.CQNegationConverter; -import com.bakdata.conquery.sql.conversion.cqelement.CQOrConverter; -import com.bakdata.conquery.sql.conversion.cqelement.CQYesConverter; -import com.bakdata.conquery.sql.conversion.cqelement.concept.CQConceptConverter; -import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; -import com.bakdata.conquery.sql.conversion.model.QueryStepTransformer; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; -import com.bakdata.conquery.sql.conversion.query.AbsoluteFormQueryConverter; -import com.bakdata.conquery.sql.conversion.query.CQReusedQueryConverter; -import com.bakdata.conquery.sql.conversion.query.ConceptQueryConverter; -import com.bakdata.conquery.sql.conversion.query.EntityDateQueryConverter; -import com.bakdata.conquery.sql.conversion.query.FormConversionHelper; -import com.bakdata.conquery.sql.conversion.query.RelativFormQueryConverter; -import com.bakdata.conquery.sql.conversion.query.SecondaryIdQueryConverter; -import com.bakdata.conquery.sql.conversion.query.TableExportQueryConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; -import com.bakdata.conquery.sql.execution.SqlCDateSetParser; -import org.jooq.DSLContext; import org.jooq.Field; import org.jooq.SQLDialect; -public interface DialectBundle { - - private static > List customize(List defaults, List substitutes) { - Map, C> substituteMap = getSubstituteMap(substitutes); - return defaults.stream() - .map(converter -> substituteMap.getOrDefault(converter.getConversionClass(), converter)) - .toList(); - } - - private static > Map, C> getSubstituteMap(List substitutes) { - return substitutes.stream() - .collect(Collectors.toMap( - Converter::getConversionClass, - Function.identity() - )); - } +/** + * Compatibility aggregate used when wiring all database-specific services for a backend namespace. + * + *

The bundle exposes both the legacy compiler adapter inherited from {@link LegacyCompilerDialect} and the runtime + * methods declared here. Legacy compiler code must depend on {@code LegacyCompilerDialect}, not on this aggregate. + * Runtime wiring may retain the bundle when it needs to construct both compiler and execution services for the same + * database.

+ * + *

This interface keeps the existing HANA and ClickHouse integrations source-compatible while their compiler and + * execution capabilities are separated. It is an application composition boundary, not a compiler dependency.

+ */ +public interface DialectBundle extends LegacyCompilerDialect { ResultSetProcessor getResultSetProcessor(ConqueryConfig config); Dialect getDialect(); - int getNameMaxLength(); - String getConnectionTestString(); SQLDialect getJooqDialect(); - - StratificationFunctions getStratificationFunctions(); - boolean isTypeCompatible(Field field, MajorTypeId type); - - SqlFunctionProvider getFunctionProvider(); - - IntervalPacker getIntervalPacker(); - - SqlDateAggregator getDateAggregator(); - - List> getNodeConverters(DSLContext context); - - default boolean supportsSingleColumnRanges() { - return false; - } - - default List> getDefaultNodeConverters(DSLContext dslContext) { - - QueryStepTransformer queryStepTransformer = new QueryStepTransformer(dslContext); - FormConversionHelper formConversionUtil = new FormConversionHelper(queryStepTransformer); - - return List.of( - new CQDateRestrictionConverter(), - new CQAndConverter(), - new CQOrConverter(), - new CQNegationConverter(), - new CQYesConverter(), - new CQConceptConverter(), - new CQExternalConverter(), - new CQReusedQueryConverter(), - new ConceptQueryConverter(queryStepTransformer), - new SecondaryIdQueryConverter(), - new AbsoluteFormQueryConverter(formConversionUtil), - new EntityDateQueryConverter(formConversionUtil), - new RelativFormQueryConverter(formConversionUtil), - new TableExportQueryConverter(queryStepTransformer) - ); - } - - default Map, ? extends SelectConverter> getSelectConverterOverrides(){ - return Collections.emptyMap(); - } - - default SelectConverter maybeOverride = (SelectConverter getSelectConverter(Select select) { + SelectConverter) getSelectConverterOverrides().get(select.getClass()); + + if (maybeOverride != null) { + return maybeOverride; + } + + return select.createConverter(); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java index 79f1e6532e..6ab0db36ed 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java @@ -38,7 +38,7 @@ public class QueryStepJoiner { */ public static QueryStep antiJoinWithAllIdsTable(QueryStep queryStep, ConversionContext context, DateAggregationAction dateAggregationAction) { - SqlFunctionProvider functionProvider = context.getConversionContext().getDialectBundle().getFunctionProvider(); + SqlFunctionProvider functionProvider = context.getConversionContext().getCompilerDialect().getFunctionProvider(); Field queryStepPrimaryColumn = queryStep.getQualifiedSelects().getIds().getPrimaryColumn(); ColumnConfig idColumnConfig = context.getIdColumns().findPrimaryIdColumn(); @@ -190,7 +190,7 @@ private static QueryStep joinStepsContainingNegation( } // first, invert dates of negated step - SqlDateAggregator dateAggregator = context.getDialectBundle().getDateAggregator(); + SqlDateAggregator dateAggregator = context.getCompilerDialect().getDateAggregator(); negateJoined = dateAggregator.invertAggregatedIntervals(negateJoined, context); // join with all-ids table necessary @@ -204,7 +204,7 @@ private static QueryStep joinStepsContainingNegation( DateAggregationDates aggregationDates = DateAggregationDates.forValidityDates(List.of( nonNegateJoined.getQualifiedSelects().getValidityDate(), negateJoined.getQualifiedSelects().getValidityDate(), - Optional.of(context.getDialectBundle().getFunctionProvider().allRangeIf(infinityRangeCondition)) + Optional.of(context.getCompilerDialect().getFunctionProvider().allRangeIf(infinityRangeCondition)) )); ColumnDateRange merged = dateAggregator.getAggregatedValidityDate(aggregationDates, DateAggregationAction.MERGE) @@ -326,7 +326,7 @@ private static QueryStep buildStepAndAggregateDates( withAllValidityDates.addAll(dateAggregationDates.allStartsAndEnds()); QueryStep joinedStep = buildJoinedStep(ids, withAllValidityDates, Optional.empty(), Optional.empty(), builder); - SqlDateAggregator sqlDateAggregator = context.getDialectBundle().getDateAggregator(); + SqlDateAggregator sqlDateAggregator = context.getCompilerDialect().getDateAggregator(); return sqlDateAggregator.apply( joinedStep, mergedSelects, diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java index e0ebf555de..07548d1ce6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java @@ -111,7 +111,7 @@ public ConnectorSqlSelects connectorSelect(CountQuartersSelect countQuartersSele selectContext.getNameGenerator().selectName(countQuartersSelect), selectContext.getTables(), selectContext.getFunctionProvider(), - selectContext.getDialectBundle().getStratificationFunctions() + selectContext.getCompilerDialect().getStratificationFunctions() ); String finalPredecessor = selectContext.getTables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER); @@ -134,7 +134,7 @@ public SqlFilters convertToSqlFilter(CountQuartersFilter countQuartersFilter, Fi filterContext.getNameGenerator().selectName(countQuartersFilter), filterContext.getTables(), filterContext.getFunctionProvider(), - filterContext.getDialectBundle().getStratificationFunctions() + filterContext.getCompilerDialect().getStratificationFunctions() ); ConnectorSqlSelects selects = ConnectorSqlSelects.builder() diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java index a40af4eeda..5f975b3cbf 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java @@ -137,7 +137,7 @@ private static List getRequiredColumns(Map flags, Set< @Override public ConnectorSqlSelects connectorSelect(FlagSelect flagSelect, SelectContext selectContext) { - SqlFunctionProvider functionProvider = selectContext.getConversionContext().getDialectBundle().getFunctionProvider(); + SqlFunctionProvider functionProvider = selectContext.getConversionContext().getCompilerDialect().getFunctionProvider(); SqlTables connectorTables = selectContext.getTables(); Map rootSelects = createFlagRootSelectMap(flagSelect, connectorTables.getRootTable()); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java index 005be1cd75..c16a91dc76 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java @@ -24,7 +24,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.IntervalPackingSelectsCte; import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingContext; import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep; -import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.CteStep; @@ -57,7 +57,7 @@ public static ConnectorSqlSelects c .collect(Collectors.toList()); SqlTables daterangeSelectTables = createTables(alias, context.getTables(), context); - QueryStep lastIntervalPackingStep = applyIntervalPacking(daterange, daterangeSelectTables, context.getIds(), context.getTables(), context.getDialectBundle()); + QueryStep lastIntervalPackingStep = applyIntervalPacking(daterange, daterangeSelectTables, context.getIds(), context.getTables(), context.getCompilerDialect()); ColumnDateRange qualified = daterange.qualify(daterangeSelectTables.getPredecessor(INTERVAL_PACKING_SELECTS)); FieldWrapper aggregationField = aggregationFunction.apply(qualified, alias, functionProvider); @@ -67,7 +67,7 @@ public static ConnectorSqlSelects c qualified, aggregationField, daterangeSelectTables, - context.getDialectBundle() + context.getCompilerDialect() ); ConnectorSqlTables tables = context.getTables(); @@ -91,7 +91,7 @@ public static SqlFilters createForFilter( FilterContext context ) { String alias = context.getNameGenerator().selectName((LabeledNamespaceIdentifiable) filter); - SqlFunctionProvider functionProvider = context.getDialectBundle().getFunctionProvider(); + SqlFunctionProvider functionProvider = context.getCompilerDialect().getFunctionProvider(); ColumnDateRange daterange = functionProvider.forArbitraryDateRange(filter).as(alias); List rootSelects = daterange.toFields().stream() @@ -99,7 +99,7 @@ public static SqlFilters createForFilter( .collect(Collectors.toList()); SqlTables daterangeSelectTables = createTables(alias, context.getTables(), context); - QueryStep lastIntervalPackingStep = applyIntervalPacking(daterange, daterangeSelectTables, context.getIds(), context.getTables(), context.getDialectBundle()); + QueryStep lastIntervalPackingStep = applyIntervalPacking(daterange, daterangeSelectTables, context.getIds(), context.getTables(), context.getCompilerDialect()); ColumnDateRange qualified = daterange.qualify(daterangeSelectTables.getPredecessor(INTERVAL_PACKING_SELECTS)); FieldWrapper aggregationField = aggregationFunction.apply(qualified, alias, functionProvider); @@ -109,7 +109,7 @@ public static SqlFilters createForFilter( qualified, aggregationField, daterangeSelectTables, - context.getDialectBundle() + context.getCompilerDialect() ); ConnectorSqlSelects sqlSelects = ConnectorSqlSelects.builder() @@ -143,8 +143,8 @@ private static Condition containsInfinityDate(ColumnDateRange validityDate, SqlF private static SqlTables createTables(String alias, ConnectorSqlTables connectorTables, Context context) { Map predecessorMapping = new HashMap<>(); String preprocessingCteName = connectorTables.cteName(PREPROCESSING); - predecessorMapping.putAll(IntervalPackingCteStep.getMappings(context.getDialectBundle())); - if (context.getDialectBundle().supportsSingleColumnRanges()) { + predecessorMapping.putAll(IntervalPackingCteStep.getMappings(context.getCompilerDialect())); + if (context.getCompilerDialect().supportsSingleColumnRanges()) { predecessorMapping.put(UNNEST_DATE, INTERVAL_COMPLETE); predecessorMapping.put(INTERVAL_PACKING_SELECTS, UNNEST_DATE); } @@ -160,7 +160,7 @@ private static QueryStep applyIntervalPacking( SqlTables dateUnionTables, SqlIdColumns idColumns, ConnectorSqlTables connectorSqlTables, - DialectBundle sqlDialect + LegacyCompilerDialect sqlDialect ) { String preprocessingCteName = connectorSqlTables.cteName(PREPROCESSING); IntervalPackingContext intervalPackingContext = IntervalPackingContext.builder() diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java index 1239ef8d87..446bc81397 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; -import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; @@ -25,8 +25,8 @@ public class SelectContext implements Context { ConversionContext conversionContext; @Override - public DialectBundle getDialectBundle() { - return getConversionContext().getDialectBundle(); + public LegacyCompilerDialect getCompilerDialect() { + return getConversionContext().getCompilerDialect(); } public static SelectContext create( @@ -48,7 +48,7 @@ public static SelectContext create( } public SqlFunctionProvider getFunctionProvider() { - return getDialectBundle().getFunctionProvider(); + return getCompilerDialect().getFunctionProvider(); } public SqlIdColumns getIds() { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java index e63356c199..7afb49e42a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java @@ -55,7 +55,7 @@ private static QueryStep convertPrerequisite(AbsoluteFormQuery absoluteForm, Con Preconditions.checkArgument(withConvertedPrerequisite.getQuerySteps().size() == 1, "Base query conversion should produce exactly 1 QueryStep"); QueryStep convertedPrerequisite = withConvertedPrerequisite.getLastConvertedStep(); - ColumnDateRange bounds = context.getDialectBundle() + ColumnDateRange bounds = context.getCompilerDialect() .getFunctionProvider() .forCDateRange(CDateRange.of(absoluteForm.getDateRange())).as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java index 343b70a2ae..5aeff1a381 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java @@ -56,7 +56,7 @@ public Class getConversionClass() { @Override public ConversionContext convert(ConceptQuery conceptQuery, ConversionContext context) { - SqlFunctionProvider functionProvider = context.getDialectBundle().getFunctionProvider(); + SqlFunctionProvider functionProvider = context.getCompilerDialect().getFunctionProvider(); ConversionContext contextAfterConversion = context.getNodeConversions().convert(conceptQuery.getRoot(), context); QueryStep preFinalStep = contextAfterConversion.getLastConvertedStep(); diff --git a/sql-connector/README.md b/sql-connector/README.md index c29418cf29..62d49ee2eb 100644 --- a/sql-connector/README.md +++ b/sql-connector/README.md @@ -21,7 +21,8 @@ The module uses top-level packages as architectural boundaries: - `model` contains only the immutable, fully resolved compiler input; - `validation` contains Bean Validation constraints and the explicit validation boundary; -- SQL business logic belongs in sibling packages such as `conversion`, `dialect`, and `execution`, not below `model`. +- `compiler` contains framework-neutral compilation contracts, including the public dialect capabilities; +- SQL conversion and execution implementations belong in sibling packages, not below `model`. The resolved model is organized by responsibility: diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java new file mode 100644 index 0000000000..f22d696aa0 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java @@ -0,0 +1,22 @@ +package com.bakdata.conquery.sql.compiler.dialect; + +/** + * Database-specific capabilities exposed to the framework-neutral SQL compiler. + * + *

This contract belongs to the SQL connector and must only use types owned by the connector or its public model + * dependencies. Backend query DTOs, converter registries, runtime connections, and result processing are deliberately + * excluded.

+ * + *

Concrete backend integrations may extend this contract with temporary adapter interfaces while the legacy compiler + * is migrated. Those adapters must not become dependencies of the framework-neutral compiler.

+ */ +public interface CompilerDialect { + + /** Maximum identifier length supported by the target database. */ + int getNameMaxLength(); + + /** Whether the dialect represents date ranges in one database column instead of separate start and end columns. */ + default boolean supportsSingleColumnRanges() { + return false; + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/package-info.java new file mode 100644 index 0000000000..fbdeb27198 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/package-info.java @@ -0,0 +1,2 @@ +/** Database-specific capabilities exposed through the SQL compiler API. */ +package com.bakdata.conquery.sql.compiler.dialect; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/package-info.java new file mode 100644 index 0000000000..b9027865d6 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/package-info.java @@ -0,0 +1,2 @@ +/** Framework-neutral contracts for compiling resolved queries into executable SQL. */ +package com.bakdata.conquery.sql.compiler; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/package-info.java index 92d232da46..f3fb3af679 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/package-info.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/package-info.java @@ -1,8 +1,8 @@ /** * Framework-neutral SQL connector contracts and services. * - *

{@code model} defines the fully resolved input, {@code validation} verifies that input, and sibling packages such - * as {@code conversion}, {@code dialect}, and {@code execution} contain SQL business logic. Business logic may depend on - * the model and validation boundary; model code must not depend on compiler or execution services.

+ *

{@code model} defines the fully resolved input, {@code validation} verifies that input, and {@code compiler} + * defines framework-neutral compilation contracts. Conversion and execution implementations may depend on these + * contracts; model code must not depend on compiler or execution services.

*/ package com.bakdata.conquery.sql; From 16061d4ac491c4f249fef53bcb0b1aeaf94a4251 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:09:20 +0200 Subject: [PATCH 06/81] refactor(sql): decouple query steps from dialect Remove the unused SQL function provider from union-step construction so the compiler IR no longer carries a false runtime dependency. Also replace Javadoc-only links to select DTOs with IR terminology. Co-authored-by: Codex GPT-5 --- .../sql/conversion/cqelement/CQExternalConverter.java | 6 +++--- .../sql/conversion/forms/RelativeStratification.java | 4 ++-- .../conversion/forms/StratificationTableFactory.java | 2 +- .../conquery/sql/conversion/model/QueryStep.java | 11 +++++------ .../conquery/sql/conversion/model/Selects.java | 5 ++--- .../model/select/ConceptColumnSelectConverter.java | 2 +- .../sql/conversion/model/select/SqlSelect.java | 5 ++--- .../conversion/query/TableExportQueryConverter.java | 2 +- 8 files changed, 17 insertions(+), 20 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java index 0b70eb2435..e06ed92761 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java @@ -41,7 +41,7 @@ private static QueryStep createExternalIdsCte(CQExternal external, SqlFunctionPr unions.addAll(rowSelects); } Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 resolved row when converting a CQExternal"); - QueryStep allStep = QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_IDS_CTE_NAME, Collections.emptyList(), context.isNegation(), functionProvider); + QueryStep allStep = QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_IDS_CTE_NAME, Collections.emptyList(), context.isNegation()); Optional maybeValidityDate = allStep.getSelects().getValidityDate(); @@ -173,8 +173,8 @@ private QueryStep createExternalExtrasCte(CQExternal external, SqlFunctionProvid unions.add(rowSelects); } Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 converted resolved row when converting a CQExternal"); - return QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_EXTRAS_CTE_NAME, Collections.emptyList(), context.isNegation(), functionProvider); + return QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_EXTRAS_CTE_NAME, Collections.emptyList(), context.isNegation()); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java index d06755cbdc..8731afabb7 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java @@ -163,7 +163,7 @@ private QueryStep createCompleteTable(QueryStep totalBoundsStep, RelativeFormQue Stream.concat(Stream.ofNullable(outcomeTable), Stream.ofNullable(featureTable)).toList(), FormCteStep.COMPLETE.getSuffix(), Collections.emptyList(), - context.isNegation(), functionProvider + context.isNegation() ); } @@ -196,7 +196,7 @@ private QueryStep createIntervalTable(QueryStep totalBoundsStep, Resolution reso List.of(timeBeforeStep, timeAfterStep), FormCteStep.stratificationCte(resolution).getSuffix(), Collections.emptyList(), - context.isNegation(), functionProvider + context.isNegation() ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java index 4287a5ace8..046f300e3d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java @@ -55,7 +55,7 @@ protected static QueryStep unionResolutionTables(List unionSteps, Lis withQualifiedSelects, FormCteStep.FULL_STRATIFICATION.getSuffix(), Stream.concat(predecessors.stream(), unionSteps.stream()).toList(), - context.isNegation(), context.getFunctionProvider() + context.isNegation() ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java index ee3d96bcde..70d9d81279 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.sql.conversion.model; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; import lombok.Builder; import lombok.Singular; @@ -62,15 +61,15 @@ public class QueryStep { @Singular List predecessors; - public static QueryStep createUnionAllStep(List unionSteps, String cteName, List predecessors, boolean negation, SqlFunctionProvider functionProvider) { - return createUnionStep(unionSteps, cteName, predecessors, true, negation, functionProvider); + public static QueryStep createUnionAllStep(List unionSteps, String cteName, List predecessors, boolean negation) { + return createUnionStep(unionSteps, cteName, predecessors, true, negation); } - public static QueryStep createUnionStep(List unionSteps, String cteName, List predecessors, boolean negation, SqlFunctionProvider functionProvider) { - return createUnionStep(unionSteps, cteName, predecessors, false, negation, functionProvider); + public static QueryStep createUnionStep(List unionSteps, String cteName, List predecessors, boolean negation) { + return createUnionStep(unionSteps, cteName, predecessors, false, negation); } - private static QueryStep createUnionStep(List unionSteps, String cteName, List predecessors, boolean unionAll, boolean negation, SqlFunctionProvider functionProvider) { + private static QueryStep createUnionStep(List unionSteps, String cteName, List predecessors, boolean unionAll, boolean negation) { QueryStep first = unionSteps.getFirst(); return first.toBuilder() .cteName(cteName) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java index 160bae7711..a13db9b221 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.sql.conversion.model; -import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; @@ -87,7 +86,7 @@ public List> all() { } /** - * All {@link Select}s that have not been explicitly selected (IDs, validity/stratification dates). + * All fields that have not been explicitly selected (IDs, validity/stratification dates). */ public List> nonExplicitSelects() { return Stream.of( @@ -102,7 +101,7 @@ public List> nonExplicitSelects() { } /** - * All explicitly selected and converted {@link Select}s. + * All explicitly selected and converted fields. */ public List> explicitSelects() { return this.sqlSelects.stream() diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java index 56b5301cbe..ff06d0116e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java @@ -115,7 +115,7 @@ private static QueryStep createUnionConnectorConnectorsStep( ) { List unionSteps = connectors.stream().map(connector -> createConnectorColumnSelectQuery(connector, select, alias, selectContext)).toList(); String unionedColumnsCteName = selectContext.getNameGenerator().cteStepName(CONCEPT_COLUMN_STEPS.UNIONED_COLUMNS, alias); - return QueryStep.createUnionStep(unionSteps, unionedColumnsCteName, Collections.emptyList(), false, selectContext.getFunctionProvider()); //TODO is false correct here? + return QueryStep.createUnionStep(unionSteps, unionedColumnsCteName, Collections.emptyList(), false); //TODO is false correct here? } private static QueryStep createConnectorColumnSelectQuery( diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java index 5ca54b46c4..cfb17e6cf2 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.sql.conversion.model.select; -import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.ExistsSelect; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.Qualifiable; import org.jooq.Field; @@ -24,14 +23,14 @@ default boolean isUniversal() { } /** - * Special selects like {@link ExistsSelect} require an extra aggregation when joining them with other connectors. + * Special selects like {@link ExistsSqlSelect} require an extra aggregation when joining them with other connectors. */ default SqlSelect connectorAggregate() { return this; } /** - * Special selects like {@link ExistsSelect} require to be converted into a specific format before executing the + * Special selects like {@link ExistsSqlSelect} require to be converted into a specific format before executing the * final query. */ default SqlSelect toFinalRepresentation() { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java index 5fe2b44e41..7c0c643e45 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java @@ -206,7 +206,7 @@ public ConversionContext convert(TableExportQuery tableExportQuery, ConversionCo convertedTables, null, // no CTE name required as this step will be the final select List.of(convertedPrerequisite), - context.isNegation(), context.getFunctionProvider() + context.isNegation() ); final Select selectQuery = queryStepTransformer.toSelectQuery(unionedTables, context.getFunctionProvider()); From f50349cdc06b7ea7e113fc6ebe30b695e02c5998 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:26:51 +0200 Subject: [PATCH 07/81] refactor(sql): expose aggregation through compiler dialect Make dialect-specific final select aggregation part of CompilerDialect and bridge the legacy backend provider to the new contract. Add jOOQ to the connector module so compiler IR can depend directly on SQL expression types. Co-authored-by: Codex GPT-5 --- backend/pom.xml | 2 +- .../conversion/dialect/LegacyCompilerDialect.java | 6 ++++++ .../conversion/model/select/ExistsSqlSelect.java | 4 ++-- .../sql/conversion/model/select/SqlSelect.java | 6 +++--- .../sql/conversion/query/ConceptQueryConverter.java | 13 ++++++++----- pom.xml | 1 + sql-connector/pom.xml | 5 +++++ .../sql/compiler/dialect/CompilerDialect.java | 10 ++++++++++ 8 files changed, 36 insertions(+), 11 deletions(-) diff --git a/backend/pom.xml b/backend/pom.xml index 6bfa236947..835a22ff89 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -351,7 +351,7 @@ org.jooq jooq - 3.20.2 + ${jooq.version} com.zaxxer diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java index 4aed1ff758..38db1fd994 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java @@ -27,6 +27,7 @@ import com.bakdata.conquery.sql.conversion.query.SecondaryIdQueryConverter; import com.bakdata.conquery.sql.conversion.query.TableExportQueryConverter; import org.jooq.DSLContext; +import org.jooq.Field; /** * Temporary backend adapter exposing services required by the legacy SQL compiler. @@ -37,6 +38,11 @@ */ public interface LegacyCompilerDialect extends CompilerDialect { + @Override + default Field anyValue(Field field) { + return getFunctionProvider().anyValue(field); + } + StratificationFunctions getStratificationFunctions(); SqlFunctionProvider getFunctionProvider(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSqlSelect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSqlSelect.java index b6a15eaab5..2b7c01ef2b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSqlSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSqlSelect.java @@ -1,6 +1,6 @@ package com.bakdata.conquery.sql.conversion.model.select; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import org.jooq.Field; @@ -61,7 +61,7 @@ public SqlSelect toFinalRepresentation() { } @Override - public List> aggregateForFinalQuery(SqlFunctionProvider functionProvider) { + public List> aggregateForFinalQuery(CompilerDialect dialect) { // We have to coalesce at the end of the query because full-outer-joins will create null values, which we want to avoid. Field coalesced = coalesce(max(select()), inline(0)); return List.of(coalesced.as(alias)); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java index cfb17e6cf2..08738c3d50 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java @@ -1,6 +1,6 @@ package com.bakdata.conquery.sql.conversion.model.select; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import com.bakdata.conquery.sql.conversion.model.Qualifiable; import org.jooq.Field; @@ -40,9 +40,9 @@ default SqlSelect toFinalRepresentation() { /** * Aggregate this select to one value per ID group in the final concept query. */ - default List> aggregateForFinalQuery(SqlFunctionProvider functionProvider) { + default List> aggregateForFinalQuery(CompilerDialect dialect) { return toFinalRepresentation().toFields().stream() - .>map(field -> functionProvider.anyValue(field).as(field.getName())) + .>map(field -> dialect.anyValue(field).as(field.getName())) .toList(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java index 5aeff1a381..bcd28f93c7 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java @@ -3,8 +3,10 @@ import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.concept.specific.CQNegation; import com.bakdata.conquery.models.query.DateAggregationMode; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; +import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.*; import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; @@ -70,7 +72,7 @@ public ConversionContext convert(ConceptQuery conceptQuery, ConversionContext co QueryStep finalStep = QueryStep.builder() .cteName(null) // the final QueryStep won't be converted to a CTE - .selects(getFinalSelects(conceptQuery, preFinalSelects, functionProvider)) + .selects(getFinalSelects(conceptQuery, preFinalSelects, context.getCompilerDialect())) .fromTable(getFinalTable(preFinalStep, contextAfterConversion)) .groupBy(getFinalGroupBySelects(preFinalSelects)) .predecessors(predecessors) @@ -80,7 +82,8 @@ public ConversionContext convert(ConceptQuery conceptQuery, ConversionContext co return contextAfterConversion.withFinalQuery(new SqlQuery(finalQuery, conceptQuery.getResultInfos())); } - private Selects getFinalSelects(ConceptQuery conceptQuery, Selects preFinalSelects, SqlFunctionProvider functionProvider) { + private Selects getFinalSelects(ConceptQuery conceptQuery, Selects preFinalSelects, LegacyCompilerDialect dialect) { + SqlFunctionProvider functionProvider = dialect.getFunctionProvider(); Selects finalSelects = preFinalSelects; if (conceptQuery.getDateAggregationMode() == DateAggregationMode.NONE) { finalSelects = preFinalSelects.blockValidityDate(); @@ -96,13 +99,13 @@ private Selects getFinalSelects(ConceptQuery conceptQuery, Selects preFinalSelec .ids(finalSelects.getIds()) .validityDate(finalSelects.getValidityDate()) .stratificationDate(finalSelects.getStratificationDate()) - .sqlSelects(getFinalAggregatedSelects(finalSelects, functionProvider)) + .sqlSelects(getFinalAggregatedSelects(finalSelects, dialect)) .build(); } - private List> getFinalAggregatedSelects(Selects finalSelects, SqlFunctionProvider functionProvider) { + private List> getFinalAggregatedSelects(Selects finalSelects, CompilerDialect dialect) { return finalSelects.getSqlSelects().stream() - .flatMap(sqlSelect -> sqlSelect.aggregateForFinalQuery(functionProvider).stream()) + .flatMap(sqlSelect -> sqlSelect.aggregateForFinalQuery(dialect).stream()) .map(this::toFieldWrapper) .toList(); } diff --git a/pom.xml b/pom.xml index 5e2cd7311e..814b239874 100644 --- a/pom.xml +++ b/pom.xml @@ -18,6 +18,7 @@ 21 + 3.20.2 1.18.36 UTF-8 ${java.required} diff --git a/sql-connector/pom.xml b/sql-connector/pom.xml index 073afc8767..aa3fee0be6 100644 --- a/sql-connector/pom.xml +++ b/sql-connector/pom.xml @@ -24,6 +24,11 @@ jakarta.validation jakarta.validation-api + + org.jooq + jooq + ${jooq.version} + org.junit.jupiter junit-jupiter-api diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java index f22d696aa0..caa03b93b0 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.sql.compiler.dialect; +import org.jooq.Field; + /** * Database-specific capabilities exposed to the framework-neutral SQL compiler. * @@ -12,6 +14,14 @@ */ public interface CompilerDialect { + /** + * Aggregate a field to an arbitrary value from its group. + * + *

The exact expression is dialect-specific. For example, a database without an {@code ANY_VALUE} aggregate may + * use {@code MIN} as an equivalent deterministic representative where the compiler only requires one value.

+ */ + Field anyValue(Field field); + /** Maximum identifier length supported by the target database. */ int getNameMaxLength(); From ab1b6e8ad1295db812eeafa85f700c21f6f30e52 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 1 Sep 2026 08:17:58 +0200 Subject: [PATCH 08/81] feat(sql): define compiled query output contract Add a self-contained compiled query with an ordered physical column layout so backends can execute SQL and correlate results without compiler-owned result processing. Replace presentation metadata in resolved result columns with stable output identifiers and validate identifiers and SQL aliases for uniqueness. Co-authored-by: Codex GPT-5 --- .../conquery/sql/compiler/ColumnRole.java | 11 ++++ .../conquery/sql/compiler/CompiledColumn.java | 22 +++++++ .../conquery/sql/compiler/CompiledQuery.java | 42 +++++++++++++ .../conquery/sql/compiler/package-info.java | 5 +- .../conquery/sql/model/ResolvedQuery.java | 9 +++ .../sql/model/result/ResultColumn.java | 28 +++------ .../sql/model/result/package-info.java | 2 +- .../sql/compiler/CompiledQueryTest.java | 61 +++++++++++++++++++ .../conquery/sql/model/ResolvedQueryTest.java | 21 +++++-- .../sql/model/ValidationTestSupport.java | 8 +-- 10 files changed, 179 insertions(+), 30 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ColumnRole.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/CompiledColumn.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/CompiledQuery.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/CompiledQueryTest.java diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ColumnRole.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ColumnRole.java new file mode 100644 index 0000000000..b95a879037 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ColumnRole.java @@ -0,0 +1,11 @@ +package com.bakdata.conquery.sql.compiler; + +/** Describes how a compiled output column contributes to a result row. */ +public enum ColumnRole { + + /** Column containing an entity identifier. Multiple columns may form a composite identifier. */ + ENTITY_ID, + + /** Query result value decoded and post-processed by the backend. */ + RESULT +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/CompiledColumn.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/CompiledColumn.java new file mode 100644 index 0000000000..88e368f327 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/CompiledColumn.java @@ -0,0 +1,22 @@ +package com.bakdata.conquery.sql.compiler; + +import com.bakdata.conquery.sql.model.result.ResultType; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; + +/** + * Ordered description of one column produced by a {@link CompiledQuery}. + * + * @param outputId stable, backend-independent key used to correlate the column with backend metadata + * @param sqlAlias alias rendered into the SQL statement + * @param type logical value type expected by backend result decoding + * @param role role of the column in the result row + */ +public record CompiledColumn( + @NotBlank String outputId, + @NotBlank String sqlAlias, + @NotNull @Valid ResultType type, + @NotNull ColumnRole role +) { +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/CompiledQuery.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/CompiledQuery.java new file mode 100644 index 0000000000..1e8faac0c0 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/CompiledQuery.java @@ -0,0 +1,42 @@ +package com.bakdata.conquery.sql.compiler; + +import java.util.List; + +import com.bakdata.conquery.sql.model.internal.ModelNormalization; +import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; + +/** + * Self-contained result of SQL compilation. + * + *

The backend executes {@link #sql()} and uses the ordered {@link #columns()} to correlate and decode the JDBC + * result. Datasource selection, execution, and result post-processing are deliberately excluded.

+ */ +public record CompiledQuery( + @NotBlank String sql, + @NotEmpty List<@NotNull @Valid CompiledColumn> columns +) { + + public CompiledQuery { + columns = ModelNormalization.immutableCopy(columns); + } + + @AssertTrue(message = "compiled column outputIds must be unique") + public boolean isOutputIdsUnique() { + return columns == null || columns.stream() + .map(CompiledColumn::outputId) + .distinct() + .count() == columns.size(); + } + + @AssertTrue(message = "compiled column SQL aliases must be unique") + public boolean isSqlAliasesUnique() { + return columns == null || columns.stream() + .map(CompiledColumn::sqlAlias) + .distinct() + .count() == columns.size(); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/package-info.java index b9027865d6..9bf4ac5306 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/package-info.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/package-info.java @@ -1,2 +1,5 @@ -/** Framework-neutral contracts for compiling resolved queries into executable SQL. */ +/** + * Framework-neutral contracts for compiling resolved queries into self-contained SQL and an ordered physical result + * layout. Query execution and result post-processing are backend responsibilities. + */ package com.bakdata.conquery.sql.compiler; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/ResolvedQuery.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/ResolvedQuery.java index 3216636ab8..61022bc442 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/ResolvedQuery.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/ResolvedQuery.java @@ -7,6 +7,7 @@ import com.bakdata.conquery.sql.model.result.ResultColumn; import com.bakdata.conquery.sql.model.schema.EntitySchema; import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.NotNull; /** @@ -25,4 +26,12 @@ public record ResolvedQuery( public ResolvedQuery { resultColumns = ModelNormalization.immutableCopy(resultColumns); } + + @AssertTrue(message = "result column outputIds must be unique") + public boolean isResultColumnOutputIdsUnique() { + return resultColumns == null || resultColumns.stream() + .map(ResultColumn::outputId) + .distinct() + .count() == resultColumns.size(); + } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/result/ResultColumn.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/result/ResultColumn.java index d67dff0702..f2af05a985 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/result/ResultColumn.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/result/ResultColumn.java @@ -1,29 +1,17 @@ package com.bakdata.conquery.sql.model.result; -import java.util.Optional; -import java.util.Set; - -import com.bakdata.conquery.sql.model.internal.ModelNormalization; import jakarta.validation.Valid; -import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; -/** Ordered output-column metadata, with presentation names already resolved for the query locale. */ +/** + * Backend-independent identity and logical type of a requested result column. + * + *

Labels, descriptions, semantics, and formatting instructions are deliberately excluded. Backends retain that + * presentation metadata and correlate it with compiled columns through {@link #outputId()}.

+ */ public record ResultColumn( - @NotBlank String label, - @NotNull Optional defaultLabel, - @NotNull Optional description, - @NotNull @Valid ResultType type, - @NotNull Set<@NotBlank String> semantics + @NotBlank String outputId, + @NotNull @Valid ResultType type ) { - - public ResultColumn { - semantics = ModelNormalization.immutableCopy(semantics); - } - - @AssertTrue(message = "defaultLabel must not be blank when present") - public boolean isDefaultLabelValid() { - return defaultLabel == null || defaultLabel.isEmpty() || !defaultLabel.get().isBlank(); - } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/result/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/result/package-info.java index f20ad2f630..32eca5222f 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/result/package-info.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/result/package-info.java @@ -1,2 +1,2 @@ -/** Ordered result-column metadata and logical result types. */ +/** Backend-independent identities and logical types of requested result columns. */ package com.bakdata.conquery.sql.model.result; diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/CompiledQueryTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/CompiledQueryTest.java new file mode 100644 index 0000000000..feaedaaf16 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/CompiledQueryTest.java @@ -0,0 +1,61 @@ +package com.bakdata.conquery.sql.compiler; + +import static com.bakdata.conquery.sql.model.ValidationTestSupport.assertInvalid; +import static com.bakdata.conquery.sql.model.ValidationTestSupport.assertValid; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.ArrayList; +import java.util.List; + +import com.bakdata.conquery.sql.model.result.ResultType; +import org.junit.jupiter.api.Test; + +class CompiledQueryTest { + + private static final CompiledColumn ENTITY_ID = new CompiledColumn( + "entity-id", "entity_id", ResultType.Primitive.STRING, ColumnRole.ENTITY_ID + ); + private static final CompiledColumn RESULT = new CompiledColumn( + "concept.exists", "concept_exists", ResultType.Primitive.BOOLEAN, ColumnRole.RESULT + ); + + @Test + void shouldDescribeTheOrderedPhysicalResult() { + CompiledQuery query = new CompiledQuery("select entity_id, concept_exists from result", List.of(ENTITY_ID, RESULT)); + + assertValid(query); + assertEquals(List.of(ENTITY_ID, RESULT), query.columns()); + } + + @Test + void shouldDefensivelyCopyColumns() { + List columns = new ArrayList<>(List.of(ENTITY_ID)); + CompiledQuery query = new CompiledQuery("select entity_id from result", columns); + + columns.clear(); + + assertEquals(List.of(ENTITY_ID), query.columns()); + assertThrows(UnsupportedOperationException.class, () -> query.columns().clear()); + } + + @Test + void shouldRejectIncompleteOutputContracts() { + assertInvalid(new CompiledColumn("", "entity_id", ResultType.Primitive.STRING, ColumnRole.ENTITY_ID)); + assertInvalid(new CompiledQuery("", List.of(ENTITY_ID))); + assertInvalid(new CompiledQuery("select 1", List.of())); + } + + @Test + void shouldRejectAmbiguousColumnMappings() { + CompiledColumn duplicateOutputId = new CompiledColumn( + ENTITY_ID.outputId(), "other_id", ResultType.Primitive.STRING, ColumnRole.RESULT + ); + CompiledColumn duplicateAlias = new CompiledColumn( + "other-id", ENTITY_ID.sqlAlias(), ResultType.Primitive.STRING, ColumnRole.RESULT + ); + + assertInvalid(new CompiledQuery("select entity_id, other_id from result", List.of(ENTITY_ID, duplicateOutputId))); + assertInvalid(new CompiledQuery("select entity_id, entity_id from result", List.of(ENTITY_ID, duplicateAlias))); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java index 4bcb23d39e..b191fd0676 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java @@ -9,7 +9,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.Set; import com.bakdata.conquery.models.datasets.ColumnType; import com.bakdata.conquery.sql.model.node.AndNode; @@ -56,9 +55,7 @@ void shouldCreateResolvedQueryWithoutFrameworkTypes() { new EntitySchema(EVENTS, ENTITY_ID), concept, true, - List.of(new ResultColumn( - "Entity", Optional.of("Entity"), Optional.empty(), ResultType.Primitive.STRING, Set.of("ID") - )) + List.of(new ResultColumn("entity", ResultType.Primitive.STRING)) ); assertValid(query); @@ -139,6 +136,22 @@ void shouldValidateTheCompleteQueryGraphAtTheCompilerBoundary() { ); } + @Test + void shouldRejectAmbiguousResultColumnIds() { + ResolvedQuery query = new ResolvedQuery( + new ExecutionTarget("dataset", "warehouse"), + new EntitySchema(EVENTS, ENTITY_ID), + conceptNode(EVENTS), + false, + List.of( + new ResultColumn("duplicate", ResultType.Primitive.STRING), + new ResultColumn("duplicate", ResultType.Primitive.BOOLEAN) + ) + ); + + assertInvalid(query); + } + private static ConceptNode conceptNode(SqlTable table) { ResolvedColumn id = new ResolvedColumn("concept.id", table, "entity_id", ColumnType.STRING, false); ResolvedConnector connector = new ResolvedConnector( diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ValidationTestSupport.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ValidationTestSupport.java index 9b57aeae8a..f15fb5ddcd 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ValidationTestSupport.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ValidationTestSupport.java @@ -9,19 +9,19 @@ import jakarta.validation.Validation; import jakarta.validation.Validator; -final class ValidationTestSupport { +public final class ValidationTestSupport { - static final Validator VALIDATOR = Validation.buildDefaultValidatorFactory().getValidator(); + public static final Validator VALIDATOR = Validation.buildDefaultValidatorFactory().getValidator(); private ValidationTestSupport() { } - static void assertValid(T value) { + public static void assertValid(T value) { Set> violations = VALIDATOR.validate(value); assertTrue(violations.isEmpty(), () -> "Expected no constraint violations, but got: " + violations); } - static void assertInvalid(T value) { + public static void assertInvalid(T value) { Set> violations = VALIDATOR.validate(value); assertFalse(violations.isEmpty(), "Expected at least one constraint violation"); } From dbffb7beff5d01b3f08a2f4b3d863b2fac9fda34 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 1 Sep 2026 08:33:57 +0200 Subject: [PATCH 09/81] feat(sql): define compiler invocation boundary Add a backend-neutral SqlCompiler that compiles a resolved query for an explicitly selected dialect. Remove execution routing from ResolvedQuery so dataset revisions and datasource selection remain backend responsibilities. Co-authored-by: Codex GPT-5 --- .../conquery/sql/compiler/SqlCompiler.java | 17 +++++++++++++ .../conquery/sql/compiler/package-info.java | 2 +- .../conquery/sql/model/ExecutionTarget.java | 24 ------------------- .../conquery/sql/model/ResolvedQuery.java | 1 - .../conquery/sql/model/package-info.java | 4 ++-- .../conquery/sql/model/ResolvedQueryTest.java | 4 ---- 6 files changed, 20 insertions(+), 32 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/SqlCompiler.java delete mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/model/ExecutionTarget.java diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/SqlCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/SqlCompiler.java new file mode 100644 index 0000000000..1c6bc06076 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/SqlCompiler.java @@ -0,0 +1,17 @@ +package com.bakdata.conquery.sql.compiler; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.model.ResolvedQuery; + +/** + * Compiles a fully resolved query into self-contained SQL for a selected database dialect. + * + *

The caller owns dataset and datasource selection, query execution, and result post-processing. Implementations + * must compile exclusively from the supplied query and dialect and must not resolve identifiers or access backend + * repositories.

+ */ +@FunctionalInterface +public interface SqlCompiler { + + CompiledQuery compile(ResolvedQuery query, CompilerDialect dialect); +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/package-info.java index 9bf4ac5306..86869b1af8 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/package-info.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/package-info.java @@ -1,5 +1,5 @@ /** * Framework-neutral contracts for compiling resolved queries into self-contained SQL and an ordered physical result - * layout. Query execution and result post-processing are backend responsibilities. + * layout. Dataset and datasource selection, query execution, and result post-processing are backend responsibilities. */ package com.bakdata.conquery.sql.compiler; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/ExecutionTarget.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/ExecutionTarget.java deleted file mode 100644 index c4eed00aa1..0000000000 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/ExecutionTarget.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.bakdata.conquery.sql.model; - -import java.util.Optional; - -import jakarta.validation.constraints.AssertTrue; -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; - -/** Selects the dataset snapshot and configured datasource used to execute a query. */ -public record ExecutionTarget( - @NotBlank String datasetId, - @NotBlank String dataSource, - @NotNull Optional catalogRevision -) { - - @AssertTrue(message = "catalogRevision must not be blank when present") - public boolean isCatalogRevisionValid() { - return catalogRevision == null || catalogRevision.isEmpty() || !catalogRevision.get().isBlank(); - } - - public ExecutionTarget(String datasetId, String dataSource) { - this(datasetId, dataSource, Optional.empty()); - } -} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/ResolvedQuery.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/ResolvedQuery.java index 61022bc442..e4db885fcd 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/ResolvedQuery.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/ResolvedQuery.java @@ -16,7 +16,6 @@ * @param includeValidityDate whether the final result exposes an aggregated validity-date column */ public record ResolvedQuery( - @NotNull @Valid ExecutionTarget target, @NotNull @Valid EntitySchema entitySchema, @NotNull @Valid QueryNode root, boolean includeValidityDate, diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/package-info.java index 9944bce082..247a51dd68 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/package-info.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/package-info.java @@ -2,8 +2,8 @@ * Framework-neutral, fully resolved input model for SQL query compilation. * *

Instances in this package must not require repository access or identifier resolution during compilation. They - * describe query semantics and physical database objects, but deliberately exclude connections, dialects, and SQL - * compiler state.

+ * describe query semantics and physical database objects, but deliberately exclude dataset routing, datasource + * selection, connections, dialects, and SQL compiler state.

* *

The root query aggregate is complemented by dedicated packages for query nodes, operations, ranges, result * metadata, and physical schema references. Validation and SQL business logic live in sibling packages so this package diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java index b191fd0676..aeea03f5e2 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java @@ -51,7 +51,6 @@ void shouldCreateResolvedQueryWithoutFrameworkTypes() { ); ResolvedQuery query = new ResolvedQuery( - new ExecutionTarget("dataset", "warehouse"), new EntitySchema(EVENTS, ENTITY_ID), concept, true, @@ -61,7 +60,6 @@ void shouldCreateResolvedQueryWithoutFrameworkTypes() { assertValid(query); new ResolvedQueryValidation(ValidationTestSupport.VALIDATOR).validate(query); assertEquals(concept, query.root()); - assertEquals("warehouse", query.target().dataSource()); } @Test @@ -123,7 +121,6 @@ void shouldValidateTheCompleteQueryGraphAtTheCompilerBoundary() { List.of() ); ResolvedQuery query = new ResolvedQuery( - new ExecutionTarget("dataset", "warehouse"), new EntitySchema(EVENTS, ENTITY_ID), new ConceptNode("concept", List.of(invalidConnector), List.of(), DateAggregationAction.MERGE), true, @@ -139,7 +136,6 @@ void shouldValidateTheCompleteQueryGraphAtTheCompilerBoundary() { @Test void shouldRejectAmbiguousResultColumnIds() { ResolvedQuery query = new ResolvedQuery( - new ExecutionTarget("dataset", "warehouse"), new EntitySchema(EVENTS, ENTITY_ID), conceptNode(EVENTS), false, From 2f0fba78fa51567a4667451c789584e7f6e5ecde Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 1 Sep 2026 10:48:47 +0200 Subject: [PATCH 10/81] refactor(sql): move select IR contracts into connector Move the qualification and select intermediate-representation contracts into sql-connector so compiler transformations no longer depend on backend-owned types. Update the legacy backend to consume the connector contracts. Co-authored-by: Codex GPT-5.6-sol --- .../specific/MappableSingleColumnSelect.java | 2 +- .../cqelement/CQExternalConverter.java | 2 +- .../aggregation/AnsiSqlDateAggregator.java | 2 +- .../aggregation/DateAggregationContext.java | 4 +- .../aggregation/DateAggregationDates.java | 2 +- .../aggregation/IntermediateTableCte.java | 2 +- .../IntersectAggregationAction.java | 2 +- .../aggregation/MergeAggregateAction.java | 2 +- .../cqelement/aggregation/RowNumberCte.java | 2 +- .../aggregation/SqlAggregationAction.java | 2 +- .../concept/AggregationFilterCte.java | 2 +- .../concept/AggregationSelectCte.java | 2 +- .../cqelement/concept/CQConceptConverter.java | 2 +- .../concept/IntervalPackingSelectsCte.java | 2 +- .../cqelement/concept/JoinBranchesCte.java | 2 +- .../cqelement/concept/PreprocessingCte.java | 2 +- .../intervalpacking/AggregationMode.java | 2 +- .../AnsiSqlIntervalPacker.java | 2 +- .../IntervalPackingContext.java | 2 +- .../conversion/dialect/IntervalPacker.java | 2 +- .../conversion/dialect/SqlDateAggregator.java | 2 +- .../ClickhouseDistinctSelectConverter.java | 2 +- .../sql/conversion/model/ColumnDateRange.java | 2 +- .../sql/conversion/model/Qualifiable.java | 7 --- .../sql/conversion/model/QualifyingUtil.java | 2 +- .../sql/conversion/model/QueryStep.java | 2 +- .../sql/conversion/model/QueryStepJoiner.java | 2 +- .../sql/conversion/model/Selects.java | 2 +- .../sql/conversion/model/SqlIdColumns.java | 1 + .../aggregator/CommonAggregationSelect.java | 4 +- .../model/aggregator/FlagSqlAggregator.java | 2 +- .../model/aggregator/SumSqlAggregator.java | 2 +- .../model/select/ConceptSqlSelects.java | 1 + .../model/select/ConnectorSqlSelects.java | 1 + .../model/select/DaterangeSelectUtil.java | 1 + .../model/select/DistinctSelectConverter.java | 1 + .../model/select/ExistsSqlSelect.java | 2 + .../model/select/ExtractingSqlSelect.java | 1 + .../conversion/model/select/FieldWrapper.java | 1 + .../model/select/SingleColumnSqlSelect.java | 29 ----------- .../conversion/model/select/SqlSelect.java | 49 ------------------- .../model/select/ValueSelectUtil.java | 2 + .../query/ConceptQueryConverter.java | 2 +- .../conquery/sql/compiler/ir/Qualifiable.java | 7 +++ .../sql/compiler/ir/package-info.java | 2 + .../ir/select/SingleColumnSqlSelect.java | 28 +++++++++++ .../sql/compiler/ir/select/SqlSelect.java | 38 ++++++++++++++ .../sql/compiler/ir/select/package-info.java | 2 + 48 files changed, 121 insertions(+), 118 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Qualifiable.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SingleColumnSqlSelect.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/Qualifiable.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/package-info.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/SingleColumnSqlSelect.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/SqlSelect.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/package-info.java diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java index 8900fa8d2f..5f4dd5e91f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java @@ -28,7 +28,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; -import com.bakdata.conquery.sql.conversion.model.select.SingleColumnSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java index e06ed92761..bb6b95bd30 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java @@ -19,7 +19,7 @@ import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.google.common.base.Preconditions; import org.jooq.Field; import org.jooq.Name; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java index 8656cd9bf4..98581b0e2c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java @@ -17,7 +17,7 @@ import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlTables; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Data; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java index f00fe6d120..3add526456 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java @@ -8,12 +8,12 @@ import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; -import com.bakdata.conquery.sql.conversion.model.Qualifiable; +import com.bakdata.conquery.sql.compiler.ir.Qualifiable; import com.bakdata.conquery.sql.conversion.model.QualifyingUtil; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Builder; import lombok.Value; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java index c3c7e708e6..fc24b5d182 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java @@ -10,7 +10,7 @@ import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java index d7ccea9966..0aeb233c68 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Getter; import org.jooq.Condition; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java index 70ee2023fb..bb0c0f650d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java @@ -12,7 +12,7 @@ import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.RequiredArgsConstructor; import org.jooq.Field; import org.jooq.impl.DSL; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java index d469015d41..8a22291e27 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java @@ -11,7 +11,7 @@ import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlTables; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.RequiredArgsConstructor; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java index 01c1dd5299..b9131625e2 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java @@ -8,7 +8,7 @@ import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Getter; import org.jooq.Field; import org.jooq.impl.DSL; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java index 5b98c52122..2825aeb2f4 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java @@ -8,7 +8,7 @@ import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlTables; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; /** * Represents a subset of {@link DateAggregationAction}. diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java index 85179b7043..9187b6c2f8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import org.jooq.Condition; class AggregationFilterCte extends ConnectorCte { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java index 26ae3c6772..5d89475328 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java @@ -8,7 +8,7 @@ import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import org.jooq.Field; class AggregationSelectCte extends ConnectorCte { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index e43c830d81..302b2d09d0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -30,7 +30,7 @@ import com.bakdata.conquery.sql.conversion.model.select.ConceptSqlSelects; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.util.TablePrimaryColumnUtil; import com.google.common.base.Preconditions; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java index 77572d97b2..8f2d508069 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.conversion.model.select.ConceptSqlSelects; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.google.common.base.Preconditions; public class IntervalPackingSelectsCte { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java index 8a78d1dbd1..886826cb4b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.aggregator.SumSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import org.jooq.Record; import org.jooq.TableLike; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java index c31d3e6b51..5434578289 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java @@ -12,7 +12,7 @@ import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import org.jooq.Condition; import org.jooq.Record; import org.jooq.Table; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AggregationMode.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AggregationMode.java index 45e6f3b03f..d1797b23dd 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AggregationMode.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AggregationMode.java @@ -1,7 +1,7 @@ package com.bakdata.conquery.sql.conversion.cqelement.intervalpacking; import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; /** * Determines if the date that is getting interval-packed is stored as validity date of {@link Selects} or as arbitrary {@link SqlSelect}. diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java index 178136ea1a..c9944bdcd6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.RequiredArgsConstructor; import org.jooq.Field; import org.jooq.impl.DSL; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java index ee0f7e45ab..6fad39405d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java @@ -12,7 +12,7 @@ import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Builder; import lombok.Value; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/IntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/IntervalPacker.java index 84652c01a6..508487ba34 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/IntervalPacker.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/IntervalPacker.java @@ -3,7 +3,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingContext; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; /** * Packing intervals involves packing groups of intersecting validity date intervals into their respective continuous intervals. diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java index 175ca894a5..225453ce14 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.aggregation.DateAggregationDates; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; public interface SqlDateAggregator { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java index fb6ddc89a4..5959d5c6f0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java @@ -10,7 +10,7 @@ import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; -import com.bakdata.conquery.sql.conversion.model.select.SingleColumnSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; public class ClickhouseDistinctSelectConverter implements SelectConverter { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java index f8679840ce..f2bb27ad46 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java @@ -5,7 +5,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Getter; import lombok.ToString; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Qualifiable.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Qualifiable.java deleted file mode 100644 index a5c54fa5e2..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Qualifiable.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model; - -public interface Qualifiable { - - T qualify(String qualifier); - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QualifyingUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QualifyingUtil.java index a4665c4957..599c81faf6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QualifyingUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QualifyingUtil.java @@ -3,7 +3,7 @@ import java.util.List; import java.util.stream.Collectors; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import org.jooq.Field; import org.jooq.impl.DSL; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java index 70d9d81279..7f1161fba3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java @@ -1,6 +1,6 @@ package com.bakdata.conquery.sql.conversion.model; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Builder; import lombok.Singular; import lombok.Value; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java index 6ab0db36ed..bbde527966 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java @@ -18,7 +18,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.aggregation.DateAggregationDates; import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.google.common.base.Preconditions; import org.jooq.Condition; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java index a13db9b221..22bce87cdb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java @@ -2,7 +2,7 @@ import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Builder; import lombok.Singular; import lombok.Value; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java index 99af726ccb..3b1b195393 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java @@ -10,6 +10,7 @@ import javax.annotation.Nullable; import com.bakdata.conquery.models.forms.util.Resolution; +import com.bakdata.conquery.sql.compiler.ir.Qualifiable; import com.bakdata.conquery.sql.conversion.SharedAliases; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CommonAggregationSelect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CommonAggregationSelect.java index 20661511b9..74860a3a44 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CommonAggregationSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CommonAggregationSelect.java @@ -8,8 +8,8 @@ import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; -import com.bakdata.conquery.sql.conversion.model.select.SingleColumnSqlSelect; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Builder; import lombok.Singular; import lombok.Value; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java index 5f975b3cbf..0d9fab1a42 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java @@ -26,7 +26,7 @@ import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; -import com.bakdata.conquery.sql.conversion.model.select.SingleColumnSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; import org.jooq.Condition; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java index 80c74df4fa..2e9e851d56 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java @@ -30,7 +30,7 @@ import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; -import com.bakdata.conquery.sql.conversion.model.select.SingleColumnSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; import lombok.Getter; import lombok.RequiredArgsConstructor; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptSqlSelects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptSqlSelects.java index df250611e7..65efbf98b1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptSqlSelects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptSqlSelects.java @@ -3,6 +3,7 @@ import java.util.List; import java.util.Optional; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.sql.conversion.model.QueryStep; import lombok.Builder; import lombok.Singular; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConnectorSqlSelects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConnectorSqlSelects.java index 4d360d62ec..22fe539410 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConnectorSqlSelects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConnectorSqlSelects.java @@ -3,6 +3,7 @@ import java.util.List; import java.util.Optional; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.sql.conversion.model.QueryStep; import lombok.Builder; import lombok.Singular; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java index c16a91dc76..8afe907ba9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java @@ -17,6 +17,7 @@ import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter; import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.identifiable.LabeledNamespaceIdentifiable; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java index c875bc4b32..6219fe229e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java @@ -8,6 +8,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.connector.DistinctSelect; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSqlSelect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSqlSelect.java index 2b7c01ef2b..1f95b87568 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSqlSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSqlSelect.java @@ -1,6 +1,8 @@ package com.bakdata.conquery.sql.conversion.model.select; import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExtractingSqlSelect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExtractingSqlSelect.java index 620c84efda..5968c71d41 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExtractingSqlSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExtractingSqlSelect.java @@ -2,6 +2,7 @@ import java.util.List; +import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; import lombok.Value; import org.jooq.Field; import org.jooq.impl.DSL; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FieldWrapper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FieldWrapper.java index 42a877a2e0..99f243c0f7 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FieldWrapper.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FieldWrapper.java @@ -2,6 +2,7 @@ import java.util.List; +import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; import lombok.EqualsAndHashCode; import org.jooq.Field; import org.jooq.impl.DSL; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SingleColumnSqlSelect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SingleColumnSqlSelect.java deleted file mode 100644 index 6004b196ee..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SingleColumnSqlSelect.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model.select; - -import java.util.List; - -import org.jooq.Field; - -public interface SingleColumnSqlSelect extends SqlSelect { - - /** - * @return The whole (aliased) SQL expression of this {@link SqlSelect}. - * For example, {@code DSL.firstValue(DSL.field(DSL.name("foo", "bar"))).as("foobar")}. - */ - Field select(); - - /** - * @return Aliased column name that can be used to reference the created select. - * For example, {@code DSL.field("foobar")}. - */ - Field aliased(); - - @Override - SingleColumnSqlSelect qualify(String qualifier); - - @Override - default List> toFields() { - return List.of(select()); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java deleted file mode 100644 index 08738c3d50..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model.select; - -import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; -import com.bakdata.conquery.sql.conversion.model.Qualifiable; -import org.jooq.Field; - -import java.util.List; - -public interface SqlSelect extends Qualifiable { - - List> toFields(); - - /** - * All column names this {@link SqlSelect} requires. - */ - List requiredColumns(); - - /** - * @return Determines if this SqlSelect is only part of the final concept conversion CTE and has no predeceasing selects. - */ - default boolean isUniversal() { - return false; - } - - /** - * Special selects like {@link ExistsSqlSelect} require an extra aggregation when joining them with other connectors. - */ - default SqlSelect connectorAggregate() { - return this; - } - - /** - * Special selects like {@link ExistsSqlSelect} require to be converted into a specific format before executing the - * final query. - */ - default SqlSelect toFinalRepresentation() { - return this; - } - - /** - * Aggregate this select to one value per ID group in the final concept query. - */ - default List> aggregateForFinalQuery(CompilerDialect dialect) { - return toFinalRepresentation().toFields().stream() - .>map(field -> dialect.anyValue(field).as(field.getName())) - .toList(); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java index e37d70b2a8..3d3405ffbd 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java @@ -9,6 +9,8 @@ import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java index bcd28f93c7..6c5c65674c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java @@ -10,7 +10,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.*; import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.RequiredArgsConstructor; import org.jooq.*; import org.jooq.Record; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/Qualifiable.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/Qualifiable.java new file mode 100644 index 0000000000..092dce642a --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/Qualifiable.java @@ -0,0 +1,7 @@ +package com.bakdata.conquery.sql.compiler.ir; + +/** An intermediate representation that can be rebound to a qualified SQL table or CTE name. */ +public interface Qualifiable { + + T qualify(String qualifier); +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/package-info.java new file mode 100644 index 0000000000..33e6fa2b60 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/package-info.java @@ -0,0 +1,2 @@ +/** Internal representations produced and transformed while compiling a resolved query. */ +package com.bakdata.conquery.sql.compiler.ir; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/SingleColumnSqlSelect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/SingleColumnSqlSelect.java new file mode 100644 index 0000000000..b75c2ece99 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/SingleColumnSqlSelect.java @@ -0,0 +1,28 @@ +package com.bakdata.conquery.sql.compiler.ir.select; + +import java.util.List; + +import org.jooq.Field; + +/** Intermediate select represented by one SQL field. */ +public interface SingleColumnSqlSelect extends SqlSelect { + + /** + * @return the complete aliased SQL expression, for example + * {@code DSL.firstValue(DSL.field(DSL.name("foo", "bar"))).as("foobar")} + */ + Field select(); + + /** + * @return the aliased column reference, for example {@code DSL.field("foobar")} + */ + Field aliased(); + + @Override + SingleColumnSqlSelect qualify(String qualifier); + + @Override + default List> toFields() { + return List.of(select()); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/SqlSelect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/SqlSelect.java new file mode 100644 index 0000000000..6c7f1a47c0 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/SqlSelect.java @@ -0,0 +1,38 @@ +package com.bakdata.conquery.sql.compiler.ir.select; + +import java.util.List; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.Qualifiable; +import org.jooq.Field; + +/** Intermediate representation of one or more SQL select expressions. */ +public interface SqlSelect extends Qualifiable { + + List> toFields(); + + /** All column names this select requires. */ + List requiredColumns(); + + /** Whether this select only occurs in the final concept CTE and has no preceding select. */ + default boolean isUniversal() { + return false; + } + + /** Apply an extra aggregation required when joining this select across connectors. */ + default SqlSelect connectorAggregate() { + return this; + } + + /** Convert this select into the representation returned by the final SQL query. */ + default SqlSelect toFinalRepresentation() { + return this; + } + + /** Aggregate this select to one value per ID group in the final concept query. */ + default List> aggregateForFinalQuery(CompilerDialect dialect) { + return toFinalRepresentation().toFields().stream() + .>map(field -> dialect.anyValue(field).as(field.getName())) + .toList(); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/package-info.java new file mode 100644 index 0000000000..65d930f995 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/package-info.java @@ -0,0 +1,2 @@ +/** SQL select expressions used by the compiler intermediate representation. */ +package com.bakdata.conquery.sql.compiler.ir.select; From 3bd0bad5c5fd1ead9beb7741cbb2a82c04d1ed58 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 1 Sep 2026 12:48:10 +0200 Subject: [PATCH 11/81] refactor(sql): move select IR implementations into connector Move the field-backed, extracting, and existence select representations into the connector-owned compiler IR. Use plain Java implementations, preserve the legacy equality behavior, and cover the relocated transformations with tests. Co-authored-by: Codex GPT-5.6-sol --- .../specific/MappableSingleColumnSelect.java | 2 +- .../conversion/cqelement/CQAndConverter.java | 2 +- .../cqelement/CQExternalConverter.java | 2 +- .../conversion/cqelement/CQOrConverter.java | 2 +- .../aggregation/DateAggregationDates.java | 2 +- .../IntersectAggregationAction.java | 2 +- .../cqelement/aggregation/RowNumberCte.java | 2 +- .../AnsiSqlIntervalPacker.java | 2 +- .../ClickhouseDistinctSelectConverter.java | 2 +- .../forms/AbsoluteStratification.java | 2 +- .../forms/RelativeStratification.java | 2 +- .../aggregator/CommonAggregationSelect.java | 2 +- .../CountQuartersSqlAggregator.java | 4 +- .../model/aggregator/CountSqlAggregator.java | 4 +- .../aggregator/DateDistanceSqlAggregator.java | 2 + .../model/aggregator/FlagSqlAggregator.java | 4 +- .../model/aggregator/SumSqlAggregator.java | 4 +- .../filter/AbstractSelectFilterConverter.java | 2 +- .../model/filter/NumberFilterConverter.java | 2 +- .../select/ConceptColumnSelectConverter.java | 2 + .../select/DateUnionSelectConverter.java | 1 + .../model/select/DaterangeSelectUtil.java | 2 + .../model/select/DistinctSelectConverter.java | 1 + .../select/EventDateUnionSelectConverter.java | 2 + .../EventDurationSumSelectConverter.java | 2 + .../model/select/ExistsSelectConverter.java | 1 + .../model/select/ExtractingSqlSelect.java | 49 -------------- .../conversion/model/select/FieldWrapper.java | 63 ------------------ .../select/RandomValueSelectConverter.java | 2 + .../model/select/ValueSelectUtil.java | 1 + .../query/ConceptQueryConverter.java | 2 +- .../query/FormConversionHelper.java | 2 +- .../query/TableExportQueryConverter.java | 2 +- .../sql/compiler/ir/package-info.java | 7 +- .../compiler/ir}/select/ExistsSqlSelect.java | 37 ++++++----- .../ir/select/ExtractingSqlSelect.java | 31 +++++++++ .../sql/compiler/ir/select/FieldWrapper.java | 66 +++++++++++++++++++ .../sql/compiler/ir/select/package-info.java | 2 +- .../select/SqlSelectImplementationsTest.java | 55 ++++++++++++++++ 39 files changed, 220 insertions(+), 156 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExtractingSqlSelect.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FieldWrapper.java rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/model => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/select/ExistsSqlSelect.java (63%) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/ExtractingSqlSelect.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/FieldWrapper.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/select/SqlSelectImplementationsTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java index 5f4dd5e91f..18bf21b494 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java @@ -26,7 +26,7 @@ import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; import com.bakdata.conquery.sql.execution.ResultSetProcessor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java index 6c8f98ced3..d70deb0efc 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java @@ -5,7 +5,7 @@ import com.bakdata.conquery.sql.conversion.model.ConqueryJoinType; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; -import com.bakdata.conquery.sql.conversion.model.select.ExistsSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.ExistsSqlSelect; public class CQAndConverter implements NodeConverter { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java index bb6b95bd30..05f47c7df0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java @@ -18,7 +18,7 @@ import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.google.common.base.Preconditions; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java index d77d52a207..594de48460 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java @@ -5,7 +5,7 @@ import com.bakdata.conquery.sql.conversion.model.ConqueryJoinType; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; -import com.bakdata.conquery.sql.conversion.model.select.ExistsSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.ExistsSqlSelect; public class CQOrConverter implements NodeConverter { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java index fc24b5d182..3b71007745 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.AccessLevel; import lombok.AllArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java index bb0c0f650d..f54bee36cb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java @@ -11,7 +11,7 @@ import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlTables; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.RequiredArgsConstructor; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java index b9131625e2..e159bafe0e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Getter; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java index c9944bdcd6..81230a0d05 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java @@ -12,7 +12,7 @@ import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.RequiredArgsConstructor; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java index 5959d5c6f0..1c6c77cf4d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java index 190aa2c512..4d4b8b7127 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java @@ -15,7 +15,7 @@ import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.google.common.base.Preconditions; import lombok.RequiredArgsConstructor; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java index 8731afabb7..3e75139cd9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java @@ -26,7 +26,7 @@ import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.google.common.base.Preconditions; import lombok.RequiredArgsConstructor; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CommonAggregationSelect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CommonAggregationSelect.java index 74860a3a44..c3d379b8fb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CommonAggregationSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CommonAggregationSelect.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Builder; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java index 07548d1ce6..cc530f2d81 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java @@ -20,8 +20,8 @@ import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; -import com.bakdata.conquery.sql.conversion.model.select.ExtractingSqlSelect; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java index b1bc5222e8..59bf61c842 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java @@ -12,8 +12,8 @@ import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; -import com.bakdata.conquery.sql.conversion.model.select.ExtractingSqlSelect; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import lombok.NoArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java index f540b01e41..ac66d083ed 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java @@ -5,6 +5,8 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.filters.specific.DateDistanceFilter; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DateDistanceSelect; +import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java index 0d9fab1a42..056f8c3ebb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java @@ -22,8 +22,8 @@ import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; -import com.bakdata.conquery.sql.conversion.model.select.ExtractingSqlSelect; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java index 2e9e851d56..df20b8a43f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java @@ -26,8 +26,8 @@ import com.bakdata.conquery.sql.conversion.model.filter.SumCondition; import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; -import com.bakdata.conquery.sql.conversion.model.select.ExtractingSqlSelect; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java index c43ae52ae9..ceae7497cf 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java @@ -5,7 +5,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; -import com.bakdata.conquery.sql.conversion.model.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import org.jooq.Condition; import org.jooq.Field; import org.jooq.impl.DSL; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java index f4e24084a2..5de7b72ab3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.model.NumberMapUtil; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; -import com.bakdata.conquery.sql.conversion.model.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import org.jooq.Condition; import org.jooq.Field; import org.jooq.impl.DSL; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java index ff06d0116e..9c305f2e21 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java @@ -8,6 +8,8 @@ import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.select.concept.ConceptColumnSelect; import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; +import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptIdMapping; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java index a03f853483..31f0c94bdb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.sql.conversion.model.select; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DateUnionSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; public class DateUnionSelectConverter implements SelectConverter { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java index 8afe907ba9..fc6f9be593 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java @@ -17,6 +17,8 @@ import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter; import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.identifiable.LabeledNamespaceIdentifiable; +import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java index 6219fe229e..f77223db97 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java @@ -8,6 +8,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.connector.DistinctSelect; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java index 73cd607ae0..c356537fcb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java @@ -1,6 +1,8 @@ package com.bakdata.conquery.sql.conversion.model.select; import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDateUnionSelect; +import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java index 7a9b21307b..0c60624d4e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java @@ -5,6 +5,8 @@ import java.time.temporal.ChronoUnit; import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDurationSumSelect; +import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java index f79444f66e..f4c0fbecd7 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.sql.conversion.model.select; import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.ExistsSelect; +import com.bakdata.conquery.sql.compiler.ir.select.ExistsSqlSelect; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExtractingSqlSelect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExtractingSqlSelect.java deleted file mode 100644 index 5968c71d41..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExtractingSqlSelect.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model.select; - -import java.util.List; - -import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; -import lombok.Value; -import org.jooq.Field; -import org.jooq.impl.DSL; - -/** - * Select that does nothing but reference an existing column. - *

- * This can be used if another select requires a column in a later step. - * - * @param type of column - */ -@Value -public class ExtractingSqlSelect implements SingleColumnSqlSelect { - - String table; - String column; - Class columnClass; - - @Override - public Field select() { - return DSL.field(DSL.name(table, column), columnClass); - } - - @Override - public Field aliased() { - return DSL.field(DSL.name(column), columnClass); - } - - @Override - public List requiredColumns() { - return List.of(column); - } - - @Override - public ExtractingSqlSelect qualify(String qualifier) { - Field aliased = aliased(); - return new ExtractingSqlSelect<>( - qualifier, - aliased.getName(), - aliased.getType() - ); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FieldWrapper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FieldWrapper.java deleted file mode 100644 index 99f243c0f7..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FieldWrapper.java +++ /dev/null @@ -1,63 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model.select; - -import java.util.List; - -import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; -import lombok.EqualsAndHashCode; -import org.jooq.Field; -import org.jooq.impl.DSL; - -/** - * Wrapper for a {@link Field}. - */ -@EqualsAndHashCode -public class FieldWrapper implements SingleColumnSqlSelect { - - private final Field field; - @EqualsAndHashCode.Exclude - private final List requiredColumns; - - /** - * @param field The field to wrap, e.g. {@code DSL.sum(DSL.field(DSL.name("foo", "bar"))).as("foo_bar")}; - * @param requiredColumns All columns this {@link FieldWrapper} requires in the previous CTE/table to be present. - */ - public FieldWrapper(Field field, String... requiredColumns) { - this.field = field; - this.requiredColumns = List.of(requiredColumns); - } - - /** - * @param field @param field The field to wrap, e.g. {@code DSL.field(DSL.name("fizz", "buzz"))).as("fizz_buzz")}; The column for - * {@link FieldWrapper#requiredColumns()} will be taken from the alias of the wrapped field. - */ - public FieldWrapper(Field field) { - this.field = field; - this.requiredColumns = List.of(field.getName()); - } - - @Override - public Field select() { - return field; - } - - @Override - public Field aliased() { - return DSL.field(DSL.name(field.getName()), field.getType()); - } - - @Override - public List requiredColumns() { - return this.requiredColumns; - } - - @Override - public ExtractingSqlSelect qualify(String qualifier) { - Field aliased = aliased(); - return new ExtractingSqlSelect<>( - qualifier, - aliased.getName(), - aliased.getType() - ); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java index 100c74327d..0356879936 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java @@ -1,6 +1,8 @@ package com.bakdata.conquery.sql.conversion.model.select; import com.bakdata.conquery.models.datasets.concepts.select.connector.RandomValueSelect; +import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java index 3d3405ffbd..8bcea25e6d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java @@ -9,6 +9,7 @@ import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java index 6c5c65674c..7171395693 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.*; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.RequiredArgsConstructor; import org.jooq.*; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java index 75b32f027b..16ffecf616 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java @@ -12,7 +12,7 @@ import com.bakdata.conquery.sql.conversion.forms.FormCteStep; import com.bakdata.conquery.sql.conversion.forms.FormType; import com.bakdata.conquery.sql.conversion.model.*; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.google.common.base.Preconditions; import lombok.RequiredArgsConstructor; import org.jooq.*; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java index 7c0c643e45..a6c764c706 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.forms.FormCteStep; import com.bakdata.conquery.sql.conversion.model.*; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.util.TablePrimaryColumnUtil; import com.google.common.base.Preconditions; import lombok.RequiredArgsConstructor; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/package-info.java index 33e6fa2b60..416eb14035 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/package-info.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/package-info.java @@ -1,2 +1,7 @@ -/** Internal representations produced and transformed while compiling a resolved query. */ +/** + * Internal representations produced and transformed while compiling a resolved query. + * + *

Types in this package and its subpackages are implementation details of {@code sql-connector}, not integration + * contracts for backends or plugins. Backend references may exist temporarily while the legacy compiler is migrated.

+ */ package com.bakdata.conquery.sql.compiler.ir; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSqlSelect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/ExistsSqlSelect.java similarity index 63% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSqlSelect.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/ExistsSqlSelect.java index 1f95b87568..a09b60e79a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSqlSelect.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/ExistsSqlSelect.java @@ -1,20 +1,18 @@ -package com.bakdata.conquery.sql.conversion.model.select; +package com.bakdata.conquery.sql.compiler.ir.select; -import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; -import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import org.jooq.Field; -import org.jooq.Name; +import static org.jooq.impl.DSL.coalesce; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.inline; +import static org.jooq.impl.DSL.max; +import static org.jooq.impl.DSL.name; -import java.util.Collections; import java.util.List; -import static org.jooq.impl.DSL.*; -import static org.jooq.impl.DSL.field; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import org.jooq.Field; +import org.jooq.Name; -@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +/** A single-column marker select indicating whether a row exists. */ public class ExistsSqlSelect implements SingleColumnSqlSelect { private static final Field EXISTS = inline(1); @@ -22,13 +20,18 @@ public class ExistsSqlSelect implements SingleColumnSqlSelect { private final Field exists; private final Name alias; - public static ExistsSqlSelect withAlias(final String alias) { + private ExistsSqlSelect(Field exists, Name alias) { + this.exists = exists; + this.alias = alias; + } + + public static ExistsSqlSelect withAlias(String alias) { return new ExistsSqlSelect(EXISTS.as(alias), name(alias)); } @Override public List requiredColumns() { - return Collections.emptyList(); + return List.of(); } @Override @@ -47,8 +50,8 @@ public Field aliased() { } @Override - public SingleColumnSqlSelect qualify(final String qualifier) { - final Field qualified = field(name(name(qualifier), alias), exists.getType()); + public SingleColumnSqlSelect qualify(String qualifier) { + Field qualified = field(name(name(qualifier), alias), exists.getType()); return new ExistsSqlSelect(qualified, alias); } @@ -64,7 +67,7 @@ public SqlSelect toFinalRepresentation() { @Override public List> aggregateForFinalQuery(CompilerDialect dialect) { - // We have to coalesce at the end of the query because full-outer-joins will create null values, which we want to avoid. + // Full outer joins can introduce null values, which an existence marker represents as zero. Field coalesced = coalesce(max(select()), inline(0)); return List.of(coalesced.as(alias)); } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/ExtractingSqlSelect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/ExtractingSqlSelect.java new file mode 100644 index 0000000000..15719254e1 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/ExtractingSqlSelect.java @@ -0,0 +1,31 @@ +package com.bakdata.conquery.sql.compiler.ir.select; + +import java.util.List; + +import org.jooq.Field; +import org.jooq.impl.DSL; + +/** A single-column select that references an existing column. */ +public record ExtractingSqlSelect(String table, String column, Class columnClass) implements SingleColumnSqlSelect { + + @Override + public Field select() { + return DSL.field(DSL.name(table, column), columnClass); + } + + @Override + public Field aliased() { + return DSL.field(DSL.name(column), columnClass); + } + + @Override + public List requiredColumns() { + return List.of(column); + } + + @Override + public ExtractingSqlSelect qualify(String qualifier) { + Field aliased = aliased(); + return new ExtractingSqlSelect<>(qualifier, aliased.getName(), aliased.getType()); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/FieldWrapper.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/FieldWrapper.java new file mode 100644 index 0000000000..c7cc871f02 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/FieldWrapper.java @@ -0,0 +1,66 @@ +package com.bakdata.conquery.sql.compiler.ir.select; + +import java.util.List; +import java.util.Objects; + +import org.jooq.Field; +import org.jooq.impl.DSL; + +/** A single-column select backed by an arbitrary jOOQ field expression. */ +public class FieldWrapper implements SingleColumnSqlSelect { + + private final Field field; + private final List requiredColumns; + + /** + * @param field field to wrap + * @param requiredColumns columns that must be present in the preceding table or CTE + */ + public FieldWrapper(Field field, String... requiredColumns) { + this.field = field; + this.requiredColumns = List.of(requiredColumns); + } + + /** Create a wrapper whose required column is derived from the field alias. */ + public FieldWrapper(Field field) { + this.field = field; + this.requiredColumns = List.of(field.getName()); + } + + @Override + public Field select() { + return field; + } + + @Override + public Field aliased() { + return DSL.field(DSL.name(field.getName()), field.getType()); + } + + @Override + public List requiredColumns() { + return requiredColumns; + } + + @Override + public ExtractingSqlSelect qualify(String qualifier) { + Field aliased = aliased(); + return new ExtractingSqlSelect<>(qualifier, aliased.getName(), aliased.getType()); + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (!(object instanceof FieldWrapper that)) { + return false; + } + return Objects.equals(field, that.field); + } + + @Override + public int hashCode() { + return Objects.hash(field); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/package-info.java index 65d930f995..b61bb2c072 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/package-info.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/package-info.java @@ -1,2 +1,2 @@ -/** SQL select expressions used by the compiler intermediate representation. */ +/** Internal SQL select expressions used while compiling a resolved query. */ package com.bakdata.conquery.sql.compiler.ir.select; diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/select/SqlSelectImplementationsTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/select/SqlSelectImplementationsTest.java new file mode 100644 index 0000000000..0aba547349 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/select/SqlSelectImplementationsTest.java @@ -0,0 +1,55 @@ +package com.bakdata.conquery.sql.compiler.ir.select; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; + +import org.jooq.Field; +import org.junit.jupiter.api.Test; + +class SqlSelectImplementationsTest { + + @Test + void shouldDeriveRequiredColumnFromFieldAlias() { + Field field = field(name("source"), Integer.class).as("result"); + + FieldWrapper select = new FieldWrapper<>(field); + + assertEquals(List.of("result"), select.requiredColumns()); + assertEquals("result", select.aliased().getName()); + } + + @Test + void shouldKeepFieldWrapperIdentityIndependentOfRequiredColumns() { + Field field = field(name("source"), Integer.class).as("result"); + + assertEquals(new FieldWrapper<>(field, "first"), new FieldWrapper<>(field, "second")); + } + + @Test + void shouldQualifyExtractingSelects() { + ExtractingSqlSelect select = new ExtractingSqlSelect<>("source", "result", Integer.class); + + ExtractingSqlSelect qualified = select.qualify("next_step"); + + assertEquals("next_step", qualified.table()); + assertEquals("result", qualified.column()); + assertEquals(Integer.class, qualified.columnClass()); + assertEquals(name("next_step", "result"), qualified.select().getQualifiedName()); + } + + @Test + void shouldRepresentExistenceAsUniversalSelect() { + ExistsSqlSelect select = ExistsSqlSelect.withAlias("present"); + + assertTrue(select.isUniversal()); + assertEquals(List.of(), select.requiredColumns()); + assertEquals("present", select.select().getName()); + assertEquals(name("next_step", "present"), select.qualify("next_step").select().getQualifiedName()); + assertEquals("present", select.connectorAggregate().toFields().getFirst().getName()); + assertEquals("present", select.toFinalRepresentation().toFields().getFirst().getName()); + } +} From 2abae33bef7075ae4370820f13c1df08430006ee Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 1 Sep 2026 13:26:07 +0200 Subject: [PATCH 12/81] refactor(sql): move date range IR into connector Move the two-column date range representation and field qualification utility into the connector-owned compiler IR. Replace Lombok-generated accessors with plain Java and cover qualification, aliasing, and coalescing behavior with tests. Co-authored-by: Codex GPT-5.6-sol --- .../cqelement/CQExternalConverter.java | 2 +- .../conversion/cqelement/CQYesConverter.java | 2 +- .../aggregation/AnsiSqlDateAggregator.java | 2 +- .../aggregation/DateAggregationContext.java | 2 +- .../aggregation/DateAggregationDates.java | 2 +- .../IntersectAggregationAction.java | 2 +- .../cqelement/aggregation/InvertCte.java | 4 +- .../aggregation/MergeAggregateAction.java | 2 +- .../aggregation/NodeNoOverlapCte.java | 2 +- .../cqelement/aggregation/OverlapCte.java | 2 +- .../cqelement/aggregation/RowNumberCte.java | 2 +- .../aggregation/SqlAggregationAction.java | 2 +- .../concept/AggregationSelectCte.java | 2 +- .../cqelement/concept/CQConceptConverter.java | 3 +- .../cqelement/concept/CQTableContext.java | 2 +- .../concept/IntervalPackingSelectsCte.java | 2 +- .../cqelement/concept/JoinBranchesCte.java | 2 +- .../cqelement/concept/PreprocessingCte.java | 2 +- .../AnsiSqlIntervalPacker.java | 4 +- .../IntervalPackingContext.java | 2 +- .../IntervalPackingResult.java | 2 +- .../PostgreSqlIntervalPacker.java | 2 +- .../conversion/dialect/SqlDateAggregator.java | 2 +- .../dialect/SqlFunctionProvider.java | 2 +- .../ClickhouseFunctionProvider.java | 2 +- .../ClickhouseStratificationFunctions.java | 2 +- .../dialect/hana/HanaSqlFunctionProvider.java | 2 +- .../hana/HanaStratificationFunctions.java | 2 +- .../forms/AbsoluteStratification.java | 2 +- .../forms/RelativeStratification.java | 2 +- .../forms/StratificationFunctions.java | 2 +- .../sql/conversion/model/ColumnDateRange.java | 99 ------------------- .../sql/conversion/model/QueryStepJoiner.java | 3 +- .../sql/conversion/model/Selects.java | 3 +- .../sql/conversion/model/SqlIdColumns.java | 1 + .../model/StratificationSqlIdColumns.java | 1 + .../aggregator/DateDistanceSqlAggregator.java | 2 +- .../aggregator/DurationSumSqlAggregator.java | 2 +- .../filter/DateRestrictionCondition.java | 2 +- .../model/select/DaterangeSelectUtil.java | 2 +- .../model/select/DistinctSelectConverter.java | 2 +- .../select/EventDateUnionSelectConverter.java | 2 +- .../EventDurationSumSelectConverter.java | 2 +- .../model/select/SelectContext.java | 2 +- .../model/select/ValueSelectUtil.java | 2 +- .../query/AbsoluteFormQueryConverter.java | 2 +- .../query/EntityDateQueryConverter.java | 2 +- .../query/FormConversionHelper.java | 3 +- .../query/TableExportQueryConverter.java | 3 +- .../sql/compiler/ir}/QualifyingUtil.java | 13 ++- .../compiler/ir/select/ColumnDateRange.java | 98 ++++++++++++++++++ .../sql/compiler/ir/QualifyingUtilTest.java | 31 ++++++ .../ir/select/ColumnDateRangeTest.java | 45 +++++++++ 53 files changed, 237 insertions(+), 152 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/model => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/QualifyingUtil.java (60%) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/ColumnDateRange.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/QualifyingUtilTest.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/select/ColumnDateRangeTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java index 05f47c7df0..2355bd13bd 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java @@ -14,7 +14,7 @@ import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java index f45cba7d1b..c5e36e3ee7 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.apiv1.query.CQYes; import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java index 98581b0e2c..6ff056e800 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java index 3add526456..3182e22641 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.compiler.ir.Qualifiable; -import com.bakdata.conquery.sql.conversion.model.QualifyingUtil; +import com.bakdata.conquery.sql.compiler.ir.QualifyingUtil; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java index 3b71007745..a53f27be61 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java @@ -7,7 +7,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java index f54bee36cb..1b0ff7c716 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java @@ -7,7 +7,7 @@ import java.util.stream.Stream; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java index abd998364e..c62ad2596c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java @@ -9,8 +9,8 @@ import java.util.stream.Stream; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QualifyingUtil; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.QualifyingUtil; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java index 8a22291e27..67acb1d5e3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java @@ -7,7 +7,7 @@ import java.util.stream.Stream; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java index 0da7709148..97a8e052b2 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java @@ -5,7 +5,7 @@ import java.util.List; import java.util.Optional; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java index 1d6790ea23..9ce61b8f23 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java @@ -5,7 +5,7 @@ import java.util.Optional; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java index e159bafe0e..e945ae7214 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java @@ -3,7 +3,7 @@ import java.util.ArrayList; import java.util.Optional; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java index 2825aeb2f4..a9f7c4f172 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java @@ -4,7 +4,7 @@ import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java index 5d89475328..645939926b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java @@ -4,7 +4,7 @@ import java.util.Optional; import java.util.stream.Stream; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 302b2d09d0..7765cce0a7 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -19,6 +19,8 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ConceptElementId; import com.bakdata.conquery.models.identifiable.ids.specific.SelectId; import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; @@ -30,7 +32,6 @@ import com.bakdata.conquery.sql.conversion.model.select.ConceptSqlSelects; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.util.TablePrimaryColumnUtil; import com.google.common.base.Preconditions; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java index 71199d2e0c..18801d2d48 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java index 8f2d508069..107c286c31 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java @@ -8,7 +8,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java index 886826cb4b..e740eb70bd 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingContext; import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.ConqueryJoinType; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java index 5434578289..c61d03465b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java index 81230a0d05..3664de8ec5 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java @@ -7,8 +7,8 @@ import java.util.Optional; import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QualifyingUtil; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.QualifyingUtil; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java index 6fad39405d..c0ae679ea1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java @@ -8,7 +8,7 @@ import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingResult.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingResult.java index c8e71a3166..68134c0dc7 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingResult.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingResult.java @@ -1,6 +1,6 @@ package com.bakdata.conquery.sql.conversion.cqelement.intervalpacking; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import lombok.Value; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java index 8fae49bdb8..d7de1cc0b0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java @@ -4,7 +4,7 @@ import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java index 225453ce14..07a4ab1334 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java @@ -5,7 +5,7 @@ import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.cqelement.aggregation.DateAggregationDates; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index 0a2b1552fd..980c7103fb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -17,7 +17,7 @@ import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; import com.bakdata.conquery.sql.conversion.SharedAliases; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java index 0131145f01..d7dde0cb09 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter; import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import org.jetbrains.annotations.NotNull; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java index 3126dcae20..1066555a3f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.forms.Offset; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import lombok.Getter; import lombok.RequiredArgsConstructor; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java index cc2fdb4a7e..a885d8c478 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java @@ -16,7 +16,7 @@ import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter; import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import org.jetbrains.annotations.NotNull; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java index 2bae80d371..cc82c62fee 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.sql.conversion.dialect.Interval; import com.bakdata.conquery.sql.conversion.forms.Offset; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import lombok.Getter; import lombok.RequiredArgsConstructor; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java index 4d4b8b7127..afdc10ab9f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java @@ -11,7 +11,7 @@ import com.bakdata.conquery.models.forms.util.Resolution; import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java index 3e75139cd9..4a64fe6d6c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java @@ -22,7 +22,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.Interval; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java index 1a2478b4f1..1cd9d390a1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java @@ -29,7 +29,7 @@ import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.dialect.Interval; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import org.jooq.Condition; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java deleted file mode 100644 index f2bb27ad46..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java +++ /dev/null @@ -1,99 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model; - -import java.sql.Date; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import lombok.Getter; -import lombok.ToString; -import org.jooq.Condition; -import org.jooq.Field; -import org.jooq.impl.DSL; - -//TODO split this class up into Dialect specific versions. -@Getter -@ToString(onlyExplicitlyIncluded = true) -public class ColumnDateRange implements SqlSelect { - - private static final String VALIDITY_DATE_COLUMN_NAME_SUFFIX = "_validity_date"; - private static final String START_SUFFIX = "_start"; - private static final String END_SUFFIX = "_end"; - - @ToString.Include - private final Field start; - @ToString.Include - private final Field end; - private final String alias; - - protected ColumnDateRange(Field startColumn, Field endColumn, String alias) { - start = startColumn; - end = endColumn; - this.alias = alias; - } - - public static ColumnDateRange of(Field startColumn, Field endColumn) { - return new ColumnDateRange(startColumn, endColumn, ""); - } - - public static ColumnDateRange of(Field startColumn, Field endColumn, String alias) { - return new ColumnDateRange(startColumn, endColumn, alias); - } - - - public ColumnDateRange asValidityDateRange(String alias) { - return as(alias + VALIDITY_DATE_COLUMN_NAME_SUFFIX); - } - - @Override - public List> toFields() { - return Stream.of(start, end) - .collect(Collectors.toList()); - } - - @Override - public ColumnDateRange qualify(String qualifier) { - return new ColumnDateRange( - QualifyingUtil.qualify(getStart(), qualifier), - QualifyingUtil.qualify(getEnd(), qualifier), - getAlias() - ); - } - - @Override - public List requiredColumns() { - return toFields().stream() - .map(Field::getName) - .distinct() - .toList(); - } - - public ColumnDateRange as(String alias) { - return new ColumnDateRange( - start.as(alias + START_SUFFIX), - end.as(alias + END_SUFFIX), - alias - ); - } - - public ColumnDateRange coalesce(ColumnDateRange right) { - return ColumnDateRange.of( - DSL.coalesce(start, right.getStart()), - DSL.coalesce(end, right.getEnd()) - ).as(alias); - } - - public Condition join(ColumnDateRange right) { - return start.eq(right.getStart()).and(end.eq(right.getEnd())); - } - - public Condition isNotNull() { - return start.isNotNull().and(end.isNotNull()); - } - - public static Condition isNotEmpty(ColumnDateRange columnDateRange) { - return columnDateRange.getStart().isNotNull().and(columnDateRange.getEnd().isNotNull()); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java index bbde527966..104f736772 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java @@ -13,12 +13,13 @@ import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.models.config.IdColumnConfig; import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.cqelement.aggregation.DateAggregationDates; import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.google.common.base.Preconditions; import org.jooq.Condition; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java index 22bce87cdb..f83f8399c2 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java @@ -1,8 +1,9 @@ package com.bakdata.conquery.sql.conversion.model; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Builder; import lombok.Singular; import lombok.Value; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java index 3b1b195393..8810aa9be3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.models.forms.util.Resolution; import com.bakdata.conquery.sql.compiler.ir.Qualifiable; +import com.bakdata.conquery.sql.compiler.ir.QualifyingUtil; import com.bakdata.conquery.sql.conversion.SharedAliases; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java index 678afefab5..7bf3611b56 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java @@ -10,6 +10,7 @@ import javax.annotation.Nullable; import com.bakdata.conquery.models.forms.util.Resolution; +import com.bakdata.conquery.sql.compiler.ir.QualifyingUtil; import com.bakdata.conquery.sql.conversion.SharedAliases; import com.google.common.base.Preconditions; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java index ac66d083ed..3c8518d87e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java @@ -12,7 +12,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.*; import com.bakdata.conquery.sql.conversion.model.select.*; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DurationSumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DurationSumSqlAggregator.java index 93f9c41151..77cf07f83c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DurationSumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DurationSumSqlAggregator.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.filter.DateDistanceCondition; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/DateRestrictionCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/DateRestrictionCondition.java index 4a03e0dc94..387bd24608 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/DateRestrictionCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/DateRestrictionCondition.java @@ -1,7 +1,7 @@ package com.bakdata.conquery.sql.conversion.model.filter; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import lombok.RequiredArgsConstructor; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java index fc6f9be593..e4d08205f8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java @@ -29,7 +29,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.CteStep; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java index f77223db97..84e45f7488 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.CteStep; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java index c356537fcb..0163c6d94b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.google.common.base.Preconditions; public class EventDateUnionSelectConverter implements SelectConverter { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java index 0c60624d4e..34be5bac11 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java @@ -11,7 +11,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.google.common.base.Preconditions; import org.jooq.Condition; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java index 446bc81397..973e441411 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java @@ -8,7 +8,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; import lombok.AccessLevel; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java index 8bcea25e6d..62f963243c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java @@ -15,7 +15,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.CteStep; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java index 7afb49e42a..6cfaf36f83 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java @@ -12,7 +12,7 @@ import com.bakdata.conquery.sql.conversion.forms.FormCteStep; import com.bakdata.conquery.sql.conversion.forms.FormType; import com.bakdata.conquery.sql.conversion.forms.StratificationTableFactory; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java index 71bb59a32c..ed8590d683 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java @@ -11,7 +11,7 @@ import com.bakdata.conquery.sql.conversion.forms.FormCteStep; import com.bakdata.conquery.sql.conversion.forms.FormType; import com.bakdata.conquery.sql.conversion.forms.StratificationTableFactory; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.google.common.base.Preconditions; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java index 16ffecf616..3899f2466f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java @@ -6,13 +6,14 @@ import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.forms.FormCteStep; import com.bakdata.conquery.sql.conversion.forms.FormType; import com.bakdata.conquery.sql.conversion.model.*; -import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.google.common.base.Preconditions; import lombok.RequiredArgsConstructor; import org.jooq.*; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java index a6c764c706..9765000136 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java @@ -7,13 +7,14 @@ import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.forms.FormCteStep; import com.bakdata.conquery.sql.conversion.model.*; -import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.util.TablePrimaryColumnUtil; import com.google.common.base.Preconditions; import lombok.RequiredArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QualifyingUtil.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QualifyingUtil.java similarity index 60% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QualifyingUtil.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QualifyingUtil.java index 599c81faf6..6c593d3253 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QualifyingUtil.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QualifyingUtil.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.model; +package com.bakdata.conquery.sql.compiler.ir; import java.util.List; import java.util.stream.Collectors; @@ -7,7 +7,11 @@ import org.jooq.Field; import org.jooq.impl.DSL; -public class QualifyingUtil { +/** Utilities for rebinding compiler IR fields and selects to a table or CTE qualifier. */ +public final class QualifyingUtil { + + private QualifyingUtil() { + } public static Field qualify(Field field, String qualifier) { return DSL.field(DSL.name(qualifier, field.getName()), field.getType()); @@ -15,8 +19,7 @@ public static Field qualify(Field field, String qualifier) { public static List qualify(List sqlSelects, String qualifier) { return sqlSelects.stream() - .map(sqlSelect -> sqlSelect.qualify(qualifier)) - .collect(Collectors.toList()); + .map(sqlSelect -> sqlSelect.qualify(qualifier)) + .collect(Collectors.toList()); } - } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/ColumnDateRange.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/ColumnDateRange.java new file mode 100644 index 0000000000..977b4508f3 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/select/ColumnDateRange.java @@ -0,0 +1,98 @@ +package com.bakdata.conquery.sql.compiler.ir.select; + +import java.sql.Date; +import java.util.List; + +import com.bakdata.conquery.sql.compiler.ir.QualifyingUtil; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.impl.DSL; + +/** A two-column date-range representation used by compiler transformations. */ +public final class ColumnDateRange implements SqlSelect { + + private static final String VALIDITY_DATE_COLUMN_NAME_SUFFIX = "_validity_date"; + private static final String START_SUFFIX = "_start"; + private static final String END_SUFFIX = "_end"; + + private final Field start; + private final Field end; + private final String alias; + + private ColumnDateRange(Field start, Field end, String alias) { + this.start = start; + this.end = end; + this.alias = alias; + } + + public static ColumnDateRange of(Field start, Field end) { + return new ColumnDateRange(start, end, ""); + } + + public static ColumnDateRange of(Field start, Field end, String alias) { + return new ColumnDateRange(start, end, alias); + } + + public Field getStart() { + return start; + } + + public Field getEnd() { + return end; + } + + public String getAlias() { + return alias; + } + + public ColumnDateRange asValidityDateRange(String alias) { + return as(alias + VALIDITY_DATE_COLUMN_NAME_SUFFIX); + } + + @Override + public List> toFields() { + return List.of(start, end); + } + + @Override + public ColumnDateRange qualify(String qualifier) { + return new ColumnDateRange( + QualifyingUtil.qualify(start, qualifier), + QualifyingUtil.qualify(end, qualifier), + alias + ); + } + + @Override + public List requiredColumns() { + return toFields().stream() + .map(Field::getName) + .distinct() + .toList(); + } + + public ColumnDateRange as(String alias) { + return new ColumnDateRange(start.as(alias + START_SUFFIX), end.as(alias + END_SUFFIX), alias); + } + + public ColumnDateRange coalesce(ColumnDateRange right) { + return ColumnDateRange.of(DSL.coalesce(start, right.start), DSL.coalesce(end, right.end)).as(alias); + } + + public Condition join(ColumnDateRange right) { + return start.eq(right.start).and(end.eq(right.end)); + } + + public Condition isNotNull() { + return start.isNotNull().and(end.isNotNull()); + } + + public static Condition isNotEmpty(ColumnDateRange columnDateRange) { + return columnDateRange.start.isNotNull().and(columnDateRange.end.isNotNull()); + } + + @Override + public String toString() { + return "ColumnDateRange(start=%s, end=%s)".formatted(start, end); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/QualifyingUtilTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/QualifyingUtilTest.java new file mode 100644 index 0000000000..02e8f5484e --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/QualifyingUtilTest.java @@ -0,0 +1,31 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; + +import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import org.junit.jupiter.api.Test; + +class QualifyingUtilTest { + + @Test + void shouldQualifyFieldsWithoutChangingTheirType() { + var qualified = QualifyingUtil.qualify(field(name("result"), Integer.class), "query_step"); + + assertEquals(name("query_step", "result"), qualified.getQualifiedName()); + assertEquals(Integer.class, qualified.getType()); + } + + @Test + void shouldQualifySelectLists() { + SqlSelect select = new ExtractingSqlSelect<>("source", "result", Integer.class); + + List qualified = QualifyingUtil.qualify(List.of(select), "query_step"); + + assertEquals(name("query_step", "result"), qualified.getFirst().toFields().getFirst().getQualifiedName()); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/select/ColumnDateRangeTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/select/ColumnDateRangeTest.java new file mode 100644 index 0000000000..7248bff712 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/select/ColumnDateRangeTest.java @@ -0,0 +1,45 @@ +package com.bakdata.conquery.sql.compiler.ir.select; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.sql.Date; +import java.util.List; + +import org.junit.jupiter.api.Test; + +class ColumnDateRangeTest { + + @Test + void shouldAliasValidityDateColumns() { + ColumnDateRange range = range().asValidityDateRange("event"); + + assertEquals("event_validity_date", range.getAlias()); + assertEquals("event_validity_date_start", range.getStart().getName()); + assertEquals("event_validity_date_end", range.getEnd().getName()); + assertEquals(List.of("event_validity_date_start", "event_validity_date_end"), range.requiredColumns()); + } + + @Test + void shouldQualifyAliasedDateRangeColumns() { + ColumnDateRange qualified = range().as("event").qualify("query_step"); + + assertEquals(name("query_step", "event_start"), qualified.getStart().getQualifiedName()); + assertEquals(name("query_step", "event_end"), qualified.getEnd().getQualifiedName()); + assertEquals("event", qualified.getAlias()); + } + + @Test + void shouldPreserveLeftAliasWhenCoalescing() { + ColumnDateRange coalesced = range().as("event").coalesce(range().as("fallback")); + + assertEquals("event", coalesced.getAlias()); + assertEquals("event_start", coalesced.getStart().getName()); + assertEquals("event_end", coalesced.getEnd().getName()); + } + + private static ColumnDateRange range() { + return ColumnDateRange.of(field(name("start"), Date.class), field(name("end"), Date.class)); + } +} From 04aea92c593a2907ba7dccbcb526a9e73bb0d235 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 1 Sep 2026 13:59:12 +0200 Subject: [PATCH 13/81] refactor(sql): move ID column IR into connector Move shared compiler aliases and plain and stratified ID-column representations into the connector-owned IR. Normalize backend resolution enums before crossing the boundary and cover aliasing, qualification, and stratification with tests. Co-authored-by: Codex GPT-5.6-sol --- .../query/concept/filter/FilterValue.java | 2 +- .../mode/local/SqlEntityResolver.java | 2 +- .../cqelement/CQExternalConverter.java | 4 +- .../conversion/cqelement/CQYesConverter.java | 2 +- .../aggregation/DateAggregationContext.java | 2 +- .../cqelement/aggregation/InvertCte.java | 2 +- .../cqelement/aggregation/RowNumberCte.java | 2 +- .../concept/AggregationSelectCte.java | 2 +- .../cqelement/concept/CQConceptConverter.java | 1 + .../cqelement/concept/CQTableContext.java | 2 +- .../cqelement/concept/FilterContext.java | 2 +- .../cqelement/concept/JoinBranchesCte.java | 2 +- .../cqelement/concept/PreprocessingCte.java | 2 +- .../AnsiSqlIntervalPacker.java | 2 +- .../IntervalPackingContext.java | 2 +- .../PostgreSqlIntervalPacker.java | 2 +- .../dialect/SqlFunctionProvider.java | 2 +- .../forms/AbsoluteStratification.java | 8 +- .../sql/conversion/forms/FormConstants.java | 2 +- .../forms/RelativeStratification.java | 10 +- .../forms/StratificationFunctions.java | 4 +- .../sql/conversion/model/QueryStepJoiner.java | 3 +- .../sql/conversion/model/Selects.java | 3 +- .../sql/conversion/model/SqlIdColumns.java | 165 ----------------- .../model/StratificationSqlIdColumns.java | 167 ------------------ .../model/aggregator/SumSqlAggregator.java | 2 +- .../select/ConceptColumnSelectConverter.java | 2 +- .../model/select/DaterangeSelectUtil.java | 2 +- .../model/select/DistinctSelectConverter.java | 2 +- .../model/select/SelectContext.java | 2 +- .../model/select/ValueSelectUtil.java | 2 +- .../query/AbsoluteFormQueryConverter.java | 4 +- .../query/EntityDateQueryConverter.java | 2 +- .../query/FormConversionHelper.java | 6 +- .../query/TableExportQueryConverter.java | 3 +- .../sql/compiler/ir}/SharedAliases.java | 18 +- .../sql/compiler/ir/SqlIdColumns.java | 137 ++++++++++++++ .../ir/StratificationSqlIdColumns.java | 113 ++++++++++++ .../sql/compiler/ir/SqlIdColumnsTest.java | 71 ++++++++ 39 files changed, 380 insertions(+), 383 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java rename {backend/src/main/java/com/bakdata/conquery/sql/conversion => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/SharedAliases.java (76%) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/SqlIdColumns.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/StratificationSqlIdColumns.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/SqlIdColumnsTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java index 8e597eb813..2e8ecfe384 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java @@ -18,7 +18,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java index 8768f24c98..92b5aafbcc 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java @@ -17,7 +17,7 @@ import com.bakdata.conquery.models.config.IdColumnConfig; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.identifiable.mapping.ExternalId; -import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; import com.bakdata.conquery.sql.execution.SqlExecutionService; import com.bakdata.conquery.util.DateReader; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java index 2355bd13bd..853ca74f6b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java @@ -12,12 +12,12 @@ import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.google.common.base.Preconditions; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java index c5e36e3ee7..0a59e4a3ca 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java @@ -10,7 +10,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import org.jooq.Field; import org.jooq.Record; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java index 3182e22641..f6c462841d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java @@ -11,7 +11,7 @@ import com.bakdata.conquery.sql.compiler.ir.Qualifiable; import com.bakdata.conquery.sql.compiler.ir.QualifyingUtil; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Builder; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java index c62ad2596c..49a8483c27 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.sql.compiler.ir.QualifyingUtil; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import lombok.Getter; import org.jooq.Condition; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java index e945ae7214..55bd70ec1e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java index 645939926b..39c1a878b0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 7765cce0a7..ae79cb7394 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -19,6 +19,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ConceptElementId; import com.bakdata.conquery.models.identifiable.ids.specific.SelectId; import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.sql.conversion.NodeConverter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java index 18801d2d48..22408801b3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java @@ -11,7 +11,7 @@ import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; import lombok.Builder; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java index ac0a1eeb5d..9108d93013 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Value; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java index e740eb70bd..dbda348b52 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java @@ -11,7 +11,7 @@ import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.aggregator.SumSqlAggregator; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import org.jooq.Record; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java index c61d03465b..4570747992 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java index 3664de8ec5..fa3403e26b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java @@ -11,7 +11,7 @@ import com.bakdata.conquery.sql.compiler.ir.QualifyingUtil; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.RequiredArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java index c0ae679ea1..a869876b32 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java @@ -10,7 +10,7 @@ import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Builder; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java index d7de1cc0b0..d7135cbf72 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import lombok.RequiredArgsConstructor; /** diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index 980c7103fb..d6a0871129 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -16,7 +16,7 @@ import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter; import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; -import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.execution.ResultSetProcessor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java index afdc10ab9f..5e3df52c20 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java @@ -9,12 +9,12 @@ import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; import com.bakdata.conquery.models.forms.util.Resolution; -import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.google.common.base.Preconditions; import lombok.RequiredArgsConstructor; @@ -118,7 +118,7 @@ private QueryStep createCompleteTable() { // complete range shall have a null index because it spans the complete range, but we set it to 1 to ensure we can join tables on index, // because a condition involving null in a join (e.g., null = some_value or null = null) always evaluates to false Field index = field(inline(1)).as(SharedAliases.INDEX.getAlias()); - SqlIdColumns ids = baseStepSelects.getIds().withAbsoluteStratification(Resolution.COMPLETE, index); + SqlIdColumns ids = baseStepSelects.getIds().withStratification(Resolution.COMPLETE.name(), index); ColumnDateRange completeRange = baseStepSelects.getStratificationDate().get(); @@ -146,7 +146,7 @@ private QueryStep createIntervalTable(QueryStep indexStartStep, ExportForm.Resol ); Field index = stratificationFunctions.index(countsCteSelects.getIds(), countsCte.getQualifiedSelects().getStratificationDate()); - SqlIdColumns ids = countsCteSelects.getIds().withAbsoluteStratification(resolutionAndAlignment.getResolution(), index); + SqlIdColumns ids = countsCteSelects.getIds().withStratification(resolutionAndAlignment.getResolution().name(), index); Selects selects = Selects.builder() .ids(ids) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormConstants.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormConstants.java index e51fe3e87f..38984a7cb3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormConstants.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormConstants.java @@ -5,7 +5,7 @@ import com.bakdata.conquery.apiv1.query.TemporalSamplerFactory; import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery; import com.bakdata.conquery.models.forms.managed.EntityDateQuery; -import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import org.jooq.Field; import org.jooq.impl.DSL; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java index 4a64fe6d6c..cbe0cc53bb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java @@ -18,14 +18,14 @@ import com.bakdata.conquery.models.forms.managed.RelativeFormQuery; import com.bakdata.conquery.models.forms.util.CalendarUnit; import com.bakdata.conquery.models.forms.util.Resolution; -import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.Interval; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.google.common.base.Preconditions; import lombok.RequiredArgsConstructor; @@ -169,14 +169,14 @@ private QueryStep createCompleteTable(QueryStep totalBoundsStep, RelativeFormQue private QueryStep createCompleteFeatureTable(Selects predecessorSelects, Interval interval, Range intRange, QueryStep totalBoundsStep) { Field featureIndex = DSL.field(DSL.inline(-1)).as(SharedAliases.INDEX.getAlias()); - SqlIdColumns featureIds = predecessorSelects.getIds().withRelativeStratification(Resolution.COMPLETE, featureIndex, INDEX_SELECTOR); + SqlIdColumns featureIds = predecessorSelects.getIds().withStratification(Resolution.COMPLETE.name(), featureIndex, INDEX_SELECTOR); Field rangeStart = stratificationFunctions.shiftByInterval(INDEX_START_NEGATIVE, interval, DSL.inline(intRange.getMin()), Offset.NONE); return createIntervalStep(featureIds, rangeStart, INDEX_START_NEGATIVE, Optional.empty(), totalBoundsStep); } private QueryStep createCompleteOutcomeTable(Selects predecessorSelects, Interval interval, Range intRange, QueryStep totalBoundsStep) { Field outcomeIndex = DSL.field(DSL.inline(1)).as(SharedAliases.INDEX.getAlias()); - SqlIdColumns outcomeIds = predecessorSelects.getIds().withRelativeStratification(Resolution.COMPLETE, outcomeIndex, INDEX_SELECTOR); + SqlIdColumns outcomeIds = predecessorSelects.getIds().withStratification(Resolution.COMPLETE.name(), outcomeIndex, INDEX_SELECTOR); Field rangeEnd = stratificationFunctions.shiftByInterval(INDEX_START_POSITIVE, interval, DSL.inline(intRange.getMax()), Offset.NONE); return createIntervalStep(outcomeIds, INDEX_START_POSITIVE, rangeEnd, Optional.empty(), totalBoundsStep); } @@ -185,7 +185,7 @@ private QueryStep createIntervalTable(QueryStep totalBoundsStep, Resolution reso Field seriesIndex = stratificationFunctions.intSeriesField(); Selects predecessorSelects = totalBoundsStep.getQualifiedSelects(); - SqlIdColumns ids = predecessorSelects.getIds().withRelativeStratification(resolution, seriesIndex, INDEX_SELECTOR); + SqlIdColumns ids = predecessorSelects.getIds().withStratification(resolution.name(), seriesIndex, INDEX_SELECTOR); Interval interval = getInterval(form.getTimeUnit(), resolution); Range bounds = toGenerateSeriesBounds(form, resolution); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java index 1cd9d390a1..2b56a34f1e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java @@ -26,11 +26,11 @@ import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; import com.bakdata.conquery.apiv1.query.TemporalSamplerFactory; import com.bakdata.conquery.models.forms.util.CalendarUnit; -import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.conversion.dialect.Interval; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import org.jooq.Condition; import org.jooq.Field; import org.jooq.Record; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java index 104f736772..c456a95b7d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java @@ -13,9 +13,10 @@ import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.models.config.IdColumnConfig; import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.cqelement.aggregation.DateAggregationDates; import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java index f83f8399c2..dc072ad613 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java @@ -1,8 +1,9 @@ package com.bakdata.conquery.sql.conversion.model; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import lombok.Builder; import lombok.Singular; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java deleted file mode 100644 index 8810aa9be3..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java +++ /dev/null @@ -1,165 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model; - -import java.sql.Date; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import javax.annotation.Nullable; - -import com.bakdata.conquery.models.forms.util.Resolution; -import com.bakdata.conquery.sql.compiler.ir.Qualifiable; -import com.bakdata.conquery.sql.compiler.ir.QualifyingUtil; -import com.bakdata.conquery.sql.conversion.SharedAliases; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.ToString; -import lombok.experimental.SuperBuilder; -import org.jooq.Condition; -import org.jooq.Field; -import org.jooq.impl.DSL; - -@SuperBuilder -@AllArgsConstructor -@ToString -public class SqlIdColumns implements Qualifiable { - - @Getter - private final Field primaryColumn; - - @Nullable - private final Field secondaryId; - - @Nullable - @ToString.Exclude - private final SqlIdColumns predecessor; - - public SqlIdColumns(Field primaryColumn, Field secondaryId) { - this.primaryColumn = primaryColumn; - this.secondaryId = secondaryId; - this.predecessor = null; - } - - public SqlIdColumns(Field primaryColumn) { - this.primaryColumn = primaryColumn; - this.secondaryId = null; - this.predecessor = null; - } - - public SqlIdColumns withAlias() { - if (this.secondaryId == null) { - return new SqlIdColumns(this.primaryColumn.as(SharedAliases.PRIMARY_COLUMN.getAlias()), null, this); - } - return new SqlIdColumns( - this.primaryColumn.as(SharedAliases.PRIMARY_COLUMN.getAlias()), - this.secondaryId.as(SharedAliases.SECONDARY_ID.getAlias()), - this - ); - } - - @Override - public SqlIdColumns qualify(String qualifier) { - Field primaryColumn = QualifyingUtil.qualify(this.primaryColumn, qualifier); - if (secondaryId == null) { - return new SqlIdColumns(primaryColumn, null, this); - } - Field secondaryId = QualifyingUtil.qualify(this.secondaryId, qualifier); - return new SqlIdColumns(primaryColumn, secondaryId, this); - } - - public SqlIdColumns withAbsoluteStratification(Resolution resolution, Field index) { - Field resolutionField = DSL.inline(resolution.toString()).as(SharedAliases.RESOLUTION.getAlias()); - return StratificationSqlIdColumns.builder() - .primaryColumn(this.primaryColumn) - .secondaryId(this.secondaryId) - .resolution(resolutionField) - .index(index) - .eventDate(null) - .build(); - } - - public SqlIdColumns withRelativeStratification(Resolution resolution, Field index, Field eventDate) { - Field resolutionField = DSL.inline(resolution.toString()).as(SharedAliases.RESOLUTION.getAlias()); - return StratificationSqlIdColumns.builder() - .primaryColumn(this.primaryColumn) - .secondaryId(this.secondaryId) - .resolution(resolutionField) - .index(index) - .eventDate(eventDate) - .build(); - } - - public SqlIdColumns forFinalSelect() { - return this; - } - - public Optional> getSecondaryId() { - return Optional.ofNullable(this.secondaryId); - } - - public Optional getPredecessor() { - return Optional.ofNullable(this.predecessor); - } - - public boolean isWithStratification() { - return false; - } - - public List> toFields() { - if (getSecondaryId().isEmpty()){ - return List.of(getPrimaryColumn()); - } - - return List.of(getPrimaryColumn(), getSecondaryId().get()); - } - - public List join(SqlIdColumns rightIds) { - - // always join on primary columns - Condition joinPrimariesCondition = primaryColumn.eq(rightIds.getPrimaryColumn()); - - // join on secondary IDs if both are present - Condition joinSecondaries = getSecondaryId() - .flatMap(leftSecondaryId -> rightIds.getSecondaryId().map(leftSecondaryId::eq)) - .orElse(DSL.noCondition()); - - return List.of(joinPrimariesCondition, joinSecondaries); - } - - public SqlIdColumns coalesce(List selectsIds) { - - List> primaryColumns = new ArrayList<>(); - List> secondaryIds = new ArrayList<>(); - - // add this ids - primaryColumns.add(this.primaryColumn); - getSecondaryId().ifPresent(secondaryIds::add); - - // add all other ids to coalesce with - selectsIds.forEach(ids -> { - primaryColumns.add(ids.getPrimaryColumn()); - ids.getSecondaryId().ifPresent(secondaryIds::add); - }); - - Field coalescedPrimaryColumn = coalesceFields(primaryColumns, String.class).coerce(String.class).as(SharedAliases.PRIMARY_COLUMN.getAlias()); - if (secondaryIds.isEmpty()) { - return new SqlIdColumns(coalescedPrimaryColumn); - } - Field coalescedSecondaryIds = coalesceFields(secondaryIds, String.class).coerce(String.class).as(SharedAliases.SECONDARY_ID.getAlias()); - return new SqlIdColumns(coalescedPrimaryColumn, coalescedSecondaryIds); - } - - - protected static Field coalesceFields(List> fields, Class type) { - Field out = fields.getFirst().coerce(type); - - for (int index = 1; index < fields.size(); index++) { - out = DSL.coalesce(out, fields.get(index).coerce(type)); - } - - return out; - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java deleted file mode 100644 index 7bf3611b56..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java +++ /dev/null @@ -1,167 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model; - -import java.sql.Date; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import javax.annotation.Nullable; - -import com.bakdata.conquery.models.forms.util.Resolution; -import com.bakdata.conquery.sql.compiler.ir.QualifyingUtil; -import com.bakdata.conquery.sql.conversion.SharedAliases; -import com.google.common.base.Preconditions; -import lombok.Getter; -import lombok.experimental.SuperBuilder; -import org.jooq.Condition; -import org.jooq.Field; -import org.jooq.impl.DSL; - -@Getter -@SuperBuilder -class StratificationSqlIdColumns extends SqlIdColumns { - - private final Field resolution; - - private final Field index; - - /** - * Optional event date. Only set for relative forms. - */ - @Nullable - private final Field eventDate; - - @Override - public SqlIdColumns qualify(String qualifier) { - - Field primaryColumn = QualifyingUtil.qualify(getPrimaryColumn(), qualifier); - Field resolution = QualifyingUtil.qualify(this.resolution, qualifier); - Field index = QualifyingUtil.qualify(this.index, qualifier); - Field eventDate = null; - if (this.eventDate != null) { - eventDate = QualifyingUtil.qualify(this.eventDate, qualifier); - } - - return StratificationSqlIdColumns.builder() - .primaryColumn(primaryColumn) - .secondaryId(null) - .resolution(resolution) - .index(index) - .eventDate(eventDate) - .build(); - } - - /** - * Replaces the integer value of the {@link Resolution#COMPLETE} with null values. - */ - @Override - public SqlIdColumns forFinalSelect() { - - Field withNulledCompleteIndex = DSL.when( - this.resolution.eq(DSL.inline(Resolution.COMPLETE.toString().toUpperCase())), - DSL.inline(null, Integer.class) - ) - .otherwise(this.index) - .as(SharedAliases.INDEX.getAlias()); - - return StratificationSqlIdColumns.builder() - .primaryColumn(getPrimaryColumn()) - .secondaryId(null) - .resolution(this.resolution) - .index(withNulledCompleteIndex) - .eventDate(this.eventDate) - .build(); - } - - @Override - public boolean isWithStratification() { - return true; - } - - @Override - public List> toFields() { - return Stream.of( - getPrimaryColumn(), - this.resolution, - this.index, - this.eventDate - ) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - } - - @Override - public List join(SqlIdColumns rightIds) { - - if (!rightIds.isWithStratification()) { - return super.join(rightIds); - } - - StratificationSqlIdColumns rightStratificationIds = (StratificationSqlIdColumns) rightIds; - Condition joinResolutionAndIndex = this.resolution.eq(rightStratificationIds.getResolution()).and(this.index.eq(rightStratificationIds.getIndex())); - - Condition joinEventDateCondition; - if (this.eventDate != null) { - joinEventDateCondition = this.eventDate.eq(rightStratificationIds.getEventDate()); - } - else { - joinEventDateCondition = DSL.noCondition(); - } - - return Stream.concat( - super.join(rightIds).stream(), - Stream.of(joinResolutionAndIndex, joinEventDateCondition) - ) - .toList(); - } - - @Override - public SqlIdColumns coalesce(List selectsIds) { - - Preconditions.checkArgument( - selectsIds.stream().allMatch(SqlIdColumns::isWithStratification), - "Can only coalesce SqlIdColumns if all are with stratification" - ); - - List> primaryColumns = new ArrayList<>(); - List> resolutions = new ArrayList<>(); - List> indices = new ArrayList<>(); - List> eventDates = new ArrayList<>(); - - // add this ids - primaryColumns.add(getPrimaryColumn()); - resolutions.add(this.resolution); - indices.add(this.index); - if (this.eventDate != null) { - eventDates.add(this.eventDate); - } - - for (SqlIdColumns ids : selectsIds) { - StratificationSqlIdColumns stratificationIds = (StratificationSqlIdColumns) ids; - primaryColumns.add(stratificationIds.getPrimaryColumn()); - resolutions.add(stratificationIds.getResolution()); - indices.add(stratificationIds.getIndex()); - if (stratificationIds.getEventDate() != null) { - eventDates.add(stratificationIds.getEventDate()); - } - } - - Field coalescedPrimaryColumn = coalesceFields(primaryColumns, String.class).coerce(String.class).as(SharedAliases.PRIMARY_COLUMN.getAlias()); - Field coalescedResolutions = coalesceFields(resolutions, String.class).as(SharedAliases.RESOLUTION.getAlias()); - Field coalescedIndices = coalesceFields(indices, Integer.class).as(SharedAliases.INDEX.getAlias()); - Field eventDate = null; - if (!eventDates.isEmpty()) { - eventDate = coalesceFields(eventDates, Date.class).as(SharedAliases.INDEX_SELECTOR.getAlias()); - } - - return StratificationSqlIdColumns.builder() - .primaryColumn(coalescedPrimaryColumn) - .secondaryId(null) - .resolution(coalescedResolutions) - .index(coalescedIndices) - .eventDate(eventDate) - .build(); - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java index df20b8a43f..c455f1a636 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java @@ -19,7 +19,7 @@ import com.bakdata.conquery.sql.conversion.model.NumberMapUtil; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java index 9c305f2e21..18efd2be87 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java @@ -19,7 +19,7 @@ import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java index e4d08205f8..54ab2b4e9c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java @@ -32,7 +32,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.CteStep; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java index 84e45f7488..47b03b1684 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java @@ -17,7 +17,7 @@ import com.bakdata.conquery.sql.conversion.model.CteStep; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java index 973e441411..08e188f56d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; import lombok.AccessLevel; import lombok.AllArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java index 62f963243c..b6917dc491 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java @@ -19,7 +19,7 @@ import com.bakdata.conquery.sql.conversion.model.CteStep; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import lombok.Getter; import lombok.RequiredArgsConstructor; import org.jetbrains.annotations.NotNull; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java index 6cfaf36f83..ce788a2c70 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery; import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.forms.FormCteStep; import com.bakdata.conquery.sql.conversion.forms.FormType; @@ -15,7 +15,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.google.common.base.Preconditions; import lombok.RequiredArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java index ed8590d683..76535c7993 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java @@ -5,7 +5,7 @@ import com.bakdata.conquery.models.forms.managed.EntityDateQuery; import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.forms.FormCteStep; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java index 3899f2466f..c992c300e6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java @@ -4,11 +4,13 @@ import com.bakdata.conquery.apiv1.query.ArrayConceptQuery; import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.Query; +import com.bakdata.conquery.models.forms.util.Resolution; import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; -import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.forms.FormCteStep; @@ -43,7 +45,7 @@ private static Selects getFinalSelects( Selects preFinalSelects = convertedFeatures.getQualifiedSelects(); Selects stratificationSelects = stratificationTable.getQualifiedSelects(); - SqlIdColumns ids = stratificationSelects.getIds().forFinalSelect(); + SqlIdColumns ids = stratificationSelects.getIds().forFinalSelect(Resolution.COMPLETE.name()); Selects.SelectsBuilder selects = Selects.builder() .ids(ids) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java index 9765000136..9f40ac8799 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java @@ -7,10 +7,11 @@ import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.forms.FormCteStep; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SharedAliases.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/SharedAliases.java similarity index 76% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/SharedAliases.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/SharedAliases.java index 191d96ecae..ff1cf3167e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SharedAliases.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/SharedAliases.java @@ -1,10 +1,6 @@ -package com.bakdata.conquery.sql.conversion; +package com.bakdata.conquery.sql.compiler.ir; -import lombok.Getter; -import lombok.RequiredArgsConstructor; - -@Getter -@RequiredArgsConstructor +/** Shared SQL aliases used by compiler transformations and their generated output columns. */ public enum SharedAliases { PRIMARY_COLUMN("primary_id"), @@ -13,7 +9,6 @@ public enum SharedAliases { NOP_TABLE("nop_table"), - // form related INDEX_START("index_start"), QUARTER_START("quarter_start"), QUARTER_END("quarter_end"), @@ -32,8 +27,15 @@ public enum SharedAliases { STRATIFICATION_BOUNDS("stratification_bounds"), OBSERVATION_SCOPE("scope"), - // full export form SOURCE("source"); private final String alias; + + SharedAliases(String alias) { + this.alias = alias; + } + + public String getAlias() { + return alias; + } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/SqlIdColumns.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/SqlIdColumns.java new file mode 100644 index 0000000000..bfb122a4a2 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/SqlIdColumns.java @@ -0,0 +1,137 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import java.sql.Date; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.impl.DSL; + +/** Compiler IR for the columns identifying an entity. */ +public class SqlIdColumns implements Qualifiable { + + private final Field primaryColumn; + private final Field secondaryId; + private final SqlIdColumns predecessor; + + public SqlIdColumns(Field primaryColumn, Field secondaryId) { + this(primaryColumn, secondaryId, null); + } + + public SqlIdColumns(Field primaryColumn) { + this(primaryColumn, null, null); + } + + protected SqlIdColumns(Field primaryColumn, Field secondaryId, SqlIdColumns predecessor) { + this.primaryColumn = primaryColumn; + this.secondaryId = secondaryId; + this.predecessor = predecessor; + } + + public Field getPrimaryColumn() { + return primaryColumn; + } + + public SqlIdColumns withAlias() { + Field primary = primaryColumn.as(SharedAliases.PRIMARY_COLUMN.getAlias()); + if (secondaryId == null) { + return new SqlIdColumns(primary, null, this); + } + return new SqlIdColumns(primary, secondaryId.as(SharedAliases.SECONDARY_ID.getAlias()), this); + } + + @Override + public SqlIdColumns qualify(String qualifier) { + Field primary = QualifyingUtil.qualify(primaryColumn, qualifier); + if (secondaryId == null) { + return new SqlIdColumns(primary, null, this); + } + return new SqlIdColumns(primary, QualifyingUtil.qualify(secondaryId, qualifier), this); + } + + public SqlIdColumns withStratification(String resolution, Field index) { + return withStratification(resolution, index, null); + } + + /** + * Add stratification columns using a backend-independent resolution token. + * + * @param resolution normalized resolution value supplied at the compiler boundary + * @param index stratification index + * @param eventDate optional event date used by relative stratification + */ + public SqlIdColumns withStratification(String resolution, Field index, Field eventDate) { + Field resolutionField = DSL.inline(resolution).as(SharedAliases.RESOLUTION.getAlias()); + return new StratificationSqlIdColumns(primaryColumn, resolutionField, index, eventDate); + } + + /** Convert stratified IDs into their final output form using the normalized complete-resolution token. */ + public SqlIdColumns forFinalSelect(String completeResolution) { + return this; + } + + public Optional> getSecondaryId() { + return Optional.ofNullable(secondaryId); + } + + public Optional getPredecessor() { + return Optional.ofNullable(predecessor); + } + + public boolean isWithStratification() { + return false; + } + + public List> toFields() { + if (secondaryId == null) { + return List.of(primaryColumn); + } + return List.of(primaryColumn, secondaryId); + } + + public List join(SqlIdColumns rightIds) { + Condition joinPrimaries = primaryColumn.eq(rightIds.primaryColumn); + Condition joinSecondaries = getSecondaryId() + .flatMap(leftSecondary -> rightIds.getSecondaryId().map(leftSecondary::eq)) + .orElse(DSL.noCondition()); + return List.of(joinPrimaries, joinSecondaries); + } + + public SqlIdColumns coalesce(List selectsIds) { + List> primaryColumns = new ArrayList<>(); + List> secondaryIds = new ArrayList<>(); + + primaryColumns.add(primaryColumn); + getSecondaryId().ifPresent(secondaryIds::add); + selectsIds.forEach(ids -> { + primaryColumns.add(ids.primaryColumn); + ids.getSecondaryId().ifPresent(secondaryIds::add); + }); + + Field coalescedPrimary = coalesceFields(primaryColumns, String.class) + .coerce(String.class) + .as(SharedAliases.PRIMARY_COLUMN.getAlias()); + if (secondaryIds.isEmpty()) { + return new SqlIdColumns(coalescedPrimary); + } + Field coalescedSecondary = coalesceFields(secondaryIds, String.class) + .coerce(String.class) + .as(SharedAliases.SECONDARY_ID.getAlias()); + return new SqlIdColumns(coalescedPrimary, coalescedSecondary); + } + + protected static Field coalesceFields(List> fields, Class type) { + Field result = fields.getFirst().coerce(type); + for (int index = 1; index < fields.size(); index++) { + result = DSL.coalesce(result, fields.get(index).coerce(type)); + } + return result; + } + + @Override + public String toString() { + return "SqlIdColumns(primaryColumn=%s, secondaryId=%s)".formatted(primaryColumn, secondaryId); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/StratificationSqlIdColumns.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/StratificationSqlIdColumns.java new file mode 100644 index 0000000000..61f70fa8e3 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/StratificationSqlIdColumns.java @@ -0,0 +1,113 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import java.sql.Date; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; + +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.impl.DSL; + +/** Stratified ID-column representation used internally by the compiler. */ +final class StratificationSqlIdColumns extends SqlIdColumns { + + private final Field resolution; + private final Field index; + private final Field eventDate; + + StratificationSqlIdColumns(Field primaryColumn, Field resolution, Field index, Field eventDate) { + super(primaryColumn); + this.resolution = resolution; + this.index = index; + this.eventDate = eventDate; + } + + @Override + public SqlIdColumns qualify(String qualifier) { + Field qualifiedEventDate = eventDate == null ? null : QualifyingUtil.qualify(eventDate, qualifier); + return new StratificationSqlIdColumns( + QualifyingUtil.qualify(getPrimaryColumn(), qualifier), + QualifyingUtil.qualify(resolution, qualifier), + QualifyingUtil.qualify(index, qualifier), + qualifiedEventDate + ); + } + + @Override + public SqlIdColumns forFinalSelect(String completeResolution) { + Field finalIndex = DSL.when( + resolution.eq(DSL.inline(completeResolution)), + DSL.inline(null, Integer.class) + ) + .otherwise(index) + .as(SharedAliases.INDEX.getAlias()); + return new StratificationSqlIdColumns(getPrimaryColumn(), resolution, finalIndex, eventDate); + } + + @Override + public boolean isWithStratification() { + return true; + } + + @Override + public List> toFields() { + return Stream.>of(getPrimaryColumn(), resolution, index, eventDate) + .filter(Objects::nonNull) + .toList(); + } + + @Override + public List join(SqlIdColumns rightIds) { + if (!(rightIds instanceof StratificationSqlIdColumns right)) { + return super.join(rightIds); + } + + Condition joinResolutionAndIndex = resolution.eq(right.resolution).and(index.eq(right.index)); + Condition joinEventDate = eventDate == null ? DSL.noCondition() : eventDate.eq(right.eventDate); + return Stream.concat(super.join(rightIds).stream(), Stream.of(joinResolutionAndIndex, joinEventDate)).toList(); + } + + @Override + public SqlIdColumns coalesce(List selectsIds) { + if (!selectsIds.stream().allMatch(StratificationSqlIdColumns.class::isInstance)) { + throw new IllegalArgumentException("Can only coalesce SqlIdColumns if all are with stratification"); + } + + List> primaryColumns = new ArrayList<>(); + List> resolutions = new ArrayList<>(); + List> indices = new ArrayList<>(); + List> eventDates = new ArrayList<>(); + + primaryColumns.add(getPrimaryColumn()); + resolutions.add(resolution); + indices.add(index); + if (eventDate != null) { + eventDates.add(eventDate); + } + + for (SqlIdColumns ids : selectsIds) { + StratificationSqlIdColumns stratified = (StratificationSqlIdColumns) ids; + primaryColumns.add(stratified.getPrimaryColumn()); + resolutions.add(stratified.resolution); + indices.add(stratified.index); + if (stratified.eventDate != null) { + eventDates.add(stratified.eventDate); + } + } + + Field primary = coalesceFields(primaryColumns, String.class) + .coerce(String.class) + .as(SharedAliases.PRIMARY_COLUMN.getAlias()); + Field coalescedResolution = coalesceFields(resolutions, String.class) + .as(SharedAliases.RESOLUTION.getAlias()); + Field coalescedIndex = coalesceFields(indices, Integer.class) + .as(SharedAliases.INDEX.getAlias()); + Field coalescedEventDate = eventDates.isEmpty() + ? null + : coalesceFields(eventDates, Date.class).as(SharedAliases.INDEX_SELECTOR.getAlias()); + + return new StratificationSqlIdColumns(primary, coalescedResolution, coalescedIndex, coalescedEventDate); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/SqlIdColumnsTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/SqlIdColumnsTest.java new file mode 100644 index 0000000000..0d82732661 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/SqlIdColumnsTest.java @@ -0,0 +1,71 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.Date; +import java.util.List; + +import org.junit.jupiter.api.Test; + +class SqlIdColumnsTest { + + @Test + void shouldAliasIdColumnsAndRetainTheirPredecessor() { + SqlIdColumns original = new SqlIdColumns( + field(name("source", "person"), String.class), + field(name("source", "visit"), String.class) + ); + + SqlIdColumns aliased = original.withAlias(); + + assertEquals(SharedAliases.PRIMARY_COLUMN.getAlias(), aliased.getPrimaryColumn().getName()); + assertEquals(SharedAliases.SECONDARY_ID.getAlias(), aliased.getSecondaryId().orElseThrow().getName()); + assertSame(original, aliased.getPredecessor().orElseThrow()); + } + + @Test + void shouldQualifyIdColumns() { + SqlIdColumns ids = new SqlIdColumns(field(name("person"), String.class)); + + SqlIdColumns qualified = ids.qualify("query_step"); + + assertEquals(name("query_step", "person"), qualified.getPrimaryColumn().getQualifiedName()); + assertSame(ids, qualified.getPredecessor().orElseThrow()); + } + + @Test + void shouldCreateStratifiedIdColumnsFromNormalizedResolution() { + SqlIdColumns ids = new SqlIdColumns(field(name("person"), String.class)); + + SqlIdColumns stratified = ids.withStratification( + "COMPLETE", + field(name(SharedAliases.INDEX.getAlias()), Integer.class), + field(name(SharedAliases.INDEX_SELECTOR.getAlias()), Date.class) + ); + + assertTrue(stratified.isWithStratification()); + assertEquals( + List.of( + "person", + SharedAliases.RESOLUTION.getAlias(), + SharedAliases.INDEX.getAlias(), + SharedAliases.INDEX_SELECTOR.getAlias() + ), + stratified.toFields().stream().map(org.jooq.Field::getName).toList() + ); + assertEquals(SharedAliases.INDEX.getAlias(), stratified.forFinalSelect("COMPLETE").toFields().get(2).getName()); + } + + @Test + void shouldRejectCoalescingStratifiedAndPlainIds() { + SqlIdColumns plain = new SqlIdColumns(field(name("person"), String.class)); + SqlIdColumns stratified = plain.withStratification("YEARS", field(name("index"), Integer.class)); + + assertThrows(IllegalArgumentException.class, () -> stratified.coalesce(List.of(plain))); + } +} From 19bdd86e4e2e06b449720a86423c497b5373f52d Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:33:12 +0200 Subject: [PATCH 14/81] refactor(sql): move select projection IR into connector Move the Selects aggregate and its pure transformations into the connector-owned IR while preserving its Lombok value and builder API. Keep dialect-aware final projection rendering in the backend transformer and cover the moved aggregate with focused tests. Co-authored-by: Codex GPT-5.6-sol --- .../cqelement/CQExternalConverter.java | 2 +- .../conversion/cqelement/CQYesConverter.java | 2 +- .../aggregation/AnsiSqlDateAggregator.java | 2 +- .../aggregation/IntermediateTableCte.java | 2 +- .../cqelement/aggregation/InvertCte.java | 2 +- .../aggregation/NodeNoOverlapCte.java | 2 +- .../cqelement/aggregation/OverlapCte.java | 2 +- .../cqelement/aggregation/RowNumberCte.java | 2 +- .../concept/AggregationFilterCte.java | 2 +- .../concept/AggregationSelectCte.java | 2 +- .../cqelement/concept/CQConceptConverter.java | 1 + .../concept/IntervalPackingSelectsCte.java | 2 +- .../cqelement/concept/JoinBranchesCte.java | 2 +- .../cqelement/concept/PreprocessingCte.java | 2 +- .../intervalpacking/AggregationMode.java | 2 +- .../AnsiSqlIntervalPacker.java | 2 +- .../PostgreSqlIntervalPacker.java | 2 +- .../forms/AbsoluteStratification.java | 2 +- .../forms/RelativeStratification.java | 2 +- .../sql/conversion/model/QueryStep.java | 1 + .../sql/conversion/model/QueryStepJoiner.java | 1 + .../model/QueryStepTransformer.java | 35 +++++++++- .../model/aggregator/SumSqlAggregator.java | 2 +- .../select/ConceptColumnSelectConverter.java | 2 +- .../model/select/DistinctSelectConverter.java | 2 +- .../model/select/ValueSelectUtil.java | 2 +- .../query/AbsoluteFormQueryConverter.java | 2 +- .../query/ConceptQueryConverter.java | 1 + .../query/EntityDateQueryConverter.java | 2 +- .../query/FormConversionHelper.java | 1 + .../query/TableExportQueryConverter.java | 1 + sql-connector/pom.xml | 6 ++ .../conquery/sql/compiler/ir}/Selects.java | 59 +++++----------- .../conquery/sql/compiler/ir/SelectsTest.java | 68 +++++++++++++++++++ 34 files changed, 155 insertions(+), 67 deletions(-) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/model => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/Selects.java (52%) create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/SelectsTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java index 853ca74f6b..5b15ee3d1a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java @@ -16,7 +16,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java index 0a59e4a3ca..17a93fcae4 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import org.jooq.Field; import org.jooq.Record; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java index 6ff056e800..51eb5cfe47 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java @@ -15,7 +15,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Data; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java index 0aeb233c68..afff1c1983 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Getter; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java index 49a8483c27..36ce1efe35 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java @@ -12,7 +12,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QualifyingUtil; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import lombok.Getter; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java index 97a8e052b2..07bf08b999 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.conversion.model.SqlTables; import lombok.Getter; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java index 9ce61b8f23..5388d688ea 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import lombok.Getter; import org.jooq.Condition; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java index 55bd70ec1e..b51e28f648 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java @@ -5,7 +5,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java index 9187b6c2f8..cb332d22fa 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java @@ -4,7 +4,7 @@ import java.util.stream.Collectors; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java index 39c1a878b0..0e8e5abe67 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index ae79cb7394..295d9020af 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; +import com.bakdata.conquery.sql.compiler.ir.Selects; import java.util.ArrayList; import java.util.List; import java.util.Optional; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java index 107c286c31..0b0d7a8239 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java @@ -10,7 +10,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.conversion.model.select.ConceptSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java index dbda348b52..4b467167d8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java @@ -10,7 +10,7 @@ import com.bakdata.conquery.sql.conversion.model.ConqueryJoinType; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.aggregator.SumSqlAggregator; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java index 4570747992..c1a30a006f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java @@ -8,7 +8,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AggregationMode.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AggregationMode.java index d1797b23dd..4592d34294 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AggregationMode.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AggregationMode.java @@ -1,6 +1,6 @@ package com.bakdata.conquery.sql.conversion.cqelement.intervalpacking; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; /** diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java index fa3403e26b..d9c0ab1146 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java @@ -10,7 +10,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QualifyingUtil; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java index d7135cbf72..26071551b0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import lombok.RequiredArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java index 5e3df52c20..99a633eff8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.google.common.base.Preconditions; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java index cbe0cc53bb..0961d6fbc8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java @@ -24,7 +24,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.google.common.base.Preconditions; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java index 7f1161fba3..9e12193867 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.sql.conversion.model; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Builder; import lombok.Singular; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java index c456a95b7d..cf69a12042 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.sql.conversion.model; +import com.bakdata.conquery.sql.compiler.ir.Selects; import static org.jooq.impl.DSL.*; import java.util.ArrayList; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java index fa34d101b0..5a72fe62ec 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.sql.conversion.model; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import lombok.RequiredArgsConstructor; import org.jooq.*; @@ -7,6 +9,9 @@ import org.jooq.impl.DSL; import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; import java.util.stream.Stream; /** @@ -22,7 +27,7 @@ public class QueryStepTransformer { */ public Select toSelectQuery(QueryStep queryStep, SqlFunctionProvider functionProvider) { - List> finalRepresentation = queryStep.getSelects().toFinalRepresentation(functionProvider, queryStep.isForTableExport()); + List> finalRepresentation = renderFinalRepresentation(queryStep.getSelects(), functionProvider, queryStep.isForTableExport()); SelectConditionStep queryBase = this.dslContext.with(constructPredecessorCteList(queryStep, functionProvider)) .select(finalRepresentation) @@ -69,7 +74,9 @@ private Select toSelectStep(QueryStep queryStep, SqlFunctionProvider fun SelectSelectStep selectClause; - List> allSelects = queryStep.isForTableExport() ? queryStep.getSelects().toFinalRepresentation(functionProvider, true) : queryStep.getSelects().all(); + List> allSelects = queryStep.isForTableExport() + ? renderFinalRepresentation(queryStep.getSelects(), functionProvider, true) + : queryStep.getSelects().all(); if (queryStep.isSelectDistinct()) { selectClause = dslContext.selectDistinct(allSelects); @@ -90,6 +97,30 @@ private Select toSelectStep(QueryStep queryStep, SqlFunctionProvider fun return selectStep; } + private static List> renderFinalRepresentation( + Selects selects, + SqlFunctionProvider functionProvider, + boolean forTableExport + ) { + final Optional> validityDateRendered = !forTableExport + ? selects.getValidityDate().map(dateRange -> functionProvider.dateRangeAggregation(dateRange).as(SharedAliases.DATES_COLUMN.getAlias())) + : selects.getValidityDate().map(dateRange -> functionProvider.dateRangeToField(dateRange).as(SharedAliases.DATES_COLUMN.getAlias())); + + final Optional> stratificationDateRendered = selects.getStratificationDate() + .map(dateRange -> functionProvider.dateRangeToField(dateRange).as(SharedAliases.STRATIFICATION_BOUNDS.getAlias())); + + return Stream.of( + selects.getIds().toFields().stream(), + stratificationDateRendered.stream(), + validityDateRendered.stream(), + selects.getSqlSelects().stream().flatMap(sqlSelect -> sqlSelect.toFinalRepresentation().toFields().stream()) + ) + .flatMap(Function.identity()) + .map(select -> (Field) select) + .distinct() + .collect(Collectors.toList()); + } + private Select union(QueryStep queryStep, Select base, SqlFunctionProvider functionProvider) { for (QueryStep unionStep : queryStep.getUnion()) { Select selectStep = diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java index c455f1a636..94c3ffeb1a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java @@ -18,7 +18,7 @@ import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.conversion.model.NumberMapUtil; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java index 18efd2be87..13c9201e6d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java @@ -18,7 +18,7 @@ import com.bakdata.conquery.sql.conversion.model.CteStep; import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java index 47b03b1684..faaea6c74a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java @@ -16,7 +16,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.CteStep; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java index b6917dc491..8afaefd21a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java @@ -18,7 +18,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.CteStep; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java index ce788a2c70..e912b300ef 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java @@ -14,7 +14,7 @@ import com.bakdata.conquery.sql.conversion.forms.StratificationTableFactory; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.google.common.base.Preconditions; import lombok.RequiredArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java index 7171395693..d5d8606089 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.sql.conversion.query; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.concept.specific.CQNegation; import com.bakdata.conquery.models.query.DateAggregationMode; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java index 76535c7993..16627292b8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.sql.conversion.forms.StratificationTableFactory; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.google.common.base.Preconditions; import lombok.RequiredArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java index c992c300e6..12d02a26dd 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.sql.conversion.query; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.apiv1.forms.FeatureGroup; import com.bakdata.conquery.apiv1.query.ArrayConceptQuery; import com.bakdata.conquery.apiv1.query.ConceptQuery; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java index 9f40ac8799..2f2f897230 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.sql.conversion.query; +import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.apiv1.query.TableExportQuery; import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; diff --git a/sql-connector/pom.xml b/sql-connector/pom.xml index aa3fee0be6..9da92ddf79 100644 --- a/sql-connector/pom.xml +++ b/sql-connector/pom.xml @@ -29,6 +29,12 @@ jooq ${jooq.version} + + org.projectlombok + lombok + ${lombok.version} + provided + org.junit.jupiter junit-jupiter-api diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/Selects.java similarity index 52% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/Selects.java index dc072ad613..de5e7a55d8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/Selects.java @@ -1,10 +1,7 @@ -package com.bakdata.conquery.sql.conversion.model; +package com.bakdata.conquery.sql.compiler.ir; -import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import com.bakdata.conquery.sql.compiler.ir.SharedAliases; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import lombok.Builder; import lombok.Singular; import lombok.Value; @@ -16,6 +13,12 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +/** + * Fields selected by a compiler step before dialect-specific rendering. + * + *

This type only describes and transforms compiler state. Turning date ranges and select expressions into the + * final SQL projection belongs to the compiler layer that owns the dialect.

+ */ @Value @Builder(toBuilder = true) public class Selects { @@ -28,27 +31,6 @@ public class Selects { @Singular List sqlSelects; - public List> toFinalRepresentation(SqlFunctionProvider functionProvider, boolean forTableExport) { - - final Optional> validityDateRendered = - !forTableExport - ? getValidityDate().map(vdd -> functionProvider.dateRangeAggregation(vdd).as(SharedAliases.DATES_COLUMN.getAlias())) - : getValidityDate().map(vdd -> functionProvider.dateRangeToField(vdd).as(SharedAliases.DATES_COLUMN.getAlias())); - - final Optional> stratificationDateRendered = getStratificationDate().map(vdd -> functionProvider.dateRangeToField(vdd).as(SharedAliases.STRATIFICATION_BOUNDS.getAlias())); - - return Stream.of( - getIds().toFields().stream(), - stratificationDateRendered.stream(), - validityDateRendered.stream(), - getSqlSelects().stream().flatMap(sqlSelect -> sqlSelect.toFinalRepresentation().toFields().stream()) - ) - .flatMap(Function.identity()) - .map(select -> (Field) select) - .distinct() - .collect(Collectors.toList()); - } - public Selects blockValidityDate() { return this.toBuilder() .validityDate(Optional.empty()) @@ -56,22 +38,17 @@ public Selects blockValidityDate() { } public Selects qualify(String qualifier) { - SqlIdColumns ids = this.ids.qualify(qualifier); - List sqlSelects = this.sqlSelects.stream().map(sqlSelect -> sqlSelect.qualify(qualifier)).collect(Collectors.toList()); - - SelectsBuilder builder = Selects.builder() - .ids(ids) - .sqlSelects(sqlSelects); - - if (this.validityDate.isPresent()) { - builder = builder.validityDate(this.validityDate.map(_validityDate -> _validityDate.qualify(qualifier))); - } - - if (this.stratificationDate.isPresent()) { - builder = builder.stratificationDate(this.stratificationDate.map(_validityDate -> _validityDate.qualify(qualifier))); - } - - return builder.build(); + SqlIdColumns qualifiedIds = this.ids.qualify(qualifier); + List qualifiedSqlSelects = this.sqlSelects.stream() + .map(sqlSelect -> sqlSelect.qualify(qualifier)) + .toList(); + + return Selects.builder() + .ids(qualifiedIds) + .validityDate(this.validityDate.map(dateRange -> dateRange.qualify(qualifier))) + .stratificationDate(this.stratificationDate.map(dateRange -> dateRange.qualify(qualifier))) + .sqlSelects(qualifiedSqlSelects) + .build(); } public List> all() { diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/SelectsTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/SelectsTest.java new file mode 100644 index 0000000000..3eb7e5d21a --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/SelectsTest.java @@ -0,0 +1,68 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import java.sql.Date; +import java.util.List; +import java.util.Optional; +import org.jooq.Field; +import org.junit.jupiter.api.Test; + +class SelectsTest { + + @Test + void shouldBuildImmutableSelectsWithDefaultDates() { + FieldWrapper explicit = new FieldWrapper<>(field(name("value"), Integer.class)); + Selects selects = Selects.builder() + .ids(new SqlIdColumns(field(name("person"), String.class))) + .sqlSelect(explicit) + .build(); + + assertTrue(selects.getValidityDate().isEmpty()); + assertTrue(selects.getStratificationDate().isEmpty()); + assertEquals(List.of(explicit), selects.getSqlSelects()); + assertThrows(UnsupportedOperationException.class, () -> selects.getSqlSelects().clear()); + } + + @Test + void shouldPreserveAdditiveLombokBuilderBehavior() { + FieldWrapper first = new FieldWrapper<>(field(name("first"), Integer.class)); + FieldWrapper second = new FieldWrapper<>(field(name("second"), Integer.class)); + Selects selects = Selects.builder() + .ids(new SqlIdColumns(field(name("person"), String.class))) + .sqlSelect(first) + .build(); + + Selects extended = selects.toBuilder().sqlSelects(List.of(second)).build(); + + assertEquals(List.of(first, second), extended.getSqlSelects()); + } + + @Test + void shouldQualifyAllCompilerFields() { + ColumnDateRange validityDate = ColumnDateRange.of( + field(name("valid_from"), Date.class), + field(name("valid_to"), Date.class) + ); + Selects selects = Selects.builder() + .ids(new SqlIdColumns(field(name("person"), String.class))) + .validityDate(Optional.of(validityDate)) + .sqlSelect(new FieldWrapper<>(field(name("value"), Integer.class))) + .build(); + + Selects qualified = selects.qualify("step"); + + assertEquals( + List.of(name("step", "person"), name("step", "valid_from"), name("step", "valid_to"), name("step", "value")), + qualified.all().stream().map(Field::getQualifiedName).toList() + ); + assertFalse(qualified.blockValidityDate().getValidityDate().isPresent()); + } +} From c067cb1a2314fb2438a1e7be669bcfc6128db906 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:48:57 +0200 Subject: [PATCH 15/81] refactor(sql): render select projections in connector Move dialect-aware Selects rendering into the connector compiler. Expose physical date-range rendering through CompilerDialect, bridge the legacy function provider, and cover aggregated and individual validity-date modes. Co-authored-by: Codex GPT-5.6-sol --- .../dialect/LegacyCompilerDialect.java | 12 +++ .../model/QueryStepTransformer.java | 74 +++++--------- .../query/ConceptQueryConverter.java | 3 +- .../query/FormConversionHelper.java | 5 +- .../query/TableExportQueryConverter.java | 2 +- .../sql/compiler/dialect/CompilerDialect.java | 12 +++ .../rendering/SelectProjectionRenderer.java | 69 +++++++++++++ .../sql/compiler/rendering/package-info.java | 7 ++ .../SelectProjectionRendererTest.java | 96 +++++++++++++++++++ 9 files changed, 224 insertions(+), 56 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRenderer.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/rendering/package-info.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRendererTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java index 38db1fd994..b299373aa6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.sql.conversion.dialect; +import java.sql.Date; import java.util.Collections; import java.util.List; import java.util.Map; @@ -7,6 +8,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.cqelement.CQAndConverter; import com.bakdata.conquery.sql.conversion.cqelement.CQDateRestrictionConverter; @@ -43,6 +45,16 @@ default Field anyValue(Field field) { return getFunctionProvider().anyValue(field); } + @Override + default Field renderDateRange(Field start, Field end) { + return getFunctionProvider().dateRangeToField(ColumnDateRange.of(start, end)); + } + + @Override + default Field aggregateDateRanges(Field start, Field end) { + return getFunctionProvider().dateRangeAggregation(ColumnDateRange.of(start, end)); + } + StratificationFunctions getStratificationFunctions(); SqlFunctionProvider getFunctionProvider(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java index 5a72fe62ec..cf838a39d3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java @@ -1,17 +1,14 @@ package com.bakdata.conquery.sql.conversion.model; -import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.bakdata.conquery.sql.compiler.ir.SharedAliases; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.rendering.SelectProjectionRenderer; +import com.bakdata.conquery.sql.compiler.rendering.SelectProjectionRenderer.ValidityDateRendering; import lombok.RequiredArgsConstructor; import org.jooq.*; import org.jooq.Record; import org.jooq.impl.DSL; import java.util.List; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; import java.util.stream.Stream; /** @@ -25,11 +22,14 @@ public class QueryStepTransformer { /** * Converts a given {@link QueryStep} into an executable SELECT statement. */ - public Select toSelectQuery(QueryStep queryStep, SqlFunctionProvider functionProvider) { + public Select toSelectQuery(QueryStep queryStep, CompilerDialect dialect) { - List> finalRepresentation = renderFinalRepresentation(queryStep.getSelects(), functionProvider, queryStep.isForTableExport()); + ValidityDateRendering validityDateRendering = queryStep.isForTableExport() + ? ValidityDateRendering.INDIVIDUAL + : ValidityDateRendering.AGGREGATED; + List> finalRepresentation = SelectProjectionRenderer.render(queryStep.getSelects(), dialect, validityDateRendering); - SelectConditionStep queryBase = this.dslContext.with(constructPredecessorCteList(queryStep, functionProvider)) + SelectConditionStep queryBase = this.dslContext.with(constructPredecessorCteList(queryStep, dialect)) .select(finalRepresentation) .from(queryStep.getFromTables()) .where(queryStep.getConditions()); @@ -42,40 +42,40 @@ public Select toSelectQuery(QueryStep queryStep, SqlFunctionProvider fun // union if (queryStep.isUnion()) { - return union(queryStep, grouped, functionProvider); + return union(queryStep, grouped, dialect); } return grouped; } - private List> constructPredecessorCteList(QueryStep queryStep, SqlFunctionProvider functionProvider) { - return predecessorCtes(queryStep, functionProvider) + private List> constructPredecessorCteList(QueryStep queryStep, CompilerDialect dialect) { + return predecessorCtes(queryStep, dialect) .toList(); } - private List> toCteList(QueryStep queryStep, SqlFunctionProvider functionProvider) { + private List> toCteList(QueryStep queryStep, CompilerDialect dialect) { return Stream.concat( - this.predecessorCtes(queryStep, functionProvider), - Stream.of(toCte(queryStep, functionProvider)) + this.predecessorCtes(queryStep, dialect), + Stream.of(toCte(queryStep, dialect)) ).toList(); } - private Stream> predecessorCtes(QueryStep queryStep, SqlFunctionProvider functionProvider) { + private Stream> predecessorCtes(QueryStep queryStep, CompilerDialect dialect) { return queryStep.getPredecessors().stream() - .flatMap(predecessor -> toCteList(predecessor, functionProvider).stream()); + .flatMap(predecessor -> toCteList(predecessor, dialect).stream()); } - private CommonTableExpression toCte(QueryStep queryStep, SqlFunctionProvider functionProvider) { - Select selectStep = toSelectStep(queryStep, functionProvider); + private CommonTableExpression toCte(QueryStep queryStep, CompilerDialect dialect) { + Select selectStep = toSelectStep(queryStep, dialect); return DSL.name(queryStep.getCteName()).as(selectStep); } - private Select toSelectStep(QueryStep queryStep, SqlFunctionProvider functionProvider) { + private Select toSelectStep(QueryStep queryStep, CompilerDialect dialect) { SelectSelectStep selectClause; List> allSelects = queryStep.isForTableExport() - ? renderFinalRepresentation(queryStep.getSelects(), functionProvider, true) + ? SelectProjectionRenderer.render(queryStep.getSelects(), dialect, ValidityDateRendering.INDIVIDUAL) : queryStep.getSelects().all(); if (queryStep.isSelectDistinct()) { @@ -91,43 +91,19 @@ private Select toSelectStep(QueryStep queryStep, SqlFunctionProvider fun } if (queryStep.isUnion()) { - selectStep = union(queryStep, selectStep, functionProvider); + selectStep = union(queryStep, selectStep, dialect); } return selectStep; } - private static List> renderFinalRepresentation( - Selects selects, - SqlFunctionProvider functionProvider, - boolean forTableExport - ) { - final Optional> validityDateRendered = !forTableExport - ? selects.getValidityDate().map(dateRange -> functionProvider.dateRangeAggregation(dateRange).as(SharedAliases.DATES_COLUMN.getAlias())) - : selects.getValidityDate().map(dateRange -> functionProvider.dateRangeToField(dateRange).as(SharedAliases.DATES_COLUMN.getAlias())); - - final Optional> stratificationDateRendered = selects.getStratificationDate() - .map(dateRange -> functionProvider.dateRangeToField(dateRange).as(SharedAliases.STRATIFICATION_BOUNDS.getAlias())); - - return Stream.of( - selects.getIds().toFields().stream(), - stratificationDateRendered.stream(), - validityDateRendered.stream(), - selects.getSqlSelects().stream().flatMap(sqlSelect -> sqlSelect.toFinalRepresentation().toFields().stream()) - ) - .flatMap(Function.identity()) - .map(select -> (Field) select) - .distinct() - .collect(Collectors.toList()); - } - - private Select union(QueryStep queryStep, Select base, SqlFunctionProvider functionProvider) { + private Select union(QueryStep queryStep, Select base, CompilerDialect dialect) { for (QueryStep unionStep : queryStep.getUnion()) { Select selectStep = queryStep.isForTableExport() ? // TODO this feels like a leaked abstraction, but i am not able to find the proper injection layer at the moment. - toSelectQuery(unionStep, functionProvider) : - toSelectStep(unionStep, functionProvider); + toSelectQuery(unionStep, dialect) : + toSelectStep(unionStep, dialect); if (queryStep.isUnionAll()) { base = base.unionAll(selectStep); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java index d5d8606089..0cf7ffca15 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java @@ -59,7 +59,6 @@ public Class getConversionClass() { @Override public ConversionContext convert(ConceptQuery conceptQuery, ConversionContext context) { - SqlFunctionProvider functionProvider = context.getCompilerDialect().getFunctionProvider(); ConversionContext contextAfterConversion = context.getNodeConversions().convert(conceptQuery.getRoot(), context); QueryStep preFinalStep = contextAfterConversion.getLastConvertedStep(); @@ -79,7 +78,7 @@ public ConversionContext convert(ConceptQuery conceptQuery, ConversionContext co .predecessors(predecessors) .build(); - Select finalQuery = this.queryStepTransformer.toSelectQuery(finalStep, functionProvider); + Select finalQuery = this.queryStepTransformer.toSelectQuery(finalStep, context.getCompilerDialect()); return contextAfterConversion.withFinalQuery(new SqlQuery(finalQuery, conceptQuery.getResultInfos())); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java index 12d02a26dd..3ac74ff0bf 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java @@ -13,7 +13,6 @@ import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.forms.FormCteStep; import com.bakdata.conquery.sql.conversion.forms.FormType; import com.bakdata.conquery.sql.conversion.model.*; @@ -150,8 +149,6 @@ private ConversionContext createFinalSelect( List queriesToJoin = List.of(stratificationTable, convertedFeatures); TableLike joinedTable = QueryStepJoiner.constructJoinedTable(queriesToJoin, ConqueryJoinType.LEFT_JOIN, context); - SqlFunctionProvider functionProvider = context.getFunctionProvider(); - QueryStep finalStep = QueryStep.builder() .cteName(null) // the final QueryStep won't be converted to a CTE .selects(getFinalSelects(formType, stratificationTable, convertedFeatures)) @@ -159,7 +156,7 @@ private ConversionContext createFinalSelect( .predecessors(queriesToJoin) .build(); - Select selectQuery = queryStepTransformer.toSelectQuery(finalStep, functionProvider); + Select selectQuery = queryStepTransformer.toSelectQuery(finalStep, context.getCompilerDialect()); return context.withFinalQuery(new SqlQuery(selectQuery, resultInfos)); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java index 2f2f897230..9dbe26987d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java @@ -211,7 +211,7 @@ public ConversionContext convert(TableExportQuery tableExportQuery, ConversionCo List.of(convertedPrerequisite), context.isNegation() ); - final Select selectQuery = queryStepTransformer.toSelectQuery(unionedTables, context.getFunctionProvider()); + final Select selectQuery = queryStepTransformer.toSelectQuery(unionedTables, context.getCompilerDialect()); return context.withFinalQuery(new SqlQuery(selectQuery, tableExportQuery.getResultInfos())); } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java index caa03b93b0..53ac40268e 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.sql.compiler.dialect; +import java.sql.Date; + import org.jooq.Field; /** @@ -22,6 +24,16 @@ public interface CompilerDialect { */ Field anyValue(Field field); + /** + * Render the bounds of one date range into the database-specific physical result representation. + */ + Field renderDateRange(Field start, Field end); + + /** + * Aggregate multiple date ranges into the database-specific physical result representation. + */ + Field aggregateDateRanges(Field start, Field end); + /** Maximum identifier length supported by the target database. */ int getNameMaxLength(); diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRenderer.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRenderer.java new file mode 100644 index 0000000000..ad1a4734a8 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRenderer.java @@ -0,0 +1,69 @@ +package com.bakdata.conquery.sql.compiler.rendering; + +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SharedAliases; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import org.jooq.Field; + +/** Renders a select projection from compiler IR into database-specific physical fields. */ +public final class SelectProjectionRenderer { + + private SelectProjectionRenderer() { + } + + /** + * Render the final projection fields. + * + * @param selects compiler projection state + * @param dialect database-specific compiler capabilities + * @param validityDateRendering whether validity ranges represent individual or grouped rows + */ + public static List> render( + Selects selects, + CompilerDialect dialect, + ValidityDateRendering validityDateRendering + ) { + Optional> validityDateRendered = selects.getValidityDate() + .map(dateRange -> renderValidityDate(dateRange, dialect, validityDateRendering) + .as(SharedAliases.DATES_COLUMN.getAlias())); + + Optional> stratificationDateRendered = selects.getStratificationDate() + .map(dateRange -> dialect.renderDateRange(dateRange.getStart(), dateRange.getEnd()) + .as(SharedAliases.STRATIFICATION_BOUNDS.getAlias())); + + return Stream.of( + selects.getIds().toFields().stream(), + stratificationDateRendered.stream(), + validityDateRendered.stream(), + selects.getSqlSelects().stream().flatMap(sqlSelect -> sqlSelect.toFinalRepresentation().toFields().stream()) + ) + .flatMap(Function.identity()) + .map(select -> (Field) select) + .distinct() + .collect(Collectors.toList()); + } + + private static Field renderValidityDate( + ColumnDateRange dateRange, + CompilerDialect dialect, + ValidityDateRendering validityDateRendering + ) { + return switch (validityDateRendering) { + case AGGREGATED -> dialect.aggregateDateRanges(dateRange.getStart(), dateRange.getEnd()); + case INDIVIDUAL -> dialect.renderDateRange(dateRange.getStart(), dateRange.getEnd()); + }; + } + + /** Physical representation required for validity dates in the projection. */ + public enum ValidityDateRendering { + AGGREGATED, + INDIVIDUAL + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/rendering/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/rendering/package-info.java new file mode 100644 index 0000000000..ea0369673d --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/rendering/package-info.java @@ -0,0 +1,7 @@ +/** + * Internal services that render compiler IR into database-specific SQL expressions. + * + *

Backend references are temporary while the remaining legacy compiler transformations move into the SQL + * connector.

+ */ +package com.bakdata.conquery.sql.compiler.rendering; diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRendererTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRendererTest.java new file mode 100644 index 0000000000..b072420159 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRendererTest.java @@ -0,0 +1,96 @@ +package com.bakdata.conquery.sql.compiler.rendering; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.sql.Date; +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SharedAliases; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.rendering.SelectProjectionRenderer.ValidityDateRendering; +import org.jooq.Field; +import org.junit.jupiter.api.Test; + +class SelectProjectionRendererTest { + + @Test + void shouldAggregateValidityDatesForGroupedResults() { + RecordingDialect dialect = new RecordingDialect(); + + List> fields = SelectProjectionRenderer.render(selectsWithDates(), dialect, ValidityDateRendering.AGGREGATED); + + assertEquals( + List.of( + "person", + SharedAliases.STRATIFICATION_BOUNDS.getAlias(), + SharedAliases.DATES_COLUMN.getAlias(), + "value" + ), + fields.stream().map(Field::getName).toList() + ); + assertEquals(1, dialect.aggregatedRanges); + assertEquals(1, dialect.renderedRanges); + } + + @Test + void shouldRenderValidityDatesWithoutAggregation() { + RecordingDialect dialect = new RecordingDialect(); + + SelectProjectionRenderer.render(selectsWithDates(), dialect, ValidityDateRendering.INDIVIDUAL); + + assertEquals(0, dialect.aggregatedRanges); + assertEquals(2, dialect.renderedRanges); + } + + private static Selects selectsWithDates() { + ColumnDateRange validityDate = ColumnDateRange.of( + field(name("valid_from"), Date.class), + field(name("valid_to"), Date.class) + ); + ColumnDateRange stratificationDate = ColumnDateRange.of( + field(name("window_from"), Date.class), + field(name("window_to"), Date.class) + ); + return Selects.builder() + .ids(new SqlIdColumns(field(name("person"), String.class))) + .validityDate(Optional.of(validityDate)) + .stratificationDate(Optional.of(stratificationDate)) + .sqlSelect(new FieldWrapper<>(field(name("value"), Integer.class))) + .build(); + } + + private static final class RecordingDialect implements CompilerDialect { + + private int renderedRanges; + private int aggregatedRanges; + + @Override + public Field anyValue(Field field) { + return field; + } + + @Override + public Field renderDateRange(Field start, Field end) { + this.renderedRanges++; + return field(name("rendered_range"), Object.class); + } + + @Override + public Field aggregateDateRanges(Field start, Field end) { + this.aggregatedRanges++; + return field(name("aggregated_ranges"), Object.class); + } + + @Override + public int getNameMaxLength() { + return 128; + } + } +} From 2b14316b9c3b2e8e4ee35898beaac72274792dd7 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:51:34 +0200 Subject: [PATCH 16/81] refactor(sql): move query step IR into connector Move QueryStep into the connector-owned IR and rewire legacy compiler consumers. Replace the backend-specific table-export flag with an explicit projection mode and cover qualification, unions, and final rendering modes. Co-authored-by: Codex GPT-5.6-sol --- .../conversion/cqelement/CQAndConverter.java | 2 +- .../cqelement/CQExternalConverter.java | 2 +- .../conversion/cqelement/CQOrConverter.java | 2 +- .../conversion/cqelement/CQYesConverter.java | 2 +- .../cqelement/ConversionContext.java | 2 +- .../aggregation/AnsiSqlDateAggregator.java | 2 +- .../aggregation/DateAggregationContext.java | 2 +- .../aggregation/DateAggregationCte.java | 2 +- .../aggregation/DateAggregationCteStep.java | 2 +- .../aggregation/DateAggregationDates.java | 2 +- .../aggregation/IntermediateTableCte.java | 2 +- .../IntersectAggregationAction.java | 2 +- .../cqelement/aggregation/InvertCte.java | 2 +- .../aggregation/MergeAggregateAction.java | 2 +- .../cqelement/aggregation/MergeCte.java | 2 +- .../aggregation/NodeNoOverlapCte.java | 2 +- .../cqelement/aggregation/OverlapCte.java | 2 +- .../cqelement/aggregation/RowNumberCte.java | 2 +- .../aggregation/SqlAggregationAction.java | 2 +- .../concept/AggregationFilterCte.java | 2 +- .../concept/AggregationSelectCte.java | 2 +- .../cqelement/concept/CQConceptConverter.java | 1 + .../cqelement/concept/CQTableContext.java | 2 +- .../cqelement/concept/ConnectorCte.java | 2 +- .../concept/IntervalPackingSelectsCte.java | 2 +- .../cqelement/concept/JoinBranchesCte.java | 2 +- .../cqelement/concept/PreprocessingCte.java | 2 +- .../cqelement/concept/TablePath.java | 2 +- .../AnsiSqlIntervalPacker.java | 2 +- .../IntervalPackingContext.java | 2 +- .../IntervalPackingCteStep.java | 2 +- .../IntervalPackingResult.java | 2 +- .../PostgreSqlIntervalPacker.java | 2 +- .../conversion/dialect/IntervalPacker.java | 2 +- .../conversion/dialect/SqlDateAggregator.java | 2 +- .../dialect/SqlFunctionProvider.java | 2 +- .../ClickhouseFunctionProvider.java | 2 +- .../dialect/hana/HanaSqlFunctionProvider.java | 2 +- .../forms/AbsoluteStratification.java | 2 +- .../forms/RelativeStratification.java | 2 +- .../forms/StratificationTableFactory.java | 2 +- .../sql/conversion/model/QueryStepJoiner.java | 1 + .../model/QueryStepTransformer.java | 22 +++--- .../aggregator/CommonAggregationSelect.java | 2 +- .../model/aggregator/SumSqlAggregator.java | 2 +- .../select/ConceptColumnSelectConverter.java | 2 +- .../model/select/ConceptSqlSelects.java | 2 +- .../model/select/ConnectorSqlSelects.java | 2 +- .../model/select/DaterangeSelectUtil.java | 2 +- .../model/select/DistinctSelectConverter.java | 2 +- .../model/select/ValueSelectUtil.java | 2 +- .../query/AbsoluteFormQueryConverter.java | 2 +- .../query/ConceptQueryConverter.java | 3 + .../query/EntityDateQueryConverter.java | 2 +- .../query/FormConversionHelper.java | 3 + .../query/RelativFormQueryConverter.java | 2 +- .../query/TableExportQueryConverter.java | 4 +- .../sql/compiler/ir/ProjectionMode.java | 11 +++ .../conquery/sql/compiler/ir}/QueryStep.java | 31 +++----- .../rendering/SelectProjectionRenderer.java | 24 +++--- .../sql/compiler/ir/QueryStepTest.java | 77 +++++++++++++++++++ .../SelectProjectionRendererTest.java | 23 +++++- 62 files changed, 204 insertions(+), 98 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/ProjectionMode.java rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/model => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/QueryStep.java (77%) create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/QueryStepTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java index d70deb0efc..5cd5af0273 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java @@ -3,7 +3,7 @@ import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.model.ConqueryJoinType; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; import com.bakdata.conquery.sql.compiler.ir.select.ExistsSqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java index 5b15ee3d1a..3bc9107cfa 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java @@ -15,7 +15,7 @@ import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java index 594de48460..246fa5a08e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java @@ -3,7 +3,7 @@ import com.bakdata.conquery.apiv1.query.concept.specific.CQOr; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.model.ConqueryJoinType; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; import com.bakdata.conquery.sql.compiler.ir.select.ExistsSqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java index 17a93fcae4..ea1e1ef23a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java @@ -8,7 +8,7 @@ import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java index 49612303ef..37d856dbfb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java @@ -15,7 +15,7 @@ import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.conversion.model.NameGenerator; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlQuery; import lombok.Builder; import lombok.NonNull; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java index 51eb5cfe47..fb24f7becc 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java @@ -14,7 +14,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java index f6c462841d..9d397c2146 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java @@ -10,7 +10,7 @@ import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.compiler.ir.Qualifiable; import com.bakdata.conquery.sql.compiler.ir.QualifyingUtil; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java index 721d99c84f..d8a34a6b21 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java @@ -2,7 +2,7 @@ import java.util.List; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlTables; /** diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteStep.java index 0c156762f0..089edfaeaa 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteStep.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.sql.conversion.model.CteStep; import com.bakdata.conquery.sql.conversion.model.NameGenerator; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlTables; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java index a53f27be61..7727308f4c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java @@ -8,7 +8,7 @@ import java.util.stream.Collectors; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.AccessLevel; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java index afff1c1983..5bf64af4f3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java @@ -5,7 +5,7 @@ import java.util.List; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java index 1b0ff7c716..b3eb77f7f9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.NameGenerator; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java index 36ce1efe35..1b702d8676 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java @@ -11,7 +11,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QualifyingUtil; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java index 67acb1d5e3..62acdd2b07 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.NameGenerator; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.RequiredArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeCte.java index 4fd1bc36cf..27091786df 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeCte.java @@ -3,7 +3,7 @@ import java.util.List; import java.util.stream.Collectors; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import lombok.Getter; @Getter diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java index 07bf08b999..0ad0a8960c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java @@ -6,7 +6,7 @@ import java.util.Optional; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.conversion.model.SqlTables; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java index 5388d688ea..2fe1a0928e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import lombok.Getter; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java index b51e28f648..d2ffd735ae 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java @@ -4,7 +4,7 @@ import java.util.Optional; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java index a9f7c4f172..c4abded10d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.NameGenerator; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java index cb332d22fa..459b33496c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java @@ -3,7 +3,7 @@ import java.util.List; import java.util.stream.Collectors; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java index 0e8e5abe67..d15cd80281 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java @@ -5,7 +5,7 @@ import java.util.stream.Stream; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 295d9020af..f41c1bd405 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import java.util.ArrayList; import java.util.List; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java index 22408801b3..f4614e5142 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java @@ -10,7 +10,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java index 0840b875e3..f5010d4fbd 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java @@ -3,7 +3,7 @@ import java.util.List; import java.util.Optional; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlTables; abstract class ConnectorCte { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java index 0b0d7a8239..541c745831 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.conversion.model.select.ConceptSqlSelects; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java index 4b467167d8..9d3c12ef24 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java @@ -8,7 +8,7 @@ import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.ConqueryJoinType; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java index c1a30a006f..4c6d09b6d8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java index a021c2db28..6e860a7349 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java @@ -15,7 +15,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep; import com.bakdata.conquery.sql.conversion.model.CteStep; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.google.common.base.Preconditions; import lombok.Data; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java index d9c0ab1146..c7ab0dc4ba 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QualifyingUtil; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java index a869876b32..f076ec9e2c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java index 640e6b789d..75c3c4cc04 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.model.CteStep; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlTables; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingResult.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingResult.java index 68134c0dc7..8467ac7f13 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingResult.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingResult.java @@ -1,7 +1,7 @@ package com.bakdata.conquery.sql.conversion.cqelement.intervalpacking; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import lombok.Value; @Value diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java index 26071551b0..c537c89b1a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java @@ -5,7 +5,7 @@ import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import lombok.RequiredArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/IntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/IntervalPacker.java index 508487ba34..5379b76d14 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/IntervalPacker.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/IntervalPacker.java @@ -2,7 +2,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingContext; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; /** diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java index 07a4ab1334..185065793b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.cqelement.aggregation.DateAggregationDates; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; public interface SqlDateAggregator { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index d6a0871129..dd0b7cf4d2 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -18,7 +18,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import org.jooq.Condition; import org.jooq.DataType; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java index d7dde0cb09..9c8aa663fc 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java @@ -14,7 +14,7 @@ import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import org.jetbrains.annotations.NotNull; import org.jooq.Condition; import org.jooq.DataType; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java index a885d8c478..70b4ce9ebd 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java @@ -17,7 +17,7 @@ import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import org.jetbrains.annotations.NotNull; import org.jooq.Condition; import org.jooq.DataType; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java index 99a633eff8..e7eab81037 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java @@ -12,7 +12,7 @@ import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java index 0961d6fbc8..2b398664d0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java @@ -23,7 +23,7 @@ import com.bakdata.conquery.sql.conversion.dialect.Interval; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java index 046f300e3d..81f80138ce 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.models.forms.managed.RelativeFormQuery; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.google.common.base.Preconditions; import lombok.AccessLevel; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java index cf69a12042..1f61f55be4 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.sql.conversion.model; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import static org.jooq.impl.DSL.*; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java index cf838a39d3..6db12f7b98 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java @@ -1,8 +1,9 @@ package com.bakdata.conquery.sql.conversion.model; import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.ProjectionMode; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.rendering.SelectProjectionRenderer; -import com.bakdata.conquery.sql.compiler.rendering.SelectProjectionRenderer.ValidityDateRendering; import lombok.RequiredArgsConstructor; import org.jooq.*; import org.jooq.Record; @@ -24,10 +25,11 @@ public class QueryStepTransformer { */ public Select toSelectQuery(QueryStep queryStep, CompilerDialect dialect) { - ValidityDateRendering validityDateRendering = queryStep.isForTableExport() - ? ValidityDateRendering.INDIVIDUAL - : ValidityDateRendering.AGGREGATED; - List> finalRepresentation = SelectProjectionRenderer.render(queryStep.getSelects(), dialect, validityDateRendering); + List> finalRepresentation = SelectProjectionRenderer.renderFinal( + queryStep.getSelects(), + dialect, + queryStep.getProjectionMode() + ); SelectConditionStep queryBase = this.dslContext.with(constructPredecessorCteList(queryStep, dialect)) .select(finalRepresentation) @@ -74,9 +76,9 @@ private Select toSelectStep(QueryStep queryStep, CompilerDialect dialect SelectSelectStep selectClause; - List> allSelects = queryStep.isForTableExport() - ? SelectProjectionRenderer.render(queryStep.getSelects(), dialect, ValidityDateRendering.INDIVIDUAL) - : queryStep.getSelects().all(); + List> allSelects = queryStep.getProjectionMode() == ProjectionMode.INTERMEDIATE + ? queryStep.getSelects().all() + : SelectProjectionRenderer.renderFinal(queryStep.getSelects(), dialect, queryStep.getProjectionMode()); if (queryStep.isSelectDistinct()) { selectClause = dslContext.selectDistinct(allSelects); @@ -100,8 +102,8 @@ private Select toSelectStep(QueryStep queryStep, CompilerDialect dialect private Select union(QueryStep queryStep, Select base, CompilerDialect dialect) { for (QueryStep unionStep : queryStep.getUnion()) { Select selectStep = - queryStep.isForTableExport() ? - // TODO this feels like a leaked abstraction, but i am not able to find the proper injection layer at the moment. + queryStep.getProjectionMode() != ProjectionMode.INTERMEDIATE ? + // Final projections must use the same physical representation in every union branch. toSelectQuery(unionStep, dialect) : toSelectStep(unionStep, dialect); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CommonAggregationSelect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CommonAggregationSelect.java index c3d379b8fb..dd9fa6d2eb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CommonAggregationSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CommonAggregationSelect.java @@ -5,7 +5,7 @@ import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.datasets.concepts.select.Select; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java index 94c3ffeb1a..e52badeaf0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java @@ -17,7 +17,7 @@ import com.bakdata.conquery.sql.conversion.model.CteStep; import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.conversion.model.NumberMapUtil; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java index 13c9201e6d..43768e2b13 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java @@ -17,7 +17,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.CteStep; import com.bakdata.conquery.sql.conversion.model.NameGenerator; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.execution.ResultSetProcessor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptSqlSelects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptSqlSelects.java index 65efbf98b1..d8cf3fb3c9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptSqlSelects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptSqlSelects.java @@ -4,7 +4,7 @@ import java.util.Optional; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import lombok.Builder; import lombok.Singular; import lombok.Value; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConnectorSqlSelects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConnectorSqlSelects.java index 22fe539410..da9eeb060e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConnectorSqlSelects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConnectorSqlSelects.java @@ -4,7 +4,7 @@ import java.util.Optional; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import lombok.Builder; import lombok.Singular; import lombok.Value; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java index 54ab2b4e9c..6279a89526 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java @@ -31,7 +31,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.CteStep; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java index faaea6c74a..658e0a2c08 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java @@ -15,7 +15,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.CteStep; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.execution.ResultSetProcessor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java index 8afaefd21a..e171703250 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java @@ -17,7 +17,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.CteStep; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java index e912b300ef..0070516eec 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.sql.conversion.forms.FormType; import com.bakdata.conquery.sql.conversion.forms.StratificationTableFactory; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.google.common.base.Preconditions; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java index 0cf7ffca15..aabc35327f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.sql.conversion.query; +import com.bakdata.conquery.sql.compiler.ir.ProjectionMode; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.concept.specific.CQNegation; @@ -72,6 +74,7 @@ public ConversionContext convert(ConceptQuery conceptQuery, ConversionContext co QueryStep finalStep = QueryStep.builder() .cteName(null) // the final QueryStep won't be converted to a CTE + .projectionMode(ProjectionMode.AGGREGATED) .selects(getFinalSelects(conceptQuery, preFinalSelects, context.getCompilerDialect())) .fromTable(getFinalTable(preFinalStep, contextAfterConversion)) .groupBy(getFinalGroupBySelects(preFinalSelects)) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java index 16627292b8..11d2d0ed5a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java @@ -12,7 +12,7 @@ import com.bakdata.conquery.sql.conversion.forms.FormType; import com.bakdata.conquery.sql.conversion.forms.StratificationTableFactory; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.google.common.base.Preconditions; import lombok.RequiredArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java index 3ac74ff0bf..2e707f1b0d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.sql.conversion.query; +import com.bakdata.conquery.sql.compiler.ir.ProjectionMode; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.apiv1.forms.FeatureGroup; import com.bakdata.conquery.apiv1.query.ArrayConceptQuery; @@ -151,6 +153,7 @@ private ConversionContext createFinalSelect( TableLike joinedTable = QueryStepJoiner.constructJoinedTable(queriesToJoin, ConqueryJoinType.LEFT_JOIN, context); QueryStep finalStep = QueryStep.builder() .cteName(null) // the final QueryStep won't be converted to a CTE + .projectionMode(ProjectionMode.AGGREGATED) .selects(getFinalSelects(formType, stratificationTable, convertedFeatures)) .fromTable(joinedTable) .predecessors(queriesToJoin) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java index e236e0d054..68b54a37a5 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java @@ -5,7 +5,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.forms.FormType; import com.bakdata.conquery.sql.conversion.forms.StratificationTableFactory; -import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import lombok.RequiredArgsConstructor; @RequiredArgsConstructor diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java index 9dbe26987d..a393110df0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.sql.conversion.query; +import com.bakdata.conquery.sql.compiler.ir.ProjectionMode; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.apiv1.query.TableExportQuery; @@ -102,7 +104,7 @@ private static QueryStep convertTable( .selects(selects) .fromTable(joinedTable) .conditions(filters) - .forTableExport(true) + .projectionMode(ProjectionMode.INDIVIDUAL) .build(); } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/ProjectionMode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/ProjectionMode.java new file mode 100644 index 0000000000..7d851187e3 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/ProjectionMode.java @@ -0,0 +1,11 @@ +package com.bakdata.conquery.sql.compiler.ir; + +/** Determines how a query step's projection is represented during SQL rendering. */ +public enum ProjectionMode { + /** Keep the separate compiler fields required by subsequent query steps. */ + INTERMEDIATE, + /** Render a final projection and aggregate validity ranges across grouped rows. */ + AGGREGATED, + /** Render a final projection while preserving the validity range of each row. */ + INDIVIDUAL +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStep.java similarity index 77% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStep.java index 9e12193867..1af727263e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStep.java @@ -1,6 +1,5 @@ -package com.bakdata.conquery.sql.conversion.model; +package com.bakdata.conquery.sql.compiler.ir; -import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Builder; import lombok.Singular; @@ -14,9 +13,7 @@ import java.util.Collections; import java.util.List; -/** - * Intermediate representation of an SQL query. - */ +/** Intermediate representation of one SQL query step. */ @Value @Builder(toBuilder = true) public class QueryStep { @@ -28,37 +25,30 @@ public class QueryStep { @Builder.Default List conditions = Collections.emptyList(); /** - * All {@link Field}s that should be part of the SQL GROUPY BY clause. + * All {@link Field}s that should be part of the SQL GROUP BY clause. */ @Builder.Default List> groupBy = Collections.emptyList(); /** - * All {@link QueryStep}s that should be connected via a SQL UNION operator + * All {@link QueryStep}s that should be connected via a SQL UNION operator. */ @Builder.Default List union = Collections.emptyList(); /** - * Determines if this steps union steps should be unioned using a UNION ALL. Default is true. + * Determines whether this step's union steps should be unioned using UNION ALL. Defaults to {@code true}. */ @Builder.Default boolean unionAll = true; @Builder.Default - boolean forTableExport = false; + ProjectionMode projectionMode = ProjectionMode.INTERMEDIATE; - - /** - * If the query should be negated or not. - */ + /** Whether the query should be negated. */ @Builder.Default boolean negate = false; - /** - * Determines if the select should be distinct. - */ + /** Whether the SELECT should be distinct. */ boolean selectDistinct; - /** - * All {@link QueryStep}'s that shall be converted before this {@link QueryStep}. - */ + /** All {@link QueryStep}s that must be converted before this step. */ @Singular List predecessors; @@ -91,7 +81,7 @@ public QueryStep addSqlSelect(SqlSelect sqlSelect) { } /** - * @return All selects re-mapped to a qualifier, which is the cteName of this QueryStep. + * @return all selects qualified by this step's CTE name */ public Selects getQualifiedSelects() { return this.selects.qualify(this.cteName); @@ -104,5 +94,4 @@ public boolean isGroupBy() { public boolean isUnion() { return !this.union.isEmpty(); } - } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRenderer.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRenderer.java index ad1a4734a8..916c18982c 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRenderer.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRenderer.java @@ -7,6 +7,7 @@ import java.util.stream.Stream; import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.ProjectionMode; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; @@ -23,15 +24,19 @@ private SelectProjectionRenderer() { * * @param selects compiler projection state * @param dialect database-specific compiler capabilities - * @param validityDateRendering whether validity ranges represent individual or grouped rows + * @param projectionMode final physical representation required for the projection */ - public static List> render( + public static List> renderFinal( Selects selects, CompilerDialect dialect, - ValidityDateRendering validityDateRendering + ProjectionMode projectionMode ) { + if (projectionMode == ProjectionMode.INTERMEDIATE) { + throw new IllegalArgumentException("An intermediate projection cannot be rendered as final output"); + } + Optional> validityDateRendered = selects.getValidityDate() - .map(dateRange -> renderValidityDate(dateRange, dialect, validityDateRendering) + .map(dateRange -> renderValidityDate(dateRange, dialect, projectionMode) .as(SharedAliases.DATES_COLUMN.getAlias())); Optional> stratificationDateRendered = selects.getStratificationDate() @@ -53,17 +58,12 @@ public static List> render( private static Field renderValidityDate( ColumnDateRange dateRange, CompilerDialect dialect, - ValidityDateRendering validityDateRendering + ProjectionMode projectionMode ) { - return switch (validityDateRendering) { + return switch (projectionMode) { case AGGREGATED -> dialect.aggregateDateRanges(dateRange.getStart(), dateRange.getEnd()); case INDIVIDUAL -> dialect.renderDateRange(dateRange.getStart(), dateRange.getEnd()); + case INTERMEDIATE -> throw new IllegalArgumentException("An intermediate projection cannot be rendered as final output"); }; } - - /** Physical representation required for validity dates in the projection. */ - public enum ValidityDateRendering { - AGGREGATED, - INDIVIDUAL - } } diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/QueryStepTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/QueryStepTest.java new file mode 100644 index 0000000000..2926468ee8 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/QueryStepTest.java @@ -0,0 +1,77 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; + +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import org.junit.jupiter.api.Test; + +class QueryStepTest { + + @Test + void shouldQualifyItsProjectionByCteName() { + QueryStep step = QueryStep.builder() + .cteName("filtered") + .selects(selects()) + .build(); + + Selects qualified = step.getQualifiedSelects(); + + assertEquals(name("filtered", "person"), qualified.getIds().getPrimaryColumn().getQualifiedName()); + assertEquals(name("filtered", "value"), qualified.explicitSelects().getFirst().getQualifiedName()); + } + + @Test + void shouldAddAnExplicitSelectWithoutChangingTheOriginalStep() { + QueryStep original = QueryStep.builder() + .cteName("source") + .selects(Selects.builder() + .ids(new SqlIdColumns(field(name("person"), String.class))) + .build()) + .build(); + + QueryStep extended = original.addSqlSelect(new FieldWrapper<>(field(name("value"), Integer.class))); + + assertTrue(original.getSelects().getSqlSelects().isEmpty()); + assertEquals(List.of("value"), extended.getSelects().explicitSelects().stream().map(org.jooq.Field::getName).toList()); + } + + @Test + void shouldCreateUnionAllStepFromFirstProjection() { + QueryStep first = QueryStep.builder() + .cteName("first") + .selects(selects()) + .projectionMode(ProjectionMode.INDIVIDUAL) + .build(); + QueryStep second = QueryStep.builder().cteName("second").selects(selects()).build(); + QueryStep predecessor = QueryStep.builder().cteName("predecessor").selects(selects()).build(); + + QueryStep union = QueryStep.createUnionAllStep( + List.of(first, second), + "combined", + List.of(predecessor), + true + ); + + assertEquals("combined", union.getCteName()); + assertEquals(List.of(second), union.getUnion()); + assertEquals(List.of(predecessor), union.getPredecessors()); + assertTrue(union.isUnion()); + assertTrue(union.isUnionAll()); + assertTrue(union.isNegate()); + assertFalse(union.isGroupBy()); + assertEquals(ProjectionMode.INDIVIDUAL, union.getProjectionMode()); + } + + private static Selects selects() { + return Selects.builder() + .ids(new SqlIdColumns(field(name("person"), String.class))) + .sqlSelect(new FieldWrapper<>(field(name("value"), Integer.class))) + .build(); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRendererTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRendererTest.java index b072420159..790d566f9a 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRendererTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRendererTest.java @@ -3,18 +3,19 @@ import static org.jooq.impl.DSL.field; import static org.jooq.impl.DSL.name; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.sql.Date; import java.util.List; import java.util.Optional; import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.ProjectionMode; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; -import com.bakdata.conquery.sql.compiler.rendering.SelectProjectionRenderer.ValidityDateRendering; import org.jooq.Field; import org.junit.jupiter.api.Test; @@ -24,7 +25,7 @@ class SelectProjectionRendererTest { void shouldAggregateValidityDatesForGroupedResults() { RecordingDialect dialect = new RecordingDialect(); - List> fields = SelectProjectionRenderer.render(selectsWithDates(), dialect, ValidityDateRendering.AGGREGATED); + List> fields = SelectProjectionRenderer.renderFinal(selectsWithDates(), dialect, ProjectionMode.AGGREGATED); assertEquals( List.of( @@ -43,12 +44,28 @@ void shouldAggregateValidityDatesForGroupedResults() { void shouldRenderValidityDatesWithoutAggregation() { RecordingDialect dialect = new RecordingDialect(); - SelectProjectionRenderer.render(selectsWithDates(), dialect, ValidityDateRendering.INDIVIDUAL); + SelectProjectionRenderer.renderFinal(selectsWithDates(), dialect, ProjectionMode.INDIVIDUAL); assertEquals(0, dialect.aggregatedRanges); assertEquals(2, dialect.renderedRanges); } + @Test + void shouldRejectIntermediateProjectionAsFinalOutput() { + RecordingDialect dialect = new RecordingDialect(); + + assertThrows( + IllegalArgumentException.class, + () -> SelectProjectionRenderer.renderFinal( + Selects.builder() + .ids(new SqlIdColumns(field(name("person"), String.class))) + .build(), + dialect, + ProjectionMode.INTERMEDIATE + ) + ); + } + private static Selects selectsWithDates() { ColumnDateRange validityDate = ColumnDateRange.of( field(name("valid_from"), Date.class), From f01c82de088d92c360c88da2e4c68bb243166c44 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:41:21 +0200 Subject: [PATCH 17/81] refactor(sql): move query step rendering into connector Move recursive CTE, grouping, and union rendering into the connector compiler. Keep backend integration limited to constructing and invoking the renderer, and cover predecessor and union rendering with focused connector tests. Co-authored-by: Codex GPT-5.6-sol --- .../dialect/LegacyCompilerDialect.java | 10 +- .../query/ConceptQueryConverter.java | 5 +- .../query/FormConversionHelper.java | 5 +- .../query/TableExportQueryConverter.java | 5 +- .../compiler/rendering/QueryStepRenderer.java | 63 ++++++------ .../rendering/QueryStepRendererTest.java | 99 +++++++++++++++++++ 6 files changed, 141 insertions(+), 46 deletions(-) rename backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/rendering/QueryStepRenderer.java (71%) create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/rendering/QueryStepRendererTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java index b299373aa6..33282dd05f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java @@ -9,6 +9,7 @@ import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.rendering.QueryStepRenderer; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.cqelement.CQAndConverter; import com.bakdata.conquery.sql.conversion.cqelement.CQDateRestrictionConverter; @@ -18,7 +19,6 @@ import com.bakdata.conquery.sql.conversion.cqelement.CQYesConverter; import com.bakdata.conquery.sql.conversion.cqelement.concept.CQConceptConverter; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; -import com.bakdata.conquery.sql.conversion.model.QueryStepTransformer; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.conversion.query.AbsoluteFormQueryConverter; import com.bakdata.conquery.sql.conversion.query.CQReusedQueryConverter; @@ -67,8 +67,8 @@ default Field aggregateDateRanges(Field start, Field end) { default List> getDefaultNodeConverters(DSLContext dslContext) { - QueryStepTransformer queryStepTransformer = new QueryStepTransformer(dslContext); - FormConversionHelper formConversionUtil = new FormConversionHelper(queryStepTransformer); + QueryStepRenderer queryStepRenderer = new QueryStepRenderer(dslContext); + FormConversionHelper formConversionUtil = new FormConversionHelper(queryStepRenderer); return List.of( new CQDateRestrictionConverter(), @@ -79,12 +79,12 @@ default List> getDefaultNodeConverters(DSLCon new CQConceptConverter(), new CQExternalConverter(), new CQReusedQueryConverter(), - new ConceptQueryConverter(queryStepTransformer), + new ConceptQueryConverter(queryStepRenderer), new SecondaryIdQueryConverter(), new AbsoluteFormQueryConverter(formConversionUtil), new EntityDateQueryConverter(formConversionUtil), new RelativFormQueryConverter(formConversionUtil), - new TableExportQueryConverter(queryStepTransformer) + new TableExportQueryConverter(queryStepRenderer) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java index aabc35327f..f20fe9b52c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java @@ -3,6 +3,7 @@ import com.bakdata.conquery.sql.compiler.ir.ProjectionMode; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.rendering.QueryStepRenderer; import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.concept.specific.CQNegation; import com.bakdata.conquery.models.query.DateAggregationMode; @@ -25,7 +26,7 @@ @RequiredArgsConstructor public class ConceptQueryConverter implements NodeConverter { - private final QueryStepTransformer queryStepTransformer; + private final QueryStepRenderer queryStepRenderer; private static Selects getPreFinalSelects(QueryStep preFinalStep, ConversionContext context) { Selects preFinalStepSelects = preFinalStep.getQualifiedSelects(); @@ -81,7 +82,7 @@ public ConversionContext convert(ConceptQuery conceptQuery, ConversionContext co .predecessors(predecessors) .build(); - Select finalQuery = this.queryStepTransformer.toSelectQuery(finalStep, context.getCompilerDialect()); + Select finalQuery = this.queryStepRenderer.toSelectQuery(finalStep, context.getCompilerDialect()); return contextAfterConversion.withFinalQuery(new SqlQuery(finalQuery, conceptQuery.getResultInfos())); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java index 2e707f1b0d..c7056ba364 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java @@ -3,6 +3,7 @@ import com.bakdata.conquery.sql.compiler.ir.ProjectionMode; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.rendering.QueryStepRenderer; import com.bakdata.conquery.apiv1.forms.FeatureGroup; import com.bakdata.conquery.apiv1.query.ArrayConceptQuery; import com.bakdata.conquery.apiv1.query.ConceptQuery; @@ -33,7 +34,7 @@ @RequiredArgsConstructor public class FormConversionHelper { - private final QueryStepTransformer queryStepTransformer; + private final QueryStepRenderer queryStepRenderer; /** * Selects the ID, resolution, index and date range from stratification table plus all explicit selects from the converted features step. @@ -159,7 +160,7 @@ private ConversionContext createFinalSelect( .predecessors(queriesToJoin) .build(); - Select selectQuery = queryStepTransformer.toSelectQuery(finalStep, context.getCompilerDialect()); + Select selectQuery = queryStepRenderer.toSelectQuery(finalStep, context.getCompilerDialect()); return context.withFinalQuery(new SqlQuery(selectQuery, resultInfos)); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java index a393110df0..ec49f9e141 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java @@ -3,6 +3,7 @@ import com.bakdata.conquery.sql.compiler.ir.ProjectionMode; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.rendering.QueryStepRenderer; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.apiv1.query.TableExportQuery; import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; @@ -43,7 +44,7 @@ public class TableExportQueryConverter implements NodeConverter selectQuery = queryStepTransformer.toSelectQuery(unionedTables, context.getCompilerDialect()); + final Select selectQuery = queryStepRenderer.toSelectQuery(unionedTables, context.getCompilerDialect()); return context.withFinalQuery(new SqlQuery(selectQuery, tableExportQuery.getResultInfos())); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/rendering/QueryStepRenderer.java similarity index 71% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/rendering/QueryStepRenderer.java index 6db12f7b98..d9574db96b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/rendering/QueryStepRenderer.java @@ -1,27 +1,32 @@ -package com.bakdata.conquery.sql.conversion.model; +package com.bakdata.conquery.sql.compiler.rendering; + +import java.util.List; +import java.util.stream.Stream; import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import com.bakdata.conquery.sql.compiler.ir.ProjectionMode; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.rendering.SelectProjectionRenderer; import lombok.RequiredArgsConstructor; -import org.jooq.*; +import org.jooq.CommonTableExpression; +import org.jooq.DSLContext; +import org.jooq.Field; import org.jooq.Record; +import org.jooq.Select; +import org.jooq.SelectConditionStep; +import org.jooq.SelectHavingStep; +import org.jooq.SelectSelectStep; import org.jooq.impl.DSL; -import java.util.List; -import java.util.stream.Stream; - -/** - * Transformer for translating the intermediate representation of {@link QueryStep} into the final SQL query. - */ +/** Renders a tree of compiler query-step IR into a final jOOQ SELECT statement. */ @RequiredArgsConstructor -public class QueryStepTransformer { +public class QueryStepRenderer { private final DSLContext dslContext; /** - * Converts a given {@link QueryStep} into an executable SELECT statement. + * Render a final query step and all of its predecessor CTEs. + * + * @throws IllegalArgumentException when the supplied root step has an intermediate projection */ public Select toSelectQuery(QueryStep queryStep, CompilerDialect dialect) { @@ -36,13 +41,11 @@ public Select toSelectQuery(QueryStep queryStep, CompilerDialect dialect .from(queryStep.getFromTables()) .where(queryStep.getConditions()); - // grouping SelectHavingStep grouped = queryBase; if (queryStep.isGroupBy()) { grouped = queryBase.groupBy(queryStep.getGroupBy()); } - // union if (queryStep.isUnion()) { return union(queryStep, grouped, dialect); } @@ -51,8 +54,7 @@ public Select toSelectQuery(QueryStep queryStep, CompilerDialect dialect } private List> constructPredecessorCteList(QueryStep queryStep, CompilerDialect dialect) { - return predecessorCtes(queryStep, dialect) - .toList(); + return predecessorCtes(queryStep, dialect).toList(); } private List> toCteList(QueryStep queryStep, CompilerDialect dialect) { @@ -73,18 +75,13 @@ private CommonTableExpression toCte(QueryStep queryStep, CompilerDialect } private Select toSelectStep(QueryStep queryStep, CompilerDialect dialect) { - - SelectSelectStep selectClause; - List> allSelects = queryStep.getProjectionMode() == ProjectionMode.INTERMEDIATE ? queryStep.getSelects().all() : SelectProjectionRenderer.renderFinal(queryStep.getSelects(), dialect, queryStep.getProjectionMode()); - if (queryStep.isSelectDistinct()) { - selectClause = dslContext.selectDistinct(allSelects); - } else { - selectClause = dslContext.select(allSelects); - } + SelectSelectStep selectClause = queryStep.isSelectDistinct() + ? this.dslContext.selectDistinct(allSelects) + : this.dslContext.select(allSelects); Select selectStep = selectClause.from(queryStep.getFromTables()).where(queryStep.getConditions()); @@ -101,19 +98,15 @@ private Select toSelectStep(QueryStep queryStep, CompilerDialect dialect private Select union(QueryStep queryStep, Select base, CompilerDialect dialect) { for (QueryStep unionStep : queryStep.getUnion()) { - Select selectStep = - queryStep.getProjectionMode() != ProjectionMode.INTERMEDIATE ? - // Final projections must use the same physical representation in every union branch. - toSelectQuery(unionStep, dialect) : - toSelectStep(unionStep, dialect); - - if (queryStep.isUnionAll()) { - base = base.unionAll(selectStep); - } else { - base = base.union(selectStep); - } + // Final projections must use the same physical representation in every union branch. + Select selectStep = queryStep.getProjectionMode() != ProjectionMode.INTERMEDIATE + ? toSelectQuery(unionStep, dialect) + : toSelectStep(unionStep, dialect); + + base = queryStep.isUnionAll() + ? base.unionAll(selectStep) + : base.union(selectStep); } return base; } - } diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/rendering/QueryStepRendererTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/rendering/QueryStepRendererTest.java new file mode 100644 index 0000000000..8109a3082e --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/rendering/QueryStepRendererTest.java @@ -0,0 +1,99 @@ +package com.bakdata.conquery.sql.compiler.rendering; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.jooq.impl.DSL.table; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.Date; +import java.util.List; +import java.util.Locale; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.ProjectionMode; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import org.jooq.Field; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Test; + +class QueryStepRendererTest { + + private static final CompilerDialect DIALECT = new TestDialect(); + + @Test + void shouldRenderPredecessorAsCommonTableExpression() { + QueryStep predecessor = queryStep("source_step", "events"); + QueryStep resultStep = QueryStep.builder() + .selects(predecessor.getQualifiedSelects()) + .fromTable(QueryStep.toTableLike("source_step")) + .predecessor(predecessor) + .projectionMode(ProjectionMode.AGGREGATED) + .build(); + + String sql = render(resultStep); + + assertTrue(sql.contains("with \"source_step\" as")); + assertTrue(sql.contains("from \"events\"")); + assertTrue(sql.contains("from \"source_step\"")); + } + + @Test + void shouldRenderUnionAll() { + QueryStep first = queryStep(null, "first_events").toBuilder() + .projectionMode(ProjectionMode.INDIVIDUAL) + .build(); + QueryStep second = queryStep(null, "second_events").toBuilder() + .projectionMode(ProjectionMode.INDIVIDUAL) + .build(); + QueryStep union = QueryStep.createUnionAllStep(List.of(first, second), null, List.of(), false); + + String sql = render(union); + + assertTrue(sql.contains("from \"first_events\"")); + assertTrue(sql.contains("union all")); + assertTrue(sql.contains("from \"second_events\"")); + } + + private static QueryStep queryStep(String cteName, String tableName) { + return QueryStep.builder() + .cteName(cteName) + .selects(Selects.builder() + .ids(new SqlIdColumns(field(name("person"), String.class))) + .build()) + .fromTable(table(name(tableName))) + .build(); + } + + private static String render(QueryStep queryStep) { + return new QueryStepRenderer(DSL.using(SQLDialect.POSTGRES)) + .toSelectQuery(queryStep, DIALECT) + .getSQL() + .toLowerCase(Locale.ROOT); + } + + private static final class TestDialect implements CompilerDialect { + + @Override + public Field anyValue(Field field) { + return field; + } + + @Override + public Field renderDateRange(Field start, Field end) { + return field(name("rendered_range"), Object.class); + } + + @Override + public Field aggregateDateRanges(Field start, Field end) { + return field(name("aggregated_ranges"), Object.class); + } + + @Override + public int getNameMaxLength() { + return 128; + } + } +} From 42c1d27859058ac377894f23c96340827ea2ceed Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:37:37 +0200 Subject: [PATCH 18/81] refactor(sql): move query step joins into connector Move join modes, table joining, ID coalescing, and select merging into the connector compiler IR. Rename the remaining backend helper to QueryStepComposer and cover portable join construction with focused connector tests. Co-authored-by: Codex GPT-5.6-sol --- .../conversion/cqelement/CQAndConverter.java | 10 +- .../conversion/cqelement/CQOrConverter.java | 10 +- .../cqelement/concept/CQConceptConverter.java | 6 +- .../cqelement/concept/JoinBranchesCte.java | 8 +- .../conversion/model/ConqueryJoinType.java | 10 -- .../sql/conversion/model/NameGenerator.java | 9 +- ...StepJoiner.java => QueryStepComposer.java} | 108 ++++-------------- .../query/ConceptQueryConverter.java | 13 ++- .../query/FormConversionHelper.java | 6 +- .../conquery/sql/compiler/ir/JoinMode.java | 8 ++ .../sql/compiler/ir/QueryStepJoiner.java | 91 +++++++++++++++ .../sql/compiler/ir/QueryStepJoinerTest.java | 90 +++++++++++++++ 12 files changed, 250 insertions(+), 119 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ConqueryJoinType.java rename backend/src/main/java/com/bakdata/conquery/sql/conversion/model/{QueryStepJoiner.java => QueryStepComposer.java} (76%) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/JoinMode.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepJoiner.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/QueryStepJoinerTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java index 5cd5af0273..5ab611ca0e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java @@ -1,11 +1,11 @@ package com.bakdata.conquery.sql.conversion.cqelement; import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; -import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.model.ConqueryJoinType; +import com.bakdata.conquery.sql.compiler.ir.JoinMode; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; import com.bakdata.conquery.sql.compiler.ir.select.ExistsSqlSelect; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.model.QueryStepComposer; public class CQAndConverter implements NodeConverter { @@ -23,10 +23,10 @@ public ConversionContext convert(CQAnd andNode, ConversionContext context) { joined = withConvertedChild.getLastConvertedStep(); } else { - joined = QueryStepJoiner.joinChildren( + joined = QueryStepComposer.joinChildren( andNode.getChildren(), context, - ConqueryJoinType.INNER_JOIN, + JoinMode.INNER, andNode.getDateAction() ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java index 246fa5a08e..8c3b939de6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java @@ -1,11 +1,11 @@ package com.bakdata.conquery.sql.conversion.cqelement; import com.bakdata.conquery.apiv1.query.concept.specific.CQOr; -import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.model.ConqueryJoinType; +import com.bakdata.conquery.sql.compiler.ir.JoinMode; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; import com.bakdata.conquery.sql.compiler.ir.select.ExistsSqlSelect; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.model.QueryStepComposer; public class CQOrConverter implements NodeConverter { @@ -23,10 +23,10 @@ public ConversionContext convert(CQOr orNode, ConversionContext context) { joined = withConvertedChild.getLastConvertedStep(); } else { - joined = QueryStepJoiner.joinChildren( + joined = QueryStepComposer.joinChildren( orNode.getChildren(), context, - ConqueryJoinType.OUTER_JOIN, + JoinMode.FULL_OUTER, orNode.getDateAction() ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index f41c1bd405..3e9c502897 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -1,6 +1,8 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; +import com.bakdata.conquery.sql.compiler.ir.JoinMode; import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStepJoiner; import com.bakdata.conquery.sql.compiler.ir.Selects; import java.util.ArrayList; import java.util.List; @@ -97,7 +99,7 @@ private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcep .sqlSelects(allConceptSelects) .build(); - TableLike joinedTable = QueryStepJoiner.constructJoinedTable(queriesToJoin, ConqueryJoinType.INNER_JOIN, context); + TableLike joinedTable = QueryStepJoiner.join(queriesToJoin, JoinMode.INNER); // group by everything which is not part of an aggregation in this step List> groupByFields = @@ -241,7 +243,7 @@ public ConversionContext convert(CQConcept cqConcept, ConversionContext context) .flatMap(cqTable -> convertCqTable(tablePath, cqConcept, cqTable, context).stream()) .toList(); - QueryStep joinedStep = QueryStepJoiner.joinSteps(convertedCQTables, ConqueryJoinType.OUTER_JOIN, DateAggregationAction.MERGE, context); + QueryStep joinedStep = QueryStepComposer.joinSteps(convertedCQTables, JoinMode.FULL_OUTER, DateAggregationAction.MERGE, context); QueryStep lastConceptStep = finishConceptConversion(joinedStep, cqConcept, tablePath, context); return context.withQueryStep(lastConceptStep); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java index 9d3c12ef24..8a1400d872 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java @@ -6,12 +6,12 @@ import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingContext; import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.ConqueryJoinType; +import com.bakdata.conquery.sql.compiler.ir.JoinMode; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; +import com.bakdata.conquery.sql.compiler.ir.QueryStepJoiner; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.aggregator.SumSqlAggregator; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import org.jooq.Record; @@ -85,7 +85,7 @@ protected QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { .forEach(queriesToJoin::add); Selects selects = collectSelects(validityDate, queriesToJoin, tableContext); - TableLike fromTable = QueryStepJoiner.constructJoinedTable(queriesToJoin, ConqueryJoinType.OUTER_JOIN, tableContext.getConversionContext()); + TableLike fromTable = QueryStepJoiner.join(queriesToJoin, JoinMode.FULL_OUTER); return QueryStep.builder() .selects(selects) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ConqueryJoinType.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ConqueryJoinType.java deleted file mode 100644 index 3dee591ad9..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ConqueryJoinType.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model; - -/** - * A subset of {@link org.jooq.JoinType} for join types that Conquery supports. - */ -public enum ConqueryJoinType { - INNER_JOIN, - OUTER_JOIN, - LEFT_JOIN -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java index 20004a4ff2..429068aea6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java @@ -8,6 +8,7 @@ import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.identifiable.LabeledNamespaceIdentifiable; +import com.bakdata.conquery.sql.compiler.ir.JoinMode; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -65,11 +66,11 @@ public String conceptConnectorName(CQConcept concept, Connector connector, Local return ensureValidLength("concept_%s_%s-%d".formatted(conceptLabel, connectorName, conceptCount)); } - public String joinedNodeName(ConqueryJoinType logicalOperation) { + public String joinedNodeName(JoinMode logicalOperation) { return switch (logicalOperation) { - case INNER_JOIN -> "AND-%d".formatted(++andCount); - case OUTER_JOIN -> "OR-%d".formatted(++orCount); - case LEFT_JOIN -> throw new UnsupportedOperationException("Creating CTE names for LEFT_JOIN nodes is not supported"); + case INNER -> "AND-%d".formatted(++andCount); + case FULL_OUTER -> "OR-%d".formatted(++orCount); + case LEFT -> throw new UnsupportedOperationException("Creating CTE names for LEFT joins is not supported"); }; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java similarity index 76% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java rename to backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java index 1f61f55be4..077abe13de 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.sql.conversion.model; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.Selects; import static org.jooq.impl.DSL.*; import java.util.ArrayList; @@ -9,34 +7,43 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; -import java.util.stream.Stream; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.models.config.IdColumnConfig; import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.sql.compiler.ir.JoinMode; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStepJoiner; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.cqelement.aggregation.DateAggregationDates; import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.google.common.base.Preconditions; import org.jooq.Condition; import org.jooq.Field; -import org.jooq.Function3; import org.jooq.Record; import org.jooq.Table; import org.jooq.TableLike; -import org.jooq.TableOnConditionStep; import org.jooq.impl.DSL; -public class QueryStepJoiner { +/** + * Composes converted backend query nodes, including negation and date aggregation, into compiler query steps. + * + *

TODO Move this composition into {@code sql-connector} after its remaining backend query, schema, and legacy + * dialect dependencies have framework-neutral compiler contracts.

+ */ +public class QueryStepComposer { private static final String NEGATED_CTE_SUFFIX = "_negated"; + private QueryStepComposer() { + } + /** * Implements an antijoin against the allIdsTable defined in {@link IdColumnConfig#getTable()}, for negation without siblings. */ @@ -79,7 +86,7 @@ public static QueryStep antiJoinWithAllIdsTable(QueryStep queryStep, ConversionC public static QueryStep joinChildren( Iterable children, ConversionContext context, - ConqueryJoinType logicalOperation, + JoinMode logicalOperation, DateAggregationAction dateAggregationAction ) { ConversionContext childrenContext = context.createChildContext(); @@ -94,7 +101,7 @@ public static QueryStep joinChildren( public static QueryStep joinSteps( List queriesToJoin, - ConqueryJoinType logicalOperation, + JoinMode logicalOperation, DateAggregationAction dateAggregationAction, ConversionContext context ) { @@ -111,7 +118,7 @@ public static QueryStep joinSteps( private static QueryStep doJoin( List queriesToJoin, - ConqueryJoinType logicalOperation, + JoinMode logicalOperation, DateAggregationAction dateAggregationAction, ConversionContext context) { // no join required @@ -120,9 +127,9 @@ private static QueryStep doJoin( } String joinedNodeName = context.getNameGenerator().joinedNodeName(logicalOperation); - SqlIdColumns ids = coalesceIds(queriesToJoin); - List mergedSelects = mergeSelects(queriesToJoin); - TableLike joinedTable = constructJoinedTable(queriesToJoin, logicalOperation, context); + SqlIdColumns ids = QueryStepJoiner.coalesceIds(queriesToJoin); + List mergedSelects = QueryStepJoiner.mergeSelects(queriesToJoin); + TableLike joinedTable = QueryStepJoiner.join(queriesToJoin, logicalOperation); QueryStep joinedStep; QueryStep.QueryStepBuilder joinedStepBuilder = QueryStep.builder() @@ -155,7 +162,7 @@ else if (dateAggregationDates.getValidityDates().size() == 1) { */ private static QueryStep joinStepsContainingNegation( List queriesToJoin, - ConqueryJoinType logicalOperation, + JoinMode logicalOperation, DateAggregationAction dateAggregationAction, ConversionContext context ) { @@ -174,11 +181,11 @@ private static QueryStep joinStepsContainingNegation( QueryStep negateJoined = doJoin(withNegation, logicalOperation, dateAggregationAction, context); Selects.SelectsBuilder selects = nonNegateJoined.getQualifiedSelects() - .toBuilder() - .sqlSelects(mergeSelects(List.of(nonNegateJoined, negateJoined))); + .toBuilder() + .sqlSelects(QueryStepJoiner.mergeSelects(List.of(nonNegateJoined, negateJoined))); - if (logicalOperation == ConqueryJoinType.INNER_JOIN) { + if (logicalOperation == JoinMode.INNER) { Table table = table(name(nonNegateJoined.getCteName())) .leftOuterJoin(table(name(negateJoined.getCteName()))) .on(nonNegateJoined.getQualifiedSelects().getIds().getPrimaryColumn() @@ -237,71 +244,6 @@ private static QueryStep joinStepsContainingNegation( } - public static TableLike constructJoinedTable(List queriesToJoin, ConqueryJoinType logicalOperation, ConversionContext context) { - - SqlFunctionProvider functionProvider = context.getFunctionProvider(); - - Function3, Table, List, TableOnConditionStep> joinType = - switch (logicalOperation) { - case INNER_JOIN -> functionProvider::innerJoin; - case OUTER_JOIN -> functionProvider::fullOuterJoin; - case LEFT_JOIN -> functionProvider::leftJoin; - }; - - Table joinedQuery = getIntitialJoinTable(queriesToJoin); - - for (int i = 0; i < queriesToJoin.size() - 1; i++) { - - QueryStep leftPartQS = queriesToJoin.get(i); - QueryStep rightPartQS = queriesToJoin.get(i + 1); - - SqlIdColumns leftIds = leftPartQS.getQualifiedSelects().getIds(); - SqlIdColumns rightIds = rightPartQS.getQualifiedSelects().getIds(); - - List joinIdsCondition = leftIds.join(rightIds); - - Condition joinDateCondition = joinOnStratification(leftPartQS, rightPartQS); - - List joinConditions = Stream.concat(joinIdsCondition.stream(), Stream.of(joinDateCondition)).collect(Collectors.toList()); - - Table rightPartTable = table(name(rightPartQS.getCteName())); - joinedQuery = joinType.apply(joinedQuery, rightPartTable, joinConditions); - } - - return joinedQuery; - } - - /** - * join on stratification date if present - */ - private static Condition joinOnStratification(QueryStep leftPartQS, QueryStep rightPartQS) { - if (leftPartQS.getSelects().getStratificationDate().isEmpty() || rightPartQS.getSelects().getStratificationDate().isEmpty()) { - //TODO use unconditionalJoin in the future. Hana does not like noCondition joins - return DSL.noCondition(); - } - - ColumnDateRange leftStratificationDate = leftPartQS.getQualifiedSelects().getStratificationDate().get(); - ColumnDateRange rightStratificationDate = rightPartQS.getQualifiedSelects().getStratificationDate().get(); - - return leftStratificationDate.join(rightStratificationDate); - } - - public static List mergeSelects(List querySteps) { - return querySteps.stream() - .flatMap(queryStep -> queryStep.getQualifiedSelects().getSqlSelects().stream()) - .collect(Collectors.toList()); - } - - public static SqlIdColumns coalesceIds(List querySteps) { - List ids = querySteps.stream().map(QueryStep::getQualifiedSelects).map(Selects::getIds).toList(); - Preconditions.checkArgument(!ids.isEmpty(), "Need at least 1 query step in the list to coalesce Ids"); - return ids.getFirst().coalesce(ids.subList(1, ids.size())); - } - - private static Table getIntitialJoinTable(List queriesToJoin) { - return table(name(queriesToJoin.getFirst().getCteName())); - } - private static QueryStep buildJoinedStep( SqlIdColumns ids, List mergedSelects, diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java index f20fe9b52c..41ad012ba7 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java @@ -1,7 +1,9 @@ package com.bakdata.conquery.sql.conversion.query; +import com.bakdata.conquery.sql.compiler.ir.JoinMode; import com.bakdata.conquery.sql.compiler.ir.ProjectionMode; import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStepJoiner; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.rendering.QueryStepRenderer; import com.bakdata.conquery.apiv1.query.ConceptQuery; @@ -47,10 +49,9 @@ private static TableLike getFinalTable(QueryStep preFinalStep, Conversio if (externalExtras == null) { return QueryStep.toTableLike(preFinalStep.getCteName()); } - return QueryStepJoiner.constructJoinedTable( + return QueryStepJoiner.join( List.of(preFinalStep, externalExtras), - ConqueryJoinType.INNER_JOIN, - context + JoinMode.INNER ); } @@ -67,7 +68,11 @@ public ConversionContext convert(ConceptQuery conceptQuery, ConversionContext co QueryStep preFinalStep = contextAfterConversion.getLastConvertedStep(); // negation of a single node results in an anti-join with all ids table if (preFinalStep.isNegate()) { - preFinalStep = QueryStepJoiner.antiJoinWithAllIdsTable(preFinalStep, contextAfterConversion, CQNegation.determineDateAction(conceptQuery.getDateAggregationMode())); + preFinalStep = QueryStepComposer.antiJoinWithAllIdsTable( + preFinalStep, + contextAfterConversion, + CQNegation.determineDateAction(conceptQuery.getDateAggregationMode()) + ); } Selects preFinalSelects = getPreFinalSelects(preFinalStep, contextAfterConversion); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java index c7056ba364..932c559df9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java @@ -1,7 +1,9 @@ package com.bakdata.conquery.sql.conversion.query; +import com.bakdata.conquery.sql.compiler.ir.JoinMode; import com.bakdata.conquery.sql.compiler.ir.ProjectionMode; import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStepJoiner; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.rendering.QueryStepRenderer; import com.bakdata.conquery.apiv1.forms.FeatureGroup; @@ -127,7 +129,7 @@ public ConversionContext convertForm( // child context contains the converted feature's QuerySteps List queriesToJoin = childContext.getQuerySteps(); - QueryStep joinedFeatures = QueryStepJoiner.joinSteps(queriesToJoin, ConqueryJoinType.OUTER_JOIN, DateAggregationAction.BLOCK, context); + QueryStep joinedFeatures = QueryStepComposer.joinSteps(queriesToJoin, JoinMode.FULL_OUTER, DateAggregationAction.BLOCK, context); return createFinalSelect(formType, stratificationTable, joinedFeatures, resultInfos, context); } @@ -151,7 +153,7 @@ private ConversionContext createFinalSelect( ); List queriesToJoin = List.of(stratificationTable, convertedFeatures); - TableLike joinedTable = QueryStepJoiner.constructJoinedTable(queriesToJoin, ConqueryJoinType.LEFT_JOIN, context); + TableLike joinedTable = QueryStepJoiner.join(queriesToJoin, JoinMode.LEFT); QueryStep finalStep = QueryStep.builder() .cteName(null) // the final QueryStep won't be converted to a CTE .projectionMode(ProjectionMode.AGGREGATED) diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/JoinMode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/JoinMode.java new file mode 100644 index 0000000000..571554a445 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/JoinMode.java @@ -0,0 +1,8 @@ +package com.bakdata.conquery.sql.compiler.ir; + +/** Supported ways to combine the tables produced by compiler query steps. */ +public enum JoinMode { + INNER, + FULL_OUTER, + LEFT +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepJoiner.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepJoiner.java new file mode 100644 index 0000000000..c957f93eda --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepJoiner.java @@ -0,0 +1,91 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import static org.jooq.impl.DSL.name; +import static org.jooq.impl.DSL.noCondition; +import static org.jooq.impl.DSL.table; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import org.jooq.Condition; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.TableLike; +import org.jooq.TableOnConditionStep; + +/** Combines compiler query-step IR without applying backend query semantics. */ +public final class QueryStepJoiner { + + private QueryStepJoiner() { + } + + /** + * Join the CTE tables represented by the supplied steps on their IDs and, where present, stratification dates. + */ + public static TableLike join(List querySteps, JoinMode joinMode) { + if (querySteps.isEmpty()) { + throw new IllegalArgumentException("Need at least one query step to construct a join"); + } + + Table joinedTable = table(name(querySteps.getFirst().getCteName())); + for (int index = 0; index < querySteps.size() - 1; index++) { + QueryStep leftStep = querySteps.get(index); + QueryStep rightStep = querySteps.get(index + 1); + + List joinConditions = Stream.concat( + leftStep.getQualifiedSelects().getIds().join(rightStep.getQualifiedSelects().getIds()).stream(), + Stream.of(joinOnStratification(leftStep, rightStep)) + ).toList(); + + Table rightTable = table(name(rightStep.getCteName())); + joinedTable = join(joinedTable, rightTable, joinConditions, joinMode); + } + + return joinedTable; + } + + /** Merge the explicitly selected fields from the supplied query steps. */ + public static List mergeSelects(List querySteps) { + return querySteps.stream() + .flatMap(queryStep -> queryStep.getQualifiedSelects().getSqlSelects().stream()) + .toList(); + } + + /** Coalesce the entity IDs selected by the supplied query steps. */ + public static SqlIdColumns coalesceIds(List querySteps) { + List ids = querySteps.stream() + .map(QueryStep::getQualifiedSelects) + .map(Selects::getIds) + .toList(); + if (ids.isEmpty()) { + throw new IllegalArgumentException("Need at least one query step to coalesce IDs"); + } + return ids.getFirst().coalesce(ids.subList(1, ids.size())); + } + + private static TableOnConditionStep join( + Table left, + Table right, + List conditions, + JoinMode joinMode + ) { + Condition[] conditionArray = conditions.toArray(Condition[]::new); + return switch (joinMode) { + case INNER -> left.innerJoin(right).on(conditionArray); + case FULL_OUTER -> left.fullOuterJoin(right).on(conditionArray); + case LEFT -> left.leftJoin(right).on(conditionArray); + }; + } + + private static Condition joinOnStratification(QueryStep leftStep, QueryStep rightStep) { + Optional leftDate = leftStep.getQualifiedSelects().getStratificationDate(); + Optional rightDate = rightStep.getQualifiedSelects().getStratificationDate(); + if (leftDate.isEmpty() || rightDate.isEmpty()) { + return noCondition(); + } + return leftDate.get().join(rightDate.get()); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/QueryStepJoinerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/QueryStepJoinerTest.java new file mode 100644 index 0000000000..60aec0b400 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/QueryStepJoinerTest.java @@ -0,0 +1,90 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.Date; +import java.util.List; +import java.util.Locale; +import java.util.Optional; + +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Test; + +class QueryStepJoinerTest { + + @Test + void shouldJoinStepsOnIdsAndStratificationDates() { + QueryStep first = queryStep("first", "first_value"); + QueryStep second = queryStep("second", "second_value"); + + String sql = renderJoin(first, second, JoinMode.FULL_OUTER); + + assertTrue(sql.contains("full outer join \"second\"")); + assertTrue(sql.contains("\"first\".\"person\" = \"second\".\"person\"")); + assertTrue(sql.contains("\"first\".\"window_start\" = \"second\".\"window_start\"")); + assertTrue(sql.contains("\"first\".\"window_end\" = \"second\".\"window_end\"")); + } + + @Test + void shouldRenderEverySupportedJoinMode() { + QueryStep first = queryStep("first", "first_value"); + QueryStep second = queryStep("second", "second_value"); + + assertTrue(renderJoin(first, second, JoinMode.INNER).contains("join \"second\"")); + assertTrue(renderJoin(first, second, JoinMode.FULL_OUTER).contains("full outer join \"second\"")); + assertTrue(renderJoin(first, second, JoinMode.LEFT).contains("left outer join \"second\"")); + } + + @Test + void shouldMergeSelectsAndCoalesceIds() { + QueryStep first = queryStep("first", "first_value"); + QueryStep second = queryStep("second", "second_value"); + + assertEquals( + List.of(name("first", "first_value"), name("second", "second_value")), + QueryStepJoiner.mergeSelects(List.of(first, second)).stream() + .flatMap(select -> select.toFields().stream()) + .map(org.jooq.Field::getQualifiedName) + .toList() + ); + assertEquals( + SharedAliases.PRIMARY_COLUMN.getAlias(), + QueryStepJoiner.coalesceIds(List.of(first, second)).getPrimaryColumn().getName() + ); + } + + @Test + void shouldRejectEmptyInput() { + assertThrows(IllegalArgumentException.class, () -> QueryStepJoiner.join(List.of(), JoinMode.INNER)); + assertThrows(IllegalArgumentException.class, () -> QueryStepJoiner.coalesceIds(List.of())); + } + + private static String renderJoin(QueryStep first, QueryStep second, JoinMode joinMode) { + return DSL.using(SQLDialect.POSTGRES) + .select() + .from(QueryStepJoiner.join(List.of(first, second), joinMode)) + .getSQL() + .toLowerCase(Locale.ROOT); + } + + private static QueryStep queryStep(String cteName, String valueColumn) { + return QueryStep.builder() + .cteName(cteName) + .selects(Selects.builder() + .ids(new SqlIdColumns(field(name("person"), String.class))) + .stratificationDate(Optional.of(ColumnDateRange.of( + field(name("window_start"), Date.class), + field(name("window_end"), Date.class) + ))) + .sqlSelect(new FieldWrapper<>(field(name(valueColumn), Integer.class))) + .build()) + .build(); + } +} From 1fc63fedad607443be40bbaaba4d954466bbcc17 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 3 Sep 2026 08:33:27 +0200 Subject: [PATCH 19/81] refactor(sql): move date aggregation inputs into connector Move DateAggregationDates into the connector compiler IR as a qualifiable value object. Rewire the legacy aggregation pipeline and cover filtering, query-step qualification, selected bounds, and empty inputs with focused tests. Co-authored-by: Codex GPT-5.6-sol --- .../aggregation/AnsiSqlDateAggregator.java | 1 + .../aggregation/DateAggregationContext.java | 1 + .../aggregation/IntermediateTableCte.java | 1 + .../IntersectAggregationAction.java | 1 + .../aggregation/MergeAggregateAction.java | 1 + .../aggregation/NodeNoOverlapCte.java | 1 + .../cqelement/aggregation/OverlapCte.java | 1 + .../cqelement/aggregation/RowNumberCte.java | 1 + .../aggregation/SqlAggregationAction.java | 1 + .../conversion/dialect/SqlDateAggregator.java | 2 +- .../conversion/model/QueryStepComposer.java | 2 +- .../compiler/ir}/DateAggregationDates.java | 43 +++++----- .../compiler/ir/DateAggregationDatesTest.java | 83 +++++++++++++++++++ 13 files changed, 114 insertions(+), 25 deletions(-) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/DateAggregationDates.java (53%) create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/DateAggregationDatesTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java index fb24f7becc..b12ee2330e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java @@ -7,6 +7,7 @@ import java.util.List; import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingContext; import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java index 9d397c2146..39139b4f5a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.Map; +import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java index 5bf64af4f3..ab9370f4fb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java @@ -4,6 +4,7 @@ import java.sql.Date; import java.util.List; +import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java index b3eb77f7f9..b2dc011a24 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java @@ -6,6 +6,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.NameGenerator; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java index 62acdd2b07..8c691e9c74 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java @@ -6,6 +6,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.NameGenerator; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java index 0ad0a8960c..0b4acbca20 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.Optional; +import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java index 2fe1a0928e..87a86fea5d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java @@ -4,6 +4,7 @@ import java.util.List; import java.util.Optional; +import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QueryStep; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java index d2ffd735ae..d13b5c3b06 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java @@ -3,6 +3,7 @@ import java.util.ArrayList; import java.util.Optional; +import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java index c4abded10d..44ec9cae10 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java @@ -3,6 +3,7 @@ import java.util.List; import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.NameGenerator; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java index 185065793b..620d7a28ee 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java @@ -4,7 +4,7 @@ import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.cqelement.aggregation.DateAggregationDates; +import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java index 077abe13de..dad5bc2015 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java @@ -12,6 +12,7 @@ import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.models.config.IdColumnConfig; import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; import com.bakdata.conquery.sql.compiler.ir.JoinMode; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.QueryStepJoiner; @@ -21,7 +22,6 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.cqelement.aggregation.DateAggregationDates; import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/DateAggregationDates.java similarity index 53% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/DateAggregationDates.java index 7727308f4c..a7095c810a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/DateAggregationDates.java @@ -1,14 +1,11 @@ -package com.bakdata.conquery.sql.conversion.cqelement.aggregation; +package com.bakdata.conquery.sql.compiler.ir; import java.sql.Date; import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.AccessLevel; @@ -17,34 +14,34 @@ import org.jooq.Field; /** - * {@link DateAggregationDates} keep track of all validity dates of list of {@link QueryStep}s that need to be aggregated. + * Validity-date inputs carried between query steps while the compiler prepares date aggregation. */ @Getter @AllArgsConstructor(access = AccessLevel.PRIVATE) -public class DateAggregationDates { +public final class DateAggregationDates implements Qualifiable { private final List validityDates; - public static DateAggregationDates forValidityDates(final List> validityDates) { - final List filtered = validityDates.stream() - .filter(Optional::isPresent) - .map(Optional::get) - .toList(); + public static DateAggregationDates forValidityDates(List> validityDates) { + List filtered = validityDates.stream() + .filter(Optional::isPresent) + .map(Optional::get) + .toList(); return new DateAggregationDates(filtered); } public static DateAggregationDates forSingleStep(QueryStep queryStep) { List validityDates = queryStep.getSelects() - .getValidityDate() - .map(List::of) - .orElse(Collections.emptyList()); + .getValidityDate() + .map(List::of) + .orElse(Collections.emptyList()); return new DateAggregationDates(validityDates); } public static DateAggregationDates forSteps(List querySteps) { - final List validityDates = querySteps.stream() - .flatMap(queryStep -> queryStep.getQualifiedSelects().getValidityDate().stream()) - .toList(); + List validityDates = querySteps.stream() + .flatMap(queryStep -> queryStep.getQualifiedSelects().getValidityDate().stream()) + .toList(); return new DateAggregationDates(validityDates); } @@ -62,16 +59,16 @@ public List> allEnds() { public List allStartsAndEnds() { return this.validityDates.stream() - .flatMap(validityDate -> validityDate.toFields().stream()) - .map(FieldWrapper::new) - .collect(Collectors.toList()); + .flatMap(validityDate -> validityDate.toFields().stream()) + .map(FieldWrapper::new) + .toList(); } + @Override public DateAggregationDates qualify(String qualifier) { List qualified = this.validityDates.stream() - .map(validityDate -> validityDate.qualify(qualifier)) - .toList(); - // validity dates will already be numerated, no we don't need no apply a counter again + .map(validityDate -> validityDate.qualify(qualifier)) + .toList(); return new DateAggregationDates(qualified); } diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/DateAggregationDatesTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/DateAggregationDatesTest.java new file mode 100644 index 0000000000..3d7e462368 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/DateAggregationDatesTest.java @@ -0,0 +1,83 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.Date; +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import org.jooq.Field; +import org.junit.jupiter.api.Test; + +class DateAggregationDatesTest { + + @Test + void shouldCollectPresentValidityDates() { + ColumnDateRange first = dateRange("first_start", "first_end"); + ColumnDateRange second = dateRange("second_start", "second_end"); + + DateAggregationDates dates = DateAggregationDates.forValidityDates( + List.of(Optional.empty(), Optional.of(first), Optional.of(second)) + ); + + assertEquals(List.of(first, second), dates.getValidityDates()); + assertEquals(List.of("first_start", "second_start"), names(dates.allStarts())); + assertEquals(List.of("first_end", "second_end"), names(dates.allEnds())); + assertEquals( + List.of("first_start", "first_end", "second_start", "second_end"), + dates.allStartsAndEnds().stream() + .flatMap(select -> select.toFields().stream()) + .map(Field::getName) + .toList() + ); + assertFalse(dates.dateAggregationImpossible()); + } + + @Test + void shouldCollectQualifiedValidityDatesFromQuerySteps() { + QueryStep first = queryStep("first", dateRange("valid_from", "valid_to")); + QueryStep second = queryStep("second", dateRange("valid_from", "valid_to")); + + DateAggregationDates dates = DateAggregationDates.forSteps(List.of(first, second)); + + assertEquals( + List.of(name("first", "valid_from"), name("second", "valid_from")), + dates.allStarts().stream().map(Field::getQualifiedName).toList() + ); + } + + @Test + void shouldQualifyDatesAndRecognizeEmptyInput() { + DateAggregationDates dates = DateAggregationDates.forSingleStep( + queryStep("source", dateRange("valid_from", "valid_to")) + ); + + DateAggregationDates qualified = dates.qualify("qualified"); + + assertEquals(name("qualified", "valid_from"), qualified.allStarts().getFirst().getQualifiedName()); + assertTrue(DateAggregationDates.forValidityDates(List.of(Optional.empty())).dateAggregationImpossible()); + } + + private static QueryStep queryStep(String cteName, ColumnDateRange validityDate) { + return QueryStep.builder() + .cteName(cteName) + .selects(Selects.builder() + .ids(new SqlIdColumns(field(name("person"), String.class))) + .validityDate(Optional.of(validityDate)) + .build()) + .build(); + } + + private static ColumnDateRange dateRange(String start, String end) { + return ColumnDateRange.of(field(name(start), Date.class), field(name(end), Date.class)); + } + + private static List names(List> fields) { + return fields.stream().map(Field::getName).toList(); + } +} From cbed222c837e1ab93e05f8faeb9f15a89e951535 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:32:38 +0200 Subject: [PATCH 20/81] refactor(model): share date aggregation actions Move the pure date-propagation vocabulary into the dependency-free shared model so the legacy and SQL query models use one enum. Keep CDateSet operations in the backend DateAggregator and preserve their behavior with focused tests. Co-authored-by: Codex GPT-5.6-sol --- backend/pom.xml | 5 ++ .../conquery/apiv1/QueryProcessor.java | 2 +- .../apiv1/query/concept/specific/CQAnd.java | 2 +- .../query/concept/specific/CQConcept.java | 2 +- .../query/concept/specific/CQNegation.java | 2 +- .../apiv1/query/concept/specific/CQOr.java | 2 +- .../forms/managed/RelativeFormQueryPlan.java | 2 +- .../models/query/DateAggregationMode.java | 2 +- .../queryplan/ArrayConceptQueryPlan.java | 1 + .../query/queryplan/ConceptQueryPlan.java | 3 +- .../queryplan/DateAggregationAction.java | 73 ------------------- .../query/queryplan/DateAggregator.java | 40 +++++++++- .../models/query/queryplan/QPParentNode.java | 1 + .../query/queryplan/SecondaryIdQueryPlan.java | 1 + .../query/queryplan/specific/AndNode.java | 2 +- .../queryplan/specific/NegatingNode.java | 2 +- .../query/queryplan/specific/OrNode.java | 2 +- .../aggregation/AnsiSqlDateAggregator.java | 2 +- .../aggregation/SqlAggregationAction.java | 2 +- .../cqelement/concept/CQConceptConverter.java | 2 +- .../conversion/dialect/SqlDateAggregator.java | 2 +- .../conversion/model/QueryStepComposer.java | 2 +- .../query/FormConversionHelper.java | 2 +- .../query/queryplan/DateAggregatorTest.java | 55 ++++++++++++++ .../models/query/DateAggregationAction.java | 9 +++ .../conquery/models/query/package-info.java | 2 + .../conquery/sql/model/node/AndNode.java | 1 + .../conquery/sql/model/node/ConceptNode.java | 1 + .../sql/model/node/DateAggregationAction.java | 9 --- .../conquery/sql/model/node/OrNode.java | 1 + .../conquery/sql/model/ResolvedQueryTest.java | 2 +- 31 files changed, 134 insertions(+), 102 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregationAction.java create mode 100644 backend/src/test/java/com/bakdata/conquery/models/query/queryplan/DateAggregatorTest.java create mode 100644 dataset-model/src/main/java/com/bakdata/conquery/models/query/DateAggregationAction.java create mode 100644 dataset-model/src/main/java/com/bakdata/conquery/models/query/package-info.java delete mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/DateAggregationAction.java diff --git a/backend/pom.xml b/backend/pom.xml index 835a22ff89..740ba1ea49 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -112,6 +112,11 @@ + + com.bakdata.conquery + dataset-model + ${project.version} + com.bakdata.conquery sql-connector diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index 890aa5c2b2..02f33540eb 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -54,7 +54,7 @@ import com.bakdata.conquery.models.query.*; import com.bakdata.conquery.models.query.preview.EntityPreviewExecution; import com.bakdata.conquery.models.query.preview.EntityPreviewForm; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; import com.bakdata.conquery.models.query.resultinfo.printers.JavaResultPrinters; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java index cf9577ff15..e75059630a 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java @@ -24,7 +24,7 @@ import com.bakdata.conquery.models.query.RequiredEntities; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.ExistsAggregator; import com.bakdata.conquery.models.query.queryplan.specific.AndNode; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java index fb9756aa23..ec3836ca3f 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java @@ -17,7 +17,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.*; import com.bakdata.conquery.models.query.*; import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.EventDateUnionAggregator; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java index 190b021d50..b64c2ac79c 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java @@ -15,7 +15,7 @@ import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.specific.NegatingNode; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java index cb019ee307..68f7b51284 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java @@ -24,7 +24,7 @@ import com.bakdata.conquery.models.query.RequiredEntities; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.ExistsAggregator; import com.bakdata.conquery.models.query.queryplan.specific.OrNode; diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java index a1da6bcc8a..8a3f0e7061 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java @@ -17,7 +17,7 @@ import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.ArrayConceptQueryPlan; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.queryplan.DateAggregator; import com.bakdata.conquery.models.query.queryplan.QueryPlan; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/DateAggregationMode.java b/backend/src/main/java/com/bakdata/conquery/models/query/DateAggregationMode.java index d54bf4242f..3ec42e5cb0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/DateAggregationMode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/DateAggregationMode.java @@ -4,7 +4,7 @@ /** * Determines how the dates of hit events are aggregated for a representation as a column the resulting table. - * This mode causes a specific {@link com.bakdata.conquery.models.query.queryplan.DateAggregationAction} to be set + * This mode causes a specific {@link DateAggregationAction} to be set * in certain query nodes that influence the date aggregation (see {@link CQOr} and {@link com.bakdata.conquery.models.query.queryplan.specific.OrNode} and others). */ public enum DateAggregationMode { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java index e59852310a..2860461f55 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java @@ -13,6 +13,7 @@ import com.bakdata.conquery.models.forms.util.ResultModifier; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.results.SinglelineEntityResult; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java index f23ed72130..133f0c5a0e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.EmptyBucket; import com.bakdata.conquery.models.identifiable.ids.specific.BucketId; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; @@ -211,4 +212,4 @@ public Set collectRequiredTables() { return child.collectRequiredTables(); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregationAction.java deleted file mode 100644 index 98fea55885..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregationAction.java +++ /dev/null @@ -1,73 +0,0 @@ -package com.bakdata.conquery.models.query.queryplan; - -import java.util.Iterator; -import java.util.Set; - -import com.bakdata.conquery.models.common.CDateSet; - -/** - * The action that is used in certain nodes of the query plan to combine {@link CDateSet}s collected from lower - * levels of the query plan. - */ -public enum DateAggregationAction { - /** - * Not propagate dates from lower levels to higher. - */ - BLOCK() { - @Override - public CDateSet aggregate(Set all) { - return null; - } - }, - /** - * Merge all collected dates from a lower level into a union. - */ - MERGE() { - @Override - public CDateSet aggregate(Set all) { - CDateSet combined = CDateSet.createEmpty(); - all.forEach(combined::addAll); - return combined; - } - }, - /** - * Intersect all collected date sets from lower level with each other. - */ - INTERSECT() { - @Override - public CDateSet aggregate(Set all) { - if (all.size() < 1) { - return CDateSet.createEmpty(); - } - - Iterator it = all.iterator(); - CDateSet intersection = it.next(); - - if (all.size() == 1) { - return intersection; - } - // Use the first range as mask and subtract all other ranges from it - - - // Intersect - while (it.hasNext()) { - final CDateSet next = it.next(); - intersection.retainAll(next); - } - return intersection; - } - }, - /** - * Build the negative of the the union of all collected dates - */ - NEGATE() { - @Override - public CDateSet aggregate(Set all) { - CDateSet negative = CDateSet.createFull(); - all.forEach(negative::removeAll); - return negative; - } - }; - - public abstract CDateSet aggregate(Set all); -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java index 46dac03a6a..9bd1216401 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java @@ -2,10 +2,12 @@ import java.util.Collection; import java.util.HashSet; +import java.util.Iterator; import java.util.Set; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.events.Bucket; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; @@ -61,10 +63,44 @@ public CDateSet createAggregationResult() { } }); - // Repackage to get the results sorted. Might need some optimization. - return action.aggregate(all); + // Repackage to get the results sorted. Might need some optimization. + return aggregate(action, all); } + static CDateSet aggregate(DateAggregationAction action, Set dateSets) { + return switch (action) { + case BLOCK -> null; + case MERGE -> merge(dateSets); + case INTERSECT -> intersect(dateSets); + case NEGATE -> negate(dateSets); + }; + } + + private static CDateSet merge(Set dateSets) { + CDateSet combined = CDateSet.createEmpty(); + dateSets.forEach(combined::addAll); + return combined; + } + + private static CDateSet intersect(Set dateSets) { + if (dateSets.isEmpty()) { + return CDateSet.createEmpty(); + } + + Iterator iterator = dateSets.iterator(); + CDateSet intersection = iterator.next(); + while (iterator.hasNext()) { + intersection.retainAll(iterator.next()); + } + return intersection; + } + + private static CDateSet negate(Set dateSets) { + CDateSet negative = CDateSet.createFull(); + dateSets.forEach(negative::removeAll); + return negative; + } + public boolean hasChildren() { return !children.isEmpty(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPParentNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPParentNode.java index 529e86860e..33e4cf476d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPParentNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPParentNode.java @@ -8,6 +8,7 @@ import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.events.Bucket; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java index 31b8431500..8674effe33 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java @@ -18,6 +18,7 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.identifiable.ids.specific.BucketId; import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.entity.Entity; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/AndNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/AndNode.java index 1066deefef..5a3258b601 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/AndNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/AndNode.java @@ -4,7 +4,7 @@ import java.util.Collection; import java.util.List; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.QPParentNode; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/NegatingNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/NegatingNode.java index c9989ab2e4..654e831a06 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/NegatingNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/NegatingNode.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.events.Bucket; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.queryplan.DateAggregator; import com.bakdata.conquery.models.query.queryplan.QPChainNode; import com.bakdata.conquery.models.query.queryplan.QPNode; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/OrNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/OrNode.java index 2a403d3c23..1155d842d7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/OrNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/OrNode.java @@ -5,7 +5,7 @@ import java.util.List; import com.bakdata.conquery.models.datasets.Table; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.queryplan.DateAggregator; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.QPParentNode; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java index b12ee2330e..fb0896ef59 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java @@ -6,7 +6,6 @@ import java.sql.Date; import java.util.List; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingContext; @@ -19,6 +18,7 @@ import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import com.bakdata.conquery.models.query.DateAggregationAction; import lombok.Data; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java index 44ec9cae10..d4b0f4e54f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java @@ -2,7 +2,6 @@ import java.util.List; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; @@ -10,6 +9,7 @@ import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import com.bakdata.conquery.models.query.DateAggregationAction; /** * Represents a subset of {@link DateAggregationAction}. diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 3e9c502897..569f995766 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -22,10 +22,10 @@ import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptElementId; import com.bakdata.conquery.models.identifiable.ids.specific.SelectId; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java index 620d7a28ee..50259f2894 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java @@ -2,12 +2,12 @@ import java.util.List; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import com.bakdata.conquery.models.query.DateAggregationAction; public interface SqlDateAggregator { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java index dad5bc2015..26e96d4253 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java @@ -11,7 +11,6 @@ import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.models.config.IdColumnConfig; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; import com.bakdata.conquery.sql.compiler.ir.JoinMode; import com.bakdata.conquery.sql.compiler.ir.QueryStep; @@ -21,6 +20,7 @@ import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java index 932c559df9..69fd11ac22 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java @@ -11,12 +11,12 @@ import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.models.forms.util.Resolution; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.SharedAliases; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.forms.FormCteStep; import com.bakdata.conquery.sql.conversion.forms.FormType; diff --git a/backend/src/test/java/com/bakdata/conquery/models/query/queryplan/DateAggregatorTest.java b/backend/src/test/java/com/bakdata/conquery/models/query/queryplan/DateAggregatorTest.java new file mode 100644 index 0000000000..bd4456611a --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/models/query/queryplan/DateAggregatorTest.java @@ -0,0 +1,55 @@ +package com.bakdata.conquery.models.query.queryplan; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Set; + +import com.bakdata.conquery.models.common.CDateSet; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.query.DateAggregationAction; +import org.junit.jupiter.api.Test; + +class DateAggregatorTest { + + @Test + void shouldBlockDates() { + assertNull(DateAggregator.aggregate(DateAggregationAction.BLOCK, Set.of(dateSet(1, 2)))); + } + + @Test + void shouldMergeDates() { + CDateSet result = DateAggregator.aggregate( + DateAggregationAction.MERGE, + Set.of(dateSet(1, 3), dateSet(5, 7)) + ); + + assertEquals(CDateSet.create(Set.of(CDateRange.of(1, 3), CDateRange.of(5, 7))), result); + } + + @Test + void shouldIntersectDates() { + CDateSet result = DateAggregator.aggregate( + DateAggregationAction.INTERSECT, + Set.of(dateSet(1, 5), dateSet(3, 7)) + ); + + assertEquals(dateSet(3, 5), result); + } + + @Test + void shouldNegateDates() { + CDateSet result = DateAggregator.aggregate(DateAggregationAction.NEGATE, Set.of(dateSet(3, 5))); + + assertTrue(result.contains(2)); + assertFalse(result.contains(3)); + assertFalse(result.contains(5)); + assertTrue(result.contains(6)); + } + + private static CDateSet dateSet(int minimum, int maximum) { + return CDateSet.create(CDateRange.of(minimum, maximum)); + } +} diff --git a/dataset-model/src/main/java/com/bakdata/conquery/models/query/DateAggregationAction.java b/dataset-model/src/main/java/com/bakdata/conquery/models/query/DateAggregationAction.java new file mode 100644 index 0000000000..eda7dc06f5 --- /dev/null +++ b/dataset-model/src/main/java/com/bakdata/conquery/models/query/DateAggregationAction.java @@ -0,0 +1,9 @@ +package com.bakdata.conquery.models.query; + +/** Defines how validity dates produced by child query nodes are propagated. */ +public enum DateAggregationAction { + BLOCK, + MERGE, + INTERSECT, + NEGATE +} diff --git a/dataset-model/src/main/java/com/bakdata/conquery/models/query/package-info.java b/dataset-model/src/main/java/com/bakdata/conquery/models/query/package-info.java new file mode 100644 index 0000000000..f168ccc270 --- /dev/null +++ b/dataset-model/src/main/java/com/bakdata/conquery/models/query/package-info.java @@ -0,0 +1,2 @@ +/** Framework-neutral query vocabulary shared by query producers and execution engines. */ +package com.bakdata.conquery.models.query; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/AndNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/AndNode.java index 9e0de0c8dc..dffe132f8b 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/AndNode.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/AndNode.java @@ -2,6 +2,7 @@ import java.util.List; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.sql.model.internal.ModelNormalization; import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/ConceptNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/ConceptNode.java index 9985c95874..6a0a1a35c5 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/ConceptNode.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/ConceptNode.java @@ -2,6 +2,7 @@ import java.util.List; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.sql.model.internal.ModelNormalization; import com.bakdata.conquery.sql.model.operation.ResolvedSelect; import com.bakdata.conquery.sql.model.schema.ResolvedConnector; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/DateAggregationAction.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/DateAggregationAction.java deleted file mode 100644 index 688608c2de..0000000000 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/DateAggregationAction.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.bakdata.conquery.sql.model.node; - -/** Defines how validity dates produced by child nodes are propagated. */ -public enum DateAggregationAction { - BLOCK, - MERGE, - INTERSECT, - NEGATE -} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/OrNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/OrNode.java index 9f0c98e74e..212803de32 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/OrNode.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/OrNode.java @@ -2,6 +2,7 @@ import java.util.List; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.sql.model.internal.ModelNormalization; import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java index aeea03f5e2..5b5305ce38 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.models.datasets.ColumnType; import com.bakdata.conquery.sql.model.node.AndNode; import com.bakdata.conquery.sql.model.node.ConceptNode; -import com.bakdata.conquery.sql.model.node.DateAggregationAction; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.sql.model.node.QueryNode; import com.bakdata.conquery.sql.model.range.DateRange; import com.bakdata.conquery.sql.model.result.ResultColumn; From 6eb250c9b8029045ec63b8205ba1db4e96ce6ea9 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:04:57 +0200 Subject: [PATCH 21/81] refactor(sql): move CTE graph into connector Move CTE step and table-resolution structures into the connector compiler IR. Decouple CTE name mapping from the backend name generator through a naming function and cover predecessor, naming, and table resolution behavior. Co-authored-by: Codex GPT-5.6-sol --- .../aggregation/AnsiSqlDateAggregator.java | 2 +- .../aggregation/DateAggregationContext.java | 2 +- .../aggregation/DateAggregationCte.java | 2 +- .../aggregation/DateAggregationCteStep.java | 6 +- .../IntersectAggregationAction.java | 2 +- .../aggregation/MergeAggregateAction.java | 2 +- .../aggregation/NodeNoOverlapCte.java | 2 +- .../aggregation/SqlAggregationAction.java | 2 +- .../cqelement/concept/ConceptCteStep.java | 2 +- .../cqelement/concept/ConceptSqlTables.java | 4 +- .../cqelement/concept/ConnectorCte.java | 2 +- .../cqelement/concept/ConnectorSqlTables.java | 4 +- .../concept/IntervalPackingSelectsCte.java | 2 +- .../cqelement/concept/TablePath.java | 14 +++- .../IntervalPackingContext.java | 2 +- .../IntervalPackingCteStep.java | 10 ++- .../sql/conversion/forms/FormCteStep.java | 2 +- .../sql/conversion/model/CteStep.java | 50 ------------ .../sql/conversion/model/NameGenerator.java | 1 + .../aggregator/DateDistanceSqlAggregator.java | 2 +- .../model/aggregator/FlagSqlAggregator.java | 2 +- .../model/aggregator/SumSqlAggregator.java | 4 +- .../select/ConceptColumnSelectConverter.java | 2 +- .../model/select/DaterangeSelectUtil.java | 10 ++- .../model/select/DistinctSelectConverter.java | 8 +- .../model/select/SelectContext.java | 2 +- .../model/select/ValueSelectUtil.java | 2 +- .../conquery/sql/compiler/ir/CteStep.java | 48 ++++++++++++ .../conquery/sql/compiler/ir}/SqlTables.java | 21 ++--- .../sql/compiler/ir/CteGraphTest.java | 76 +++++++++++++++++++ 30 files changed, 182 insertions(+), 108 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/CteStep.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/CteStep.java rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/model => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/SqlTables.java (52%) create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/CteGraphTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java index fb0896ef59..42108ccd44 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java @@ -16,7 +16,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.models.query.DateAggregationAction; import lombok.Data; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java index 39139b4f5a..4243b91342 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.sql.compiler.ir.QualifyingUtil; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Builder; import lombok.Value; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java index d8a34a6b21..aa17adb282 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java @@ -3,7 +3,7 @@ import java.util.List; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; /** * Base class for a CTE that is part of the date aggregation process. diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteStep.java index 089edfaeaa..91363be02a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteStep.java @@ -5,10 +5,10 @@ import java.util.Map; import java.util.Set; -import com.bakdata.conquery.sql.conversion.model.CteStep; +import com.bakdata.conquery.sql.compiler.ir.CteStep; import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -80,7 +80,7 @@ private static List createCtes(List private static SqlTables createTables(List requiredSteps, QueryStep joinedTable, NameGenerator nameGenerator) { Set asSet = new HashSet<>(requiredSteps); - Map cteNameMap = CteStep.createCteNameMap(asSet, joinedTable.getCteName(), nameGenerator); + Map cteNameMap = CteStep.createCteNameMap(asSet, joinedTable.getCteName(), nameGenerator::cteStepName); Map predecessorMap = CteStep.getDefaultPredecessorMap(asSet); return new SqlTables(joinedTable.getCteName(), cteNameMap, predecessorMap); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java index b2dc011a24..ed34eda55f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java @@ -11,7 +11,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.RequiredArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java index 8c691e9c74..a4cc312642 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java @@ -11,7 +11,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.RequiredArgsConstructor; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java index 0b4acbca20..3a11348615 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import lombok.Getter; import org.jooq.Condition; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java index d4b0f4e54f..195b2d293b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.models.query.DateAggregationAction; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteStep.java index 191dadec6f..68d71d0f9c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteStep.java @@ -3,7 +3,7 @@ import java.util.Set; -import com.bakdata.conquery.sql.conversion.model.CteStep; +import com.bakdata.conquery.sql.compiler.ir.CteStep; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java index 2fda1a3295..ab2aa2f475 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java @@ -3,8 +3,8 @@ import java.util.List; import java.util.Map; -import com.bakdata.conquery.sql.conversion.model.CteStep; -import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.CteStep; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import lombok.Getter; @Getter diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java index f5010d4fbd..62003cdbeb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java @@ -4,7 +4,7 @@ import java.util.Optional; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; abstract class ConnectorCte { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java index 162473a945..529256b88f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java @@ -4,8 +4,8 @@ import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep; -import com.bakdata.conquery.sql.conversion.model.CteStep; -import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.CteStep; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import lombok.Getter; @Getter diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java index 541c745831..8df5197f00 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java @@ -11,7 +11,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.conversion.model.select.ConceptSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.google.common.base.Preconditions; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java index 6e860a7349..dc22ec42ac 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java @@ -14,7 +14,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.SelectId; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep; -import com.bakdata.conquery.sql.conversion.model.CteStep; +import com.bakdata.conquery.sql.compiler.ir.CteStep; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.google.common.base.Preconditions; import lombok.Data; @@ -49,7 +49,11 @@ private static ConnectorSqlTables createConnectorTables(CQConcept cqConcept, CQT String connectorName = context.getNameGenerator().conceptConnectorName(cqConcept, cqTable.getConnector().resolve(), context.getSqlPrintSettings() .getLocale()); TablePathInfo tableInfo = collectConnectorTables(cqConcept, cqTable, context); - Map cteNameMap = CteStep.createCteNameMap(tableInfo.getMappings().keySet(), connectorName, context.getNameGenerator()); + Map cteNameMap = CteStep.createCteNameMap( + tableInfo.getMappings().keySet(), + connectorName, + context.getNameGenerator()::cteStepName + ); return new ConnectorSqlTables( cqTable.getConnector().resolve(), @@ -66,7 +70,11 @@ public ConceptSqlTables createConceptTables(QueryStep predecessor) { TablePathInfo tableInfo = collectConceptTables(predecessor); String conceptName = context.getNameGenerator().conceptName(cqConcept, context.getSqlPrintSettings().getLocale()); - Map cteNameMap = CteStep.createCteNameMap(tableInfo.getMappings().keySet(), conceptName, context.getNameGenerator()); + Map cteNameMap = CteStep.createCteNameMap( + tableInfo.getMappings().keySet(), + conceptName, + context.getNameGenerator()::cteStepName + ); List connectorSqlTables = this.connectorTableMap.values().stream().toList(); return new ConceptSqlTables( diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java index f076ec9e2c..1aa57b4f23 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java @@ -11,7 +11,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Builder; import lombok.Value; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java index 75c3c4cc04..fab86621fb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java @@ -6,9 +6,9 @@ import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; -import com.bakdata.conquery.sql.conversion.model.CteStep; +import com.bakdata.conquery.sql.compiler.ir.CteStep; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import lombok.AllArgsConstructor; import lombok.Getter; @@ -35,7 +35,11 @@ public static SqlTables createTables(QueryStep predecessor, Context context) { ? Set.of(INTERVAL_COMPLETE) : Set.of(values()); - Map cteNameMap = CteStep.createCteNameMap(requiredSteps, rootTable, context.getNameGenerator()); + Map cteNameMap = CteStep.createCteNameMap( + requiredSteps, + rootTable, + context.getNameGenerator()::cteStepName + ); Map predecessorMap = CteStep.getDefaultPredecessorMap(requiredSteps); return new SqlTables(rootTable, cteNameMap, predecessorMap); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java index 7dbd2baedb..76fab89fa0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java @@ -1,7 +1,7 @@ package com.bakdata.conquery.sql.conversion.forms; import com.bakdata.conquery.models.forms.util.Resolution; -import com.bakdata.conquery.sql.conversion.model.CteStep; +import com.bakdata.conquery.sql.compiler.ir.CteStep; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/CteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/CteStep.java deleted file mode 100644 index 5f6833df01..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/CteStep.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model; - -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -import javax.annotation.Nullable; - -/** - * A CteStep represents a common table expression. - */ -public interface CteStep { - - String getSuffix(); - - /** - * Maps each given required step to its default predecessor (see {@link CteStep#getPredecessor()}. The map will contain all given required steps as keys, - * but values might be null. - */ - static Map getDefaultPredecessorMap(Set requiredSteps) { - return requiredSteps.stream().collect( - HashMap::new, - (map, cteStep) -> map.put(cteStep, cteStep.getPredecessor()), // value might be null - Map::putAll - ); - } - - /** - * Generates a CTE name for each of the given required steps. Combines the given label with the CTE step suffix (@link CteStep#getSuffix()). - */ - static Map createCteNameMap(Set requiredSteps, String label, NameGenerator nameGenerator) { - return requiredSteps.stream().collect( - Collectors.toMap( - Function.identity(), - step -> nameGenerator.cteStepName(step, label) - )); - } - - @Nullable - default CteStep getPredecessor() { - return null; - } - - default String cteName(String nodeLabel) { - return "%s-%s".formatted(nodeLabel, getSuffix()); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java index 429068aea6..c48555c133 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java @@ -8,6 +8,7 @@ import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.identifiable.LabeledNamespaceIdentifiable; +import com.bakdata.conquery.sql.compiler.ir.CteStep; import com.bakdata.conquery.sql.compiler.ir.JoinMode; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import lombok.Data; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java index 3c8518d87e..bb083ef784 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.*; import com.bakdata.conquery.sql.conversion.model.select.*; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java index 056f8c3ebb..b1c85c30f8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java @@ -16,7 +16,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.bakdata.conquery.sql.conversion.model.filter.FlagCondition; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java index e52badeaf0..701b77d84c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java @@ -14,13 +14,13 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; -import com.bakdata.conquery.sql.conversion.model.CteStep; +import com.bakdata.conquery.sql.compiler.ir.CteStep; import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.conversion.model.NumberMapUtil; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.sql.conversion.model.filter.SumCondition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java index 43768e2b13..360bc31e38 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java @@ -15,7 +15,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.CteStep; +import com.bakdata.conquery.sql.compiler.ir.CteStep; import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java index 6279a89526..d50e5a2290 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java @@ -30,10 +30,10 @@ import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.CteStep; +import com.bakdata.conquery.sql.compiler.ir.CteStep; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; @@ -154,7 +154,11 @@ private static SqlTables createTables(String alias, ConnectorSqlTables connector else { predecessorMapping.put(INTERVAL_PACKING_SELECTS, INTERVAL_COMPLETE); } - Map cteNameMap = CteStep.createCteNameMap(predecessorMapping.keySet(), alias, context.getNameGenerator()); + Map cteNameMap = CteStep.createCteNameMap( + predecessorMapping.keySet(), + alias, + context.getNameGenerator()::cteStepName + ); return new SqlTables(preprocessingCteName, cteNameMap, predecessorMapping); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java index 658e0a2c08..d5e707081c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java @@ -1,8 +1,5 @@ package com.bakdata.conquery.sql.conversion.model.select; -import static org.jooq.impl.DSL.*; - -import java.util.Collections; import java.util.List; import java.util.Optional; @@ -13,16 +10,13 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.CteStep; +import com.bakdata.conquery.sql.compiler.ir.CteStep; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import lombok.Getter; import lombok.RequiredArgsConstructor; -import org.jetbrains.annotations.NotNull; -import org.jooq.Condition; import org.jooq.Field; import org.jooq.impl.DSL; import org.jooq.impl.SQLDataType; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java index 08e188f56d..ae16baf4ee 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java @@ -10,7 +10,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Value; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java index e171703250..759988f1c6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java @@ -16,7 +16,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.CteStep; +import com.bakdata.conquery.sql.compiler.ir.CteStep; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/CteStep.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/CteStep.java new file mode 100644 index 0000000000..79379085a5 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/CteStep.java @@ -0,0 +1,48 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** A logical step in a common-table-expression graph. */ +public interface CteStep { + + String getSuffix(); + + /** + * Maps every required step to its default predecessor. Values may be {@code null} for root steps. + */ + static Map getDefaultPredecessorMap(Set requiredSteps) { + return requiredSteps.stream().collect( + HashMap::new, + (map, cteStep) -> map.put(cteStep, cteStep.getPredecessor()), + Map::putAll + ); + } + + /** Generates a unique CTE name for every required step using the supplied naming strategy. */ + static Map createCteNameMap( + Set requiredSteps, + String label, + BiFunction namingFunction + ) { + return requiredSteps.stream().collect( + Collectors.toMap( + Function.identity(), + step -> namingFunction.apply(step, label) + ) + ); + } + + /** Returns this step's default predecessor, or {@code null} if it reads from the graph's root table. */ + default CteStep getPredecessor() { + return null; + } + + default String cteName(String nodeLabel) { + return "%s-%s".formatted(nodeLabel, getSuffix()); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlTables.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/SqlTables.java similarity index 52% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlTables.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/SqlTables.java index 48f2e39fa2..d2dd272d75 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlTables.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/SqlTables.java @@ -1,14 +1,11 @@ -package com.bakdata.conquery.sql.conversion.model; +package com.bakdata.conquery.sql.compiler.ir; import java.util.Map; import lombok.Getter; import lombok.RequiredArgsConstructor; -/** - * SqlTables provide a mapping from {@link CteStep}s to their respective table names/cte names and from a {@link CteStep} to the respective preceding step - * in the generated SQL query. - */ +/** Resolves CTE graph steps to their generated table names and predecessor tables. */ @RequiredArgsConstructor public class SqlTables { @@ -17,24 +14,17 @@ public class SqlTables { private final Map cteNameMap; private final Map predecessorMap; - /** - * @return The CTE name for a {@link CteStep}. - */ + /** Returns the generated CTE name for the supplied step. */ public String cteName(CteStep cteStep) { return cteNameMap.get(cteStep); } - /** - * @return True if the given {@link CteStep} is part of these {@link SqlTables}. - */ + /** Returns whether the supplied step participates in this CTE graph. */ public boolean isRequiredStep(CteStep cteStep) { return cteNameMap.containsKey(cteStep); } - /** - * @return The name of the table the given {@link CteStep} will select from. If their exists no mapped preceding {@link CteStep} for the given - * {@link CteStep}, the root table is returned. - */ + /** Returns the predecessor CTE name, or the root table when the step has no mapped predecessor. */ public String getPredecessor(CteStep cteStep) { CteStep predecessor = predecessorMap.get(cteStep); if (predecessor == null) { @@ -42,5 +32,4 @@ public String getPredecessor(CteStep cteStep) { } return cteNameMap.get(predecessor); } - } diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/CteGraphTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/CteGraphTest.java new file mode 100644 index 0000000000..473896f35e --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/CteGraphTest.java @@ -0,0 +1,76 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Map; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +class CteGraphTest { + + @Test + void shouldCreateDefaultPredecessorMapIncludingRootSteps() { + Map predecessors = CteStep.getDefaultPredecessorMap(Set.of(Step.ROOT, Step.CHILD)); + + assertTrue(predecessors.containsKey(Step.ROOT)); + assertNull(predecessors.get(Step.ROOT)); + assertEquals(Step.ROOT, predecessors.get(Step.CHILD)); + } + + @Test + void shouldDelegateCteNameGeneration() { + Map names = CteStep.createCteNameMap( + Set.of(Step.ROOT, Step.CHILD), + "query", + (step, label) -> "generated_" + step.cteName(label) + ); + + assertEquals("generated_query-root", names.get(Step.ROOT)); + assertEquals("generated_query-child", names.get(Step.CHILD)); + } + + @Test + void shouldResolveCteAndPredecessorTables() { + Map names = Map.of(Step.ROOT, "query-root", Step.CHILD, "query-child"); + SqlTables tables = new SqlTables( + "source_table", + names, + CteStep.getDefaultPredecessorMap(Set.of(Step.ROOT, Step.CHILD)) + ); + + assertEquals("source_table", tables.getRootTable()); + assertEquals("query-child", tables.cteName(Step.CHILD)); + assertEquals("source_table", tables.getPredecessor(Step.ROOT)); + assertEquals("query-root", tables.getPredecessor(Step.CHILD)); + assertTrue(tables.isRequiredStep(Step.CHILD)); + assertFalse(tables.isRequiredStep(Step.UNUSED)); + } + + private enum Step implements CteStep { + ROOT("root", null), + CHILD("child", ROOT), + UNUSED("unused", null); + + private final String suffix; + private final CteStep predecessor; + + Step(String suffix, CteStep predecessor) { + this.suffix = suffix; + this.predecessor = predecessor; + } + + @Override + public String getSuffix() { + return suffix; + } + + @Override + public CteStep getPredecessor() { + return predecessor; + } + } +} From 628136d1c1e7e24b0858c1f595936be6901cbf27 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 7 Sep 2026 08:39:54 +0200 Subject: [PATCH 22/81] refactor(sql): move identifier generation into connector Move stateful SQL alias and CTE naming into the connector compiler. Add stable technical names to resolved selects and filters, expose typed naming methods, and retain explicit legacy bridges until backend conversion uses this model. Co-authored-by: Codex GPT-5.6-sol --- .../conquery/sql/conversion/Context.java | 4 +- .../sql/conversion/NodeConversions.java | 6 +- .../cqelement/ConversionContext.java | 4 +- .../aggregation/DateAggregationCteStep.java | 10 +- .../IntersectAggregationAction.java | 4 +- .../aggregation/MergeAggregateAction.java | 4 +- .../aggregation/SqlAggregationAction.java | 4 +- .../cqelement/concept/TablePath.java | 8 +- .../ClickhouseDistinctSelectConverter.java | 2 +- .../sql/conversion/model/NameGenerator.java | 92 --------------- .../CountQuartersSqlAggregator.java | 4 +- .../model/aggregator/CountSqlAggregator.java | 4 +- .../aggregator/DateDistanceSqlAggregator.java | 2 +- .../model/aggregator/FlagSqlAggregator.java | 2 +- .../model/aggregator/SumSqlAggregator.java | 14 +-- .../select/ConceptColumnSelectConverter.java | 6 +- .../model/select/DaterangeSelectUtil.java | 4 +- .../model/select/DistinctSelectConverter.java | 2 +- .../select/EventDateUnionSelectConverter.java | 2 +- .../EventDurationSumSelectConverter.java | 2 +- .../model/select/ExistsSelectConverter.java | 2 +- .../select/FirstValueSelectConverter.java | 2 +- .../select/LastValueSelectConverter.java | 2 +- .../select/RandomValueSelectConverter.java | 2 +- .../query/TableExportQueryConverter.java | 6 +- .../sql/compiler/naming/SqlNameGenerator.java | 108 ++++++++++++++++++ .../sql/compiler/naming/package-info.java | 2 + .../sql/model/operation/BuiltInFilters.java | 14 ++- .../sql/model/operation/BuiltInSelects.java | 20 +++- .../sql/model/operation/ResolvedFilter.java | 6 + .../sql/model/operation/ResolvedSelect.java | 6 + .../compiler/naming/SqlNameGeneratorTest.java | 90 +++++++++++++++ .../sql/model/BuiltInOperationsTest.java | 45 +++++--- 33 files changed, 317 insertions(+), 168 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/naming/SqlNameGenerator.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/naming/package-info.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/naming/SqlNameGeneratorTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Context.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/Context.java index 9d679eb311..26856c9ead 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Context.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/Context.java @@ -3,7 +3,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.NameGenerator; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; /** * Marker for a conversion context. @@ -14,7 +14,7 @@ public interface Context { LegacyCompilerDialect getCompilerDialect(); - default NameGenerator getNameGenerator() { + default SqlNameGenerator getNameGenerator() { return getConversionContext().getNameGenerator(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java index 6ec5be439f..71ea205adf 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; -import com.bakdata.conquery.sql.conversion.model.NameGenerator; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; import lombok.NonNull; import org.jooq.DSLContext; @@ -21,7 +21,7 @@ public class NodeConversions implements NodeConversionDispatcher { private final Conversions conversions; private final IdColumnConfig idColumns; private final LegacyCompilerDialect dialect; - private final NameGenerator nameGenerator; + private final SqlNameGenerator nameGenerator; private final Clock clock; @NonNull private final String defaultPrimaryColumn; @@ -36,7 +36,7 @@ public NodeConversions( this.conversions = new Conversions<>(compilerDialect.getNodeConverters(dslContext)); this.idColumns = idColumns; this.dialect = compilerDialect; - this.nameGenerator = new NameGenerator(compilerDialect.getNameMaxLength()); + this.nameGenerator = new SqlNameGenerator(compilerDialect.getNameMaxLength()); this.clock = clock; this.defaultPrimaryColumn = defaultPrimaryColumn; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java index 37d856dbfb..0a22bd78a1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java @@ -14,7 +14,7 @@ import com.bakdata.conquery.sql.conversion.NodeConversionDispatcher; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; -import com.bakdata.conquery.sql.conversion.model.NameGenerator; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlQuery; import lombok.Builder; @@ -38,7 +38,7 @@ public class ConversionContext implements Context { PrintSettings sqlPrintSettings; NodeConversionDispatcher nodeConversions; LegacyCompilerDialect compilerDialect; - NameGenerator nameGenerator; + SqlNameGenerator nameGenerator; @Singular List querySteps; @Nullable diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteStep.java index 91363be02a..43e19eca7c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteStep.java @@ -6,7 +6,7 @@ import java.util.Set; import com.bakdata.conquery.sql.compiler.ir.CteStep; -import com.bakdata.conquery.sql.conversion.model.NameGenerator; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.SqlTables; import lombok.Getter; @@ -60,15 +60,15 @@ public static List createInvertCtes() { return createCtes(INVERT_STEPS); } - public static SqlTables createMergeTables(QueryStep joinedTable, NameGenerator nameGenerator) { + public static SqlTables createMergeTables(QueryStep joinedTable, SqlNameGenerator nameGenerator) { return createTables(MERGE_STEPS, joinedTable, nameGenerator); } - public static SqlTables createIntersectTables(QueryStep joinedTable, NameGenerator nameGenerator) { + public static SqlTables createIntersectTables(QueryStep joinedTable, SqlNameGenerator nameGenerator) { return createTables(INTERSECT_STEPS, joinedTable, nameGenerator); } - public static SqlTables createInvertTables(QueryStep joinedTable, NameGenerator nameGenerator) { + public static SqlTables createInvertTables(QueryStep joinedTable, SqlNameGenerator nameGenerator) { return createTables(INVERT_STEPS, joinedTable, nameGenerator); } @@ -78,7 +78,7 @@ private static List createCtes(List .toList(); } - private static SqlTables createTables(List requiredSteps, QueryStep joinedTable, NameGenerator nameGenerator) { + private static SqlTables createTables(List requiredSteps, QueryStep joinedTable, SqlNameGenerator nameGenerator) { Set asSet = new HashSet<>(requiredSteps); Map cteNameMap = CteStep.createCteNameMap(asSet, joinedTable.getCteName(), nameGenerator::cteStepName); Map predecessorMap = CteStep.getDefaultPredecessorMap(asSet); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java index ed34eda55f..e7e89010f6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.NameGenerator; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; @@ -24,7 +24,7 @@ class IntersectAggregationAction implements SqlAggregationAction { private final QueryStep joinedStep; @Override - public SqlTables tableNames(NameGenerator nameGenerator) { + public SqlTables tableNames(SqlNameGenerator nameGenerator) { return DateAggregationCteStep.createIntersectTables(this.joinedStep, nameGenerator); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java index a4cc312642..07de768d34 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.NameGenerator; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; @@ -22,7 +22,7 @@ class MergeAggregateAction implements SqlAggregationAction { private final QueryStep joinedStep; @Override - public SqlTables tableNames(NameGenerator nameGenerator) { + public SqlTables tableNames(SqlNameGenerator nameGenerator) { return DateAggregationCteStep.createMergeTables(this.joinedStep, nameGenerator); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java index 195b2d293b..41c5f2918a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java @@ -5,7 +5,7 @@ import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.NameGenerator; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; @@ -16,7 +16,7 @@ */ interface SqlAggregationAction { - SqlTables tableNames(NameGenerator nameGenerator); + SqlTables tableNames(SqlNameGenerator nameGenerator); List dateAggregationCtes(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java index dc22ec42ac..7486225abf 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java @@ -46,8 +46,10 @@ public ConnectorSqlTables getConnectorTables(CQTable cqTable) { private static ConnectorSqlTables createConnectorTables(CQConcept cqConcept, CQTable cqTable, ConversionContext context) { - String connectorName = context.getNameGenerator().conceptConnectorName(cqConcept, cqTable.getConnector().resolve(), context.getSqlPrintSettings() - .getLocale()); + String connectorName = context.getNameGenerator().legacyConceptConnectorName( + cqConcept.userLabel(context.getSqlPrintSettings().getLocale()), + cqTable.getConnector().resolve().getName() + ); TablePathInfo tableInfo = collectConnectorTables(cqConcept, cqTable, context); Map cteNameMap = CteStep.createCteNameMap( tableInfo.getMappings().keySet(), @@ -69,7 +71,7 @@ private static ConnectorSqlTables createConnectorTables(CQConcept cqConcept, CQT public ConceptSqlTables createConceptTables(QueryStep predecessor) { TablePathInfo tableInfo = collectConceptTables(predecessor); - String conceptName = context.getNameGenerator().conceptName(cqConcept, context.getSqlPrintSettings().getLocale()); + String conceptName = context.getNameGenerator().legacyConceptName(cqConcept.userLabel(context.getSqlPrintSettings().getLocale())); Map cteNameMap = CteStep.createCteNameMap( tableInfo.getMappings().keySet(), conceptName, diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java index 1c6c77cf4d..91f37055ca 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java @@ -18,7 +18,7 @@ public class ClickhouseDistinctSelectConverter implements SelectConverter selectContext) { - String alias = selectContext.getNameGenerator().selectName(distinctSelect); + String alias = selectContext.getNameGenerator().legacyOperationName(distinctSelect.getName()); ConnectorSqlTables tables = selectContext.getTables(); SingleColumnSqlSelect preprocessingSelect = diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java deleted file mode 100644 index c48555c133..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java +++ /dev/null @@ -1,92 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model; - -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; -import java.util.regex.Pattern; - -import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; -import com.bakdata.conquery.models.datasets.concepts.Connector; -import com.bakdata.conquery.models.identifiable.LabeledNamespaceIdentifiable; -import com.bakdata.conquery.sql.compiler.ir.CteStep; -import com.bakdata.conquery.sql.compiler.ir.JoinMode; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; - -/** - * Utility to ensure no generated database identifier, like a CTE name or a select alias, is unique and does not exceed the maximum allowed length of the DBMS. - *

- * Carries counters for CONCEPT, AND and OR nodes as well as for all distinct select aliases. For every generated identifier, the counter will be - * incremented to ensure unique names. - *

    - *
  • Always reuse the same NameGenerator during the lifetime of a {@link ConversionContext}.
    • - *
  • NameGenerator is not thread-safe!
    • - *
- */ -@Slf4j -@Data -public class NameGenerator { - - private static final Pattern WHITESPACE = Pattern.compile("\\s+"); - - private final Map selectCountMap; - private final int nameMaxLength; - - private int conceptCount; - private int andCount; - private int orCount; - - public NameGenerator(int nameMaxLength) { - this.nameMaxLength = nameMaxLength; - this.conceptCount = 0; - this.andCount = 0; - this.orCount = 0; - this.selectCountMap = new HashMap<>(); - } - - public String cteStepName(CteStep cteStep, String nodeLabel) { - return ensureValidLength(cteStep.cteName(nodeLabel)); - } - - public String selectName(LabeledNamespaceIdentifiable selectOrFilter) { - int selectCount = this.selectCountMap.merge(selectOrFilter.getName(), 1, Integer::sum); - String name = lowerAndReplaceWhitespace(selectOrFilter.getName()); - return ensureValidLength("%s-%d".formatted(name, selectCount)); - } - - public String conceptName(CQConcept concept, Locale locale) { - String conceptLabel = lowerAndReplaceWhitespace(concept.userLabel(locale)); - return ensureValidLength("concept_%s-%d".formatted(conceptLabel, ++conceptCount)); - } - - public String conceptConnectorName(CQConcept concept, Connector connector, Locale locale) { - //TODO consider reworking this (only name with disambiguation or label should be sufficient) - String conceptLabel = lowerAndReplaceWhitespace(concept.userLabel(locale)); - String connectorName = lowerAndReplaceWhitespace(connector.getName()); - return ensureValidLength("concept_%s_%s-%d".formatted(conceptLabel, connectorName, conceptCount)); - } - - public String joinedNodeName(JoinMode logicalOperation) { - return switch (logicalOperation) { - case INNER -> "AND-%d".formatted(++andCount); - case FULL_OUTER -> "OR-%d".formatted(++orCount); - case LEFT -> throw new UnsupportedOperationException("Creating CTE names for LEFT joins is not supported"); - }; - } - - private String ensureValidLength(String input) { - if (input.length() > nameMaxLength) { - log.trace("CTE or select name too long: {}", input); - int removeIndex = input.length() - nameMaxLength; - return input.substring(removeIndex); - } - return input; - } - - private static String lowerAndReplaceWhitespace(String name) { - return WHITESPACE.matcher(name.toLowerCase()) - .replaceAll("_"); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java index cc530f2d81..960a890c48 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java @@ -108,7 +108,7 @@ public ConnectorSqlSelects connectorSelect(CountQuartersSelect countQuartersSele buildSqlSelect(countQuartersSelect.getColumn(), countQuartersSelect.getStartColumn(), countQuartersSelect.getEndColumn(), - selectContext.getNameGenerator().selectName(countQuartersSelect), + selectContext.getNameGenerator().legacyOperationName(countQuartersSelect.getName()), selectContext.getTables(), selectContext.getFunctionProvider(), selectContext.getCompilerDialect().getStratificationFunctions() @@ -131,7 +131,7 @@ public SqlFilters convertToSqlFilter(CountQuartersFilter countQuartersFilter, Fi buildSqlSelect(countQuartersFilter.getColumn(), countQuartersFilter.getStartColumn(), countQuartersFilter.getEndColumn(), - filterContext.getNameGenerator().selectName(countQuartersFilter), + filterContext.getNameGenerator().legacyOperationName(countQuartersFilter.getName()), filterContext.getTables(), filterContext.getFunctionProvider(), filterContext.getCompilerDialect().getStratificationFunctions() diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java index 59bf61c842..a818be3029 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java @@ -31,7 +31,7 @@ public ConnectorSqlSelects connectorSelect(CountSelect countSelect, SelectContex ConnectorSqlTables tables = selectContext.getTables(); boolean distinct = countSelect.isDistinct(); Column countColumn = countSelect.getColumn().resolve(); - String alias = selectContext.getNameGenerator().selectName(countSelect); + String alias = selectContext.getNameGenerator().legacyOperationName(countSelect.getName()); CommonAggregationSelect countAggregationSelect = createCountAggregationSelect(countColumn, distinct, alias, tables); @@ -68,7 +68,7 @@ public SqlFilters convertToSqlFilter(CountFilter countFilter, FilterContext countAggregationSelect = createCountAggregationSelect(countColumn, distinct, alias, tables); ConnectorSqlSelects selects = ConnectorSqlSelects.builder() diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java index bb083ef784..7051d966aa 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java @@ -32,7 +32,7 @@ public class DateDistanceSqlAggregator implements SelectConverter selectContext) { Column column = select.getColumn().resolve(); - String alias = selectContext.getNameGenerator().selectName(select); + String alias = selectContext.getNameGenerator().legacyOperationName(select.getName()); ConnectorSqlTables tables = selectContext.getTables(); Field dateDistanceCalculation = createDateDistanceCalculation(column, select.getTimeUnit(), tables, selectContext.getConversionContext()); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java index b1c85c30f8..22b8f8a9db 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java @@ -142,7 +142,7 @@ public ConnectorSqlSelects connectorSelect(FlagSelect flagSelect, SelectContext< Map rootSelects = createFlagRootSelectMap(flagSelect, connectorTables.getRootTable()); - String alias = selectContext.getNameGenerator().selectName(flagSelect); + String alias = selectContext.getNameGenerator().legacyOperationName(flagSelect.getName()); FieldWrapper flagAggregation = createFlagSelect(alias, connectorTables, functionProvider, rootSelects); ExtractingSqlSelect finalSelect = flagAggregation.qualify(connectorTables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java index 701b77d84c..33457acb95 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java @@ -15,7 +15,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.compiler.ir.CteStep; -import com.bakdata.conquery.sql.conversion.model.NameGenerator; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; import com.bakdata.conquery.sql.conversion.model.NumberMapUtil; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; @@ -87,8 +87,8 @@ public class SumSqlAggregator> impleme @Override public ConnectorSqlSelects connectorSelect(SumSelect sumSelect, SelectContext selectContext) { - NameGenerator nameGenerator = selectContext.getNameGenerator(); - String alias = nameGenerator.selectName(sumSelect); + SqlNameGenerator nameGenerator = selectContext.getNameGenerator(); + String alias = nameGenerator.legacyOperationName(sumSelect.getName()); Column sumColumn = sumSelect.getColumn().resolve(); Column subtractColumn = sumSelect.getSubtractColumn() != null ? sumSelect.getSubtractColumn().resolve() : null; @@ -126,7 +126,7 @@ private CommonAggregationSelect createDistinctSumAggregationSelect( String alias, SqlIdColumns ids, ConnectorSqlTables tables, - NameGenerator nameGenerator + SqlNameGenerator nameGenerator ) { List> preprocessingSelects = new ArrayList<>(); @@ -211,7 +211,7 @@ private static QueryStep createRowNumberCte( List> distinctByRootSelects, String alias, SqlTables connectorTables, - NameGenerator nameGenerator + SqlNameGenerator nameGenerator ) { String predecessor = connectorTables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT); SqlIdColumns qualifiedIds = ids.qualify(predecessor); @@ -246,7 +246,7 @@ private static QueryStep createRowNumberFilteredCte( QueryStep rowNumberCte, FieldWrapper sumSelect, String alias, - NameGenerator nameGenerator + SqlNameGenerator nameGenerator ) { SqlIdColumns ids = rowNumberCte.getQualifiedSelects().getIds(); @@ -274,7 +274,7 @@ public SqlFilters convertToSqlFilter(SumFilter sumFilter, FilterContext distinctByColumns = sumFilter.getDistinctByColumn().stream().map(ColumnId::resolve).toList(); - String alias = filterContext.getNameGenerator().selectName(sumFilter); + String alias = filterContext.getNameGenerator().legacyOperationName(sumFilter.getName()); ConnectorSqlTables tables = filterContext.getTables(); CommonAggregationSelect sumAggregationSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java index 360bc31e38..82c70b9d55 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java @@ -16,7 +16,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.CteStep; -import com.bakdata.conquery.sql.conversion.model.NameGenerator; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; @@ -77,8 +77,8 @@ public ConceptSqlSelects conceptSelect(ConceptColumnSelect select, SelectContext connectors = selectContext.getTables().getConnectorTables().stream().map(ConnectorSqlTables::getConnector).toList(); } - NameGenerator nameGenerator = selectContext.getNameGenerator(); - String alias = nameGenerator.selectName(select); + SqlNameGenerator nameGenerator = selectContext.getNameGenerator(); + String alias = nameGenerator.legacyOperationName(select.getName()); QueryStep unionStep = createUnionConnectorConnectorsStep(connectors, select, alias, selectContext); FieldWrapper conceptColumnSelect = createConnectorColumnStringAgg(selectContext, unionStep, alias); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java index d50e5a2290..b11654a3c3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java @@ -51,7 +51,7 @@ public static ConnectorSqlSelects c AggregationFunction aggregationFunction, SelectContext context ) { - String alias = context.getNameGenerator().selectName(select); + String alias = context.getNameGenerator().legacyOperationName(select.getName()); SqlFunctionProvider functionProvider = context.getFunctionProvider(); ColumnDateRange daterange = functionProvider.forArbitraryDateRange(select).as(alias); @@ -93,7 +93,7 @@ public static SqlFilters createForFilter( Function, WhereCondition> filterFunction, FilterContext context ) { - String alias = context.getNameGenerator().selectName((LabeledNamespaceIdentifiable) filter); + String alias = context.getNameGenerator().legacyOperationName(((LabeledNamespaceIdentifiable) filter).getName()); SqlFunctionProvider functionProvider = context.getCompilerDialect().getFunctionProvider(); ColumnDateRange daterange = functionProvider.forArbitraryDateRange(filter).as(alias); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java index d5e707081c..a4dd792bfe 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java @@ -65,7 +65,7 @@ private enum DistinctSelectCteStep implements CteStep { @Override public ConnectorSqlSelects connectorSelect(DistinctSelect distinctSelect, SelectContext selectContext) { - String alias = selectContext.getNameGenerator().selectName(distinctSelect); + String alias = selectContext.getNameGenerator().legacyOperationName(distinctSelect.getName()); ConnectorSqlTables tables = selectContext.getTables(); SingleColumnSqlSelect preprocessingSelect = MappableSingleColumnSelect.getSubstringSelect(distinctSelect.getColumn().get(), distinctSelect.getSubstringRange(), selectContext, alias); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java index 0163c6d94b..dc467b5727 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java @@ -42,7 +42,7 @@ private static FieldWrapper createEventDateUnionAggregation(EventDateUnionSel ColumnDateRange validityDate = selectContext.getValidityDate().get(); SqlFunctionProvider functionProvider = selectContext.getFunctionProvider(); - String alias = selectContext.getNameGenerator().selectName(select); + String alias = selectContext.getNameGenerator().legacyOperationName(select.getName()); ColumnDateRange qualified = validityDate.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.INTERVAL_PACKING_SELECTS)); return new FieldWrapper<>(functionProvider.dateRangeAggregation(qualified).as(alias)); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java index 34be5bac11..ce23d9f028 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java @@ -51,7 +51,7 @@ private FieldWrapper createEventDurationSumAggregation(EventDuration ColumnDateRange asDualColumn = selectContext.getFunctionProvider().toDualColumn(qualified); SqlFunctionProvider functionProvider = selectContext.getFunctionProvider(); - String alias = selectContext.getNameGenerator().selectName(select); + String alias = selectContext.getNameGenerator().legacyOperationName(select.getName()); Field durationSum = DSL.sum( DSL.when(containsInfinityDate(asDualColumn, functionProvider), DSL.inline(null, Integer.class)) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java index f4c0fbecd7..28c64b432c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java @@ -24,7 +24,7 @@ public ConceptSqlSelects conceptSelect(ExistsSelect select, SelectContext selectContext) { - String alias = selectContext.getNameGenerator().selectName(select); + String alias = selectContext.getNameGenerator().legacyOperationName(select.getName()); return ExistsSqlSelect.withAlias(alias); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java index f4e29c5431..a8c120df13 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java @@ -10,7 +10,7 @@ public class FirstValueSelectConverter implements SelectConverter selectContext) { return ValueSelectUtil.createValueSelect( select.getColumn().resolve(), - selectContext.getNameGenerator().selectName(select), + selectContext.getNameGenerator().legacyOperationName(select.getName()), Field::asc, select.getSubstringRange(), selectContext diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java index a98a8aa9e4..c955232b8f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java @@ -10,7 +10,7 @@ public class LastValueSelectConverter implements SelectConverter selectContext) { return ValueSelectUtil.createValueSelect( select.getColumn().resolve(), - selectContext.getNameGenerator().selectName(select), + selectContext.getNameGenerator().legacyOperationName(select.getName()), Field::desc, select.getSubstringRange(), selectContext ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java index 0356879936..8e20fc6bb2 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java @@ -18,7 +18,7 @@ public ConnectorSqlSelects connectorSelect(RandomValueSelect select, SelectConte String columnName = select.getColumn().getColumn(); ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(rootTableName, columnName, Object.class); - String alias = selectContext.getNameGenerator().selectName(select); + String alias = selectContext.getNameGenerator().legacyOperationName(select.getName()); Field qualifiedRootSelect = rootSelect.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT)).select(); Field firstAggregation = selectContext.getFunctionProvider().random(qualifiedRootSelect).as(alias); FieldWrapper firstAggregationSqlSelect = new FieldWrapper<>(firstAggregation, columnName); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java index ec49f9e141..47da80f4a3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java @@ -85,8 +85,10 @@ private static QueryStep convertTable( context.getDefaultPrimaryColumn() ); final SqlIdColumns ids = new SqlIdColumns(primaryColumn); - final String conceptConnectorName = - context.getNameGenerator().conceptConnectorName(concept, cqTable.getConnector().resolve(), context.getSqlPrintSettings().getLocale()); + final String conceptConnectorName = context.getNameGenerator().legacyConceptConnectorName( + concept.userLabel(context.getSqlPrintSettings().getLocale()), + cqTable.getConnector().resolve().getName() + ); final Optional validityDate = convertTablesValidityDate(cqTable, conceptConnectorName, context); final List> exportColumns = initializeFields(cqTable, positions); diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/naming/SqlNameGenerator.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/naming/SqlNameGenerator.java new file mode 100644 index 0000000000..5ad9ec5f77 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/naming/SqlNameGenerator.java @@ -0,0 +1,108 @@ +package com.bakdata.conquery.sql.compiler.naming; + +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Pattern; + +import com.bakdata.conquery.sql.compiler.ir.CteStep; +import com.bakdata.conquery.sql.compiler.ir.JoinMode; +import com.bakdata.conquery.sql.model.node.ConceptNode; +import com.bakdata.conquery.sql.model.operation.ResolvedFilter; +import com.bakdata.conquery.sql.model.operation.ResolvedSelect; +import com.bakdata.conquery.sql.model.schema.ResolvedConnector; + +/** + * Stateful generator for unique, length-constrained SQL aliases and CTE names. + * + *

The same instance must be reused throughout a conversion context tree. Naming operations accept resolved SQL + * model types so their stable technical identifiers remain part of the compiler contract. Instances are not + * thread-safe.

+ */ +public class SqlNameGenerator { + + private static final Pattern WHITESPACE = Pattern.compile("\\s+"); + + private final Map operationCountMap = new HashMap<>(); + private final int nameMaxLength; + + private int conceptCount; + private int andCount; + private int orCount; + + public SqlNameGenerator(int nameMaxLength) { + this.nameMaxLength = nameMaxLength; + } + + public String cteStepName(CteStep cteStep, String nodeLabel) { + return ensureValidLength(cteStep.cteName(nodeLabel)); + } + + public String selectName(ResolvedSelect select) { + return operationName(select.name()); + } + + public String filterName(ResolvedFilter filter) { + return operationName(filter.name()); + } + + /** TODO Remove this legacy-backend bridge once backend converters consume resolved operations. */ + public String legacyOperationName(String operationName) { + return operationName(operationName); + } + + private String operationName(String operationName) { + int operationCount = operationCountMap.merge(operationName, 1, Integer::sum); + String normalizedName = lowerAndReplaceWhitespace(operationName); + return ensureValidLength("%s-%d".formatted(normalizedName, operationCount)); + } + + public String conceptName(ConceptNode concept) { + return conceptName(concept.logicalId()); + } + + /** TODO Remove this legacy-backend bridge once backend converters consume resolved query nodes. */ + public String legacyConceptName(String conceptName) { + return conceptName(conceptName); + } + + private String conceptName(String conceptLabel) { + String normalizedLabel = lowerAndReplaceWhitespace(conceptLabel); + return ensureValidLength("concept_%s-%d".formatted(normalizedLabel, ++conceptCount)); + } + + public String conceptConnectorName(ConceptNode concept, ResolvedConnector connector) { + return conceptConnectorName(concept.logicalId(), connector.logicalId()); + } + + /** TODO Remove this legacy-backend bridge once backend converters consume resolved query nodes. */ + public String legacyConceptConnectorName(String conceptName, String connectorName) { + return conceptConnectorName(conceptName, connectorName); + } + + private String conceptConnectorName(String conceptLabel, String connectorName) { + String normalizedConceptLabel = lowerAndReplaceWhitespace(conceptLabel); + String normalizedConnectorName = lowerAndReplaceWhitespace(connectorName); + return ensureValidLength("concept_%s_%s-%d".formatted(normalizedConceptLabel, normalizedConnectorName, conceptCount)); + } + + public String joinedNodeName(JoinMode logicalOperation) { + return switch (logicalOperation) { + case INNER -> "AND-%d".formatted(++andCount); + case FULL_OUTER -> "OR-%d".formatted(++orCount); + case LEFT -> throw new UnsupportedOperationException("Creating CTE names for LEFT joins is not supported"); + }; + } + + private String ensureValidLength(String input) { + if (input.length() > nameMaxLength) { + int removeIndex = input.length() - nameMaxLength; + return input.substring(removeIndex); + } + return input; + } + + private static String lowerAndReplaceWhitespace(String name) { + return WHITESPACE.matcher(name.toLowerCase()) + .replaceAll("_"); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/naming/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/naming/package-info.java new file mode 100644 index 0000000000..f723193081 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/naming/package-info.java @@ -0,0 +1,2 @@ +/** Stateful generation of SQL identifiers used while building compiler IR. */ +package com.bakdata.conquery.sql.compiler.naming; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInFilters.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInFilters.java index 6c4a3c6cea..68eee9c3e0 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInFilters.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInFilters.java @@ -13,6 +13,7 @@ import com.bakdata.conquery.sql.validation.AllowedColumnTypes; import jakarta.validation.Valid; import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; @@ -23,6 +24,7 @@ private BuiltInFilters() { } public record StringValues( + @NotBlank String name, @NotNull @Valid @AllowedColumnTypes(ColumnType.STRING) ResolvedColumn column, @NotEmpty Set<@NotNull String> values, @NotNull Optional<@Valid SubstringRange> substring @@ -32,12 +34,13 @@ public record StringValues( values = ModelNormalization.immutableCopy(values); } - public StringValues(ResolvedColumn column, Set values) { - this(column, values, Optional.empty()); + public StringValues(String name, ResolvedColumn column, Set values) { + this(name, column, values, Optional.empty()); } } public record NumericColumnRange( + @NotBlank String name, @NotNull @Valid @AllowedColumnTypes({ColumnType.INTEGER, ColumnType.REAL, ColumnType.DECIMAL, ColumnType.MONEY}) ResolvedColumn column, @NotNull @Valid NumberRange range @@ -45,6 +48,7 @@ public record NumericColumnRange( } public record AggregationRange( + @NotBlank String name, @NotNull @Valid ResolvedAggregation aggregation, @NotNull @Valid NumberRange range ) implements ResolvedFilter { @@ -52,6 +56,7 @@ public record AggregationRange( /** Filters by distance to an end date that was frozen while resolving the query. */ public record DateDistanceRange( + @NotBlank String name, @NotNull @Valid @AllowedColumnTypes(ColumnType.DATE) ResolvedColumn column, @NotNull ChronoUnit unit, @NotNull LocalDate endDate, @@ -60,6 +65,7 @@ public record DateDistanceRange( } public record Flags( + @NotBlank String name, @NotNull @Valid BuiltInAggregations.Flags availableFlags, @NotEmpty Set<@NotNull String> selectedFlags ) implements ResolvedFilter { @@ -68,8 +74,8 @@ public record Flags( selectedFlags = ModelNormalization.immutableCopy(selectedFlags); } - public Flags(java.util.Map availableFlags, Set selectedFlags) { - this(new BuiltInAggregations.Flags(availableFlags), selectedFlags); + public Flags(String name, java.util.Map availableFlags, Set selectedFlags) { + this(name, new BuiltInAggregations.Flags(availableFlags), selectedFlags); } @AssertTrue(message = "selectedFlags must be defined by availableFlags") diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInSelects.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInSelects.java index 6bf44ff043..3b38dfaa68 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInSelects.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInSelects.java @@ -13,6 +13,7 @@ import com.bakdata.conquery.sql.validation.AllowedColumnTypes; import jakarta.validation.Valid; import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; @@ -29,10 +30,14 @@ public enum ValueOperation { RANDOM } - public record Aggregation(@NotNull @Valid ResolvedAggregation aggregation) implements ResolvedSelect { + public record Aggregation( + @NotBlank String name, + @NotNull @Valid ResolvedAggregation aggregation + ) implements ResolvedSelect { } public record Values( + @NotBlank String name, @NotNull @Valid ResolvedColumn column, @NotNull ValueOperation operation, @NotNull Optional<@Valid SubstringRange> substring @@ -44,11 +49,15 @@ public boolean isSubstringOnStringColumn() { } } - public record DateUnion(@NotNull @Valid DateColumns dates) implements ResolvedSelect { + public record DateUnion( + @NotBlank String name, + @NotNull @Valid DateColumns dates + ) implements ResolvedSelect { } /** Selects the distance to an end date that was frozen while resolving the query. */ public record DateDistance( + @NotBlank String name, @NotNull @Valid @AllowedColumnTypes({ColumnType.DATE, ColumnType.DATE_RANGE}) ResolvedColumn column, @NotNull ChronoUnit unit, @NotNull LocalDate endDate @@ -57,6 +66,7 @@ public record DateDistance( /** Aggregates the resolved concept-value columns across connector tables. */ public record ConceptValues( + @NotBlank String name, @NotEmpty List<@NotNull @Valid ResolvedColumn> columns ) implements ResolvedSelect { @@ -65,12 +75,12 @@ public record ConceptValues( } } - public record EventDateUnion() implements ResolvedSelect { + public record EventDateUnion(@NotBlank String name) implements ResolvedSelect { } - public record EventDurationSum() implements ResolvedSelect { + public record EventDurationSum(@NotBlank String name) implements ResolvedSelect { } - public record Exists() implements ResolvedSelect { + public record Exists(@NotBlank String name) implements ResolvedSelect { } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedFilter.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedFilter.java index 8b42def1ff..98054daeef 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedFilter.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedFilter.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.sql.model.operation; +import jakarta.validation.constraints.NotBlank; + /** * Marker for an immutable, validated filter operation. * @@ -7,4 +9,8 @@ * repository identifiers.

*/ public interface ResolvedFilter { + + /** Stable technical name used to derive internal SQL aliases; it is not a presentation label. */ + @NotBlank + String name(); } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedSelect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedSelect.java index 20ff85d949..779afa550c 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedSelect.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedSelect.java @@ -1,9 +1,15 @@ package com.bakdata.conquery.sql.model.operation; +import jakarta.validation.constraints.NotBlank; + /** * Marker for an immutable, validated select operation. * *

Implementations may be supplied by extensions and are dispatched to matching SQL converters.

*/ public interface ResolvedSelect { + + /** Stable technical name used to derive SQL aliases; it is not a presentation label. */ + @NotBlank + String name(); } diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/naming/SqlNameGeneratorTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/naming/SqlNameGeneratorTest.java new file mode 100644 index 0000000000..84880aae2b --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/naming/SqlNameGeneratorTest.java @@ -0,0 +1,90 @@ +package com.bakdata.conquery.sql.compiler.naming; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.models.query.DateAggregationAction; +import com.bakdata.conquery.sql.compiler.ir.CteStep; +import com.bakdata.conquery.sql.compiler.ir.JoinMode; +import com.bakdata.conquery.sql.model.node.ConceptNode; +import com.bakdata.conquery.sql.model.operation.ResolvedFilter; +import com.bakdata.conquery.sql.model.operation.ResolvedSelect; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.ResolvedConnector; +import com.bakdata.conquery.sql.model.schema.ResolvedValidityDate; +import com.bakdata.conquery.sql.model.schema.SqlTable; +import org.junit.jupiter.api.Test; + +class SqlNameGeneratorTest { + + private static final SqlTable EVENTS = SqlTable.of("events", "analytics", "events"); + private static final ResolvedColumn PRIMARY_ID = new ResolvedColumn( + "events.id", EVENTS, "id", ColumnType.STRING, false + ); + private static final ResolvedConnector CONNECTOR = new ResolvedConnector( + "Events", + EVENTS, + PRIMARY_ID, + Optional.empty(), + new ResolvedValidityDate.None(), + List.of(), + List.of(), + List.of() + ); + private static final ConceptNode CONCEPT = new ConceptNode( + "My Concept", List.of(CONNECTOR), List.of(), DateAggregationAction.BLOCK + ); + + @Test + void shouldNormalizeAndDisambiguateOperationNames() { + SqlNameGenerator generator = new SqlNameGenerator(64); + + assertEquals("total_count-1", generator.selectName(new TestSelect("Total Count"))); + assertEquals("total_count-2", generator.filterName(new TestFilter("Total Count"))); + } + + @Test + void shouldKeepConceptAndConnectorNamesInTheSameSequence() { + SqlNameGenerator generator = new SqlNameGenerator(64); + + assertEquals("concept_my_concept-1", generator.conceptName(CONCEPT)); + assertEquals("concept_my_concept_events-1", generator.conceptConnectorName(CONCEPT, CONNECTOR)); + } + + @Test + void shouldCountJoinedNodeTypesIndependently() { + SqlNameGenerator generator = new SqlNameGenerator(64); + + assertEquals("AND-1", generator.joinedNodeName(JoinMode.INNER)); + assertEquals("OR-1", generator.joinedNodeName(JoinMode.FULL_OUTER)); + assertEquals("AND-2", generator.joinedNodeName(JoinMode.INNER)); + assertThrows(UnsupportedOperationException.class, () -> generator.joinedNodeName(JoinMode.LEFT)); + } + + @Test + void shouldLimitGeneratedNamesFromTheRight() { + SqlNameGenerator generator = new SqlNameGenerator(8); + + assertEquals("_label-1", generator.selectName(new TestSelect("Long Label"))); + assertEquals("fix-step", generator.cteStepName(Step.VALUE, "long-prefix")); + } + + private record TestSelect(String name) implements ResolvedSelect { + } + + private record TestFilter(String name) implements ResolvedFilter { + } + + private enum Step implements CteStep { + VALUE; + + @Override + public String getSuffix() { + return "step"; + } + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/model/BuiltInOperationsTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/BuiltInOperationsTest.java index 5826a63192..d729fc9097 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/model/BuiltInOperationsTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/BuiltInOperationsTest.java @@ -45,18 +45,19 @@ class BuiltInOperationsTest { @Test void shouldRepresentStandardFiltersWithResolvedInputs() { ResolvedFilter values = new BuiltInFilters.StringValues( - CODE, Set.of("A", "B"), Optional.of(SubstringRange.between(0, 3)) + "values", CODE, Set.of("A", "B"), Optional.of(SubstringRange.between(0, 3)) ); - ResolvedFilter number = new BuiltInFilters.NumericColumnRange(AMOUNT, NumberRange.closed(1, 10)); + ResolvedFilter number = new BuiltInFilters.NumericColumnRange("number", AMOUNT, NumberRange.closed(1, 10)); ResolvedFilter aggregation = new BuiltInFilters.AggregationRange( + "aggregation", new BuiltInAggregations.Sum(AMOUNT, Optional.empty(), List.of()), NumberRange.atLeast(BigDecimal.TEN) ); ResolvedFilter distance = new BuiltInFilters.DateDistanceRange( - START, ChronoUnit.YEARS, LocalDate.of(2026, 8, 27), NumberRange.atMost(65) + "distance", START, ChronoUnit.YEARS, LocalDate.of(2026, 8, 27), NumberRange.atMost(65) ); ResolvedFilter flags = new BuiltInFilters.Flags( - Map.of("A", FLAG_A, "B", FLAG_B), Set.of("A") + "flags", Map.of("A", FLAG_A, "B", FLAG_B), Set.of("A") ); assertValid(values); @@ -74,19 +75,21 @@ void shouldRepresentStandardFiltersWithResolvedInputs() { @Test void shouldRepresentStandardSelectsWithResolvedInputs() { List selects = List.of( - new BuiltInSelects.Aggregation(new BuiltInAggregations.Count(CODE, List.of(CODE))), - new BuiltInSelects.Aggregation(new BuiltInAggregations.CountQuarters(new DateColumns.Pair(START, END))), - new BuiltInSelects.Aggregation(new BuiltInAggregations.DurationSum(new DateColumns.Pair(START, END), List.of(CODE))), - new BuiltInSelects.Aggregation(new BuiltInAggregations.Flags(Map.of("A", FLAG_A, "B", FLAG_B))), + new BuiltInSelects.Aggregation("count", new BuiltInAggregations.Count(CODE, List.of(CODE))), + new BuiltInSelects.Aggregation("quarters", new BuiltInAggregations.CountQuarters(new DateColumns.Pair(START, END))), + new BuiltInSelects.Aggregation( + "duration", new BuiltInAggregations.DurationSum(new DateColumns.Pair(START, END), List.of(CODE)) + ), + new BuiltInSelects.Aggregation("flags", new BuiltInAggregations.Flags(Map.of("A", FLAG_A, "B", FLAG_B))), new BuiltInSelects.Values( - CODE, BuiltInSelects.ValueOperation.DISTINCT, Optional.of(SubstringRange.between(0, 3)) + "values", CODE, BuiltInSelects.ValueOperation.DISTINCT, Optional.of(SubstringRange.between(0, 3)) ), - new BuiltInSelects.DateUnion(new DateColumns.Pair(START, END)), - new BuiltInSelects.DateDistance(START, ChronoUnit.DAYS, LocalDate.of(2026, 8, 27)), - new BuiltInSelects.ConceptValues(List.of(CODE)), - new BuiltInSelects.EventDateUnion(), - new BuiltInSelects.EventDurationSum(), - new BuiltInSelects.Exists() + new BuiltInSelects.DateUnion("date-union", new DateColumns.Pair(START, END)), + new BuiltInSelects.DateDistance("date-distance", START, ChronoUnit.DAYS, LocalDate.of(2026, 8, 27)), + new BuiltInSelects.ConceptValues("concept-values", List.of(CODE)), + new BuiltInSelects.EventDateUnion("event-date-union"), + new BuiltInSelects.EventDurationSum("event-duration-sum"), + new BuiltInSelects.Exists("exists") ); selects.forEach(ValidationTestSupport::assertValid); @@ -139,16 +142,22 @@ void shouldRejectColumnsFromDifferentTablesInOneOperation() { void shouldRejectInvalidColumnTypes() { assertInvalid(new BuiltInAggregations.Sum(CODE, Optional.empty(), List.of())); assertInvalid(new BuiltInFilters.DateDistanceRange( - CODE, ChronoUnit.DAYS, LocalDate.of(2026, 8, 27), NumberRange.unbounded() + "distance", CODE, ChronoUnit.DAYS, LocalDate.of(2026, 8, 27), NumberRange.unbounded() )); assertInvalid(new BuiltInSelects.Values( - AMOUNT, BuiltInSelects.ValueOperation.FIRST, Optional.of(SubstringRange.from(1)) + "values", AMOUNT, BuiltInSelects.ValueOperation.FIRST, Optional.of(SubstringRange.from(1)) )); } + @Test + void shouldRejectBlankOperationNames() { + assertInvalid(new BuiltInSelects.Exists(" ")); + assertInvalid(new BuiltInFilters.StringValues(" ", CODE, Set.of("A"))); + } + @Test void shouldRejectUnknownSelectedFlags() { - assertInvalid(new BuiltInFilters.Flags(Map.of("A", FLAG_A), Set.of("missing"))); + assertInvalid(new BuiltInFilters.Flags("flags", Map.of("A", FLAG_A), Set.of("missing"))); } @Test From 5378dad9c1c9559877063ab24fd8945b4165b820 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 7 Sep 2026 12:10:49 +0200 Subject: [PATCH 23/81] refactor(sql): move condition IR into connector Move composable WHERE conditions and phase-specific clause grouping out of the legacy backend. Keep backend conversion on connector-owned IR and cover logical composition, inversion, flag matching, and clause grouping. Co-authored-by: Codex GPT-5.6-sol --- .../concepts/conditions/AndCondition.java | 2 +- .../concepts/conditions/CTCondition.java | 2 +- .../conditions/ColumnEqualCondition.java | 2 +- .../concepts/conditions/EqualCondition.java | 2 +- .../concepts/conditions/GroovyCondition.java | 2 +- .../concepts/conditions/IsEmptyCondition.java | 4 +- .../conditions/IsPresentCondition.java | 4 +- .../concepts/conditions/NotCondition.java | 2 +- .../concepts/conditions/PrefixCondition.java | 4 +- .../conditions/PrefixRangeCondition.java | 4 +- .../concept/AggregationFilterCte.java | 2 +- .../cqelement/concept/CQConceptConverter.java | 4 +- .../cqelement/concept/PreprocessingCte.java | 2 +- .../CountQuartersSqlAggregator.java | 2 +- .../model/aggregator/CountSqlAggregator.java | 2 +- .../aggregator/DateDistanceSqlAggregator.java | 2 + .../model/aggregator/FlagSqlAggregator.java | 4 +- .../model/aggregator/SumSqlAggregator.java | 2 +- .../filter/AbstractSelectFilterConverter.java | 2 + .../model/filter/ConditionUtil.java | 2 + .../filter/DateRestrictionCondition.java | 3 +- .../model/filter/MultiSelectCondition.java | 1 + .../model/filter/NumberFilterConverter.java | 1 + .../model/filter/RangeCondition.java | 1 + .../conversion/model/filter/SqlFilters.java | 1 + .../model/select/DaterangeSelectUtil.java | 4 +- .../ConditionWrappingWhereCondition.java | 4 +- .../compiler/ir/condition}/FlagCondition.java | 12 ++-- .../ir/condition}/InvertedCondition.java | 4 +- .../compiler/ir/condition}/WhereClauses.java | 4 +- .../ir/condition}/WhereCondition.java | 4 +- .../compiler/ir/condition/package-info.java | 2 + .../ir/condition/WhereConditionTest.java | 57 +++++++++++++++++++ 33 files changed, 110 insertions(+), 40 deletions(-) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition}/ConditionWrappingWhereCondition.java (59%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition}/FlagCondition.java (51%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition}/InvertedCondition.java (78%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition}/WhereClauses.java (73%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition}/WhereCondition.java (83%) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/package-info.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereConditionTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/AndCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/AndCondition.java index 6e06ea5118..01fa1ed1a3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/AndCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/AndCondition.java @@ -10,7 +10,7 @@ import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.models.exceptions.ConceptConfigurationException; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import lombok.Getter; import lombok.Setter; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/CTCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/CTCondition.java index 241031bf34..967bbb9028 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/CTCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/CTCondition.java @@ -15,7 +15,7 @@ import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.models.exceptions.ConceptConfigurationException; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.collect.Sets; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java index 1f8e49dc43..c383a8b398 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java @@ -15,7 +15,7 @@ import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; import com.bakdata.conquery.sql.conversion.model.filter.MultiSelectCondition; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import com.bakdata.conquery.util.CollectionsUtil; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java index 5afdfae99c..64857a282c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; import com.bakdata.conquery.sql.conversion.model.filter.MultiSelectCondition; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import com.bakdata.conquery.util.CollectionsUtil; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/GroovyCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/GroovyCondition.java index 13bbb04a45..0e0799ed3a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/GroovyCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/GroovyCondition.java @@ -10,7 +10,7 @@ import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.models.exceptions.ConceptConfigurationException; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import com.fasterxml.jackson.annotation.JsonIgnore; import groovy.lang.GroovyShell; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsEmptyCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsEmptyCondition.java index cc967ab09c..0e061fb1c8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsEmptyCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsEmptyCondition.java @@ -9,8 +9,8 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; -import com.bakdata.conquery.sql.conversion.model.filter.ConditionWrappingWhereCondition; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.ConditionWrappingWhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import lombok.Getter; import lombok.NonNull; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsPresentCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsPresentCondition.java index 5a1ffb7746..ca6995d0a2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsPresentCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsPresentCondition.java @@ -10,8 +10,8 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; -import com.bakdata.conquery.sql.conversion.model.filter.ConditionWrappingWhereCondition; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.ConditionWrappingWhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import lombok.Getter; import lombok.NonNull; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/NotCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/NotCondition.java index 3dfb4d902e..454e2d15d1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/NotCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/NotCondition.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.models.exceptions.ConceptConfigurationException; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import lombok.Getter; import lombok.Setter; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixCondition.java index 28a72733e2..957d75a982 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixCondition.java @@ -10,8 +10,8 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; -import com.bakdata.conquery.sql.conversion.model.filter.ConditionWrappingWhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.ConditionWrappingWhereCondition; import com.bakdata.conquery.util.CalculatedValue; import jakarta.validation.constraints.NotEmpty; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixRangeCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixRangeCondition.java index c8a653828c..51c9d4362a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixRangeCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixRangeCondition.java @@ -10,8 +10,8 @@ import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; -import com.bakdata.conquery.sql.conversion.model.filter.ConditionWrappingWhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.ConditionWrappingWhereCondition; import com.bakdata.conquery.util.CalculatedValue; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java index 459b33496c..cc37fe6791 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java @@ -5,7 +5,7 @@ import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 569f995766..6f0f2d310e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -32,8 +32,8 @@ import com.bakdata.conquery.sql.conversion.model.*; import com.bakdata.conquery.sql.conversion.model.filter.ConditionUtil; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; -import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.sql.conversion.model.select.ConceptSqlSelects; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java index 4c6d09b6d8..ccd84d0097 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java @@ -11,7 +11,7 @@ import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import org.jooq.Condition; import org.jooq.Record; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java index 960a890c48..fd5795d9b6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java @@ -18,7 +18,7 @@ import com.bakdata.conquery.sql.conversion.model.filter.CountCondition; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; -import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java index a818be3029..748b2f8af3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java @@ -10,7 +10,7 @@ import com.bakdata.conquery.sql.conversion.model.filter.CountCondition; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; -import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java index 7051d966aa..a4ba3e3fc7 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java @@ -5,6 +5,8 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.filters.specific.DateDistanceFilter; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DateDistanceSelect; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java index 22b8f8a9db..9d53974230 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java @@ -18,9 +18,9 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; -import com.bakdata.conquery.sql.conversion.model.filter.FlagCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.FlagCondition; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; -import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java index 33457acb95..162622bffd 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java @@ -24,7 +24,7 @@ import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.sql.conversion.model.filter.SumCondition; -import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java index ceae7497cf..a00b75410c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java @@ -2,6 +2,8 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.filters.specific.SelectFilter; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/ConditionUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/ConditionUtil.java index 2cb8fe04f0..84879a059d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/ConditionUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/ConditionUtil.java @@ -4,6 +4,8 @@ import java.util.stream.Stream; import com.bakdata.conquery.models.common.IRange; +import com.bakdata.conquery.sql.compiler.ir.condition.ConditionWrappingWhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import lombok.experimental.UtilityClass; import org.jooq.Condition; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/DateRestrictionCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/DateRestrictionCondition.java index 387bd24608..74442d311c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/DateRestrictionCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/DateRestrictionCondition.java @@ -1,7 +1,8 @@ package com.bakdata.conquery.sql.conversion.model.filter; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import lombok.RequiredArgsConstructor; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/MultiSelectCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/MultiSelectCondition.java index 9d778877c7..7e13f938ae 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/MultiSelectCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/MultiSelectCondition.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.sql.conversion.model.filter; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.google.common.base.Strings; import lombok.RequiredArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java index 5de7b72ab3..10b85d7b8b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java @@ -3,6 +3,7 @@ import com.bakdata.conquery.models.common.IRange; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.filters.specific.NumberFilter; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.model.NumberMapUtil; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/RangeCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/RangeCondition.java index bd537efb93..17cf4d3ecd 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/RangeCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/RangeCondition.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.sql.conversion.model.filter; import com.bakdata.conquery.models.common.IRange; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import lombok.RequiredArgsConstructor; import org.jooq.Condition; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/SqlFilters.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/SqlFilters.java index 5f8a1e6a6b..4cd4e4d41b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/SqlFilters.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/SqlFilters.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.sql.conversion.model.filter; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; import lombok.Value; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java index b11654a3c3..74da3d6943 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java @@ -35,8 +35,8 @@ import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; -import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import org.jooq.Condition; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/ConditionWrappingWhereCondition.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/ConditionWrappingWhereCondition.java similarity index 59% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/ConditionWrappingWhereCondition.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/ConditionWrappingWhereCondition.java index 85a6a3642c..f33af151db 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/ConditionWrappingWhereCondition.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/ConditionWrappingWhereCondition.java @@ -1,8 +1,9 @@ -package com.bakdata.conquery.sql.conversion.model.filter; +package com.bakdata.conquery.sql.compiler.ir.condition; import lombok.Value; import org.jooq.Condition; +/** Adapts an arbitrary jOOQ condition to the compiler's composable condition abstraction. */ @Value public class ConditionWrappingWhereCondition implements WhereCondition { @@ -12,5 +13,4 @@ public class ConditionWrappingWhereCondition implements WhereCondition { public Condition condition() { return condition; } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/FlagCondition.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/FlagCondition.java similarity index 51% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/FlagCondition.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/FlagCondition.java index 9a1d24fb06..8de85b1484 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/FlagCondition.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/FlagCondition.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.model.filter; +package com.bakdata.conquery.sql.compiler.ir.condition; import java.util.List; @@ -7,6 +7,7 @@ import org.jooq.Field; import org.jooq.impl.DSL; +/** Matches an event when at least one selected Boolean flag is true. */ @RequiredArgsConstructor public class FlagCondition implements WhereCondition { @@ -15,10 +16,9 @@ public class FlagCondition implements WhereCondition { @Override public Condition condition() { return flagFields.stream() - .map(DSL::condition) - .map(Field::isTrue) - .reduce(Condition::or) - .orElseThrow(() -> new IllegalArgumentException("Can't construct a FlagCondition with an empty flag field list.")); + .map(DSL::condition) + .map(Field::isTrue) + .reduce(Condition::or) + .orElseThrow(() -> new IllegalArgumentException("Can't construct a FlagCondition with an empty flag field list.")); } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/InvertedCondition.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/InvertedCondition.java similarity index 78% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/InvertedCondition.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/InvertedCondition.java index 7af9fb52ed..bb288eaebc 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/InvertedCondition.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/InvertedCondition.java @@ -1,9 +1,10 @@ -package com.bakdata.conquery.sql.conversion.model.filter; +package com.bakdata.conquery.sql.compiler.ir.condition; import lombok.RequiredArgsConstructor; import org.jooq.Condition; import org.jooq.impl.DSL; +/** Logical negation of another compiler condition. */ @RequiredArgsConstructor public class InvertedCondition implements WhereCondition { @@ -18,5 +19,4 @@ public Condition condition() { public WhereCondition negate() { return filterCondition; } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/WhereClauses.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereClauses.java similarity index 73% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/WhereClauses.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereClauses.java index ddc4198017..ead0a6db89 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/WhereClauses.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereClauses.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.model.filter; +package com.bakdata.conquery.sql.compiler.ir.condition; import java.util.List; @@ -6,6 +6,7 @@ import lombok.Singular; import lombok.Value; +/** Conditions grouped by the compiler phase in which they must be applied. */ @Builder @Value public class WhereClauses { @@ -20,5 +21,4 @@ public class WhereClauses { public static WhereClauses empty() { return WhereClauses.builder().build(); } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/WhereCondition.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereCondition.java similarity index 83% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/WhereCondition.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereCondition.java index d5c4cb9885..ef55a718d9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/WhereCondition.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereCondition.java @@ -1,9 +1,10 @@ -package com.bakdata.conquery.sql.conversion.model.filter; +package com.bakdata.conquery.sql.compiler.ir.condition; import java.util.function.BiFunction; import org.jooq.Condition; +/** A composable condition rendered in a SQL {@code WHERE} or {@code HAVING} clause. */ public interface WhereCondition { Condition condition(); @@ -24,5 +25,4 @@ private WhereCondition combineConditions(WhereCondition whereCondition, BiFuncti Condition combinedCondition = operation.apply(this.condition(), whereCondition.condition()); return new ConditionWrappingWhereCondition(combinedCondition); } - } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/package-info.java new file mode 100644 index 0000000000..9b556e07ed --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/package-info.java @@ -0,0 +1,2 @@ +/** Internal SQL conditions and phase-specific clause groupings used by the compiler. */ +package com.bakdata.conquery.sql.compiler.ir.condition; diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereConditionTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereConditionTest.java new file mode 100644 index 0000000000..b9c137160b --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereConditionTest.java @@ -0,0 +1,57 @@ +package com.bakdata.conquery.sql.compiler.ir.condition; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.jooq.impl.DSL.condition; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.jooq.impl.DSL.not; + +import java.util.List; + +import org.jooq.Field; +import org.junit.jupiter.api.Test; + +class WhereConditionTest { + + private static final Field FIRST_FLAG = field(name("first_flag"), Boolean.class); + private static final Field SECOND_FLAG = field(name("second_flag"), Boolean.class); + + @Test + void shouldComposeAndInvertConditions() { + WhereCondition first = new ConditionWrappingWhereCondition(FIRST_FLAG.isTrue()); + WhereCondition second = new ConditionWrappingWhereCondition(SECOND_FLAG.isTrue()); + + assertEquals(FIRST_FLAG.isTrue().and(SECOND_FLAG.isTrue()), first.and(second).condition()); + assertEquals(FIRST_FLAG.isTrue().or(SECOND_FLAG.isTrue()), first.or(second).condition()); + assertEquals(not(FIRST_FLAG.isTrue()), first.negate().condition()); + assertSame(first, first.negate().negate()); + } + + @Test + void shouldMatchAnySelectedFlag() { + FlagCondition flags = new FlagCondition(List.of(FIRST_FLAG, SECOND_FLAG)); + + assertEquals(condition(FIRST_FLAG).isTrue().or(condition(SECOND_FLAG).isTrue()), flags.condition()); + assertThrows(IllegalArgumentException.class, () -> new FlagCondition(List.of()).condition()); + } + + @Test + void shouldGroupConditionsByCompilationPhase() { + WhereCondition condition = new ConditionWrappingWhereCondition(FIRST_FLAG.isTrue()); + WhereClauses clauses = WhereClauses.builder() + .preprocessingCondition(condition) + .eventFilter(condition) + .groupFilter(condition) + .build(); + + assertEquals(List.of(condition), clauses.getPreprocessingConditions()); + assertEquals(List.of(condition), clauses.getEventFilters()); + assertEquals(List.of(condition), clauses.getGroupFilters()); + assertTrue(WhereClauses.empty().getPreprocessingConditions().isEmpty()); + assertTrue(WhereClauses.empty().getEventFilters().isEmpty()); + assertTrue(WhereClauses.empty().getGroupFilters().isEmpty()); + } +} From 670544a432cfe6dd5d05372c0dfb249b798fddd5 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 8 Sep 2026 07:36:58 +0200 Subject: [PATCH 24/81] refactor(sql): move string value conditions into connector Move string membership rendering into the connector's condition IR. Remove the unused legacy dialect indirection, align the internal name with the resolved model, and cover single, multiple, empty, null, and negated selections. Co-authored-by: Codex GPT-5.6-sol --- .../conditions/ColumnEqualCondition.java | 6 +- .../concepts/conditions/EqualCondition.java | 6 +- .../dialect/SqlFunctionProvider.java | 6 +- .../filter/AbstractSelectFilterConverter.java | 8 +-- .../model/filter/MultiSelectCondition.java | 61 ------------------- .../ir/condition/StringValuesCondition.java | 49 +++++++++++++++ .../ir/condition/WhereConditionTest.java | 25 ++++++++ 7 files changed, 85 insertions(+), 76 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/MultiSelectCondition.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/StringValuesCondition.java diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java index c383a8b398..ed2d7e0c03 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java @@ -13,9 +13,9 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; -import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; -import com.bakdata.conquery.sql.conversion.model.filter.MultiSelectCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.StringValuesCondition; import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; +import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; import com.bakdata.conquery.util.CalculatedValue; import com.bakdata.conquery.util.CollectionsUtil; import com.fasterxml.jackson.annotation.JsonCreator; @@ -59,7 +59,7 @@ public boolean matches(String value, CalculatedValue> rowMap @Override public WhereCondition convertToSqlCondition(CTConditionContext context) { Field field = field(name(column), String.class); - return new MultiSelectCondition(field, values.toArray(String[]::new), context.getFunctionProvider()); + return new StringValuesCondition(field, values.toArray(String[]::new)); } private int fieldLength() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java index 64857a282c..408c176f3a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java @@ -11,9 +11,9 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; -import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; -import com.bakdata.conquery.sql.conversion.model.filter.MultiSelectCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.StringValuesCondition; import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; +import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; import com.bakdata.conquery.util.CalculatedValue; import com.bakdata.conquery.util.CollectionsUtil; import com.fasterxml.jackson.annotation.JsonCreator; @@ -46,7 +46,7 @@ public boolean matches(String value, CalculatedValue> rowMap @Override public WhereCondition convertToSqlCondition(CTConditionContext context) { - return new MultiSelectCondition(field(context.getConnectorColumn(), VARCHAR), values.toArray(String[]::new), context.getFunctionProvider()); + return new StringValuesCondition(field(context.getConnectorColumn(), VARCHAR), values.toArray(String[]::new)); } private int fieldLength() { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index dd0b7cf4d2..1a7e7e88c9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -264,10 +264,6 @@ default Field greatest(List> fields) { return function("greatest", fieldArray[0].getType(), fieldArray); } - default Condition in(Field column, String[] values) { - return column.in(values); - } - default TableOnConditionStep innerJoin(Table leftPart, Table rightPart, List joinConditions) { return leftPart.innerJoin(rightPart).on(joinConditions.toArray(Condition[]::new)); } @@ -318,4 +314,4 @@ default Condition unconditionalJoinCondition(){ default Field isNull(Field field){ return field.isNull(); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java index a00b75410c..7e59ebace9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java @@ -2,6 +2,7 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.filters.specific.SelectFilter; +import com.bakdata.conquery.sql.compiler.ir.condition.StringValuesCondition; import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; @@ -23,10 +24,9 @@ public SqlFilters convertToSqlFilter(F filter, FilterContext filterContext) { String.class ); - WhereCondition condition = new MultiSelectCondition( + WhereCondition condition = new StringValuesCondition( rootSelect.select(), - getValues(filterContext), - filterContext.getFunctionProvider() + getValues(filterContext) ); return new SqlFilters( @@ -43,7 +43,7 @@ public Condition convertForTableExport(F filter, FilterContext filterContext) String tableName = column.getTable().getName(); String columnName = column.getName(); Field field = DSL.field(DSL.name(tableName, columnName), String.class); - return new MultiSelectCondition(field, getValues(filterContext), filterContext.getFunctionProvider()).condition(); + return new StringValuesCondition(field, getValues(filterContext)).condition(); } protected abstract String[] getValues(FilterContext filterContext); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/MultiSelectCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/MultiSelectCondition.java deleted file mode 100644 index 7e13f938ae..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/MultiSelectCondition.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model.filter; - -import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.google.common.base.Strings; -import lombok.RequiredArgsConstructor; -import org.jooq.Condition; -import org.jooq.Field; - -import java.util.Arrays; - -import static org.jooq.impl.DSL.*; - -@RequiredArgsConstructor -public class MultiSelectCondition implements WhereCondition { - - private final Field column; - private final String[] values; - private final SqlFunctionProvider functionProvider; - - @Override - public WhereCondition negate() { - // we want all entries that don't satisfy a condition - because in SQL a comparison with NULL equals UNKNOWN and not FALSE, - // we need to check if the entry is NULL or does not fulfil the condition - Condition valueIsNull = column.isNull(); - Condition notOrNull = not(condition()).or(valueIsNull); - return ConditionUtil.wrap(notOrNull); - } - - @Override - public Condition condition() { - - // values can contain empty or null Strings - String[] valuesWithoutNull = Arrays.stream(values) - .filter(value -> !Strings.isNullOrEmpty(value)) - .toArray(String[]::new); - - - Condition inCondition; - - if (valuesWithoutNull.length == 0) { - inCondition = noCondition(); - } else if (valuesWithoutNull.length == 1) { - inCondition = field(column).eq(val(valuesWithoutNull[0])); - } else { - inCondition = functionProvider.in(column, valuesWithoutNull); - } - - Condition emptyCondition; - - if (Arrays.stream(values).anyMatch(Strings::isNullOrEmpty)) { - emptyCondition = field(column).isNull(); - } else { - emptyCondition = noCondition(); - } - - return inCondition.or(emptyCondition); - } - - -} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/StringValuesCondition.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/StringValuesCondition.java new file mode 100644 index 0000000000..1e42496e61 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/StringValuesCondition.java @@ -0,0 +1,49 @@ +package com.bakdata.conquery.sql.compiler.ir.condition; + +import static org.jooq.impl.DSL.noCondition; +import static org.jooq.impl.DSL.not; +import static org.jooq.impl.DSL.val; + +import java.util.Arrays; + +import lombok.RequiredArgsConstructor; +import org.jooq.Condition; +import org.jooq.Field; + +/** Matches a string column against selected values, treating null and empty selections as an empty database value. */ +@RequiredArgsConstructor +public class StringValuesCondition implements WhereCondition { + + private final Field column; + private final String[] values; + + @Override + public WhereCondition negate() { + // SQL comparisons with NULL yield UNKNOWN, so explicitly retain NULL rows when negating a value condition. + Condition notOrNull = not(condition()).or(column.isNull()); + return new ConditionWrappingWhereCondition(notOrNull); + } + + @Override + public Condition condition() { + String[] nonEmptyValues = Arrays.stream(values) + .filter(value -> !isNullOrEmpty(value)) + .toArray(String[]::new); + + Condition valueCondition = switch (nonEmptyValues.length) { + case 0 -> noCondition(); + case 1 -> column.eq(val(nonEmptyValues[0])); + default -> column.in(nonEmptyValues); + }; + + Condition emptyCondition = Arrays.stream(values).anyMatch(StringValuesCondition::isNullOrEmpty) + ? column.isNull() + : noCondition(); + + return valueCondition.or(emptyCondition); + } + + private static boolean isNullOrEmpty(String value) { + return value == null || value.isEmpty(); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereConditionTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereConditionTest.java index b9c137160b..4dbdbcac8c 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereConditionTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereConditionTest.java @@ -11,13 +11,17 @@ import java.util.List; +import org.jooq.Condition; import org.jooq.Field; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; import org.junit.jupiter.api.Test; class WhereConditionTest { private static final Field FIRST_FLAG = field(name("first_flag"), Boolean.class); private static final Field SECOND_FLAG = field(name("second_flag"), Boolean.class); + private static final Field VALUE = field(name("value"), String.class); @Test void shouldComposeAndInvertConditions() { @@ -38,6 +42,20 @@ void shouldMatchAnySelectedFlag() { assertThrows(IllegalArgumentException.class, () -> new FlagCondition(List.of()).condition()); } + @Test + void shouldMatchSelectedStringValuesWithoutADialectService() { + assertConditionEquals(VALUE.eq("A"), new StringValuesCondition(VALUE, new String[]{"A"}).condition()); + assertConditionEquals(VALUE.in("A", "B"), new StringValuesCondition(VALUE, new String[]{"A", "B"}).condition()); + assertConditionEquals(VALUE.isNull(), new StringValuesCondition(VALUE, new String[]{null, ""}).condition()); + } + + @Test + void shouldIncludeNullValuesWhenNegatingAStringValueCondition() { + WhereCondition condition = new StringValuesCondition(VALUE, new String[]{"A"}); + + assertConditionEquals(not(VALUE.eq("A")).or(VALUE.isNull()), condition.negate().condition()); + } + @Test void shouldGroupConditionsByCompilationPhase() { WhereCondition condition = new ConditionWrappingWhereCondition(FIRST_FLAG.isTrue()); @@ -54,4 +72,11 @@ void shouldGroupConditionsByCompilationPhase() { assertTrue(WhereClauses.empty().getEventFilters().isEmpty()); assertTrue(WhereClauses.empty().getGroupFilters().isEmpty()); } + + private static void assertConditionEquals(Condition expected, Condition actual) { + assertEquals( + DSL.using(SQLDialect.POSTGRES).renderInlined(expected), + DSL.using(SQLDialect.POSTGRES).renderInlined(actual) + ); + } } From 132bd0327decc5694cc2fe0ddd1dbb481ff0c74a Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 8 Sep 2026 07:50:41 +0200 Subject: [PATCH 25/81] refactor(sql): move date restriction conditions into connector Centralize strict date-range overlap rendering in the connector's condition IR. Remove identical HANA and ClickHouse dialect hooks, migrate legacy callers, and cover the exclusive-end overlap expression. Co-authored-by: Codex GPT-5.6-sol --- .../cqelement/CQExternalConverter.java | 5 ++++- .../cqelement/concept/CQConceptConverter.java | 3 ++- .../cqelement/concept/PreprocessingCte.java | 3 ++- .../dialect/SqlFunctionProvider.java | 6 ------ .../ClickhouseFunctionProvider.java | 9 -------- .../dialect/hana/HanaSqlFunctionProvider.java | 11 +--------- .../filter/DateRestrictionCondition.java | 21 ------------------- .../query/TableExportQueryConverter.java | 3 ++- .../condition/DateRestrictionCondition.java | 21 +++++++++++++++++++ .../ir/condition/WhereConditionTest.java | 16 ++++++++++++++ 10 files changed, 48 insertions(+), 50 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/DateRestrictionCondition.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/DateRestrictionCondition.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java index 3bc9107cfa..dbcbf0682b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; import com.bakdata.conquery.models.common.CDateSet; +import com.bakdata.conquery.sql.compiler.ir.condition.DateRestrictionCondition; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; @@ -51,7 +52,9 @@ private static QueryStep createExternalIdsCte(CQExternal external, SqlFunctionPr return QueryStep.builder() .predecessors(List.of(allStep)) - .conditions(List.of(functionProvider.dateRestriction(functionProvider.forCDateRange(context.getDateRestrictionRange()), maybeValidityDate.get()))) + .conditions(List.of(new DateRestrictionCondition( + functionProvider.forCDateRange(context.getDateRestrictionRange()), maybeValidityDate.get() + ).condition())) .selects(allStep.getQualifiedSelects()) .fromTable(table(name(allStep.getCteName()))) .cteName(CQ_EXTERNAL_IDS_CTE_NAME + "_date_restriction") diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 6f0f2d310e..d9df760139 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -22,6 +22,7 @@ import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptElementId; import com.bakdata.conquery.models.identifiable.ids.specific.SelectId; +import com.bakdata.conquery.sql.compiler.ir.condition.DateRestrictionCondition; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; @@ -208,7 +209,7 @@ private static SqlFilters dateRestrictionFilter(ConversionContext context, Colum if (context.getDateRestrictionRange() != null) { ColumnDateRange dateRestriction = functionProvider.forCDateRange(context.getDateRestrictionRange()); - conditions.add(ConditionUtil.wrap(functionProvider.dateRestriction(dateRestriction, validityDate))); + conditions.add(new DateRestrictionCondition(dateRestriction, validityDate)); } return new SqlFilters( diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java index ccd84d0097..0b3960a687 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java @@ -5,6 +5,7 @@ import java.util.Optional; import com.bakdata.conquery.models.datasets.concepts.Connector; +import com.bakdata.conquery.sql.compiler.ir.condition.DateRestrictionCondition; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QueryStep; @@ -42,7 +43,7 @@ private static QueryStep.QueryStepBuilder joinWithStratificationTable( "Stratification table must provide a stratification date" )); // Both expressions are available from the joined source tables; do not reference aliases produced by this SELECT. - conditions.add(functionProvider.dateRestriction(stratificationDate, tableContext.getRawValidityDate())); + conditions.add(new DateRestrictionCondition(stratificationDate, tableContext.getRawValidityDate()).condition()); Table connectorTable = createConceptJoin(tableContext); TableLike joinedTable = functionProvider.innerJoin(connectorTable, stratificationTable, idConditions); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index 1a7e7e88c9..53a3b59b4d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -86,12 +86,6 @@ default Table getNoOpTable() { return noTable(); } - /** - * A date restriction condition is true if holds: dateRestrictionStart < daterangeEnd and dateRestrictionEnd > daterangeStart. The ends of both ranges are - * exclusive. - */ - Condition dateRestriction(ColumnDateRange dateRestriction, ColumnDateRange daterange); - /** * Creates a {@link ColumnDateRange} as a SQL representation of the {@link CDateRange}. */ diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java index 9c8aa663fc..b2d746df2e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java @@ -49,15 +49,6 @@ public Field externalId(String id) { return field("{0}::Nullable(String)", String.class, inline(id, String.class)); } - @Override - public Condition dateRestriction(ColumnDateRange dateRestriction, ColumnDateRange daterange) { - Condition dateRestrictionStartsBeforeDate = dateRestriction.getStart().lessThan(daterange.getEnd()); - Condition dateRestrictionEndsAfterDate = dateRestriction.getEnd().greaterThan(daterange.getStart()); - - return condition(dateRestrictionStartsBeforeDate.and(dateRestrictionEndsAfterDate)); - } - - @Override public ColumnDateRange forCDateRange(CDateRange daterange) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java index 70b4ce9ebd..e74666de95 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java @@ -57,15 +57,6 @@ public Condition unconditionalJoinCondition() { return inline(true).eq(inline(true)); } - @Override - public Condition dateRestriction(ColumnDateRange dateRestriction, ColumnDateRange daterange) { - Condition dateRestrictionStartsBeforeDate = dateRestriction.getStart().lessThan(daterange.getEnd()); - Condition dateRestrictionEndsAfterDate = dateRestriction.getEnd().greaterThan(daterange.getStart()); - - return condition(dateRestrictionStartsBeforeDate.and(dateRestrictionEndsAfterDate)); - } - - @Override public ColumnDateRange forCDateRange(CDateRange daterange) { @@ -363,4 +354,4 @@ public Field isNull(Field field) { // DSl.isNull does not work in some cases for Hana. This accomplishes the same thing with extra steps (: return DSL.function("IFNULL", Boolean.class, field, inline(true)); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/DateRestrictionCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/DateRestrictionCondition.java deleted file mode 100644 index 74442d311c..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/DateRestrictionCondition.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model.filter; - -import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import lombok.RequiredArgsConstructor; -import org.jooq.Condition; - -@RequiredArgsConstructor -public class DateRestrictionCondition implements WhereCondition { - - private final SqlFunctionProvider functionProvider; - private final ColumnDateRange dateRestriction; - private final ColumnDateRange validityDate; - - @Override - public Condition condition() { - return this.functionProvider.dateRestriction(dateRestriction, validityDate); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java index 47da80f4a3..281f81ed40 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; +import com.bakdata.conquery.sql.compiler.ir.condition.DateRestrictionCondition; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; @@ -156,7 +157,7 @@ private static Table joinConnectorTableWithPrerequisite( final ColumnDateRange validityDate = functionProvider.forValidityDate(cqTable.findValidityDate()); final List joinConditions = Stream.concat( ids.join(convertedPrerequisite.getQualifiedSelects().getIds()).stream(), - Stream.of(functionProvider.dateRestriction(functionProvider.forCDateRange(dateRestriction), validityDate)) + Stream.of(new DateRestrictionCondition(functionProvider.forCDateRange(dateRestriction), validityDate).condition()) ).toList(); return functionProvider.innerJoin(connectorTable, convertedPrerequisiteTable, joinConditions); diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/DateRestrictionCondition.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/DateRestrictionCondition.java new file mode 100644 index 0000000000..174341c58b --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/DateRestrictionCondition.java @@ -0,0 +1,21 @@ +package com.bakdata.conquery.sql.compiler.ir.condition; + +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import lombok.RequiredArgsConstructor; +import org.jooq.Condition; + +/** Matches validity dates that overlap a date restriction with exclusive range ends. */ +@RequiredArgsConstructor +public class DateRestrictionCondition implements WhereCondition { + + private final ColumnDateRange dateRestriction; + private final ColumnDateRange validityDate; + + @Override + public Condition condition() { + Condition restrictionStartsBeforeValidityEnds = dateRestriction.getStart().lessThan(validityDate.getEnd()); + Condition restrictionEndsAfterValidityStarts = dateRestriction.getEnd().greaterThan(validityDate.getStart()); + + return restrictionStartsBeforeValidityEnds.and(restrictionEndsAfterValidityStarts); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereConditionTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereConditionTest.java index 4dbdbcac8c..e9e761d1b8 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereConditionTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereConditionTest.java @@ -9,8 +9,10 @@ import static org.jooq.impl.DSL.name; import static org.jooq.impl.DSL.not; +import java.sql.Date; import java.util.List; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import org.jooq.Condition; import org.jooq.Field; import org.jooq.SQLDialect; @@ -22,6 +24,12 @@ class WhereConditionTest { private static final Field FIRST_FLAG = field(name("first_flag"), Boolean.class); private static final Field SECOND_FLAG = field(name("second_flag"), Boolean.class); private static final Field VALUE = field(name("value"), String.class); + private static final ColumnDateRange DATE_RESTRICTION = ColumnDateRange.of( + field(name("restriction_start"), Date.class), field(name("restriction_end"), Date.class) + ); + private static final ColumnDateRange VALIDITY_DATE = ColumnDateRange.of( + field(name("validity_start"), Date.class), field(name("validity_end"), Date.class) + ); @Test void shouldComposeAndInvertConditions() { @@ -56,6 +64,14 @@ void shouldIncludeNullValuesWhenNegatingAStringValueCondition() { assertConditionEquals(not(VALUE.eq("A")).or(VALUE.isNull()), condition.negate().condition()); } + @Test + void shouldMatchStrictlyOverlappingDateRanges() { + Condition expected = DATE_RESTRICTION.getStart().lessThan(VALIDITY_DATE.getEnd()) + .and(DATE_RESTRICTION.getEnd().greaterThan(VALIDITY_DATE.getStart())); + + assertConditionEquals(expected, new DateRestrictionCondition(DATE_RESTRICTION, VALIDITY_DATE).condition()); + } + @Test void shouldGroupConditionsByCompilationPhase() { WhereCondition condition = new ConditionWrappingWhereCondition(FIRST_FLAG.isTrue()); From ec1f4a22f020f19133f6c344ed525c2ec7f74f51 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 8 Sep 2026 09:18:03 +0200 Subject: [PATCH 26/81] refactor(sql): move inclusive range conditions into connector Centralize inclusive-bound rendering in the connector's condition IR. Introduce a shared, order-validating range contract implemented by both resolved and legacy models, then remove the remaining backend condition utility. Co-authored-by: Codex GPT-5.6-sol --- .../conquery/models/common/IRange.java | 14 ++++++- .../bakdata/conquery/models/common/Range.java | 12 +++--- .../models/query/filter/RangeFilterNode.java | 2 +- .../cqelement/concept/CQConceptConverter.java | 4 +- .../model/filter/ConditionUtil.java | 41 ------------------- .../model/filter/RangeCondition.java | 5 ++- dataset-model/pom.xml | 7 ++++ .../models/common/InclusiveRange.java | 20 +++++++++ .../ir/condition/InclusiveRangeCondition.java | 30 ++++++++++++++ .../conquery/sql/model/range/NumberRange.java | 11 +---- .../ir/condition/WhereConditionTest.java | 24 +++++++++++ 11 files changed, 108 insertions(+), 62 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/ConditionUtil.java create mode 100644 dataset-model/src/main/java/com/bakdata/conquery/models/common/InclusiveRange.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/InclusiveRangeCondition.java diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/IRange.java b/backend/src/main/java/com/bakdata/conquery/models/common/IRange.java index 073da17389..a52999f9b5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/common/IRange.java +++ b/backend/src/main/java/com/bakdata/conquery/models/common/IRange.java @@ -1,13 +1,25 @@ package com.bakdata.conquery.models.common; +import java.util.Optional; + import lombok.NonNull; -public interface IRange> { +public interface IRange, CLASS extends IRange> extends InclusiveRange { VALUE getMin(); VALUE getMax(); + @Override + default Optional minimum() { + return Optional.ofNullable(getMin()); + } + + @Override + default Optional maximum() { + return Optional.ofNullable(getMax()); + } + boolean contains(VALUE value); boolean contains(CLASS other); diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/Range.java b/backend/src/main/java/com/bakdata/conquery/models/common/Range.java index c085c4aec2..6481b4fb8e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/common/Range.java +++ b/backend/src/main/java/com/bakdata/conquery/models/common/Range.java @@ -16,7 +16,7 @@ @With @Getter @EqualsAndHashCode -public class Range implements IRange> { +public class Range> implements IRange> { private final T min; private final T max; @@ -42,24 +42,24 @@ public boolean isOpen() { return max == null || min == null; } - public static > Range exactly(T exactly) { + public static > Range exactly(T exactly) { return new Range<>(exactly, exactly); } @JsonCreator - public static > Range of(@JsonProperty("min") T min, @JsonProperty("max") T max) { + public static > Range of(@JsonProperty("min") T min, @JsonProperty("max") T max) { return new Range<>(min, max); } - public static > Range atMost(T bound) { + public static > Range atMost(T bound) { return new Range<>(null, bound); } - public static > Range atLeast(T bound) { + public static > Range atLeast(T bound) { return new Range<>(bound, null); } - public static > Range all() { + public static > Range all() { return new Range<>(null, null); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/RangeFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/RangeFilterNode.java index d011e6bf4d..cb0ecaa85a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/RangeFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/RangeFilterNode.java @@ -9,7 +9,7 @@ * Entity is included, when the result of the aggregator is contained in the range. */ @ToString(callSuper = true) -public class RangeFilterNode> extends AggregationFilterNode, IRange> { +public class RangeFilterNode> extends AggregationFilterNode, IRange> { public RangeFilterNode(IRange filterValue, Aggregator aggregator) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index d9df760139..d39d5794fe 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -22,6 +22,7 @@ import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptElementId; import com.bakdata.conquery.models.identifiable.ids.specific.SelectId; +import com.bakdata.conquery.sql.compiler.ir.condition.ConditionWrappingWhereCondition; import com.bakdata.conquery.sql.compiler.ir.condition.DateRestrictionCondition; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; @@ -31,7 +32,6 @@ import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.*; -import com.bakdata.conquery.sql.conversion.model.filter.ConditionUtil; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; @@ -190,7 +190,7 @@ private static SqlFilters collectConditionFilters( return new SqlFilters(ConnectorSqlSelects.none(), WhereClauses.builder() .preprocessingConditions(conditions) - .preprocessingCondition(ConditionUtil.wrap(validityDateFilter)) + .preprocessingCondition(new ConditionWrappingWhereCondition(validityDateFilter)) .build() ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/ConditionUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/ConditionUtil.java deleted file mode 100644 index 84879a059d..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/ConditionUtil.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model.filter; - -import java.util.Optional; -import java.util.stream.Stream; - -import com.bakdata.conquery.models.common.IRange; -import com.bakdata.conquery.sql.compiler.ir.condition.ConditionWrappingWhereCondition; -import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; -import lombok.experimental.UtilityClass; -import org.jooq.Condition; -import org.jooq.Field; -import org.jooq.impl.DSL; - -@UtilityClass -public class ConditionUtil { - - public > Condition rangeCondition(final Field column, final IRange range) { - Condition condition = DSL.noCondition(); - - if (range.hasLowerBound()){ - condition = condition.and(column.greaterOrEqual(DSL.inline(range.getMin()))); - } - - if (range.hasUpperBound()){ - condition = condition.and(column.lessOrEqual(DSL.inline(range.getMax()))); - } - - - return condition; - } - - /** - * Use if you want to wrap a {@link Condition} without implementing a {@link WhereCondition} in an own class. - * - * @return A {@link WhereCondition} instance encapsulating the provided condition and type. - */ - public WhereCondition wrap(final Condition condition) { - return new ConditionWrappingWhereCondition(condition); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/RangeCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/RangeCondition.java index 17cf4d3ecd..4f36cd82ee 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/RangeCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/RangeCondition.java @@ -1,20 +1,21 @@ package com.bakdata.conquery.sql.conversion.model.filter; import com.bakdata.conquery.models.common.IRange; +import com.bakdata.conquery.sql.compiler.ir.condition.InclusiveRangeCondition; import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import lombok.RequiredArgsConstructor; import org.jooq.Condition; import org.jooq.Field; @RequiredArgsConstructor -abstract class RangeCondition> implements WhereCondition { +abstract class RangeCondition> implements WhereCondition { private final Field column; private final IRange range; @Override public Condition condition() { - return ConditionUtil.rangeCondition(column, range); + return new InclusiveRangeCondition<>(column, range).condition(); } } diff --git a/dataset-model/pom.xml b/dataset-model/pom.xml index e567eca481..013174d1f6 100644 --- a/dataset-model/pom.xml +++ b/dataset-model/pom.xml @@ -13,4 +13,11 @@ dataset-model Conquery Dataset Model + + + + jakarta.validation + jakarta.validation-api + + diff --git a/dataset-model/src/main/java/com/bakdata/conquery/models/common/InclusiveRange.java b/dataset-model/src/main/java/com/bakdata/conquery/models/common/InclusiveRange.java new file mode 100644 index 0000000000..aee6941041 --- /dev/null +++ b/dataset-model/src/main/java/com/bakdata/conquery/models/common/InclusiveRange.java @@ -0,0 +1,20 @@ +package com.bakdata.conquery.models.common; + +import java.util.Optional; + +import jakarta.validation.constraints.AssertTrue; + +/** A possibly unbounded value range whose present lower and upper bounds are inclusive. */ +public interface InclusiveRange> { + + Optional minimum(); + + Optional maximum(); + + @AssertTrue(message = "minimum must not be greater than maximum") + default boolean isOrdered() { + return minimum() == null || maximum() == null + || minimum().isEmpty() || maximum().isEmpty() + || minimum().get().compareTo(maximum().get()) <= 0; + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/InclusiveRangeCondition.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/InclusiveRangeCondition.java new file mode 100644 index 0000000000..86b2d2e189 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/InclusiveRangeCondition.java @@ -0,0 +1,30 @@ +package com.bakdata.conquery.sql.compiler.ir.condition; + +import com.bakdata.conquery.models.common.InclusiveRange; +import lombok.RequiredArgsConstructor; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.impl.DSL; + +/** Matches values within optional inclusive lower and upper bounds. */ +@RequiredArgsConstructor +public class InclusiveRangeCondition> implements WhereCondition { + + private final Field field; + private final InclusiveRange range; + + @Override + public Condition condition() { + Condition condition = DSL.noCondition(); + + if (range.minimum().isPresent()) { + condition = condition.and(field.greaterOrEqual(DSL.inline(range.minimum().get()))); + } + + if (range.maximum().isPresent()) { + condition = condition.and(field.lessOrEqual(DSL.inline(range.maximum().get()))); + } + + return condition; + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/range/NumberRange.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/range/NumberRange.java index 040be4dd72..dfb961ccd4 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/range/NumberRange.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/range/NumberRange.java @@ -4,21 +4,14 @@ import java.util.Objects; import java.util.Optional; -import jakarta.validation.constraints.AssertTrue; +import com.bakdata.conquery.models.common.InclusiveRange; import jakarta.validation.constraints.NotNull; /** Inclusive numeric range. An empty bound represents an unbounded side. */ public record NumberRange( @NotNull Optional minimum, @NotNull Optional maximum -) { - - @AssertTrue(message = "minimum must not be greater than maximum") - public boolean isOrdered() { - return minimum == null || maximum == null - || minimum.isEmpty() || maximum.isEmpty() - || minimum.get().compareTo(maximum.get()) <= 0; - } +) implements InclusiveRange { public static NumberRange closed(Number minimum, Number maximum) { return new NumberRange(Optional.of(decimal(minimum)), Optional.of(decimal(maximum))); diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereConditionTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereConditionTest.java index e9e761d1b8..eefd0b0dfa 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereConditionTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereConditionTest.java @@ -9,10 +9,12 @@ import static org.jooq.impl.DSL.name; import static org.jooq.impl.DSL.not; +import java.math.BigDecimal; import java.sql.Date; import java.util.List; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.model.range.NumberRange; import org.jooq.Condition; import org.jooq.Field; import org.jooq.SQLDialect; @@ -72,6 +74,28 @@ void shouldMatchStrictlyOverlappingDateRanges() { assertConditionEquals(expected, new DateRestrictionCondition(DATE_RESTRICTION, VALIDITY_DATE).condition()); } + @Test + void shouldMatchInclusiveRangeBounds() { + Field value = field(name("number"), BigDecimal.class); + + assertConditionEquals( + value.greaterOrEqual(BigDecimal.ONE).and(value.lessOrEqual(BigDecimal.TEN)), + new InclusiveRangeCondition<>(value, NumberRange.closed(1, 10)).condition() + ); + assertConditionEquals( + value.greaterOrEqual(BigDecimal.ONE), + new InclusiveRangeCondition<>(value, NumberRange.atLeast(1)).condition() + ); + assertConditionEquals( + value.lessOrEqual(BigDecimal.TEN), + new InclusiveRangeCondition<>(value, NumberRange.atMost(10)).condition() + ); + assertConditionEquals( + DSL.noCondition(), + new InclusiveRangeCondition<>(value, NumberRange.unbounded()).condition() + ); + } + @Test void shouldGroupConditionsByCompilationPhase() { WhereCondition condition = new ConditionWrappingWhereCondition(FIRST_FLAG.isTrue()); From e2d137cecc03fb24adf89d7ba68bc86c9203ce1b Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:51:01 +0200 Subject: [PATCH 27/81] refactor(sql): consolidate legacy range adapters Replace the type-named backend range condition wrappers with one explicit compatibility adapter. Keep the wildcard bridge isolated until backend conversion produces connector model ranges directly. Co-authored-by: Codex GPT-5.6-sol --- .../CountQuartersSqlAggregator.java | 6 ++--- .../model/aggregator/CountSqlAggregator.java | 6 ++--- .../aggregator/DateDistanceSqlAggregator.java | 4 +-- .../aggregator/DurationSumSqlAggregator.java | 7 +++-- .../model/aggregator/SumSqlAggregator.java | 8 +++--- .../model/filter/CountCondition.java | 12 --------- .../model/filter/DateDistanceCondition.java | 12 --------- .../filter/LegacyInclusiveRangeCondition.java | 26 +++++++++++++++++++ .../model/filter/NumberCondition.java | 12 --------- .../model/filter/NumberFilterConverter.java | 4 +-- .../model/filter/RangeCondition.java | 21 --------------- .../conversion/model/filter/SumCondition.java | 12 --------- 12 files changed, 43 insertions(+), 87 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/CountCondition.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/DateDistanceCondition.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/LegacyInclusiveRangeCondition.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberCondition.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/RangeCondition.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/SumCondition.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java index fd5795d9b6..7367e58bae 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java @@ -15,8 +15,8 @@ import com.bakdata.conquery.sql.conversion.dialect.Interval; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; -import com.bakdata.conquery.sql.conversion.model.filter.CountCondition; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; +import com.bakdata.conquery.sql.conversion.model.filter.LegacyInclusiveRangeCondition; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; @@ -144,7 +144,7 @@ public SqlFilters convertToSqlFilter(CountQuartersFilter countQuartersFilter, Fi String predecessorTableName = filterContext.getTables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER); Field qualifiedCountSelect = countAggregationSelect.getGroupBy().qualify(predecessorTableName).select(); - CountCondition countCondition = new CountCondition(qualifiedCountSelect, filterContext.getValue()); + LegacyInclusiveRangeCondition countCondition = new LegacyInclusiveRangeCondition(qualifiedCountSelect, filterContext.getValue()); WhereClauses whereClauses = WhereClauses.builder().groupFilter(countCondition).build(); return new SqlFilters(selects, whereClauses); @@ -153,7 +153,7 @@ public SqlFilters convertToSqlFilter(CountQuartersFilter countQuartersFilter, Fi @Override public Condition convertForTableExport(CountQuartersFilter filter, FilterContext filterContext) { Param field = DSL.inline(1); // no grouping, count is always 1 per row - return new CountCondition(field, filterContext.getValue()).condition(); + return new LegacyInclusiveRangeCondition(field, filterContext.getValue()).condition(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java index 748b2f8af3..71feafd9ee 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java @@ -7,8 +7,8 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; -import com.bakdata.conquery.sql.conversion.model.filter.CountCondition; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; +import com.bakdata.conquery.sql.conversion.model.filter.LegacyInclusiveRangeCondition; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; @@ -77,7 +77,7 @@ public SqlFilters convertToSqlFilter(CountFilter countFilter, FilterContext qualifiedCountSelect = countAggregationSelect.getGroupBy().qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)).select(); - CountCondition countCondition = new CountCondition(qualifiedCountSelect, filterContext.getValue()); + LegacyInclusiveRangeCondition countCondition = new LegacyInclusiveRangeCondition(qualifiedCountSelect, filterContext.getValue()); WhereClauses whereClauses = WhereClauses.builder() .groupFilter(countCondition) .build(); @@ -88,7 +88,7 @@ public SqlFilters convertToSqlFilter(CountFilter countFilter, FilterContext filterContext) { Param field = DSL.inline(1); // no grouping, count is always 1 per row - return new CountCondition(field, filterContext.getValue()).condition(); + return new LegacyInclusiveRangeCondition(field, filterContext.getValue()).condition(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java index a4ba3e3fc7..2b01ec23b9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java @@ -60,7 +60,7 @@ public SqlFilters convertToSqlFilter(DateDistanceFilter filter, FilterContext dateDistanceCalculation = createDateDistanceCalculation(column, filter.getTimeUnit(), tables, filterContext.getConversionContext()); - WhereCondition dateDistanceCondition = new DateDistanceCondition(dateDistanceCalculation, filterContext.getValue()); + WhereCondition dateDistanceCondition = new LegacyInclusiveRangeCondition(dateDistanceCalculation, filterContext.getValue()); WhereClauses whereClauses = WhereClauses.builder().eventFilter(dateDistanceCondition).build(); @@ -78,7 +78,7 @@ public Condition convertForTableExport(DateDistanceFilter filter, FilterContext< Field endDate = getEndDate(filterContext.getConversionContext()); Field dateDistance = filterContext.getFunctionProvider().dateDistance(filter.getTimeUnit(), startDateField, endDate); - return new DateDistanceCondition(dateDistance, filterContext.getValue()).condition(); + return new LegacyInclusiveRangeCondition(dateDistance, filterContext.getValue()).condition(); } private Field createDateDistanceCalculation( diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DurationSumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DurationSumSqlAggregator.java index 77cf07f83c..3cb5de1bb1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DurationSumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DurationSumSqlAggregator.java @@ -14,10 +14,9 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.filter.DateDistanceCondition; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; +import com.bakdata.conquery.sql.conversion.model.filter.LegacyInclusiveRangeCondition; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; -import com.bakdata.conquery.sql.conversion.model.filter.SumCondition; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; import com.bakdata.conquery.sql.conversion.model.select.DaterangeSelectUtil; import com.bakdata.conquery.sql.conversion.model.select.DaterangeSelectUtil.AggregationFunction; @@ -39,7 +38,7 @@ public SqlFilters convertToSqlFilter(DurationSumFilter filter, FilterContext new SumCondition((Field) aggregationField, context.getValue())), + (aggregationField -> new LegacyInclusiveRangeCondition(aggregationField, context.getValue())), context); } @@ -66,7 +65,7 @@ public Condition convertForTableExport(DurationSumFilter filter, FilterContext dateDistance = functionProvider.dateDistance(ChronoUnit.DAYS, startDateField, endDateField); // no need so compute a sum here - the duration sum of a single row is simply the date distance - return new DateDistanceCondition(dateDistance, filterContext.getValue()).condition(); + return new LegacyInclusiveRangeCondition(dateDistance, filterContext.getValue()).condition(); } private static AggregationFunction durationSumSelectFunction() { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java index 162622bffd..c2e2cbafc9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java @@ -22,8 +22,8 @@ import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; +import com.bakdata.conquery.sql.conversion.model.filter.LegacyInclusiveRangeCondition; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; -import com.bakdata.conquery.sql.conversion.model.filter.SumCondition; import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; @@ -298,7 +298,7 @@ public SqlFilters convertToSqlFilter(SumFilter sumFilter, FilterContext qualifiedSumSelect = sumAggregationSelect.getGroupBy().qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)).select(); - SumCondition sumCondition = new SumCondition(qualifiedSumSelect, filterContext.getValue()); + LegacyInclusiveRangeCondition sumCondition = new LegacyInclusiveRangeCondition(qualifiedSumSelect, filterContext.getValue()); WhereClauses whereClauses = WhereClauses.builder() .groupFilter(sumCondition) .build(); @@ -318,14 +318,14 @@ public Condition convertForTableExport(SumFilter filter, FilterContext subtractField = DSL.field(DSL.name(subtractTableName, subtractColumnName), numberClass); - return new SumCondition(field.minus(subtractField), filterContext.getValue()).condition(); + return new LegacyInclusiveRangeCondition(field.minus(subtractField), filterContext.getValue()).condition(); } @Getter diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/CountCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/CountCondition.java deleted file mode 100644 index a54ae1525f..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/CountCondition.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model.filter; - -import com.bakdata.conquery.models.common.IRange; -import org.jooq.Field; - -public class CountCondition extends RangeCondition { - - public CountCondition(Field column, IRange range) { - super(column, range); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/DateDistanceCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/DateDistanceCondition.java deleted file mode 100644 index b99ebd18d2..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/DateDistanceCondition.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model.filter; - -import com.bakdata.conquery.models.common.Range; -import org.jooq.Field; - -public class DateDistanceCondition extends RangeCondition { - - public DateDistanceCondition(Field column, Range.LongRange range) { - super(column, range); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/LegacyInclusiveRangeCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/LegacyInclusiveRangeCondition.java new file mode 100644 index 0000000000..d4c87d4a86 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/LegacyInclusiveRangeCondition.java @@ -0,0 +1,26 @@ +package com.bakdata.conquery.sql.conversion.model.filter; + +import com.bakdata.conquery.models.common.IRange; +import com.bakdata.conquery.sql.compiler.ir.condition.InclusiveRangeCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; +import lombok.RequiredArgsConstructor; +import org.jooq.Condition; +import org.jooq.Field; + +/** + * Adapts legacy backend ranges to the connector-owned inclusive range condition. + * + *

TODO Remove this adapter when backend conversion produces connector model ranges directly.

+ */ +@RequiredArgsConstructor +public final class LegacyInclusiveRangeCondition implements WhereCondition { + + private final Field field; + private final IRange range; + + @Override + @SuppressWarnings({"rawtypes", "unchecked"}) + public Condition condition() { + return new InclusiveRangeCondition((Field) field, range).condition(); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberCondition.java deleted file mode 100644 index 087d06afed..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberCondition.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model.filter; - -import com.bakdata.conquery.models.common.IRange; -import org.jooq.Field; - -public class NumberCondition extends RangeCondition { - - public NumberCondition(Field column, IRange range) { - super(column, range); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java index 10b85d7b8b..117956f6f9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java @@ -25,7 +25,7 @@ public SqlFilters convertToSqlFilter(NumberFilter filter, FilterContext rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), column.getName(), numberClass); IRange filterValue = filterContext.getValue(); - NumberCondition condition = new NumberCondition(rootSelect.select(), filterValue); + LegacyInclusiveRangeCondition condition = new LegacyInclusiveRangeCondition(rootSelect.select(), filterValue); WhereClauses whereClauses = WhereClauses.builder().eventFilter(condition).build(); @@ -42,6 +42,6 @@ public Condition convertForTableExport(NumberFilter filter, FilterContext Field field = DSL.field(DSL.name(tableName, columnName), numberClass); IRange range = filterContext.getValue(); - return new NumberCondition(field, range).condition(); + return new LegacyInclusiveRangeCondition(field, range).condition(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/RangeCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/RangeCondition.java deleted file mode 100644 index 4f36cd82ee..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/RangeCondition.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model.filter; - -import com.bakdata.conquery.models.common.IRange; -import com.bakdata.conquery.sql.compiler.ir.condition.InclusiveRangeCondition; -import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; -import lombok.RequiredArgsConstructor; -import org.jooq.Condition; -import org.jooq.Field; - -@RequiredArgsConstructor -abstract class RangeCondition> implements WhereCondition { - - private final Field column; - private final IRange range; - - @Override - public Condition condition() { - return new InclusiveRangeCondition<>(column, range).condition(); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/SumCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/SumCondition.java deleted file mode 100644 index c999ad5b10..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/SumCondition.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model.filter; - -import com.bakdata.conquery.models.common.IRange; -import org.jooq.Field; - -public class SumCondition extends RangeCondition { - - public SumCondition(Field column, IRange range) { - super(column, range); - } - -} From 2541a504f9dc14293d58a06ec6c79db565f495e0 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:08:11 +0200 Subject: [PATCH 28/81] refactor(sql): narrow date aggregation dependencies Move dialect-independent field expressions into connector IR and expose date sentinels through CompilerDialect. Replace DateAggregationContext's broad backend conversion state with the narrow compiler dialect dependency. Co-authored-by: Codex GPT-5.6-sol --- .../aggregation/AnsiSqlDateAggregator.java | 21 ++++++----- .../aggregation/DateAggregationContext.java | 13 ++----- .../aggregation/IntermediateTableCte.java | 5 ++- .../IntersectAggregationAction.java | 8 ++--- .../cqelement/aggregation/InvertCte.java | 8 ++--- .../aggregation/MergeAggregateAction.java | 8 ++--- .../cqelement/aggregation/OverlapCte.java | 8 ++--- .../aggregation/SqlAggregationAction.java | 3 +- .../IntervalPackingCteStep.java | 13 ++++--- .../dialect/LegacyCompilerDialect.java | 10 ++++++ .../dialect/SqlFunctionProvider.java | 18 ---------- .../sql/compiler/dialect/CompilerDialect.java | 6 ++++ .../sql/compiler/ir/FieldExpressions.java | 30 ++++++++++++++++ .../sql/compiler/ir/FieldExpressionsTest.java | 35 +++++++++++++++++++ .../rendering/QueryStepRendererTest.java | 10 ++++++ .../SelectProjectionRendererTest.java | 10 ++++++ 16 files changed, 144 insertions(+), 62 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/FieldExpressions.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/FieldExpressionsTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java index 42108ccd44..262027ee73 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java @@ -7,6 +7,7 @@ import java.util.List; import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; +import com.bakdata.conquery.sql.compiler.ir.FieldExpressions; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingContext; import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep; @@ -50,7 +51,7 @@ public QueryStep apply( .dateAggregationDates(dateAggregationDates) .dateAggregationTables(aggregationAction.tableNames(conversionContext.getNameGenerator())) .ids(joinedStep.getQualifiedSelects().getIds()) - .conversionContext(conversionContext) + .compilerDialect(conversionContext.getCompilerDialect()) .build(); QueryStep finalDateAggregationStep = convertSteps(joinedStep, aggregationAction.dateAggregationCtes(), context); @@ -59,7 +60,11 @@ public QueryStep apply( } Selects predecessorSelects = finalDateAggregationStep.getSelects(); - SqlTables intervalPackingTables = IntervalPackingCteStep.createTables(finalDateAggregationStep, context); + SqlTables intervalPackingTables = IntervalPackingCteStep.createTables( + finalDateAggregationStep, + conversionContext.getCompilerDialect(), + conversionContext.getNameGenerator() + ); IntervalPackingContext intervalPackingContext = IntervalPackingContext.builder() @@ -77,8 +82,8 @@ public QueryStep apply( @Override public ColumnDateRange getAggregatedValidityDate(DateAggregationDates dateAggregationDates, DateAggregationAction dateAggregationAction) { //TODO(FK): i think this is only ever relevant with dateMode=Logical which i want to remove - Field rangeStart = functionProvider.least(dateAggregationDates.allStarts()); - Field rangeEnd = functionProvider.greatest(dateAggregationDates.allEnds()); + Field rangeStart = FieldExpressions.least(dateAggregationDates.allStarts()); + Field rangeEnd = FieldExpressions.greatest(dateAggregationDates.allEnds()); return ColumnDateRange.of( rangeStart.as(DateAggregationCte.RANGE_START), @@ -101,10 +106,10 @@ public QueryStep invertAggregatedIntervals(QueryStep baseStep, ConversionContext .sqlAggregationAction(null) // when inverting, an aggregation has already been applied .carryThroughSelects(baseStepQualifiedSelects.getSqlSelects()) .dateAggregationDates(dateAggregationDates) - .dateAggregationTables(dateAggregationTables) - .ids(baseStepQualifiedSelects.getIds()) - .conversionContext(conversionContext) - .build(); + .dateAggregationTables(dateAggregationTables) + .ids(baseStepQualifiedSelects.getIds()) + .compilerDialect(conversionContext.getCompilerDialect()) + .build(); return convertSteps(baseStep, DateAggregationCteStep.createInvertCtes(), context); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java index 4243b91342..d9b4a52448 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java @@ -5,10 +5,8 @@ import java.util.List; import java.util.Map; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; -import com.bakdata.conquery.sql.conversion.Context; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.compiler.ir.Qualifiable; import com.bakdata.conquery.sql.compiler.ir.QualifyingUtil; import com.bakdata.conquery.sql.compiler.ir.QueryStep; @@ -20,7 +18,7 @@ @Value @Builder(toBuilder = true) -class DateAggregationContext implements Context, Qualifiable { +class DateAggregationContext implements Qualifiable { SqlIdColumns ids; List carryThroughSelects; @@ -29,7 +27,7 @@ class DateAggregationContext implements Context, Qualifiable> intervalMergeSteps = new HashMap<>(); SqlAggregationAction sqlAggregationAction; - ConversionContext conversionContext; + CompilerDialect compilerDialect; public DateAggregationContext withStep(DateAggregationCteStep dateAggregationCteStep, QueryStep queryStep) { this.intervalMergeSteps.computeIfAbsent(dateAggregationCteStep, k -> new ArrayList<>()) @@ -63,9 +61,4 @@ public DateAggregationContext qualify(String qualifier) { .build(); } - @Override - public LegacyCompilerDialect getCompilerDialect() { - return getConversionContext().getCompilerDialect(); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java index ab9370f4fb..e1eac9b0d9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java @@ -5,7 +5,7 @@ import java.util.List; import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.compiler.ir.FieldExpressions; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; @@ -39,8 +39,7 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, List> allStarts = dateAggregationDates.allStarts(); List> allEnds = dateAggregationDates.allEnds(); - SqlFunctionProvider functionProvider = context.getFunctionProvider(); - Condition startBeforeEnd = functionProvider.greatest(allStarts).lessThan(functionProvider.least(allEnds)); + Condition startBeforeEnd = FieldExpressions.greatest(allStarts).lessThan(FieldExpressions.least(allEnds)); Condition startIsNull = allStarts.stream() .map(Field::isNull) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java index e7e89010f6..257ce7675e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java @@ -7,7 +7,7 @@ import java.util.stream.Stream; import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.compiler.ir.FieldExpressions; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; import com.bakdata.conquery.sql.compiler.ir.QueryStep; @@ -34,10 +34,10 @@ public List dateAggregationCtes() { } @Override - public ColumnDateRange getOverlapValidityDate(DateAggregationDates dateAggregationDates, SqlFunctionProvider functionProvider) { + public ColumnDateRange getOverlapValidityDate(DateAggregationDates dateAggregationDates) { - Field rangeStart = functionProvider.greatest(dateAggregationDates.allStarts()); - Field rangeEnd = functionProvider.least(dateAggregationDates.allEnds()); + Field rangeStart = FieldExpressions.greatest(dateAggregationDates.allStarts()); + Field rangeEnd = FieldExpressions.least(dateAggregationDates.allEnds()); return ColumnDateRange.of( rangeStart.as(DateAggregationCte.RANGE_START), diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java index 1b702d8676..dadab95852 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java @@ -8,7 +8,7 @@ import java.util.Optional; import java.util.stream.Stream; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QualifyingUtil; import com.bakdata.conquery.sql.compiler.ir.QueryStep; @@ -57,17 +57,17 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, private Selects getInvertSelects(QueryStep rowNumberStep, SqlIdColumns coalescedIds, DateAggregationContext context) { - SqlFunctionProvider functionProvider = context.getFunctionProvider(); + CompilerDialect dialect = context.getCompilerDialect(); ColumnDateRange validityDate = rowNumberStep.getSelects().getValidityDate().get(); Field rangeStart = coalesce( QualifyingUtil.qualify(validityDate.getEnd(), ROWS_LEFT_TABLE_NAME), - functionProvider.getMinDateExpression() + dialect.minimumDate() ).as(DateAggregationCte.RANGE_START); Field rangeEnd = coalesce( QualifyingUtil.qualify(validityDate.getStart(), ROWS_RIGHT_TABLE_NAME), - functionProvider.getMaxDateExpression() + dialect.maximumDate() ).as(DateAggregationCte.RANGE_END); return Selects.builder() diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java index 07de768d34..77672662a1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java @@ -7,7 +7,7 @@ import java.util.stream.Stream; import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.compiler.ir.FieldExpressions; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; import com.bakdata.conquery.sql.compiler.ir.QueryStep; @@ -32,10 +32,10 @@ public List dateAggregationCtes() { } @Override - public ColumnDateRange getOverlapValidityDate(DateAggregationDates dateAggregationDates, SqlFunctionProvider functionProvider) { + public ColumnDateRange getOverlapValidityDate(DateAggregationDates dateAggregationDates) { - Field rangeStart = functionProvider.least(dateAggregationDates.allStarts()); - Field rangeEnd = functionProvider.greatest(dateAggregationDates.allEnds()); + Field rangeStart = FieldExpressions.least(dateAggregationDates.allStarts()); + Field rangeEnd = FieldExpressions.greatest(dateAggregationDates.allEnds()); return ColumnDateRange.of( rangeStart.as(DateAggregationCte.RANGE_START), diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java index 87a86fea5d..e305f43d30 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java @@ -5,7 +5,7 @@ import java.util.Optional; import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.compiler.ir.FieldExpressions; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; @@ -25,20 +25,18 @@ public OverlapCte(DateAggregationCteStep cteStep) { @Override protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, String predecessor) { - SqlFunctionProvider functionProvider = context.getFunctionProvider(); - DateAggregationDates dateAggregationDates = context.getDateAggregationDates(); List> allStarts = dateAggregationDates.allStarts(); List> allEnds = dateAggregationDates.allEnds(); - ColumnDateRange overlapValidityDate = context.getSqlAggregationAction().getOverlapValidityDate(context.getDateAggregationDates(), functionProvider); + ColumnDateRange overlapValidityDate = context.getSqlAggregationAction().getOverlapValidityDate(context.getDateAggregationDates()); Selects overlapSelects = Selects.builder() .ids(context.getIds()) .validityDate(Optional.of(overlapValidityDate.asValidityDateRange(predecessor))) .sqlSelects(context.getCarryThroughSelects()) .build(); - Condition startBeforeEnd = functionProvider.greatest(allStarts).lessThan(functionProvider.least(allEnds)); + Condition startBeforeEnd = FieldExpressions.greatest(allStarts).lessThan(FieldExpressions.least(allEnds)); Condition allStartsNotNull = allStarts.stream() .map(Field::isNotNull) .reduce(Condition::and) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java index 41c5f2918a..665fb134d7 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java @@ -3,7 +3,6 @@ import java.util.List; import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; import com.bakdata.conquery.sql.compiler.ir.QueryStep; @@ -20,7 +19,7 @@ interface SqlAggregationAction { List dateAggregationCtes(); - ColumnDateRange getOverlapValidityDate(DateAggregationDates dateAggregationDates, SqlFunctionProvider functionProvider); + ColumnDateRange getOverlapValidityDate(DateAggregationDates dateAggregationDates); List getIntermediateTableSelects(DateAggregationDates dateAggregationDates, List carryThroughSelects); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java index fab86621fb..be56b8333a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java @@ -4,7 +4,8 @@ import java.util.Map; import java.util.Set; -import com.bakdata.conquery.sql.conversion.Context; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.compiler.ir.CteStep; import com.bakdata.conquery.sql.compiler.ir.QueryStep; @@ -28,17 +29,21 @@ public enum IntervalPackingCteStep implements CteStep { /** * Create {@link SqlTables} based on a preceding {@link QueryStep}, that must contain a validity date which shall be interval-packed. */ - public static SqlTables createTables(QueryStep predecessor, Context context) { + public static SqlTables createTables( + QueryStep predecessor, + CompilerDialect dialect, + SqlNameGenerator nameGenerator + ) { String rootTable = predecessor.getCteName(); - Set requiredSteps = context.getCompilerDialect().supportsSingleColumnRanges() + Set requiredSteps = dialect.supportsSingleColumnRanges() ? Set.of(INTERVAL_COMPLETE) : Set.of(values()); Map cteNameMap = CteStep.createCteNameMap( requiredSteps, rootTable, - context.getNameGenerator()::cteStepName + nameGenerator::cteStepName ); Map predecessorMap = CteStep.getDefaultPredecessorMap(requiredSteps); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java index 33282dd05f..174e3f66c6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java @@ -40,6 +40,16 @@ */ public interface LegacyCompilerDialect extends CompilerDialect { + @Override + default Field minimumDate() { + return getFunctionProvider().getMinDateExpression(); + } + + @Override + default Field maximumDate() { + return getFunctionProvider().getMaxDateExpression(); + } + @Override default Field anyValue(Field field) { return getFunctionProvider().anyValue(field); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index 53a3b59b4d..e977bedd1a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -240,24 +240,6 @@ default Field arrayOut(List> fields) { return field(concatenated, String.class); } - default Field least(List> fields) { - if (fields.isEmpty()) { - return null; - } - Field[] fieldArray = fields.toArray(Field[]::new); - // signature only accepts arrays/varargs - return function("least", fieldArray[0].getType(), fieldArray); - } - - default Field greatest(List> fields) { - if (fields.isEmpty()) { - return null; - } - Field[] fieldArray = fields.toArray(Field[]::new); - // signature only accepts arrays/varargs - return function("greatest", fieldArray[0].getType(), fieldArray); - } - default TableOnConditionStep innerJoin(Table leftPart, Table rightPart, List joinConditions) { return leftPart.innerJoin(rightPart).on(joinConditions.toArray(Condition[]::new)); } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java index 53ac40268e..750da3df74 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java @@ -16,6 +16,12 @@ */ public interface CompilerDialect { + /** Expression used as the database-specific lower infinity sentinel for dates. */ + Field minimumDate(); + + /** Expression used as the database-specific upper infinity sentinel for dates. */ + Field maximumDate(); + /** * Aggregate a field to an arbitrary value from its group. * diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/FieldExpressions.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/FieldExpressions.java new file mode 100644 index 0000000000..36eee3f26c --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/FieldExpressions.java @@ -0,0 +1,30 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import java.util.List; + +import lombok.experimental.UtilityClass; +import org.jooq.Field; +import org.jooq.impl.DSL; + +/** Dialect-independent field expressions used while composing compiler IR. */ +@UtilityClass +public class FieldExpressions { + + public Field least(List> fields) { + return function("least", fields); + } + + public Field greatest(List> fields) { + return function("greatest", fields); + } + + @SuppressWarnings("unchecked") + private Field function(String name, List> fields) { + if (fields.isEmpty()) { + return null; + } + + Field[] fieldArray = fields.toArray(Field[]::new); + return DSL.function(name, fieldArray[0].getType(), fieldArray); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/FieldExpressionsTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/FieldExpressionsTest.java new file mode 100644 index 0000000000..0f3ca92aa0 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/FieldExpressionsTest.java @@ -0,0 +1,35 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; + +import java.util.List; + +import org.jooq.Field; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Test; + +class FieldExpressionsTest { + + @Test + void shouldCreateLeastAndGreatestExpressions() { + Field first = field(name("first"), Integer.class); + Field second = field(name("second"), Integer.class); + + assertEquals("least(\"first\", \"second\")", render(FieldExpressions.least(List.of(first, second)))); + assertEquals("greatest(\"first\", \"second\")", render(FieldExpressions.greatest(List.of(first, second)))); + } + + @Test + void shouldPreserveEmptyFieldListBehavior() { + assertNull(FieldExpressions.least(List.of())); + assertNull(FieldExpressions.greatest(List.of())); + } + + private static String render(Field expression) { + return DSL.using(SQLDialect.POSTGRES).renderInlined(expression); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/rendering/QueryStepRendererTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/rendering/QueryStepRendererTest.java index 8109a3082e..7117e63b3c 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/rendering/QueryStepRendererTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/rendering/QueryStepRendererTest.java @@ -76,6 +76,16 @@ private static String render(QueryStep queryStep) { private static final class TestDialect implements CompilerDialect { + @Override + public Field minimumDate() { + return field(name("minimum_date"), Date.class); + } + + @Override + public Field maximumDate() { + return field(name("maximum_date"), Date.class); + } + @Override public Field anyValue(Field field) { return field; diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRendererTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRendererTest.java index 790d566f9a..e85c9f3f90 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRendererTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRendererTest.java @@ -88,6 +88,16 @@ private static final class RecordingDialect implements CompilerDialect { private int renderedRanges; private int aggregatedRanges; + @Override + public Field minimumDate() { + return field(name("minimum_date"), Date.class); + } + + @Override + public Field maximumDate() { + return field(name("maximum_date"), Date.class); + } + @Override public Field anyValue(Field field) { return field; From 4b0f0c6edffcdf5027025d194fd33681404f1d55 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:40:24 +0200 Subject: [PATCH 29/81] refactor(sql): move interval packing into connector Move the framework-neutral interval-packing transformation and its CTE graph into the connector IR. Remove the unused PostgreSQL strategy and obsolete dialect plumbing now that HANA and ClickHouse share the ANSI implementation. Cover both multi-step and single-column CTE mappings and fix the latter to reference the root table when intermediate steps are omitted. Co-authored-by: Codex GPT-5.6-sol --- .../aggregation/AnsiSqlDateAggregator.java | 13 +- .../cqelement/concept/ConnectorSqlTables.java | 2 +- .../cqelement/concept/JoinBranchesCte.java | 7 +- .../cqelement/concept/TablePath.java | 4 +- .../IntervalPackingResult.java | 13 -- .../PostgreSqlIntervalPacker.java | 59 -------- .../conversion/dialect/IntervalPacker.java | 33 ----- .../dialect/LegacyCompilerDialect.java | 2 - .../clickhouse/ClickhouseDialectBundle.java | 11 +- .../dialect/hana/HanaDialectBundle.java | 11 +- .../model/select/DaterangeSelectUtil.java | 19 ++- .../ir/interval}/AggregationMode.java | 2 +- .../ir/interval}/AnsiSqlIntervalPacker.java | 36 ++--- .../ir/interval}/IntervalPackingContext.java | 25 +--- .../ir/interval}/IntervalPackingCteStep.java | 11 +- .../compiler/ir/interval}/interval-packing.md | 2 +- .../compiler/ir/interval/package-info.java | 2 + .../ir/interval/IntervalPackingTest.java | 130 ++++++++++++++++++ 18 files changed, 185 insertions(+), 197 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingResult.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/IntervalPacker.java rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval}/AggregationMode.java (81%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval}/AnsiSqlIntervalPacker.java (80%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval}/IntervalPackingContext.java (59%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval}/IntervalPackingCteStep.java (81%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval}/interval-packing.md (99%) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/package-info.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java index 262027ee73..c2f0636a35 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java @@ -5,13 +5,14 @@ import java.sql.Date; import java.util.List; +import java.util.Optional; import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; import com.bakdata.conquery.sql.compiler.ir.FieldExpressions; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingContext; -import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep; -import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; +import com.bakdata.conquery.sql.compiler.ir.interval.AnsiSqlIntervalPacker; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingContext; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep; import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; @@ -26,7 +27,6 @@ @Data public class AnsiSqlDateAggregator implements SqlDateAggregator { - private final IntervalPacker intervalPacker; private final SqlFunctionProvider functionProvider; @@ -70,13 +70,12 @@ public QueryStep apply( IntervalPackingContext.builder() .ids(predecessorSelects.getIds()) .daterange(predecessorSelects.getValidityDate().get()) - .predecessor(finalDateAggregationStep) + .predecessor(Optional.of(finalDateAggregationStep)) .carryThroughSelects(carryThroughSelects) .tables(intervalPackingTables) - .conversionContext(conversionContext) .build(); - return this.intervalPacker.aggregateAsValidityDate(intervalPackingContext); + return AnsiSqlIntervalPacker.aggregateAsValidityDate(intervalPackingContext); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java index 529256b88f..1e70769242 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java @@ -3,7 +3,7 @@ import java.util.Map; import com.bakdata.conquery.models.datasets.concepts.Connector; -import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep; import com.bakdata.conquery.sql.compiler.ir.CteStep; import com.bakdata.conquery.sql.compiler.ir.SqlTables; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java index 8a1400d872..dec50dcca4 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java @@ -4,8 +4,8 @@ import java.util.List; import java.util.Optional; -import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingContext; -import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; +import com.bakdata.conquery.sql.compiler.ir.interval.AnsiSqlIntervalPacker; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingContext; import com.bakdata.conquery.sql.compiler.ir.JoinMode; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.QueryStepJoiner; @@ -63,8 +63,7 @@ protected QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { } else { IntervalPackingContext intervalPackingContext = createIntervalPackingContext(tableContext); - IntervalPacker intervalPacker = tableContext.getConversionContext().getCompilerDialect().getIntervalPacker(); - QueryStep lastIntervalPackingStep = intervalPacker.aggregateAsValidityDate(intervalPackingContext); + QueryStep lastIntervalPackingStep = AnsiSqlIntervalPacker.aggregateAsValidityDate(intervalPackingContext); queriesToJoin.add(lastIntervalPackingStep); validityDate = lastIntervalPackingStep.getQualifiedSelects().getValidityDate(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java index 7486225abf..bcdb846abc 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java @@ -1,7 +1,7 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; import static com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep.*; -import static com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep.INTERVAL_COMPLETE; +import static com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep.INTERVAL_COMPLETE; import java.util.HashMap; import java.util.List; @@ -13,7 +13,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.identifiable.ids.specific.SelectId; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep; import com.bakdata.conquery.sql.compiler.ir.CteStep; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.google.common.base.Preconditions; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingResult.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingResult.java deleted file mode 100644 index 8467ac7f13..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingResult.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.intervalpacking; - -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import lombok.Value; - -@Value -public class IntervalPackingResult { - - QueryStep finalIntervalPackingStep; - ColumnDateRange aggregatedRange; - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java deleted file mode 100644 index c537c89b1a..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.intervalpacking; - -import java.util.Optional; - -import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import lombok.RequiredArgsConstructor; - -/** - * PostgreSql supports interval packing with a native range function. - *

- * See && range function operator - */ -@RequiredArgsConstructor -public class PostgreSqlIntervalPacker implements IntervalPacker { - - private final SqlFunctionProvider functionProvider; - - @Override - public QueryStep aggregateAsValidityDate(IntervalPackingContext context) { - return aggregateDate(context, AggregationMode.VALIDITY_DATE); - } - - @Override - public QueryStep aggregateAsArbitrarySelect(IntervalPackingContext context) { - return aggregateDate(context, AggregationMode.ARBITRARY_SELECT); - } - - private QueryStep aggregateDate(IntervalPackingContext context, AggregationMode aggregationMode) { - - String sourceTableName = context.getTables().getPredecessor(IntervalPackingCteStep.INTERVAL_COMPLETE); - SqlIdColumns ids = context.getIds().qualify(sourceTableName); - ColumnDateRange qualifiedDaterange = context.getDaterange().qualify(sourceTableName); - ColumnDateRange aggregatedDaterange = this.functionProvider.aggregated(qualifiedDaterange); - - Selects.SelectsBuilder selectsBuilder = Selects.builder() - .ids(ids) - .sqlSelects(context.getCarryThroughSelects()); - - switch (aggregationMode) { - case VALIDITY_DATE -> selectsBuilder.validityDate(Optional.of(aggregatedDaterange)); - case ARBITRARY_SELECT -> selectsBuilder.sqlSelect(aggregatedDaterange); - } - - return QueryStep.builder() - .cteName(context.getTables().cteName(IntervalPackingCteStep.INTERVAL_COMPLETE)) - .selects(selectsBuilder.build()) - .fromTable(QueryStep.toTableLike(sourceTableName)) - .groupBy(ids.toFields()) - .predecessors(Optional.ofNullable(context.getPredecessor()).stream().toList()) - .build(); - } - - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/IntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/IntervalPacker.java deleted file mode 100644 index 5379b76d14..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/IntervalPacker.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.bakdata.conquery.sql.conversion.dialect; - - -import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingContext; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; - -/** - * Packing intervals involves packing groups of intersecting validity date intervals into their respective continuous intervals. - *

- * See Interval Packing - */ -public interface IntervalPacker { - - String PREVIOUS_END_FIELD_NAME = "previous_end"; - String RANGE_INDEX_FIELD_NAME = "range_index"; - - /** - * Depending on the dialect, one or more {@link QueryStep}s are created to aggregate multiple validity date entries of the same subject. The given - * {@link IntervalPackingContext#getDaterange()} is set as the validity date of the returned {@link QueryStep}. - * - *

- * Only the last {@link QueryStep} containing the aggregated validity dates will be returned. - */ - QueryStep aggregateAsValidityDate(IntervalPackingContext context); - - /** - * Depending on the dialect, one or more {@link QueryStep}s are created to aggregate multiple date entries of the same subject. The given - * {@link IntervalPackingContext#getDaterange()} is NOT set as the validity date of the returned {@link QueryStep}, but as a {@link SqlSelect}. - */ - QueryStep aggregateAsArbitrarySelect(IntervalPackingContext context); - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java index 174e3f66c6..55b49b9ded 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java @@ -69,8 +69,6 @@ default Field aggregateDateRanges(Field start, Field end) { SqlFunctionProvider getFunctionProvider(); - IntervalPacker getIntervalPacker(); - SqlDateAggregator getDateAggregator(); List> getNodeConverters(DSLContext context); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java index 9efe105302..1c7b00defb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java @@ -11,9 +11,7 @@ import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.cqelement.aggregation.AnsiSqlDateAggregator; -import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.AnsiSqlIntervalPacker; import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; -import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; @@ -29,13 +27,11 @@ public class ClickhouseDialectBundle implements DialectBundle { private final SqlFunctionProvider functionProvider; - private final IntervalPacker intervalPacker; private final SqlDateAggregator dateAggregator; public ClickhouseDialectBundle() { this.functionProvider = new ClickhouseFunctionProvider(); - this.intervalPacker = new AnsiSqlIntervalPacker(); - this.dateAggregator = new AnsiSqlDateAggregator(this.intervalPacker, this.functionProvider); + this.dateAggregator = new AnsiSqlDateAggregator(this.functionProvider); } @Override @@ -78,11 +74,6 @@ public SqlFunctionProvider getFunctionProvider() { return this.functionProvider; } - @Override - public IntervalPacker getIntervalPacker() { - return this.intervalPacker; - } - @Override public SqlDateAggregator getDateAggregator() { return this.dateAggregator; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaDialectBundle.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaDialectBundle.java index f271ca3003..fe9806ee80 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaDialectBundle.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaDialectBundle.java @@ -8,9 +8,7 @@ import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.cqelement.aggregation.AnsiSqlDateAggregator; -import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.AnsiSqlIntervalPacker; import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; -import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; @@ -25,14 +23,12 @@ public class HanaDialectBundle implements DialectBundle { private final SqlFunctionProvider functionProvider; - private final IntervalPacker intervalPacker; private final SqlDateAggregator dateAggregator; private final SqlCDateSetParser dateSetParser; public HanaDialectBundle() { this.functionProvider = new HanaSqlFunctionProvider(); - this.intervalPacker = new AnsiSqlIntervalPacker(); - this.dateAggregator = new AnsiSqlDateAggregator(this.intervalPacker, functionProvider); + this.dateAggregator = new AnsiSqlDateAggregator(functionProvider); this.dateSetParser = new HanaSqlCDateSetParser(); } @@ -90,11 +86,6 @@ public SqlFunctionProvider getFunctionProvider() { return this.functionProvider; } - @Override - public IntervalPacker getIntervalPacker() { - return this.intervalPacker; - } - @Override public SqlDateAggregator getDateAggregator() { return this.dateAggregator; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java index 74da3d6943..da40bff687 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java @@ -1,7 +1,7 @@ package com.bakdata.conquery.sql.conversion.model.select; import static com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep.*; -import static com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep.INTERVAL_COMPLETE; +import static com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep.INTERVAL_COMPLETE; import static org.jooq.impl.DSL.*; import java.math.BigDecimal; @@ -25,9 +25,9 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.cqelement.concept.IntervalPackingSelectsCte; -import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingContext; -import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep; -import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingContext; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep; +import com.bakdata.conquery.sql.compiler.ir.interval.AnsiSqlIntervalPacker; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.CteStep; @@ -60,7 +60,7 @@ public static ConnectorSqlSelects c .collect(Collectors.toList()); SqlTables daterangeSelectTables = createTables(alias, context.getTables(), context); - QueryStep lastIntervalPackingStep = applyIntervalPacking(daterange, daterangeSelectTables, context.getIds(), context.getTables(), context.getCompilerDialect()); + QueryStep lastIntervalPackingStep = applyIntervalPacking(daterange, daterangeSelectTables, context.getIds(), context.getTables()); ColumnDateRange qualified = daterange.qualify(daterangeSelectTables.getPredecessor(INTERVAL_PACKING_SELECTS)); FieldWrapper aggregationField = aggregationFunction.apply(qualified, alias, functionProvider); @@ -102,7 +102,7 @@ public static SqlFilters createForFilter( .collect(Collectors.toList()); SqlTables daterangeSelectTables = createTables(alias, context.getTables(), context); - QueryStep lastIntervalPackingStep = applyIntervalPacking(daterange, daterangeSelectTables, context.getIds(), context.getTables(), context.getCompilerDialect()); + QueryStep lastIntervalPackingStep = applyIntervalPacking(daterange, daterangeSelectTables, context.getIds(), context.getTables()); ColumnDateRange qualified = daterange.qualify(daterangeSelectTables.getPredecessor(INTERVAL_PACKING_SELECTS)); FieldWrapper aggregationField = aggregationFunction.apply(qualified, alias, functionProvider); @@ -166,8 +166,7 @@ private static QueryStep applyIntervalPacking( ColumnDateRange daterange, SqlTables dateUnionTables, SqlIdColumns idColumns, - ConnectorSqlTables connectorSqlTables, - LegacyCompilerDialect sqlDialect + ConnectorSqlTables connectorSqlTables ) { String preprocessingCteName = connectorSqlTables.cteName(PREPROCESSING); IntervalPackingContext intervalPackingContext = IntervalPackingContext.builder() @@ -176,9 +175,7 @@ private static QueryStep applyIntervalPacking( .tables(dateUnionTables) .build(); - return sqlDialect - .getIntervalPacker() - .aggregateAsArbitrarySelect(intervalPackingContext); + return AnsiSqlIntervalPacker.aggregateAsArbitrarySelect(intervalPackingContext); } @FunctionalInterface diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AggregationMode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/AggregationMode.java similarity index 81% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AggregationMode.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/AggregationMode.java index 4592d34294..c88fa460e4 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AggregationMode.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/AggregationMode.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.cqelement.intervalpacking; +package com.bakdata.conquery.sql.compiler.ir.interval; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/AnsiSqlIntervalPacker.java similarity index 80% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/AnsiSqlIntervalPacker.java index c7ab0dc4ba..c950d2fb6c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/AnsiSqlIntervalPacker.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.cqelement.intervalpacking; +package com.bakdata.conquery.sql.compiler.ir.interval; import java.math.BigDecimal; import java.sql.Date; @@ -6,7 +6,6 @@ import java.util.List; import java.util.Optional; -import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QualifyingUtil; import com.bakdata.conquery.sql.compiler.ir.QueryStep; @@ -14,31 +13,32 @@ import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import lombok.RequiredArgsConstructor; +import lombok.experimental.UtilityClass; import org.jooq.Field; import org.jooq.impl.DSL; -@RequiredArgsConstructor -public class AnsiSqlIntervalPacker implements IntervalPacker { +@UtilityClass +public class AnsiSqlIntervalPacker { - @Override - public QueryStep aggregateAsValidityDate(IntervalPackingContext context) { + private static final String PREVIOUS_END_FIELD_NAME = "previous_end"; + private static final String RANGE_INDEX_FIELD_NAME = "range_index"; + + public static QueryStep aggregateAsValidityDate(IntervalPackingContext context) { return aggregateDate(context, AggregationMode.VALIDITY_DATE); } - @Override - public QueryStep aggregateAsArbitrarySelect(IntervalPackingContext context) { + public static QueryStep aggregateAsArbitrarySelect(IntervalPackingContext context) { return aggregateDate(context, AggregationMode.ARBITRARY_SELECT); } - private QueryStep aggregateDate(IntervalPackingContext context, AggregationMode aggregationMode) { + private static QueryStep aggregateDate(IntervalPackingContext context, AggregationMode aggregationMode) { QueryStep previousEndStep = createPreviousEndStep(context, aggregationMode); QueryStep rangeIndexStep = createRangeIndexStep(previousEndStep, context, aggregationMode); QueryStep intervalCompleteStep = createIntervalCompleteStep(rangeIndexStep, context, aggregationMode); return intervalCompleteStep; } - private QueryStep createPreviousEndStep(IntervalPackingContext context, AggregationMode aggregationMode) { + private static QueryStep createPreviousEndStep(IntervalPackingContext context, AggregationMode aggregationMode) { String sourceTableName = context.getTables().getPredecessor(IntervalPackingCteStep.PREVIOUS_END); SqlIdColumns ids = context.getIds().qualify(sourceTableName); @@ -49,7 +49,7 @@ private QueryStep createPreviousEndStep(IntervalPackingContext context, Aggregat .orderBy(daterange.getStart(), daterange.getEnd()) .rowsBetweenUnboundedPreceding() .andPreceding(1)) - .as(IntervalPacker.PREVIOUS_END_FIELD_NAME); + .as(PREVIOUS_END_FIELD_NAME); List qualifiedSelects = new ArrayList<>(QualifyingUtil.qualify(context.getCarryThroughSelects(), sourceTableName)); qualifiedSelects.add(new FieldWrapper<>(previousEnd, daterange.getStart().getName(), daterange.getEnd().getName())); @@ -60,17 +60,17 @@ private QueryStep createPreviousEndStep(IntervalPackingContext context, Aggregat .cteName(context.getTables().cteName(IntervalPackingCteStep.PREVIOUS_END)) .selects(previousEndSelects) .fromTable(QueryStep.toTableLike(sourceTableName)) - .predecessors(Optional.ofNullable(context.getPredecessor()).stream().toList()) + .predecessors(context.getPredecessor().stream().toList()) .build(); } - private QueryStep createRangeIndexStep(QueryStep previousEndStep, IntervalPackingContext context, AggregationMode aggregationMode) { + private static QueryStep createRangeIndexStep(QueryStep previousEndStep, IntervalPackingContext context, AggregationMode aggregationMode) { String previousEndCteName = previousEndStep.getCteName(); Selects previousEndSelects = previousEndStep.getQualifiedSelects(); SqlIdColumns ids = previousEndSelects.getIds(); ColumnDateRange daterange = context.getDaterange().qualify(previousEndCteName); - Field previousEnd = DSL.field(DSL.name(previousEndCteName, IntervalPacker.PREVIOUS_END_FIELD_NAME), Date.class); + Field previousEnd = DSL.field(DSL.name(previousEndCteName, PREVIOUS_END_FIELD_NAME), Date.class); Field rangeIndex = DSL.sum( @@ -79,7 +79,7 @@ private QueryStep createRangeIndexStep(QueryStep previousEndStep, IntervalPackin .over(DSL.partitionBy(ids.toFields()) .orderBy(daterange.getStart(), daterange.getEnd()) .rowsUnboundedPreceding()) - .as(IntervalPacker.RANGE_INDEX_FIELD_NAME); + .as(RANGE_INDEX_FIELD_NAME); List qualifiedSelects = new ArrayList<>(QualifyingUtil.qualify(context.getCarryThroughSelects(), previousEndCteName)); qualifiedSelects.add(new FieldWrapper<>(rangeIndex)); @@ -94,7 +94,7 @@ private QueryStep createRangeIndexStep(QueryStep previousEndStep, IntervalPackin .build(); } - private QueryStep createIntervalCompleteStep(QueryStep rangeIndexStep, IntervalPackingContext context, AggregationMode aggregationMode) { + private static QueryStep createIntervalCompleteStep(QueryStep rangeIndexStep, IntervalPackingContext context, AggregationMode aggregationMode) { String rangeIndexCteName = rangeIndexStep.getCteName(); Selects rangeIndexSelects = rangeIndexStep.getQualifiedSelects(); @@ -104,7 +104,7 @@ private QueryStep createIntervalCompleteStep(QueryStep rangeIndexStep, IntervalP Field rangeStart = DSL.min(daterange.getStart()).as(daterange.getStart().getName()); Field rangeEnd = DSL.max(daterange.getEnd()).as(daterange.getEnd().getName()); ColumnDateRange minMax = ColumnDateRange.of(rangeStart, rangeEnd); - Field rangeIndex = DSL.field(DSL.name(rangeIndexCteName, IntervalPacker.RANGE_INDEX_FIELD_NAME), BigDecimal.class); + Field rangeIndex = DSL.field(DSL.name(rangeIndexCteName, RANGE_INDEX_FIELD_NAME), BigDecimal.class); List qualifiedSelects = QualifyingUtil.qualify(context.getCarryThroughSelects(), rangeIndexCteName); Selects intervalCompleteSelects = buildSelects(ids, minMax, qualifiedSelects, aggregationMode); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingContext.java similarity index 59% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingContext.java index 1aa57b4f23..25c0417abf 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingContext.java @@ -1,24 +1,21 @@ -package com.bakdata.conquery.sql.conversion.cqelement.intervalpacking; +package com.bakdata.conquery.sql.compiler.ir.interval; import java.util.Collections; import java.util.List; +import java.util.Optional; -import javax.annotation.Nullable; - -import com.bakdata.conquery.sql.conversion.Context; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Builder; import lombok.Value; +/** Inputs required to transform a date range into interval-packing query steps. */ @Value @Builder -public class IntervalPackingContext implements Context { +public class IntervalPackingContext { SqlIdColumns ids; @@ -30,8 +27,8 @@ public class IntervalPackingContext implements Context { /** * An optional predecessor of the first interval packing CTE. */ - @Nullable - QueryStep predecessor; + @Builder.Default + Optional predecessor = Optional.empty(); SqlTables tables; @@ -41,12 +38,4 @@ public class IntervalPackingContext implements Context { @Builder.Default List carryThroughSelects = Collections.emptyList(); - @Nullable - ConversionContext conversionContext; - - @Override - public LegacyCompilerDialect getCompilerDialect() { - return getConversionContext().getCompilerDialect(); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingCteStep.java similarity index 81% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingCteStep.java index be56b8333a..08ce1b2f58 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingCteStep.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.cqelement.intervalpacking; +package com.bakdata.conquery.sql.compiler.ir.interval; import java.util.HashMap; import java.util.Map; @@ -6,7 +6,6 @@ import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; -import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.compiler.ir.CteStep; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.SqlTables; @@ -21,8 +20,6 @@ public enum IntervalPackingCteStep implements CteStep { RANGE_INDEX("range_index", PREVIOUS_END), INTERVAL_COMPLETE("interval_complete", RANGE_INDEX); - private static final Set STEPS = Set.of(values()); - private final String suffix; private final CteStep predecessor; @@ -45,7 +42,7 @@ public static SqlTables createTables( rootTable, nameGenerator::cteStepName ); - Map predecessorMap = CteStep.getDefaultPredecessorMap(requiredSteps); + Map predecessorMap = getMappings(dialect); return new SqlTables(rootTable, cteNameMap, predecessorMap); } @@ -54,7 +51,7 @@ public static SqlTables createTables( * Create predecessor mappings for these interval packing {@link CteStep}s based on a preceding root step that must contain a validity date which * shall be interval-packed. */ - public static Map getMappings(CteStep root, LegacyCompilerDialect dialect) { + public static Map getMappings(CteStep root, CompilerDialect dialect) { if (dialect.supportsSingleColumnRanges()) { return Map.of(INTERVAL_COMPLETE, root); } @@ -66,7 +63,7 @@ public static Map getMappings(CteStep root, LegacyCompilerDial /** * Create predecessor mappings for these interval packing {@link CteStep}s based on the default mapping. */ - public static Map getMappings(LegacyCompilerDialect dialect) { + public static Map getMappings(CompilerDialect dialect) { if (dialect.supportsSingleColumnRanges()) { Map mappings = new HashMap<>(); mappings.put(INTERVAL_COMPLETE, null); // final step directly mapped onto root table diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/interval-packing.md b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/interval-packing.md similarity index 99% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/interval-packing.md rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/interval-packing.md index a667b3c6ec..24ce6dc133 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/interval-packing.md +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/interval-packing.md @@ -48,7 +48,7 @@ create 3 consecutive common table conceptConditions (CTE): This table generates the `previous_end` select: The `previous_end` value for each row will be the maximum `date_end` within the same `id` partition that occurs before the current row's `date_start`. -- `partion by "id"` creates a window frame for each subject. +- `partition by "id"` creates a window frame for each subject. - `order by "date_start", "date_end"`: arranges the rows in ascending order of their start dates and if start dates are the same, it ensures that the event with the earlier end date is considered first within the window frame. - `rows between unbounded preceding and preceding 1` ensures that the maximum `date_end` that occurred before diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/package-info.java new file mode 100644 index 0000000000..fe98146abb --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/package-info.java @@ -0,0 +1,2 @@ +/** Interval-packing transformations over the connector's internal query-step representation. */ +package com.bakdata.conquery.sql.compiler.ir.interval; diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingTest.java new file mode 100644 index 0000000000..1e8d3cdb16 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingTest.java @@ -0,0 +1,130 @@ +package com.bakdata.conquery.sql.compiler.ir.interval; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.Date; +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import org.jooq.Field; +import org.junit.jupiter.api.Test; + +class IntervalPackingTest { + + private static final SqlIdColumns IDS = new SqlIdColumns(field(name("person"), String.class)); + private static final ColumnDateRange DATE_RANGE = ColumnDateRange.of( + field(name("valid_from"), Date.class), + field(name("valid_to"), Date.class) + ); + + @Test + void shouldCreateThreeStepGraphForSeparateRangeColumns() { + SqlTables tables = IntervalPackingCteStep.createTables( + sourceStep(), + new TestDialect(false), + new SqlNameGenerator(128) + ); + + assertEquals("events-previous_end", tables.cteName(IntervalPackingCteStep.PREVIOUS_END)); + assertEquals("events-range_index", tables.cteName(IntervalPackingCteStep.RANGE_INDEX)); + assertEquals("events-interval_complete", tables.cteName(IntervalPackingCteStep.INTERVAL_COMPLETE)); + assertEquals("events", tables.getPredecessor(IntervalPackingCteStep.PREVIOUS_END)); + assertEquals("events-previous_end", tables.getPredecessor(IntervalPackingCteStep.RANGE_INDEX)); + assertEquals("events-range_index", tables.getPredecessor(IntervalPackingCteStep.INTERVAL_COMPLETE)); + } + + @Test + void shouldSkipIntermediateStepsForSingleColumnRanges() { + SqlTables tables = IntervalPackingCteStep.createTables( + sourceStep(), + new TestDialect(true), + new SqlNameGenerator(128) + ); + + assertFalse(tables.isRequiredStep(IntervalPackingCteStep.PREVIOUS_END)); + assertFalse(tables.isRequiredStep(IntervalPackingCteStep.RANGE_INDEX)); + assertTrue(tables.isRequiredStep(IntervalPackingCteStep.INTERVAL_COMPLETE)); + assertEquals("events", tables.getPredecessor(IntervalPackingCteStep.INTERVAL_COMPLETE)); + } + + @Test + void shouldBuildAnsiPackingStepsOverConnectorIr() { + QueryStep source = sourceStep(); + SqlTables tables = IntervalPackingCteStep.createTables( + source, + new TestDialect(false), + new SqlNameGenerator(128) + ); + IntervalPackingContext context = IntervalPackingContext.builder() + .ids(IDS) + .daterange(DATE_RANGE) + .predecessor(Optional.of(source)) + .tables(tables) + .build(); + + QueryStep intervalComplete = AnsiSqlIntervalPacker.aggregateAsValidityDate(context); + QueryStep rangeIndex = intervalComplete.getPredecessors().getFirst(); + QueryStep previousEnd = rangeIndex.getPredecessors().getFirst(); + + assertEquals("events-interval_complete", intervalComplete.getCteName()); + assertEquals("events-range_index", rangeIndex.getCteName()); + assertEquals("events-previous_end", previousEnd.getCteName()); + assertEquals(List.of(source), previousEnd.getPredecessors()); + assertTrue(intervalComplete.getSelects().getValidityDate().isPresent()); + assertEquals(DATE_RANGE.getStart().getName(), intervalComplete.getSelects().getValidityDate().orElseThrow().getStart().getName()); + } + + private static QueryStep sourceStep() { + return QueryStep.builder() + .cteName("events") + .selects(Selects.builder() + .ids(IDS) + .validityDate(Optional.of(DATE_RANGE)) + .build()) + .build(); + } + + private record TestDialect(boolean supportsSingleColumnRanges) implements CompilerDialect { + + @Override + public Field minimumDate() { + return field(name("minimum_date"), Date.class); + } + + @Override + public Field maximumDate() { + return field(name("maximum_date"), Date.class); + } + + @Override + public Field anyValue(Field value) { + return value; + } + + @Override + public Field renderDateRange(Field start, Field end) { + return field(name("rendered_range"), Object.class); + } + + @Override + public Field aggregateDateRanges(Field start, Field end) { + return field(name("aggregated_ranges"), Object.class); + } + + @Override + public int getNameMaxLength() { + return 128; + } + } +} From 22b78b50308b375d9cdb2295f53edf7e98301cc0 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:55:16 +0200 Subject: [PATCH 30/81] refactor(sql): move date aggregation into connector Move the date aggregation and inversion CTE graph into the connector IR and expose it through a framework-neutral DateAggregationCompiler. Retain a small legacy backend adapter that supplies dialect and naming dependencies. Add direct connector tests for merge, intersect, inversion, empty date input, and unsupported aggregation actions. Co-authored-by: Codex GPT-5.6-sol --- .../aggregation/AnsiSqlDateAggregator.java | 120 +++----------- .../clickhouse/ClickhouseDialectBundle.java | 2 +- .../dialect/hana/HanaDialectBundle.java | 2 +- .../aggregation/DateAggregationCompiler.java | 118 +++++++++++++ .../aggregation/DateAggregationContext.java | 2 +- .../ir}/aggregation/DateAggregationCte.java | 2 +- .../DateAggregationCteConstructor.java | 2 +- .../aggregation/DateAggregationCteStep.java | 2 +- .../ir}/aggregation/IntermediateTableCte.java | 2 +- .../IntersectAggregationAction.java | 2 +- .../compiler/ir}/aggregation/InvertCte.java | 2 +- .../ir}/aggregation/MergeAggregateAction.java | 2 +- .../compiler/ir}/aggregation/MergeCte.java | 2 +- .../ir}/aggregation/NodeNoOverlapCte.java | 2 +- .../compiler/ir}/aggregation/OverlapCte.java | 2 +- .../ir}/aggregation/RowNumberCte.java | 2 +- .../ir}/aggregation/SqlAggregationAction.java | 2 +- .../compiler/ir/aggregation/package-info.java | 2 + .../ir}/aggregation/sql_date_aggregation.md | 2 +- .../DateAggregationCompilerTest.java | 155 ++++++++++++++++++ 20 files changed, 316 insertions(+), 111 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationCompiler.java rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/aggregation/DateAggregationContext.java (97%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/aggregation/DateAggregationCte.java (95%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/aggregation/DateAggregationCteConstructor.java (65%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/aggregation/DateAggregationCteStep.java (97%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/aggregation/IntermediateTableCte.java (96%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/aggregation/IntersectAggregationAction.java (97%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/aggregation/InvertCte.java (98%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/aggregation/MergeAggregateAction.java (97%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/aggregation/MergeCte.java (94%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/aggregation/NodeNoOverlapCte.java (97%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/aggregation/OverlapCte.java (96%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/aggregation/RowNumberCte.java (96%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/aggregation/SqlAggregationAction.java (94%) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/package-info.java rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/aggregation/sql_date_aggregation.md (99%) create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationCompilerTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java index c2f0636a35..3a73348de6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java @@ -1,35 +1,23 @@ package com.bakdata.conquery.sql.conversion.cqelement.aggregation; -import static org.jooq.impl.DSL.field; -import static org.jooq.impl.DSL.inline; - -import java.sql.Date; import java.util.List; -import java.util.Optional; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; -import com.bakdata.conquery.sql.compiler.ir.FieldExpressions; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.compiler.ir.interval.AnsiSqlIntervalPacker; -import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingContext; -import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep; -import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.aggregation.DateAggregationCompiler; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import com.bakdata.conquery.models.query.DateAggregationAction; -import lombok.Data; -import org.jooq.Field; +import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; +import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; -@Data +/** + * Adapts the legacy backend date-aggregation contract to the connector compiler. + * + *

TODO Remove this adapter when backend converters invoke {@link DateAggregationCompiler} directly.

+ */ public class AnsiSqlDateAggregator implements SqlDateAggregator { - private final SqlFunctionProvider functionProvider; - - @Override public QueryStep apply( QueryStep joinedStep, @@ -38,88 +26,30 @@ public QueryStep apply( DateAggregationAction dateAggregationAction, ConversionContext conversionContext ) { - SqlAggregationAction aggregationAction = switch (dateAggregationAction) { - case MERGE -> new MergeAggregateAction(joinedStep); - case INTERSECT -> new IntersectAggregationAction(joinedStep); - default -> throw new IllegalStateException("Unexpected date aggregation action: %s".formatted(dateAggregationAction)); - }; - - DateAggregationContext context = - DateAggregationContext.builder() - .sqlAggregationAction(aggregationAction) - .carryThroughSelects(carryThroughSelects) - .dateAggregationDates(dateAggregationDates) - .dateAggregationTables(aggregationAction.tableNames(conversionContext.getNameGenerator())) - .ids(joinedStep.getQualifiedSelects().getIds()) - .compilerDialect(conversionContext.getCompilerDialect()) - .build(); - - QueryStep finalDateAggregationStep = convertSteps(joinedStep, aggregationAction.dateAggregationCtes(), context); - if (!aggregationAction.requiresIntervalPackingAfterwards()) { - return finalDateAggregationStep; - } - - Selects predecessorSelects = finalDateAggregationStep.getSelects(); - SqlTables intervalPackingTables = IntervalPackingCteStep.createTables( - finalDateAggregationStep, + return DateAggregationCompiler.aggregate( + joinedStep, + carryThroughSelects, + dateAggregationDates, + dateAggregationAction, conversionContext.getCompilerDialect(), conversionContext.getNameGenerator() ); - - IntervalPackingContext intervalPackingContext = - IntervalPackingContext.builder() - .ids(predecessorSelects.getIds()) - .daterange(predecessorSelects.getValidityDate().get()) - .predecessor(Optional.of(finalDateAggregationStep)) - .carryThroughSelects(carryThroughSelects) - .tables(intervalPackingTables) - .build(); - - return AnsiSqlIntervalPacker.aggregateAsValidityDate(intervalPackingContext); } @Override - public ColumnDateRange getAggregatedValidityDate(DateAggregationDates dateAggregationDates, DateAggregationAction dateAggregationAction) { - //TODO(FK): i think this is only ever relevant with dateMode=Logical which i want to remove - Field rangeStart = FieldExpressions.least(dateAggregationDates.allStarts()); - Field rangeEnd = FieldExpressions.greatest(dateAggregationDates.allEnds()); - - return ColumnDateRange.of( - rangeStart.as(DateAggregationCte.RANGE_START), - rangeEnd.as(DateAggregationCte.RANGE_END) - ); + public ColumnDateRange getAggregatedValidityDate( + DateAggregationDates dateAggregationDates, + DateAggregationAction dateAggregationAction + ) { + return DateAggregationCompiler.getAggregatedValidityDate(dateAggregationDates); } @Override public QueryStep invertAggregatedIntervals(QueryStep baseStep, ConversionContext conversionContext) { - - DateAggregationDates dateAggregationDates = DateAggregationDates.forSingleStep(baseStep); - if (dateAggregationDates.dateAggregationImpossible()) { - return baseStep; - } - - Selects baseStepQualifiedSelects = baseStep.getQualifiedSelects(); - SqlTables dateAggregationTables = DateAggregationCteStep.createInvertTables(baseStep, conversionContext.getNameGenerator()); - - DateAggregationContext context = DateAggregationContext.builder() - .sqlAggregationAction(null) // when inverting, an aggregation has already been applied - .carryThroughSelects(baseStepQualifiedSelects.getSqlSelects()) - .dateAggregationDates(dateAggregationDates) - .dateAggregationTables(dateAggregationTables) - .ids(baseStepQualifiedSelects.getIds()) - .compilerDialect(conversionContext.getCompilerDialect()) - .build(); - - return convertSteps(baseStep, DateAggregationCteStep.createInvertCtes(), context); - } - - private QueryStep convertSteps(QueryStep baseStep, List dateAggregationCTEs, DateAggregationContext context) { - QueryStep finalDateAggregationStep = baseStep; - for (DateAggregationCte step : dateAggregationCTEs) { - finalDateAggregationStep = step.convert(context, finalDateAggregationStep); - context = context.withStep(step.getCteStep(), finalDateAggregationStep); - } - return finalDateAggregationStep; + return DateAggregationCompiler.invert( + baseStep, + conversionContext.getCompilerDialect(), + conversionContext.getNameGenerator() + ); } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java index 1c7b00defb..167aefc80e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java @@ -31,7 +31,7 @@ public class ClickhouseDialectBundle implements DialectBundle { public ClickhouseDialectBundle() { this.functionProvider = new ClickhouseFunctionProvider(); - this.dateAggregator = new AnsiSqlDateAggregator(this.functionProvider); + this.dateAggregator = new AnsiSqlDateAggregator(); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaDialectBundle.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaDialectBundle.java index fe9806ee80..e9049d46fd 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaDialectBundle.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaDialectBundle.java @@ -28,7 +28,7 @@ public class HanaDialectBundle implements DialectBundle { public HanaDialectBundle() { this.functionProvider = new HanaSqlFunctionProvider(); - this.dateAggregator = new AnsiSqlDateAggregator(functionProvider); + this.dateAggregator = new AnsiSqlDateAggregator(); this.dateSetParser = new HanaSqlCDateSetParser(); } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationCompiler.java new file mode 100644 index 0000000000..d7f79f535b --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationCompiler.java @@ -0,0 +1,118 @@ +package com.bakdata.conquery.sql.compiler.ir.aggregation; + +import java.sql.Date; +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.models.query.DateAggregationAction; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; +import com.bakdata.conquery.sql.compiler.ir.FieldExpressions; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.interval.AnsiSqlIntervalPacker; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingContext; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import lombok.experimental.UtilityClass; +import org.jooq.Field; + +/** Builds date aggregation and inversion query-step graphs from connector IR. */ +@UtilityClass +public class DateAggregationCompiler { + + public static QueryStep aggregate( + QueryStep joinedStep, + List carryThroughSelects, + DateAggregationDates dateAggregationDates, + DateAggregationAction dateAggregationAction, + CompilerDialect compilerDialect, + SqlNameGenerator nameGenerator + ) { + SqlAggregationAction aggregationAction = switch (dateAggregationAction) { + case MERGE -> new MergeAggregateAction(joinedStep); + case INTERSECT -> new IntersectAggregationAction(joinedStep); + default -> throw new IllegalStateException("Unexpected date aggregation action: %s".formatted(dateAggregationAction)); + }; + + DateAggregationContext context = DateAggregationContext.builder() + .sqlAggregationAction(aggregationAction) + .carryThroughSelects(carryThroughSelects) + .dateAggregationDates(dateAggregationDates) + .dateAggregationTables(aggregationAction.tableNames(nameGenerator)) + .ids(joinedStep.getQualifiedSelects().getIds()) + .compilerDialect(compilerDialect) + .build(); + + QueryStep finalDateAggregationStep = convertSteps(joinedStep, aggregationAction.dateAggregationCtes(), context); + if (!aggregationAction.requiresIntervalPackingAfterwards()) { + return finalDateAggregationStep; + } + + Selects predecessorSelects = finalDateAggregationStep.getSelects(); + SqlTables intervalPackingTables = IntervalPackingCteStep.createTables( + finalDateAggregationStep, + compilerDialect, + nameGenerator + ); + IntervalPackingContext intervalPackingContext = IntervalPackingContext.builder() + .ids(predecessorSelects.getIds()) + .daterange(predecessorSelects.getValidityDate().orElseThrow()) + .predecessor(Optional.of(finalDateAggregationStep)) + .carryThroughSelects(carryThroughSelects) + .tables(intervalPackingTables) + .build(); + + return AnsiSqlIntervalPacker.aggregateAsValidityDate(intervalPackingContext); + } + + public static ColumnDateRange getAggregatedValidityDate(DateAggregationDates dateAggregationDates) { + Field rangeStart = FieldExpressions.least(dateAggregationDates.allStarts()); + Field rangeEnd = FieldExpressions.greatest(dateAggregationDates.allEnds()); + + return ColumnDateRange.of( + rangeStart.as(DateAggregationCte.RANGE_START), + rangeEnd.as(DateAggregationCte.RANGE_END) + ); + } + + public static QueryStep invert( + QueryStep baseStep, + CompilerDialect compilerDialect, + SqlNameGenerator nameGenerator + ) { + DateAggregationDates dateAggregationDates = DateAggregationDates.forSingleStep(baseStep); + if (dateAggregationDates.dateAggregationImpossible()) { + return baseStep; + } + + Selects baseStepQualifiedSelects = baseStep.getQualifiedSelects(); + SqlTables dateAggregationTables = DateAggregationCteStep.createInvertTables(baseStep, nameGenerator); + DateAggregationContext context = DateAggregationContext.builder() + .sqlAggregationAction(null) + .carryThroughSelects(baseStepQualifiedSelects.getSqlSelects()) + .dateAggregationDates(dateAggregationDates) + .dateAggregationTables(dateAggregationTables) + .ids(baseStepQualifiedSelects.getIds()) + .compilerDialect(compilerDialect) + .build(); + + return convertSteps(baseStep, DateAggregationCteStep.createInvertCtes(), context); + } + + private static QueryStep convertSteps( + QueryStep baseStep, + List dateAggregationCtes, + DateAggregationContext context + ) { + QueryStep finalDateAggregationStep = baseStep; + for (DateAggregationCte step : dateAggregationCtes) { + finalDateAggregationStep = step.convert(context, finalDateAggregationStep); + context = context.withStep(step.getCteStep(), finalDateAggregationStep); + } + return finalDateAggregationStep; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationContext.java similarity index 97% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationContext.java index d9b4a52448..eb15990c7d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationContext.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.cqelement.aggregation; +package com.bakdata.conquery.sql.compiler.ir.aggregation; import java.util.ArrayList; import java.util.HashMap; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationCte.java similarity index 95% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationCte.java index aa17adb282..3ed7e5147b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationCte.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.cqelement.aggregation; +package com.bakdata.conquery.sql.compiler.ir.aggregation; import java.util.List; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteConstructor.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationCteConstructor.java similarity index 65% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteConstructor.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationCteConstructor.java index 8cd7eb996f..bf1585b7ed 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteConstructor.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationCteConstructor.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.cqelement.aggregation; +package com.bakdata.conquery.sql.compiler.ir.aggregation; @FunctionalInterface interface DateAggregationCteConstructor { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteStep.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationCteStep.java similarity index 97% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteStep.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationCteStep.java index 43e19eca7c..3e50239a30 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteStep.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationCteStep.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.cqelement.aggregation; +package com.bakdata.conquery.sql.compiler.ir.aggregation; import java.util.HashSet; import java.util.List; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/IntermediateTableCte.java similarity index 96% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/IntermediateTableCte.java index e1eac9b0d9..322d2dc673 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/IntermediateTableCte.java @@ -1,5 +1,5 @@ -package com.bakdata.conquery.sql.conversion.cqelement.aggregation; +package com.bakdata.conquery.sql.compiler.ir.aggregation; import java.sql.Date; import java.util.List; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/IntersectAggregationAction.java similarity index 97% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/IntersectAggregationAction.java index 257ce7675e..e82bd28913 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/IntersectAggregationAction.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.cqelement.aggregation; +package com.bakdata.conquery.sql.compiler.ir.aggregation; import java.sql.Date; import java.util.Collection; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/InvertCte.java similarity index 98% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/InvertCte.java index dadab95852..d42e5256b6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/InvertCte.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.cqelement.aggregation; +package com.bakdata.conquery.sql.compiler.ir.aggregation; import static org.jooq.impl.DSL.*; import static org.jooq.impl.DSL.field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/MergeAggregateAction.java similarity index 97% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/MergeAggregateAction.java index 77672662a1..af7f819602 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/MergeAggregateAction.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.cqelement.aggregation; +package com.bakdata.conquery.sql.compiler.ir.aggregation; import java.sql.Date; import java.util.Collection; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeCte.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/MergeCte.java similarity index 94% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeCte.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/MergeCte.java index 27091786df..955354d018 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeCte.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/MergeCte.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.cqelement.aggregation; +package com.bakdata.conquery.sql.compiler.ir.aggregation; import java.util.List; import java.util.stream.Collectors; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/NodeNoOverlapCte.java similarity index 97% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/NodeNoOverlapCte.java index 3a11348615..c8b53a79c7 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/NodeNoOverlapCte.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.cqelement.aggregation; +package com.bakdata.conquery.sql.compiler.ir.aggregation; import java.sql.Date; import java.util.Iterator; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/OverlapCte.java similarity index 96% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/OverlapCte.java index e305f43d30..c7a9945b10 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/OverlapCte.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.cqelement.aggregation; +package com.bakdata.conquery.sql.compiler.ir.aggregation; import java.sql.Date; import java.util.List; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/RowNumberCte.java similarity index 96% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/RowNumberCte.java index d13b5c3b06..1557509d6a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/RowNumberCte.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.cqelement.aggregation; +package com.bakdata.conquery.sql.compiler.ir.aggregation; import java.util.ArrayList; import java.util.Optional; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/SqlAggregationAction.java similarity index 94% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/SqlAggregationAction.java index 665fb134d7..db4be54816 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/SqlAggregationAction.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.cqelement.aggregation; +package com.bakdata.conquery.sql.compiler.ir.aggregation; import java.util.List; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/package-info.java new file mode 100644 index 0000000000..d66c1f36e6 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/package-info.java @@ -0,0 +1,2 @@ +/** Date aggregation transformations over the connector's internal query-step representation. */ +package com.bakdata.conquery.sql.compiler.ir.aggregation; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/sql_date_aggregation.md b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/sql_date_aggregation.md similarity index 99% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/sql_date_aggregation.md rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/sql_date_aggregation.md index ec4a707087..fb29832d5f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/sql_date_aggregation.md +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/sql_date_aggregation.md @@ -10,7 +10,7 @@ Besides that, there is also the possibility to `NEGATE` a date aggregation: this a node. Finally, dates of a certain node can also be `BLOCKED` which will block the upwards aggregation of a nodes' validity dates. -The document outlines the process the [AnsiSqlDateAggregator](./AnsiSqlDateAggregator.java) employs to realise the +The document outlines the process the [DateAggregationCompiler](./DateAggregationCompiler.java) employs to realise the different aggregations. Suppose we have two concepts, each containing a date range set with only unique date range values. diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationCompilerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationCompilerTest.java new file mode 100644 index 0000000000..0e8cbe6478 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationCompilerTest.java @@ -0,0 +1,155 @@ +package com.bakdata.conquery.sql.compiler.ir.aggregation; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.Date; +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.models.query.DateAggregationAction; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import org.jooq.Field; +import org.junit.jupiter.api.Test; + +class DateAggregationCompilerTest { + + private static final CompilerDialect DIALECT = new TestDialect(); + + @Test + void shouldMergeDatesAndPackTheResultingIntervals() { + QueryStep result = DateAggregationCompiler.aggregate( + joinedStep(), + List.of(), + dateInputs(), + DateAggregationAction.MERGE, + DIALECT, + new SqlNameGenerator(128) + ); + + assertEquals("joined-merge-interval_complete", result.getCteName()); + assertTrue(result.getSelects().getValidityDate().isPresent()); + assertEquals("joined-merge-range_index", result.getPredecessors().getFirst().getCteName()); + } + + @Test + void shouldIntersectDatesWithoutAdditionalIntervalPacking() { + QueryStep result = DateAggregationCompiler.aggregate( + joinedStep(), + List.of(), + dateInputs(), + DateAggregationAction.INTERSECT, + DIALECT, + new SqlNameGenerator(128) + ); + + assertEquals("joined-merge", result.getCteName()); + assertTrue(result.isUnion()); + } + + @Test + void shouldInvertAggregatedIntervals() { + QueryStep base = stepWithValidityDate("base", dateRange("valid_from", "valid_to")); + + QueryStep result = DateAggregationCompiler.invert(base, DIALECT, new SqlNameGenerator(128)); + + assertEquals("base-inverted_dates", result.getCteName()); + assertEquals("base-row_numbers", result.getPredecessors().getFirst().getCteName()); + assertTrue(result.getSelects().getValidityDate().isPresent()); + } + + @Test + void shouldReturnOriginalStepWhenThereAreNoDatesToInvert() { + QueryStep base = joinedStep(); + + assertSame(base, DateAggregationCompiler.invert(base, DIALECT, new SqlNameGenerator(128))); + } + + @Test + void shouldRejectNonAggregationActions() { + assertThrows( + IllegalStateException.class, + () -> DateAggregationCompiler.aggregate( + joinedStep(), + List.of(), + dateInputs(), + DateAggregationAction.BLOCK, + DIALECT, + new SqlNameGenerator(128) + ) + ); + } + + private static QueryStep joinedStep() { + return QueryStep.builder() + .cteName("joined") + .selects(Selects.builder() + .ids(new SqlIdColumns(field(name("person"), String.class))) + .build()) + .build(); + } + + private static QueryStep stepWithValidityDate(String cteName, ColumnDateRange dateRange) { + return QueryStep.builder() + .cteName(cteName) + .selects(Selects.builder() + .ids(new SqlIdColumns(field(name("person"), String.class))) + .validityDate(Optional.of(dateRange)) + .build()) + .build(); + } + + private static DateAggregationDates dateInputs() { + return DateAggregationDates.forValidityDates(List.of( + Optional.of(dateRange("first_start", "first_end")), + Optional.of(dateRange("second_start", "second_end")) + )); + } + + private static ColumnDateRange dateRange(String start, String end) { + return ColumnDateRange.of(field(name(start), Date.class), field(name(end), Date.class)); + } + + private static final class TestDialect implements CompilerDialect { + + @Override + public Field minimumDate() { + return field(name("minimum_date"), Date.class); + } + + @Override + public Field maximumDate() { + return field(name("maximum_date"), Date.class); + } + + @Override + public Field anyValue(Field value) { + return value; + } + + @Override + public Field renderDateRange(Field start, Field end) { + return field(name("rendered_range"), Object.class); + } + + @Override + public Field aggregateDateRanges(Field start, Field end) { + return field(name("aggregated_ranges"), Object.class); + } + + @Override + public int getNameMaxLength() { + return 128; + } + } +} From 1866bc4f42b6a8ab611e12d33d0b848d6803dbba Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:32:09 +0200 Subject: [PATCH 31/81] refactor(sql): remove legacy date aggregation adapter Invoke DateAggregationCompiler directly from the backend query-step composer. Remove the obsolete SqlDateAggregator contract, its adapter, and the associated HANA and ClickHouse dialect plumbing. Co-authored-by: Codex GPT-5.6-sol --- .../aggregation/AnsiSqlDateAggregator.java | 55 ------------------- .../dialect/LegacyCompilerDialect.java | 2 - .../conversion/dialect/SqlDateAggregator.java | 34 ------------ .../clickhouse/ClickhouseDialectBundle.java | 9 --- .../dialect/hana/HanaDialectBundle.java | 9 --- .../conversion/model/QueryStepComposer.java | 22 ++++---- 6 files changed, 12 insertions(+), 119 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java deleted file mode 100644 index 3a73348de6..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.aggregation; - -import java.util.List; - -import com.bakdata.conquery.models.query.DateAggregationAction; -import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.aggregation.DateAggregationCompiler; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; - -/** - * Adapts the legacy backend date-aggregation contract to the connector compiler. - * - *

TODO Remove this adapter when backend converters invoke {@link DateAggregationCompiler} directly.

- */ -public class AnsiSqlDateAggregator implements SqlDateAggregator { - - @Override - public QueryStep apply( - QueryStep joinedStep, - List carryThroughSelects, - DateAggregationDates dateAggregationDates, - DateAggregationAction dateAggregationAction, - ConversionContext conversionContext - ) { - return DateAggregationCompiler.aggregate( - joinedStep, - carryThroughSelects, - dateAggregationDates, - dateAggregationAction, - conversionContext.getCompilerDialect(), - conversionContext.getNameGenerator() - ); - } - - @Override - public ColumnDateRange getAggregatedValidityDate( - DateAggregationDates dateAggregationDates, - DateAggregationAction dateAggregationAction - ) { - return DateAggregationCompiler.getAggregatedValidityDate(dateAggregationDates); - } - - @Override - public QueryStep invertAggregatedIntervals(QueryStep baseStep, ConversionContext conversionContext) { - return DateAggregationCompiler.invert( - baseStep, - conversionContext.getCompilerDialect(), - conversionContext.getNameGenerator() - ); - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java index 55b49b9ded..99c7ca45a2 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java @@ -69,8 +69,6 @@ default Field aggregateDateRanges(Field start, Field end) { SqlFunctionProvider getFunctionProvider(); - SqlDateAggregator getDateAggregator(); - List> getNodeConverters(DSLContext context); default List> getDefaultNodeConverters(DSLContext dslContext) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java deleted file mode 100644 index 50259f2894..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.bakdata.conquery.sql.conversion.dialect; - -import java.util.List; - -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import com.bakdata.conquery.models.query.DateAggregationAction; - -public interface SqlDateAggregator { - - /** - * MERGE or INTERSECT the validity dates of the given {@link QueryStep}s based on the given {@link DateAggregationAction}. - * - * @param carryThroughSelects The selects that should be carried through the date aggregation process. They remain unchanged. - */ - QueryStep apply( - QueryStep joinedStep, - List carryThroughSelects, - DateAggregationDates dateAggregationDates, - DateAggregationAction dateAggregationAction, - ConversionContext conversionContext - ); - - ColumnDateRange getAggregatedValidityDate(DateAggregationDates dateAggregationDates, DateAggregationAction dateAggregationAction); - - /** - * Inverts the validity date of the given base step. - */ - QueryStep invertAggregatedIntervals(QueryStep baseStep, ConversionContext conversionContext); - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java index 167aefc80e..e6a1db452b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java @@ -10,9 +10,7 @@ import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.cqelement.aggregation.AnsiSqlDateAggregator; import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; -import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; @@ -27,11 +25,9 @@ public class ClickhouseDialectBundle implements DialectBundle { private final SqlFunctionProvider functionProvider; - private final SqlDateAggregator dateAggregator; public ClickhouseDialectBundle() { this.functionProvider = new ClickhouseFunctionProvider(); - this.dateAggregator = new AnsiSqlDateAggregator(); } @Override @@ -74,11 +70,6 @@ public SqlFunctionProvider getFunctionProvider() { return this.functionProvider; } - @Override - public SqlDateAggregator getDateAggregator() { - return this.dateAggregator; - } - @Override public ResultSetProcessor getResultSetProcessor(ConqueryConfig config) { return new ClickhouseResultSetProcessor(config); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaDialectBundle.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaDialectBundle.java index e9049d46fd..018012a139 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaDialectBundle.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaDialectBundle.java @@ -7,9 +7,7 @@ import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.cqelement.aggregation.AnsiSqlDateAggregator; import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; -import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.execution.DefaultResultSetProcessor; @@ -23,12 +21,10 @@ public class HanaDialectBundle implements DialectBundle { private final SqlFunctionProvider functionProvider; - private final SqlDateAggregator dateAggregator; private final SqlCDateSetParser dateSetParser; public HanaDialectBundle() { this.functionProvider = new HanaSqlFunctionProvider(); - this.dateAggregator = new AnsiSqlDateAggregator(); this.dateSetParser = new HanaSqlCDateSetParser(); } @@ -86,9 +82,4 @@ public SqlFunctionProvider getFunctionProvider() { return this.functionProvider; } - @Override - public SqlDateAggregator getDateAggregator() { - return this.dateAggregator; - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java index 26e96d4253..ea0d04fc6f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.models.config.IdColumnConfig; +import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; import com.bakdata.conquery.sql.compiler.ir.JoinMode; import com.bakdata.conquery.sql.compiler.ir.QueryStep; @@ -18,11 +19,10 @@ import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.aggregation.DateAggregationCompiler; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import org.jooq.Condition; import org.jooq.Field; @@ -201,8 +201,11 @@ private static QueryStep joinStepsContainingNegation( } // first, invert dates of negated step - SqlDateAggregator dateAggregator = context.getCompilerDialect().getDateAggregator(); - negateJoined = dateAggregator.invertAggregatedIntervals(negateJoined, context); + negateJoined = DateAggregationCompiler.invert( + negateJoined, + context.getCompilerDialect(), + context.getNameGenerator() + ); // join with all-ids table necessary ColumnConfig columnConfig = context.getIdColumns().findPrimaryIdColumn(); @@ -217,9 +220,8 @@ private static QueryStep joinStepsContainingNegation( negateJoined.getQualifiedSelects().getValidityDate(), Optional.of(context.getCompilerDialect().getFunctionProvider().allRangeIf(infinityRangeCondition)) )); - ColumnDateRange merged = - dateAggregator.getAggregatedValidityDate(aggregationDates, DateAggregationAction.MERGE) - .as(cteName + SharedAliases.DATES_COLUMN.getAlias()); + ColumnDateRange merged = DateAggregationCompiler.getAggregatedValidityDate(aggregationDates) + .as(cteName + SharedAliases.DATES_COLUMN.getAlias()); Field coalescedId = DSL.coalesce(nonNegatePrimaryColumn, allIdsPrimaryColumn) .as(SharedAliases.PRIMARY_COLUMN.getAlias()); @@ -272,13 +274,13 @@ private static QueryStep buildStepAndAggregateDates( withAllValidityDates.addAll(dateAggregationDates.allStartsAndEnds()); QueryStep joinedStep = buildJoinedStep(ids, withAllValidityDates, Optional.empty(), Optional.empty(), builder); - SqlDateAggregator sqlDateAggregator = context.getCompilerDialect().getDateAggregator(); - return sqlDateAggregator.apply( + return DateAggregationCompiler.aggregate( joinedStep, mergedSelects, dateAggregationDates, dateAggregationAction, - context + context.getCompilerDialect(), + context.getNameGenerator() ); } From 6028df38dc4db7148d0703dfa8cdaf18dcfc5e83 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 10 Sep 2026 10:27:21 +0200 Subject: [PATCH 32/81] refactor(sql): move date range expressions into dialect Define empty, unbounded, and conditional unbounded date ranges on the connector-owned CompilerDialect. Use those primitives from query composition and remove the duplicate legacy function-provider implementations. Add focused tests for the shared range expressions. Co-authored-by: Codex GPT-5.6-sol --- .../cqelement/concept/CQConceptConverter.java | 2 +- .../dialect/SqlFunctionProvider.java | 7 -- .../ClickhouseFunctionProvider.java | 17 ---- .../dialect/hana/HanaSqlFunctionProvider.java | 17 ---- .../conversion/model/QueryStepComposer.java | 9 +- .../sql/compiler/dialect/CompilerDialect.java | 23 +++++ .../compiler/dialect/CompilerDialectTest.java | 87 +++++++++++++++++++ 7 files changed, 114 insertions(+), 48 deletions(-) create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialectTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index d39d5794fe..34a3ecc932 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -164,7 +164,7 @@ private static ColumnDateRange convertValidityDate(ConversionContext context, Va if (hasDateRestriction) { sqlValidityDate = functionProvider.forCDateRange(context.getDateRestrictionRange()); } else { - sqlValidityDate = functionProvider.allRange(); + sqlValidityDate = context.getCompilerDialect().unboundedDateRange(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index e977bedd1a..42b6ba765a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -132,11 +132,6 @@ default Condition isNotEmptyValidityDate(ValidityDate validityDate) { return isNotEmptyStart.or(isNotEmptyEnd); } - /** - * Creates a {@link ColumnDateRange} of maximum range. - */ - ColumnDateRange allRange(); - Field anyValue(Field field); /** @@ -225,8 +220,6 @@ default Field stringAggregation(Field stringField, Field ); } - ColumnDateRange allRangeIf(Condition condition); - /** * Render an array for Conquery processing. */ diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java index b2d746df2e..818fd629cb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java @@ -88,11 +88,6 @@ public ColumnDateRange forValidityDate(ValidityDate validityDate) { } - @Override - public ColumnDateRange allRange() { - return ColumnDateRange.of(getMinDateExpression(), getMaxDateExpression()); - } - @Override public Field anyValue(Field field) { return DSL.anyValue(field); @@ -317,16 +312,4 @@ public Field yearQuarter(Field dateField) { return field("formatDateTime({0}, '%Y-Q%Q')", String.class, dateField); } - @Override - public ColumnDateRange allRangeIf(Condition condition) { - return ColumnDateRange.of( - when(condition.isTrue(), - getMinDateExpression() - ), - when(condition.isTrue(), - getMaxDateExpression() - ) - ); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java index e74666de95..c38b90a576 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java @@ -101,11 +101,6 @@ public ColumnDateRange forValidityDate(ValidityDate validityDate) { return toColumnDateRange(validityDate); } - @Override - public ColumnDateRange allRange() { - return ColumnDateRange.of(getMinDateExpression(), getMaxDateExpression()); - } - @Override public Field anyValue(Field field) { // Hana does not have any_value @@ -158,18 +153,6 @@ public Field addDays(Field dateColumn, Field amountOfDays) ); } - @Override - public ColumnDateRange allRangeIf(Condition condition) { - return ColumnDateRange.of( - when(condition.isTrue(), - getMinDateExpression() - ), - when(condition.isTrue(), - getMaxDateExpression() - ) - ); - } - @Override public ColumnDateRange forValidityDate(ValidityDate validityDate, CDateRange dateRestriction) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java index ea0d04fc6f..9e6a113c11 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java @@ -23,7 +23,6 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import org.jooq.Condition; import org.jooq.Field; import org.jooq.Record; @@ -49,8 +48,6 @@ private QueryStepComposer() { */ public static QueryStep antiJoinWithAllIdsTable(QueryStep queryStep, ConversionContext context, DateAggregationAction dateAggregationAction) { - SqlFunctionProvider functionProvider = context.getConversionContext().getCompilerDialect().getFunctionProvider(); - Field queryStepPrimaryColumn = queryStep.getQualifiedSelects().getIds().getPrimaryColumn(); ColumnConfig idColumnConfig = context.getIdColumns().findPrimaryIdColumn(); @@ -65,8 +62,8 @@ public static QueryStep antiJoinWithAllIdsTable(QueryStep queryStep, ConversionC String cteName = queryStep.getCteName() + NEGATED_CTE_SUFFIX; Optional validityDate = switch (dateAggregationAction) { - case BLOCK, MERGE, INTERSECT -> Optional.of(functionProvider.emptyColumnDateRange()); - case NEGATE -> Optional.of(functionProvider.allRange()); + case BLOCK, MERGE, INTERSECT -> Optional.of(context.getCompilerDialect().emptyDateRange()); + case NEGATE -> Optional.of(context.getCompilerDialect().unboundedDateRange()); }; Selects selects = Selects.builder() @@ -218,7 +215,7 @@ private static QueryStep joinStepsContainingNegation( DateAggregationDates aggregationDates = DateAggregationDates.forValidityDates(List.of( nonNegateJoined.getQualifiedSelects().getValidityDate(), negateJoined.getQualifiedSelects().getValidityDate(), - Optional.of(context.getCompilerDialect().getFunctionProvider().allRangeIf(infinityRangeCondition)) + Optional.of(context.getCompilerDialect().conditionalUnboundedDateRange(infinityRangeCondition)) )); ColumnDateRange merged = DateAggregationCompiler.getAggregatedValidityDate(aggregationDates) .as(cteName + SharedAliases.DATES_COLUMN.getAlias()); diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java index 750da3df74..064e7739a2 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java @@ -2,7 +2,10 @@ import java.sql.Date; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import org.jooq.Condition; import org.jooq.Field; +import org.jooq.impl.DSL; /** * Database-specific capabilities exposed to the framework-neutral SQL compiler. @@ -22,6 +25,26 @@ public interface CompilerDialect { /** Expression used as the database-specific upper infinity sentinel for dates. */ Field maximumDate(); + /** Date range representing no date value. */ + default ColumnDateRange emptyDateRange() { + return ColumnDateRange.of(DSL.inline(null, Date.class), DSL.inline(null, Date.class)); + } + + /** Logically unbounded date range using the database-specific date sentinels. */ + default ColumnDateRange unboundedDateRange() { + return ColumnDateRange.of(minimumDate(), maximumDate()); + } + + /** + * Logically unbounded date range when the supplied condition holds, and an empty range otherwise. + */ + default ColumnDateRange conditionalUnboundedDateRange(Condition condition) { + return ColumnDateRange.of( + DSL.when(condition.isTrue(), minimumDate()), + DSL.when(condition.isTrue(), maximumDate()) + ); + } + /** * Aggregate a field to an arbitrary value from its group. * diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialectTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialectTest.java new file mode 100644 index 0000000000..4ac416a2c3 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialectTest.java @@ -0,0 +1,87 @@ +package com.bakdata.conquery.sql.compiler.dialect; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.Date; +import java.util.Locale; + +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import org.jooq.Field; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Test; + +class CompilerDialectTest { + + private static final CompilerDialect DIALECT = new TestDialect(); + + @Test + void shouldCreateEmptyDateRange() { + ColumnDateRange dateRange = DIALECT.emptyDateRange(); + + assertEquals("null", render(dateRange.getStart())); + assertEquals("null", render(dateRange.getEnd())); + } + + @Test + void shouldCreateUnboundedDateRangeFromDialectSentinels() { + ColumnDateRange dateRange = DIALECT.unboundedDateRange(); + + assertEquals("\"minimum_date\"", render(dateRange.getStart())); + assertEquals("\"maximum_date\"", render(dateRange.getEnd())); + } + + @Test + void shouldCreateConditionalUnboundedDateRange() { + ColumnDateRange dateRange = DIALECT.conditionalUnboundedDateRange( + field(name("include_all"), Boolean.class).isTrue() + ); + + String start = render(dateRange.getStart()); + String end = render(dateRange.getEnd()); + assertTrue(start.contains("case when")); + assertTrue(start.contains("\"minimum_date\"")); + assertTrue(end.contains("case when")); + assertTrue(end.contains("\"maximum_date\"")); + } + + private static String render(Field expression) { + return DSL.using(SQLDialect.POSTGRES).renderInlined(expression).toLowerCase(Locale.ROOT); + } + + private static final class TestDialect implements CompilerDialect { + + @Override + public Field minimumDate() { + return field(name("minimum_date"), Date.class); + } + + @Override + public Field maximumDate() { + return field(name("maximum_date"), Date.class); + } + + @Override + public Field anyValue(Field value) { + return value; + } + + @Override + public Field renderDateRange(Field start, Field end) { + return field(name("rendered_range"), Object.class); + } + + @Override + public Field aggregateDateRanges(Field start, Field end) { + return field(name("aggregated_ranges"), Object.class); + } + + @Override + public int getNameMaxLength() { + return 128; + } + } +} From 616de38dabf9ba0bdd1c83c4c254b3c700056faa Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:06:31 +0200 Subject: [PATCH 33/81] refactor(sql): move query step composition into connector Compose joins, negation, and date aggregation from connector-owned IR and the resolved entity schema. Keep the backend wrapper limited to converting legacy query children and ID configuration before delegating. Add connector-level tests for positive joins and both negation strategies. Co-authored-by: Codex GPT-5.6-sol --- .../conversion/model/QueryStepComposer.java | 282 +++-------------- .../sql/compiler/ir/QueryStepComposer.java | 291 ++++++++++++++++++ .../compiler/ir/QueryStepComposerTest.java | 175 +++++++++++ 3 files changed, 501 insertions(+), 247 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposer.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposerTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java index 9e6a113c11..2e3af2b10a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java @@ -1,83 +1,41 @@ package com.bakdata.conquery.sql.conversion.model; -import static org.jooq.impl.DSL.*; - -import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.models.config.IdColumnConfig; +import com.bakdata.conquery.models.datasets.ColumnType; import com.bakdata.conquery.models.query.DateAggregationAction; -import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; import com.bakdata.conquery.sql.compiler.ir.JoinMode; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.QueryStepJoiner; -import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.bakdata.conquery.sql.compiler.ir.SharedAliases; -import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import com.bakdata.conquery.sql.compiler.ir.aggregation.DateAggregationCompiler; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import org.jooq.Condition; -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Table; -import org.jooq.TableLike; -import org.jooq.impl.DSL; +import com.bakdata.conquery.sql.model.schema.EntitySchema; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.SqlTable; /** - * Composes converted backend query nodes, including negation and date aggregation, into compiler query steps. + * Adapts legacy backend query nodes and ID configuration to connector-owned query-step composition. * - *

TODO Move this composition into {@code sql-connector} after its remaining backend query, schema, and legacy - * dialect dependencies have framework-neutral compiler contracts.

+ *

TODO Remove this adapter once the backend conversion entry point consumes resolved query nodes and an + * {@link EntitySchema} directly.

*/ -public class QueryStepComposer { - - private static final String NEGATED_CTE_SUFFIX = "_negated"; +public final class QueryStepComposer { private QueryStepComposer() { } - /** - * Implements an antijoin against the allIdsTable defined in {@link IdColumnConfig#getTable()}, for negation without siblings. - */ - public static QueryStep antiJoinWithAllIdsTable(QueryStep queryStep, ConversionContext context, DateAggregationAction dateAggregationAction) { - - Field queryStepPrimaryColumn = queryStep.getQualifiedSelects().getIds().getPrimaryColumn(); - ColumnConfig idColumnConfig = context.getIdColumns().findPrimaryIdColumn(); - - String allIdsTable = context.getIdColumns().getTable(); - - Field allIdsPrimaryColumn = field(name(allIdsTable, idColumnConfig.getField()), String.class); - - Table table = table(name(allIdsTable)) - .leftOuterJoin(table(name(queryStep.getCteName()))) - .on(allIdsPrimaryColumn.eq(queryStepPrimaryColumn)); - - String cteName = queryStep.getCteName() + NEGATED_CTE_SUFFIX; - - Optional validityDate = switch (dateAggregationAction) { - case BLOCK, MERGE, INTERSECT -> Optional.of(context.getCompilerDialect().emptyDateRange()); - case NEGATE -> Optional.of(context.getCompilerDialect().unboundedDateRange()); - }; - - Selects selects = Selects.builder() - .ids(new SqlIdColumns(allIdsPrimaryColumn)) - .validityDate(validityDate.map(cdr -> cdr.asValidityDateRange(cteName))) - .build(); - - return QueryStep.builder() - .cteName(cteName) - .selects(selects) - .fromTable(table) - .conditions(List.of(queryStepPrimaryColumn.isNull())) - .predecessor(queryStep) - .build(); + public static QueryStep antiJoinWithAllIdsTable( + QueryStep queryStep, + ConversionContext context, + DateAggregationAction dateAggregationAction + ) { + return com.bakdata.conquery.sql.compiler.ir.QueryStepComposer.antiJoinWithAllEntities( + queryStep, + toEntitySchema(context.getIdColumns()), + dateAggregationAction, + context.getCompilerDialect() + ); } public static QueryStep joinChildren( @@ -87,13 +45,10 @@ public static QueryStep joinChildren( DateAggregationAction dateAggregationAction ) { ConversionContext childrenContext = context.createChildContext(); - for (CQElement childNode : children) { childrenContext = context.getNodeConversions().convert(childNode, childrenContext); } - - List queriesToJoin = childrenContext.getQuerySteps(); - return joinSteps(queriesToJoin, logicalOperation, dateAggregationAction, context); + return joinSteps(childrenContext.getQuerySteps(), logicalOperation, dateAggregationAction, context); } public static QueryStep joinSteps( @@ -102,193 +57,26 @@ public static QueryStep joinSteps( DateAggregationAction dateAggregationAction, ConversionContext context ) { - //TODO somehow Negation does not show up here. - - // keep all entries from group A (non-negate steps), whose entity has no matching entry in group B (negate steps) - if (queriesToJoin.stream().anyMatch(QueryStep::isNegate)) { - return joinStepsContainingNegation(queriesToJoin, logicalOperation, dateAggregationAction, context); - } - - // splitting out the actual join to be able to recur here - return doJoin(queriesToJoin, logicalOperation, dateAggregationAction, context); - } - - private static QueryStep doJoin( - List queriesToJoin, - JoinMode logicalOperation, - DateAggregationAction dateAggregationAction, - ConversionContext context) { - // no join required - if (queriesToJoin.size() == 1) { - return queriesToJoin.getFirst(); - } - - String joinedNodeName = context.getNameGenerator().joinedNodeName(logicalOperation); - SqlIdColumns ids = QueryStepJoiner.coalesceIds(queriesToJoin); - List mergedSelects = QueryStepJoiner.mergeSelects(queriesToJoin); - TableLike joinedTable = QueryStepJoiner.join(queriesToJoin, logicalOperation); - - QueryStep joinedStep; - QueryStep.QueryStepBuilder joinedStepBuilder = QueryStep.builder() - .cteName(joinedNodeName) - .fromTable(joinedTable) - .predecessors(queriesToJoin); - - DateAggregationDates dateAggregationDates = DateAggregationDates.forSteps(queriesToJoin); - - if (dateAggregationAction == DateAggregationAction.BLOCK || dateAggregationDates.dateAggregationImpossible()) { - // for forms, date aggregation is allways blocked, but dates need to be coalesced in case we do a fulll outer join - Optional stratificationDate = coalesceStratificationDates(queriesToJoin); - joinedStep = buildJoinedStep(ids, mergedSelects, Optional.empty(), stratificationDate, joinedStepBuilder); - } - // if there is only 1 child node containing a validity date, we just keep it as overall validity date for the joined node - else if (dateAggregationDates.getValidityDates().size() == 1) { - ColumnDateRange validityDate = dateAggregationDates.getValidityDates().getFirst(); - joinedStep = buildJoinedStep(ids, mergedSelects, Optional.of(validityDate), Optional.empty(), joinedStepBuilder); - } - else { - joinedStep = buildStepAndAggregateDates(ids, mergedSelects, joinedStepBuilder, dateAggregationDates, dateAggregationAction, context); - } - return joinedStep; - } - - /** - * If queriesToJoin contains any negated queries, they will be used in an antijoin against their siblings. - * - * That means the negation works by removing the entities in the negated portion from the unnegated ones. - */ - private static QueryStep joinStepsContainingNegation( - List queriesToJoin, - JoinMode logicalOperation, - DateAggregationAction dateAggregationAction, - ConversionContext context - ) { - Map> byNegation = queriesToJoin.stream().collect(Collectors.groupingBy(QueryStep::isNegate)); - - List withoutNegation = byNegation.get(false); - List withNegation = byNegation.get(true); - - if (withoutNegation == null) { - QueryStep negateJoined = doJoin(withNegation, logicalOperation, dateAggregationAction, context); - return antiJoinWithAllIdsTable(negateJoined, context, dateAggregationAction); - } - - String cteName = context.getNameGenerator().joinedNodeName(logicalOperation) + NEGATED_CTE_SUFFIX; - QueryStep nonNegateJoined = doJoin(withoutNegation, logicalOperation, dateAggregationAction, context); - QueryStep negateJoined = doJoin(withNegation, logicalOperation, dateAggregationAction, context); - - Selects.SelectsBuilder selects = nonNegateJoined.getQualifiedSelects() - .toBuilder() - .sqlSelects(QueryStepJoiner.mergeSelects(List.of(nonNegateJoined, negateJoined))); - - - if (logicalOperation == JoinMode.INNER) { - Table table = table(name(nonNegateJoined.getCteName())) - .leftOuterJoin(table(name(negateJoined.getCteName()))) - .on(nonNegateJoined.getQualifiedSelects().getIds().getPrimaryColumn() - .eq(negateJoined.getQualifiedSelects().getIds().getPrimaryColumn())); - - return QueryStep.builder() - .cteName(cteName) - .selects(selects.build()) - .fromTable(table) - .conditions(List.of(negateJoined.getQualifiedSelects().getIds().getPrimaryColumn().isNull())) - .predecessors(List.of(nonNegateJoined, negateJoined)) - .build(); - } - - // first, invert dates of negated step - negateJoined = DateAggregationCompiler.invert( - negateJoined, - context.getCompilerDialect(), - context.getNameGenerator() - ); - - // join with all-ids table necessary - ColumnConfig columnConfig = context.getIdColumns().findPrimaryIdColumn(); - Field allIdsPrimaryColumn = field(name(columnConfig.getField())); - Field negatePrimaryColumn = negateJoined.getQualifiedSelects().getIds().getPrimaryColumn(); - Field nonNegatePrimaryColumn = nonNegateJoined.getQualifiedSelects().getIds().getPrimaryColumn(); - - // prepare date aggregation - Condition infinityRangeCondition = negatePrimaryColumn.isNull().or(nonNegatePrimaryColumn.isNull()); - DateAggregationDates aggregationDates = DateAggregationDates.forValidityDates(List.of( - nonNegateJoined.getQualifiedSelects().getValidityDate(), - negateJoined.getQualifiedSelects().getValidityDate(), - Optional.of(context.getCompilerDialect().conditionalUnboundedDateRange(infinityRangeCondition)) - )); - ColumnDateRange merged = DateAggregationCompiler.getAggregatedValidityDate(aggregationDates) - .as(cteName + SharedAliases.DATES_COLUMN.getAlias()); - - Field coalescedId = DSL.coalesce(nonNegatePrimaryColumn, allIdsPrimaryColumn) - .as(SharedAliases.PRIMARY_COLUMN.getAlias()); - - selects = selects - .ids(new SqlIdColumns(coalescedId)) - .validityDate(Optional.of(merged)); - - Table table = table(name(context.getIdColumns().getTable())) - .leftOuterJoin(table(name(negateJoined.getCteName()))) - .on(allIdsPrimaryColumn.eq(negatePrimaryColumn)) - .leftOuterJoin(table(name(nonNegateJoined.getCteName()))) - .on(allIdsPrimaryColumn.eq(nonNegatePrimaryColumn)); - - return QueryStep.builder() - .cteName(cteName) - .selects(selects.build()) - .fromTable(table) - .conditions(List.of(negatePrimaryColumn.isNull().or(nonNegatePrimaryColumn.isNotNull()))) - .predecessors(List.of(nonNegateJoined, negateJoined)) - .build(); - - } - - private static QueryStep buildJoinedStep( - SqlIdColumns ids, - List mergedSelects, - Optional validityDate, - Optional stratificationDate, - QueryStep.QueryStepBuilder builder - ) { - Selects selects = Selects.builder() - .ids(ids) - .stratificationDate(stratificationDate) - .validityDate(validityDate) - .sqlSelects(mergedSelects) - .build(); - return builder.selects(selects).build(); - } - - private static QueryStep buildStepAndAggregateDates( - SqlIdColumns ids, - List mergedSelects, - QueryStep.QueryStepBuilder builder, - DateAggregationDates dateAggregationDates, - DateAggregationAction dateAggregationAction, - ConversionContext context - ) { - List withAllValidityDates = new ArrayList<>(mergedSelects); - withAllValidityDates.addAll(dateAggregationDates.allStartsAndEnds()); - QueryStep joinedStep = buildJoinedStep(ids, withAllValidityDates, Optional.empty(), Optional.empty(), builder); - - return DateAggregationCompiler.aggregate( - joinedStep, - mergedSelects, - dateAggregationDates, + return com.bakdata.conquery.sql.compiler.ir.QueryStepComposer.joinSteps( + queriesToJoin, + logicalOperation, dateAggregationAction, + toEntitySchema(context.getIdColumns()), context.getCompilerDialect(), context.getNameGenerator() ); } - private static Optional coalesceStratificationDates(List queriesToJoin) { - return queriesToJoin.stream() - .map(QueryStep::getQualifiedSelects) - .map(Selects::getStratificationDate) - .filter(Optional::isPresent) - .map(Optional::get) - .reduce(ColumnDateRange::coalesce); + private static EntitySchema toEntitySchema(IdColumnConfig idColumns) { + ColumnConfig primaryId = idColumns.findPrimaryIdColumn(); + SqlTable entityTable = SqlTable.of(idColumns.getTable(), idColumns.getTable()); + ResolvedColumn primaryIdColumn = new ResolvedColumn( + primaryId.getName(), + entityTable, + primaryId.getField(), + ColumnType.STRING, + false + ); + return new EntitySchema(entityTable, primaryIdColumn); } - - } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposer.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposer.java new file mode 100644 index 0000000000..8f7d4ae09f --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposer.java @@ -0,0 +1,291 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.jooq.impl.DSL.table; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import com.bakdata.conquery.models.query.DateAggregationAction; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.aggregation.DateAggregationCompiler; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import com.bakdata.conquery.sql.model.schema.EntitySchema; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.Name; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.TableLike; +import org.jooq.impl.DSL; + +/** Composes query-step IR, including negation and date aggregation, from resolved compiler inputs. */ +public final class QueryStepComposer { + + private static final String NEGATED_CTE_SUFFIX = "_negated"; + + private QueryStepComposer() { + } + + /** Implements an anti-join against the entity schema used to enumerate all known entities. */ + public static QueryStep antiJoinWithAllEntities( + QueryStep queryStep, + EntitySchema entitySchema, + DateAggregationAction dateAggregationAction, + CompilerDialect compilerDialect + ) { + Field queryStepPrimaryColumn = queryStep.getQualifiedSelects().getIds().getPrimaryColumn(); + Field allIdsPrimaryColumn = primaryIdField(entitySchema); + + Table joinedTable = table(tableName(entitySchema)) + .leftOuterJoin(table(name(queryStep.getCteName()))) + .on(allIdsPrimaryColumn.eq(queryStepPrimaryColumn)); + + String cteName = queryStep.getCteName() + NEGATED_CTE_SUFFIX; + Optional validityDate = switch (dateAggregationAction) { + case BLOCK, MERGE, INTERSECT -> Optional.of(compilerDialect.emptyDateRange()); + case NEGATE -> Optional.of(compilerDialect.unboundedDateRange()); + }; + + Selects selects = Selects.builder() + .ids(new SqlIdColumns(allIdsPrimaryColumn)) + .validityDate(validityDate.map(dateRange -> dateRange.asValidityDateRange(cteName))) + .build(); + + return QueryStep.builder() + .cteName(cteName) + .selects(selects) + .fromTable(joinedTable) + .conditions(List.of(queryStepPrimaryColumn.isNull())) + .predecessor(queryStep) + .build(); + } + + public static QueryStep joinSteps( + List queriesToJoin, + JoinMode logicalOperation, + DateAggregationAction dateAggregationAction, + EntitySchema entitySchema, + CompilerDialect compilerDialect, + SqlNameGenerator nameGenerator + ) { + if (queriesToJoin.isEmpty()) { + throw new IllegalArgumentException("Need at least one query step to compose"); + } + if (queriesToJoin.stream().anyMatch(QueryStep::isNegate)) { + return joinStepsContainingNegation( + queriesToJoin, + logicalOperation, + dateAggregationAction, + entitySchema, + compilerDialect, + nameGenerator + ); + } + + return doJoin(queriesToJoin, logicalOperation, dateAggregationAction, compilerDialect, nameGenerator); + } + + private static QueryStep doJoin( + List queriesToJoin, + JoinMode logicalOperation, + DateAggregationAction dateAggregationAction, + CompilerDialect compilerDialect, + SqlNameGenerator nameGenerator + ) { + if (queriesToJoin.size() == 1) { + return queriesToJoin.getFirst(); + } + + String joinedNodeName = nameGenerator.joinedNodeName(logicalOperation); + SqlIdColumns ids = QueryStepJoiner.coalesceIds(queriesToJoin); + List mergedSelects = QueryStepJoiner.mergeSelects(queriesToJoin); + TableLike joinedTable = QueryStepJoiner.join(queriesToJoin, logicalOperation); + + QueryStep.QueryStepBuilder joinedStepBuilder = QueryStep.builder() + .cteName(joinedNodeName) + .fromTable(joinedTable) + .predecessors(queriesToJoin); + + DateAggregationDates dateAggregationDates = DateAggregationDates.forSteps(queriesToJoin); + if (dateAggregationAction == DateAggregationAction.BLOCK || dateAggregationDates.dateAggregationImpossible()) { + Optional stratificationDate = coalesceStratificationDates(queriesToJoin); + return buildJoinedStep(ids, mergedSelects, Optional.empty(), stratificationDate, joinedStepBuilder); + } + if (dateAggregationDates.getValidityDates().size() == 1) { + ColumnDateRange validityDate = dateAggregationDates.getValidityDates().getFirst(); + return buildJoinedStep(ids, mergedSelects, Optional.of(validityDate), Optional.empty(), joinedStepBuilder); + } + return buildStepAndAggregateDates( + ids, + mergedSelects, + joinedStepBuilder, + dateAggregationDates, + dateAggregationAction, + compilerDialect, + nameGenerator + ); + } + + /** + * Uses negated steps in an anti-join against their positive siblings, or against all entities when no positive + * sibling exists. + */ + private static QueryStep joinStepsContainingNegation( + List queriesToJoin, + JoinMode logicalOperation, + DateAggregationAction dateAggregationAction, + EntitySchema entitySchema, + CompilerDialect compilerDialect, + SqlNameGenerator nameGenerator + ) { + Map> byNegation = queriesToJoin.stream() + .collect(Collectors.groupingBy(QueryStep::isNegate)); + + List withoutNegation = byNegation.get(false); + List withNegation = byNegation.get(true); + if (withoutNegation == null) { + QueryStep negateJoined = doJoin( + withNegation, + logicalOperation, + dateAggregationAction, + compilerDialect, + nameGenerator + ); + return antiJoinWithAllEntities(negateJoined, entitySchema, dateAggregationAction, compilerDialect); + } + + String cteName = nameGenerator.joinedNodeName(logicalOperation) + NEGATED_CTE_SUFFIX; + QueryStep nonNegateJoined = doJoin( + withoutNegation, + logicalOperation, + dateAggregationAction, + compilerDialect, + nameGenerator + ); + QueryStep negateJoined = doJoin( + withNegation, + logicalOperation, + dateAggregationAction, + compilerDialect, + nameGenerator + ); + + Selects.SelectsBuilder selects = nonNegateJoined.getQualifiedSelects() + .toBuilder() + .sqlSelects(QueryStepJoiner.mergeSelects(List.of(nonNegateJoined, negateJoined))); + + if (logicalOperation == JoinMode.INNER) { + Table joinedTable = table(name(nonNegateJoined.getCteName())) + .leftOuterJoin(table(name(negateJoined.getCteName()))) + .on(nonNegateJoined.getQualifiedSelects().getIds().getPrimaryColumn() + .eq(negateJoined.getQualifiedSelects().getIds().getPrimaryColumn())); + + return QueryStep.builder() + .cteName(cteName) + .selects(selects.build()) + .fromTable(joinedTable) + .conditions(List.of(negateJoined.getQualifiedSelects().getIds().getPrimaryColumn().isNull())) + .predecessors(List.of(nonNegateJoined, negateJoined)) + .build(); + } + + negateJoined = DateAggregationCompiler.invert(negateJoined, compilerDialect, nameGenerator); + Field allIdsPrimaryColumn = primaryIdField(entitySchema); + Field negatePrimaryColumn = negateJoined.getQualifiedSelects().getIds().getPrimaryColumn(); + Field nonNegatePrimaryColumn = nonNegateJoined.getQualifiedSelects().getIds().getPrimaryColumn(); + + Condition infinityRangeCondition = negatePrimaryColumn.isNull().or(nonNegatePrimaryColumn.isNull()); + DateAggregationDates aggregationDates = DateAggregationDates.forValidityDates(List.of( + nonNegateJoined.getQualifiedSelects().getValidityDate(), + negateJoined.getQualifiedSelects().getValidityDate(), + Optional.of(compilerDialect.conditionalUnboundedDateRange(infinityRangeCondition)) + )); + ColumnDateRange merged = DateAggregationCompiler.getAggregatedValidityDate(aggregationDates) + .as(cteName + SharedAliases.DATES_COLUMN.getAlias()); + + Field coalescedId = DSL.coalesce(nonNegatePrimaryColumn, allIdsPrimaryColumn) + .as(SharedAliases.PRIMARY_COLUMN.getAlias()); + selects = selects + .ids(new SqlIdColumns(coalescedId)) + .validityDate(Optional.of(merged)); + + Table joinedTable = table(tableName(entitySchema)) + .leftOuterJoin(table(name(negateJoined.getCteName()))) + .on(allIdsPrimaryColumn.eq(negatePrimaryColumn)) + .leftOuterJoin(table(name(nonNegateJoined.getCteName()))) + .on(allIdsPrimaryColumn.eq(nonNegatePrimaryColumn)); + + return QueryStep.builder() + .cteName(cteName) + .selects(selects.build()) + .fromTable(joinedTable) + .conditions(List.of(negatePrimaryColumn.isNull().or(nonNegatePrimaryColumn.isNotNull()))) + .predecessors(List.of(nonNegateJoined, negateJoined)) + .build(); + } + + private static QueryStep buildJoinedStep( + SqlIdColumns ids, + List mergedSelects, + Optional validityDate, + Optional stratificationDate, + QueryStep.QueryStepBuilder builder + ) { + Selects selects = Selects.builder() + .ids(ids) + .stratificationDate(stratificationDate) + .validityDate(validityDate) + .sqlSelects(mergedSelects) + .build(); + return builder.selects(selects).build(); + } + + private static QueryStep buildStepAndAggregateDates( + SqlIdColumns ids, + List mergedSelects, + QueryStep.QueryStepBuilder builder, + DateAggregationDates dateAggregationDates, + DateAggregationAction dateAggregationAction, + CompilerDialect compilerDialect, + SqlNameGenerator nameGenerator + ) { + List withAllValidityDates = new ArrayList<>(mergedSelects); + withAllValidityDates.addAll(dateAggregationDates.allStartsAndEnds()); + QueryStep joinedStep = buildJoinedStep(ids, withAllValidityDates, Optional.empty(), Optional.empty(), builder); + + return DateAggregationCompiler.aggregate( + joinedStep, + mergedSelects, + dateAggregationDates, + dateAggregationAction, + compilerDialect, + nameGenerator + ); + } + + private static Optional coalesceStratificationDates(List queriesToJoin) { + return queriesToJoin.stream() + .map(QueryStep::getQualifiedSelects) + .map(Selects::getStratificationDate) + .filter(Optional::isPresent) + .map(Optional::get) + .reduce(ColumnDateRange::coalesce); + } + + private static Name tableName(EntitySchema entitySchema) { + return name(entitySchema.table().physicalName().toArray(String[]::new)); + } + + private static Field primaryIdField(EntitySchema entitySchema) { + List qualifiedName = new ArrayList<>(entitySchema.table().physicalName()); + qualifiedName.add(entitySchema.primaryId().physicalName()); + return field(name(qualifiedName.toArray(String[]::new)), String.class); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposerTest.java new file mode 100644 index 0000000000..2385b44d59 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposerTest.java @@ -0,0 +1,175 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.Date; +import java.util.List; +import java.util.Locale; + +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.models.query.DateAggregationAction; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import com.bakdata.conquery.sql.model.schema.EntitySchema; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.SqlTable; +import org.jooq.Field; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Test; + +class QueryStepComposerTest { + + private static final CompilerDialect DIALECT = new TestDialect(); + private static final SqlTable ENTITY_TABLE = SqlTable.of("entities", "catalog", "entities"); + private static final EntitySchema ENTITY_SCHEMA = new EntitySchema( + ENTITY_TABLE, + new ResolvedColumn("person", ENTITY_TABLE, "person_id", ColumnType.STRING, false) + ); + + @Test + void shouldReturnSinglePositiveStepUnchanged() { + QueryStep step = queryStep("included", false); + + QueryStep result = compose(List.of(step), JoinMode.INNER, DateAggregationAction.BLOCK); + + assertSame(step, result); + } + + @Test + void shouldRejectEmptyInput() { + assertThrows( + IllegalArgumentException.class, + () -> compose(List.of(), JoinMode.INNER, DateAggregationAction.BLOCK) + ); + } + + @Test + void shouldJoinPositiveSteps() { + QueryStep result = compose( + List.of(queryStep("first", false), queryStep("second", false)), + JoinMode.INNER, + DateAggregationAction.BLOCK + ); + + assertEquals("AND-1", result.getCteName()); + assertEquals(2, result.getPredecessors().size()); + assertTrue(render(result).contains("join \"second\"")); + } + + @Test + void shouldAntiJoinOnlyNegatedStepAgainstEntitySchema() { + QueryStep excluded = queryStep("excluded", true); + + QueryStep result = compose(List.of(excluded), JoinMode.INNER, DateAggregationAction.BLOCK); + + assertEquals("excluded_negated", result.getCteName()); + assertEquals(List.of(excluded), result.getPredecessors()); + assertTrue(result.getSelects().getValidityDate().isPresent()); + String sql = render(result); + assertTrue(sql.contains("from \"catalog\".\"entities\" left outer join \"excluded\"")); + assertTrue(sql.contains("\"excluded\".\"person_id\" is null")); + } + + @Test + void shouldAntiJoinNegatedSiblingFromPositiveInnerJoin() { + QueryStep result = compose( + List.of(queryStep("included", false), queryStep("excluded", true)), + JoinMode.INNER, + DateAggregationAction.BLOCK + ); + + assertEquals("AND-1_negated", result.getCteName()); + assertEquals(2, result.getPredecessors().size()); + String sql = render(result); + assertTrue(sql.contains("from \"included\" left outer join \"excluded\"")); + assertTrue(sql.contains("\"excluded\".\"person_id\" is null")); + } + + @Test + void shouldComposePositiveAndNegatedFullOuterJoinWithAllEntities() { + QueryStep result = compose( + List.of(queryStep("included", false), queryStep("excluded", true)), + JoinMode.FULL_OUTER, + DateAggregationAction.MERGE + ); + + assertEquals("OR-1_negated", result.getCteName()); + assertTrue(result.getSelects().getValidityDate().isPresent()); + String sql = render(result); + assertTrue(sql.contains("from \"catalog\".\"entities\" left outer join \"excluded\"")); + assertTrue(sql.contains("left outer join \"included\"")); + } + + private static QueryStep compose( + List steps, + JoinMode joinMode, + DateAggregationAction dateAggregationAction + ) { + return QueryStepComposer.joinSteps( + steps, + joinMode, + dateAggregationAction, + ENTITY_SCHEMA, + DIALECT, + new SqlNameGenerator(128) + ); + } + + private static QueryStep queryStep(String cteName, boolean negate) { + return QueryStep.builder() + .cteName(cteName) + .selects(Selects.builder() + .ids(new SqlIdColumns(field(name("person_id"), String.class))) + .build()) + .negate(negate) + .build(); + } + + private static String render(QueryStep queryStep) { + return DSL.using(SQLDialect.POSTGRES) + .select() + .from(queryStep.getFromTables().getFirst()) + .where(queryStep.getConditions()) + .getSQL() + .toLowerCase(Locale.ROOT); + } + + private static final class TestDialect implements CompilerDialect { + + @Override + public Field minimumDate() { + return field(name("minimum_date"), Date.class); + } + + @Override + public Field maximumDate() { + return field(name("maximum_date"), Date.class); + } + + @Override + public Field anyValue(Field value) { + return value; + } + + @Override + public Field renderDateRange(Field start, Field end) { + return field(name("rendered_range"), Object.class); + } + + @Override + public Field aggregateDateRanges(Field start, Field end) { + return field(name("aggregated_ranges"), Object.class); + } + + @Override + public int getNameMaxLength() { + return 128; + } + } +} From 4dd07bc918cc8f7edcb355c74414033fff5a2da2 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Fri, 11 Sep 2026 07:49:46 +0200 Subject: [PATCH 34/81] refactor(sql): move final query composition into connector Build the final aggregated query step from connector-owned IR while leaving legacy query resolution, result metadata, and rendering orchestration in the backend converter. Avoid duplicating explicit selects when joining additional result columns. Always aggregate explicit selects and only add an empty validity projection when the result includes dates. Co-authored-by: Codex GPT-5.6-sol --- .../query/ConceptQueryConverter.java | 100 ++--------- .../compiler/ir/FinalQueryStepComposer.java | 100 +++++++++++ .../ir/FinalQueryStepComposerTest.java | 165 ++++++++++++++++++ 3 files changed, 277 insertions(+), 88 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/FinalQueryStepComposer.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/FinalQueryStepComposerTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java index 41ad012ba7..0264c499bf 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java @@ -1,60 +1,26 @@ package com.bakdata.conquery.sql.conversion.query; -import com.bakdata.conquery.sql.compiler.ir.JoinMode; -import com.bakdata.conquery.sql.compiler.ir.ProjectionMode; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.QueryStepJoiner; -import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.bakdata.conquery.sql.compiler.rendering.QueryStepRenderer; import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.concept.specific.CQNegation; import com.bakdata.conquery.models.query.DateAggregationMode; -import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.FinalQueryStepComposer; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.rendering.QueryStepRenderer; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.*; -import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import com.bakdata.conquery.sql.conversion.model.QueryStepComposer; +import com.bakdata.conquery.sql.conversion.model.SqlQuery; import lombok.RequiredArgsConstructor; -import org.jooq.*; import org.jooq.Record; +import org.jooq.Select; -import java.util.List; import java.util.Optional; -import java.util.stream.Stream; @RequiredArgsConstructor public class ConceptQueryConverter implements NodeConverter { private final QueryStepRenderer queryStepRenderer; - private static Selects getPreFinalSelects(QueryStep preFinalStep, ConversionContext context) { - Selects preFinalStepSelects = preFinalStep.getQualifiedSelects(); - QueryStep externalExtras = context.getExternalExtras(); - if (externalExtras == null) { - return preFinalStepSelects; - } - // adding extra selects - List concatenated = Stream.concat( - preFinalStepSelects.getSqlSelects().stream(), - externalExtras.getQualifiedSelects().getSqlSelects().stream() - ).toList(); - return preFinalStepSelects.toBuilder().sqlSelects(concatenated).build(); - } - - private static TableLike getFinalTable(QueryStep preFinalStep, ConversionContext context) { - QueryStep externalExtras = context.getExternalExtras(); - if (externalExtras == null) { - return QueryStep.toTableLike(preFinalStep.getCteName()); - } - return QueryStepJoiner.join( - List.of(preFinalStep, externalExtras), - JoinMode.INNER - ); - } - @Override public Class getConversionClass() { return ConceptQuery.class; @@ -75,56 +41,14 @@ public ConversionContext convert(ConceptQuery conceptQuery, ConversionContext co ); } - Selects preFinalSelects = getPreFinalSelects(preFinalStep, contextAfterConversion); - List predecessors = Stream.concat(Stream.of(preFinalStep), Stream.ofNullable(contextAfterConversion.getExternalExtras())).toList(); - - QueryStep finalStep = QueryStep.builder() - .cteName(null) // the final QueryStep won't be converted to a CTE - .projectionMode(ProjectionMode.AGGREGATED) - .selects(getFinalSelects(conceptQuery, preFinalSelects, context.getCompilerDialect())) - .fromTable(getFinalTable(preFinalStep, contextAfterConversion)) - .groupBy(getFinalGroupBySelects(preFinalSelects)) - .predecessors(predecessors) - .build(); + QueryStep finalStep = FinalQueryStepComposer.compose( + preFinalStep, + Optional.ofNullable(contextAfterConversion.getExternalExtras()), + conceptQuery.getDateAggregationMode() != DateAggregationMode.NONE, + context.getCompilerDialect() + ); Select finalQuery = this.queryStepRenderer.toSelectQuery(finalStep, context.getCompilerDialect()); return contextAfterConversion.withFinalQuery(new SqlQuery(finalQuery, conceptQuery.getResultInfos())); } - - private Selects getFinalSelects(ConceptQuery conceptQuery, Selects preFinalSelects, LegacyCompilerDialect dialect) { - SqlFunctionProvider functionProvider = dialect.getFunctionProvider(); - Selects finalSelects = preFinalSelects; - if (conceptQuery.getDateAggregationMode() == DateAggregationMode.NONE) { - finalSelects = preFinalSelects.blockValidityDate(); - } - -// In case all final selects have no validity-date, we convert it to infinity. - if (preFinalSelects.getValidityDate().isEmpty()) { - return preFinalSelects.toBuilder() - .validityDate(Optional.of(functionProvider.emptyColumnDateRange())) - .build(); - } - return Selects.builder() - .ids(finalSelects.getIds()) - .validityDate(finalSelects.getValidityDate()) - .stratificationDate(finalSelects.getStratificationDate()) - .sqlSelects(getFinalAggregatedSelects(finalSelects, dialect)) - .build(); - } - - private List> getFinalAggregatedSelects(Selects finalSelects, CompilerDialect dialect) { - return finalSelects.getSqlSelects().stream() - .flatMap(sqlSelect -> sqlSelect.aggregateForFinalQuery(dialect).stream()) - .map(this::toFieldWrapper) - .toList(); - } - - private FieldWrapper toFieldWrapper(Field field) { - return new FieldWrapper<>(field); - } - - private List> getFinalGroupBySelects(Selects preFinalSelects) { - return preFinalSelects.getIds().toFields(); - } - } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/FinalQueryStepComposer.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/FinalQueryStepComposer.java new file mode 100644 index 0000000000..15e8157d4d --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/FinalQueryStepComposer.java @@ -0,0 +1,100 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.TableLike; + +/** Builds the final aggregated query-step projection from compiled connector IR. */ +public final class FinalQueryStepComposer { + + private FinalQueryStepComposer() { + } + + /** + * Compose a final, non-CTE query step. + * + * @param preFinalStep last step of the compiled query-node graph + * @param additionalStep optional step whose explicit selects are joined to the result by entity ID + * @param includeValidityDate whether the result should retain its aggregated validity date + * @param compilerDialect dialect capabilities used to aggregate explicit selects + */ + public static QueryStep compose( + QueryStep preFinalStep, + Optional additionalStep, + boolean includeValidityDate, + CompilerDialect compilerDialect + ) { + Selects preFinalSelects = combineSelects(preFinalStep, additionalStep); + List predecessors = Stream.concat(Stream.of(preFinalStep), additionalStep.stream()).toList(); + + return QueryStep.builder() + .cteName(null) + .projectionMode(ProjectionMode.AGGREGATED) + .selects(finalSelects(preFinalSelects, includeValidityDate, compilerDialect)) + .fromTable(finalTable(preFinalStep, additionalStep)) + .groupBy(preFinalSelects.getIds().toFields()) + .predecessors(predecessors) + .build(); + } + + private static Selects combineSelects(QueryStep preFinalStep, Optional additionalStep) { + Selects preFinalSelects = preFinalStep.getQualifiedSelects(); + if (additionalStep.isEmpty()) { + return preFinalSelects; + } + List combinedExplicitSelects = Stream.concat( + preFinalSelects.getSqlSelects().stream(), + additionalStep.orElseThrow().getQualifiedSelects().getSqlSelects().stream() + ).toList(); + return preFinalSelects.toBuilder() + .clearSqlSelects() + .sqlSelects(combinedExplicitSelects) + .build(); + } + + private static TableLike finalTable(QueryStep preFinalStep, Optional additionalStep) { + if (additionalStep.isEmpty()) { + return QueryStep.toTableLike(preFinalStep.getCteName()); + } + return QueryStepJoiner.join(List.of(preFinalStep, additionalStep.orElseThrow()), JoinMode.INNER); + } + + private static Selects finalSelects( + Selects preFinalSelects, + boolean includeValidityDate, + CompilerDialect compilerDialect + ) { + Optional validityDate = includeValidityDate + ? Optional.of(preFinalSelects.getValidityDate().orElseGet(compilerDialect::emptyDateRange)) + : Optional.empty(); + + return Selects.builder() + .ids(preFinalSelects.getIds()) + .validityDate(validityDate) + .stratificationDate(preFinalSelects.getStratificationDate()) + .sqlSelects(aggregateExplicitSelects(preFinalSelects, compilerDialect)) + .build(); + } + + private static List> aggregateExplicitSelects( + Selects finalSelects, + CompilerDialect compilerDialect + ) { + return finalSelects.getSqlSelects().stream() + .flatMap(sqlSelect -> sqlSelect.aggregateForFinalQuery(compilerDialect).stream()) + .map(FinalQueryStepComposer::toFieldWrapper) + .toList(); + } + + private static FieldWrapper toFieldWrapper(Field field) { + return new FieldWrapper<>(field); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/FinalQueryStepComposerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/FinalQueryStepComposerTest.java new file mode 100644 index 0000000000..50e5869a9a --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/FinalQueryStepComposerTest.java @@ -0,0 +1,165 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.Date; +import java.util.List; +import java.util.Locale; +import java.util.Optional; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import org.jooq.Field; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Test; + +class FinalQueryStepComposerTest { + + private static final CompilerDialect DIALECT = new TestDialect(); + + @Test + void shouldBuildAggregatedFinalStep() { + QueryStep preFinalStep = queryStep("compiled", "value", true); + TestDialect dialect = new TestDialect(); + + QueryStep result = FinalQueryStepComposer.compose(preFinalStep, Optional.empty(), true, dialect); + + assertNull(result.getCteName()); + assertEquals(ProjectionMode.AGGREGATED, result.getProjectionMode()); + assertEquals(List.of(preFinalStep), result.getPredecessors()); + assertEquals(List.of(name("compiled", "person_id")), result.getGroupBy().stream() + .map(Field::getQualifiedName) + .toList()); + assertTrue(result.getSelects().getValidityDate().isPresent()); + assertEquals(1, dialect.getAnyValueCalls()); + } + + @Test + void shouldOmitValidityDateWhenDisabled() { + QueryStep result = FinalQueryStepComposer.compose( + queryStep("compiled", "value", true), + Optional.empty(), + false, + DIALECT + ); + + assertFalse(result.getSelects().getValidityDate().isPresent()); + } + + @Test + void shouldKeepStableEmptyDateProjectionWhenInputHasNoValidityDate() { + TestDialect dialect = new TestDialect(); + QueryStep result = FinalQueryStepComposer.compose( + queryStep("compiled", "value", false), + Optional.empty(), + true, + dialect + ); + + ColumnDateRange validityDate = result.getSelects().getValidityDate().orElseThrow(); + assertEquals("null", render(validityDate.getStart())); + assertEquals("null", render(validityDate.getEnd())); + assertEquals(1, dialect.getAnyValueCalls()); + } + + @Test + void shouldNotAddEmptyDateProjectionWhenValidityDateIsDisabled() { + QueryStep result = FinalQueryStepComposer.compose( + queryStep("compiled", "value", false), + Optional.empty(), + false, + DIALECT + ); + + assertFalse(result.getSelects().getValidityDate().isPresent()); + } + + @Test + void shouldJoinAdditionalStepAndIncludeItsExplicitSelects() { + QueryStep preFinalStep = queryStep("compiled", "value", true); + QueryStep additionalStep = queryStep("extras", "extra_value", false); + + QueryStep result = FinalQueryStepComposer.compose( + preFinalStep, + Optional.of(additionalStep), + true, + DIALECT + ); + + assertEquals(List.of(preFinalStep, additionalStep), result.getPredecessors()); + assertEquals(2, result.getSelects().getSqlSelects().size()); + String tableSql = DSL.using(SQLDialect.POSTGRES) + .select() + .from(result.getFromTables().getFirst()) + .getSQL() + .toLowerCase(Locale.ROOT); + assertTrue(tableSql.contains("from \"compiled\" join \"extras\"")); + assertTrue(tableSql.contains("\"compiled\".\"person_id\" = \"extras\".\"person_id\"")); + } + + private static QueryStep queryStep(String cteName, String valueColumn, boolean withValidityDate) { + Selects.SelectsBuilder selects = Selects.builder() + .ids(new SqlIdColumns(field(name("person_id"), String.class))) + .sqlSelect(new FieldWrapper<>(field(name(valueColumn), Integer.class))); + if (withValidityDate) { + selects.validityDate(Optional.of(ColumnDateRange.of( + field(name("valid_from"), Date.class), + field(name("valid_to"), Date.class) + ))); + } + return QueryStep.builder() + .cteName(cteName) + .selects(selects.build()) + .build(); + } + + private static String render(Field field) { + return DSL.using(SQLDialect.POSTGRES).renderInlined(field).toLowerCase(Locale.ROOT); + } + + private static final class TestDialect implements CompilerDialect { + private int anyValueCalls; + + int getAnyValueCalls() { + return anyValueCalls; + } + + @Override + public Field minimumDate() { + return field(name("minimum_date"), Date.class); + } + + @Override + public Field maximumDate() { + return field(name("maximum_date"), Date.class); + } + + @Override + public Field anyValue(Field value) { + anyValueCalls++; + return DSL.field("any_value({0})", value.getType(), value); + } + + @Override + public Field renderDateRange(Field start, Field end) { + return field(name("rendered_range"), Object.class); + } + + @Override + public Field aggregateDateRanges(Field start, Field end) { + return field(name("aggregated_ranges"), Object.class); + } + + @Override + public int getNameMaxLength() { + return 128; + } + } +} From ef9821559581ffaa3e1688c7a0454dc277289381 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Fri, 11 Sep 2026 14:54:15 +0200 Subject: [PATCH 35/81] refactor(sql): move external node compilation into connector Represent resolved external entity uploads in the framework-neutral query model and compile their IDs, date ranges, restrictions, and result values into connector-owned query-step IR. Keep upload resolution in the legacy backend adapter and extend the dialect contract with the literal rendering capabilities required by external nodes. Co-authored-by: Codex GPT-5.6-sol --- .../concept/specific/external/CQExternal.java | 41 ++-- .../cqelement/CQExternalConverter.java | 203 ++++-------------- .../dialect/LegacyCompilerDialect.java | 28 +++ .../sql/compiler/dialect/CompilerDialect.java | 40 ++++ .../ir/ExternalQueryStepCompiler.java | 130 +++++++++++ .../sql/compiler/ir/ExternalQuerySteps.java | 10 + .../sql/model/node/ExternalEntity.java | 34 +++ .../conquery/sql/model/node/ExternalNode.java | 51 +++++ .../conquery/sql/model/node/QueryNode.java | 2 +- .../compiler/dialect/CompilerDialectTest.java | 13 ++ .../ir/ExternalQueryStepCompilerTest.java | 125 +++++++++++ .../conquery/sql/model/ExternalNodeTest.java | 78 +++++++ 12 files changed, 574 insertions(+), 181 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/ExternalQueryStepCompiler.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/ExternalQuerySteps.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/ExternalEntity.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/ExternalNode.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/ExternalQueryStepCompilerTest.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/model/ExternalNodeTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index 3a3b619cdd..2ad1c46154 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -8,9 +8,9 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.IntStream; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.io.cps.CPSType; @@ -32,7 +32,6 @@ import com.bakdata.conquery.models.types.ResultType; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonView; -import com.google.common.collect.Streams; import io.dropwizard.validation.ValidationMethod; import jakarta.validation.constraints.NotEmpty; import lombok.AccessLevel; @@ -106,13 +105,7 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { throw new IllegalStateException("CQExternal needs to be resolved before creating a plan"); } - final String[] extraHeaders = Streams.zip( - Arrays.stream(headers), - format.stream(), - (header, format) -> format.equals(EntityResolverUtil.FORMAT_EXTRA) ? header : null - ) - .filter(Objects::nonNull) - .toArray(String[]::new); + final String[] extraHeaders = getExtraHeaders().toArray(String[]::new); if (onlySingles) { return createExternalNodeOnlySingle(context, plan, extraHeaders); @@ -218,19 +211,24 @@ public List getResultInfos() { if (extra == null) { return Collections.emptyList(); } - List resultInfos = new ArrayList<>(); - for (int col = 0; col < format.size(); col++) { - if (!format.get(col).equals(EntityResolverUtil.FORMAT_EXTRA)) { - continue; - } - - final String column = headers[col]; + final ResultType type = onlySingles + ? ResultType.Primitive.STRING + : new ResultType.ListT<>(ResultType.Primitive.STRING); + return getExtraHeaders().stream() + .map(column -> new ExternalResultInfo(column, type)) + .toList(); + } - final ResultType type = onlySingles ? ResultType.Primitive.STRING : new ResultType.ListT<>(ResultType.Primitive.STRING); - resultInfos.add(new ExternalResultInfo(column, type)); + /** Ordered names of the resolved columns that are returned as external result values. */ + @JsonIgnore + public List getExtraHeaders() { + if (headers == null) { + return List.of(); } - - return resultInfos; + return IntStream.range(0, format.size()) + .filter(index -> EntityResolverUtil.FORMAT_EXTRA.equals(format.get(index))) + .mapToObj(index -> headers[index]) + .toList(); } @@ -268,6 +266,9 @@ public boolean isWithExtras() { } public List>> getExtrasForId(String id) { + if (extra == null) { + return List.of(); + } Map> extras = extra.getOrDefault(id, Collections.emptyMap()); // we need to bring the extras in the correct order List>> inOrder = new ArrayList<>(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java index dbcbf0682b..469ce34225 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java @@ -1,153 +1,27 @@ package com.bakdata.conquery.sql.conversion.cqelement; -import static org.jooq.impl.DSL.*; - -import java.util.ArrayList; -import java.util.Collections; +import java.util.Collection; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; -import com.bakdata.conquery.models.common.CDateSet; -import com.bakdata.conquery.sql.compiler.ir.condition.DateRestrictionCondition; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.sql.compiler.ir.ExternalQueryStepCompiler; +import com.bakdata.conquery.sql.compiler.ir.ExternalQuerySteps; import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.compiler.ir.SharedAliases; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import com.google.common.base.Preconditions; -import org.jooq.Field; -import org.jooq.Name; -import org.jooq.Record; -import org.jooq.Table; -import org.jooq.impl.DSL; - +import com.bakdata.conquery.sql.model.node.ExternalEntity; +import com.bakdata.conquery.sql.model.node.ExternalNode; +import com.bakdata.conquery.sql.model.range.DateRange; + +/** + * Adapts the resolved legacy external-query payload to connector-owned compilation. + * + *

TODO Remove this adapter once backend query resolution produces {@link ExternalNode} directly.

+ */ public class CQExternalConverter implements NodeConverter { - private static final String CQ_EXTERNAL_IDS_CTE_NAME = "external_ids"; - private static final String CQ_EXTERNAL_EXTRAS_CTE_NAME = "external_extra"; - private static final String UNDERSCORE = "_"; - private static final String WHITESPACE = " "; - - private static QueryStep createExternalIdsCte(CQExternal external, SqlFunctionProvider functionProvider, ConversionContext context) { - List unions = new ArrayList<>(); - for (Map.Entry entry : external.getValuesResolved().entrySet()) { - List rowSelects = createRowSelects(entry, functionProvider); - unions.addAll(rowSelects); - } - Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 resolved row when converting a CQExternal"); - QueryStep allStep = QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_IDS_CTE_NAME, Collections.emptyList(), context.isNegation()); - - Optional maybeValidityDate = allStep.getSelects().getValidityDate(); - - if (maybeValidityDate.isEmpty() || !context.dateRestrictionActive()) { - return allStep; - } - - return QueryStep.builder() - .predecessors(List.of(allStep)) - .conditions(List.of(new DateRestrictionCondition( - functionProvider.forCDateRange(context.getDateRestrictionRange()), maybeValidityDate.get() - ).condition())) - .selects(allStep.getQualifiedSelects()) - .fromTable(table(name(allStep.getCteName()))) - .cteName(CQ_EXTERNAL_IDS_CTE_NAME + "_date_restriction") - .build(); - - } - - /** - * For each entry, we need to create a SELECT statement of static values for each pid -> date set. For dialects that support date multiranges, - * 1 row per ID is sufficient. For other dialects there can be multiple rows with the same pid -> date range from the date set. - */ - private static List createRowSelects(Map.Entry entry, SqlFunctionProvider functionProvider) { - SqlIdColumns ids = createIdSelect(entry, functionProvider); - List validityDateEntries = functionProvider.forCDateSet(entry.getValue(), SharedAliases.DATES_COLUMN); - return validityDateEntries.stream() - .map(validityDateEntry -> createIdRowSelect(ids, validityDateEntry, functionProvider)) - .toList(); - } - - private static SqlIdColumns createIdSelect(Map.Entry entry, SqlFunctionProvider functionProvider) { - Field primaryColumn = functionProvider.externalId(entry.getKey()).as(SharedAliases.PRIMARY_COLUMN.getAlias()); - return new SqlIdColumns(primaryColumn); - } - - /** - * For each entry, we need to create a SELECT statement of static values for each pid -> extras. - */ - private QueryStep createRowSelects( - Map.Entry entry, - List>> extra, - SqlFunctionProvider functionProvider - ) { - SqlIdColumns ids = createIdSelect(entry); - List extraSelects = extra.stream().map(extraEntry -> createExtraColumnValue(extraEntry, functionProvider)).collect(Collectors.toList()); - return createExtraRowSelect(ids, extraSelects, functionProvider); - } - - private static SqlIdColumns createIdSelect(Map.Entry entry) { - Field primaryColumn = DSL.val(entry.getKey()).coerce(String.class).as(SharedAliases.PRIMARY_COLUMN.getAlias()); - return new SqlIdColumns(primaryColumn); - } - - private static FieldWrapper createExtraColumnValue(Map.Entry> extraEntry, SqlFunctionProvider functionProvider) { - Field extraValues = functionProvider.asArrayRepr(extraEntry.getValue()); - final Name alias = name(extraEntry.getKey().replace(WHITESPACE, UNDERSCORE)); - final Field withAlias = extraValues.as(alias); - return new FieldWrapper<>(withAlias); - } - - /** - * Creates a SELECT statement of static values for each pid -> date entry, like - * 
{@code select 1 as "pid", '[2021-01-01,2022-01-01)'::daterange as "date_range"}
- */ - private static QueryStep createIdRowSelect( - SqlIdColumns ids, - ColumnDateRange validityDate, - SqlFunctionProvider functionProvider - ) { - Selects selects = Selects.builder() - .ids(ids) - .validityDate(Optional.ofNullable(validityDate)) - .build(); - return wrapInQueryStep(selects, functionProvider); - } - - /** - * Creates a SELECT statement of static values for each pid -> extras, like - * 
{@code select 1 as "pid", '1' as "payload_1"}
- */ - private static QueryStep createExtraRowSelect( - SqlIdColumns ids, - List extraSelects, - SqlFunctionProvider functionProvider - ) { - Selects selects = Selects.builder() - .ids(ids) - .sqlSelects(extraSelects) - .build(); - return wrapInQueryStep(selects, functionProvider); - } - - private static QueryStep wrapInQueryStep(Selects selects, SqlFunctionProvider functionProvider) { - - // not all SQL dialects can create a SELECT statement without a FROM clause, - // so we ensure there is some no-op table to select the static values from - Table table = functionProvider.getNoOpTable(); - - return QueryStep.builder() - .selects(selects) - .fromTable(table) - .build(); - } - @Override public Class getConversionClass() { return CQExternal.class; @@ -155,29 +29,38 @@ public Class getConversionClass() { @Override public ConversionContext convert(CQExternal external, ConversionContext context) { - SqlFunctionProvider functionProvider = context.getCompilerDialect().getFunctionProvider(); - - QueryStep externalIdsCte = createExternalIdsCte(external, functionProvider, context); - ConversionContext withExternalIdCte = context.withQueryStep(externalIdsCte); - - if (!external.isWithExtras()) { - return withExternalIdCte; - } - - QueryStep externalExtrasCte = createExternalExtrasCte(external, functionProvider, context); - return withExternalIdCte.withExternalExtras(externalExtrasCte); + ExternalQuerySteps steps = ExternalQueryStepCompiler.compile( + toExternalNode(external), + context.isNegation(), + Optional.ofNullable(context.getDateRestrictionRange()).map(CQExternalConverter::toDateRange), + context.getCompilerDialect() + ); + ConversionContext converted = context.withQueryStep(steps.entities()); + return steps.values().map(converted::withExternalExtras).orElse(converted); } - private QueryStep createExternalExtrasCte(CQExternal external, SqlFunctionProvider functionProvider, ConversionContext context) { - List unions = new ArrayList<>(); - for (Map.Entry entry : external.getValuesResolved().entrySet()) { - List>> extrasForId = external.getExtrasForId(entry.getKey()); - QueryStep rowSelects = createRowSelects(entry, extrasForId, functionProvider); - unions.add(rowSelects); - } - Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 converted resolved row when converting a CQExternal"); - return QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_EXTRAS_CTE_NAME, Collections.emptyList(), context.isNegation()); + private static ExternalNode toExternalNode(CQExternal external) { + List entities = external.getValuesResolved().entrySet().stream() + .map(entry -> toExternalEntity(external, entry.getKey(), entry.getValue().asRanges())) + .toList(); + return new ExternalNode(entities, external.getExtraHeaders()); } + private static ExternalEntity toExternalEntity( + CQExternal external, + String entityId, + Collection validityDates + ) { + Map> values = new LinkedHashMap<>(); + external.getExtrasForId(entityId).forEach(entry -> values.put(entry.getKey(), entry.getValue())); + return new ExternalEntity( + entityId, + validityDates.stream().map(CQExternalConverter::toDateRange).toList(), + values + ); + } + private static DateRange toDateRange(CDateRange range) { + return new DateRange(Optional.ofNullable(range.getMin()), Optional.ofNullable(range.getMax())); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java index 99c7ca45a2..e774f7a77f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java @@ -5,11 +5,13 @@ import java.util.List; import java.util.Map; +import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.rendering.QueryStepRenderer; +import com.bakdata.conquery.sql.model.range.DateRange; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.cqelement.CQAndConverter; import com.bakdata.conquery.sql.conversion.cqelement.CQDateRestrictionConverter; @@ -30,6 +32,8 @@ import com.bakdata.conquery.sql.conversion.query.TableExportQueryConverter; import org.jooq.DSLContext; import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; /** * Temporary backend adapter exposing services required by the legacy SQL compiler. @@ -65,6 +69,30 @@ default Field aggregateDateRanges(Field start, Field end) { return getFunctionProvider().dateRangeAggregation(ColumnDateRange.of(start, end)); } + @Override + default Field externalId(String id) { + return getFunctionProvider().externalId(id); + } + + @Override + default Field externalStringValues(List values) { + return getFunctionProvider().asArrayRepr(values); + } + + @Override + default Table literalSelectTable() { + return getFunctionProvider().getNoOpTable(); + } + + @Override + default ColumnDateRange dateRangeLiteral(DateRange dateRange) { + CDateRange legacyDateRange = CDateRange.of( + dateRange.startInclusive().orElse(null), + dateRange.endInclusive().orElse(null) + ); + return getFunctionProvider().forCDateRange(legacyDateRange); + } + StratificationFunctions getStratificationFunctions(); SqlFunctionProvider getFunctionProvider(); diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java index 064e7739a2..12a6756bc4 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java @@ -1,11 +1,16 @@ package com.bakdata.conquery.sql.compiler.dialect; import java.sql.Date; +import java.util.List; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.model.range.DateRange; import org.jooq.Condition; import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; /** * Database-specific capabilities exposed to the framework-neutral SQL compiler. @@ -45,6 +50,41 @@ default ColumnDateRange conditionalUnboundedDateRange(Condition condition) { ); } + /** SQL expression for an already-resolved external entity ID. */ + default Field externalId(String id) { + return DSL.inline(id, SQLDataType.VARCHAR); + } + + /** + * Render ordered external string values in the physical representation expected by this dialect's result reader. + */ + default Field externalStringValues(List values) { + if (values.isEmpty()) { + return DSL.inline(null, String.class); + } + Field concatenated = DSL.inline(values.getFirst()); + for (String value : values.subList(1, values.size())) { + concatenated = DSL.concat(concatenated, DSL.inline(String.valueOf((char) 31)), DSL.inline(value)); + } + return concatenated; + } + + /** Dummy table used by dialects that cannot select literal values without a FROM clause. */ + default Table literalSelectTable() { + return DSL.noTable(); + } + + /** Convert an inclusive resolved date range to the compiler's half-open date-range representation. */ + default ColumnDateRange dateRangeLiteral(DateRange dateRange) { + Field start = dateRange.startInclusive() + .>map(value -> DSL.inline(Date.valueOf(value))) + .orElseGet(this::minimumDate); + Field end = dateRange.endInclusive() + .>map(value -> DSL.inline(Date.valueOf(value.plusDays(1)))) + .orElseGet(this::maximumDate); + return ColumnDateRange.of(start, end); + } + /** * Aggregate a field to an arbitrary value from its group. * diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/ExternalQueryStepCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/ExternalQueryStepCompiler.java new file mode 100644 index 0000000000..7132ecda46 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/ExternalQueryStepCompiler.java @@ -0,0 +1,130 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import static org.jooq.impl.DSL.name; +import static org.jooq.impl.DSL.table; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.condition.DateRestrictionCondition; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import com.bakdata.conquery.sql.model.node.ExternalEntity; +import com.bakdata.conquery.sql.model.node.ExternalNode; +import com.bakdata.conquery.sql.model.range.DateRange; +import org.jooq.Field; +import org.jooq.Name; + +/** Compiles resolved external entity values into literal-backed query-step IR. */ +public final class ExternalQueryStepCompiler { + + private static final String EXTERNAL_IDS_CTE_NAME = "external_ids"; + private static final String EXTERNAL_VALUES_CTE_NAME = "external_extra"; + + private ExternalQueryStepCompiler() { + } + + public static ExternalQuerySteps compile( + ExternalNode externalNode, + boolean negate, + Optional dateRestriction, + CompilerDialect compilerDialect + ) { + QueryStep entities = createEntityStep(externalNode, negate, dateRestriction, compilerDialect); + Optional values = externalNode.valueColumns().isEmpty() + ? Optional.empty() + : Optional.of(createValuesStep(externalNode, negate, compilerDialect)); + return new ExternalQuerySteps(entities, values); + } + + private static QueryStep createEntityStep( + ExternalNode externalNode, + boolean negate, + Optional dateRestriction, + CompilerDialect compilerDialect + ) { + List rows = externalNode.entities().stream() + .flatMap(entity -> entityRows(entity, compilerDialect).stream()) + .toList(); + QueryStep allEntities = QueryStep.createUnionAllStep( + rows, + EXTERNAL_IDS_CTE_NAME, + List.of(), + negate + ); + + if (dateRestriction.isEmpty()) { + return allEntities; + } + ColumnDateRange validityDate = allEntities.getSelects().getValidityDate().orElseThrow(); + return QueryStep.builder() + .predecessor(allEntities) + .conditions(List.of(new DateRestrictionCondition( + compilerDialect.dateRangeLiteral(dateRestriction.orElseThrow()), + validityDate + ).condition())) + .selects(allEntities.getQualifiedSelects()) + .fromTable(table(name(allEntities.getCteName()))) + .cteName(EXTERNAL_IDS_CTE_NAME + "_date_restriction") + .build(); + } + + private static List entityRows(ExternalEntity entity, CompilerDialect compilerDialect) { + SqlIdColumns ids = new SqlIdColumns( + compilerDialect.externalId(entity.entityId()).as(SharedAliases.PRIMARY_COLUMN.getAlias()) + ); + List validityDates = entity.validityDates().isEmpty() + ? List.of(compilerDialect.emptyDateRange()) + : entity.validityDates().stream().map(compilerDialect::dateRangeLiteral).toList(); + + return validityDates.stream() + .map(validityDate -> literalRow( + Selects.builder() + .ids(ids) + .validityDate(Optional.of(validityDate.as(SharedAliases.DATES_COLUMN.getAlias()))) + .build(), + compilerDialect + )) + .toList(); + } + + private static QueryStep createValuesStep( + ExternalNode externalNode, + boolean negate, + CompilerDialect compilerDialect + ) { + List rows = externalNode.entities().stream() + .map(entity -> valueRow(entity, externalNode.valueColumns(), compilerDialect)) + .toList(); + return QueryStep.createUnionAllStep(rows, EXTERNAL_VALUES_CTE_NAME, List.of(), negate); + } + + private static QueryStep valueRow( + ExternalEntity entity, + List valueColumns, + CompilerDialect compilerDialect + ) { + SqlIdColumns ids = new SqlIdColumns( + compilerDialect.externalId(entity.entityId()).as(SharedAliases.PRIMARY_COLUMN.getAlias()) + ); + List values = new ArrayList<>(valueColumns.size()); + for (String valueColumn : valueColumns) { + Field expression = compilerDialect.externalStringValues( + entity.values().getOrDefault(valueColumn, List.of()) + ); + Name alias = name(valueColumn.replace(' ', '_')); + values.add(new FieldWrapper<>(expression.as(alias))); + } + return literalRow(Selects.builder().ids(ids).sqlSelects(values).build(), compilerDialect); + } + + private static QueryStep literalRow(Selects selects, CompilerDialect compilerDialect) { + return QueryStep.builder() + .selects(selects) + .fromTable(compilerDialect.literalSelectTable()) + .build(); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/ExternalQuerySteps.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/ExternalQuerySteps.java new file mode 100644 index 0000000000..8f27b599d2 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/ExternalQuerySteps.java @@ -0,0 +1,10 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import java.util.Optional; + +/** Query-step graph fragments produced for a resolved external node. */ +public record ExternalQuerySteps( + QueryStep entities, + Optional values +) { +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/ExternalEntity.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/ExternalEntity.java new file mode 100644 index 0000000000..fe14c84db5 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/ExternalEntity.java @@ -0,0 +1,34 @@ +package com.bakdata.conquery.sql.model.node; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.bakdata.conquery.sql.model.internal.ModelNormalization; +import com.bakdata.conquery.sql.model.range.DateRange; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; + +/** One resolved external entity together with its validity ranges and ordered result values. */ +public record ExternalEntity( + @NotBlank String entityId, + @NotNull List<@NotNull @Valid DateRange> validityDates, + @NotNull Map<@NotBlank String, @NotNull List<@NotNull String>> values +) { + + public ExternalEntity { + validityDates = ModelNormalization.immutableCopy(validityDates); + values = deepImmutableCopy(values); + } + + private static Map> deepImmutableCopy(Map> values) { + if (values == null) { + return null; + } + Map> copy = new LinkedHashMap<>(); + values.forEach((column, columnValues) -> copy.put(column, ModelNormalization.immutableCopy(columnValues))); + return Collections.unmodifiableMap(copy); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/ExternalNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/ExternalNode.java new file mode 100644 index 0000000000..9aff0ffddb --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/ExternalNode.java @@ -0,0 +1,51 @@ +package com.bakdata.conquery.sql.model.node; + +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +import com.bakdata.conquery.sql.model.internal.ModelNormalization; +import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; + +/** A resolved upload of entity IDs, optional validity dates, and additional result columns. */ +public record ExternalNode( + @NotEmpty List<@NotNull @Valid ExternalEntity> entities, + @NotNull List<@NotBlank String> valueColumns +) implements QueryNode { + + public ExternalNode { + entities = ModelNormalization.immutableCopy(entities); + valueColumns = ModelNormalization.immutableCopy(valueColumns); + } + + @AssertTrue(message = "external entity IDs must be unique") + public boolean isEntityIdsUnique() { + return entities == null || entities.stream() + .filter(Objects::nonNull) + .map(ExternalEntity::entityId) + .distinct() + .count() == entities.size(); + } + + @AssertTrue(message = "external value column names must be unique") + public boolean isValueColumnsUnique() { + return valueColumns == null || new HashSet<>(valueColumns).size() == valueColumns.size(); + } + + @AssertTrue(message = "external entity values must reference declared value columns") + public boolean isEntityValueColumnsDeclared() { + if (entities == null || valueColumns == null) { + return true; + } + Set declaredColumns = new HashSet<>(valueColumns); + return entities.stream() + .filter(entity -> entity != null && entity.values() != null) + .flatMap(entity -> entity.values().keySet().stream()) + .allMatch(declaredColumns::contains); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/QueryNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/QueryNode.java index 8aa8152bb0..c41a301b7f 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/QueryNode.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/QueryNode.java @@ -1,5 +1,5 @@ package com.bakdata.conquery.sql.model.node; /** A normalized query-tree node accepted by SQL compilation. */ -public sealed interface QueryNode permits AndNode, OrNode, NegationNode, DateRestrictionNode, ConceptNode { +public sealed interface QueryNode permits AndNode, OrNode, NegationNode, DateRestrictionNode, ConceptNode, ExternalNode { } diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialectTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialectTest.java index 4ac416a2c3..23ff5f8e27 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialectTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialectTest.java @@ -6,9 +6,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.sql.Date; +import java.time.LocalDate; import java.util.Locale; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.model.range.DateRange; import org.jooq.Field; import org.jooq.SQLDialect; import org.jooq.impl.DSL; @@ -48,6 +50,17 @@ void shouldCreateConditionalUnboundedDateRange() { assertTrue(end.contains("\"maximum_date\"")); } + @Test + void shouldConvertInclusiveResolvedDateRangeToExclusiveSqlEnd() { + ColumnDateRange dateRange = DIALECT.dateRangeLiteral(DateRange.closed( + LocalDate.of(2025, 1, 1), + LocalDate.of(2025, 1, 31) + )); + + assertEquals("date '2025-01-01'", render(dateRange.getStart())); + assertEquals("date '2025-02-01'", render(dateRange.getEnd())); + } + private static String render(Field expression) { return DSL.using(SQLDialect.POSTGRES).renderInlined(expression).toLowerCase(Locale.ROOT); } diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/ExternalQueryStepCompilerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/ExternalQueryStepCompilerTest.java new file mode 100644 index 0000000000..9ff5b17151 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/ExternalQueryStepCompilerTest.java @@ -0,0 +1,125 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.Date; +import java.time.LocalDate; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.model.node.ExternalEntity; +import com.bakdata.conquery.sql.model.node.ExternalNode; +import com.bakdata.conquery.sql.model.range.DateRange; +import org.jooq.Field; +import org.junit.jupiter.api.Test; + +class ExternalQueryStepCompilerTest { + + private static final CompilerDialect DIALECT = new TestDialect(); + private static final DateRange JANUARY = DateRange.closed( + LocalDate.of(2025, 1, 1), + LocalDate.of(2025, 1, 31) + ); + + @Test + void shouldCompileOneEntityRowForEveryValidityRange() { + ExternalNode node = new ExternalNode( + List.of( + new ExternalEntity("person-1", List.of(JANUARY, DateRange.closed( + LocalDate.of(2025, 3, 1), LocalDate.of(2025, 3, 31) + )), Map.of()), + new ExternalEntity("person-2", List.of(), Map.of()) + ), + List.of() + ); + + ExternalQuerySteps compiled = ExternalQueryStepCompiler.compile(node, true, Optional.empty(), DIALECT); + + assertEquals("external_ids", compiled.entities().getCteName()); + assertEquals(2, compiled.entities().getUnion().size()); + assertTrue(compiled.entities().isNegate()); + assertTrue(compiled.entities().getSelects().getValidityDate().isPresent()); + assertTrue(compiled.values().isEmpty()); + } + + @Test + void shouldCompileOrderedExternalValueColumnsForEveryEntity() { + ExternalNode node = new ExternalNode( + List.of( + new ExternalEntity("person-1", List.of(JANUARY), Map.of("group name", List.of("A", "B"))), + new ExternalEntity("person-2", List.of(JANUARY), Map.of()) + ), + List.of("group name") + ); + + QueryStep values = ExternalQueryStepCompiler.compile(node, false, Optional.empty(), DIALECT) + .values() + .orElseThrow(); + + assertEquals("external_extra", values.getCteName()); + assertEquals(1, values.getUnion().size()); + assertEquals(List.of("group_name"), values.getSelects().explicitSelects().stream().map(Field::getName).toList()); + assertEquals(List.of("group_name"), values.getUnion().getFirst().getSelects().explicitSelects().stream() + .map(Field::getName) + .toList()); + } + + @Test + void shouldApplyResolvedDateRestrictionInASeparateStep() { + ExternalNode node = new ExternalNode( + List.of(new ExternalEntity("person-1", List.of(JANUARY), Map.of())), + List.of() + ); + + QueryStep restricted = ExternalQueryStepCompiler.compile( + node, + false, + Optional.of(DateRange.closed(LocalDate.of(2025, 1, 10), LocalDate.of(2025, 1, 20))), + DIALECT + ).entities(); + + assertEquals("external_ids_date_restriction", restricted.getCteName()); + assertEquals(List.of("external_ids"), restricted.getPredecessors().stream().map(QueryStep::getCteName).toList()); + assertEquals(1, restricted.getConditions().size()); + assertFalse(restricted.isNegate()); + } + + private static final class TestDialect implements CompilerDialect { + + @Override + public Field minimumDate() { + return field(name("minimum_date"), Date.class); + } + + @Override + public Field maximumDate() { + return field(name("maximum_date"), Date.class); + } + + @Override + public Field anyValue(Field value) { + return value; + } + + @Override + public Field renderDateRange(Field start, Field end) { + return field(name("rendered_range"), Object.class); + } + + @Override + public Field aggregateDateRanges(Field start, Field end) { + return field(name("aggregated_ranges"), Object.class); + } + + @Override + public int getNameMaxLength() { + return 128; + } + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ExternalNodeTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ExternalNodeTest.java new file mode 100644 index 0000000000..e0e500d16c --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ExternalNodeTest.java @@ -0,0 +1,78 @@ +package com.bakdata.conquery.sql.model; + +import static com.bakdata.conquery.sql.model.ValidationTestSupport.assertInvalid; +import static com.bakdata.conquery.sql.model.ValidationTestSupport.assertValid; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.bakdata.conquery.sql.model.node.ExternalEntity; +import com.bakdata.conquery.sql.model.node.ExternalNode; +import com.bakdata.conquery.sql.model.range.DateRange; +import org.junit.jupiter.api.Test; + +class ExternalNodeTest { + + private static final DateRange VALIDITY = DateRange.closed( + LocalDate.of(2025, 1, 1), + LocalDate.of(2025, 1, 31) + ); + + @Test + void shouldAcceptFullyResolvedExternalValues() { + ExternalNode externalNode = new ExternalNode( + List.of(entity("person-1", Map.of("group", List.of("A", "B")))), + List.of("group") + ); + + assertValid(externalNode); + } + + @Test + void shouldDefensivelyCopyNestedValues() { + List columnValues = new ArrayList<>(List.of("A")); + Map> values = new LinkedHashMap<>(); + values.put("group", columnValues); + ExternalEntity entity = new ExternalEntity("person-1", List.of(VALIDITY), values); + + columnValues.add("B"); + values.put("other", List.of("value")); + + assertEquals(Map.of("group", List.of("A")), entity.values()); + assertThrows(UnsupportedOperationException.class, () -> entity.values().put("other", List.of())); + assertThrows(UnsupportedOperationException.class, () -> entity.values().get("group").add("B")); + } + + @Test + void shouldRejectDuplicateEntityIds() { + assertInvalid(new ExternalNode( + List.of(entity("person-1", Map.of()), entity("person-1", Map.of())), + List.of() + )); + } + + @Test + void shouldRejectDuplicateValueColumns() { + assertInvalid(new ExternalNode( + List.of(entity("person-1", Map.of("group", List.of("A")))), + List.of("group", "group") + )); + } + + @Test + void shouldRejectUndeclaredValueColumns() { + assertInvalid(new ExternalNode( + List.of(entity("person-1", Map.of("undeclared", List.of("A")))), + List.of("group") + )); + } + + private static ExternalEntity entity(String id, Map> values) { + return new ExternalEntity(id, List.of(VALIDITY), values); + } +} From 72caab9e31500786b4ff2d7881d8170b3406071f Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:08:07 +0200 Subject: [PATCH 36/81] refactor(sql): move converter dispatch into connector Make runtime conversion dispatch independent of the legacy backend context so it can support connector-owned converters for resolved model operations. Keep legacy node conversion as a backend adapter and verify that dispatch selects exactly one matching converter. Co-authored-by: Codex GPT-5.6-sol --- .../conquery/sql/conversion/Conversions.java | 45 ----------- .../conquery/sql/conversion/Converter.java | 25 ------- .../sql/conversion/NodeConversions.java | 7 +- .../sql/conversion/NodeConverter.java | 1 + .../conversion/ConversionDispatcher.java | 37 +++++++++ .../sql/compiler/conversion/Converter.java | 27 +++++++ .../sql/compiler/conversion/package-info.java | 4 + .../conversion/ConversionDispatcherTest.java | 75 +++++++++++++++++++ 8 files changed, 148 insertions(+), 73 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/Conversions.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/Converter.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/ConversionDispatcher.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/Converter.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/package-info.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/ConversionDispatcherTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Conversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/Conversions.java deleted file mode 100644 index 48a59e2b84..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Conversions.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.bakdata.conquery.sql.conversion; - -import java.util.List; -import java.util.Optional; - -import lombok.Getter; - -/** - * Converts an input to a result with an applicable converter. - * - * @param type that can be converted - * @param type of the result - * @param context of the convertible - * @see Converter - */ -public class Conversions { - - @Getter - private final List> converters; - - protected Conversions(List> converters) { - this.converters = converters; - } - - public R convert(C node, X context) { - R converted = null; - for (Converter converter : converters) { - Optional maybeConverted = converter.tryConvert(node, context); - if (maybeConverted.isPresent()) { - if (converted == null) { - converted = maybeConverted.get(); - } else { - throw new IllegalStateException("Multiple Converters for %s".formatted(node)); - } - } - } - - if (converted == null) { - throw new IllegalStateException("No converter found for %s".formatted(node)); - } - - return converted; - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Converter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/Converter.java deleted file mode 100644 index cd34b0af27..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Converter.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.bakdata.conquery.sql.conversion; - -import java.util.Optional; - -/** - * A converter converts an input into a result object if the input matches the conversion class. - * - * @param type that can be converted - * @param type of the result - * @param context of the convertible - */ -public interface Converter { - - default Optional tryConvert(I input, X context) { - if (getConversionClass().isInstance(input)) { - return Optional.ofNullable(convert(getConversionClass().cast(input), context)); - } - return Optional.empty(); - } - - Class getConversionClass(); - - R convert(final C convert, final X context); - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java index 71ea205adf..ab44c944a4 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java @@ -4,6 +4,7 @@ import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.IdColumnConfig; import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.sql.compiler.conversion.ConversionDispatcher; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; @@ -18,7 +19,7 @@ */ public class NodeConversions implements NodeConversionDispatcher { - private final Conversions conversions; + private final ConversionDispatcher dispatcher; private final IdColumnConfig idColumns; private final LegacyCompilerDialect dialect; private final SqlNameGenerator nameGenerator; @@ -33,7 +34,7 @@ public NodeConversions( Clock clock, String defaultPrimaryColumn ) { - this.conversions = new Conversions<>(compilerDialect.getNodeConverters(dslContext)); + this.dispatcher = new ConversionDispatcher<>(compilerDialect.getNodeConverters(dslContext)); this.idColumns = idColumns; this.dialect = compilerDialect; this.nameGenerator = new SqlNameGenerator(compilerDialect.getNameMaxLength()); @@ -43,7 +44,7 @@ public NodeConversions( @Override public ConversionContext convert(Object node, ConversionContext context) { - return conversions.convert(node, context); + return dispatcher.convert(node, context); } public ConversionContext convert(QueryDescription queryDescription, ConqueryConfig conqueryConfig) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverter.java index 8b49b696fb..514c7190d0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverter.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.sql.conversion; import com.bakdata.conquery.models.query.Visitable; +import com.bakdata.conquery.sql.compiler.conversion.Converter; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; /** diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/ConversionDispatcher.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/ConversionDispatcher.java new file mode 100644 index 0000000000..44f7223904 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/ConversionDispatcher.java @@ -0,0 +1,37 @@ +package com.bakdata.conquery.sql.compiler.conversion; + +import java.util.List; +import java.util.Optional; + +/** Dispatches a value to exactly one converter supporting its runtime type. */ +public final class ConversionDispatcher { + + private final List> converters; + + public ConversionDispatcher(List> converters) { + this.converters = List.copyOf(converters); + } + + public List> getConverters() { + return converters; + } + + public R convert(C input, X context) { + R converted = null; + for (Converter converter : converters) { + Optional maybeConverted = converter.tryConvert(input, context); + if (maybeConverted.isEmpty()) { + continue; + } + if (converted != null) { + throw new IllegalStateException("Multiple converters for %s".formatted(input)); + } + converted = maybeConverted.orElseThrow(); + } + + if (converted == null) { + throw new IllegalStateException("No converter found for %s".formatted(input)); + } + return converted; + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/Converter.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/Converter.java new file mode 100644 index 0000000000..156ed38577 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/Converter.java @@ -0,0 +1,27 @@ +package com.bakdata.conquery.sql.compiler.conversion; + +import java.util.Optional; + +/** + * Converts values of one supported runtime type using an explicit compilation context. + * + *

This small SPI is independent of query-framework classes so connector extensions can contribute converters for + * resolved model operations.

+ * + * @param supported input type + * @param conversion result type + * @param conversion context type + */ +public interface Converter { + + default Optional tryConvert(I input, X context) { + if (getConversionClass().isInstance(input)) { + return Optional.ofNullable(convert(getConversionClass().cast(input), context)); + } + return Optional.empty(); + } + + Class getConversionClass(); + + R convert(C input, X context); +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/package-info.java new file mode 100644 index 0000000000..1f1bb1302a --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/package-info.java @@ -0,0 +1,4 @@ +/** + * Runtime dispatch contracts for converting extensible resolved-model operations into compiler representations. + */ +package com.bakdata.conquery.sql.compiler.conversion; diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/ConversionDispatcherTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/ConversionDispatcherTest.java new file mode 100644 index 0000000000..5882a15deb --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/ConversionDispatcherTest.java @@ -0,0 +1,75 @@ +package com.bakdata.conquery.sql.compiler.conversion; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; + +class ConversionDispatcherTest { + + @Test + void shouldDispatchToConverterSupportingRuntimeType() { + ConversionDispatcher dispatcher = new ConversionDispatcher<>(List.of( + converter(Integer.class, (value, prefix) -> prefix + value) + )); + + assertEquals("value=42", dispatcher.convert(42, "value=")); + } + + @Test + void shouldRejectMissingConverter() { + ConversionDispatcher dispatcher = new ConversionDispatcher<>(List.of( + converter(Integer.class, (value, context) -> value.toString()) + )); + + assertThrows(IllegalStateException.class, () -> dispatcher.convert(1.5, null)); + } + + @Test + void shouldRejectAmbiguousConverters() { + ConversionDispatcher dispatcher = new ConversionDispatcher<>(List.of( + converter(Number.class, (value, context) -> value.toString()), + converter(Integer.class, (value, context) -> value.toString()) + )); + + assertThrows(IllegalStateException.class, () -> dispatcher.convert(42, null)); + } + + @Test + void shouldDefensivelyCopyConverters() { + List> converters = new ArrayList<>(); + converters.add(converter(Integer.class, (value, context) -> value.toString())); + ConversionDispatcher dispatcher = new ConversionDispatcher<>(converters); + + converters.clear(); + + assertEquals("42", dispatcher.convert(42, null)); + assertThrows(UnsupportedOperationException.class, () -> dispatcher.getConverters().clear()); + } + + private static Converter converter( + Class conversionClass, + Conversion conversion + ) { + return new Converter<>() { + @Override + public Class getConversionClass() { + return conversionClass; + } + + @Override + public String convert(C input, X context) { + return conversion.convert(input, context); + } + }; + } + + @FunctionalInterface + private interface Conversion { + + String convert(C input, X context); + } +} From 628f213e73d8e1ce2201d34158e53c69eb8f4ffb Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 14 Sep 2026 07:35:31 +0200 Subject: [PATCH 37/81] refactor(sql): move all-entities compilation into connector Represent all-entities queries in the resolved query model and compile their entity ID and empty validity range from the resolved entity schema. Translate legacy ID configuration into EntitySchema at the backend boundary. Reuse the same entity-table and primary-ID mapping for all-entities queries and negation anti-joins. Co-authored-by: Codex GPT-5.6-sol --- .../conversion/cqelement/CQYesConverter.java | 44 ++++------- .../conversion/model/EntitySchemaAdapter.java | 28 +++++++ .../conversion/model/QueryStepComposer.java | 21 +---- .../ir/AllEntitiesQueryStepCompiler.java | 36 +++++++++ .../sql/compiler/ir/EntitySchemaSql.java | 35 +++++++++ .../sql/compiler/ir/QueryStepComposer.java | 19 +---- .../sql/model/node/AllEntitiesNode.java | 5 ++ .../conquery/sql/model/node/QueryNode.java | 2 +- .../sql/model/schema/EntitySchema.java | 4 +- .../ir/AllEntitiesQueryStepCompilerTest.java | 78 +++++++++++++++++++ .../conquery/sql/model/ResolvedQueryTest.java | 9 +++ 11 files changed, 215 insertions(+), 66 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/EntitySchemaAdapter.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/AllEntitiesQueryStepCompiler.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/EntitySchemaSql.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/AllEntitiesNode.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/AllEntitiesQueryStepCompilerTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java index ea1e1ef23a..3627dfa7fa 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java @@ -1,23 +1,18 @@ package com.bakdata.conquery.sql.conversion.cqelement; -import static org.jooq.impl.DSL.*; - -import java.util.Optional; - import com.bakdata.conquery.apiv1.query.CQYes; -import com.bakdata.conquery.models.config.ColumnConfig; +import com.bakdata.conquery.sql.compiler.ir.AllEntitiesQueryStepCompiler; import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import org.jooq.Field; -import org.jooq.Record; - +import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; +import com.bakdata.conquery.sql.model.node.AllEntitiesNode; + +/** + * Adapts the legacy all-entities query node to connector-owned compilation. + * + *

TODO Remove this adapter once backend query resolution produces the connector's all-entities node directly.

+ */ public class CQYesConverter implements NodeConverter { - private static final String ALL_IDS_CTE = "all_ids"; - @Override public Class getConversionClass() { return CQYes.class; @@ -25,21 +20,10 @@ public Class getConversionClass() { @Override public ConversionContext convert(CQYes cqYes, ConversionContext context) { - - ColumnConfig primaryColumnConfig = context.getIdColumns().findPrimaryIdColumn(); - Field primaryColumn = field(name(primaryColumnConfig.getField()), String.class); - SqlIdColumns ids = new SqlIdColumns(primaryColumn); - - Selects selects = Selects.builder().ids(ids) - .validityDate(Optional.of(context.getFunctionProvider().emptyColumnDateRange().asValidityDateRange(ALL_IDS_CTE))) - .build(); - org.jooq.Table fromTable = table(name(context.getIdColumns().getTable())); - - QueryStep cqYesTep = QueryStep.builder() - .cteName(ALL_IDS_CTE) - .selects(selects) - .fromTable(fromTable) - .build(); - return context.withQueryStep(cqYesTep); + return context.withQueryStep(AllEntitiesQueryStepCompiler.compile( + new AllEntitiesNode(), + EntitySchemaAdapter.from(context.getIdColumns()), + context.getCompilerDialect() + )); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/EntitySchemaAdapter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/EntitySchemaAdapter.java new file mode 100644 index 0000000000..beef7d5811 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/EntitySchemaAdapter.java @@ -0,0 +1,28 @@ +package com.bakdata.conquery.sql.conversion.model; + +import com.bakdata.conquery.models.config.ColumnConfig; +import com.bakdata.conquery.models.config.IdColumnConfig; +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.sql.model.schema.EntitySchema; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.SqlTable; + +/** Converts the legacy entity-ID configuration into the connector's resolved entity schema. */ +public final class EntitySchemaAdapter { + + private EntitySchemaAdapter() { + } + + public static EntitySchema from(IdColumnConfig idColumns) { + ColumnConfig primaryId = idColumns.findPrimaryIdColumn(); + SqlTable entityTable = SqlTable.of(idColumns.getTable(), idColumns.getTable()); + ResolvedColumn primaryIdColumn = new ResolvedColumn( + primaryId.getName(), + entityTable, + primaryId.getField(), + ColumnType.STRING, + false + ); + return new EntitySchema(entityTable, primaryIdColumn); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java index 2e3af2b10a..9a2266163e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java @@ -3,16 +3,11 @@ import java.util.List; import com.bakdata.conquery.apiv1.query.CQElement; -import com.bakdata.conquery.models.config.ColumnConfig; -import com.bakdata.conquery.models.config.IdColumnConfig; -import com.bakdata.conquery.models.datasets.ColumnType; import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.sql.compiler.ir.JoinMode; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.model.schema.EntitySchema; -import com.bakdata.conquery.sql.model.schema.ResolvedColumn; -import com.bakdata.conquery.sql.model.schema.SqlTable; /** * Adapts legacy backend query nodes and ID configuration to connector-owned query-step composition. @@ -32,7 +27,7 @@ public static QueryStep antiJoinWithAllIdsTable( ) { return com.bakdata.conquery.sql.compiler.ir.QueryStepComposer.antiJoinWithAllEntities( queryStep, - toEntitySchema(context.getIdColumns()), + EntitySchemaAdapter.from(context.getIdColumns()), dateAggregationAction, context.getCompilerDialect() ); @@ -61,22 +56,10 @@ public static QueryStep joinSteps( queriesToJoin, logicalOperation, dateAggregationAction, - toEntitySchema(context.getIdColumns()), + EntitySchemaAdapter.from(context.getIdColumns()), context.getCompilerDialect(), context.getNameGenerator() ); } - private static EntitySchema toEntitySchema(IdColumnConfig idColumns) { - ColumnConfig primaryId = idColumns.findPrimaryIdColumn(); - SqlTable entityTable = SqlTable.of(idColumns.getTable(), idColumns.getTable()); - ResolvedColumn primaryIdColumn = new ResolvedColumn( - primaryId.getName(), - entityTable, - primaryId.getField(), - ColumnType.STRING, - false - ); - return new EntitySchema(entityTable, primaryIdColumn); - } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/AllEntitiesQueryStepCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/AllEntitiesQueryStepCompiler.java new file mode 100644 index 0000000000..6bf9399239 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/AllEntitiesQueryStepCompiler.java @@ -0,0 +1,36 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import java.util.Objects; +import java.util.Optional; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.model.node.AllEntitiesNode; +import com.bakdata.conquery.sql.model.schema.EntitySchema; + +/** Compiles the resolved entity schema into the query step used to select every entity. */ +public final class AllEntitiesQueryStepCompiler { + + private static final String ALL_ENTITIES_CTE = "all_ids"; + + private AllEntitiesQueryStepCompiler() { + } + + public static QueryStep compile( + AllEntitiesNode allEntitiesNode, + EntitySchema entitySchema, + CompilerDialect compilerDialect + ) { + Objects.requireNonNull(allEntitiesNode, "allEntitiesNode"); + Selects selects = Selects.builder() + .ids(new SqlIdColumns(EntitySchemaSql.primaryId(entitySchema))) + .validityDate(Optional.of( + compilerDialect.emptyDateRange().asValidityDateRange(ALL_ENTITIES_CTE) + )) + .build(); + return QueryStep.builder() + .cteName(ALL_ENTITIES_CTE) + .selects(selects) + .fromTable(EntitySchemaSql.table(entitySchema)) + .build(); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/EntitySchemaSql.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/EntitySchemaSql.java new file mode 100644 index 0000000000..f56e458c01 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/EntitySchemaSql.java @@ -0,0 +1,35 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; + +import java.util.ArrayList; +import java.util.List; + +import com.bakdata.conquery.sql.model.schema.EntitySchema; +import org.jooq.Field; +import org.jooq.Name; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.impl.DSL; + +/** Creates SQL identifiers for a resolved entity schema. */ +final class EntitySchemaSql { + + private EntitySchemaSql() { + } + + static Table table(EntitySchema entitySchema) { + return DSL.table(tableName(entitySchema)); + } + + static Name tableName(EntitySchema entitySchema) { + return name(entitySchema.table().physicalName().toArray(String[]::new)); + } + + static Field primaryId(EntitySchema entitySchema) { + List qualifiedName = new ArrayList<>(entitySchema.table().physicalName()); + qualifiedName.add(entitySchema.primaryId().physicalName()); + return field(name(qualifiedName.toArray(String[]::new)), String.class); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposer.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposer.java index 8f7d4ae09f..de9807fdc6 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposer.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposer.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.sql.compiler.ir; -import static org.jooq.impl.DSL.field; import static org.jooq.impl.DSL.name; import static org.jooq.impl.DSL.table; @@ -19,7 +18,6 @@ import com.bakdata.conquery.sql.model.schema.EntitySchema; import org.jooq.Condition; import org.jooq.Field; -import org.jooq.Name; import org.jooq.Record; import org.jooq.Table; import org.jooq.TableLike; @@ -41,9 +39,9 @@ public static QueryStep antiJoinWithAllEntities( CompilerDialect compilerDialect ) { Field queryStepPrimaryColumn = queryStep.getQualifiedSelects().getIds().getPrimaryColumn(); - Field allIdsPrimaryColumn = primaryIdField(entitySchema); + Field allIdsPrimaryColumn = EntitySchemaSql.primaryId(entitySchema); - Table joinedTable = table(tableName(entitySchema)) + Table joinedTable = table(EntitySchemaSql.tableName(entitySchema)) .leftOuterJoin(table(name(queryStep.getCteName()))) .on(allIdsPrimaryColumn.eq(queryStepPrimaryColumn)); @@ -197,7 +195,7 @@ private static QueryStep joinStepsContainingNegation( } negateJoined = DateAggregationCompiler.invert(negateJoined, compilerDialect, nameGenerator); - Field allIdsPrimaryColumn = primaryIdField(entitySchema); + Field allIdsPrimaryColumn = EntitySchemaSql.primaryId(entitySchema); Field negatePrimaryColumn = negateJoined.getQualifiedSelects().getIds().getPrimaryColumn(); Field nonNegatePrimaryColumn = nonNegateJoined.getQualifiedSelects().getIds().getPrimaryColumn(); @@ -216,7 +214,7 @@ private static QueryStep joinStepsContainingNegation( .ids(new SqlIdColumns(coalescedId)) .validityDate(Optional.of(merged)); - Table joinedTable = table(tableName(entitySchema)) + Table joinedTable = table(EntitySchemaSql.tableName(entitySchema)) .leftOuterJoin(table(name(negateJoined.getCteName()))) .on(allIdsPrimaryColumn.eq(negatePrimaryColumn)) .leftOuterJoin(table(name(nonNegateJoined.getCteName()))) @@ -279,13 +277,4 @@ private static Optional coalesceStratificationDates(List primaryIdField(EntitySchema entitySchema) { - List qualifiedName = new ArrayList<>(entitySchema.table().physicalName()); - qualifiedName.add(entitySchema.primaryId().physicalName()); - return field(name(qualifiedName.toArray(String[]::new)), String.class); - } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/AllEntitiesNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/AllEntitiesNode.java new file mode 100644 index 0000000000..d75c746207 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/AllEntitiesNode.java @@ -0,0 +1,5 @@ +package com.bakdata.conquery.sql.model.node; + +/** Selects every entity from the resolved entity schema. */ +public record AllEntitiesNode() implements QueryNode { +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/QueryNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/QueryNode.java index c41a301b7f..9bcd45964d 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/QueryNode.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/QueryNode.java @@ -1,5 +1,5 @@ package com.bakdata.conquery.sql.model.node; /** A normalized query-tree node accepted by SQL compilation. */ -public sealed interface QueryNode permits AndNode, OrNode, NegationNode, DateRestrictionNode, ConceptNode, ExternalNode { +public sealed interface QueryNode permits AllEntitiesNode, AndNode, OrNode, NegationNode, DateRestrictionNode, ConceptNode, ExternalNode { } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/EntitySchema.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/EntitySchema.java index 4b0d51e4b6..70c1d918f6 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/EntitySchema.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/EntitySchema.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.sql.model.schema; +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.sql.validation.AllowedColumnTypes; import jakarta.validation.Valid; import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.NotNull; @@ -7,7 +9,7 @@ /** Physical source used to enumerate all entities, for example when compiling a root-level negation. */ public record EntitySchema( @NotNull @Valid SqlTable table, - @NotNull @Valid ResolvedColumn primaryId + @NotNull @Valid @AllowedColumnTypes(ColumnType.STRING) ResolvedColumn primaryId ) { @AssertTrue(message = "primaryId must belong to the entity table") diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/AllEntitiesQueryStepCompilerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/AllEntitiesQueryStepCompilerTest.java new file mode 100644 index 0000000000..7c1428977c --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/AllEntitiesQueryStepCompilerTest.java @@ -0,0 +1,78 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.sql.Date; + +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.model.node.AllEntitiesNode; +import com.bakdata.conquery.sql.model.schema.EntitySchema; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.SqlTable; +import org.jooq.Field; +import org.junit.jupiter.api.Test; + +class AllEntitiesQueryStepCompilerTest { + + private static final SqlTable ENTITIES = SqlTable.of("entities", "catalog", "entities"); + private static final EntitySchema ENTITY_SCHEMA = new EntitySchema( + ENTITIES, + new ResolvedColumn("entity-id", ENTITIES, "person_id", ColumnType.STRING, false) + ); + + @Test + void shouldCompileResolvedEntitySchema() { + QueryStep step = AllEntitiesQueryStepCompiler.compile( + new AllEntitiesNode(), + ENTITY_SCHEMA, + new TestDialect() + ); + + assertEquals("all_ids", step.getCteName()); + assertEquals( + name("catalog", "entities", "person_id"), + step.getSelects().getIds().getPrimaryColumn().getQualifiedName() + ); + assertEquals(name("catalog", "entities"), step.getFromTables().getFirst().asTable().getQualifiedName()); + ColumnDateRange validityDate = step.getSelects().getValidityDate().orElseThrow(); + assertEquals("all_ids_validity_date_start", validityDate.getStart().getName()); + assertEquals("all_ids_validity_date_end", validityDate.getEnd().getName()); + } + + private static final class TestDialect implements CompilerDialect { + + @Override + public Field minimumDate() { + return field(name("minimum_date"), Date.class); + } + + @Override + public Field maximumDate() { + return field(name("maximum_date"), Date.class); + } + + @Override + public Field anyValue(Field value) { + return value; + } + + @Override + public Field renderDateRange(Field start, Field end) { + return field(name("rendered_range"), Object.class); + } + + @Override + public Field aggregateDateRanges(Field start, Field end) { + return field(name("aggregated_ranges"), Object.class); + } + + @Override + public int getNameMaxLength() { + return 128; + } + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java index 5b5305ce38..4971d83b43 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java @@ -95,6 +95,15 @@ void shouldRejectColumnsFromAnotherConnectorTable() { assertInvalid(connector); } + @Test + void shouldRejectNonStringEntityIds() { + ResolvedColumn numericId = new ResolvedColumn( + "dataset.events.numeric_id", EVENTS, "numeric_id", ColumnType.INTEGER, false + ); + + assertInvalid(new EntitySchema(EVENTS, numericId)); + } + @Test void shouldRejectNonDateValidityColumns() { assertInvalid(new ResolvedValidityDate.Point(ENTITY_ID)); From 68008939300d2d8627c40cda5a3ef43543313ceb Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 14 Sep 2026 08:28:00 +0200 Subject: [PATCH 38/81] refactor(sql): move logical node composition into connector Compose conjunctions and disjunctions from compiled child steps in the SQL connector, including date aggregation, negation handling, and optional existence projections. Keep traversal of legacy query DTOs in backend adapters and honor an explicit false value for existence projection. Co-authored-by: Codex GPT-5.6-sol --- .../conversion/cqelement/CQAndConverter.java | 44 ++++--- .../conversion/cqelement/CQOrConverter.java | 45 ++++--- .../conversion/model/QueryStepComposer.java | 8 +- .../compiler/ir/LogicalQueryStepCompiler.java | 38 ++++++ .../ir/LogicalQueryStepCompilerTest.java | 113 ++++++++++++++++++ 5 files changed, 194 insertions(+), 54 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/LogicalQueryStepCompiler.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/LogicalQueryStepCompilerTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java index 5ab611ca0e..5379b282f3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java @@ -1,12 +1,20 @@ package com.bakdata.conquery.sql.conversion.cqelement; +import java.util.List; + import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; import com.bakdata.conquery.sql.compiler.ir.JoinMode; +import com.bakdata.conquery.sql.compiler.ir.LogicalQueryStepCompiler; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.select.ExistsSqlSelect; import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; import com.bakdata.conquery.sql.conversion.model.QueryStepComposer; +/** + * Adapts a resolved legacy conjunction to connector-owned logical query-step compilation. + * + *

TODO Remove this adapter once backend query resolution produces connector query nodes directly.

+ */ public class CQAndConverter implements NodeConverter { @Override @@ -16,28 +24,16 @@ public Class getConversionClass() { @Override public ConversionContext convert(CQAnd andNode, ConversionContext context) { - - QueryStep joined; - if (andNode.getChildren().size() == 1) { - ConversionContext withConvertedChild = context.getNodeConversions().convert(andNode.getChildren().get(0), context); - joined = withConvertedChild.getLastConvertedStep(); - } - else { - joined = QueryStepComposer.joinChildren( - andNode.getChildren(), - context, - JoinMode.INNER, - andNode.getDateAction() - ); - } - - if (andNode.getCreateExists().isEmpty()) { - return context.withQueryStep(joined); - } - - String joinedNodeName = joined.getCteName(); - ExistsSqlSelect existsSqlSelect = ExistsSqlSelect.withAlias(joinedNodeName); - return context.withQueryStep(joined.addSqlSelect(existsSqlSelect)); + List children = QueryStepComposer.convertChildren(andNode.getChildren(), context); + QueryStep joined = LogicalQueryStepCompiler.compile( + children, + JoinMode.INNER, + andNode.getDateAction(), + andNode.getCreateExists().orElse(false), + EntitySchemaAdapter.from(context.getIdColumns()), + context.getCompilerDialect(), + context.getNameGenerator() + ); + return context.withQueryStep(joined); } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java index 8c3b939de6..009fbae701 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java @@ -1,12 +1,20 @@ package com.bakdata.conquery.sql.conversion.cqelement; +import java.util.List; + import com.bakdata.conquery.apiv1.query.concept.specific.CQOr; import com.bakdata.conquery.sql.compiler.ir.JoinMode; +import com.bakdata.conquery.sql.compiler.ir.LogicalQueryStepCompiler; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.select.ExistsSqlSelect; import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; import com.bakdata.conquery.sql.conversion.model.QueryStepComposer; +/** + * Adapts a resolved legacy disjunction to connector-owned logical query-step compilation. + * + *

TODO Remove this adapter once backend query resolution produces connector query nodes directly.

+ */ public class CQOrConverter implements NodeConverter { @Override @@ -16,29 +24,16 @@ public Class getConversionClass() { @Override public ConversionContext convert(CQOr orNode, ConversionContext context) { - - QueryStep joined; - if (orNode.getChildren().size() == 1) { - ConversionContext withConvertedChild = context.getNodeConversions().convert(orNode.getChildren().get(0), context); - joined = withConvertedChild.getLastConvertedStep(); - } - else { - joined = QueryStepComposer.joinChildren( - orNode.getChildren(), - context, - JoinMode.FULL_OUTER, - orNode.getDateAction() - ); - } - - if (orNode.getCreateExists().isEmpty()) { - return context.withQueryStep(joined); - } - - String joinedNodeName = joined.getCteName(); - ExistsSqlSelect existsSqlSelect = ExistsSqlSelect.withAlias(joinedNodeName); - return context.withQueryStep(joined.addSqlSelect(existsSqlSelect)); + List children = QueryStepComposer.convertChildren(orNode.getChildren(), context); + QueryStep joined = LogicalQueryStepCompiler.compile( + children, + JoinMode.FULL_OUTER, + orNode.getDateAction(), + orNode.getCreateExists().orElse(false), + EntitySchemaAdapter.from(context.getIdColumns()), + context.getCompilerDialect(), + context.getNameGenerator() + ); + return context.withQueryStep(joined); } - - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java index 9a2266163e..720a88e0d8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java @@ -33,17 +33,15 @@ public static QueryStep antiJoinWithAllIdsTable( ); } - public static QueryStep joinChildren( + public static List convertChildren( Iterable children, - ConversionContext context, - JoinMode logicalOperation, - DateAggregationAction dateAggregationAction + ConversionContext context ) { ConversionContext childrenContext = context.createChildContext(); for (CQElement childNode : children) { childrenContext = context.getNodeConversions().convert(childNode, childrenContext); } - return joinSteps(childrenContext.getQuerySteps(), logicalOperation, dateAggregationAction, context); + return childrenContext.getQuerySteps(); } public static QueryStep joinSteps( diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/LogicalQueryStepCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/LogicalQueryStepCompiler.java new file mode 100644 index 0000000000..61d211b829 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/LogicalQueryStepCompiler.java @@ -0,0 +1,38 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import java.util.List; + +import com.bakdata.conquery.models.query.DateAggregationAction; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.select.ExistsSqlSelect; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import com.bakdata.conquery.sql.model.schema.EntitySchema; +import lombok.experimental.UtilityClass; + +/** Composes compiled child steps for a logical conjunction or disjunction. */ +@UtilityClass +public final class LogicalQueryStepCompiler { + + public static QueryStep compile( + List childSteps, + JoinMode joinMode, + DateAggregationAction dateAggregationAction, + boolean createExists, + EntitySchema entitySchema, + CompilerDialect compilerDialect, + SqlNameGenerator nameGenerator + ) { + QueryStep joined = QueryStepComposer.joinSteps( + childSteps, + joinMode, + dateAggregationAction, + entitySchema, + compilerDialect, + nameGenerator + ); + if (!createExists) { + return joined; + } + return joined.addSqlSelect(ExistsSqlSelect.withAlias(joined.getCteName())); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/LogicalQueryStepCompilerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/LogicalQueryStepCompilerTest.java new file mode 100644 index 0000000000..062ad556c2 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/LogicalQueryStepCompilerTest.java @@ -0,0 +1,113 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; + +import java.sql.Date; +import java.util.List; + +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.models.query.DateAggregationAction; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import com.bakdata.conquery.sql.model.schema.EntitySchema; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.SqlTable; +import org.jooq.Field; +import org.junit.jupiter.api.Test; + +class LogicalQueryStepCompilerTest { + + private static final CompilerDialect DIALECT = new TestDialect(); + private static final SqlTable ENTITY_TABLE = SqlTable.of("entities", "catalog", "entities"); + private static final EntitySchema ENTITY_SCHEMA = new EntitySchema( + ENTITY_TABLE, + new ResolvedColumn("person", ENTITY_TABLE, "person_id", ColumnType.STRING, false) + ); + + @Test + void shouldReturnSingleChildWithoutExistsProjection() { + QueryStep child = queryStep("child"); + + QueryStep compiled = compile(List.of(child), JoinMode.INNER, false); + + assertSame(child, compiled); + } + + @Test + void shouldAddExistsProjectionForSingleChild() { + QueryStep compiled = compile(List.of(queryStep("child")), JoinMode.INNER, true); + + assertEquals(1, compiled.getSelects().getSqlSelects().size()); + assertEquals("child", compiled.getSelects().explicitSelects().getFirst().getName()); + } + + @Test + void shouldComposeDisjunctionBeforeAddingExistsProjection() { + QueryStep compiled = compile( + List.of(queryStep("first"), queryStep("second")), + JoinMode.FULL_OUTER, + true + ); + + assertEquals("OR-1", compiled.getCteName()); + assertEquals(2, compiled.getPredecessors().size()); + assertEquals("OR-1", compiled.getSelects().explicitSelects().getFirst().getName()); + } + + private static QueryStep compile(List children, JoinMode joinMode, boolean createExists) { + return LogicalQueryStepCompiler.compile( + children, + joinMode, + DateAggregationAction.BLOCK, + createExists, + ENTITY_SCHEMA, + DIALECT, + new SqlNameGenerator(128) + ); + } + + private static QueryStep queryStep(String cteName) { + return QueryStep.builder() + .cteName(cteName) + .selects(Selects.builder() + .ids(new SqlIdColumns(field(name("person_id"), String.class))) + .build()) + .build(); + } + + private static final class TestDialect implements CompilerDialect { + + @Override + public Field minimumDate() { + return field(name("minimum_date"), Date.class); + } + + @Override + public Field maximumDate() { + return field(name("maximum_date"), Date.class); + } + + @Override + public Field anyValue(Field value) { + return value; + } + + @Override + public Field renderDateRange(Field start, Field end) { + return field(name("rendered_range"), Object.class); + } + + @Override + public Field aggregateDateRanges(Field start, Field end) { + return field(name("aggregated_ranges"), Object.class); + } + + @Override + public int getNameMaxLength() { + return 128; + } + } +} From 41285f57ccafd5db1f36b66f8041c3e684415d0d Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 14 Sep 2026 08:46:21 +0200 Subject: [PATCH 39/81] refactor(sql): mark utility-only classes explicitly Use Lombok utility classes for connector types that expose only static model helpers, compiler operations, and rendering functions. Remove their manual private constructors so the non-instantiable intent is declared consistently. Co-authored-by: Codex GPT-5.6-sol --- .../sql/compiler/ir/AllEntitiesQueryStepCompiler.java | 5 ++--- .../bakdata/conquery/sql/compiler/ir/EntitySchemaSql.java | 5 ++--- .../conquery/sql/compiler/ir/ExternalQueryStepCompiler.java | 5 ++--- .../conquery/sql/compiler/ir/FinalQueryStepComposer.java | 5 ++--- .../bakdata/conquery/sql/compiler/ir/QualifyingUtil.java | 5 ++--- .../bakdata/conquery/sql/compiler/ir/QueryStepComposer.java | 5 ++--- .../bakdata/conquery/sql/compiler/ir/QueryStepJoiner.java | 5 ++--- .../sql/compiler/rendering/SelectProjectionRenderer.java | 5 ++--- .../conquery/sql/model/internal/ModelNormalization.java | 6 +++--- .../conquery/sql/model/operation/BuiltInAggregations.java | 5 ++--- .../conquery/sql/model/operation/BuiltInConditions.java | 5 ++--- .../conquery/sql/model/operation/BuiltInFilters.java | 5 ++--- .../conquery/sql/model/operation/BuiltInSelects.java | 5 ++--- 13 files changed, 27 insertions(+), 39 deletions(-) diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/AllEntitiesQueryStepCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/AllEntitiesQueryStepCompiler.java index 6bf9399239..87d935f35c 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/AllEntitiesQueryStepCompiler.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/AllEntitiesQueryStepCompiler.java @@ -6,15 +6,14 @@ import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import com.bakdata.conquery.sql.model.node.AllEntitiesNode; import com.bakdata.conquery.sql.model.schema.EntitySchema; +import lombok.experimental.UtilityClass; /** Compiles the resolved entity schema into the query step used to select every entity. */ +@UtilityClass public final class AllEntitiesQueryStepCompiler { private static final String ALL_ENTITIES_CTE = "all_ids"; - private AllEntitiesQueryStepCompiler() { - } - public static QueryStep compile( AllEntitiesNode allEntitiesNode, EntitySchema entitySchema, diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/EntitySchemaSql.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/EntitySchemaSql.java index f56e458c01..bb822b485f 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/EntitySchemaSql.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/EntitySchemaSql.java @@ -7,6 +7,7 @@ import java.util.List; import com.bakdata.conquery.sql.model.schema.EntitySchema; +import lombok.experimental.UtilityClass; import org.jooq.Field; import org.jooq.Name; import org.jooq.Record; @@ -14,11 +15,9 @@ import org.jooq.impl.DSL; /** Creates SQL identifiers for a resolved entity schema. */ +@UtilityClass final class EntitySchemaSql { - private EntitySchemaSql() { - } - static Table table(EntitySchema entitySchema) { return DSL.table(tableName(entitySchema)); } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/ExternalQueryStepCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/ExternalQueryStepCompiler.java index 7132ecda46..f81aba0a7c 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/ExternalQueryStepCompiler.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/ExternalQueryStepCompiler.java @@ -15,18 +15,17 @@ import com.bakdata.conquery.sql.model.node.ExternalEntity; import com.bakdata.conquery.sql.model.node.ExternalNode; import com.bakdata.conquery.sql.model.range.DateRange; +import lombok.experimental.UtilityClass; import org.jooq.Field; import org.jooq.Name; /** Compiles resolved external entity values into literal-backed query-step IR. */ +@UtilityClass public final class ExternalQueryStepCompiler { private static final String EXTERNAL_IDS_CTE_NAME = "external_ids"; private static final String EXTERNAL_VALUES_CTE_NAME = "external_extra"; - private ExternalQueryStepCompiler() { - } - public static ExternalQuerySteps compile( ExternalNode externalNode, boolean negate, diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/FinalQueryStepComposer.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/FinalQueryStepComposer.java index 15e8157d4d..6a29a46309 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/FinalQueryStepComposer.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/FinalQueryStepComposer.java @@ -8,16 +8,15 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import lombok.experimental.UtilityClass; import org.jooq.Field; import org.jooq.Record; import org.jooq.TableLike; /** Builds the final aggregated query-step projection from compiled connector IR. */ +@UtilityClass public final class FinalQueryStepComposer { - private FinalQueryStepComposer() { - } - /** * Compose a final, non-CTE query step. * diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QualifyingUtil.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QualifyingUtil.java index 6c593d3253..7202750993 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QualifyingUtil.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QualifyingUtil.java @@ -4,15 +4,14 @@ import java.util.stream.Collectors; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import lombok.experimental.UtilityClass; import org.jooq.Field; import org.jooq.impl.DSL; /** Utilities for rebinding compiler IR fields and selects to a table or CTE qualifier. */ +@UtilityClass public final class QualifyingUtil { - private QualifyingUtil() { - } - public static Field qualify(Field field, String qualifier) { return DSL.field(DSL.name(qualifier, field.getName()), field.getType()); } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposer.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposer.java index de9807fdc6..2b96658b47 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposer.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposer.java @@ -16,6 +16,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; import com.bakdata.conquery.sql.model.schema.EntitySchema; +import lombok.experimental.UtilityClass; import org.jooq.Condition; import org.jooq.Field; import org.jooq.Record; @@ -24,13 +25,11 @@ import org.jooq.impl.DSL; /** Composes query-step IR, including negation and date aggregation, from resolved compiler inputs. */ +@UtilityClass public final class QueryStepComposer { private static final String NEGATED_CTE_SUFFIX = "_negated"; - private QueryStepComposer() { - } - /** Implements an anti-join against the entity schema used to enumerate all known entities. */ public static QueryStep antiJoinWithAllEntities( QueryStep queryStep, diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepJoiner.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepJoiner.java index c957f93eda..bee249fe4b 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepJoiner.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepJoiner.java @@ -10,6 +10,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import lombok.experimental.UtilityClass; import org.jooq.Condition; import org.jooq.Record; import org.jooq.Table; @@ -17,11 +18,9 @@ import org.jooq.TableOnConditionStep; /** Combines compiler query-step IR without applying backend query semantics. */ +@UtilityClass public final class QueryStepJoiner { - private QueryStepJoiner() { - } - /** * Join the CTE tables represented by the supplied steps on their IDs and, where present, stratification dates. */ diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRenderer.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRenderer.java index 916c18982c..db3f629633 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRenderer.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/rendering/SelectProjectionRenderer.java @@ -11,14 +11,13 @@ import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import lombok.experimental.UtilityClass; import org.jooq.Field; /** Renders a select projection from compiler IR into database-specific physical fields. */ +@UtilityClass public final class SelectProjectionRenderer { - private SelectProjectionRenderer() { - } - /** * Render the final projection fields. * diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/internal/ModelNormalization.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/internal/ModelNormalization.java index 1f311ab6e8..d7b1bf13b8 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/internal/ModelNormalization.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/internal/ModelNormalization.java @@ -8,10 +8,10 @@ import java.util.Map; import java.util.Set; -public final class ModelNormalization { +import lombok.experimental.UtilityClass; - private ModelNormalization() { - } +@UtilityClass +public final class ModelNormalization { public static List immutableCopy(List values) { return values == null ? null : Collections.unmodifiableList(new ArrayList<>(values)); diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInAggregations.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInAggregations.java index 849378f558..48160098e3 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInAggregations.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInAggregations.java @@ -16,13 +16,12 @@ import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; +import lombok.experimental.UtilityClass; /** Framework-neutral aggregation operations supported by the SQL connector. */ +@UtilityClass public final class BuiltInAggregations { - private BuiltInAggregations() { - } - public record Count( @NotNull @Valid ResolvedColumn column, @NotNull List<@NotNull @Valid ResolvedColumn> distinctBy diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInConditions.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInConditions.java index af4e34591b..b9da183134 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInConditions.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInConditions.java @@ -12,13 +12,12 @@ import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; +import lombok.experimental.UtilityClass; /** Declarative connector and concept-element conditions supported by the SQL connector. */ +@UtilityClass public final class BuiltInConditions { - private BuiltInConditions() { - } - public record AllOf( @NotEmpty List<@NotNull @Valid ResolvedCondition> conditions ) implements ResolvedCondition { diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInFilters.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInFilters.java index 68eee9c3e0..ae5e706825 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInFilters.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInFilters.java @@ -16,13 +16,12 @@ import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; +import lombok.experimental.UtilityClass; /** Framework-neutral filter operations supported by the SQL connector. */ +@UtilityClass public final class BuiltInFilters { - private BuiltInFilters() { - } - public record StringValues( @NotBlank String name, @NotNull @Valid @AllowedColumnTypes(ColumnType.STRING) ResolvedColumn column, diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInSelects.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInSelects.java index 3b38dfaa68..378f8fe089 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInSelects.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInSelects.java @@ -16,13 +16,12 @@ import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; +import lombok.experimental.UtilityClass; /** Framework-neutral select operations supported by the SQL connector. */ +@UtilityClass public final class BuiltInSelects { - private BuiltInSelects() { - } - public enum ValueOperation { DISTINCT, FIRST, From 03533b6d4b2528ecfd435aa0ed2a711b263fa0aa Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 14 Sep 2026 09:00:18 +0200 Subject: [PATCH 40/81] refactor(sql): move concept CTE steps into connector Move the concept compilation CTE vocabulary into the connector's internal IR package. Keep the legacy backend conversion pipeline as a consumer while the remaining concept compiler implementation is extracted incrementally. Co-authored-by: Codex GPT-5.6-sol --- .../cqelement/concept/AggregationFilterCte.java | 1 + .../cqelement/concept/AggregationSelectCte.java | 1 + .../conversion/cqelement/concept/CQConceptConverter.java | 1 + .../sql/conversion/cqelement/concept/ConnectorCte.java | 1 + .../cqelement/concept/IntervalPackingSelectsCte.java | 1 + .../sql/conversion/cqelement/concept/JoinBranchesCte.java | 1 + .../conversion/cqelement/concept/PreprocessingCte.java | 1 + .../sql/conversion/cqelement/concept/TablePath.java | 2 +- .../clickhouse/ClickhouseDistinctSelectConverter.java | 2 +- .../model/aggregator/CountQuartersSqlAggregator.java | 2 +- .../conversion/model/aggregator/CountSqlAggregator.java | 2 +- .../model/aggregator/DateDistanceSqlAggregator.java | 2 +- .../conversion/model/aggregator/FlagSqlAggregator.java | 2 +- .../sql/conversion/model/aggregator/SumSqlAggregator.java | 2 +- .../model/filter/AbstractSelectFilterConverter.java | 2 +- .../model/select/ConceptColumnSelectConverter.java | 2 +- .../sql/conversion/model/select/DaterangeSelectUtil.java | 4 ++-- .../conversion/model/select/DistinctSelectConverter.java | 2 +- .../model/select/EventDateUnionSelectConverter.java | 2 +- .../model/select/EventDurationSumSelectConverter.java | 2 +- .../model/select/RandomValueSelectConverter.java | 2 +- .../sql/conversion/model/select/ValueSelectUtil.java | 2 +- .../conquery/sql/compiler/ir}/concept/ConceptCteStep.java | 8 ++------ .../conquery/sql/compiler/ir/concept/package-info.java | 2 ++ 24 files changed, 27 insertions(+), 22 deletions(-) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir}/concept/ConceptCteStep.java (84%) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/package-info.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java index cc37fe6791..7af324215b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java @@ -5,6 +5,7 @@ import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java index d15cd80281..66c87104fa 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java @@ -8,6 +8,7 @@ import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 34a3ecc932..6683c3c364 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -4,6 +4,7 @@ import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.QueryStepJoiner; import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import java.util.ArrayList; import java.util.List; import java.util.Optional; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java index 62003cdbeb..99eb4e3ccc 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java @@ -5,6 +5,7 @@ import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; abstract class ConnectorCte { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java index 8df5197f00..975c10a411 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java @@ -12,6 +12,7 @@ import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.model.select.ConceptSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.google.common.base.Preconditions; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java index dec50dcca4..b2f36f2c4b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.sql.compiler.ir.QueryStepJoiner; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.aggregator.SumSqlAggregator; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java index 0b3960a687..700f3bcda0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java index bcdb846abc..bf5762d882 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java @@ -1,6 +1,6 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; -import static com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep.*; +import static com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep.*; import static com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep.INTERVAL_COMPLETE; import java.util.HashMap; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java index 91f37055ca..fc600a2d2c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java @@ -4,7 +4,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.connector.DistinctSelect; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java index 7367e58bae..ca483bf27a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.models.datasets.concepts.filters.specific.CountQuartersFilter; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.CountQuartersSelect; import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.dialect.Interval; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java index 71feafd9ee..28d27bfdc9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java @@ -4,7 +4,7 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.filters.specific.CountFilter; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.CountSelect; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java index 2b01ec23b9..7ec085c7f5 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java @@ -10,7 +10,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java index 9d53974230..07bb9ef9ff 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java @@ -12,7 +12,7 @@ import com.bakdata.conquery.models.datasets.concepts.filters.specific.FlagFilter; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.FlagSelect; import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java index c2e2cbafc9..6b9b48e2e5 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java @@ -11,7 +11,7 @@ import com.bakdata.conquery.models.datasets.concepts.filters.specific.SumFilter; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.SumSelect; import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.compiler.ir.CteStep; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java index 7e59ebace9..b8ae29ef89 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java @@ -5,7 +5,7 @@ import com.bakdata.conquery.sql.compiler.ir.condition.StringValuesCondition; import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java index 82c70b9d55..0c39d49d0f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java @@ -10,7 +10,7 @@ import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptIdMapping; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java index da40bff687..e1f14b3edb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java @@ -1,6 +1,6 @@ package com.bakdata.conquery.sql.conversion.model.select; -import static com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep.*; +import static com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep.*; import static com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep.INTERVAL_COMPLETE; import static org.jooq.impl.DSL.*; @@ -21,7 +21,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.sql.conversion.Context; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.cqelement.concept.IntervalPackingSelectsCte; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java index a4dd792bfe..cdc444f675 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.CteStep; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java index dc467b5727..29f9f6cd46 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java @@ -3,7 +3,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDateUnionSelect; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java index ce23d9f028..6f829b8ccb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDurationSumSelect; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java index 8e20fc6bb2..a96c69eac8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java @@ -3,7 +3,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.connector.RandomValueSelect; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java index 759988f1c6..1a3b322fe5 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java @@ -12,7 +12,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteStep.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteStep.java similarity index 84% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteStep.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteStep.java index 68d71d0f9c..48364c2187 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteStep.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteStep.java @@ -1,5 +1,4 @@ - -package com.bakdata.conquery.sql.conversion.cqelement.concept; +package com.bakdata.conquery.sql.compiler.ir.concept; import java.util.Set; @@ -7,21 +6,19 @@ import lombok.AllArgsConstructor; import lombok.Getter; +/** Defines the connector-internal CTE graph used to compile a resolved concept node. */ @Getter @AllArgsConstructor public enum ConceptCteStep implements CteStep { - // connector PREPROCESSING("preprocessing", null), AGGREGATION_SELECT("group_select", PREPROCESSING), JOIN_BRANCHES("join_branches", AGGREGATION_SELECT), AGGREGATION_FILTER("group_filter", JOIN_BRANCHES), - // interval packing selects UNNEST_DATE("unnested", null), INTERVAL_PACKING_SELECTS("interval_packing_selects", null), - // universal selects / final step UNIVERSAL_SELECTS("universal_selects", null); public static final Set MANDATORY_STEPS = Set.of( @@ -33,5 +30,4 @@ public enum ConceptCteStep implements CteStep { private final String suffix; private final CteStep predecessor; - } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/package-info.java new file mode 100644 index 0000000000..e8caee045c --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/package-info.java @@ -0,0 +1,2 @@ +/** Internal CTE vocabulary and intermediate representations used while compiling resolved concept nodes. */ +package com.bakdata.conquery.sql.compiler.ir.concept; From 97643931934347657003bc2d2f6739f03440cfb7 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 14 Sep 2026 09:08:53 +0200 Subject: [PATCH 41/81] refactor(sql): move concept compilation state into connector Move connector and concept select bundles together with staged filter output into the connector's internal concept IR package. Keep legacy converters as consumers while concept compilation is extracted incrementally. Co-authored-by: Codex GPT-5.6-sol --- .../apiv1/query/concept/filter/FilterValue.java | 2 +- .../cqelement/concept/CQConceptConverter.java | 6 +++--- .../cqelement/concept/CQTableContext.java | 4 ++-- .../concept/IntervalPackingSelectsCte.java | 2 +- .../cqelement/concept/PreprocessingCte.java | 2 +- .../ClickhouseDistinctSelectConverter.java | 2 +- .../model/aggregator/CommonAggregationSelect.java | 2 +- .../aggregator/CountQuartersSqlAggregator.java | 4 ++-- .../model/aggregator/CountSqlAggregator.java | 4 ++-- .../aggregator/DateDistanceSqlAggregator.java | 2 ++ .../model/aggregator/DurationSumSqlAggregator.java | 4 ++-- .../model/aggregator/FlagSqlAggregator.java | 4 ++-- .../model/aggregator/SumSqlAggregator.java | 4 ++-- .../filter/AbstractSelectFilterConverter.java | 3 ++- .../conversion/model/filter/FilterConverter.java | 1 + .../model/filter/NumberFilterConverter.java | 3 ++- .../model/select/ConceptColumnSelectConverter.java | 2 ++ .../model/select/DateUnionSelectConverter.java | 1 + .../model/select/DaterangeSelectUtil.java | 3 ++- .../model/select/DistinctSelectConverter.java | 1 + .../select/EventDateUnionSelectConverter.java | 2 ++ .../select/EventDurationSumSelectConverter.java | 2 ++ .../model/select/ExistsSelectConverter.java | 2 ++ .../model/select/FirstValueSelectConverter.java | 1 + .../model/select/LastValueSelectConverter.java | 1 + .../model/select/RandomValueSelectConverter.java | 1 + .../conversion/model/select/SelectConverter.java | 2 ++ .../conversion/model/select/ValueSelectUtil.java | 1 + .../compiler/ir/concept}/ConceptSqlSelects.java | 6 +++--- .../compiler/ir/concept}/ConnectorSqlSelects.java | 14 +++++++------- .../sql/compiler/ir/concept}/SqlFilters.java | 4 ++-- 31 files changed, 57 insertions(+), 35 deletions(-) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept}/ConceptSqlSelects.java (76%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept}/ConnectorSqlSelects.java (61%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept}/SqlFilters.java (53%) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java index 2e8ecfe384..5b34a5cf90 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java @@ -19,7 +19,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; +import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 6683c3c364..d67aa33516 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -33,11 +33,11 @@ import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.*; -import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; +import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; -import com.bakdata.conquery.sql.conversion.model.select.ConceptSqlSelects; -import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.util.TablePrimaryColumnUtil; import com.google.common.base.Preconditions; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java index f4614e5142..e0a43df8fa 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java @@ -12,8 +12,8 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; -import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; import lombok.Builder; import lombok.Value; import lombok.With; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java index 975c10a411..c5e85561ee 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; -import com.bakdata.conquery.sql.conversion.model.select.ConceptSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.google.common.base.Preconditions; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java index 700f3bcda0..237479421d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java @@ -12,7 +12,7 @@ import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; -import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; +import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java index fc600a2d2c..e4d15e5fd6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; -import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CommonAggregationSelect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CommonAggregationSelect.java index dd9fa6d2eb..fda6cee7ea 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CommonAggregationSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CommonAggregationSelect.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java index ca483bf27a..e263adc670 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java @@ -17,9 +17,9 @@ import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.bakdata.conquery.sql.conversion.model.filter.LegacyInclusiveRangeCondition; -import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; +import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; -import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java index 28d27bfdc9..b0998cf3d8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java @@ -9,9 +9,9 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.bakdata.conquery.sql.conversion.model.filter.LegacyInclusiveRangeCondition; -import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; +import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; -import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java index 7ec085c7f5..94eac1dd1a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java @@ -5,6 +5,8 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.filters.specific.DateDistanceFilter; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DateDistanceSelect; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DurationSumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DurationSumSqlAggregator.java index 3cb5de1bb1..98f791508a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DurationSumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DurationSumSqlAggregator.java @@ -16,8 +16,8 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.bakdata.conquery.sql.conversion.model.filter.LegacyInclusiveRangeCondition; -import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; -import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; import com.bakdata.conquery.sql.conversion.model.select.DaterangeSelectUtil; import com.bakdata.conquery.sql.conversion.model.select.DaterangeSelectUtil.AggregationFunction; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java index 07bb9ef9ff..619b2fef12 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java @@ -19,9 +19,9 @@ import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.bakdata.conquery.sql.compiler.ir.condition.FlagCondition; -import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; +import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; -import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java index 6b9b48e2e5..1cb7c0d442 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java @@ -23,9 +23,9 @@ import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.bakdata.conquery.sql.conversion.model.filter.LegacyInclusiveRangeCondition; -import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; +import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; -import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java index b8ae29ef89..8f290e6d56 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java @@ -7,7 +7,8 @@ import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; -import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import org.jooq.Condition; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/FilterConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/FilterConverter.java index 6ed05d1527..dbadec21fc 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/FilterConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/FilterConverter.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.sql.conversion.model.filter; import com.bakdata.conquery.models.datasets.concepts.filters.Filter; +import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java index 117956f6f9..00798b03fe 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java @@ -7,7 +7,8 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.model.NumberMapUtil; -import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import org.jooq.Condition; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java index 0c39d49d0f..6c3fe3ccf3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java @@ -8,6 +8,8 @@ import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.select.concept.ConceptColumnSelect; import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java index 31f0c94bdb..36c0eb093d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.sql.conversion.model.select; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DateUnionSelect; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java index e1f14b3edb..42055f48cc 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java @@ -17,6 +17,7 @@ import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter; import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.identifiable.LabeledNamespaceIdentifiable; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; @@ -34,7 +35,7 @@ import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.SqlTables; -import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; +import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import org.jooq.Condition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java index cdc444f675..75711c33e1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java @@ -5,6 +5,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.connector.DistinctSelect; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java index 29f9f6cd46..b6cfbe3ee1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java @@ -1,6 +1,8 @@ package com.bakdata.conquery.sql.conversion.model.select; import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDateUnionSelect; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java index 6f829b8ccb..9421c33ad3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java @@ -5,6 +5,8 @@ import java.time.temporal.ChronoUnit; import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDurationSumSelect; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java index 28c64b432c..9d0c7049c8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java @@ -1,6 +1,8 @@ package com.bakdata.conquery.sql.conversion.model.select; import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.ExistsSelect; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.ExistsSqlSelect; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java index a8c120df13..819bcf7337 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.sql.conversion.model.select; import com.bakdata.conquery.models.datasets.concepts.select.connector.FirstValueSelect; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java index c955232b8f..66b25d136a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.sql.conversion.model.select; import com.bakdata.conquery.models.datasets.concepts.select.connector.LastValueSelect; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java index a96c69eac8..0ab99f2952 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.sql.conversion.model.select; import com.bakdata.conquery.models.datasets.concepts.select.connector.RandomValueSelect; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectConverter.java index 3e8a8bfb29..f17d8e0863 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectConverter.java @@ -1,6 +1,8 @@ package com.bakdata.conquery.sql.conversion.model.select; import com.bakdata.conquery.models.datasets.concepts.select.Select; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java index 1a3b322fe5..2708cdca6e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java @@ -9,6 +9,7 @@ import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptSqlSelects.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptSqlSelects.java similarity index 76% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptSqlSelects.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptSqlSelects.java index d8cf3fb3c9..171a0b38ec 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptSqlSelects.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptSqlSelects.java @@ -1,14 +1,15 @@ -package com.bakdata.conquery.sql.conversion.model.select; +package com.bakdata.conquery.sql.compiler.ir.concept; import java.util.List; import java.util.Optional; -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Builder; import lombok.Singular; import lombok.Value; +/** Groups final concept selects and any auxiliary query step needed to calculate them. */ @Value @Builder public class ConceptSqlSelects { @@ -21,5 +22,4 @@ public class ConceptSqlSelects { @Singular List finalSelects; - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConnectorSqlSelects.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorSqlSelects.java similarity index 61% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConnectorSqlSelects.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorSqlSelects.java index da9eeb060e..5332f29a3c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConnectorSqlSelects.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorSqlSelects.java @@ -1,14 +1,15 @@ -package com.bakdata.conquery.sql.conversion.model.select; +package com.bakdata.conquery.sql.compiler.ir.concept; import java.util.List; import java.util.Optional; -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.Builder; import lombok.Singular; import lombok.Value; +/** Groups select expressions by the concept-connector CTE phase that consumes them. */ @Value @Builder public class ConnectorSqlSelects { @@ -16,24 +17,23 @@ public class ConnectorSqlSelects { @Singular List preprocessingSelects; - // Empty if only used for event filtering during preprocessing + /** Empty when the select is used only for event filtering during preprocessing. */ @Singular List aggregationSelects; - // Selects that are applied on the aggregated validity date. + /** Selects applied to the aggregated validity date. */ @Singular List eventDateSelects; - // Empty if only used in aggregation select + /** Empty when the select is used only during aggregation. */ @Singular List finalSelects; - // An additional predecessor these SqlSelects require + /** An additional query-step predecessor required by these selects. */ @Builder.Default Optional additionalPredecessor = Optional.empty(); public static ConnectorSqlSelects none() { return ConnectorSqlSelects.builder().build(); } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/SqlFilters.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/SqlFilters.java similarity index 53% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/SqlFilters.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/SqlFilters.java index 4cd4e4d41b..98398ff0de 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/SqlFilters.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/SqlFilters.java @@ -1,9 +1,9 @@ -package com.bakdata.conquery.sql.conversion.model.filter; +package com.bakdata.conquery.sql.compiler.ir.concept; import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; -import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; import lombok.Value; +/** Carries select expressions and staged where clauses produced while compiling a resolved filter. */ @Value public class SqlFilters { ConnectorSqlSelects selects; From 8680840dd0e42fe0f084fd44403690bd348e454e Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 14 Sep 2026 10:20:58 +0200 Subject: [PATCH 42/81] refactor(sql): move aggregation select state into connector Move the shared aggregation projection state into the connector's internal concept IR package. Remove its legacy model references and keep the backend aggregation converters as consumers during the incremental extraction. Co-authored-by: Codex GPT-5.6-sol --- .../aggregator/CommonAggregationSelect.java | 40 ------------------- .../CountQuartersSqlAggregator.java | 1 + .../model/aggregator/CountSqlAggregator.java | 1 + .../model/aggregator/SumSqlAggregator.java | 1 + .../ir/concept/CommonAggregationSelect.java | 32 +++++++++++++++ 5 files changed, 35 insertions(+), 40 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CommonAggregationSelect.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/CommonAggregationSelect.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CommonAggregationSelect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CommonAggregationSelect.java deleted file mode 100644 index fda6cee7ea..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CommonAggregationSelect.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model.aggregator; - -import java.util.List; -import java.util.Optional; - -import com.bakdata.conquery.models.datasets.concepts.filters.Filter; -import com.bakdata.conquery.models.datasets.concepts.select.Select; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; -import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; -import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import lombok.Builder; -import lombok.Singular; -import lombok.Value; - -/** - * Container object for parts of the {@link ConnectorSqlSelects}. - *

- * {@link Select}s and {@link Filter}s like COUNT and SUM share the majority of their {@link ConnectorSqlSelects} when being converted. This container makes - * the respective shared {@link SqlSelect}s and the optional additional preceding {@link QueryStep} accessible for building {@link ConnectorSqlSelects} on - * demand. - * - * @param The type parameter of the aggregation select. - */ -@Value -@Builder -class CommonAggregationSelect { - - @Singular - List rootSelects; - - FieldWrapper groupBy; - - QueryStep additionalPredecessor; - - public Optional getAdditionalPredecessor() { - return Optional.ofNullable(additionalPredecessor); - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java index e263adc670..85100b35a4 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java @@ -9,6 +9,7 @@ import com.bakdata.conquery.models.datasets.concepts.filters.specific.CountQuartersFilter; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.CountQuartersSelect; import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; +import com.bakdata.conquery.sql.compiler.ir.concept.CommonAggregationSelect; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java index b0998cf3d8..c132e941be 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java @@ -4,6 +4,7 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.filters.specific.CountFilter; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.CountSelect; +import com.bakdata.conquery.sql.compiler.ir.concept.CommonAggregationSelect; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java index 1cb7c0d442..82a474609a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.models.datasets.concepts.filters.specific.SumFilter; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.SumSelect; import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; +import com.bakdata.conquery.sql.compiler.ir.concept.CommonAggregationSelect; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/CommonAggregationSelect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/CommonAggregationSelect.java new file mode 100644 index 0000000000..1c8e65d76f --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/CommonAggregationSelect.java @@ -0,0 +1,32 @@ +package com.bakdata.conquery.sql.compiler.ir.concept; + +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; +import lombok.Builder; +import lombok.Singular; +import lombok.Value; + +/** + * Intermediate projections shared by aggregate select and filter compilation. + * + * @param grouped aggregation result type + */ +@Value +@Builder +public class CommonAggregationSelect { + + @Singular + List rootSelects; + + FieldWrapper groupBy; + + QueryStep additionalPredecessor; + + public Optional getAdditionalPredecessor() { + return Optional.ofNullable(additionalPredecessor); + } +} From 77bf1da52219d092036bc688b1c1ef8b2c11400d Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 14 Sep 2026 10:33:51 +0200 Subject: [PATCH 43/81] refactor(sql): move interval vocabulary into connector Move the SQL calendar interval amounts into the connector's dialect package. Keep legacy form stratification, aggregation, and database-specific renderers as consumers while their implementations are extracted incrementally. Co-authored-by: Codex GPT-5.6-sol --- .../clickhouse/ClickhouseStratificationFunctions.java | 4 ++-- .../dialect/hana/HanaStratificationFunctions.java | 4 ++-- .../sql/conversion/forms/RelativeStratification.java | 2 +- .../sql/conversion/forms/StratificationFunctions.java | 8 ++++---- .../model/aggregator/CountQuartersSqlAggregator.java | 2 +- .../bakdata/conquery/sql/compiler}/dialect/Interval.java | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler}/dialect/Interval.java (80%) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java index 1066555a3f..2ab01b6053 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java @@ -1,6 +1,6 @@ package com.bakdata.conquery.sql.conversion.dialect.clickhouse; -import static com.bakdata.conquery.sql.conversion.dialect.Interval.MONTHS_PER_QUARTER; +import static com.bakdata.conquery.sql.compiler.dialect.Interval.MONTHS_PER_QUARTER; import static com.bakdata.conquery.sql.conversion.forms.FormConstants.SERIES_INDEX; import static org.jooq.impl.DSL.*; import static org.jooq.impl.DSL.inline; @@ -9,7 +9,7 @@ import java.time.temporal.ChronoUnit; import com.bakdata.conquery.apiv1.query.TemporalSamplerFactory; -import com.bakdata.conquery.sql.conversion.dialect.Interval; +import com.bakdata.conquery.sql.compiler.dialect.Interval; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.forms.Offset; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java index cc82c62fee..f5559c7024 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java @@ -1,12 +1,12 @@ package com.bakdata.conquery.sql.conversion.dialect.hana; -import static com.bakdata.conquery.sql.conversion.dialect.Interval.MONTHS_PER_QUARTER; +import static com.bakdata.conquery.sql.compiler.dialect.Interval.MONTHS_PER_QUARTER; import java.sql.Date; import java.time.temporal.ChronoUnit; import com.bakdata.conquery.apiv1.query.TemporalSamplerFactory; -import com.bakdata.conquery.sql.conversion.dialect.Interval; +import com.bakdata.conquery.sql.compiler.dialect.Interval; import com.bakdata.conquery.sql.conversion.forms.Offset; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java index 2b398664d0..0bd70ac372 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java @@ -20,7 +20,7 @@ import com.bakdata.conquery.models.forms.util.Resolution; import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.Interval; +import com.bakdata.conquery.sql.compiler.dialect.Interval; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QueryStep; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java index 2b56a34f1e..dcbf8cbc21 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java @@ -1,8 +1,8 @@ package com.bakdata.conquery.sql.conversion.forms; -import static com.bakdata.conquery.sql.conversion.dialect.Interval.DAYS_PER_QUARTER; -import static com.bakdata.conquery.sql.conversion.dialect.Interval.DAYS_PER_YEAR; -import static com.bakdata.conquery.sql.conversion.dialect.Interval.MONTHS_PER_QUARTER; +import static com.bakdata.conquery.sql.compiler.dialect.Interval.DAYS_PER_QUARTER; +import static com.bakdata.conquery.sql.compiler.dialect.Interval.DAYS_PER_YEAR; +import static com.bakdata.conquery.sql.compiler.dialect.Interval.MONTHS_PER_QUARTER; import static com.bakdata.conquery.sql.conversion.forms.FormConstants.DAY_ALIGNED_COUNT; import static com.bakdata.conquery.sql.conversion.forms.FormConstants.INDEX_SELECTOR; import static com.bakdata.conquery.sql.conversion.forms.FormConstants.INDEX_START; @@ -27,7 +27,7 @@ import com.bakdata.conquery.apiv1.query.TemporalSamplerFactory; import com.bakdata.conquery.models.forms.util.CalendarUnit; import com.bakdata.conquery.sql.compiler.ir.SharedAliases; -import com.bakdata.conquery.sql.conversion.dialect.Interval; +import com.bakdata.conquery.sql.compiler.dialect.Interval; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java index 85100b35a4..72a2b92abb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; -import com.bakdata.conquery.sql.conversion.dialect.Interval; +import com.bakdata.conquery.sql.compiler.dialect.Interval; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/Interval.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/Interval.java similarity index 80% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/Interval.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/Interval.java index 20eece3642..f89b27761d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/Interval.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/Interval.java @@ -1,8 +1,9 @@ -package com.bakdata.conquery.sql.conversion.dialect; +package com.bakdata.conquery.sql.compiler.dialect; import lombok.Getter; import lombok.RequiredArgsConstructor; +/** Calendar interval amounts used by dialect-specific SQL date calculations. */ @Getter @RequiredArgsConstructor public enum Interval { @@ -18,5 +19,4 @@ public enum Interval { public static final int MONTHS_PER_QUARTER = Interval.QUARTER_INTERVAL.getAmount(); private final int amount; - } From b8b912de78163ea35bba45d0d7bf423223de44de Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:07:28 +0200 Subject: [PATCH 44/81] refactor(sql): move form compilation vocabulary into connector Move shared form fields and interval offsets into the connector's internal representation. Keep legacy form converters and dialect implementations as consumers while their compilation logic is extracted incrementally. Co-authored-by: Codex GPT-5.6-sol --- .../ClickhouseStratificationFunctions.java | 4 +- .../hana/HanaStratificationFunctions.java | 2 +- .../sql/conversion/forms/FormConstants.java | 92 ------------------- .../forms/RelativeStratification.java | 8 +- .../forms/StratificationFunctions.java | 21 +++-- .../sql/compiler/ir/form/FormConstants.java | 81 ++++++++++++++++ .../sql/compiler/ir/form}/Offset.java | 4 +- .../sql/compiler/ir/form/package-info.java | 2 + 8 files changed, 104 insertions(+), 110 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormConstants.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/form/FormConstants.java rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/forms => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/form}/Offset.java (66%) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/form/package-info.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java index 2ab01b6053..17a6cab018 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java @@ -1,7 +1,7 @@ package com.bakdata.conquery.sql.conversion.dialect.clickhouse; import static com.bakdata.conquery.sql.compiler.dialect.Interval.MONTHS_PER_QUARTER; -import static com.bakdata.conquery.sql.conversion.forms.FormConstants.SERIES_INDEX; +import static com.bakdata.conquery.sql.compiler.ir.form.FormConstants.SERIES_INDEX; import static org.jooq.impl.DSL.*; import static org.jooq.impl.DSL.inline; @@ -11,7 +11,7 @@ import com.bakdata.conquery.apiv1.query.TemporalSamplerFactory; import com.bakdata.conquery.sql.compiler.dialect.Interval; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.forms.Offset; +import com.bakdata.conquery.sql.compiler.ir.form.Offset; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java index f5559c7024..7ef2a9e12b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.apiv1.query.TemporalSamplerFactory; import com.bakdata.conquery.sql.compiler.dialect.Interval; -import com.bakdata.conquery.sql.conversion.forms.Offset; +import com.bakdata.conquery.sql.compiler.ir.form.Offset; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormConstants.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormConstants.java deleted file mode 100644 index 38984a7cb3..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormConstants.java +++ /dev/null @@ -1,92 +0,0 @@ -package com.bakdata.conquery.sql.conversion.forms; - -import java.sql.Date; - -import com.bakdata.conquery.apiv1.query.TemporalSamplerFactory; -import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery; -import com.bakdata.conquery.models.forms.managed.EntityDateQuery; -import com.bakdata.conquery.sql.compiler.ir.SharedAliases; -import org.jooq.Field; -import org.jooq.impl.DSL; - -/** - * Constant fields used for absolute and relative date stratification. See this modules README for more detailed information. - */ -public class FormConstants { - - /** - * The index date from which an absolute stratification for an entity starts. It is the lowest date bound of an absolute stratification. - */ - public static Field INDEX_START = DSL.field(DSL.name(SharedAliases.INDEX_START.getAlias()), Date.class); - - /** - * The index date corresponding to the {@link TemporalSamplerFactory} of a relative stratification. - */ - public static Field INDEX_SELECTOR = DSL.field(DSL.name(SharedAliases.INDEX_SELECTOR.getAlias()), Date.class); - - /** - * The index date from which we start when calculating a feature range. - */ - public static Field INDEX_START_NEGATIVE = DSL.field(DSL.name(SharedAliases.INDEX_START_NEGATIVE.getAlias()), Date.class); - - /** - * The index date from which we start when calculating an outcome range. - */ - public static Field INDEX_START_POSITIVE = DSL.field(DSL.name(SharedAliases.INDEX_START_POSITIVE.getAlias()), Date.class); - - /** - * The quarter start of the lower bound of an absolute stratification range. The stratification range this date is referring to can be an - * {@link AbsoluteFormQuery#getDateRange()} or for an {@link EntityDateQuery} the respective entities date range bound by the - * {@link EntityDateQuery#getDateRange()}. - */ - public static Field QUARTER_START = DSL.field(DSL.name(SharedAliases.QUARTER_START.getAlias()), Date.class); - - /** - * The exclusive quarter end of the upper bound of an absolute stratification range. The stratification range this date is referring to can be an - * {@link AbsoluteFormQuery#getDateRange()} or for an {@link EntityDateQuery} the respective entities date range bound by the - * {@link EntityDateQuery#getDateRange()}. - */ - public static Field QUARTER_END = DSL.field(DSL.name(SharedAliases.QUARTER_END.getAlias()), Date.class); - - /** - * The year-start of the lower bound of an absolute stratification range. The stratification range this date is referring to can be an - * {@link AbsoluteFormQuery#getDateRange()} or for an {@link EntityDateQuery} the respective entities date range bound by the - * {@link EntityDateQuery#getDateRange()}. - */ - public static Field YEAR_START = DSL.field(DSL.name(SharedAliases.YEAR_START.getAlias()), Date.class); - - /** - * The year-end of the upper bound of an absolute stratification range. The stratification range this date is referring to can be an - * {@link AbsoluteFormQuery#getDateRange()} or for an {@link EntityDateQuery} the respective entities date range bound by the - * {@link EntityDateQuery#getDateRange()}. - */ - public static Field YEAR_END = DSL.field(DSL.name(SharedAliases.YEAR_END.getAlias()), Date.class); - - /** - * The year-end of the upper bound of an absolute stratification range, but aligned on the quarter start of the lower bound. The stratification range this - * date is referring to can be an {@link AbsoluteFormQuery#getDateRange()} or for an {@link EntityDateQuery} the respective entities date range bound by - * the {@link EntityDateQuery#getDateRange()}. - */ - public static Field YEAR_END_QUARTER_ALIGNED = DSL.field(DSL.name(SharedAliases.YEAR_END_QUARTER_ALIGNED.getAlias()), Date.class); - - /** - * The amount times an interval added to a date, aligned on a year. - */ - public static Field YEAR_ALIGNED_COUNT = DSL.field(DSL.name(SharedAliases.YEAR_ALIGNED_COUNT.getAlias()), Integer.class); - - /** - * The amount times an interval added to a date, aligned on a quarter. - */ - public static Field QUARTER_ALIGNED_COUNT = DSL.field(DSL.name(SharedAliases.QUARTER_ALIGNED_COUNT.getAlias()), Integer.class); - - /** - * The amount times an interval added to a date, aligned a day. - */ - public static Field DAY_ALIGNED_COUNT = DSL.field(DSL.name(SharedAliases.DAY_ALIGNED_COUNT.getAlias()), Integer.class); - - /** - * The int field generated by a generate_series SQL set-returning function. - */ - public static Field SERIES_INDEX = DSL.field(DSL.name(SharedAliases.SERIES_INDEX.getAlias()), Integer.class); - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java index 0bd70ac372..b3c4423053 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java @@ -1,8 +1,8 @@ package com.bakdata.conquery.sql.conversion.forms; -import static com.bakdata.conquery.sql.conversion.forms.FormConstants.INDEX_SELECTOR; -import static com.bakdata.conquery.sql.conversion.forms.FormConstants.INDEX_START_NEGATIVE; -import static com.bakdata.conquery.sql.conversion.forms.FormConstants.INDEX_START_POSITIVE; +import static com.bakdata.conquery.sql.compiler.ir.form.FormConstants.INDEX_SELECTOR; +import static com.bakdata.conquery.sql.compiler.ir.form.FormConstants.INDEX_START_NEGATIVE; +import static com.bakdata.conquery.sql.compiler.ir.form.FormConstants.INDEX_START_POSITIVE; import java.sql.Date; import java.util.ArrayList; @@ -19,6 +19,8 @@ import com.bakdata.conquery.models.forms.util.CalendarUnit; import com.bakdata.conquery.models.forms.util.Resolution; import com.bakdata.conquery.sql.compiler.ir.SharedAliases; +import com.bakdata.conquery.sql.compiler.ir.form.FormConstants; +import com.bakdata.conquery.sql.compiler.ir.form.Offset; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.compiler.dialect.Interval; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java index dcbf8cbc21..4c96d70440 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java @@ -3,16 +3,16 @@ import static com.bakdata.conquery.sql.compiler.dialect.Interval.DAYS_PER_QUARTER; import static com.bakdata.conquery.sql.compiler.dialect.Interval.DAYS_PER_YEAR; import static com.bakdata.conquery.sql.compiler.dialect.Interval.MONTHS_PER_QUARTER; -import static com.bakdata.conquery.sql.conversion.forms.FormConstants.DAY_ALIGNED_COUNT; -import static com.bakdata.conquery.sql.conversion.forms.FormConstants.INDEX_SELECTOR; -import static com.bakdata.conquery.sql.conversion.forms.FormConstants.INDEX_START; -import static com.bakdata.conquery.sql.conversion.forms.FormConstants.QUARTER_ALIGNED_COUNT; -import static com.bakdata.conquery.sql.conversion.forms.FormConstants.QUARTER_END; -import static com.bakdata.conquery.sql.conversion.forms.FormConstants.QUARTER_START; -import static com.bakdata.conquery.sql.conversion.forms.FormConstants.YEAR_ALIGNED_COUNT; -import static com.bakdata.conquery.sql.conversion.forms.FormConstants.YEAR_END; -import static com.bakdata.conquery.sql.conversion.forms.FormConstants.YEAR_END_QUARTER_ALIGNED; -import static com.bakdata.conquery.sql.conversion.forms.FormConstants.YEAR_START; +import static com.bakdata.conquery.sql.compiler.ir.form.FormConstants.DAY_ALIGNED_COUNT; +import static com.bakdata.conquery.sql.compiler.ir.form.FormConstants.INDEX_SELECTOR; +import static com.bakdata.conquery.sql.compiler.ir.form.FormConstants.INDEX_START; +import static com.bakdata.conquery.sql.compiler.ir.form.FormConstants.QUARTER_ALIGNED_COUNT; +import static com.bakdata.conquery.sql.compiler.ir.form.FormConstants.QUARTER_END; +import static com.bakdata.conquery.sql.compiler.ir.form.FormConstants.QUARTER_START; +import static com.bakdata.conquery.sql.compiler.ir.form.FormConstants.YEAR_ALIGNED_COUNT; +import static com.bakdata.conquery.sql.compiler.ir.form.FormConstants.YEAR_END; +import static com.bakdata.conquery.sql.compiler.ir.form.FormConstants.YEAR_END_QUARTER_ALIGNED; +import static com.bakdata.conquery.sql.compiler.ir.form.FormConstants.YEAR_START; import java.sql.Date; import java.time.temporal.ChronoUnit; @@ -27,6 +27,7 @@ import com.bakdata.conquery.apiv1.query.TemporalSamplerFactory; import com.bakdata.conquery.models.forms.util.CalendarUnit; import com.bakdata.conquery.sql.compiler.ir.SharedAliases; +import com.bakdata.conquery.sql.compiler.ir.form.Offset; import com.bakdata.conquery.sql.compiler.dialect.Interval; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/form/FormConstants.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/form/FormConstants.java new file mode 100644 index 0000000000..a8984bc856 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/form/FormConstants.java @@ -0,0 +1,81 @@ +package com.bakdata.conquery.sql.compiler.ir.form; + +import java.sql.Date; + +import com.bakdata.conquery.sql.compiler.ir.SharedAliases; +import lombok.experimental.UtilityClass; +import org.jooq.Field; +import org.jooq.impl.DSL; + +/** + * SQL fields shared by absolute, entity-date, and relative form compilation. + */ +@UtilityClass +public class FormConstants { + + /** + * The index date from which an absolute stratification for an entity starts. It is the lowest date bound of an absolute stratification. + */ + public static final Field INDEX_START = DSL.field(DSL.name(SharedAliases.INDEX_START.getAlias()), Date.class); + + /** + * The index date selected by a relative stratification's temporal sampler. + */ + public static final Field INDEX_SELECTOR = DSL.field(DSL.name(SharedAliases.INDEX_SELECTOR.getAlias()), Date.class); + + /** + * The index date from which we start when calculating a feature range. + */ + public static final Field INDEX_START_NEGATIVE = DSL.field(DSL.name(SharedAliases.INDEX_START_NEGATIVE.getAlias()), Date.class); + + /** + * The index date from which we start when calculating an outcome range. + */ + public static final Field INDEX_START_POSITIVE = DSL.field(DSL.name(SharedAliases.INDEX_START_POSITIVE.getAlias()), Date.class); + + /** + * The quarter start of the lower bound of an absolute stratification range. + */ + public static final Field QUARTER_START = DSL.field(DSL.name(SharedAliases.QUARTER_START.getAlias()), Date.class); + + /** + * The exclusive quarter end of the upper bound of an absolute stratification range. + */ + public static final Field QUARTER_END = DSL.field(DSL.name(SharedAliases.QUARTER_END.getAlias()), Date.class); + + /** + * The year start of the lower bound of an absolute stratification range. + */ + public static final Field YEAR_START = DSL.field(DSL.name(SharedAliases.YEAR_START.getAlias()), Date.class); + + /** + * The year end of the upper bound of an absolute stratification range. + */ + public static final Field YEAR_END = DSL.field(DSL.name(SharedAliases.YEAR_END.getAlias()), Date.class); + + /** + * The year end of the upper bound of an absolute stratification range, aligned on the lower bound's quarter start. + */ + public static final Field YEAR_END_QUARTER_ALIGNED = DSL.field(DSL.name(SharedAliases.YEAR_END_QUARTER_ALIGNED.getAlias()), Date.class); + + /** + * The amount times an interval added to a date, aligned on a year. + */ + public static final Field YEAR_ALIGNED_COUNT = DSL.field(DSL.name(SharedAliases.YEAR_ALIGNED_COUNT.getAlias()), Integer.class); + + /** + * The amount times an interval added to a date, aligned on a quarter. + */ + public static final Field QUARTER_ALIGNED_COUNT = DSL.field(DSL.name(SharedAliases.QUARTER_ALIGNED_COUNT.getAlias()), Integer.class); + + /** + * The amount times an interval added to a date, aligned a day. + */ + public static final Field DAY_ALIGNED_COUNT = DSL.field(DSL.name(SharedAliases.DAY_ALIGNED_COUNT.getAlias()), Integer.class); + + /** + * The int field generated by a generate_series SQL set-returning function. + */ + public static final Field SERIES_INDEX = DSL.field(DSL.name(SharedAliases.SERIES_INDEX.getAlias()), Integer.class); + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/Offset.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/form/Offset.java similarity index 66% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/Offset.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/form/Offset.java index 92242c63bf..d421d775d3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/Offset.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/form/Offset.java @@ -1,10 +1,10 @@ -package com.bakdata.conquery.sql.conversion.forms; +package com.bakdata.conquery.sql.compiler.ir.form; import lombok.Getter; import lombok.RequiredArgsConstructor; /** - * When calculating ranges by adding intervals, Offset is used so jump to the start or end of an interval by adding the offset. + * Offset applied when a form compiler moves a date to an interval boundary. *

* Example for index = 1: * '2012-01-01'::date + (index + offset.getOffset()) * interval 3 month => 2012-01-01 for Offset.MINUS_ONE, 2012-04-01 for Offset.NONE diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/form/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/form/package-info.java new file mode 100644 index 0000000000..d57217000e --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/form/package-info.java @@ -0,0 +1,2 @@ +/** Internal SQL fields and values used while compiling form stratifications. */ +package com.bakdata.conquery.sql.compiler.ir.form; From 2822ce19645d8667cc1e38a39148f3e4b36dd05e Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 14 Sep 2026 12:50:06 +0200 Subject: [PATCH 45/81] refactor(sql): move connector aggregation CTE compilation Move aggregation-select and aggregation-filter query-step construction into the connector compiler. Keep the backend CTE wrappers responsible only for legacy context adaptation and CTE lifecycle handling. Co-authored-by: Codex GPT-5.6-sol --- .../concept/AggregationFilterCte.java | 49 +------- .../concept/AggregationSelectCte.java | 34 +----- .../ir/concept/ConnectorCteCompiler.java | 78 +++++++++++++ .../ir/concept/ConnectorCteCompilerTest.java | 108 ++++++++++++++++++ 4 files changed, 194 insertions(+), 75 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java index 7af324215b..4abd9a50fc 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java @@ -1,14 +1,8 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; -import java.util.List; -import java.util.stream.Collectors; - import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; -import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import org.jooq.Condition; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCteCompiler; class AggregationFilterCte extends ConnectorCte { @@ -19,42 +13,11 @@ public ConceptCteStep cteStep() { @Override public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { - - Selects aggregationFilterSelects = collectSelects(tableContext); - - List aggregationFilterConditions = tableContext.getSqlFilters().stream() - .flatMap(conceptFilter -> conceptFilter.getWhereClauses().getGroupFilters().stream()) - .map(WhereCondition::condition) - .toList(); - - return QueryStep.builder() - .selects(aggregationFilterSelects) - .conditions(aggregationFilterConditions); - } - - private Selects collectSelects(CQTableContext tableContext) { - - QueryStep previous = tableContext.getPrevious(); - Selects previousSelects = previous.getQualifiedSelects(); - List forAggregationFilterStep = - tableContext.allSqlSelects().stream() - .flatMap(sqlSelects -> sqlSelects.getFinalSelects().stream()) - .map(sqlSelect -> { - // universal selects like an ExistsSelect have no predecessor in preceding CTE - if (sqlSelect.isUniversal()) { - return sqlSelect; - } - return sqlSelect.qualify(previous.getCteName()); - }) - .collect(Collectors.toList()); - - return Selects.builder() - .ids(previousSelects.getIds()) - .stratificationDate(previousSelects.getStratificationDate()) - .validityDate(previousSelects.getValidityDate()) - .sqlSelects(forAggregationFilterStep) - .build(); - + return ConnectorCteCompiler.compileAggregationFilter( + tableContext.getPrevious(), + tableContext.allSqlSelects(), + tableContext.getSqlFilters() + ); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java index 66c87104fa..fc6e1f5cb0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java @@ -1,44 +1,14 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; -import java.util.List; -import java.util.Optional; -import java.util.stream.Stream; - -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import org.jooq.Field; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCteCompiler; class AggregationSelectCte extends ConnectorCte { @Override public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { - - List requiredInAggregationFilterStep = tableContext.allSqlSelects().stream() - .flatMap(sqlSelects -> sqlSelects.getAggregationSelects().stream()) - .toList(); - - Selects predecessorSelects = tableContext.getPrevious().getQualifiedSelects(); - SqlIdColumns ids = predecessorSelects.getIds(); - Optional stratificationDate = predecessorSelects.getStratificationDate(); - Selects aggregationSelectSelects = Selects.builder() - .ids(ids) - .stratificationDate(stratificationDate) - .sqlSelects(requiredInAggregationFilterStep) - .build(); - - List> groupByFields = Stream.concat( - ids.toFields().stream(), - stratificationDate.stream().flatMap(range -> range.toFields().stream()) - ) - .toList(); - - return QueryStep.builder() - .selects(aggregationSelectSelects) - .groupBy(groupByFields); + return ConnectorCteCompiler.compileAggregationSelect(tableContext.getPrevious(), tableContext.allSqlSelects()); } @Override diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java new file mode 100644 index 0000000000..ab7ee7f383 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java @@ -0,0 +1,78 @@ +package com.bakdata.conquery.sql.compiler.ir.concept; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import lombok.experimental.UtilityClass; +import org.jooq.Condition; +import org.jooq.Field; + +/** Builds connector-level concept CTE bodies from compiler IR. */ +@UtilityClass +public class ConnectorCteCompiler { + + public static QueryStep.QueryStepBuilder compileAggregationSelect( + QueryStep predecessor, + List sqlSelects + ) { + List aggregationSelects = sqlSelects.stream() + .flatMap(selects -> selects.getAggregationSelects().stream()) + .toList(); + + Selects predecessorSelects = predecessor.getQualifiedSelects(); + SqlIdColumns ids = predecessorSelects.getIds(); + Optional stratificationDate = predecessorSelects.getStratificationDate(); + Selects selects = Selects.builder() + .ids(ids) + .stratificationDate(stratificationDate) + .sqlSelects(aggregationSelects) + .build(); + + List> groupByFields = Stream.concat( + ids.toFields().stream(), + stratificationDate.stream().flatMap(range -> range.toFields().stream()) + ).toList(); + + return QueryStep.builder() + .selects(selects) + .groupBy(groupByFields); + } + + public static QueryStep.QueryStepBuilder compileAggregationFilter( + QueryStep predecessor, + List sqlSelects, + List sqlFilters + ) { + Selects predecessorSelects = predecessor.getQualifiedSelects(); + List finalSelects = sqlSelects.stream() + .flatMap(selects -> selects.getFinalSelects().stream()) + .map(sqlSelect -> qualifyUnlessUniversal(sqlSelect, predecessor.getCteName())) + .toList(); + Selects selects = Selects.builder() + .ids(predecessorSelects.getIds()) + .stratificationDate(predecessorSelects.getStratificationDate()) + .validityDate(predecessorSelects.getValidityDate()) + .sqlSelects(finalSelects) + .build(); + + List conditions = sqlFilters.stream() + .flatMap(sqlFilter -> sqlFilter.getWhereClauses().getGroupFilters().stream()) + .map(WhereCondition::condition) + .toList(); + + return QueryStep.builder() + .selects(selects) + .conditions(conditions); + } + + private static SqlSelect qualifyUnlessUniversal(SqlSelect sqlSelect, String predecessorName) { + return sqlSelect.isUniversal() ? sqlSelect : sqlSelect.qualify(predecessorName); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java new file mode 100644 index 0000000000..3d05a1f83c --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java @@ -0,0 +1,108 @@ +package com.bakdata.conquery.sql.compiler.ir.concept; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; + +import java.sql.Date; +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.condition.ConditionWrappingWhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ExistsSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.Name; +import org.junit.jupiter.api.Test; + +class ConnectorCteCompilerTest { + + private static final String PREDECESSOR_NAME = "predecessor"; + + @Test + void shouldCompileAggregationSelectsAndGroupByPredecessorDimensions() { + FieldWrapper aggregation = new FieldWrapper<>(field(name("aggregated"), Integer.class)); + QueryStep result = ConnectorCteCompiler.compileAggregationSelect( + predecessor(), + List.of(ConnectorSqlSelects.builder().aggregationSelect(aggregation).build()) + ).build(); + + assertEquals(List.of(aggregation), result.getSelects().getSqlSelects()); + assertEquals( + List.of(name(PREDECESSOR_NAME, "person"), name(PREDECESSOR_NAME, "stratification_start"), name(PREDECESSOR_NAME, "stratification_end")), + result.getGroupBy().stream().map(Field::getQualifiedName).toList() + ); + } + + @Test + void shouldCompileFinalSelectsAndGroupFilters() { + QueryStep predecessor = predecessor(); + FieldWrapper extracted = new FieldWrapper<>(field(name("metric"), Integer.class)); + ExistsSqlSelect universal = ExistsSqlSelect.withAlias("exists"); + Condition groupFilter = field(name("count"), Integer.class).greaterThan(2); + ConnectorSqlSelects sqlSelects = ConnectorSqlSelects.builder() + .finalSelect(extracted) + .finalSelect(universal) + .build(); + SqlFilters sqlFilter = new SqlFilters( + ConnectorSqlSelects.none(), + WhereClauses.builder() + .groupFilter(new ConditionWrappingWhereCondition(groupFilter)) + .build() + ); + + QueryStep result = ConnectorCteCompiler.compileAggregationFilter( + predecessor, + List.of(sqlSelects), + List.of(sqlFilter) + ).build(); + + List resultSelects = result.getSelects().getSqlSelects(); + assertEquals( + name(PREDECESSOR_NAME, "metric"), + resultSelects.getFirst().toFields().getFirst().getQualifiedName() + ); + assertSame(universal, resultSelects.getLast()); + assertEquals(List.of(groupFilter), result.getConditions()); + + Selects predecessorSelects = predecessor.getQualifiedSelects(); + assertEquals(predecessorSelects.getIds().toFields(), result.getSelects().getIds().toFields()); + assertEquals( + qualifiedNames(predecessorSelects.getStratificationDate().orElseThrow()), + qualifiedNames(result.getSelects().getStratificationDate().orElseThrow()) + ); + assertEquals( + qualifiedNames(predecessorSelects.getValidityDate().orElseThrow()), + qualifiedNames(result.getSelects().getValidityDate().orElseThrow()) + ); + } + + private static List qualifiedNames(ColumnDateRange range) { + return range.toFields().stream().map(Field::getQualifiedName).toList(); + } + + private static QueryStep predecessor() { + return QueryStep.builder() + .cteName(PREDECESSOR_NAME) + .selects(Selects.builder() + .ids(new SqlIdColumns(field(name("person"), String.class))) + .stratificationDate(Optional.of(ColumnDateRange.of( + field(name("stratification_start"), Date.class), + field(name("stratification_end"), Date.class) + ))) + .validityDate(Optional.of(ColumnDateRange.of( + field(name("validity_start"), Date.class), + field(name("validity_end"), Date.class) + ))) + .build()) + .build(); + } +} From ec5a2ad4657462e8a87eb267409b4703336b4f82 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 14 Sep 2026 13:00:06 +0200 Subject: [PATCH 46/81] refactor(sql): move final concept CTE compilation Move final concept select merging, connector aggregation, predecessor joins, grouping, and negation into the connector compiler. Keep legacy select conversion and interval-step adaptation in the backend. Co-authored-by: Codex GPT-5.6-sol --- .../cqelement/concept/CQConceptConverter.java | 55 ++--------- .../ir/concept/ConceptCteCompiler.java | 73 ++++++++++++++ .../ir/concept/ConceptCteCompilerTest.java | 96 +++++++++++++++++++ 3 files changed, 179 insertions(+), 45 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteCompiler.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteCompilerTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index d67aa33516..30e314c62d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -2,13 +2,12 @@ import com.bakdata.conquery.sql.compiler.ir.JoinMode; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.QueryStepJoiner; import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteCompiler; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.function.Predicate; import java.util.stream.Stream; import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; @@ -27,7 +26,6 @@ import com.bakdata.conquery.sql.compiler.ir.condition.DateRestrictionCondition; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; @@ -43,8 +41,6 @@ import com.google.common.base.Preconditions; import org.jooq.Condition; import org.jooq.Field; -import org.jooq.Record; -import org.jooq.TableLike; import static org.jooq.impl.DSL.*; @@ -64,7 +60,6 @@ public CQConceptConverter() { private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcept cqConcept, TablePath tablePath, ConversionContext context) { ConceptSqlTables universalTables = tablePath.createConceptTables(predecessor); - Selects predecessorSelects = predecessor.getQualifiedSelects(); Optional validityDate = predecessorSelects.getValidityDate(); SqlIdColumns ids = predecessorSelects.getIds(); @@ -77,49 +72,19 @@ private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcep }) .toList(); - List queriesToJoin = new ArrayList<>(); - queriesToJoin.add(predecessor); - converted.stream().map(ConceptSqlSelects::getAdditionalPredecessor).filter(Optional::isPresent).map(Optional::get).forEach(queriesToJoin::add); - + Optional intervalPackingSelects = Optional.empty(); if (universalTables.isRequiredStep(ConceptCteStep.INTERVAL_PACKING_SELECTS)) { QueryStep eventDateSelectsStep = IntervalPackingSelectsCte.forConcept(predecessor, universalTables, converted, context); - queriesToJoin.add(eventDateSelectsStep); + intervalPackingSelects = Optional.of(eventDateSelectsStep); } - // combine all universal selects and connector selects from preceding step - List allConceptSelects = Stream.concat( - converted.stream().flatMap(sqlSelects -> sqlSelects.getFinalSelects().stream()), - // aggregate special selects (e.g. Exists) - predecessor.getQualifiedSelects().getSqlSelects().stream().map(SqlSelect::connectorAggregate) - ) - .toList(); - - Selects finalSelects = Selects.builder() - .ids(ids) - .stratificationDate(predecessorSelects.getStratificationDate()) - .validityDate(validityDate) - .sqlSelects(allConceptSelects) - .build(); - - TableLike joinedTable = QueryStepJoiner.join(queriesToJoin, JoinMode.INNER); - - // group by everything which is not part of an aggregation in this step - List> groupByFields = - Stream.concat( - finalSelects.nonExplicitSelects().stream(), - finalSelects.getSqlSelects().stream() - .filter(Predicate.not(SqlSelect::isUniversal)) - .flatMap(sqlSelect -> sqlSelect.toFields().stream()) - ).toList(); - - return QueryStep.builder() - .cteName(universalTables.cteName(ConceptCteStep.UNIVERSAL_SELECTS)) - .selects(finalSelects) - .fromTable(joinedTable) - .groupBy(groupByFields) - .predecessors(queriesToJoin) - .negate(context.isNegation()) - .build(); + return ConceptCteCompiler.compileUniversalSelects( + predecessor, + converted, + intervalPackingSelects, + universalTables, + context.isNegation() + ); } public static SqlIdColumns convertIds(CQConcept cqConcept, CQTable cqTable, ConversionContext conversionContext) { diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteCompiler.java new file mode 100644 index 0000000000..b1d3613601 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteCompiler.java @@ -0,0 +1,73 @@ +package com.bakdata.conquery.sql.compiler.ir.concept; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Stream; + +import com.bakdata.conquery.sql.compiler.ir.JoinMode; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStepJoiner; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import lombok.experimental.UtilityClass; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.TableLike; + +/** Builds concept-level CTEs from connector compiler IR. */ +@UtilityClass +public class ConceptCteCompiler { + + public static QueryStep compileUniversalSelects( + QueryStep predecessor, + List conceptSelects, + Optional intervalPackingSelects, + SqlTables tables, + boolean negate + ) { + List queriesToJoin = new ArrayList<>(); + queriesToJoin.add(predecessor); + conceptSelects.stream() + .map(ConceptSqlSelects::getAdditionalPredecessor) + .flatMap(Optional::stream) + .forEach(queriesToJoin::add); + intervalPackingSelects.ifPresent(queriesToJoin::add); + + Selects predecessorSelects = predecessor.getQualifiedSelects(); + Optional validityDate = predecessorSelects.getValidityDate(); + SqlIdColumns ids = predecessorSelects.getIds(); + List allConceptSelects = Stream.concat( + conceptSelects.stream().flatMap(sqlSelects -> sqlSelects.getFinalSelects().stream()), + predecessorSelects.getSqlSelects().stream().map(SqlSelect::connectorAggregate) + ).toList(); + + Selects finalSelects = Selects.builder() + .ids(ids) + .stratificationDate(predecessorSelects.getStratificationDate()) + .validityDate(validityDate) + .sqlSelects(allConceptSelects) + .build(); + TableLike joinedTable = QueryStepJoiner.join(queriesToJoin, JoinMode.INNER); + + List> groupByFields = Stream.concat( + finalSelects.nonExplicitSelects().stream(), + finalSelects.getSqlSelects().stream() + .filter(Predicate.not(SqlSelect::isUniversal)) + .flatMap(sqlSelect -> sqlSelect.toFields().stream()) + ).toList(); + + return QueryStep.builder() + .cteName(tables.cteName(ConceptCteStep.UNIVERSAL_SELECTS)) + .selects(finalSelects) + .fromTable(joinedTable) + .groupBy(groupByFields) + .predecessors(queriesToJoin) + .negate(negate) + .build(); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteCompilerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteCompilerTest.java new file mode 100644 index 0000000000..5bf980d665 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteCompilerTest.java @@ -0,0 +1,96 @@ +package com.bakdata.conquery.sql.compiler.ir.concept; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.Date; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ExistsSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import org.jooq.Field; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Test; + +class ConceptCteCompilerTest { + + @Test + void shouldCompileUniversalConceptStep() { + FieldWrapper connectorMetric = new FieldWrapper<>(field(name("connector_metric"), Integer.class)); + QueryStep predecessor = queryStep("connector", connectorMetric); + QueryStep additional = queryStep("additional"); + QueryStep intervalPacking = queryStep("interval_packing"); + FieldWrapper conceptMetric = new FieldWrapper<>(field(name("concept_metric"), Integer.class)); + ExistsSqlSelect universal = ExistsSqlSelect.withAlias("exists"); + ConceptSqlSelects conceptSelects = ConceptSqlSelects.builder() + .additionalPredecessor(Optional.of(additional)) + .finalSelect(conceptMetric) + .finalSelect(universal) + .build(); + SqlTables tables = new SqlTables( + "connector", + Map.of(ConceptCteStep.UNIVERSAL_SELECTS, "concept"), + Map.of() + ); + + QueryStep result = ConceptCteCompiler.compileUniversalSelects( + predecessor, + List.of(conceptSelects), + Optional.of(intervalPacking), + tables, + true + ); + + assertEquals("concept", result.getCteName()); + assertEquals(List.of(predecessor, additional, intervalPacking), result.getPredecessors()); + assertTrue(result.isNegate()); + List sqlSelects = result.getSelects().getSqlSelects(); + assertSame(conceptMetric, sqlSelects.getFirst()); + assertSame(universal, sqlSelects.get(1)); + assertEquals(name("connector", "connector_metric"), sqlSelects.getLast().toFields().getFirst().getQualifiedName()); + assertEquals( + List.of( + name("connector", "person"), + name("connector", "validity_start"), + name("connector", "validity_end"), + name("concept_metric"), + name("connector", "connector_metric") + ), + result.getGroupBy().stream().map(Field::getQualifiedName).toList() + ); + String renderedFrom = DSL.using(SQLDialect.POSTGRES) + .select() + .from(result.getFromTables().getFirst()) + .getSQL() + .toLowerCase(Locale.ROOT); + assertTrue(renderedFrom.contains("from \"connector\" join \"additional\"")); + assertTrue(renderedFrom.contains("join \"interval_packing\"")); + } + + private static QueryStep queryStep(String cteName, SqlSelect... sqlSelects) { + return QueryStep.builder() + .cteName(cteName) + .selects(Selects.builder() + .ids(new SqlIdColumns(field(name("person"), String.class))) + .validityDate(Optional.of(ColumnDateRange.of( + field(name("validity_start"), Date.class), + field(name("validity_end"), Date.class) + ))) + .sqlSelects(List.of(sqlSelects)) + .build()) + .build(); + } +} From bca054161dd3f2da53b42267ae0e3e3627786456 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 15 Sep 2026 08:55:54 +0200 Subject: [PATCH 47/81] refactor(sql): move connector preprocessing CTE compilation Move SQL-resolved preprocessing input and CTE construction into the connector compiler. Keep the backend adapter responsible for resolving legacy context, and replace the neutral dialect join helper with direct jOOQ joins. Co-authored-by: Codex GPT-5.6-sol --- .../mode/local/SqlEntityResolver.java | 2 +- .../cqelement/concept/PreprocessingCte.java | 130 ++++-------------- .../dialect/SqlFunctionProvider.java | 4 - .../query/TableExportQueryConverter.java | 2 +- .../ir/concept/ConnectorCteCompiler.java | 63 +++++++++ .../ir/concept/PreprocessingCteInput.java | 45 ++++++ .../ir/concept/ConnectorCteCompilerTest.java | 97 +++++++++++++ 7 files changed, 233 insertions(+), 110 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/PreprocessingCteInput.java diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java index 92b5aafbcc..b4037d21f7 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java @@ -153,7 +153,7 @@ private Map resolveIds(String[][] values, List> resolveIdsQuery = context.with(unresolvedCte) .select(rowIndex, externalPrimaryColumn, isResolved) - .from(dialect.getFunctionProvider().innerJoin(allIdsTable, unresolvedCte, List.of(externalPrimaryColumn.eq(innerPrimaryColumn)))) + .from(allIdsTable.innerJoin(unresolvedCte).on(externalPrimaryColumn.eq(innerPrimaryColumn))) .where(externalPrimaryColumn.eq(innerPrimaryColumn)); return executionService.fetchStream(resolveIdsQuery) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java index 237479421d..95edbfe6c5 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java @@ -1,65 +1,37 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; -import java.util.ArrayList; import java.util.List; import java.util.Optional; import com.bakdata.conquery.models.datasets.concepts.Connector; -import com.bakdata.conquery.sql.compiler.ir.condition.DateRestrictionCondition; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; -import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; -import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCteCompiler; +import com.bakdata.conquery.sql.compiler.ir.concept.PreprocessingCteInput; import org.jooq.Condition; import org.jooq.Record; import org.jooq.Table; -import org.jooq.TableLike; import org.jooq.impl.DSL; class PreprocessingCte extends ConnectorCte { - private static QueryStep.QueryStepBuilder joinWithStratificationTable( - List preprocessingSelects, - List conditions, - CQTableContext tableContext - ) { - QueryStep stratificationTableCte = tableContext.getConversionContext().getStratificationTable(); - Table stratificationTable = DSL.table(DSL.name(stratificationTableCte.getCteName())); + @Override + public ConceptCteStep cteStep() { + return ConceptCteStep.PREPROCESSING; + } - Selects stratificationSelects = stratificationTableCte.getQualifiedSelects(); - SqlIdColumns stratificationIds = stratificationSelects.getIds(); - SqlIdColumns rootTableIds = tableContext.getIds().getPredecessor().orElseThrow(() -> new IllegalStateException( - "Id's should have been qualified during conversion and thus have a predecessor") + @Override + public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { + PreprocessingCteInput input = new PreprocessingCteInput( + createConceptJoin(tableContext), + tableContext.getIds(), + tableContext.getRawValidityDate(), + tableContext.getValidityDate(), + tableContext.allSqlSelects(), + tableContext.getSqlFilters(), + Optional.ofNullable(tableContext.getConversionContext().getStratificationTable()) ); - List idConditions = stratificationIds.join(rootTableIds); - - // join full stratification with connector table on all ID's from prerequisite query - SqlFunctionProvider functionProvider = tableContext.getConversionContext().getFunctionProvider(); - ColumnDateRange stratificationDate = stratificationSelects.getStratificationDate().orElseThrow(() -> new IllegalStateException( - "Stratification table must provide a stratification date" - )); - // Both expressions are available from the joined source tables; do not reference aliases produced by this SELECT. - conditions.add(new DateRestrictionCondition(stratificationDate, tableContext.getRawValidityDate()).condition()); - - Table connectorTable = createConceptJoin(tableContext); - TableLike joinedTable = functionProvider.innerJoin(connectorTable, stratificationTable, idConditions); - - Selects selects = Selects.builder() - .ids(stratificationSelects.getIds()) - .validityDate(Optional.of(tableContext.getValidityDate())) - .stratificationDate(stratificationSelects.getStratificationDate()) - .sqlSelects(preprocessingSelects) - .build(); - - return QueryStep.builder() - .selects(selects) - .fromTable(joinedTable) - .conditions(conditions); + return ConnectorCteCompiler.compilePreprocessing(input); } private static Table createConceptJoin(CQTableContext tableContext) { @@ -71,68 +43,18 @@ private static Table createConceptJoin(CQTableContext tableContext) { } Condition joinCondition = mapping.joinCondition(mappingConnector(tableContext)); - Condition conceptFilterCondition; - - if (!tableContext.getSelectedConceptElements().isEmpty()) { - conceptFilterCondition = mapping.resolvedId().in(mapping.includedLocalIds(tableContext.getSelectedConceptElements())); - } else { - conceptFilterCondition = DSL.noCondition(); - } - - return tableContext.getConversionContext().getFunctionProvider().innerJoin(connectorTable, mapping.table(), List.of(joinCondition.and(conceptFilterCondition))); + Condition conceptFilterCondition = tableContext.getSelectedConceptElements().isEmpty() + ? DSL.noCondition() + : mapping.resolvedId().in(mapping.includedLocalIds(tableContext.getSelectedConceptElements())); + + return tableContext.getConversionContext().getFunctionProvider().innerJoin( + connectorTable, + mapping.table(), + List.of(joinCondition.and(conceptFilterCondition)) + ); } private static Connector mappingConnector(CQTableContext tableContext) { return tableContext.getConnectorTables().getConnector(); - - - } - - @Override - public ConceptCteStep cteStep() { - return ConceptCteStep.PREPROCESSING; - } - - @Override - public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { - - List forPreprocessing = tableContext.allSqlSelects().stream() - .flatMap(sqlSelects -> sqlSelects.getPreprocessingSelects().stream()) - .toList(); - - Selects preprocessingSelects = Selects.builder() - .ids(tableContext.getIds()) - .validityDate(Optional.of(tableContext.getValidityDate())) - .sqlSelects(forPreprocessing) - .build(); - // All event-level where clauses are applied directly to the connector table while preprocessing. - List conditions = new ArrayList<>(); - - for (SqlFilters sqlFilter : tableContext.getSqlFilters()) { - for (WhereCondition whereCondition : sqlFilter.getWhereClauses().getPreprocessingConditions()) { - conditions.add(whereCondition.condition()); - } - for (WhereCondition whereCondition : sqlFilter.getWhereClauses().getEventFilters()) { - conditions.add(whereCondition.condition()); - } - } - - // The aliased secondary ID is selected in this CTE, so its root-table expression must be used in the WHERE clause. - tableContext.getIds() - .getPredecessor() - .flatMap(SqlIdColumns::getSecondaryId) - .ifPresent(secondaryId -> conditions.add(secondaryId.isNotNull())); - - QueryStep.QueryStepBuilder builder = QueryStep.builder() - .selects(preprocessingSelects) - .conditions(conditions); - - if (tableContext.getConversionContext().isWithStratification()) { - return joinWithStratificationTable(forPreprocessing, conditions, tableContext); - } - - return builder.fromTable(createConceptJoin(tableContext)); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index 42b6ba765a..bbc7b2bda0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -233,10 +233,6 @@ default Field arrayOut(List> fields) { return field(concatenated, String.class); } - default TableOnConditionStep innerJoin(Table leftPart, Table rightPart, List joinConditions) { - return leftPart.innerJoin(rightPart).on(joinConditions.toArray(Condition[]::new)); - } - default TableOnConditionStep fullOuterJoin(Table leftPart, Table rightPart, List joinConditions) { return leftPart.fullOuterJoin(rightPart).on(joinConditions.toArray(Condition[]::new)); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java index 281f81ed40..9193a185c8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java @@ -160,7 +160,7 @@ private static Table joinConnectorTableWithPrerequisite( Stream.of(new DateRestrictionCondition(functionProvider.forCDateRange(dateRestriction), validityDate).condition()) ).toList(); - return functionProvider.innerJoin(connectorTable, convertedPrerequisiteTable, joinConditions); + return connectorTable.innerJoin(convertedPrerequisiteTable).on(joinConditions.toArray(Condition[]::new)); } private static Field[] createPlaceholders(Map positions) { diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java index ab7ee7f383..8f64610ece 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java @@ -1,23 +1,86 @@ package com.bakdata.conquery.sql.compiler.ir.concept; +import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import java.util.stream.Stream; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.condition.DateRestrictionCondition; import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.experimental.UtilityClass; import org.jooq.Condition; import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.impl.DSL; /** Builds connector-level concept CTE bodies from compiler IR. */ @UtilityClass public class ConnectorCteCompiler { + public static QueryStep.QueryStepBuilder compilePreprocessing(PreprocessingCteInput input) { + List preprocessingSelects = input.sqlSelects().stream() + .flatMap(selects -> selects.getPreprocessingSelects().stream()) + .toList(); + List conditions = input.sqlFilters().stream() + .flatMap(sqlFilter -> Stream.concat( + sqlFilter.getWhereClauses().getPreprocessingConditions().stream(), + sqlFilter.getWhereClauses().getEventFilters().stream() + )) + .map(WhereCondition::condition) + .collect(Collectors.toCollection(ArrayList::new)); + + input.ids() + .getPredecessor() + .flatMap(SqlIdColumns::getSecondaryId) + .ifPresent(secondaryId -> conditions.add(secondaryId.isNotNull())); + + if (input.stratificationTable().isEmpty()) { + Selects selects = Selects.builder() + .ids(input.ids()) + .validityDate(Optional.of(input.validityDate())) + .sqlSelects(preprocessingSelects) + .build(); + return QueryStep.builder() + .selects(selects) + .fromTable(input.sourceTable()) + .conditions(conditions); + } + + QueryStep stratificationTableCte = input.stratificationTable().orElseThrow(); + Table stratificationTable = DSL.table(DSL.name(stratificationTableCte.getCteName())); + Selects stratificationSelects = stratificationTableCte.getQualifiedSelects(); + SqlIdColumns rootTableIds = input.ids().getPredecessor().orElseThrow(() -> new IllegalStateException( + "IDs must retain their source expressions when preprocessing uses stratification" + )); + List idConditions = stratificationSelects.getIds().join(rootTableIds); + ColumnDateRange stratificationDate = stratificationSelects.getStratificationDate().orElseThrow(() -> new IllegalStateException( + "Stratification table must provide a stratification date" + )); + conditions.add(new DateRestrictionCondition(stratificationDate, input.rawValidityDate()).condition()); + + Table connectorTable = input.sourceTable().asTable(); + Table joinedTable = connectorTable.innerJoin(stratificationTable) + .on(idConditions.toArray(Condition[]::new)); + Selects selects = Selects.builder() + .ids(stratificationSelects.getIds()) + .validityDate(Optional.of(input.validityDate())) + .stratificationDate(stratificationSelects.getStratificationDate()) + .sqlSelects(preprocessingSelects) + .build(); + + return QueryStep.builder() + .selects(selects) + .fromTable(joinedTable) + .conditions(conditions); + } + public static QueryStep.QueryStepBuilder compileAggregationSelect( QueryStep predecessor, List sqlSelects diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/PreprocessingCteInput.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/PreprocessingCteInput.java new file mode 100644 index 0000000000..d1c6e1d9e0 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/PreprocessingCteInput.java @@ -0,0 +1,45 @@ +package com.bakdata.conquery.sql.compiler.ir.concept; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import org.jooq.Record; +import org.jooq.TableLike; + +/** Fully SQL-resolved input needed to build a connector preprocessing CTE. */ +public record PreprocessingCteInput( + TableLike sourceTable, + SqlIdColumns ids, + ColumnDateRange rawValidityDate, + ColumnDateRange validityDate, + List sqlSelects, + List sqlFilters, + Optional stratificationTable +) { + + public PreprocessingCteInput { + Objects.requireNonNull(sourceTable, "sourceTable"); + Objects.requireNonNull(ids, "ids"); + Objects.requireNonNull(rawValidityDate, "rawValidityDate"); + Objects.requireNonNull(validityDate, "validityDate"); + sqlSelects = List.copyOf(sqlSelects); + sqlFilters = List.copyOf(sqlFilters); + Objects.requireNonNull(stratificationTable, "stratificationTable"); + } + + public PreprocessingCteInput( + String sourceTable, + SqlIdColumns ids, + ColumnDateRange rawValidityDate, + ColumnDateRange validityDate, + List sqlSelects, + List sqlFilters, + Optional stratificationTable + ) { + this(QueryStep.toTableLike(sourceTable), ids, rawValidityDate, validityDate, sqlSelects, sqlFilters, stratificationTable); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java index 3d05a1f83c..5901f7c75d 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java @@ -4,9 +4,11 @@ import static org.jooq.impl.DSL.name; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.sql.Date; import java.util.List; +import java.util.Locale; import java.util.Optional; import com.bakdata.conquery.sql.compiler.ir.QueryStep; @@ -21,12 +23,61 @@ import org.jooq.Condition; import org.jooq.Field; import org.jooq.Name; +import org.jooq.SQLDialect; +import org.jooq.conf.ParamType; +import org.jooq.impl.DSL; import org.junit.jupiter.api.Test; class ConnectorCteCompilerTest { private static final String PREDECESSOR_NAME = "predecessor"; + @Test + void shouldCompilePreprocessingAgainstConnectorTable() { + PreprocessingCteInput input = preprocessingInput(Optional.empty()); + + QueryStep result = ConnectorCteCompiler.compilePreprocessing(input).build(); + + assertEquals(List.of(name("score")), result.getSelects().getSqlSelects().stream() + .flatMap(sqlSelect -> sqlSelect.toFields().stream()) + .map(Field::getQualifiedName) + .toList()); + assertEquals(3, result.getConditions().size()); + String sql = render(result); + assertTrue(sql.contains("from \"events\"")); + assertTrue(sql.contains("\"events\".\"status\" = 'active'")); + assertTrue(sql.contains("\"events\".\"score\" > 0")); + assertTrue(sql.contains("\"events\".\"secondary\" is not null")); + } + + @Test + void shouldCompilePreprocessingWithStratificationJoin() { + QueryStep stratificationTable = QueryStep.builder() + .cteName("stratification") + .selects(Selects.builder() + .ids(new SqlIdColumns(field(name("person"), String.class))) + .stratificationDate(Optional.of(ColumnDateRange.of( + field(name("stratification_start"), Date.class), + field(name("stratification_end"), Date.class) + ))) + .build()) + .build(); + PreprocessingCteInput input = preprocessingInput(Optional.of(stratificationTable)); + + QueryStep result = ConnectorCteCompiler.compilePreprocessing(input).build(); + + assertEquals(name("stratification", "person"), result.getSelects().getIds().getPrimaryColumn().getQualifiedName()); + assertEquals( + List.of(name("stratification", "stratification_start"), name("stratification", "stratification_end")), + qualifiedNames(result.getSelects().getStratificationDate().orElseThrow()) + ); + String sql = render(result); + assertTrue(sql.contains("from \"events\" join \"stratification\"")); + assertTrue(sql.contains("\"stratification\".\"person\" = \"events\".\"person\"")); + assertTrue(sql.contains("\"stratification\".\"stratification_start\" < \"events\".\"valid_to\"")); + assertTrue(sql.contains("\"stratification\".\"stratification_end\" > \"events\".\"valid_from\"")); + } + @Test void shouldCompileAggregationSelectsAndGroupByPredecessorDimensions() { FieldWrapper aggregation = new FieldWrapper<>(field(name("aggregated"), Integer.class)); @@ -89,6 +140,52 @@ private static List qualifiedNames(ColumnDateRange range) { return range.toFields().stream().map(Field::getQualifiedName).toList(); } + private static PreprocessingCteInput preprocessingInput(Optional stratificationTable) { + SqlIdColumns ids = new SqlIdColumns( + field(name("events", "person"), String.class), + field(name("events", "secondary"), String.class) + ).withAlias(); + ColumnDateRange rawValidityDate = ColumnDateRange.of( + field(name("events", "valid_from"), Date.class), + field(name("events", "valid_to"), Date.class) + ); + ConnectorSqlSelects sqlSelects = ConnectorSqlSelects.builder() + .preprocessingSelect(new FieldWrapper<>(field(name("events", "score"), Integer.class).as("score"))) + .build(); + SqlFilters sqlFilters = new SqlFilters( + ConnectorSqlSelects.none(), + WhereClauses.builder() + .preprocessingCondition(new ConditionWrappingWhereCondition( + field(name("events", "status"), String.class).eq("active") + )) + .eventFilter(new ConditionWrappingWhereCondition( + field(name("events", "score"), Integer.class).greaterThan(0) + )) + .groupFilter(new ConditionWrappingWhereCondition( + field(name("events", "score"), Integer.class).lessThan(10) + )) + .build() + ); + return new PreprocessingCteInput( + "events", + ids, + rawValidityDate, + rawValidityDate.asValidityDateRange("connector"), + List.of(sqlSelects), + List.of(sqlFilters), + stratificationTable + ); + } + + private static String render(QueryStep queryStep) { + return DSL.using(SQLDialect.POSTGRES) + .select(queryStep.getSelects().all()) + .from(queryStep.getFromTables().getFirst()) + .where(queryStep.getConditions()) + .getSQL(ParamType.INLINED) + .toLowerCase(Locale.ROOT); + } + private static QueryStep predecessor() { return QueryStep.builder() .cteName(PREDECESSOR_NAME) From 9ff9818f90602b0bb84e46dc1ca6248f8587ee9f Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:08:47 +0200 Subject: [PATCH 48/81] refactor(sql): move interval select branch compilation Move interval-packing select branch construction into the connector compiler. Keep the backend adapter responsible for legacy select conversion and remove its now-unused conversion-context dependency. Co-authored-by: Codex GPT-5.6-sol --- .../cqelement/concept/CQConceptConverter.java | 2 +- .../concept/IntervalPackingSelectsCte.java | 56 ++---------- .../IntervalPackingSelectCompiler.java | 42 +++++++++ .../IntervalPackingSelectCompilerTest.java | 85 +++++++++++++++++++ 4 files changed, 134 insertions(+), 51 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompiler.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompilerTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 30e314c62d..9f67f665ea 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -74,7 +74,7 @@ private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcep Optional intervalPackingSelects = Optional.empty(); if (universalTables.isRequiredStep(ConceptCteStep.INTERVAL_PACKING_SELECTS)) { - QueryStep eventDateSelectsStep = IntervalPackingSelectsCte.forConcept(predecessor, universalTables, converted, context); + QueryStep eventDateSelectsStep = IntervalPackingSelectsCte.forConcept(predecessor, universalTables, converted); intervalPackingSelects = Optional.of(eventDateSelectsStep); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java index c5e85561ee..59a365786c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java @@ -1,21 +1,16 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; import java.util.List; -import java.util.Optional; -import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDateUnionSelect; -import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDurationSumSelect; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingSelectCompiler; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import com.google.common.base.Preconditions; public class IntervalPackingSelectsCte { @@ -52,62 +47,23 @@ public static QueryStep forSelect( } public static QueryStep forConnector(QueryStep predecessor, CQTableContext cqTableContext) { - return create( + return IntervalPackingSelectCompiler.compile( predecessor, cqTableContext.getSqlSelects().stream().flatMap(selects -> selects.getEventDateSelects().stream()).toList(), - cqTableContext.getConnectorTables(), - cqTableContext.getFunctionProvider() + cqTableContext.getConnectorTables() ); } public static QueryStep forConcept( QueryStep predecessor, SqlTables tables, - List sqlSelects, - ConversionContext conversionContext + List sqlSelects ) { - return create( + return IntervalPackingSelectCompiler.compile( predecessor, sqlSelects.stream().flatMap(selects -> selects.getEventDateSelects().stream()).toList(), - tables, - conversionContext.getFunctionProvider() + tables ); } - /** - * @param predecessor The preceding query step which must contain an aggregated validity date. - * @param intervalPackingSelects {@link SqlSelect}s which will be part of the returned {@link QueryStep}. - * @return A {@link QueryStep} containing converted interval packing selects, like {@link EventDurationSumSelect}, {@link EventDateUnionSelect}, etc. - * Returns the given predecessor as is if the given list of interval packing selects is empty. - */ - private static QueryStep create( - QueryStep predecessor, - List intervalPackingSelects, - SqlTables tables, - SqlFunctionProvider functionProvider - ) { - if (intervalPackingSelects.isEmpty()) { - return predecessor; - } - - Optional validityDate = predecessor.getQualifiedSelects().getValidityDate(); - Preconditions.checkArgument(validityDate.isPresent(), "Can't create a IntervalPackingSelectsCte without a validity date present."); - - // we need an additional predecessor to unnest the validity date if it is a single column range - - Selects predecessorSelects = predecessor.getQualifiedSelects(); - Selects selects = Selects.builder() - .ids(predecessorSelects.getIds()) - .sqlSelects(intervalPackingSelects) - .build(); - - return QueryStep.builder() - .cteName(tables.cteName(ConceptCteStep.INTERVAL_PACKING_SELECTS)) - .selects(selects) - .fromTable(QueryStep.toTableLike(predecessor.getCteName())) - .groupBy(predecessorSelects.getIds().toFields()) - .predecessors(List.of()) - .build(); - } - } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompiler.java new file mode 100644 index 0000000000..900fbc7e28 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompiler.java @@ -0,0 +1,42 @@ +package com.bakdata.conquery.sql.compiler.ir.interval; + +import java.util.List; + +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import lombok.experimental.UtilityClass; + +/** Builds the side branch that aggregates interval-based select expressions. */ +@UtilityClass +public class IntervalPackingSelectCompiler { + + public static QueryStep compile( + QueryStep predecessor, + List intervalPackingSelects, + SqlTables tables + ) { + if (intervalPackingSelects.isEmpty()) { + return predecessor; + } + + Selects predecessorSelects = predecessor.getQualifiedSelects(); + if (predecessorSelects.getValidityDate().isEmpty()) { + throw new IllegalArgumentException("Cannot compile interval-packing selects without a validity date"); + } + Selects selects = Selects.builder() + .ids(predecessorSelects.getIds()) + .sqlSelects(intervalPackingSelects) + .build(); + + return QueryStep.builder() + .cteName(tables.cteName(ConceptCteStep.INTERVAL_PACKING_SELECTS)) + .selects(selects) + .fromTable(QueryStep.toTableLike(predecessor.getCteName())) + .groupBy(predecessorSelects.getIds().toFields()) + .predecessors(List.of()) + .build(); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompilerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompilerTest.java new file mode 100644 index 0000000000..165e9d3a7e --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompilerTest.java @@ -0,0 +1,85 @@ +package com.bakdata.conquery.sql.compiler.ir.interval; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.sql.Date; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import org.jooq.Field; +import org.junit.jupiter.api.Test; + +class IntervalPackingSelectCompilerTest { + + private static final SqlTables TABLES = new SqlTables( + "source", + Map.of(ConceptCteStep.INTERVAL_PACKING_SELECTS, "interval_selects"), + Map.of() + ); + + @Test + void shouldReturnPredecessorWhenNoSelectRequiresIntervalPacking() { + QueryStep predecessor = predecessor(true); + + QueryStep result = IntervalPackingSelectCompiler.compile(predecessor, List.of(), TABLES); + + assertSame(predecessor, result); + } + + @Test + void shouldRequireValidityDate() { + QueryStep predecessor = predecessor(false); + + assertThrows( + IllegalArgumentException.class, + () -> IntervalPackingSelectCompiler.compile( + predecessor, + List.of(new FieldWrapper<>(field(name("event_duration"), Integer.class))), + TABLES + ) + ); + } + + @Test + void shouldCompileIntervalPackingSelectBranch() { + QueryStep predecessor = predecessor(true); + FieldWrapper eventDuration = new FieldWrapper<>(field(name("event_duration"), Integer.class)); + + QueryStep result = IntervalPackingSelectCompiler.compile(predecessor, List.of(eventDuration), TABLES); + + assertEquals("interval_selects", result.getCteName()); + assertEquals(List.of(eventDuration), result.getSelects().getSqlSelects()); + assertEquals( + List.of(name("interval_complete", "person")), + result.getGroupBy().stream().map(Field::getQualifiedName).toList() + ); + assertEquals("\"interval_complete\"", result.getFromTables().getFirst().toString()); + } + + private static QueryStep predecessor(boolean withValidityDate) { + Selects.SelectsBuilder selects = Selects.builder() + .ids(new SqlIdColumns(field(name("person"), String.class))); + if (withValidityDate) { + selects.validityDate(Optional.of(ColumnDateRange.of( + field(name("valid_from"), Date.class), + field(name("valid_to"), Date.class) + ))); + } + return QueryStep.builder() + .cteName("interval_complete") + .selects(selects.build()) + .build(); + } +} From d35c8921c70135fd4b470054a7542280c2ae53c9 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:25:00 +0200 Subject: [PATCH 49/81] refactor(sql): move connector branch composition Move interval packing and auxiliary predecessor joins into the connector compiler behind a SQL-resolved input. Keep the backend responsible for adapting legacy select and filter groups into their distinct branch inputs. Co-authored-by: Codex GPT-5.6-sol --- .../concept/IntervalPackingSelectsCte.java | 8 -- .../cqelement/concept/JoinBranchesCte.java | 94 ++++----------- .../ir/concept/ConnectorCteCompiler.java | 49 ++++++++ .../ir/concept/JoinBranchesCteInput.java | 29 +++++ .../ir/concept/ConnectorCteCompilerTest.java | 113 ++++++++++++++++++ 5 files changed, 212 insertions(+), 81 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/JoinBranchesCteInput.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java index 59a365786c..2781b24a05 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java @@ -46,14 +46,6 @@ public static QueryStep forSelect( .build(); } - public static QueryStep forConnector(QueryStep predecessor, CQTableContext cqTableContext) { - return IntervalPackingSelectCompiler.compile( - predecessor, - cqTableContext.getSqlSelects().stream().flatMap(selects -> selects.getEventDateSelects().stream()).toList(), - cqTableContext.getConnectorTables() - ); - } - public static QueryStep forConcept( QueryStep predecessor, SqlTables tables, diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java index b2f36f2c4b..161900f6b6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java @@ -1,24 +1,17 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; -import java.util.ArrayList; import java.util.List; -import java.util.Optional; -import com.bakdata.conquery.sql.compiler.ir.interval.AnsiSqlIntervalPacker; -import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingContext; -import com.bakdata.conquery.sql.compiler.ir.JoinMode; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.QueryStepJoiner; -import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.aggregator.SumSqlAggregator; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCteCompiler; +import com.bakdata.conquery.sql.compiler.ir.concept.JoinBranchesCteInput; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import org.jooq.Record; -import org.jooq.TableLike; +import com.bakdata.conquery.sql.conversion.model.aggregator.SumSqlAggregator; /** + * Adapts the legacy connector context to the SQL-ready branch-joining compiler input. + *

* Joins the {@link ConceptCteStep#AGGREGATION_SELECT} with the interval packing branch for the aggregated validity date and optional validity date selects * {@link IntervalPackingSelectsCte} as well as optional additional predecessors. *

@@ -53,67 +46,22 @@ protected ConceptCteStep cteStep() { @Override protected QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { - - List queriesToJoin = new ArrayList<>(); - queriesToJoin.add(tableContext.getPrevious()); - - // validity date aggregation - Optional validityDate; - if (!tableContext.getConnectorTables().isWithIntervalPacking()) { - validityDate = Optional.empty(); - } - else { - IntervalPackingContext intervalPackingContext = createIntervalPackingContext(tableContext); - QueryStep lastIntervalPackingStep = AnsiSqlIntervalPacker.aggregateAsValidityDate(intervalPackingContext); - queriesToJoin.add(lastIntervalPackingStep); - validityDate = lastIntervalPackingStep.getQualifiedSelects().getValidityDate(); - - QueryStep intervalPackingSelectsStep = IntervalPackingSelectsCte.forConnector(lastIntervalPackingStep, tableContext); - if (intervalPackingSelectsStep != lastIntervalPackingStep) { - queriesToJoin.add(intervalPackingSelectsStep); - } - - // interval packing was required for event date selects, but we won't propagate it - if (tableContext.getConnectorTables().isExcludedFromTimeAggregation()) { - validityDate = Optional.empty(); - } - } - - // additional preceding tables - tableContext.allSqlSelects().stream() - .flatMap(sqlSelects -> sqlSelects.getAdditionalPredecessor().stream()) - .forEach(queriesToJoin::add); - - Selects selects = collectSelects(validityDate, queriesToJoin, tableContext); - TableLike fromTable = QueryStepJoiner.join(queriesToJoin, JoinMode.FULL_OUTER); - - return QueryStep.builder() - .selects(selects) - .fromTable(fromTable) - .predecessors(queriesToJoin); - } - - private static IntervalPackingContext createIntervalPackingContext(CQTableContext tableContext) { - Selects predcessorSelects = tableContext.getPrevious().getQualifiedSelects(); - return IntervalPackingContext.builder() - .ids(predcessorSelects.getIds()) - .daterange(tableContext.getValidityDate()) - .tables(tableContext.getConnectorTables()) - .build(); - } - - private static Selects collectSelects(Optional validityDate, List queriesToJoin, CQTableContext tableContext) { - - SqlIdColumns ids = QueryStepJoiner.coalesceIds(queriesToJoin); - List mergedSqlSelects = QueryStepJoiner.mergeSelects(queriesToJoin); - Optional stratificationDate = tableContext.getPrevious().getQualifiedSelects().getStratificationDate(); - - return Selects.builder() - .ids(ids) - .stratificationDate(stratificationDate) - .validityDate(validityDate) - .sqlSelects(mergedSqlSelects) - .build(); + ConnectorSqlTables tables = tableContext.getConnectorTables(); + List eventDateSelects = tableContext.getSqlSelects().stream() + .flatMap(selects -> selects.getEventDateSelects().stream()) + .toList(); + List additionalPredecessors = tableContext.allSqlSelects().stream() + .flatMap(selects -> selects.getAdditionalPredecessor().stream()) + .toList(); + return ConnectorCteCompiler.compileJoinBranches(new JoinBranchesCteInput( + tableContext.getPrevious(), + tableContext.getValidityDate(), + tables, + tables.isWithIntervalPacking(), + tables.isExcludedFromTimeAggregation(), + eventDateSelects, + additionalPredecessors + )); } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java index 8f64610ece..7fe7026ea3 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java @@ -6,11 +6,16 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import com.bakdata.conquery.sql.compiler.ir.JoinMode; import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStepJoiner; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.condition.DateRestrictionCondition; import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; +import com.bakdata.conquery.sql.compiler.ir.interval.AnsiSqlIntervalPacker; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingContext; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingSelectCompiler; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.experimental.UtilityClass; @@ -135,6 +140,50 @@ public static QueryStep.QueryStepBuilder compileAggregationFilter( .conditions(conditions); } + public static QueryStep.QueryStepBuilder compileJoinBranches(JoinBranchesCteInput input) { + List queriesToJoin = new ArrayList<>(); + queriesToJoin.add(input.predecessor()); + + Optional validityDate = Optional.empty(); + if (input.withIntervalPacking()) { + IntervalPackingContext intervalPackingContext = IntervalPackingContext.builder() + .ids(input.predecessor().getQualifiedSelects().getIds()) + .daterange(input.validityDate()) + .tables(input.tables()) + .build(); + QueryStep lastIntervalPackingStep = AnsiSqlIntervalPacker.aggregateAsValidityDate(intervalPackingContext); + queriesToJoin.add(lastIntervalPackingStep); + validityDate = lastIntervalPackingStep.getQualifiedSelects().getValidityDate(); + + QueryStep intervalPackingSelects = IntervalPackingSelectCompiler.compile( + lastIntervalPackingStep, + input.eventDateSelects(), + input.tables() + ); + if (intervalPackingSelects != lastIntervalPackingStep) { + queriesToJoin.add(intervalPackingSelects); + } + + if (input.excludedFromTimeAggregation()) { + validityDate = Optional.empty(); + } + } + + queriesToJoin.addAll(input.additionalPredecessors()); + + Selects selects = Selects.builder() + .ids(QueryStepJoiner.coalesceIds(queriesToJoin)) + .stratificationDate(input.predecessor().getQualifiedSelects().getStratificationDate()) + .validityDate(validityDate) + .sqlSelects(QueryStepJoiner.mergeSelects(queriesToJoin)) + .build(); + + return QueryStep.builder() + .selects(selects) + .fromTable(QueryStepJoiner.join(queriesToJoin, JoinMode.FULL_OUTER)) + .predecessors(queriesToJoin); + } + private static SqlSelect qualifyUnlessUniversal(SqlSelect sqlSelect, String predecessorName) { return sqlSelect.isUniversal() ? sqlSelect : sqlSelect.qualify(predecessorName); } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/JoinBranchesCteInput.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/JoinBranchesCteInput.java new file mode 100644 index 0000000000..3a2aec1078 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/JoinBranchesCteInput.java @@ -0,0 +1,29 @@ +package com.bakdata.conquery.sql.compiler.ir.concept; + +import java.util.List; +import java.util.Objects; + +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; + +/** Fully SQL-resolved input needed to join the branches of a connector concept. */ +public record JoinBranchesCteInput( + QueryStep predecessor, + ColumnDateRange validityDate, + SqlTables tables, + boolean withIntervalPacking, + boolean excludedFromTimeAggregation, + List eventDateSelects, + List additionalPredecessors +) { + + public JoinBranchesCteInput { + Objects.requireNonNull(predecessor, "predecessor"); + Objects.requireNonNull(validityDate, "validityDate"); + Objects.requireNonNull(tables, "tables"); + eventDateSelects = List.copyOf(eventDateSelects); + additionalPredecessors = List.copyOf(additionalPredecessors); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java index 5901f7c75d..56ea897183 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java @@ -9,13 +9,16 @@ import java.sql.Date; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Optional; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.compiler.ir.condition.ConditionWrappingWhereCondition; import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.ExistsSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; @@ -136,6 +139,93 @@ void shouldCompileFinalSelectsAndGroupFilters() { ); } + @Test + void shouldJoinAdditionalPredecessors() { + FieldWrapper additionalSelect = new FieldWrapper<>(field(name("metric"), Integer.class)); + QueryStep additionalPredecessor = QueryStep.builder() + .cteName("additional") + .selects(Selects.builder() + .ids(new SqlIdColumns(field(name("person"), String.class))) + .sqlSelect(additionalSelect) + .build()) + .build(); + JoinBranchesCteInput input = new JoinBranchesCteInput( + predecessor(), + validityDate(), + new SqlTables(PREDECESSOR_NAME, Map.of(), Map.of()), + false, + false, + List.of(), + List.of(additionalPredecessor) + ); + + QueryStep result = ConnectorCteCompiler.compileJoinBranches(input).build(); + + assertEquals(List.of(PREDECESSOR_NAME, "additional"), result.getPredecessors().stream().map(QueryStep::getCteName).toList()); + assertTrue(result.getSelects().getValidityDate().isEmpty()); + assertEquals( + List.of(name("additional", "metric")), + result.getSelects().getSqlSelects().stream() + .flatMap(select -> select.toFields().stream()) + .map(Field::getQualifiedName) + .toList() + ); + assertTrue(render(result).contains("full outer join \"additional\"")); + } + + @Test + void shouldJoinIntervalPackingAndEventDateSelectBranches() { + FieldWrapper eventDuration = new FieldWrapper<>(field(name("event_duration"), Integer.class)); + JoinBranchesCteInput input = new JoinBranchesCteInput( + predecessor(), + validityDate(), + intervalPackingTables(), + true, + false, + List.of(eventDuration), + List.of() + ); + + QueryStep result = ConnectorCteCompiler.compileJoinBranches(input).build(); + + assertEquals( + List.of(PREDECESSOR_NAME, "interval_complete", "interval_selects"), + result.getPredecessors().stream().map(QueryStep::getCteName).toList() + ); + assertEquals( + List.of(name("interval_complete", "validity_start"), name("interval_complete", "validity_end")), + qualifiedNames(result.getSelects().getValidityDate().orElseThrow()) + ); + assertEquals( + List.of(name("interval_selects", "event_duration")), + result.getSelects().getSqlSelects().stream() + .flatMap(select -> select.toFields().stream()) + .map(Field::getQualifiedName) + .toList() + ); + } + + @Test + void shouldNotPropagateExcludedValidityDate() { + JoinBranchesCteInput input = new JoinBranchesCteInput( + predecessor(), + validityDate(), + intervalPackingTables(), + true, + true, + List.of(), + List.of() + ); + + QueryStep result = ConnectorCteCompiler.compileJoinBranches(input).build(); + + assertEquals( + List.of(PREDECESSOR_NAME, "interval_complete"), + result.getPredecessors().stream().map(QueryStep::getCteName).toList() + ); + assertTrue(result.getSelects().getValidityDate().isEmpty()); + } + private static List qualifiedNames(ColumnDateRange range) { return range.toFields().stream().map(Field::getQualifiedName).toList(); } @@ -186,6 +276,29 @@ private static String render(QueryStep queryStep) { .toLowerCase(Locale.ROOT); } + private static ColumnDateRange validityDate() { + return ColumnDateRange.of( + field(name("connector", "validity_start"), Date.class), + field(name("connector", "validity_end"), Date.class) + ); + } + + private static SqlTables intervalPackingTables() { + return new SqlTables( + PREDECESSOR_NAME, + Map.of( + IntervalPackingCteStep.PREVIOUS_END, "previous_end", + IntervalPackingCteStep.RANGE_INDEX, "range_index", + IntervalPackingCteStep.INTERVAL_COMPLETE, "interval_complete", + ConceptCteStep.INTERVAL_PACKING_SELECTS, "interval_selects" + ), + Map.of( + IntervalPackingCteStep.RANGE_INDEX, IntervalPackingCteStep.PREVIOUS_END, + IntervalPackingCteStep.INTERVAL_COMPLETE, IntervalPackingCteStep.RANGE_INDEX + ) + ); + } + private static QueryStep predecessor() { return QueryStep.builder() .cteName(PREDECESSOR_NAME) From 9c59b001c5a465151eea9fe9a18b29f96da31f49 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 15 Sep 2026 14:23:04 +0200 Subject: [PATCH 50/81] refactor(sql): move connector CTE orchestration Compile connector CTE phases and predecessor wiring from a SQL-resolved pipeline input. Reduce the legacy concept converter to an input adapter and remove its backend-specific pipeline state and phase classes. Co-authored-by: Codex GPT-5.6-sol --- .../concept/AggregationFilterCte.java | 23 ---- .../concept/AggregationSelectCte.java | 19 ---- .../cqelement/concept/CQConceptConverter.java | 106 ++++++++++++------ .../cqelement/concept/CQTableContext.java | 57 ---------- .../cqelement/concept/ConnectorCte.java | 45 -------- .../cqelement/concept/JoinBranchesCte.java | 67 ----------- .../cqelement/concept/PreprocessingCte.java | 60 ---------- .../ir/concept/ConnectorCteCompiler.java | 68 +++++++++++ .../ir/concept/ConnectorCtePipelineInput.java | 26 +++++ .../ir/concept/ConnectorCteCompilerTest.java | 34 ++++++ 10 files changed, 199 insertions(+), 306 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineInput.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java deleted file mode 100644 index 4abd9a50fc..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; -import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCteCompiler; - -class AggregationFilterCte extends ConnectorCte { - - @Override - public ConceptCteStep cteStep() { - return ConceptCteStep.AGGREGATION_FILTER; - } - - @Override - public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { - return ConnectorCteCompiler.compileAggregationFilter( - tableContext.getPrevious(), - tableContext.allSqlSelects(), - tableContext.getSqlFilters() - ); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java deleted file mode 100644 index fc6e1f5cb0..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; -import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCteCompiler; - -class AggregationSelectCte extends ConnectorCte { - - @Override - public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { - return ConnectorCteCompiler.compileAggregationSelect(tableContext.getPrevious(), tableContext.allSqlSelects()); - } - - @Override - public ConceptCteStep cteStep() { - return ConceptCteStep.AGGREGATION_SELECT; - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 9f67f665ea..b3c5d8224a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -5,6 +5,9 @@ import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteCompiler; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCteCompiler; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCtePipelineInput; +import com.bakdata.conquery.sql.compiler.ir.concept.PreprocessingCteInput; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -26,6 +29,7 @@ import com.bakdata.conquery.sql.compiler.ir.condition.DateRestrictionCondition; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; @@ -41,22 +45,12 @@ import com.google.common.base.Preconditions; import org.jooq.Condition; import org.jooq.Field; +import org.jooq.Record; import static org.jooq.impl.DSL.*; public class CQConceptConverter implements NodeConverter { - private final List connectorCTEs; - - public CQConceptConverter() { - this.connectorCTEs = List.of( - new PreprocessingCte(), - new AggregationSelectCte(), - new JoinBranchesCte(), - new AggregationFilterCte() - ); - } - private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcept cqConcept, TablePath tablePath, ConversionContext context) { ConceptSqlTables universalTables = tablePath.createConceptTables(predecessor); @@ -216,20 +210,16 @@ public ConversionContext convert(CQConcept cqConcept, ConversionContext context) } private Optional convertCqTable(TablePath tablePath, CQConcept cqConcept, CQTable cqTable, ConversionContext context) { - CQTableContext tableContext = createTableContext(tablePath, cqConcept, cqTable, context); - Optional lastQueryStep = Optional.empty(); - for (ConnectorCte queryStep : connectorCTEs) { - Optional convertedStep = queryStep.convert(tableContext, lastQueryStep); - if (convertedStep.isEmpty()) { - continue; - } - lastQueryStep = convertedStep; - tableContext = tableContext.withPrevious(lastQueryStep.get()); - } - return lastQueryStep; + ConnectorCtePipelineInput input = createConnectorCteInput(tablePath, cqConcept, cqTable, context); + return ConnectorCteCompiler.compileConnector(input); } - private CQTableContext createTableContext(TablePath tablePath, CQConcept cqConcept, CQTable cqTable, ConversionContext conversionContext) { + private ConnectorCtePipelineInput createConnectorCteInput( + TablePath tablePath, + CQConcept cqConcept, + CQTable cqTable, + ConversionContext conversionContext + ) { SqlIdColumns ids = convertIds(cqConcept, cqTable, conversionContext); ConnectorSqlTables connectorTables = tablePath.getConnectorTables(cqTable); @@ -266,18 +256,64 @@ private CQTableContext createTableContext(TablePath tablePath, CQConcept cqConce .map(select -> selectContext.getCompilerDialect().getSelectConverter(select).connectorSelect(select, selectContext)) .forEach(allSelectsForTable::add); - return CQTableContext.builder() - .ids(ids) - .connector(connectorTables.getName()) - .rawValidityDate(validityDateCalculation) - .sqlSelects(allSelectsForTable) - .sqlFilters(allSqlFiltersForTable) - .connectorTables(connectorTables) - .conceptIdMapping(conceptIdMapping) - .selectedConceptElements(conceptElements) - .resolveConceptIds(resolveConceptIds) - .conversionContext(conversionContext) - .build(); + List allSqlSelects = Stream.concat( + allSelectsForTable.stream(), + allSqlFiltersForTable.stream().map(SqlFilters::getSelects) + ).toList(); + PreprocessingCteInput preprocessing = new PreprocessingCteInput( + createConceptSourceTable( + connectorTables, + conceptIdMapping, + conceptElements, + resolveConceptIds, + functionProvider + ), + ids, + validityDateCalculation, + validityDateCalculation.asValidityDateRange(connectorTables.getName()), + allSqlSelects, + allSqlFiltersForTable, + Optional.ofNullable(conversionContext.getStratificationTable()) + ); + List eventDateSelects = allSelectsForTable.stream() + .flatMap(selects -> selects.getEventDateSelects().stream()) + .toList(); + List additionalPredecessors = allSqlSelects.stream() + .flatMap(selects -> selects.getAdditionalPredecessor().stream()) + .toList(); + + return new ConnectorCtePipelineInput( + connectorTables, + preprocessing, + eventDateSelects, + additionalPredecessors, + connectorTables.isWithIntervalPacking(), + connectorTables.isExcludedFromTimeAggregation() + ); + } + + private static org.jooq.Table createConceptSourceTable( + ConnectorSqlTables connectorTables, + ConceptIdMapping conceptIdMapping, + List> selectedConceptElements, + boolean resolveConceptIds, + SqlFunctionProvider functionProvider + ) { + org.jooq.Table connectorTable = table(name(connectorTables.getPredecessor(ConceptCteStep.PREPROCESSING))); + + if (conceptIdMapping.includesRoot(selectedConceptElements) && !resolveConceptIds) { + return connectorTable; + } + + Condition conceptFilterCondition = selectedConceptElements.isEmpty() + ? noCondition() + : conceptIdMapping.resolvedId().in(conceptIdMapping.includedLocalIds(selectedConceptElements)); + + return functionProvider.innerJoin( + connectorTable, + conceptIdMapping.table(), + List.of(conceptIdMapping.joinCondition(connectorTables.getConnector()).and(conceptFilterCondition)) + ); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java deleted file mode 100644 index e0a43df8fa..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import java.util.List; -import java.util.stream.Stream; - -import com.bakdata.conquery.models.datasets.concepts.ConceptElement; -import com.bakdata.conquery.models.datasets.concepts.filters.Filter; -import com.bakdata.conquery.models.datasets.concepts.select.Select; -import com.bakdata.conquery.sql.conversion.Context; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; -import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; -import lombok.Builder; -import lombok.Value; -import lombok.With; - -@Value -@Builder -class CQTableContext implements Context { - - String connector; - SqlIdColumns ids; - /** - * Unaliased validity-date expression for predicates rendered in the preprocessing SELECT. - */ - ColumnDateRange rawValidityDate; - List sqlSelects; - List sqlFilters; - ConnectorSqlTables connectorTables; - ConceptIdMapping conceptIdMapping; - List> selectedConceptElements; - boolean resolveConceptIds; - ConversionContext conversionContext; - @With - QueryStep previous; - - - public ColumnDateRange getValidityDate() { - return rawValidityDate.asValidityDateRange(connectorTables.getName()); - } - - /** - * @return All {@link ConnectorSqlSelects} that are either required for {@link Filter}'s or {@link Select}'s. - */ - public List allSqlSelects() { - return Stream.concat(sqlSelects.stream(), sqlFilters.stream().map(SqlFilters::getSelects)).toList(); - } - - @Override - public LegacyCompilerDialect getCompilerDialect() { - return getConversionContext().getCompilerDialect(); - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java deleted file mode 100644 index 99eb4e3ccc..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import java.util.List; -import java.util.Optional; - -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.SqlTables; -import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; - -abstract class ConnectorCte { - - - protected Optional convert(CQTableContext tableContext, Optional previous) { - - if (!isRequired(tableContext.getConnectorTables())) { - return Optional.empty(); - } - - String cteName = tableContext.getConnectorTables().cteName(cteStep()); - QueryStep.QueryStepBuilder queryStepBuilder = this.convertStep(tableContext).cteName(cteName); - - // only preprocessing has no predecessor - if (previous.isEmpty()) { - queryStepBuilder.predecessors(List.of()); - } - // if interval packing takes place, fromTable and predecessors of the final concept step are already set - else if (queryStepBuilder.build().getFromTables().isEmpty() && queryStepBuilder.build().getPredecessors().isEmpty()) { - queryStepBuilder.fromTable(QueryStep.toTableLike(previous.get().getCteName())) - .predecessors(List.of(previous.get())); - } - return Optional.of(queryStepBuilder.build()); - } - - /** - * @return The {@link ConceptCteStep} this instance belongs to. - */ - protected abstract ConceptCteStep cteStep(); - - protected abstract QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext); - - private boolean isRequired(SqlTables connectorTables) { - return connectorTables.isRequiredStep(cteStep()); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java deleted file mode 100644 index 161900f6b6..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import java.util.List; - -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; -import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCteCompiler; -import com.bakdata.conquery.sql.compiler.ir.concept.JoinBranchesCteInput; -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import com.bakdata.conquery.sql.conversion.model.aggregator.SumSqlAggregator; - -/** - * Adapts the legacy connector context to the SQL-ready branch-joining compiler input. - *

- * Joins the {@link ConceptCteStep#AGGREGATION_SELECT} with the interval packing branch for the aggregated validity date and optional validity date selects - * {@link IntervalPackingSelectsCte} as well as optional additional predecessors. - *

- * Joining is optional - if a validity date is not present, the node is excluded from time aggregation or if there is no additional predecessor, no join will - * take place. See {@link SumSqlAggregator} with distinct-by columns for an example of additional predecessors. - * - * 

- *     {@code
- *     "join_branches" as (
- *  	  select
- *  	    coalesce("group_select"."pid", "interval_complete"."pid", "interval_packing_selects"."pid", "row_number_filtered"."pid") as "pid",
- *  	    "interval_complete"."concept_concept-1_validity_date",
- *  	    "interval_packing_selects"."event_duration_sum",
- *  	    "row_number_filtered"."sum_distinct-1"
- *  	  from "group_select"
- *  	    join "interval_complete"
- *  	      on "group_select"."pid" = "interval_complete"."pid"
- *  	    join "interval_packing_selects"
- *    	      on "group_select"."pid" = "interval_packing_selects"."pid"
- *  	    join "row_number_filtered"
- *  	      on "interval_complete"."pid" = "row_number_filtered"."pid"
- *  	)
- *     }
- *
- */ -class JoinBranchesCte extends ConnectorCte { - - @Override - protected ConceptCteStep cteStep() { - return ConceptCteStep.JOIN_BRANCHES; - } - - @Override - protected QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { - ConnectorSqlTables tables = tableContext.getConnectorTables(); - List eventDateSelects = tableContext.getSqlSelects().stream() - .flatMap(selects -> selects.getEventDateSelects().stream()) - .toList(); - List additionalPredecessors = tableContext.allSqlSelects().stream() - .flatMap(selects -> selects.getAdditionalPredecessor().stream()) - .toList(); - return ConnectorCteCompiler.compileJoinBranches(new JoinBranchesCteInput( - tableContext.getPrevious(), - tableContext.getValidityDate(), - tables, - tables.isWithIntervalPacking(), - tables.isExcludedFromTimeAggregation(), - eventDateSelects, - additionalPredecessors - )); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java deleted file mode 100644 index 95edbfe6c5..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import java.util.List; -import java.util.Optional; - -import com.bakdata.conquery.models.datasets.concepts.Connector; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; -import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCteCompiler; -import com.bakdata.conquery.sql.compiler.ir.concept.PreprocessingCteInput; -import org.jooq.Condition; -import org.jooq.Record; -import org.jooq.Table; -import org.jooq.impl.DSL; - -class PreprocessingCte extends ConnectorCte { - - @Override - public ConceptCteStep cteStep() { - return ConceptCteStep.PREPROCESSING; - } - - @Override - public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { - PreprocessingCteInput input = new PreprocessingCteInput( - createConceptJoin(tableContext), - tableContext.getIds(), - tableContext.getRawValidityDate(), - tableContext.getValidityDate(), - tableContext.allSqlSelects(), - tableContext.getSqlFilters(), - Optional.ofNullable(tableContext.getConversionContext().getStratificationTable()) - ); - return ConnectorCteCompiler.compilePreprocessing(input); - } - - private static Table createConceptJoin(CQTableContext tableContext) { - Table connectorTable = DSL.table(DSL.name(tableContext.getConnectorTables().getPredecessor(ConceptCteStep.PREPROCESSING))); - ConceptIdMapping mapping = tableContext.getConceptIdMapping(); - - if (mapping.includesRoot(tableContext.getSelectedConceptElements()) && !tableContext.isResolveConceptIds()) { - return connectorTable; - } - - Condition joinCondition = mapping.joinCondition(mappingConnector(tableContext)); - Condition conceptFilterCondition = tableContext.getSelectedConceptElements().isEmpty() - ? DSL.noCondition() - : mapping.resolvedId().in(mapping.includedLocalIds(tableContext.getSelectedConceptElements())); - - return tableContext.getConversionContext().getFunctionProvider().innerJoin( - connectorTable, - mapping.table(), - List.of(joinCondition.and(conceptFilterCondition)) - ); - } - - private static Connector mappingConnector(CQTableContext tableContext) { - return tableContext.getConnectorTables().getConnector(); - } -} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java index 7fe7026ea3..2c9986592e 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java @@ -28,6 +28,25 @@ /** Builds connector-level concept CTE bodies from compiler IR. */ @UtilityClass public class ConnectorCteCompiler { + private static final List CONNECTOR_PIPELINE = List.of( + ConceptCteStep.PREPROCESSING, + ConceptCteStep.AGGREGATION_SELECT, + ConceptCteStep.JOIN_BRANCHES, + ConceptCteStep.AGGREGATION_FILTER + ); + + public static Optional compileConnector(ConnectorCtePipelineInput input) { + Optional previous = Optional.empty(); + for (ConceptCteStep cteStep : CONNECTOR_PIPELINE) { + if (!input.tables().isRequiredStep(cteStep)) { + continue; + } + QueryStep.QueryStepBuilder builder = compileConnectorStep(input, cteStep, previous) + .cteName(input.tables().cteName(cteStep)); + previous = Optional.of(completeStep(builder, previous)); + } + return previous; + } public static QueryStep.QueryStepBuilder compilePreprocessing(PreprocessingCteInput input) { List preprocessingSelects = input.sqlSelects().stream() @@ -184,6 +203,55 @@ public static QueryStep.QueryStepBuilder compileJoinBranches(JoinBranchesCteInpu .predecessors(queriesToJoin); } + private static QueryStep.QueryStepBuilder compileConnectorStep( + ConnectorCtePipelineInput input, + ConceptCteStep cteStep, + Optional previous + ) { + if (cteStep == ConceptCteStep.PREPROCESSING) { + // TODO this looks like it can be unwinded + return compilePreprocessing(input.preprocessing()); + } + + QueryStep predecessor = previous.orElseThrow(() -> new IllegalStateException( + "Connector CTE %s requires a predecessor".formatted(cteStep) + )); + return switch (cteStep) { + case AGGREGATION_SELECT -> compileAggregationSelect(predecessor, input.preprocessing().sqlSelects()); + case JOIN_BRANCHES -> compileJoinBranches(new JoinBranchesCteInput( + predecessor, + input.preprocessing().validityDate(), + input.tables(), + input.withIntervalPacking(), + input.excludedFromTimeAggregation(), + input.eventDateSelects(), + input.additionalPredecessors() + )); + case AGGREGATION_FILTER -> compileAggregationFilter( + predecessor, + input.preprocessing().sqlSelects(), + input.preprocessing().sqlFilters() + ); + default -> throw new IllegalArgumentException("Not a connector pipeline step: %s".formatted(cteStep)); + }; + } + + private static QueryStep completeStep(QueryStep.QueryStepBuilder builder, Optional previous) { + if (previous.isEmpty()) { + return builder.predecessors(List.of()).build(); + } + + QueryStep partialStep = builder.build(); + if (partialStep.getFromTables().isEmpty() && partialStep.getPredecessors().isEmpty()) { + QueryStep predecessor = previous.orElseThrow(); + return builder + .fromTable(QueryStep.toTableLike(predecessor.getCteName())) + .predecessors(List.of(predecessor)) + .build(); + } + return partialStep; + } + private static SqlSelect qualifyUnlessUniversal(SqlSelect sqlSelect, String predecessorName) { return sqlSelect.isUniversal() ? sqlSelect : sqlSelect.qualify(predecessorName); } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineInput.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineInput.java new file mode 100644 index 0000000000..70a977716a --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineInput.java @@ -0,0 +1,26 @@ +package com.bakdata.conquery.sql.compiler.ir.concept; + +import java.util.List; +import java.util.Objects; + +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; + +/** Fully SQL-resolved input needed to compile one connector's concept CTE pipeline. */ +public record ConnectorCtePipelineInput( + SqlTables tables, + PreprocessingCteInput preprocessing, + List eventDateSelects, + List additionalPredecessors, + boolean withIntervalPacking, + boolean excludedFromTimeAggregation +) { + + public ConnectorCtePipelineInput { + Objects.requireNonNull(tables, "tables"); + Objects.requireNonNull(preprocessing, "preprocessing"); + eventDateSelects = List.copyOf(eventDateSelects); + additionalPredecessors = List.copyOf(additionalPredecessors); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java index 56ea897183..c578decdbf 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java @@ -35,6 +35,40 @@ class ConnectorCteCompilerTest { private static final String PREDECESSOR_NAME = "predecessor"; + @Test + void shouldCompileConnectorCtePipeline() { + SqlTables tables = new SqlTables( + "events", + Map.of( + ConceptCteStep.PREPROCESSING, "preprocessing", + ConceptCteStep.AGGREGATION_SELECT, "aggregation", + ConceptCteStep.JOIN_BRANCHES, "joined", + ConceptCteStep.AGGREGATION_FILTER, "filtered" + ), + Map.of() + ); + ConnectorCtePipelineInput input = new ConnectorCtePipelineInput( + tables, + preprocessingInput(Optional.empty()), + List.of(), + List.of(), + false, + false + ); + + QueryStep aggregationFilter = ConnectorCteCompiler.compileConnector(input).orElseThrow(); + QueryStep joinedBranches = aggregationFilter.getPredecessors().getFirst(); + QueryStep aggregationSelect = joinedBranches.getPredecessors().getFirst(); + QueryStep preprocessing = aggregationSelect.getPredecessors().getFirst(); + + assertEquals("filtered", aggregationFilter.getCteName()); + assertEquals("joined", joinedBranches.getCteName()); + assertEquals("aggregation", aggregationSelect.getCteName()); + assertEquals("preprocessing", preprocessing.getCteName()); + assertEquals("\"joined\"", aggregationFilter.getFromTables().getFirst().toString()); + assertEquals("\"preprocessing\"", aggregationSelect.getFromTables().getFirst().toString()); + } + @Test void shouldCompilePreprocessingAgainstConnectorTable() { PreprocessingCteInput input = preprocessingInput(Optional.empty()); From 4b350ce5a53cb840ceeab56304d5ee546adea693 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 15 Sep 2026 16:07:10 +0200 Subject: [PATCH 51/81] refactor(sql): move concept branch orchestration Let the connector compile the final concept CTE and its optional interval select branch from SQL-resolved input. Keep the legacy backend responsible for converting concept selects and remove its concept-branch adapter. Co-authored-by: Codex GPT-5.6-sol --- .../cqelement/concept/CQConceptConverter.java | 12 ++------ .../concept/IntervalPackingSelectsCte.java | 14 ---------- .../ir/concept/ConceptCteCompiler.java | 22 +++++++++++++++ .../compiler/ir/concept/ConceptCteInput.java | 22 +++++++++++++++ .../ir/concept/ConceptCteCompilerTest.java | 28 +++++++++++++++++++ 5 files changed, 75 insertions(+), 23 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteInput.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index b3c5d8224a..0702291600 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -3,6 +3,7 @@ import com.bakdata.conquery.sql.compiler.ir.JoinMode; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteInput; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteCompiler; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCteCompiler; @@ -66,19 +67,12 @@ private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcep }) .toList(); - Optional intervalPackingSelects = Optional.empty(); - if (universalTables.isRequiredStep(ConceptCteStep.INTERVAL_PACKING_SELECTS)) { - QueryStep eventDateSelectsStep = IntervalPackingSelectsCte.forConcept(predecessor, universalTables, converted); - intervalPackingSelects = Optional.of(eventDateSelectsStep); - } - - return ConceptCteCompiler.compileUniversalSelects( + return ConceptCteCompiler.compileConcept(new ConceptCteInput( predecessor, converted, - intervalPackingSelects, universalTables, context.isNegation() - ); + )); } public static SqlIdColumns convertIds(CQConcept cqConcept, CQTable cqTable, ConversionContext conversionContext) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java index 2781b24a05..1a7cecc0ec 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java @@ -8,8 +8,6 @@ import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; -import com.bakdata.conquery.sql.compiler.ir.concept.ConceptSqlSelects; -import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingSelectCompiler; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; public class IntervalPackingSelectsCte { @@ -46,16 +44,4 @@ public static QueryStep forSelect( .build(); } - public static QueryStep forConcept( - QueryStep predecessor, - SqlTables tables, - List sqlSelects - ) { - return IntervalPackingSelectCompiler.compile( - predecessor, - sqlSelects.stream().flatMap(selects -> selects.getEventDateSelects().stream()).toList(), - tables - ); - } - } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteCompiler.java index b1d3613601..fb64ddf402 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteCompiler.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteCompiler.java @@ -12,6 +12,7 @@ import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingSelectCompiler; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.experimental.UtilityClass; @@ -23,6 +24,27 @@ @UtilityClass public class ConceptCteCompiler { + public static QueryStep compileConcept(ConceptCteInput input) { + Optional intervalPackingSelects = Optional.empty(); + if (input.tables().isRequiredStep(ConceptCteStep.INTERVAL_PACKING_SELECTS)) { + // TODO Derive the optional branch from the resolved selects instead of retaining duplicate CTE-graph state. + QueryStep eventDateSelects = IntervalPackingSelectCompiler.compile( + input.predecessor(), + input.sqlSelects().stream().flatMap(selects -> selects.getEventDateSelects().stream()).toList(), + input.tables() + ); + intervalPackingSelects = Optional.of(eventDateSelects); + } + + return compileUniversalSelects( + input.predecessor(), + input.sqlSelects(), + intervalPackingSelects, + input.tables(), + input.negate() + ); + } + public static QueryStep compileUniversalSelects( QueryStep predecessor, List conceptSelects, diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteInput.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteInput.java new file mode 100644 index 0000000000..ab50edb562 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteInput.java @@ -0,0 +1,22 @@ +package com.bakdata.conquery.sql.compiler.ir.concept; + +import java.util.List; +import java.util.Objects; + +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; + +/** Fully SQL-resolved input needed to compile the final CTE of a concept node. */ +public record ConceptCteInput( + QueryStep predecessor, + List sqlSelects, + SqlTables tables, + boolean negate +) { + + public ConceptCteInput { + Objects.requireNonNull(predecessor, "predecessor"); + sqlSelects = List.copyOf(sqlSelects); + Objects.requireNonNull(tables, "tables"); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteCompilerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteCompilerTest.java index 5bf980d665..144759bfc4 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteCompilerTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCteCompilerTest.java @@ -27,6 +27,34 @@ class ConceptCteCompilerTest { + @Test + void shouldCompileIntervalSelectBranchFromResolvedConceptInput() { + QueryStep predecessor = queryStep("connector"); + FieldWrapper eventDuration = new FieldWrapper<>(field(name("event_duration"), Integer.class)); + ConceptSqlSelects conceptSelects = ConceptSqlSelects.builder() + .eventDateSelect(eventDuration) + .build(); + SqlTables tables = new SqlTables( + "connector", + Map.of( + ConceptCteStep.INTERVAL_PACKING_SELECTS, "interval_packing", + ConceptCteStep.UNIVERSAL_SELECTS, "concept" + ), + Map.of() + ); + + QueryStep result = ConceptCteCompiler.compileConcept(new ConceptCteInput( + predecessor, + List.of(conceptSelects), + tables, + false + )); + + assertEquals(List.of("connector", "interval_packing"), result.getPredecessors().stream().map(QueryStep::getCteName).toList()); + QueryStep intervalPacking = result.getPredecessors().getLast(); + assertEquals(List.of(eventDuration), intervalPacking.getSelects().getSqlSelects()); + } + @Test void shouldCompileUniversalConceptStep() { FieldWrapper connectorMetric = new FieldWrapper<>(field(name("connector_metric"), Integer.class)); From 3596f48382cb154acfa9a097847ccd10b5ce5075 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 16 Sep 2026 08:16:29 +0200 Subject: [PATCH 52/81] refactor(sql): move arbitrary interval select compilation Compile arbitrary interval-packed select projections in the connector and expose single-column range expansion as a compiler dialect capability. Keep the backend dialect as a temporary delegate to its existing implementation. Co-authored-by: Codex GPT-5.6-sol --- .../concept/IntervalPackingSelectsCte.java | 47 ---------- .../dialect/LegacyCompilerDialect.java | 6 ++ .../model/select/DaterangeSelectUtil.java | 10 +-- .../sql/compiler/dialect/CompilerDialect.java | 11 +++ .../IntervalPackingSelectCompiler.java | 41 +++++++++ .../IntervalPackingSelectCompilerTest.java | 90 +++++++++++++++++++ 6 files changed, 153 insertions(+), 52 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java deleted file mode 100644 index 1a7cecc0ec..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import java.util.List; - -import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.bakdata.conquery.sql.compiler.ir.SqlTables; -import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; - -public class IntervalPackingSelectsCte { - - public static QueryStep forSelect( - QueryStep withAggregatedDaterange, - ColumnDateRange daterange, - SqlSelect select, - SqlTables tables, - LegacyCompilerDialect compilerDialect - ) { - List predecessors = List.of(withAggregatedDaterange); - QueryStep directPredecessor = withAggregatedDaterange; - - // we need an additional predecessor to unnest the validity date if it is a single column range - if (compilerDialect.supportsSingleColumnRanges()) { - String unnestCteName = tables.cteName(ConceptCteStep.UNNEST_DATE); - directPredecessor = compilerDialect.getFunctionProvider().unnestDaterange(daterange, withAggregatedDaterange, unnestCteName); - predecessors = List.of(withAggregatedDaterange, directPredecessor); - } - - Selects predecessorSelects = directPredecessor.getQualifiedSelects(); - Selects selects = Selects.builder() - .ids(predecessorSelects.getIds()) - .sqlSelect(select) - .build(); - - return QueryStep.builder() - .cteName(tables.cteName(ConceptCteStep.INTERVAL_PACKING_SELECTS)) - .selects(selects) - .fromTable(QueryStep.toTableLike(directPredecessor.getCteName())) - .groupBy(predecessorSelects.getIds().toFields()) - .predecessors(predecessors) - .build(); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java index e774f7a77f..4b282528cd 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java @@ -9,6 +9,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.rendering.QueryStepRenderer; import com.bakdata.conquery.sql.model.range.DateRange; @@ -93,6 +94,11 @@ default ColumnDateRange dateRangeLiteral(DateRange dateRange) { return getFunctionProvider().forCDateRange(legacyDateRange); } + @Override + default QueryStep unnestDateRange(ColumnDateRange dateRange, QueryStep predecessor, String cteName) { + return getFunctionProvider().unnestDaterange(dateRange, predecessor, cteName); + } + StratificationFunctions getStratificationFunctions(); SqlFunctionProvider getFunctionProvider(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java index 42055f48cc..35afa87b1a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java @@ -25,9 +25,9 @@ import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; -import com.bakdata.conquery.sql.conversion.cqelement.concept.IntervalPackingSelectsCte; import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingContext; import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingSelectCompiler; import com.bakdata.conquery.sql.compiler.ir.interval.AnsiSqlIntervalPacker; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; @@ -45,7 +45,7 @@ public class DaterangeSelectUtil { /** * Aggregates the daterange of a corresponding {@link DaterangeSelectOrFilter} and applies the respective converted aggregation via - * {@link IntervalPackingSelectsCte}s using additional predecessor tables. + * {@link IntervalPackingSelectCompiler} using an additional predecessor table. */ public static ConnectorSqlSelects createForSelect( S select, @@ -66,7 +66,7 @@ public static ConnectorSqlSelects c ColumnDateRange qualified = daterange.qualify(daterangeSelectTables.getPredecessor(INTERVAL_PACKING_SELECTS)); FieldWrapper aggregationField = aggregationFunction.apply(qualified, alias, functionProvider); - QueryStep intervalPackingSelectsStep = IntervalPackingSelectsCte.forSelect( + QueryStep intervalPackingSelectsStep = IntervalPackingSelectCompiler.compileArbitrarySelect( lastIntervalPackingStep, qualified, aggregationField, @@ -86,7 +86,7 @@ public static ConnectorSqlSelects c /** * Aggregates the daterange of a corresponding {@link DaterangeSelectOrFilter} and applies the respective converted aggregation via - * {@link IntervalPackingSelectsCte}s using additional predecessor tables. Finally, the filter condition is created. + * {@link IntervalPackingSelectCompiler} using an additional predecessor table. Finally, the filter condition is created. */ public static SqlFilters createForFilter( DaterangeSelectOrFilter filter, @@ -108,7 +108,7 @@ public static SqlFilters createForFilter( ColumnDateRange qualified = daterange.qualify(daterangeSelectTables.getPredecessor(INTERVAL_PACKING_SELECTS)); FieldWrapper aggregationField = aggregationFunction.apply(qualified, alias, functionProvider); - QueryStep intervalPackingSelectsStep = IntervalPackingSelectsCte.forSelect( + QueryStep intervalPackingSelectsStep = IntervalPackingSelectCompiler.compileArbitrarySelect( lastIntervalPackingStep, qualified, aggregationField, diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java index 12a6756bc4..8d0a0d753e 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java @@ -3,6 +3,7 @@ import java.sql.Date; import java.util.List; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.model.range.DateRange; import org.jooq.Condition; @@ -110,4 +111,14 @@ default ColumnDateRange dateRangeLiteral(DateRange dateRange) { default boolean supportsSingleColumnRanges() { return false; } + + /** + * Expand a physical single-column date range into a query step exposing its logical start and end fields. + * + * @throws UnsupportedOperationException when a dialect advertises single-column ranges without implementing the + * expansion + */ + default QueryStep unnestDateRange(ColumnDateRange dateRange, QueryStep predecessor, String cteName) { + throw new UnsupportedOperationException("Single-column date-range expansion is not implemented by this dialect"); + } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompiler.java index 900fbc7e28..1303ec4b15 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompiler.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompiler.java @@ -2,10 +2,12 @@ import java.util.List; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.experimental.UtilityClass; @@ -13,6 +15,45 @@ @UtilityClass public class IntervalPackingSelectCompiler { + /** + * Compile one aggregation over an arbitrary interval-packed date range. + * + *

TODO Unify this legacy single-select path with {@link #compile(QueryStep, List, SqlTables)} once arbitrary + * date-range selects are represented by the common resolved model.

+ */ + public static QueryStep compileArbitrarySelect( + QueryStep predecessor, + ColumnDateRange dateRange, + SqlSelect select, + SqlTables tables, + CompilerDialect dialect + ) { + List predecessors = List.of(predecessor); + QueryStep directPredecessor = predecessor; + if (dialect.supportsSingleColumnRanges()) { + directPredecessor = dialect.unnestDateRange( + dateRange, + predecessor, + tables.cteName(ConceptCteStep.UNNEST_DATE) + ); + predecessors = List.of(predecessor, directPredecessor); + } + + Selects predecessorSelects = directPredecessor.getQualifiedSelects(); + Selects selects = Selects.builder() + .ids(predecessorSelects.getIds()) + .sqlSelect(select) + .build(); + + return QueryStep.builder() + .cteName(tables.cteName(ConceptCteStep.INTERVAL_PACKING_SELECTS)) + .selects(selects) + .fromTable(QueryStep.toTableLike(directPredecessor.getCteName())) + .groupBy(predecessorSelects.getIds().toFields()) + .predecessors(predecessors) + .build(); + } + public static QueryStep compile( QueryStep predecessor, List intervalPackingSelects, diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompilerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompilerTest.java index 165e9d3a7e..670fa68e4f 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompilerTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompilerTest.java @@ -11,6 +11,7 @@ import java.util.Map; import java.util.Optional; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; @@ -68,6 +69,54 @@ void shouldCompileIntervalPackingSelectBranch() { assertEquals("\"interval_complete\"", result.getFromTables().getFirst().toString()); } + @Test + void shouldCompileArbitraryIntervalSelect() { + QueryStep predecessor = predecessor(true); + ColumnDateRange dateRange = predecessor.getSelects().getValidityDate().orElseThrow(); + FieldWrapper duration = new FieldWrapper<>(field(name("duration"), Integer.class)); + + QueryStep result = IntervalPackingSelectCompiler.compileArbitrarySelect( + predecessor, + dateRange, + duration, + TABLES, + new TestDialect(false) + ); + + assertEquals("interval_selects", result.getCteName()); + assertEquals(List.of(predecessor), result.getPredecessors()); + assertEquals(List.of(duration), result.getSelects().getSqlSelects()); + assertEquals("\"interval_complete\"", result.getFromTables().getFirst().toString()); + } + + @Test + void shouldExpandSingleColumnRangeBeforeCompilingArbitrarySelect() { + QueryStep predecessor = predecessor(true); + ColumnDateRange dateRange = predecessor.getSelects().getValidityDate().orElseThrow(); + FieldWrapper duration = new FieldWrapper<>(field(name("duration"), Integer.class)); + SqlTables tables = new SqlTables( + "source", + Map.of( + ConceptCteStep.UNNEST_DATE, "unnested", + ConceptCteStep.INTERVAL_PACKING_SELECTS, "interval_selects" + ), + Map.of() + ); + + QueryStep result = IntervalPackingSelectCompiler.compileArbitrarySelect( + predecessor, + dateRange, + duration, + tables, + new TestDialect(true) + ); + + QueryStep unnested = result.getPredecessors().getLast(); + assertEquals(List.of(predecessor, unnested), result.getPredecessors()); + assertEquals("unnested", unnested.getCteName()); + assertEquals("\"unnested\"", result.getFromTables().getFirst().toString()); + } + private static QueryStep predecessor(boolean withValidityDate) { Selects.SelectsBuilder selects = Selects.builder() .ids(new SqlIdColumns(field(name("person"), String.class))); @@ -82,4 +131,45 @@ private static QueryStep predecessor(boolean withValidityDate) { .selects(selects.build()) .build(); } + + private record TestDialect(boolean supportsSingleColumnRanges) implements CompilerDialect { + + @Override + public QueryStep unnestDateRange(ColumnDateRange dateRange, QueryStep predecessor, String cteName) { + return predecessor.toBuilder() + .cteName(cteName) + .predecessors(List.of(predecessor)) + .build(); + } + + @Override + public Field minimumDate() { + return field(name("minimum_date"), Date.class); + } + + @Override + public Field maximumDate() { + return field(name("maximum_date"), Date.class); + } + + @Override + public Field anyValue(Field value) { + return value; + } + + @Override + public Field renderDateRange(Field start, Field end) { + return field(name("rendered_range"), Object.class); + } + + @Override + public Field aggregateDateRanges(Field start, Field end) { + return field(name("aggregated_ranges"), Object.class); + } + + @Override + public int getNameMaxLength() { + return 128; + } + } } From 34352985791f4dfbea66baf9521a1c232644ccba Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 16 Sep 2026 10:14:48 +0200 Subject: [PATCH 53/81] refactor(sql): move interval select preparation Move auxiliary CTE planning and arbitrary date-range packing into the SQL connector. Return neutral preparation IR so the legacy adapter only creates the select or filter aggregation expression. Co-authored-by: Codex GPT-5.6-sol --- .../model/select/DaterangeSelectUtil.java | 87 ++++++++----------- .../IntervalPackingSelectCompiler.java | 37 ++++++++ .../IntervalPackingSelectPreparation.java | 21 +++++ .../IntervalPackingSelectCompilerTest.java | 23 +++-- 4 files changed, 110 insertions(+), 58 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectPreparation.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java index 35afa87b1a..d54677281a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java @@ -1,15 +1,12 @@ package com.bakdata.conquery.sql.conversion.model.select; -import static com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep.*; -import static com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep.INTERVAL_COMPLETE; +import static com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep.PREPROCESSING; import static org.jooq.impl.DSL.*; import java.math.BigDecimal; import java.sql.Date; import java.time.temporal.ChronoUnit; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; @@ -25,16 +22,12 @@ import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; -import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingContext; -import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingSelectPreparation; import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingSelectCompiler; -import com.bakdata.conquery.sql.compiler.ir.interval.AnsiSqlIntervalPacker; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.compiler.ir.CteStep; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; @@ -60,17 +53,21 @@ public static ConnectorSqlSelects c .map(FieldWrapper::new) .collect(Collectors.toList()); - SqlTables daterangeSelectTables = createTables(alias, context.getTables(), context); - QueryStep lastIntervalPackingStep = applyIntervalPacking(daterange, daterangeSelectTables, context.getIds(), context.getTables()); - - ColumnDateRange qualified = daterange.qualify(daterangeSelectTables.getPredecessor(INTERVAL_PACKING_SELECTS)); + IntervalPackingSelectPreparation preparation = prepareIntervalSelect( + alias, + daterange, + context.getIds(), + context.getTables(), + context + ); + ColumnDateRange qualified = preparation.dateRange(); FieldWrapper aggregationField = aggregationFunction.apply(qualified, alias, functionProvider); QueryStep intervalPackingSelectsStep = IntervalPackingSelectCompiler.compileArbitrarySelect( - lastIntervalPackingStep, + preparation.predecessor(), qualified, aggregationField, - daterangeSelectTables, + preparation.tables(), context.getCompilerDialect() ); @@ -102,17 +99,21 @@ public static SqlFilters createForFilter( .map(FieldWrapper::new) .collect(Collectors.toList()); - SqlTables daterangeSelectTables = createTables(alias, context.getTables(), context); - QueryStep lastIntervalPackingStep = applyIntervalPacking(daterange, daterangeSelectTables, context.getIds(), context.getTables()); - - ColumnDateRange qualified = daterange.qualify(daterangeSelectTables.getPredecessor(INTERVAL_PACKING_SELECTS)); + IntervalPackingSelectPreparation preparation = prepareIntervalSelect( + alias, + daterange, + context.getIds(), + context.getTables(), + context + ); + ColumnDateRange qualified = preparation.dateRange(); FieldWrapper aggregationField = aggregationFunction.apply(qualified, alias, functionProvider); QueryStep intervalPackingSelectsStep = IntervalPackingSelectCompiler.compileArbitrarySelect( - lastIntervalPackingStep, + preparation.predecessor(), qualified, aggregationField, - daterangeSelectTables, + preparation.tables(), context.getCompilerDialect() ); @@ -144,39 +145,21 @@ private static Condition containsInfinityDate(ColumnDateRange validityDate, SqlF return validityDate.getStart().eq(negativeInfinity).or(validityDate.getEnd().eq(positiveInfinity)); } - private static SqlTables createTables(String alias, ConnectorSqlTables connectorTables, Context context) { - Map predecessorMapping = new HashMap<>(); - String preprocessingCteName = connectorTables.cteName(PREPROCESSING); - predecessorMapping.putAll(IntervalPackingCteStep.getMappings(context.getCompilerDialect())); - if (context.getCompilerDialect().supportsSingleColumnRanges()) { - predecessorMapping.put(UNNEST_DATE, INTERVAL_COMPLETE); - predecessorMapping.put(INTERVAL_PACKING_SELECTS, UNNEST_DATE); - } - else { - predecessorMapping.put(INTERVAL_PACKING_SELECTS, INTERVAL_COMPLETE); - } - Map cteNameMap = CteStep.createCteNameMap( - predecessorMapping.keySet(), - alias, - context.getNameGenerator()::cteStepName - ); - return new SqlTables(preprocessingCteName, cteNameMap, predecessorMapping); - } - - private static QueryStep applyIntervalPacking( + private static IntervalPackingSelectPreparation prepareIntervalSelect( + String alias, ColumnDateRange daterange, - SqlTables dateUnionTables, - SqlIdColumns idColumns, - ConnectorSqlTables connectorSqlTables + SqlIdColumns ids, + ConnectorSqlTables tables, + Context context ) { - String preprocessingCteName = connectorSqlTables.cteName(PREPROCESSING); - IntervalPackingContext intervalPackingContext = IntervalPackingContext.builder() - .ids(idColumns.qualify(preprocessingCteName)) - .daterange(daterange.qualify(preprocessingCteName)) - .tables(dateUnionTables) - .build(); - - return AnsiSqlIntervalPacker.aggregateAsArbitrarySelect(intervalPackingContext); + return IntervalPackingSelectCompiler.prepareArbitrarySelect( + alias, + tables.cteName(PREPROCESSING), + ids, + daterange, + context.getCompilerDialect(), + context.getNameGenerator() + ); } @FunctionalInterface diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompiler.java index 1303ec4b15..3ae6fa397b 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompiler.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompiler.java @@ -1,20 +1,57 @@ package com.bakdata.conquery.sql.compiler.ir.interval; +import java.util.HashMap; import java.util.List; +import java.util.Map; import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.CteStep; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; import lombok.experimental.UtilityClass; /** Builds the side branch that aggregates interval-based select expressions. */ @UtilityClass public class IntervalPackingSelectCompiler { + public static IntervalPackingSelectPreparation prepareArbitrarySelect( + String label, + String sourceTable, + SqlIdColumns ids, + ColumnDateRange dateRange, + CompilerDialect dialect, + SqlNameGenerator nameGenerator + ) { + Map predecessorMapping = new HashMap<>(IntervalPackingCteStep.getMappings(dialect)); + if (dialect.supportsSingleColumnRanges()) { + predecessorMapping.put(ConceptCteStep.UNNEST_DATE, IntervalPackingCteStep.INTERVAL_COMPLETE); + predecessorMapping.put(ConceptCteStep.INTERVAL_PACKING_SELECTS, ConceptCteStep.UNNEST_DATE); + } + else { + predecessorMapping.put(ConceptCteStep.INTERVAL_PACKING_SELECTS, IntervalPackingCteStep.INTERVAL_COMPLETE); + } + Map cteNames = CteStep.createCteNameMap( + predecessorMapping.keySet(), + label, + nameGenerator::cteStepName + ); + SqlTables tables = new SqlTables(sourceTable, cteNames, predecessorMapping); + IntervalPackingContext intervalPackingContext = IntervalPackingContext.builder() + .ids(ids.qualify(sourceTable)) + .daterange(dateRange.qualify(sourceTable)) + .tables(tables) + .build(); + QueryStep predecessor = AnsiSqlIntervalPacker.aggregateAsArbitrarySelect(intervalPackingContext); + ColumnDateRange qualifiedDateRange = dateRange.qualify(tables.getPredecessor(ConceptCteStep.INTERVAL_PACKING_SELECTS)); + return new IntervalPackingSelectPreparation(predecessor, qualifiedDateRange, tables); + } + /** * Compile one aggregation over an arbitrary interval-packed date range. * diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectPreparation.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectPreparation.java new file mode 100644 index 0000000000..48a7d7bf44 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectPreparation.java @@ -0,0 +1,21 @@ +package com.bakdata.conquery.sql.compiler.ir.interval; + +import java.util.Objects; + +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; + +/** SQL-ready interval-packing state used to construct one derived select branch. */ +public record IntervalPackingSelectPreparation( + QueryStep predecessor, + ColumnDateRange dateRange, + SqlTables tables +) { + + public IntervalPackingSelectPreparation { + Objects.requireNonNull(predecessor, "predecessor"); + Objects.requireNonNull(dateRange, "dateRange"); + Objects.requireNonNull(tables, "tables"); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompilerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompilerTest.java index 670fa68e4f..61c71e43c7 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompilerTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/interval/IntervalPackingSelectCompilerTest.java @@ -19,6 +19,7 @@ import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; import org.jooq.Field; import org.junit.jupiter.api.Test; @@ -74,19 +75,29 @@ void shouldCompileArbitraryIntervalSelect() { QueryStep predecessor = predecessor(true); ColumnDateRange dateRange = predecessor.getSelects().getValidityDate().orElseThrow(); FieldWrapper duration = new FieldWrapper<>(field(name("duration"), Integer.class)); + IntervalPackingSelectPreparation preparation = IntervalPackingSelectCompiler.prepareArbitrarySelect( + "duration", + "source", + predecessor.getSelects().getIds(), + dateRange, + new TestDialect(false), + new SqlNameGenerator(128) + ); QueryStep result = IntervalPackingSelectCompiler.compileArbitrarySelect( - predecessor, - dateRange, + preparation.predecessor(), + preparation.dateRange(), duration, - TABLES, + preparation.tables(), new TestDialect(false) ); - assertEquals("interval_selects", result.getCteName()); - assertEquals(List.of(predecessor), result.getPredecessors()); + assertEquals("duration-interval_packing_selects", result.getCteName()); + assertEquals(List.of(preparation.predecessor()), result.getPredecessors()); + assertEquals("duration-interval_complete", preparation.predecessor().getCteName()); + assertEquals(name("duration-interval_complete", "valid_from"), preparation.dateRange().getStart().getQualifiedName()); assertEquals(List.of(duration), result.getSelects().getSqlSelects()); - assertEquals("\"interval_complete\"", result.getFromTables().getFirst().toString()); + assertEquals("\"duration-interval_complete\"", result.getFromTables().getFirst().toString()); } @Test From 84b1e79f1a64620167fb3073d5f298ae3779d12d Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 16 Sep 2026 13:00:50 +0200 Subject: [PATCH 54/81] refactor(sql): move concept CTE graph planning Move connector and final-concept CTE graph planning into the SQL connector. Keep the legacy table path responsible for model inspection and attaching backend-specific connector metadata. Co-authored-by: Codex GPT-5.6-sol --- .../cqelement/concept/ConceptSqlTables.java | 8 +- .../cqelement/concept/ConnectorSqlTables.java | 18 +- .../cqelement/concept/TablePath.java | 177 +++--------------- .../conquery/sql/compiler/ir/SqlTables.java | 7 + .../ir/concept/ConceptCtePlanner.java | 96 ++++++++++ .../compiler/ir/concept/ConnectorCtePlan.java | 17 ++ .../ir/concept/ConceptCtePlannerTest.java | 174 +++++++++++++++++ 7 files changed, 329 insertions(+), 168 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCtePlanner.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePlan.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCtePlannerTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java index ab2aa2f475..335fc419f6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java @@ -1,9 +1,7 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; import java.util.List; -import java.util.Map; -import com.bakdata.conquery.sql.compiler.ir.CteStep; import com.bakdata.conquery.sql.compiler.ir.SqlTables; import lombok.Getter; @@ -13,12 +11,10 @@ public class ConceptSqlTables extends SqlTables { private final List connectorTables; public ConceptSqlTables( - String rootTable, - Map cteNameMap, - Map predecessorMap, + SqlTables tables, List connectorTables ) { - super(rootTable, cteNameMap, predecessorMap); + super(tables); this.connectorTables = connectorTables; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java index 1e70769242..62fa956fe3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java @@ -1,11 +1,9 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; -import java.util.Map; - import com.bakdata.conquery.models.datasets.concepts.Connector; -import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep; -import com.bakdata.conquery.sql.compiler.ir.CteStep; import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCtePlan; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep; import lombok.Getter; @Getter @@ -34,17 +32,13 @@ public class ConnectorSqlTables extends SqlTables { public ConnectorSqlTables( Connector connector, String connectorName, - String rootTable, - Map cteNameMap, - Map predecessorMap, - boolean containsIntervalPacking, - boolean excludedFromTimeAggregation + ConnectorCtePlan plan ) { - super(rootTable, cteNameMap, predecessorMap); + super(plan.tables()); this.connector = connector; this.name = connectorName; - this.withIntervalPacking = containsIntervalPacking; - this.excludedFromTimeAggregation = excludedFromTimeAggregation; + this.withIntervalPacking = plan.withIntervalPacking(); + this.excludedFromTimeAggregation = plan.excludedFromTimeAggregation(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java index bf5762d882..6dab7af1db 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java @@ -1,28 +1,23 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; -import static com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep.*; -import static com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep.INTERVAL_COMPLETE; - import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.identifiable.ids.specific.SelectId; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep; -import com.bakdata.conquery.sql.compiler.ir.CteStep; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.google.common.base.Preconditions; -import lombok.Data; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCtePlanner; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCtePlan; +import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import lombok.Getter; /** - * Determines all table/CTE names and creates the respective required {@link ConnectorSqlTables} and {@link ConceptSqlTables} which will be created during the - * conversion of a {@link CQConcept}. + * Resolves legacy concept models and attaches their backend metadata to the connector-owned CTE plans. */ class TablePath { @@ -45,158 +40,40 @@ public ConnectorSqlTables getConnectorTables(CQTable cqTable) { } private static ConnectorSqlTables createConnectorTables(CQConcept cqConcept, CQTable cqTable, ConversionContext context) { - + Connector connector = cqTable.getConnector().resolve(); String connectorName = context.getNameGenerator().legacyConceptConnectorName( cqConcept.userLabel(context.getSqlPrintSettings().getLocale()), - cqTable.getConnector().resolve().getName() + connector.getName() ); - TablePathInfo tableInfo = collectConnectorTables(cqConcept, cqTable, context); - Map cteNameMap = CteStep.createCteNameMap( - tableInfo.getMappings().keySet(), + boolean eventDateSelectsPresent = cqTable.getSelects().stream() + .map(SelectId::resolve) + .anyMatch(Select::isEventDateSelect); + ConnectorCtePlan plan = ConceptCtePlanner.planConnector( + connector.resolveTableId().getTable(), connectorName, - context.getNameGenerator()::cteStepName + cqConcept.isAggregateEventDates(), + eventDateSelectsPresent, + context.getCompilerDialect(), + context.getNameGenerator() ); - return new ConnectorSqlTables( - cqTable.getConnector().resolve(), - connectorName, - tableInfo.getRootTable(), - cteNameMap, - tableInfo.getMappings(), - tableInfo.isContainsIntervalPacking(), - tableInfo.isExcludedFromTimeAggregation() - ); + return new ConnectorSqlTables(connector, connectorName, plan); } public ConceptSqlTables createConceptTables(QueryStep predecessor) { - - TablePathInfo tableInfo = collectConceptTables(predecessor); String conceptName = context.getNameGenerator().legacyConceptName(cqConcept.userLabel(context.getSqlPrintSettings().getLocale())); - Map cteNameMap = CteStep.createCteNameMap( - tableInfo.getMappings().keySet(), + boolean eventDateSelectsPresent = cqConcept.getSelects().stream() + .map(SelectId::resolve) + .anyMatch(Select::isEventDateSelect); + SqlTables tables = ConceptCtePlanner.planConcept( + predecessor, conceptName, - context.getNameGenerator()::cteStepName + eventDateSelectsPresent, + context.getCompilerDialect(), + context.getNameGenerator() ); List connectorSqlTables = this.connectorTableMap.values().stream().toList(); - return new ConceptSqlTables( - tableInfo.getRootTable(), - cteNameMap, - tableInfo.getMappings(), - connectorSqlTables - ); - } - - private static TablePathInfo collectConnectorTables(CQConcept cqConcept, CQTable cqTable, ConversionContext context) { - - TablePathInfo tableInfo = new TablePathInfo(); - tableInfo.setRootTable(cqTable.getConnector().resolve().resolveTableId().getTable()); - tableInfo.addWithDefaultMapping(MANDATORY_STEPS); - - boolean eventDateSelectsPresent = cqTable.getSelects().stream().map(SelectId::resolve).anyMatch(Select::isEventDateSelect); - // no validity date aggregation necessary - if (!cqConcept.isAggregateEventDates() && !eventDateSelectsPresent) { - return tableInfo; - } - - // interval packing requiredw - tableInfo.setContainsIntervalPacking(true); - tableInfo.addMappings(IntervalPackingCteStep.getMappings(PREPROCESSING, context.getCompilerDialect())); - - // validity date propagation not necessary - if (!cqConcept.isAggregateEventDates()) { - tableInfo.setExcludedFromTimeAggregation(true); - } - - if (!eventDateSelectsPresent) { - return tableInfo; - } - - // interval packing selects required with optional unnest step - if (context.getCompilerDialect().supportsSingleColumnRanges()) { - tableInfo.addMappings(Map.of( - UNNEST_DATE, INTERVAL_COMPLETE, - INTERVAL_PACKING_SELECTS, UNNEST_DATE - )); - } - else { - tableInfo.addMappings(Map.of( - INTERVAL_PACKING_SELECTS, INTERVAL_COMPLETE - )); - } - - return tableInfo; - } - - private TablePathInfo collectConceptTables(QueryStep predecessor) { - - TablePathInfo tableInfo = new TablePathInfo(); - tableInfo.setRootTable(predecessor.getCteName()); // last table of a single connector or merged and aggregated table of multiple connectors - tableInfo.addRootTableMapping(UNIVERSAL_SELECTS); - - // no event date selects present - if (cqConcept.getSelects().stream().map(SelectId::resolve).noneMatch(Select::isEventDateSelect)) { - return tableInfo; - } - - Preconditions.checkArgument( - predecessor.getSelects().getValidityDate().isPresent(), - "Can not convert Selects that require interval packing without a validity date present after converting (a) connector(s)" - ); - - // universal event date selects required with optional additional unnest step - if (context.getCompilerDialect().supportsSingleColumnRanges()) { - tableInfo.addRootTableMapping(UNNEST_DATE); - tableInfo.addMappings(Map.of(INTERVAL_PACKING_SELECTS, UNNEST_DATE)); - } - else { - tableInfo.addRootTableMapping(INTERVAL_PACKING_SELECTS); - } - - tableInfo.addMappings(Map.of(UNIVERSAL_SELECTS, INTERVAL_PACKING_SELECTS)); - - return tableInfo; + return new ConceptSqlTables(tables, connectorSqlTables); } - - @Data - private static class TablePathInfo { - - /** - * Mapping of a CTE step to their respective preceding CTE step. - */ - private final Map mappings; - - /** - * The root table is the predecessor of all CteSteps from {@link TablePathInfo#mappings} which have a null-predecessor. - */ - private String rootTable; - - /** - * True if this path info contains CTEs from {@link IntervalPackingCteStep}. - */ - private boolean containsIntervalPacking; - - /** - * True if these tables should not propagate a present validity date. - */ - private boolean excludedFromTimeAggregation; - - public TablePathInfo() { - this.mappings = new HashMap<>(); - } - - public void addMappings(Map mappings) { - this.mappings.putAll(mappings); - } - - public void addWithDefaultMapping(Set steps) { - this.mappings.putAll(CteStep.getDefaultPredecessorMap(steps)); - } - - public void addRootTableMapping(CteStep step) { - this.mappings.put(step, null); - } - - } - } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/SqlTables.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/SqlTables.java index d2dd272d75..626c68a759 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/SqlTables.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/SqlTables.java @@ -14,6 +14,13 @@ public class SqlTables { private final Map cteNameMap; private final Map predecessorMap; + /** Allow backend adapters to attach framework-specific metadata without exposing the graph maps. */ + protected SqlTables(SqlTables tables) { + this.rootTable = tables.rootTable; + this.cteNameMap = tables.cteNameMap; + this.predecessorMap = tables.predecessorMap; + } + /** Returns the generated CTE name for the supplied step. */ public String cteName(CteStep cteStep) { return cteNameMap.get(cteStep); diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCtePlanner.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCtePlanner.java new file mode 100644 index 0000000000..36cadf2a6f --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCtePlanner.java @@ -0,0 +1,96 @@ +package com.bakdata.conquery.sql.compiler.ir.concept; + +import static com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep.INTERVAL_PACKING_SELECTS; +import static com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep.PREPROCESSING; +import static com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep.UNIVERSAL_SELECTS; +import static com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep.UNNEST_DATE; +import static com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep.INTERVAL_COMPLETE; + +import java.util.HashMap; +import java.util.Map; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.CteStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import lombok.experimental.UtilityClass; + +/** Plans the CTE graphs required to compile a resolved concept and its connectors. */ +@UtilityClass +public class ConceptCtePlanner { + + /** Plan the CTE graph for one connector branch of a concept. */ + public ConnectorCtePlan planConnector( + String rootTable, + String connectorName, + boolean aggregateEventDates, + boolean eventDateSelectsPresent, + CompilerDialect dialect, + SqlNameGenerator nameGenerator + ) { + Map mappings = CteStep.getDefaultPredecessorMap(ConceptCteStep.MANDATORY_STEPS); + boolean withIntervalPacking = aggregateEventDates || eventDateSelectsPresent; + if (!withIntervalPacking) { + return new ConnectorCtePlan(createTables(rootTable, connectorName, mappings, nameGenerator), false, false); + } + + mappings.putAll(IntervalPackingCteStep.getMappings(PREPROCESSING, dialect)); + if (eventDateSelectsPresent) { + addIntervalSelectMappings(mappings, INTERVAL_COMPLETE, dialect); + } + + SqlTables tables = createTables(rootTable, connectorName, mappings, nameGenerator); + return new ConnectorCtePlan(tables, true, !aggregateEventDates); + } + + /** Plan the final concept CTE graph over the converted connector branches. */ + public SqlTables planConcept( + QueryStep predecessor, + String conceptName, + boolean eventDateSelectsPresent, + CompilerDialect dialect, + SqlNameGenerator nameGenerator + ) { + Map mappings = new HashMap<>(); + mappings.put(UNIVERSAL_SELECTS, null); + if (!eventDateSelectsPresent) { + return createTables(predecessor.getCteName(), conceptName, mappings, nameGenerator); + } + + if (predecessor.getSelects().getValidityDate().isEmpty()) { + throw new IllegalArgumentException( + "Can not convert Selects that require interval packing without a validity date present after converting (a) connector(s)" + ); + } + + addIntervalSelectMappings(mappings, null, dialect); + mappings.put(UNIVERSAL_SELECTS, INTERVAL_PACKING_SELECTS); + return createTables(predecessor.getCteName(), conceptName, mappings, nameGenerator); + } + + private void addIntervalSelectMappings(Map mappings, CteStep root, CompilerDialect dialect) { + if (dialect.supportsSingleColumnRanges()) { + mappings.put(UNNEST_DATE, root); + mappings.put(INTERVAL_PACKING_SELECTS, UNNEST_DATE); + } + else { + mappings.put(INTERVAL_PACKING_SELECTS, root); + } + } + + private SqlTables createTables( + String rootTable, + String label, + Map mappings, + SqlNameGenerator nameGenerator + ) { + Map cteNameMap = CteStep.createCteNameMap( + mappings.keySet(), + label, + nameGenerator::cteStepName + ); + return new SqlTables(rootTable, cteNameMap, mappings); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePlan.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePlan.java new file mode 100644 index 0000000000..a8cecfcf91 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePlan.java @@ -0,0 +1,17 @@ +package com.bakdata.conquery.sql.compiler.ir.concept; + +import java.util.Objects; + +import com.bakdata.conquery.sql.compiler.ir.SqlTables; + +/** Connector-owned CTE graph together with the execution flags derived while planning it. */ +public record ConnectorCtePlan( + SqlTables tables, + boolean withIntervalPacking, + boolean excludedFromTimeAggregation +) { + + public ConnectorCtePlan { + Objects.requireNonNull(tables, "tables"); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCtePlannerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCtePlannerTest.java new file mode 100644 index 0000000000..eeb1fc44bc --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCtePlannerTest.java @@ -0,0 +1,174 @@ +package com.bakdata.conquery.sql.compiler.ir.concept; + +import static com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep.AGGREGATION_FILTER; +import static com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep.INTERVAL_PACKING_SELECTS; +import static com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep.PREPROCESSING; +import static com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep.UNIVERSAL_SELECTS; +import static com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep.UNNEST_DATE; +import static com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep.INTERVAL_COMPLETE; +import static com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep.PREVIOUS_END; +import static com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep.RANGE_INDEX; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.Date; +import java.util.Optional; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import org.jooq.Field; +import org.junit.jupiter.api.Test; + +class ConceptCtePlannerTest { + + private static final ColumnDateRange DATE_RANGE = ColumnDateRange.of( + field(name("valid_from"), Date.class), + field(name("valid_to"), Date.class) + ); + + @Test + void shouldPlanMandatoryConnectorStepsWithoutIntervalPacking() { + ConnectorCtePlan plan = planConnector(false, false, false); + + assertFalse(plan.withIntervalPacking()); + assertFalse(plan.excludedFromTimeAggregation()); + assertTrue(plan.tables().isRequiredStep(PREPROCESSING)); + assertTrue(plan.tables().isRequiredStep(AGGREGATION_FILTER)); + assertFalse(plan.tables().isRequiredStep(INTERVAL_COMPLETE)); + assertEquals("events", plan.tables().getPredecessor(PREPROCESSING)); + } + + @Test + void shouldPlanConnectorIntervalPackingForAggregatedEventDates() { + ConnectorCtePlan plan = planConnector(true, false, false); + + assertTrue(plan.withIntervalPacking()); + assertFalse(plan.excludedFromTimeAggregation()); + assertTrue(plan.tables().isRequiredStep(PREVIOUS_END)); + assertTrue(plan.tables().isRequiredStep(RANGE_INDEX)); + assertTrue(plan.tables().isRequiredStep(INTERVAL_COMPLETE)); + assertFalse(plan.tables().isRequiredStep(INTERVAL_PACKING_SELECTS)); + assertEquals("connector-preprocessing", plan.tables().getPredecessor(PREVIOUS_END)); + } + + @Test + void shouldPlanConnectorIntervalSelectsWithoutPropagatingTheirValidityDate() { + ConnectorCtePlan plan = planConnector(false, true, false); + + assertTrue(plan.withIntervalPacking()); + assertTrue(plan.excludedFromTimeAggregation()); + assertTrue(plan.tables().isRequiredStep(INTERVAL_PACKING_SELECTS)); + assertEquals("connector-interval_complete", plan.tables().getPredecessor(INTERVAL_PACKING_SELECTS)); + } + + @Test + void shouldPlanUniversalSelectDirectlyOverConceptRoot() { + SqlTables tables = ConceptCtePlanner.planConcept( + queryStep(false), + "concept", + false, + new TestDialect(false), + new SqlNameGenerator(128) + ); + + assertEquals("merged", tables.getPredecessor(UNIVERSAL_SELECTS)); + assertFalse(tables.isRequiredStep(INTERVAL_PACKING_SELECTS)); + } + + @Test + void shouldPlanSingleColumnConceptIntervalSelectsThroughUnnesting() { + SqlTables tables = ConceptCtePlanner.planConcept( + queryStep(true), + "concept", + true, + new TestDialect(true), + new SqlNameGenerator(128) + ); + + assertEquals("merged", tables.getPredecessor(UNNEST_DATE)); + assertEquals("concept-unnested", tables.getPredecessor(INTERVAL_PACKING_SELECTS)); + assertEquals("concept-interval_packing_selects", tables.getPredecessor(UNIVERSAL_SELECTS)); + } + + @Test + void shouldRejectConceptIntervalSelectsWithoutValidityDate() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> ConceptCtePlanner.planConcept( + queryStep(false), + "concept", + true, + new TestDialect(false), + new SqlNameGenerator(128) + ) + ); + + assertEquals( + "Can not convert Selects that require interval packing without a validity date present after converting (a) connector(s)", + exception.getMessage() + ); + } + + private static ConnectorCtePlan planConnector(boolean aggregateEventDates, boolean eventDateSelectsPresent, boolean singleColumnRanges) { + return ConceptCtePlanner.planConnector( + "events", + "connector", + aggregateEventDates, + eventDateSelectsPresent, + new TestDialect(singleColumnRanges), + new SqlNameGenerator(128) + ); + } + + private static QueryStep queryStep(boolean withValidityDate) { + Selects.SelectsBuilder selects = Selects.builder(); + if (withValidityDate) { + selects.validityDate(Optional.of(DATE_RANGE)); + } + return QueryStep.builder() + .cteName("merged") + .selects(selects.build()) + .build(); + } + + private record TestDialect(boolean supportsSingleColumnRanges) implements CompilerDialect { + + @Override + public Field minimumDate() { + return field(name("minimum_date"), Date.class); + } + + @Override + public Field maximumDate() { + return field(name("maximum_date"), Date.class); + } + + @Override + public Field anyValue(Field value) { + return value; + } + + @Override + public Field renderDateRange(Field start, Field end) { + return field(name("rendered_range"), Object.class); + } + + @Override + public Field aggregateDateRanges(Field start, Field end) { + return field(name("aggregated_ranges"), Object.class); + } + + @Override + public int getNameMaxLength() { + return 128; + } + } +} From d4a8461b3bb82675d3090411cb03ee17f7c8f1fc Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:15:05 +0200 Subject: [PATCH 55/81] refactor(sql): move connector pipeline assembly Assemble preprocessing inputs, event-date selects, and additional predecessors inside the SQL connector. Carry the generated connector name with its CTE plan so the legacy adapter only retains backend connector metadata. Co-authored-by: Codex GPT-5.6-sol --- .../cqelement/concept/CQConceptConverter.java | 29 +---- .../cqelement/concept/ConnectorSqlTables.java | 27 ++-- .../cqelement/concept/TablePath.java | 2 +- .../ir/concept/ConceptCtePlanner.java | 4 +- .../ConnectorCtePipelineAssembler.java | 60 +++++++++ .../compiler/ir/concept/ConnectorCtePlan.java | 2 + .../ConnectorCtePipelineAssemblerTest.java | 116 ++++++++++++++++++ 7 files changed, 192 insertions(+), 48 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineAssembler.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineAssemblerTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 0702291600..efeea998cc 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -5,10 +5,9 @@ import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteInput; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteCompiler; -import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCteCompiler; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCtePipelineAssembler; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCtePipelineInput; -import com.bakdata.conquery.sql.compiler.ir.concept.PreprocessingCteInput; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -30,7 +29,6 @@ import com.bakdata.conquery.sql.compiler.ir.condition.DateRestrictionCondition; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; @@ -250,11 +248,8 @@ private ConnectorCtePipelineInput createConnectorCteInput( .map(select -> selectContext.getCompilerDialect().getSelectConverter(select).connectorSelect(select, selectContext)) .forEach(allSelectsForTable::add); - List allSqlSelects = Stream.concat( - allSelectsForTable.stream(), - allSqlFiltersForTable.stream().map(SqlFilters::getSelects) - ).toList(); - PreprocessingCteInput preprocessing = new PreprocessingCteInput( + return ConnectorCtePipelineAssembler.assemble( + connectorTables.getPlan(), createConceptSourceTable( connectorTables, conceptIdMapping, @@ -264,26 +259,10 @@ private ConnectorCtePipelineInput createConnectorCteInput( ), ids, validityDateCalculation, - validityDateCalculation.asValidityDateRange(connectorTables.getName()), - allSqlSelects, + allSelectsForTable, allSqlFiltersForTable, Optional.ofNullable(conversionContext.getStratificationTable()) ); - List eventDateSelects = allSelectsForTable.stream() - .flatMap(selects -> selects.getEventDateSelects().stream()) - .toList(); - List additionalPredecessors = allSqlSelects.stream() - .flatMap(selects -> selects.getAdditionalPredecessor().stream()) - .toList(); - - return new ConnectorCtePipelineInput( - connectorTables, - preprocessing, - eventDateSelects, - additionalPredecessors, - connectorTables.isWithIntervalPacking(), - connectorTables.isExcludedFromTimeAggregation() - ); } private static org.jooq.Table createConceptSourceTable( diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java index 62fa956fe3..5934294a87 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java @@ -3,42 +3,29 @@ import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCtePlan; -import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep; import lombok.Getter; @Getter public class ConnectorSqlTables extends SqlTables { - /** - * A unique label for these tables to create unique SQL CTE/Select names. - */ - private final String name; - - /** - * True if these tables contain interval packing CTEs {@link IntervalPackingCteStep}. - */ - private final boolean withIntervalPacking; - - /** - * True if these tables should not propagate a present validity date. - */ - private final boolean excludedFromTimeAggregation; - /** * Corresponding {@link Connector} of these {@link SqlTables}. */ private final Connector connector; + private final ConnectorCtePlan plan; public ConnectorSqlTables( Connector connector, - String connectorName, ConnectorCtePlan plan ) { super(plan.tables()); this.connector = connector; - this.name = connectorName; - this.withIntervalPacking = plan.withIntervalPacking(); - this.excludedFromTimeAggregation = plan.excludedFromTimeAggregation(); + this.plan = plan; + } + + /** A unique label for these tables to create unique SQL CTE/select names. */ + public String getName() { + return plan.connectorName(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java index 6dab7af1db..5438a011b0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java @@ -57,7 +57,7 @@ private static ConnectorSqlTables createConnectorTables(CQConcept cqConcept, CQT context.getNameGenerator() ); - return new ConnectorSqlTables(connector, connectorName, plan); + return new ConnectorSqlTables(connector, plan); } public ConceptSqlTables createConceptTables(QueryStep predecessor) { diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCtePlanner.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCtePlanner.java index 36cadf2a6f..d31345f7f4 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCtePlanner.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCtePlanner.java @@ -33,7 +33,7 @@ public ConnectorCtePlan planConnector( Map mappings = CteStep.getDefaultPredecessorMap(ConceptCteStep.MANDATORY_STEPS); boolean withIntervalPacking = aggregateEventDates || eventDateSelectsPresent; if (!withIntervalPacking) { - return new ConnectorCtePlan(createTables(rootTable, connectorName, mappings, nameGenerator), false, false); + return new ConnectorCtePlan(connectorName, createTables(rootTable, connectorName, mappings, nameGenerator), false, false); } mappings.putAll(IntervalPackingCteStep.getMappings(PREPROCESSING, dialect)); @@ -42,7 +42,7 @@ public ConnectorCtePlan planConnector( } SqlTables tables = createTables(rootTable, connectorName, mappings, nameGenerator); - return new ConnectorCtePlan(tables, true, !aggregateEventDates); + return new ConnectorCtePlan(connectorName, tables, true, !aggregateEventDates); } /** Plan the final concept CTE graph over the converted connector branches. */ diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineAssembler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineAssembler.java new file mode 100644 index 0000000000..4096ebc999 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineAssembler.java @@ -0,0 +1,60 @@ +package com.bakdata.conquery.sql.compiler.ir.concept; + +import static com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep.PREPROCESSING; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import lombok.experimental.UtilityClass; +import org.jooq.Record; +import org.jooq.TableLike; + +/** Assembles SQL-resolved connector operations into the input for the connector CTE pipeline. */ +@UtilityClass +public class ConnectorCtePipelineAssembler { + + public ConnectorCtePipelineInput assemble( + ConnectorCtePlan plan, + TableLike sourceTable, + SqlIdColumns ids, + ColumnDateRange rawValidityDate, + List selects, + List filters, + Optional stratificationTable + ) { + List allSqlSelects = Stream.concat( + selects.stream(), + filters.stream().map(SqlFilters::getSelects) + ).toList(); + ColumnDateRange validityDate = rawValidityDate.asValidityDateRange(plan.connectorName()); + PreprocessingCteInput preprocessing = new PreprocessingCteInput( + sourceTable, + ids, + rawValidityDate, + validityDate, + allSqlSelects, + filters, + stratificationTable + ); + List eventDateSelects = selects.stream() + .flatMap(sqlSelects -> sqlSelects.getEventDateSelects().stream()) + .toList(); + List additionalPredecessors = allSqlSelects.stream() + .flatMap(sqlSelects -> sqlSelects.getAdditionalPredecessor().stream()) + .toList(); + + return new ConnectorCtePipelineInput( + plan.tables(), + preprocessing, + eventDateSelects, + additionalPredecessors, + plan.withIntervalPacking(), + plan.excludedFromTimeAggregation() + ); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePlan.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePlan.java index a8cecfcf91..0e11656d48 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePlan.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePlan.java @@ -6,12 +6,14 @@ /** Connector-owned CTE graph together with the execution flags derived while planning it. */ public record ConnectorCtePlan( + String connectorName, SqlTables tables, boolean withIntervalPacking, boolean excludedFromTimeAggregation ) { public ConnectorCtePlan { + Objects.requireNonNull(connectorName, "connectorName"); Objects.requireNonNull(tables, "tables"); } } diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineAssemblerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineAssemblerTest.java new file mode 100644 index 0000000000..1c1b95ab39 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineAssemblerTest.java @@ -0,0 +1,116 @@ +package com.bakdata.conquery.sql.compiler.ir.concept; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.Date; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import org.junit.jupiter.api.Test; + +class ConnectorCtePipelineAssemblerTest { + + @Test + void shouldAssemblePreprocessingAndPipelineInputs() { + ConnectorCtePlan plan = plan(); + SqlIdColumns ids = new SqlIdColumns(field(name("events", "person"), String.class)); + ColumnDateRange rawValidityDate = ColumnDateRange.of( + field(name("events", "valid_from"), Date.class), + field(name("events", "valid_to"), Date.class) + ); + FieldWrapper directSelect = new FieldWrapper<>(field(name("events", "score"), Integer.class)); + FieldWrapper filterSelect = new FieldWrapper<>(field(name("events", "threshold"), Integer.class)); + ConnectorSqlSelects selects = ConnectorSqlSelects.builder() + .preprocessingSelect(directSelect) + .build(); + SqlFilters filters = new SqlFilters( + ConnectorSqlSelects.builder().preprocessingSelect(filterSelect).build(), + WhereClauses.builder().build() + ); + QueryStep stratificationTable = queryStep("stratification"); + + ConnectorCtePipelineInput input = ConnectorCtePipelineAssembler.assemble( + plan, + QueryStep.toTableLike("events"), + ids, + rawValidityDate, + List.of(selects), + List.of(filters), + Optional.of(stratificationTable) + ); + + assertSame(plan.tables(), input.tables()); + assertEquals("events", input.preprocessing().sourceTable().asTable().getName()); + assertSame(ids, input.preprocessing().ids()); + assertSame(rawValidityDate, input.preprocessing().rawValidityDate()); + assertEquals("connector_validity_date", input.preprocessing().validityDate().getAlias()); + assertEquals(List.of(selects, filters.getSelects()), input.preprocessing().sqlSelects()); + assertEquals(List.of(filters), input.preprocessing().sqlFilters()); + assertEquals(Optional.of(stratificationTable), input.preprocessing().stratificationTable()); + assertTrue(input.withIntervalPacking()); + assertTrue(input.excludedFromTimeAggregation()); + } + + @Test + void shouldCollectEventSelectsAndAllAdditionalPredecessors() { + SqlSelect directEventSelect = new FieldWrapper<>(field(name("direct_event"), Date.class)); + SqlSelect filterEventSelect = new FieldWrapper<>(field(name("filter_event"), Date.class)); + QueryStep directPredecessor = queryStep("direct_predecessor"); + QueryStep filterPredecessor = queryStep("filter_predecessor"); + ConnectorSqlSelects selects = ConnectorSqlSelects.builder() + .eventDateSelect(directEventSelect) + .additionalPredecessor(Optional.of(directPredecessor)) + .build(); + SqlFilters filters = new SqlFilters( + ConnectorSqlSelects.builder() + .eventDateSelect(filterEventSelect) + .additionalPredecessor(Optional.of(filterPredecessor)) + .build(), + WhereClauses.builder().build() + ); + + ConnectorCtePipelineInput input = ConnectorCtePipelineAssembler.assemble( + plan(), + QueryStep.toTableLike("events"), + new SqlIdColumns(field(name("person"), String.class)), + ColumnDateRange.of(field(name("start"), Date.class), field(name("end"), Date.class)), + List.of(selects), + List.of(filters), + Optional.empty() + ); + + assertEquals(List.of(directEventSelect), input.eventDateSelects()); + assertEquals(List.of(directPredecessor, filterPredecessor), input.additionalPredecessors()); + } + + private static ConnectorCtePlan plan() { + SqlTables tables = new SqlTables( + "events", + Map.of(ConceptCteStep.PREPROCESSING, "connector-preprocessing"), + Map.of() + ); + return new ConnectorCtePlan("connector", tables, true, true); + } + + private static QueryStep queryStep(String cteName) { + return QueryStep.builder() + .cteName(cteName) + .selects(Selects.builder() + .ids(new SqlIdColumns(field(name("person"), String.class))) + .build()) + .build(); + } +} From fb16af6210d5ae3f76a396942ca5e13e1799d495 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 17 Sep 2026 08:05:13 +0200 Subject: [PATCH 56/81] refactor(sql): decouple filter table state Expose connector-owned SqlTables directly to legacy filter conversion and shared aggregation helpers. Keep ConnectorSqlTables limited to select paths that still require backend Connector metadata. Co-authored-by: Codex GPT-5.6-sol --- .../apiv1/query/concept/filter/FilterValue.java | 4 ++-- .../conversion/cqelement/concept/FilterContext.java | 5 +++-- .../model/aggregator/CountQuartersSqlAggregator.java | 9 +++++---- .../model/aggregator/CountSqlAggregator.java | 7 ++++--- .../model/aggregator/DateDistanceSqlAggregator.java | 4 ++-- .../conversion/model/aggregator/SumSqlAggregator.java | 10 +++++----- .../conversion/model/filter/NumberFilterConverter.java | 4 ++-- .../conversion/model/select/DaterangeSelectUtil.java | 7 ++++--- 8 files changed, 27 insertions(+), 23 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java index 5b34a5cf90..3f82b387fb 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java @@ -15,8 +15,8 @@ import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.identifiable.ids.specific.FilterId; import com.bakdata.conquery.models.query.QueryResolveContext; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; @@ -58,7 +58,7 @@ public VALUE readValue() { return ((VALUE) value); } - public SqlFilters convertToSqlFilter(SqlIdColumns ids, ConversionContext context, ConnectorSqlTables tables) { + public SqlFilters convertToSqlFilter(SqlIdColumns ids, ConversionContext context, SqlTables tables) { FilterContext filterContext = FilterContext.forConceptConversion(ids, readValue(), context, tables); final Filter resolve = (Filter) filter.resolve(); SqlFilters sqlFilters = resolve.createConverter().convertToSqlFilter(resolve, filterContext); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java index 9108d93013..004306ab90 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java @@ -8,6 +8,7 @@ import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Value; @@ -29,9 +30,9 @@ public class FilterContext implements Context { * Not present if this context is for table export. */ @Nullable - ConnectorSqlTables tables; + SqlTables tables; - public static FilterContext forConceptConversion(SqlIdColumns ids, V value, ConversionContext conversionContext, ConnectorSqlTables tables) { + public static FilterContext forConceptConversion(SqlIdColumns ids, V value, ConversionContext conversionContext, SqlTables tables) { return new FilterContext<>(ids, value, conversionContext, tables); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java index 72a2b92abb..0100eb56dc 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; import com.bakdata.conquery.sql.compiler.ir.concept.CommonAggregationSelect; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.compiler.dialect.Interval; @@ -36,7 +37,7 @@ public class CountQuartersSqlAggregator implements SelectConverter createSingleDateColumnAggregationSelect( Column countColumn, String alias, - ConnectorSqlTables tables, + SqlTables tables, SqlFunctionProvider functionProvider) { ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), countColumn.getName(), Date.class); @@ -52,7 +53,7 @@ private static CommonAggregationSelect createTwoDateColumnAggregatio Column startColumn, Column endColumn, String alias, - ConnectorSqlTables tables, + SqlTables tables, SqlFunctionProvider functionProvider, StratificationFunctions stratificationFunctions) { @@ -69,7 +70,7 @@ private static CommonAggregationSelect sumQuarterCount( Field quarterStart, Field nextQuarterStart, String alias, - ConnectorSqlTables tables, + SqlTables tables, SqlFunctionProvider functionProvider) { Field quarterCount = calcQuarterCount(quarterStart, nextQuarterStart, alias, functionProvider); FieldWrapper quarterCountWrapper = new FieldWrapper<>(quarterCount); @@ -89,7 +90,7 @@ private static CommonAggregationSelect buildSqlSelect( ColumnId startColumn, ColumnId endColumn, String alias, - ConnectorSqlTables tables, + SqlTables tables, SqlFunctionProvider functionProvider, StratificationFunctions stratificationFunctions) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java index c132e941be..aa8831cb38 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java @@ -6,6 +6,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.CountSelect; import com.bakdata.conquery.sql.compiler.ir.concept.CommonAggregationSelect; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; @@ -29,7 +30,7 @@ public class CountSqlAggregator implements SelectConverter, FilterC @Override public ConnectorSqlSelects connectorSelect(CountSelect countSelect, SelectContext selectContext) { - ConnectorSqlTables tables = selectContext.getTables(); + SqlTables tables = selectContext.getTables(); boolean distinct = countSelect.isDistinct(); Column countColumn = countSelect.getColumn().resolve(); String alias = selectContext.getNameGenerator().legacyOperationName(countSelect.getName()); @@ -46,7 +47,7 @@ public ConnectorSqlSelects connectorSelect(CountSelect countSelect, SelectContex .build(); } - private CommonAggregationSelect createCountAggregationSelect(Column countColumn, boolean distinct, String alias, ConnectorSqlTables tables) { + private CommonAggregationSelect createCountAggregationSelect(Column countColumn, boolean distinct, String alias, SqlTables tables) { ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), countColumn.getName(), Object.class); @@ -66,7 +67,7 @@ private CommonAggregationSelect createCountAggregationSelect(Column cou @Override public SqlFilters convertToSqlFilter(CountFilter countFilter, FilterContext filterContext) { - ConnectorSqlTables tables = filterContext.getTables(); + SqlTables tables = filterContext.getTables(); boolean distinct = countFilter.isDistinct(); Column countColumn = countFilter.getColumn().resolve(); String alias = filterContext.getNameGenerator().legacyOperationName(countFilter.getName()); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java index 94eac1dd1a..1359ff7ee4 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java @@ -37,7 +37,7 @@ public ConnectorSqlSelects connectorSelect(DateDistanceSelect select, SelectCont Column column = select.getColumn().resolve(); String alias = selectContext.getNameGenerator().legacyOperationName(select.getName()); - ConnectorSqlTables tables = selectContext.getTables(); + SqlTables tables = selectContext.getTables(); Field dateDistanceCalculation = createDateDistanceCalculation(column, select.getTimeUnit(), tables, selectContext.getConversionContext()); FieldWrapper dateDistanceSelect = new FieldWrapper<>(dateDistanceCalculation.as(alias)); @@ -59,7 +59,7 @@ public ConnectorSqlSelects connectorSelect(DateDistanceSelect select, SelectCont public SqlFilters convertToSqlFilter(DateDistanceFilter filter, FilterContext filterContext) { Column column = filter.getColumn().resolve(); - ConnectorSqlTables tables = filterContext.getTables(); + SqlTables tables = filterContext.getTables(); Field dateDistanceCalculation = createDateDistanceCalculation(column, filter.getTimeUnit(), tables, filterContext.getConversionContext()); WhereCondition dateDistanceCondition = new LegacyInclusiveRangeCondition(dateDistanceCalculation, filterContext.getValue()); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java index 82a474609a..eb4d7281ea 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java @@ -96,7 +96,7 @@ public ConnectorSqlSelects connectorSelect(SumSelect sumSelect, SelectContext distinctByColumns = sumSelect.getDistinctByColumn().stream().map(ColumnId::resolve).toList(); - ConnectorSqlTables tables = selectContext.getTables(); + SqlTables tables = selectContext.getTables(); CommonAggregationSelect sumAggregationSelect; @@ -126,7 +126,7 @@ private CommonAggregationSelect createDistinctSumAggregationSelect( List distinctByColumns, String alias, SqlIdColumns ids, - ConnectorSqlTables tables, + SqlTables tables, SqlNameGenerator nameGenerator ) { List> preprocessingSelects = new ArrayList<>(); @@ -153,12 +153,12 @@ private CommonAggregationSelect createDistinctSumAggregationSelect( .build(); } - private static ExtractingSqlSelect createFinalSelect(CommonAggregationSelect sumAggregationSelect, ConnectorSqlTables tables) { + private static ExtractingSqlSelect createFinalSelect(CommonAggregationSelect sumAggregationSelect, SqlTables tables) { String finalPredecessor = tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER); return sumAggregationSelect.getGroupBy().qualify(finalPredecessor); } - private CommonAggregationSelect createSumAggregationSelect(Column sumColumn, Column subtractColumn, String alias, ConnectorSqlTables tables) { + private CommonAggregationSelect createSumAggregationSelect(Column sumColumn, Column subtractColumn, String alias, SqlTables tables) { Class numberClass = NumberMapUtil.getType(sumColumn); List> preprocessingSelects = new ArrayList<>(); @@ -276,7 +276,7 @@ public SqlFilters convertToSqlFilter(SumFilter sumFilter, FilterContext distinctByColumns = sumFilter.getDistinctByColumn().stream().map(ColumnId::resolve).toList(); String alias = filterContext.getNameGenerator().legacyOperationName(sumFilter.getName()); - ConnectorSqlTables tables = filterContext.getTables(); + SqlTables tables = filterContext.getTables(); CommonAggregationSelect sumAggregationSelect; ConnectorSqlSelects selects; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java index 00798b03fe..75d0b14239 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java @@ -3,8 +3,8 @@ import com.bakdata.conquery.models.common.IRange; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.filters.specific.NumberFilter; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.model.NumberMapUtil; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; @@ -20,7 +20,7 @@ public class NumberFilterConverter> im public SqlFilters convertToSqlFilter(NumberFilter filter, FilterContext filterContext) { Column column = filter.getColumn().resolve(); - ConnectorSqlTables tables = filterContext.getTables(); + SqlTables tables = filterContext.getTables(); Class numberClass = NumberMapUtil.getType(column); ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), column.getName(), numberClass); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java index d54677281a..49c0cc0624 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java @@ -28,6 +28,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; @@ -71,7 +72,7 @@ public static ConnectorSqlSelects c context.getCompilerDialect() ); - ConnectorSqlTables tables = context.getTables(); + SqlTables tables = context.getTables(); ExtractingSqlSelect finalSelect = aggregationField.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); return ConnectorSqlSelects.builder() @@ -122,7 +123,7 @@ public static SqlFilters createForFilter( .additionalPredecessor(Optional.of(intervalPackingSelectsStep)) .build(); - ConnectorSqlTables tables = context.getTables(); + SqlTables tables = context.getTables(); Field qualifiedAggregationField = aggregationField.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)).select(); WhereClauses whereClauses = WhereClauses.builder().groupFilter(filterFunction.apply(qualifiedAggregationField)).build(); @@ -149,7 +150,7 @@ private static IntervalPackingSelectPreparation prepareIntervalSelect( String alias, ColumnDateRange daterange, SqlIdColumns ids, - ConnectorSqlTables tables, + SqlTables tables, Context context ) { return IntervalPackingSelectCompiler.prepareArbitrarySelect( From ae91f791a2386687d4b899b58298f1b9f68c64b0 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 17 Sep 2026 13:49:10 +0200 Subject: [PATCH 57/81] refactor(sql): retain resolved connector tables Carry physical connector tables through CTE planning and preprocessing so SQL generation preserves schema-qualified names from resolved models. Keep the legacy string planning entry point for the backend adapter, and derive the entity table from its resolved primary-ID column. Co-authored-by: Codex GPT-5.6-sol --- .../cqelement/concept/CQConceptConverter.java | 2 +- .../cqelement/concept/TablePath.java | 3 +- .../dialect/SqlFunctionProvider.java | 4 ++ .../conversion/model/EntitySchemaAdapter.java | 13 ++++- .../ir/AllEntitiesQueryStepCompiler.java | 4 +- .../sql/compiler/ir/EntitySchemaSql.java | 34 ----------- .../sql/compiler/ir/QueryStepComposer.java | 8 +-- .../conquery/sql/compiler/ir/SchemaSql.java | 38 ++++++++++++ .../ir/concept/ConceptCtePlanner.java | 58 ++++++++++++++++++- .../ir/concept/ConnectorCteCompiler.java | 4 +- .../ConnectorCtePipelineAssembler.java | 6 +- .../compiler/ir/concept/ConnectorCtePlan.java | 4 ++ .../ir/concept/PreprocessingCteInput.java | 16 +---- .../sql/model/schema/EntitySchema.java | 8 +-- .../ir/AllEntitiesQueryStepCompilerTest.java | 1 - .../ir/LogicalQueryStepCompilerTest.java | 1 - .../compiler/ir/QueryStepComposerTest.java | 1 - .../ir/concept/ConceptCtePlannerTest.java | 16 +++++ .../ir/concept/ConnectorCteCompilerTest.java | 3 +- .../ConnectorCtePipelineAssemblerTest.java | 9 +-- .../conquery/sql/model/ResolvedQueryTest.java | 8 +-- 21 files changed, 157 insertions(+), 84 deletions(-) delete mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/EntitySchemaSql.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/SchemaSql.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index efeea998cc..634c8a3625 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -265,7 +265,7 @@ private ConnectorCtePipelineInput createConnectorCteInput( ); } - private static org.jooq.Table createConceptSourceTable( + private static org.jooq.Table createConceptSourceTable( ConnectorSqlTables connectorTables, ConceptIdMapping conceptIdMapping, List> selectedConceptElements, diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java index 5438a011b0..5a627fd8a5 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java @@ -14,6 +14,7 @@ import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCtePlanner; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCtePlan; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; +import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; import lombok.Getter; /** @@ -49,7 +50,7 @@ private static ConnectorSqlTables createConnectorTables(CQConcept cqConcept, CQT .map(SelectId::resolve) .anyMatch(Select::isEventDateSelect); ConnectorCtePlan plan = ConceptCtePlanner.planConnector( - connector.resolveTableId().getTable(), + EntitySchemaAdapter.from(connector.getResolvedTable()), connectorName, cqConcept.isAggregateEventDates(), eventDateSelectsPresent, diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index bbc7b2bda0..e291329a52 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -237,6 +237,10 @@ default TableOnConditionStep fullOuterJoin(Table leftPart, Table r return leftPart.fullOuterJoin(rightPart).on(joinConditions.toArray(Condition[]::new)); } + default TableOnConditionStep innerJoin(Table leftPart, Table rightPart, List joinConditions) { + return leftPart.innerJoin(rightPart).on(joinConditions.toArray(Condition[]::new)); + } + default TableOnConditionStep leftJoin(Table leftPart, Table rightPart, List joinConditions) { return leftPart.leftJoin(rightPart).on(joinConditions.toArray(Condition[]::new)); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/EntitySchemaAdapter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/EntitySchemaAdapter.java index beef7d5811..7a7b7aa66b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/EntitySchemaAdapter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/EntitySchemaAdapter.java @@ -3,11 +3,16 @@ import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.models.config.IdColumnConfig; import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.sql.model.schema.EntitySchema; import com.bakdata.conquery.sql.model.schema.ResolvedColumn; import com.bakdata.conquery.sql.model.schema.SqlTable; -/** Converts the legacy entity-ID configuration into the connector's resolved entity schema. */ +/** + * Converts legacy backend schema objects into the connector's resolved schema model. + * + *

TODO Remove this adapter once backend query resolution produces resolved connector schema objects directly.

+ */ public final class EntitySchemaAdapter { private EntitySchemaAdapter() { @@ -23,6 +28,10 @@ public static EntitySchema from(IdColumnConfig idColumns) { ColumnType.STRING, false ); - return new EntitySchema(entityTable, primaryIdColumn); + return new EntitySchema(primaryIdColumn); + } + + public static SqlTable from(Table table) { + return SqlTable.of(table.getId().toString(), table.getName()); } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/AllEntitiesQueryStepCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/AllEntitiesQueryStepCompiler.java index 87d935f35c..df642e2f20 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/AllEntitiesQueryStepCompiler.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/AllEntitiesQueryStepCompiler.java @@ -21,7 +21,7 @@ public static QueryStep compile( ) { Objects.requireNonNull(allEntitiesNode, "allEntitiesNode"); Selects selects = Selects.builder() - .ids(new SqlIdColumns(EntitySchemaSql.primaryId(entitySchema))) + .ids(new SqlIdColumns(SchemaSql.field(entitySchema.primaryId(), String.class))) .validityDate(Optional.of( compilerDialect.emptyDateRange().asValidityDateRange(ALL_ENTITIES_CTE) )) @@ -29,7 +29,7 @@ public static QueryStep compile( return QueryStep.builder() .cteName(ALL_ENTITIES_CTE) .selects(selects) - .fromTable(EntitySchemaSql.table(entitySchema)) + .fromTable(SchemaSql.table(entitySchema.table())) .build(); } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/EntitySchemaSql.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/EntitySchemaSql.java deleted file mode 100644 index bb822b485f..0000000000 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/EntitySchemaSql.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.bakdata.conquery.sql.compiler.ir; - -import static org.jooq.impl.DSL.field; -import static org.jooq.impl.DSL.name; - -import java.util.ArrayList; -import java.util.List; - -import com.bakdata.conquery.sql.model.schema.EntitySchema; -import lombok.experimental.UtilityClass; -import org.jooq.Field; -import org.jooq.Name; -import org.jooq.Record; -import org.jooq.Table; -import org.jooq.impl.DSL; - -/** Creates SQL identifiers for a resolved entity schema. */ -@UtilityClass -final class EntitySchemaSql { - - static Table table(EntitySchema entitySchema) { - return DSL.table(tableName(entitySchema)); - } - - static Name tableName(EntitySchema entitySchema) { - return name(entitySchema.table().physicalName().toArray(String[]::new)); - } - - static Field primaryId(EntitySchema entitySchema) { - List qualifiedName = new ArrayList<>(entitySchema.table().physicalName()); - qualifiedName.add(entitySchema.primaryId().physicalName()); - return field(name(qualifiedName.toArray(String[]::new)), String.class); - } -} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposer.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposer.java index 2b96658b47..6342b4534d 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposer.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposer.java @@ -38,9 +38,9 @@ public static QueryStep antiJoinWithAllEntities( CompilerDialect compilerDialect ) { Field queryStepPrimaryColumn = queryStep.getQualifiedSelects().getIds().getPrimaryColumn(); - Field allIdsPrimaryColumn = EntitySchemaSql.primaryId(entitySchema); + Field allIdsPrimaryColumn = SchemaSql.field(entitySchema.primaryId(), String.class); - Table joinedTable = table(EntitySchemaSql.tableName(entitySchema)) + Table joinedTable = table(SchemaSql.tableName(entitySchema.table())) .leftOuterJoin(table(name(queryStep.getCteName()))) .on(allIdsPrimaryColumn.eq(queryStepPrimaryColumn)); @@ -194,7 +194,7 @@ private static QueryStep joinStepsContainingNegation( } negateJoined = DateAggregationCompiler.invert(negateJoined, compilerDialect, nameGenerator); - Field allIdsPrimaryColumn = EntitySchemaSql.primaryId(entitySchema); + Field allIdsPrimaryColumn = SchemaSql.field(entitySchema.primaryId(), String.class); Field negatePrimaryColumn = negateJoined.getQualifiedSelects().getIds().getPrimaryColumn(); Field nonNegatePrimaryColumn = nonNegateJoined.getQualifiedSelects().getIds().getPrimaryColumn(); @@ -213,7 +213,7 @@ private static QueryStep joinStepsContainingNegation( .ids(new SqlIdColumns(coalescedId)) .validityDate(Optional.of(merged)); - Table joinedTable = table(EntitySchemaSql.tableName(entitySchema)) + Table joinedTable = table(SchemaSql.tableName(entitySchema.table())) .leftOuterJoin(table(name(negateJoined.getCteName()))) .on(allIdsPrimaryColumn.eq(negatePrimaryColumn)) .leftOuterJoin(table(name(nonNegateJoined.getCteName()))) diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/SchemaSql.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/SchemaSql.java new file mode 100644 index 0000000000..f60921a8b9 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/SchemaSql.java @@ -0,0 +1,38 @@ +package com.bakdata.conquery.sql.compiler.ir; + +import static org.jooq.impl.DSL.name; + +import java.util.ArrayList; +import java.util.List; + +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.SqlTable; +import lombok.experimental.UtilityClass; +import org.jooq.Field; +import org.jooq.Name; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.impl.DSL; + +/** Creates SQL identifiers for resolved physical schema objects. */ +@UtilityClass +public final class SchemaSql { + + public static Table table(SqlTable table) { + return DSL.table(tableName(table)); + } + + public static Name tableName(SqlTable table) { + return name(table.physicalName().toArray(String[]::new)); + } + + public static Field field(ResolvedColumn column, Class type) { + return DSL.field(columnName(column), type); + } + + public static Name columnName(ResolvedColumn column) { + List qualifiedName = new ArrayList<>(column.table().physicalName()); + qualifiedName.add(column.physicalName()); + return name(qualifiedName.toArray(String[]::new)); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCtePlanner.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCtePlanner.java index d31345f7f4..f3f324eaf3 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCtePlanner.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCtePlanner.java @@ -12,10 +12,15 @@ import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import com.bakdata.conquery.sql.compiler.ir.CteStep; import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.SchemaSql; import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingCteStep; import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import com.bakdata.conquery.sql.model.schema.SqlTable; import lombok.experimental.UtilityClass; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.impl.DSL; /** Plans the CTE graphs required to compile a resolved concept and its connectors. */ @UtilityClass @@ -29,11 +34,58 @@ public ConnectorCtePlan planConnector( boolean eventDateSelectsPresent, CompilerDialect dialect, SqlNameGenerator nameGenerator + ) { + return planConnector( + DSL.table(DSL.name(rootTable)), + rootTable, + connectorName, + aggregateEventDates, + eventDateSelectsPresent, + dialect, + nameGenerator + ); + } + + /** Plan one connector branch from a fully resolved physical source table. */ + public ConnectorCtePlan planConnector( + SqlTable sourceTable, + String connectorName, + boolean aggregateEventDates, + boolean eventDateSelectsPresent, + CompilerDialect dialect, + SqlNameGenerator nameGenerator + ) { + String rootQualifier = sourceTable.physicalName().getLast(); + return planConnector( + SchemaSql.table(sourceTable), + rootQualifier, + connectorName, + aggregateEventDates, + eventDateSelectsPresent, + dialect, + nameGenerator + ); + } + + private ConnectorCtePlan planConnector( + Table sourceTable, + String rootQualifier, + String connectorName, + boolean aggregateEventDates, + boolean eventDateSelectsPresent, + CompilerDialect dialect, + SqlNameGenerator nameGenerator ) { Map mappings = CteStep.getDefaultPredecessorMap(ConceptCteStep.MANDATORY_STEPS); boolean withIntervalPacking = aggregateEventDates || eventDateSelectsPresent; if (!withIntervalPacking) { - return new ConnectorCtePlan(connectorName, createTables(rootTable, connectorName, mappings, nameGenerator), false, false); + return new ConnectorCtePlan( + connectorName, + sourceTable, + createTables(rootQualifier, connectorName, mappings, nameGenerator), + false, + false + ); } mappings.putAll(IntervalPackingCteStep.getMappings(PREPROCESSING, dialect)); @@ -41,8 +93,8 @@ public ConnectorCtePlan planConnector( addIntervalSelectMappings(mappings, INTERVAL_COMPLETE, dialect); } - SqlTables tables = createTables(rootTable, connectorName, mappings, nameGenerator); - return new ConnectorCtePlan(connectorName, tables, true, !aggregateEventDates); + SqlTables tables = createTables(rootQualifier, connectorName, mappings, nameGenerator); + return new ConnectorCtePlan(connectorName, sourceTable, tables, true, !aggregateEventDates); } /** Plan the final concept CTE graph over the converted connector branches. */ diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java index 2c9986592e..fc71967751 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java @@ -89,8 +89,8 @@ public static QueryStep.QueryStepBuilder compilePreprocessing(PreprocessingCteIn )); conditions.add(new DateRestrictionCondition(stratificationDate, input.rawValidityDate()).condition()); - Table connectorTable = input.sourceTable().asTable(); - Table joinedTable = connectorTable.innerJoin(stratificationTable) + Table connectorTable = input.sourceTable(); + Table joinedTable = connectorTable.innerJoin(stratificationTable) .on(idConditions.toArray(Condition[]::new)); Selects selects = Selects.builder() .ids(stratificationSelects.getIds()) diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineAssembler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineAssembler.java index 4096ebc999..4f3b2eb50c 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineAssembler.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineAssembler.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.sql.compiler.ir.concept; -import static com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep.PREPROCESSING; - import java.util.List; import java.util.Optional; import java.util.stream.Stream; @@ -12,7 +10,7 @@ import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import lombok.experimental.UtilityClass; import org.jooq.Record; -import org.jooq.TableLike; +import org.jooq.Table; /** Assembles SQL-resolved connector operations into the input for the connector CTE pipeline. */ @UtilityClass @@ -20,7 +18,7 @@ public class ConnectorCtePipelineAssembler { public ConnectorCtePipelineInput assemble( ConnectorCtePlan plan, - TableLike sourceTable, + Table sourceTable, SqlIdColumns ids, ColumnDateRange rawValidityDate, List selects, diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePlan.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePlan.java index 0e11656d48..080764a632 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePlan.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePlan.java @@ -3,10 +3,13 @@ import java.util.Objects; import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import org.jooq.Record; +import org.jooq.Table; /** Connector-owned CTE graph together with the execution flags derived while planning it. */ public record ConnectorCtePlan( String connectorName, + Table sourceTable, SqlTables tables, boolean withIntervalPacking, boolean excludedFromTimeAggregation @@ -14,6 +17,7 @@ public record ConnectorCtePlan( public ConnectorCtePlan { Objects.requireNonNull(connectorName, "connectorName"); + Objects.requireNonNull(sourceTable, "sourceTable"); Objects.requireNonNull(tables, "tables"); } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/PreprocessingCteInput.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/PreprocessingCteInput.java index d1c6e1d9e0..eeab2fd607 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/PreprocessingCteInput.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/PreprocessingCteInput.java @@ -8,11 +8,11 @@ import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import org.jooq.Record; -import org.jooq.TableLike; +import org.jooq.Table; /** Fully SQL-resolved input needed to build a connector preprocessing CTE. */ public record PreprocessingCteInput( - TableLike sourceTable, + Table sourceTable, SqlIdColumns ids, ColumnDateRange rawValidityDate, ColumnDateRange validityDate, @@ -30,16 +30,4 @@ public record PreprocessingCteInput( sqlFilters = List.copyOf(sqlFilters); Objects.requireNonNull(stratificationTable, "stratificationTable"); } - - public PreprocessingCteInput( - String sourceTable, - SqlIdColumns ids, - ColumnDateRange rawValidityDate, - ColumnDateRange validityDate, - List sqlSelects, - List sqlFilters, - Optional stratificationTable - ) { - this(QueryStep.toTableLike(sourceTable), ids, rawValidityDate, validityDate, sqlSelects, sqlFilters, stratificationTable); - } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/EntitySchema.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/EntitySchema.java index 70c1d918f6..e9d3b7c642 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/EntitySchema.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/EntitySchema.java @@ -3,17 +3,15 @@ import com.bakdata.conquery.models.datasets.ColumnType; import com.bakdata.conquery.sql.validation.AllowedColumnTypes; import jakarta.validation.Valid; -import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.NotNull; /** Physical source used to enumerate all entities, for example when compiling a root-level negation. */ public record EntitySchema( - @NotNull @Valid SqlTable table, @NotNull @Valid @AllowedColumnTypes(ColumnType.STRING) ResolvedColumn primaryId ) { - @AssertTrue(message = "primaryId must belong to the entity table") - public boolean isPrimaryIdOnEntityTable() { - return table == null || primaryId == null || table.equals(primaryId.table()); + /** Physical table containing all entities. */ + public SqlTable table() { + return primaryId == null ? null : primaryId.table(); } } diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/AllEntitiesQueryStepCompilerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/AllEntitiesQueryStepCompilerTest.java index 7c1428977c..1fef7c267c 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/AllEntitiesQueryStepCompilerTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/AllEntitiesQueryStepCompilerTest.java @@ -20,7 +20,6 @@ class AllEntitiesQueryStepCompilerTest { private static final SqlTable ENTITIES = SqlTable.of("entities", "catalog", "entities"); private static final EntitySchema ENTITY_SCHEMA = new EntitySchema( - ENTITIES, new ResolvedColumn("entity-id", ENTITIES, "person_id", ColumnType.STRING, false) ); diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/LogicalQueryStepCompilerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/LogicalQueryStepCompilerTest.java index 062ad556c2..1bb922ea4f 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/LogicalQueryStepCompilerTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/LogicalQueryStepCompilerTest.java @@ -23,7 +23,6 @@ class LogicalQueryStepCompilerTest { private static final CompilerDialect DIALECT = new TestDialect(); private static final SqlTable ENTITY_TABLE = SqlTable.of("entities", "catalog", "entities"); private static final EntitySchema ENTITY_SCHEMA = new EntitySchema( - ENTITY_TABLE, new ResolvedColumn("person", ENTITY_TABLE, "person_id", ColumnType.STRING, false) ); diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposerTest.java index 2385b44d59..4fa8c64b06 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposerTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/QueryStepComposerTest.java @@ -28,7 +28,6 @@ class QueryStepComposerTest { private static final CompilerDialect DIALECT = new TestDialect(); private static final SqlTable ENTITY_TABLE = SqlTable.of("entities", "catalog", "entities"); private static final EntitySchema ENTITY_SCHEMA = new EntitySchema( - ENTITY_TABLE, new ResolvedColumn("person", ENTITY_TABLE, "person_id", ColumnType.STRING, false) ); diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCtePlannerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCtePlannerTest.java index eeb1fc44bc..03994f90c8 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCtePlannerTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConceptCtePlannerTest.java @@ -18,6 +18,7 @@ import java.sql.Date; import java.util.Optional; +import com.bakdata.conquery.sql.model.schema.SqlTable; import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; @@ -69,6 +70,21 @@ void shouldPlanConnectorIntervalSelectsWithoutPropagatingTheirValidityDate() { assertEquals("connector-interval_complete", plan.tables().getPredecessor(INTERVAL_PACKING_SELECTS)); } + @Test + void shouldRetainResolvedPhysicalConnectorTable() { + ConnectorCtePlan plan = ConceptCtePlanner.planConnector( + SqlTable.of("events", "catalog", "events"), + "connector", + false, + false, + new TestDialect(false), + new SqlNameGenerator(128) + ); + + assertEquals(name("catalog", "events"), plan.sourceTable().getQualifiedName()); + assertEquals("events", plan.tables().getPredecessor(PREPROCESSING)); + } + @Test void shouldPlanUniversalSelectDirectlyOverConceptRoot() { SqlTables tables = ConceptCtePlanner.planConcept( diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java index c578decdbf..6207d0513b 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java @@ -2,6 +2,7 @@ import static org.jooq.impl.DSL.field; import static org.jooq.impl.DSL.name; +import static org.jooq.impl.DSL.table; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -291,7 +292,7 @@ private static PreprocessingCteInput preprocessingInput(Optional stra .build() ); return new PreprocessingCteInput( - "events", + table(name("events")), ids, rawValidityDate, rawValidityDate.asValidityDateRange("connector"), diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineAssemblerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineAssemblerTest.java index 1c1b95ab39..20870a5510 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineAssemblerTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineAssemblerTest.java @@ -2,6 +2,7 @@ import static org.jooq.impl.DSL.field; import static org.jooq.impl.DSL.name; +import static org.jooq.impl.DSL.table; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -44,7 +45,7 @@ void shouldAssemblePreprocessingAndPipelineInputs() { ConnectorCtePipelineInput input = ConnectorCtePipelineAssembler.assemble( plan, - QueryStep.toTableLike("events"), + org.jooq.impl.DSL.table(name("events")), ids, rawValidityDate, List.of(selects), @@ -53,7 +54,7 @@ void shouldAssemblePreprocessingAndPipelineInputs() { ); assertSame(plan.tables(), input.tables()); - assertEquals("events", input.preprocessing().sourceTable().asTable().getName()); + assertEquals(name("events"), input.preprocessing().sourceTable().getQualifiedName()); assertSame(ids, input.preprocessing().ids()); assertSame(rawValidityDate, input.preprocessing().rawValidityDate()); assertEquals("connector_validity_date", input.preprocessing().validityDate().getAlias()); @@ -84,7 +85,7 @@ void shouldCollectEventSelectsAndAllAdditionalPredecessors() { ConnectorCtePipelineInput input = ConnectorCtePipelineAssembler.assemble( plan(), - QueryStep.toTableLike("events"), + org.jooq.impl.DSL.table(name("events")), new SqlIdColumns(field(name("person"), String.class)), ColumnDateRange.of(field(name("start"), Date.class), field(name("end"), Date.class)), List.of(selects), @@ -102,7 +103,7 @@ private static ConnectorCtePlan plan() { Map.of(ConceptCteStep.PREPROCESSING, "connector-preprocessing"), Map.of() ); - return new ConnectorCtePlan("connector", tables, true, true); + return new ConnectorCtePlan("connector", table(name("events")), tables, true, true); } private static QueryStep queryStep(String cteName) { diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java index 4971d83b43..e33bd395f3 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java @@ -51,7 +51,7 @@ void shouldCreateResolvedQueryWithoutFrameworkTypes() { ); ResolvedQuery query = new ResolvedQuery( - new EntitySchema(EVENTS, ENTITY_ID), + new EntitySchema(ENTITY_ID), concept, true, List.of(new ResultColumn("entity", ResultType.Primitive.STRING)) @@ -101,7 +101,7 @@ void shouldRejectNonStringEntityIds() { "dataset.events.numeric_id", EVENTS, "numeric_id", ColumnType.INTEGER, false ); - assertInvalid(new EntitySchema(EVENTS, numericId)); + assertInvalid(new EntitySchema(numericId)); } @Test @@ -130,7 +130,7 @@ void shouldValidateTheCompleteQueryGraphAtTheCompilerBoundary() { List.of() ); ResolvedQuery query = new ResolvedQuery( - new EntitySchema(EVENTS, ENTITY_ID), + new EntitySchema(ENTITY_ID), new ConceptNode("concept", List.of(invalidConnector), List.of(), DateAggregationAction.MERGE), true, List.of() @@ -145,7 +145,7 @@ void shouldValidateTheCompleteQueryGraphAtTheCompilerBoundary() { @Test void shouldRejectAmbiguousResultColumnIds() { ResolvedQuery query = new ResolvedQuery( - new EntitySchema(EVENTS, ENTITY_ID), + new EntitySchema(ENTITY_ID), conceptNode(EVENTS), false, List.of( From b2d19cb173982e1f812e6324b8950dc52e3240b4 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Fri, 18 Sep 2026 08:29:02 +0200 Subject: [PATCH 58/81] refactor(sql): keep query step builders internal Return complete query steps from connector CTE compilation instead of exposing mutable builders across method boundaries. Let concept and date aggregation stages own their CTE metadata and predecessor relationships, and keep individual connector stages package-private. Co-authored-by: Codex GPT-5.6-sol --- .../ir/aggregation/DateAggregationCte.java | 16 +---- .../ir/aggregation/IntermediateTableCte.java | 8 ++- .../compiler/ir/aggregation/InvertCte.java | 7 +- .../sql/compiler/ir/aggregation/MergeCte.java | 8 ++- .../ir/aggregation/NodeNoOverlapCte.java | 14 ++-- .../compiler/ir/aggregation/OverlapCte.java | 8 ++- .../compiler/ir/aggregation/RowNumberCte.java | 11 ++- .../ir/concept/ConnectorCteCompiler.java | 72 ++++++++++--------- .../ir/concept/ConnectorCteCompilerTest.java | 27 ++++--- 9 files changed, 95 insertions(+), 76 deletions(-) diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationCte.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationCte.java index 3ed7e5147b..362d40e2b1 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationCte.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/DateAggregationCte.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.sql.compiler.ir.aggregation; -import java.util.List; - import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.SqlTables; @@ -22,20 +20,10 @@ public QueryStep convert(DateAggregationContext context, QueryStep previous) { String predecessor = dateAggregationTables.getPredecessor(cteStep); context = context.qualify(predecessor); - QueryStep.QueryStepBuilder builder = this.convertStep(context, predecessor); - - if (cteStep != DateAggregationCteStep.NODE_NO_OVERLAP) { - builder = builder.cteName(dateAggregationTables.cteName(cteStep)) - .predecessors(List.of(previous)); - } - if (cteStep != DateAggregationCteStep.INVERT && cteStep != DateAggregationCteStep.NODE_NO_OVERLAP) { - builder = builder.fromTable(QueryStep.toTableLike(predecessor)); - } - - return builder.build(); + return this.convertStep(context, predecessor, previous); } - protected abstract QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, String predecessor); + protected abstract QueryStep convertStep(DateAggregationContext context, String predecessor, QueryStep previous); public abstract DateAggregationCteStep getCteStep(); diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/IntermediateTableCte.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/IntermediateTableCte.java index 322d2dc673..d0de72225b 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/IntermediateTableCte.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/IntermediateTableCte.java @@ -23,7 +23,7 @@ public IntermediateTableCte(DateAggregationCteStep cteStep) { } @Override - protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, String predecessor) { + protected QueryStep convertStep(DateAggregationContext context, String predecessor, QueryStep previous) { List intermediateTableSelects = context.getSqlAggregationAction().getIntermediateTableSelects( @@ -49,8 +49,12 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, Condition intermediateTableCondition = startIsNull.orNot(startBeforeEnd); return QueryStep.builder() + .cteName(context.getDateAggregationTables().cteName(getCteStep())) .selects(selects) - .conditions(List.of(intermediateTableCondition)); + .fromTable(QueryStep.toTableLike(predecessor)) + .conditions(List.of(intermediateTableCondition)) + .predecessors(List.of(previous)) + .build(); } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/InvertCte.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/InvertCte.java index d42e5256b6..0c35b0cb71 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/InvertCte.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/InvertCte.java @@ -38,7 +38,7 @@ public InvertCte(DateAggregationCteStep cteStep) { } @Override - protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, String predecessor) { + protected QueryStep convertStep(DateAggregationContext context, String predecessor, QueryStep previous) { QueryStep rowNumberStep = context.getStep(DateAggregationCteStep.ROW_NUMBER); @@ -51,8 +51,11 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, TableOnConditionStep fromTable = selfJoinWithShiftedRows(leftIds, rightIds, rowNumberStep); return QueryStep.builder() + .cteName(context.getDateAggregationTables().cteName(getCteStep())) .selects(invertSelects) - .fromTable(fromTable); + .fromTable(fromTable) + .predecessors(List.of(previous)) + .build(); } private Selects getInvertSelects(QueryStep rowNumberStep, SqlIdColumns coalescedIds, DateAggregationContext context) { diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/MergeCte.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/MergeCte.java index 955354d018..ee754dede1 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/MergeCte.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/MergeCte.java @@ -16,7 +16,7 @@ public MergeCte(DateAggregationCteStep cteStep) { } @Override - protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, String predecessor) { + protected QueryStep convertStep(DateAggregationContext context, String predecessor, QueryStep previous) { SqlAggregationAction aggregationAction = context.getSqlAggregationAction(); List noOverlapSteps = aggregationAction.getNoOverlapSelects(context); @@ -25,8 +25,12 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, List unionSteps = noOverlapSteps.stream().map(MergeCte::createUnionStep).collect(Collectors.toList()); return QueryStep.builder() + .cteName(context.getDateAggregationTables().cteName(getCteStep())) .selects(overlapStep.getQualifiedSelects()) - .union(unionSteps); + .fromTable(QueryStep.toTableLike(predecessor)) + .union(unionSteps) + .predecessors(List.of(previous)) + .build(); } private static QueryStep createUnionStep(QueryStep noOverlapStep) { diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/NodeNoOverlapCte.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/NodeNoOverlapCte.java index c8b53a79c7..4dee17f55e 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/NodeNoOverlapCte.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/NodeNoOverlapCte.java @@ -25,7 +25,7 @@ public NodeNoOverlapCte(DateAggregationCteStep cteStep) { } @Override - protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, String predecessor) { + protected QueryStep convertStep(DateAggregationContext context, String predecessor, QueryStep previous) { // we create a no-overlap node for each query step we need to aggregate DateAggregationDates dateAggregationDates = context.getDateAggregationDates(); @@ -33,20 +33,19 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, QueryStep intermediateTableStep = context.getStep(DateAggregationCteStep.INTERMEDIATE_TABLE); // first no-overlap step has intermediate table as predecessor - QueryStep.QueryStepBuilder noOverlapStep = createNoOverlapStep(validityDates.next(), context, intermediateTableStep); + QueryStep noOverlapStep = createNoOverlapStep(validityDates.next(), context, intermediateTableStep); // each following step has it's predeceasing no-overlap as predecessor while (validityDates.hasNext()) { counter++; - QueryStep predeceasingNoOverlapStep = noOverlapStep.build(); - context = context.withStep(getCteStep(), predeceasingNoOverlapStep); - noOverlapStep = createNoOverlapStep(validityDates.next(), context, predeceasingNoOverlapStep); + context = context.withStep(getCteStep(), noOverlapStep); + noOverlapStep = createNoOverlapStep(validityDates.next(), context, noOverlapStep); } return noOverlapStep; } - private QueryStep.QueryStepBuilder createNoOverlapStep( + private QueryStep createNoOverlapStep( ColumnDateRange validityDate, DateAggregationContext context, QueryStep predecessor @@ -73,7 +72,8 @@ private QueryStep.QueryStepBuilder createNoOverlapStep( .selects(nodeNoOverlapSelects) .fromTable(QueryStep.toTableLike(intermediateTableCteName)) .conditions(List.of(startNotNull)) - .predecessors(List.of(predecessor)); + .predecessors(List.of(predecessor)) + .build(); } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/OverlapCte.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/OverlapCte.java index c7a9945b10..2e9079905c 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/OverlapCte.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/OverlapCte.java @@ -23,7 +23,7 @@ public OverlapCte(DateAggregationCteStep cteStep) { } @Override - protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, String predecessor) { + protected QueryStep convertStep(DateAggregationContext context, String predecessor, QueryStep previous) { DateAggregationDates dateAggregationDates = context.getDateAggregationDates(); List> allStarts = dateAggregationDates.allStarts(); @@ -44,8 +44,12 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, Condition overlapConditions = allStartsNotNull.and(startBeforeEnd); return QueryStep.builder() + .cteName(context.getDateAggregationTables().cteName(getCteStep())) .selects(overlapSelects) - .conditions(List.of(overlapConditions)); + .fromTable(QueryStep.toTableLike(predecessor)) + .conditions(List.of(overlapConditions)) + .predecessors(List.of(previous)) + .build(); } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/RowNumberCte.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/RowNumberCte.java index 1557509d6a..11f1087576 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/RowNumberCte.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/aggregation/RowNumberCte.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.sql.compiler.ir.aggregation; import java.util.ArrayList; +import java.util.List; import java.util.Optional; import com.bakdata.conquery.sql.compiler.ir.DateAggregationDates; @@ -28,11 +29,11 @@ public RowNumberCte(DateAggregationCteStep cteStep) { } @Override - protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, String predecessor) { + protected QueryStep convertStep(DateAggregationContext context, String predecessor, QueryStep previous) { SqlIdColumns ids = context.getIds(); - ColumnDateRange aggregatedValidityDate = context.getDateAggregationDates().getValidityDates().get(0); + ColumnDateRange aggregatedValidityDate = context.getDateAggregationDates().getValidityDates().getFirst(); Field rowNumber = DSL.rowNumber().over(DSL.partitionBy(ids.toFields()).orderBy(aggregatedValidityDate.getStart())) .as(ROW_NUMBER_FIELD_NAME); @@ -46,7 +47,11 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, .build(); return QueryStep.builder() - .selects(rowNumberSelects); + .cteName(context.getDateAggregationTables().cteName(getCteStep())) + .selects(rowNumberSelects) + .fromTable(QueryStep.toTableLike(predecessor)) + .predecessors(List.of(previous)) + .build(); } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java index fc71967751..ac031ca9fc 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompiler.java @@ -41,14 +41,12 @@ public static Optional compileConnector(ConnectorCtePipelineInput inp if (!input.tables().isRequiredStep(cteStep)) { continue; } - QueryStep.QueryStepBuilder builder = compileConnectorStep(input, cteStep, previous) - .cteName(input.tables().cteName(cteStep)); - previous = Optional.of(completeStep(builder, previous)); + previous = Optional.of(compileConnectorStep(input, cteStep, previous)); } return previous; } - public static QueryStep.QueryStepBuilder compilePreprocessing(PreprocessingCteInput input) { + static QueryStep compilePreprocessing(PreprocessingCteInput input, String cteName) { List preprocessingSelects = input.sqlSelects().stream() .flatMap(selects -> selects.getPreprocessingSelects().stream()) .toList(); @@ -72,9 +70,12 @@ public static QueryStep.QueryStepBuilder compilePreprocessing(PreprocessingCteIn .sqlSelects(preprocessingSelects) .build(); return QueryStep.builder() + .cteName(cteName) .selects(selects) .fromTable(input.sourceTable()) - .conditions(conditions); + .conditions(conditions) + .predecessors(List.of()) + .build(); } QueryStep stratificationTableCte = input.stratificationTable().orElseThrow(); @@ -100,14 +101,18 @@ public static QueryStep.QueryStepBuilder compilePreprocessing(PreprocessingCteIn .build(); return QueryStep.builder() + .cteName(cteName) .selects(selects) .fromTable(joinedTable) - .conditions(conditions); + .conditions(conditions) + .predecessors(List.of()) + .build(); } - public static QueryStep.QueryStepBuilder compileAggregationSelect( + static QueryStep compileAggregationSelect( QueryStep predecessor, - List sqlSelects + List sqlSelects, + String cteName ) { List aggregationSelects = sqlSelects.stream() .flatMap(selects -> selects.getAggregationSelects().stream()) @@ -128,14 +133,19 @@ public static QueryStep.QueryStepBuilder compileAggregationSelect( ).toList(); return QueryStep.builder() + .cteName(cteName) .selects(selects) - .groupBy(groupByFields); + .fromTable(QueryStep.toTableLike(predecessor.getCteName())) + .groupBy(groupByFields) + .predecessors(List.of(predecessor)) + .build(); } - public static QueryStep.QueryStepBuilder compileAggregationFilter( + static QueryStep compileAggregationFilter( QueryStep predecessor, List sqlSelects, - List sqlFilters + List sqlFilters, + String cteName ) { Selects predecessorSelects = predecessor.getQualifiedSelects(); List finalSelects = sqlSelects.stream() @@ -155,11 +165,15 @@ public static QueryStep.QueryStepBuilder compileAggregationFilter( .toList(); return QueryStep.builder() + .cteName(cteName) .selects(selects) - .conditions(conditions); + .fromTable(QueryStep.toTableLike(predecessor.getCteName())) + .conditions(conditions) + .predecessors(List.of(predecessor)) + .build(); } - public static QueryStep.QueryStepBuilder compileJoinBranches(JoinBranchesCteInput input) { + static QueryStep compileJoinBranches(JoinBranchesCteInput input, String cteName) { List queriesToJoin = new ArrayList<>(); queriesToJoin.add(input.predecessor()); @@ -198,26 +212,29 @@ public static QueryStep.QueryStepBuilder compileJoinBranches(JoinBranchesCteInpu .build(); return QueryStep.builder() + .cteName(cteName) .selects(selects) .fromTable(QueryStepJoiner.join(queriesToJoin, JoinMode.FULL_OUTER)) - .predecessors(queriesToJoin); + .predecessors(queriesToJoin) + .build(); } - private static QueryStep.QueryStepBuilder compileConnectorStep( + private static QueryStep compileConnectorStep( ConnectorCtePipelineInput input, ConceptCteStep cteStep, Optional previous ) { + String cteName = input.tables().cteName(cteStep); if (cteStep == ConceptCteStep.PREPROCESSING) { // TODO this looks like it can be unwinded - return compilePreprocessing(input.preprocessing()); + return compilePreprocessing(input.preprocessing(), cteName); } QueryStep predecessor = previous.orElseThrow(() -> new IllegalStateException( "Connector CTE %s requires a predecessor".formatted(cteStep) )); return switch (cteStep) { - case AGGREGATION_SELECT -> compileAggregationSelect(predecessor, input.preprocessing().sqlSelects()); + case AGGREGATION_SELECT -> compileAggregationSelect(predecessor, input.preprocessing().sqlSelects(), cteName); case JOIN_BRANCHES -> compileJoinBranches(new JoinBranchesCteInput( predecessor, input.preprocessing().validityDate(), @@ -226,32 +243,17 @@ private static QueryStep.QueryStepBuilder compileConnectorStep( input.excludedFromTimeAggregation(), input.eventDateSelects(), input.additionalPredecessors() - )); + ), cteName); case AGGREGATION_FILTER -> compileAggregationFilter( predecessor, input.preprocessing().sqlSelects(), - input.preprocessing().sqlFilters() + input.preprocessing().sqlFilters(), + cteName ); default -> throw new IllegalArgumentException("Not a connector pipeline step: %s".formatted(cteStep)); }; } - private static QueryStep completeStep(QueryStep.QueryStepBuilder builder, Optional previous) { - if (previous.isEmpty()) { - return builder.predecessors(List.of()).build(); - } - - QueryStep partialStep = builder.build(); - if (partialStep.getFromTables().isEmpty() && partialStep.getPredecessors().isEmpty()) { - QueryStep predecessor = previous.orElseThrow(); - return builder - .fromTable(QueryStep.toTableLike(predecessor.getCteName())) - .predecessors(List.of(predecessor)) - .build(); - } - return partialStep; - } - private static SqlSelect qualifyUnlessUniversal(SqlSelect sqlSelect, String predecessorName) { return sqlSelect.isUniversal() ? sqlSelect : sqlSelect.qualify(predecessorName); } diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java index 6207d0513b..10cc02f95c 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCteCompilerTest.java @@ -74,8 +74,10 @@ void shouldCompileConnectorCtePipeline() { void shouldCompilePreprocessingAgainstConnectorTable() { PreprocessingCteInput input = preprocessingInput(Optional.empty()); - QueryStep result = ConnectorCteCompiler.compilePreprocessing(input).build(); + QueryStep result = ConnectorCteCompiler.compilePreprocessing(input, "preprocessing"); + assertEquals("preprocessing", result.getCteName()); + assertTrue(result.getPredecessors().isEmpty()); assertEquals(List.of(name("score")), result.getSelects().getSqlSelects().stream() .flatMap(sqlSelect -> sqlSelect.toFields().stream()) .map(Field::getQualifiedName) @@ -102,7 +104,7 @@ void shouldCompilePreprocessingWithStratificationJoin() { .build(); PreprocessingCteInput input = preprocessingInput(Optional.of(stratificationTable)); - QueryStep result = ConnectorCteCompiler.compilePreprocessing(input).build(); + QueryStep result = ConnectorCteCompiler.compilePreprocessing(input, "preprocessing"); assertEquals(name("stratification", "person"), result.getSelects().getIds().getPrimaryColumn().getQualifiedName()); assertEquals( @@ -121,9 +123,12 @@ void shouldCompileAggregationSelectsAndGroupByPredecessorDimensions() { FieldWrapper aggregation = new FieldWrapper<>(field(name("aggregated"), Integer.class)); QueryStep result = ConnectorCteCompiler.compileAggregationSelect( predecessor(), - List.of(ConnectorSqlSelects.builder().aggregationSelect(aggregation).build()) - ).build(); + List.of(ConnectorSqlSelects.builder().aggregationSelect(aggregation).build()), + "aggregation" + ); + assertEquals("aggregation", result.getCteName()); + assertEquals(List.of(PREDECESSOR_NAME), result.getPredecessors().stream().map(QueryStep::getCteName).toList()); assertEquals(List.of(aggregation), result.getSelects().getSqlSelects()); assertEquals( List.of(name(PREDECESSOR_NAME, "person"), name(PREDECESSOR_NAME, "stratification_start"), name(PREDECESSOR_NAME, "stratification_end")), @@ -151,9 +156,12 @@ void shouldCompileFinalSelectsAndGroupFilters() { QueryStep result = ConnectorCteCompiler.compileAggregationFilter( predecessor, List.of(sqlSelects), - List.of(sqlFilter) - ).build(); + List.of(sqlFilter), + "filtered" + ); + assertEquals("filtered", result.getCteName()); + assertEquals(List.of(predecessor), result.getPredecessors()); List resultSelects = result.getSelects().getSqlSelects(); assertEquals( name(PREDECESSOR_NAME, "metric"), @@ -194,8 +202,9 @@ void shouldJoinAdditionalPredecessors() { List.of(additionalPredecessor) ); - QueryStep result = ConnectorCteCompiler.compileJoinBranches(input).build(); + QueryStep result = ConnectorCteCompiler.compileJoinBranches(input, "joined"); + assertEquals("joined", result.getCteName()); assertEquals(List.of(PREDECESSOR_NAME, "additional"), result.getPredecessors().stream().map(QueryStep::getCteName).toList()); assertTrue(result.getSelects().getValidityDate().isEmpty()); assertEquals( @@ -221,7 +230,7 @@ void shouldJoinIntervalPackingAndEventDateSelectBranches() { List.of() ); - QueryStep result = ConnectorCteCompiler.compileJoinBranches(input).build(); + QueryStep result = ConnectorCteCompiler.compileJoinBranches(input, "joined"); assertEquals( List.of(PREDECESSOR_NAME, "interval_complete", "interval_selects"), @@ -252,7 +261,7 @@ void shouldNotPropagateExcludedValidityDate() { List.of() ); - QueryStep result = ConnectorCteCompiler.compileJoinBranches(input).build(); + QueryStep result = ConnectorCteCompiler.compileJoinBranches(input, "joined"); assertEquals( List.of(PREDECESSOR_NAME, "interval_complete"), From 156d8a39d8c4e5e6fc19608c1e96e8e7e3a4455a Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Fri, 18 Sep 2026 09:08:18 +0200 Subject: [PATCH 59/81] refactor(sql): convert resolved conditions in connector Convert built-in resolved conditions into compiler IR through the connector's runtime dispatcher and allow extensions to contribute converters for their own condition types. Expose regular-expression matching as a compiler dialect capability while preserving the legacy backend dialect implementations. Mark the prefix-range model as deprecated because matching-statistics expression generation does not support it. Co-authored-by: Codex GPT-5.6-sol --- .../dialect/LegacyCompilerDialect.java | 11 ++ .../operation/BuiltInConditionConverters.java | 111 ++++++++++++++ .../operation/ConditionConversionContext.java | 17 +++ .../operation/ResolvedConditionConverter.java | 39 +++++ .../conversion/operation/package-info.java | 2 + .../sql/compiler/dialect/CompilerDialect.java | 10 ++ .../model/operation/BuiltInConditions.java | 5 + .../model/operation/ResolvedCondition.java | 6 +- .../ResolvedConditionConverterTest.java | 138 ++++++++++++++++++ 9 files changed, 338 insertions(+), 1 deletion(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInConditionConverters.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ConditionConversionContext.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedConditionConverter.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/package-info.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedConditionConverterTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java index 4b282528cd..90ad7a9026 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java @@ -31,6 +31,7 @@ import com.bakdata.conquery.sql.conversion.query.RelativFormQueryConverter; import com.bakdata.conquery.sql.conversion.query.SecondaryIdQueryConverter; import com.bakdata.conquery.sql.conversion.query.TableExportQueryConverter; +import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.Field; import org.jooq.Record; @@ -99,6 +100,16 @@ default QueryStep unnestDateRange(ColumnDateRange dateRange, QueryStep predecess return getFunctionProvider().unnestDaterange(dateRange, predecessor, cteName); } + @Override + default String regexAnyCharacters() { + return getFunctionProvider().getAnyCharRegex(); + } + + @Override + default Condition regexMatches(Field field, String pattern) { + return getFunctionProvider().likeRegex(field, pattern); + } + StratificationFunctions getStratificationFunctions(); SqlFunctionProvider getFunctionProvider(); diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInConditionConverters.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInConditionConverters.java new file mode 100644 index 0000000000..ff0dda24f1 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInConditionConverters.java @@ -0,0 +1,111 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import java.util.List; +import java.util.function.BiFunction; + +import com.bakdata.conquery.sql.compiler.conversion.Converter; +import com.bakdata.conquery.sql.compiler.ir.SchemaSql; +import com.bakdata.conquery.sql.compiler.ir.condition.ConditionWrappingWhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.StringValuesCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; +import com.bakdata.conquery.sql.model.operation.BuiltInConditions; +import com.bakdata.conquery.sql.model.operation.ResolvedCondition; +import org.jooq.Field; + +/** Built-in mappings from resolved condition values to compiler condition IR. */ +final class BuiltInConditionConverters { + + private BuiltInConditionConverters() { + } + + static List> create() { + return List.of( + converter(BuiltInConditions.AllOf.class, BuiltInConditionConverters::convertAllOf), + converter(BuiltInConditions.Not.class, BuiltInConditionConverters::convertNot), + converter(BuiltInConditions.StringValues.class, BuiltInConditionConverters::convertStringValues), + converter(BuiltInConditions.Presence.class, BuiltInConditionConverters::convertPresence), + converter(BuiltInConditions.Prefixes.class, BuiltInConditionConverters::convertPrefixes), + converter(BuiltInConditions.PrefixRange.class, BuiltInConditionConverters::convertPrefixRange) + ); + } + + private static WhereCondition convertAllOf( + BuiltInConditions.AllOf condition, + ConditionConversionContext context + ) { + return condition.conditions().stream() + .map(child -> context.conditionConverter().convert(child, context.dialect())) + .reduce(WhereCondition::and) + .orElseThrow(); + } + + private static WhereCondition convertNot( + BuiltInConditions.Not condition, + ConditionConversionContext context + ) { + return context.conditionConverter().convert(condition.condition(), context.dialect()).negate(); + } + + private static WhereCondition convertStringValues( + BuiltInConditions.StringValues condition, + ConditionConversionContext context + ) { + Field field = SchemaSql.field(condition.column(), String.class); + return new StringValuesCondition(field, condition.values().toArray(String[]::new)); + } + + private static WhereCondition convertPresence( + BuiltInConditions.Presence condition, + ConditionConversionContext context + ) { + Field field = SchemaSql.field(condition.column(), Object.class); + return new ConditionWrappingWhereCondition(condition.present() ? field.isNotNull() : field.isNull()); + } + + private static WhereCondition convertPrefixes( + BuiltInConditions.Prefixes condition, + ConditionConversionContext context + ) { + String pattern = String.join("|", condition.prefixes()) + context.dialect().regexAnyCharacters(); + Field field = SchemaSql.field(condition.column(), String.class); + return new ConditionWrappingWhereCondition(context.dialect().regexMatches(field, pattern)); + } + + private static WhereCondition convertPrefixRange( + BuiltInConditions.PrefixRange condition, + ConditionConversionContext context + ) { + StringBuilder pattern = new StringBuilder(); + for (int index = 0; index < condition.minimum().length(); index++) { + char minimum = condition.minimum().charAt(index); + char maximum = condition.maximum().charAt(index); + if (minimum == maximum) { + pattern.append(minimum); + } + else { + pattern.append("[%s-%s]".formatted(minimum, maximum)); + } + } + pattern.append(context.dialect().regexAnyCharacters()); + + Field field = SchemaSql.field(condition.column(), String.class); + return new ConditionWrappingWhereCondition(context.dialect().regexMatches(field, pattern.toString())); + } + + private static Converter converter( + Class conversionClass, + BiFunction conversion + ) { + return new Converter<>() { + @Override + public Class getConversionClass() { + return conversionClass; + } + + @Override + public WhereCondition convert(C input, ConditionConversionContext context) { + return conversion.apply(input, context); + } + }; + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ConditionConversionContext.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ConditionConversionContext.java new file mode 100644 index 0000000000..16024e4026 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ConditionConversionContext.java @@ -0,0 +1,17 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import java.util.Objects; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; + +/** Services available while converting a resolved condition into compiler IR. */ +public record ConditionConversionContext( + CompilerDialect dialect, + ResolvedConditionConverter conditionConverter +) { + + public ConditionConversionContext { + Objects.requireNonNull(dialect, "dialect"); + Objects.requireNonNull(conditionConverter, "conditionConverter"); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedConditionConverter.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedConditionConverter.java new file mode 100644 index 0000000000..6908d5b9e6 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedConditionConverter.java @@ -0,0 +1,39 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import java.util.ArrayList; +import java.util.List; + +import com.bakdata.conquery.sql.compiler.conversion.ConversionDispatcher; +import com.bakdata.conquery.sql.compiler.conversion.Converter; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; +import com.bakdata.conquery.sql.model.operation.ResolvedCondition; + +/** Converts resolved condition-model values into compiler condition IR. */ +public final class ResolvedConditionConverter { + + private final ConversionDispatcher dispatcher; + + public ResolvedConditionConverter() { + this(List.of()); + } + + /** + * Creates a converter with the built-in condition implementations and additional extension converters. + * + *

An extension must support a distinct runtime type; ambiguous converters are rejected by the dispatcher.

+ */ + public ResolvedConditionConverter( + List> extensions + ) { + List> converters = new ArrayList<>( + BuiltInConditionConverters.create() + ); + converters.addAll(extensions); + this.dispatcher = new ConversionDispatcher<>(converters); + } + + public WhereCondition convert(ResolvedCondition condition, CompilerDialect dialect) { + return dispatcher.convert(condition, new ConditionConversionContext(dialect, this)); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/package-info.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/package-info.java new file mode 100644 index 0000000000..3ebe774491 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/package-info.java @@ -0,0 +1,2 @@ +/** Converters crossing from resolved operation models into compiler IR. */ +package com.bakdata.conquery.sql.compiler.conversion.operation; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java index 8d0a0d753e..6cfd53013f 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java @@ -51,6 +51,16 @@ default ColumnDateRange conditionalUnboundedDateRange(Condition condition) { ); } + /** Regex fragment matching any number of characters in this dialect. */ + default String regexAnyCharacters() { + return ".*"; + } + + /** Test whether a string field matches the supplied regular expression. */ + default Condition regexMatches(Field field, String pattern) { + return field.likeRegex(pattern); + } + /** SQL expression for an already-resolved external entity ID. */ default Field externalId(String id) { return DSL.inline(id, SQLDataType.VARCHAR); diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInConditions.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInConditions.java index b9da183134..80fd6800a4 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInConditions.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/BuiltInConditions.java @@ -53,6 +53,11 @@ public record Prefixes( } } + /** + * @deprecated Retained for direct SQL-conversion compatibility with the deprecated legacy prefix-range condition. + * It is not supported by matching-statistics expression generation and must not be used for new conditions. + */ + @Deprecated public record PrefixRange( @NotNull @Valid @AllowedColumnTypes(ColumnType.STRING) ResolvedColumn column, @NotBlank String minimum, diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedCondition.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedCondition.java index 8662889588..6bbda8ad8d 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedCondition.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedCondition.java @@ -1,5 +1,9 @@ package com.bakdata.conquery.sql.model.operation; -/** A resolved connector or concept-element condition. */ +/** + * Marker for an immutable, validated connector or concept-element condition. + * + *

Implementations may be supplied by extensions and are dispatched to matching compiler converters.

+ */ public interface ResolvedCondition { } diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedConditionConverterTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedConditionConverterTest.java new file mode 100644 index 0000000000..0bc39bd112 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedConditionConverterTest.java @@ -0,0 +1,138 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import static org.jooq.impl.DSL.condition; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.inline; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.sql.Date; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; + +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.sql.compiler.conversion.Converter; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.condition.ConditionWrappingWhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; +import com.bakdata.conquery.sql.model.operation.BuiltInConditions; +import com.bakdata.conquery.sql.model.operation.ResolvedCondition; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.SqlTable; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Test; + +class ResolvedConditionConverterTest { + + private static final SqlTable TABLE = SqlTable.of("events", "analytics", "events"); + private static final ResolvedColumn COLUMN = new ResolvedColumn("code", TABLE, "event_code", ColumnType.STRING, true); + private static final CompilerDialect DIALECT = new TestDialect(); + + private final ResolvedConditionConverter converter = new ResolvedConditionConverter(); + + @Test + void shouldConvertStringValuesAgainstResolvedColumn() { + BuiltInConditions.StringValues condition = new BuiltInConditions.StringValues( + COLUMN, + new LinkedHashSet<>(List.of("A", "B")) + ); + + assertEquals( + "\"analytics\".\"events\".\"event_code\" in ('a', 'b')", + render(converter.convert(condition, DIALECT)) + ); + } + + @Test + void shouldConvertCompositeConditionsRecursively() { + BuiltInConditions.AllOf condition = new BuiltInConditions.AllOf(List.of( + new BuiltInConditions.Presence(COLUMN, true), + new BuiltInConditions.Not(new BuiltInConditions.Presence(COLUMN, false)) + )); + + assertEquals( + "(\"analytics\".\"events\".\"event_code\" is not null and not (\"analytics\".\"events\".\"event_code\" is null))", + render(converter.convert(condition, DIALECT)) + ); + } + + @Test + void shouldDelegateRegexRenderingToDialect() { + BuiltInConditions.AllOf condition = new BuiltInConditions.AllOf(List.of( + new BuiltInConditions.Prefixes(COLUMN, List.of("A", "B")), + new BuiltInConditions.PrefixRange(COLUMN, "A0", "B9") + )); + + assertEquals( + "((matches(\"analytics\".\"events\".\"event_code\", 'a|b.*')) and (matches(\"analytics\".\"events\".\"event_code\", '[a-b][0-9].*')))", + render(converter.convert(condition, DIALECT)) + ); + } + + @Test + void shouldDispatchExtensionCondition() { + Converter extension = new Converter<>() { + @Override + public Class getConversionClass() { + return ExtensionCondition.class; + } + + @Override + public WhereCondition convert(ExtensionCondition input, ConditionConversionContext context) { + return new ConditionWrappingWhereCondition(condition("extension_condition")); + } + }; + ResolvedConditionConverter converterWithExtension = new ResolvedConditionConverter(List.of(extension)); + + assertEquals("(extension_condition)", render(converterWithExtension.convert(new ExtensionCondition(), DIALECT))); + } + + private static String render(WhereCondition condition) { + return DSL.using(SQLDialect.POSTGRES).renderInlined(condition.condition()).toLowerCase(Locale.ROOT); + } + + private record ExtensionCondition() implements ResolvedCondition { + } + + private static final class TestDialect implements CompilerDialect { + + @Override + public Field minimumDate() { + return field(name("minimum_date"), Date.class); + } + + @Override + public Field maximumDate() { + return field(name("maximum_date"), Date.class); + } + + @Override + public Condition regexMatches(Field field, String pattern) { + return condition("matches({0}, {1})", field, inline(pattern)); + } + + @Override + public Field anyValue(Field value) { + return value; + } + + @Override + public Field renderDateRange(Field start, Field end) { + return field(name("rendered_range"), Object.class); + } + + @Override + public Field aggregateDateRanges(Field start, Field end) { + return field(name("aggregated_ranges"), Object.class); + } + + @Override + public int getNameMaxLength() { + return 128; + } + } +} From 6618931ebff8a61f0e80eb3f5f8e1a70cbcc719a Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Fri, 18 Sep 2026 10:18:33 +0200 Subject: [PATCH 60/81] refactor(sql): convert resolved column filters Convert resolved string-value and numeric-range filters into event-level connector predicates. Preserve zero-based substring semantics when building SQL expressions and provide the compiler context and dispatcher boundary for extension-defined filter converters. Document the preprocessing, event, and post-aggregation condition buckets consumed by the connector pipeline. Co-authored-by: Codex GPT-5.6-sol --- .../BuiltInColumnFilterConverters.java | 83 +++++++++ .../operation/FilterConversionContext.java | 24 +++ .../operation/ResolvedFilterConverter.java | 38 ++++ .../compiler/ir/condition/WhereClauses.java | 5 + .../sql/model/operation/ResolvedFilter.java | 4 +- .../ResolvedFilterConverterTest.java | 162 ++++++++++++++++++ 6 files changed, 314 insertions(+), 2 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInColumnFilterConverters.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/FilterConversionContext.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedFilterConverter.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedFilterConverterTest.java diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInColumnFilterConverters.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInColumnFilterConverters.java new file mode 100644 index 0000000000..330c5ab335 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInColumnFilterConverters.java @@ -0,0 +1,83 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import java.math.BigDecimal; +import java.util.List; +import java.util.function.BiFunction; + +import com.bakdata.conquery.sql.compiler.conversion.Converter; +import com.bakdata.conquery.sql.compiler.ir.SchemaSql; +import com.bakdata.conquery.sql.compiler.ir.condition.InclusiveRangeCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.StringValuesCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; +import com.bakdata.conquery.sql.model.operation.BuiltInFilters; +import com.bakdata.conquery.sql.model.operation.ResolvedFilter; +import com.bakdata.conquery.sql.model.range.SubstringRange; +import org.jooq.Field; +import org.jooq.impl.DSL; + +/** Built-in resolved filters that operate directly on connector columns. */ +final class BuiltInColumnFilterConverters { + + private BuiltInColumnFilterConverters() { + } + + static List> create() { + return List.of( + converter(BuiltInFilters.StringValues.class, BuiltInColumnFilterConverters::convertStringValues), + converter(BuiltInFilters.NumericColumnRange.class, BuiltInColumnFilterConverters::convertNumericRange) + ); + } + + private static SqlFilters convertStringValues( + BuiltInFilters.StringValues filter, + FilterConversionContext context + ) { + Field field = SchemaSql.field(filter.column(), String.class); + if (filter.substring().isPresent()) { + field = substring(field, filter.substring().orElseThrow()); + } + return eventFilter(new StringValuesCondition(field, filter.values().toArray(String[]::new))); + } + + private static SqlFilters convertNumericRange( + BuiltInFilters.NumericColumnRange filter, + FilterConversionContext context + ) { + Field field = SchemaSql.field(filter.column(), BigDecimal.class); + return eventFilter(new InclusiveRangeCondition<>(field, filter.range())); + } + + private static Field substring(Field field, SubstringRange range) { + int start = range.startInclusive() + 1; + return range.endExclusive() + .map(end -> DSL.substring(field, start, end - range.startInclusive())) + .orElseGet(() -> DSL.substring(field, start)); + } + + private static SqlFilters eventFilter(WhereCondition condition) { + return new SqlFilters( + ConnectorSqlSelects.none(), + WhereClauses.builder().eventFilter(condition).build() + ); + } + + private static Converter converter( + Class conversionClass, + BiFunction conversion + ) { + return new Converter<>() { + @Override + public Class getConversionClass() { + return conversionClass; + } + + @Override + public SqlFilters convert(F input, FilterConversionContext context) { + return conversion.apply(input, context); + } + }; + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/FilterConversionContext.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/FilterConversionContext.java new file mode 100644 index 0000000000..d8216091eb --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/FilterConversionContext.java @@ -0,0 +1,24 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import java.util.Objects; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; + +/** Compiler state available while converting a resolved filter into connector IR. */ +public record FilterConversionContext( + CompilerDialect dialect, + SqlNameGenerator nameGenerator, + SqlTables tables, + SqlIdColumns ids +) { + + public FilterConversionContext { + Objects.requireNonNull(dialect, "dialect"); + Objects.requireNonNull(nameGenerator, "nameGenerator"); + Objects.requireNonNull(tables, "tables"); + Objects.requireNonNull(ids, "ids"); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedFilterConverter.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedFilterConverter.java new file mode 100644 index 0000000000..f3094071a2 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedFilterConverter.java @@ -0,0 +1,38 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import java.util.ArrayList; +import java.util.List; + +import com.bakdata.conquery.sql.compiler.conversion.ConversionDispatcher; +import com.bakdata.conquery.sql.compiler.conversion.Converter; +import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; +import com.bakdata.conquery.sql.model.operation.ResolvedFilter; + +/** Converts resolved filter-model values into connector filter IR. */ +public final class ResolvedFilterConverter { + + private final ConversionDispatcher dispatcher; + + public ResolvedFilterConverter() { + this(List.of()); + } + + /** + * Creates a converter with the built-in column-filter implementations and additional extension converters. + * + *

An extension must support a distinct runtime type; ambiguous converters are rejected by the dispatcher.

+ */ + public ResolvedFilterConverter( + List> extensions + ) { + List> converters = new ArrayList<>( + BuiltInColumnFilterConverters.create() + ); + converters.addAll(extensions); + this.dispatcher = new ConversionDispatcher<>(converters); + } + + public SqlFilters convert(ResolvedFilter filter, FilterConversionContext context) { + return dispatcher.convert(filter, context); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereClauses.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereClauses.java index ead0a6db89..59532a8b53 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereClauses.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/condition/WhereClauses.java @@ -11,10 +11,15 @@ @Value public class WhereClauses { + /** Connector and concept conditions applied while reading t_e raw connector table during preprocessing. */ @Singular List preprocessingConditions; + + /** User-selected filters evaluated for each raw event during preprocessing, before aggregation. */ @Singular List eventFilters; + + /** Filters evaluated after aggregation expressions have been calculated for an entity group. */ @Singular List groupFilters; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedFilter.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedFilter.java index 98054daeef..87388802d3 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedFilter.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/operation/ResolvedFilter.java @@ -5,8 +5,8 @@ /** * Marker for an immutable, validated filter operation. * - *

Implementations may be supplied by extensions. They must contain resolved columns and typed values rather than - * repository identifiers.

+ *

Implementations may be supplied by extensions and are dispatched to matching compiler converters. They must + * contain resolved columns and typed values rather than repository identifiers.

*/ public interface ResolvedFilter { diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedFilterConverterTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedFilterConverterTest.java new file mode 100644 index 0000000000..d7d962a2f0 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedFilterConverterTest.java @@ -0,0 +1,162 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import static org.jooq.impl.DSL.condition; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.Date; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.sql.compiler.conversion.Converter; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.condition.ConditionWrappingWhereCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import com.bakdata.conquery.sql.model.operation.BuiltInFilters; +import com.bakdata.conquery.sql.model.operation.ResolvedFilter; +import com.bakdata.conquery.sql.model.range.NumberRange; +import com.bakdata.conquery.sql.model.range.SubstringRange; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.SqlTable; +import org.jooq.Field; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Test; + +class ResolvedFilterConverterTest { + + private static final SqlTable TABLE = SqlTable.of("events", "analytics", "events"); + private static final ResolvedColumn STRING_COLUMN = new ResolvedColumn( + "code", TABLE, "event_code", ColumnType.STRING, true + ); + private static final ResolvedColumn NUMBER_COLUMN = new ResolvedColumn( + "score", TABLE, "score", ColumnType.DECIMAL, true + ); + private static final FilterConversionContext CONTEXT = new FilterConversionContext( + new TestDialect(), + new SqlNameGenerator(128), + new SqlTables("events", Map.of(), Map.of()), + new SqlIdColumns(field(name("analytics", "events", "id"), String.class)) + ); + + private final ResolvedFilterConverter converter = new ResolvedFilterConverter(); + + @Test + void shouldConvertStringValuesWithSubstring() { + BuiltInFilters.StringValues filter = new BuiltInFilters.StringValues( + "codes", + STRING_COLUMN, + new LinkedHashSet<>(List.of("AA", "BB")), + Optional.of(SubstringRange.between(1, 3)) + ); + + SqlFilters result = converter.convert(filter, CONTEXT); + + assertEquals( + "substring(\"analytics\".\"events\".\"event_code\", 2, 2) in ('aa', 'bb')", + render(result) + ); + assertEmptySelects(result.getSelects()); + } + + @Test + void shouldConvertNumericColumnRange() { + BuiltInFilters.NumericColumnRange filter = new BuiltInFilters.NumericColumnRange( + "score", + NUMBER_COLUMN, + NumberRange.atLeast(10.5) + ); + + SqlFilters result = converter.convert(filter, CONTEXT); + + assertEquals("\"analytics\".\"events\".\"score\" >= 10.5", render(result)); + assertEmptySelects(result.getSelects()); + } + + @Test + void shouldDispatchExtensionFilter() { + Converter extension = new Converter<>() { + @Override + public Class getConversionClass() { + return ExtensionFilter.class; + } + + @Override + public SqlFilters convert(ExtensionFilter input, FilterConversionContext context) { + return new SqlFilters( + ConnectorSqlSelects.none(), + WhereClauses.builder() + .eventFilter(new ConditionWrappingWhereCondition(condition("extension_filter"))) + .build() + ); + } + }; + ResolvedFilterConverter converterWithExtension = new ResolvedFilterConverter(List.of(extension)); + + assertEquals("(extension_filter)", render(converterWithExtension.convert(new ExtensionFilter("extension"), CONTEXT))); + } + + private static String render(SqlFilters filters) { + assertEquals(1, filters.getWhereClauses().getEventFilters().size()); + assertTrue(filters.getWhereClauses().getPreprocessingConditions().isEmpty()); + assertTrue(filters.getWhereClauses().getGroupFilters().isEmpty()); + return DSL.using(SQLDialect.POSTGRES) + .renderInlined(filters.getWhereClauses().getEventFilters().getFirst().condition()) + .toLowerCase(Locale.ROOT); + } + + private static void assertEmptySelects(ConnectorSqlSelects selects) { + assertTrue(selects.getPreprocessingSelects().isEmpty()); + assertTrue(selects.getAggregationSelects().isEmpty()); + assertTrue(selects.getEventDateSelects().isEmpty()); + assertTrue(selects.getFinalSelects().isEmpty()); + assertTrue(selects.getAdditionalPredecessor().isEmpty()); + } + + private record ExtensionFilter(String name) implements ResolvedFilter { + } + + private static final class TestDialect implements CompilerDialect { + + @Override + public Field minimumDate() { + return field(name("minimum_date"), Date.class); + } + + @Override + public Field maximumDate() { + return field(name("maximum_date"), Date.class); + } + + @Override + public Field anyValue(Field value) { + return value; + } + + @Override + public Field renderDateRange(Field start, Field end) { + return field(name("rendered_range"), Object.class); + } + + @Override + public Field aggregateDateRanges(Field start, Field end) { + return field(name("aggregated_ranges"), Object.class); + } + + @Override + public int getNameMaxLength() { + return 128; + } + } +} From fac88236b048007dc3088c9c34b2aeb6d6642890 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Fri, 18 Sep 2026 10:25:14 +0200 Subject: [PATCH 61/81] refactor(sql): convert remaining event filters Convert resolved flag and date-distance filters into event-level connector predicates. Keep range and flag semantics in the connector while delegating database-specific date-distance expressions to the compiler dialect and preserving the legacy backend adapter. Co-authored-by: Codex GPT-5.6-sol --- .../dialect/LegacyCompilerDialect.java | 11 ++++ .../BuiltInColumnFilterConverters.java | 28 ++++++++- .../sql/compiler/dialect/CompilerDialect.java | 11 ++++ .../ResolvedFilterConverterTest.java | 59 +++++++++++++++++++ 4 files changed, 108 insertions(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java index 90ad7a9026..0bcb7b511e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java @@ -1,6 +1,8 @@ package com.bakdata.conquery.sql.conversion.dialect; import java.sql.Date; +import java.time.LocalDate; +import java.time.temporal.ChronoUnit; import java.util.Collections; import java.util.List; import java.util.Map; @@ -110,6 +112,15 @@ default Condition regexMatches(Field field, String pattern) { return getFunctionProvider().likeRegex(field, pattern); } + @Override + default Field dateDistance(ChronoUnit unit, Field startDate, LocalDate endDate) { + return getFunctionProvider().dateDistance( + unit, + startDate, + getFunctionProvider().toDateField(endDate.toString()) + ); + } + StratificationFunctions getStratificationFunctions(); SqlFunctionProvider getFunctionProvider(); diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInColumnFilterConverters.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInColumnFilterConverters.java index 330c5ab335..5f3229ba74 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInColumnFilterConverters.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInColumnFilterConverters.java @@ -1,12 +1,14 @@ package com.bakdata.conquery.sql.compiler.conversion.operation; import java.math.BigDecimal; +import java.sql.Date; import java.util.List; import java.util.function.BiFunction; import com.bakdata.conquery.sql.compiler.conversion.Converter; import com.bakdata.conquery.sql.compiler.ir.SchemaSql; import com.bakdata.conquery.sql.compiler.ir.condition.InclusiveRangeCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.FlagCondition; import com.bakdata.conquery.sql.compiler.ir.condition.StringValuesCondition; import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; @@ -27,7 +29,9 @@ private BuiltInColumnFilterConverters() { static List> create() { return List.of( converter(BuiltInFilters.StringValues.class, BuiltInColumnFilterConverters::convertStringValues), - converter(BuiltInFilters.NumericColumnRange.class, BuiltInColumnFilterConverters::convertNumericRange) + converter(BuiltInFilters.NumericColumnRange.class, BuiltInColumnFilterConverters::convertNumericRange), + converter(BuiltInFilters.DateDistanceRange.class, BuiltInColumnFilterConverters::convertDateDistanceRange), + converter(BuiltInFilters.Flags.class, BuiltInColumnFilterConverters::convertFlags) ); } @@ -50,6 +54,28 @@ private static SqlFilters convertNumericRange( return eventFilter(new InclusiveRangeCondition<>(field, filter.range())); } + private static SqlFilters convertDateDistanceRange( + BuiltInFilters.DateDistanceRange filter, + FilterConversionContext context + ) { + Field startDate = SchemaSql.field(filter.column(), Date.class); + Field distance = context.dialect() + .dateDistance(filter.unit(), startDate, filter.endDate()) + .coerce(BigDecimal.class); + return eventFilter(new InclusiveRangeCondition<>(distance, filter.range())); + } + + private static SqlFilters convertFlags( + BuiltInFilters.Flags filter, + FilterConversionContext context + ) { + List> selectedFlags = filter.selectedFlags().stream() + .map(filter.availableFlags().columns()::get) + .map(column -> SchemaSql.field(column, Boolean.class)) + .toList(); + return eventFilter(new FlagCondition(selectedFlags)); + } + private static Field substring(Field field, SubstringRange range) { int start = range.startInclusive() + 1; return range.endExclusive() diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java index 6cfd53013f..b28e1b50e5 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java @@ -1,6 +1,8 @@ package com.bakdata.conquery.sql.compiler.dialect; import java.sql.Date; +import java.time.LocalDate; +import java.time.temporal.ChronoUnit; import java.util.List; import com.bakdata.conquery.sql.compiler.ir.QueryStep; @@ -61,6 +63,15 @@ default Condition regexMatches(Field field, String pattern) { return field.likeRegex(pattern); } + /** + * Calculate the distance between an inclusive start date and an inclusive end date in the requested unit. + * + * @throws UnsupportedOperationException when the dialect does not support date-distance operations + */ + default Field dateDistance(ChronoUnit unit, Field startDate, LocalDate endDate) { + throw new UnsupportedOperationException("Date-distance calculation is not implemented by this dialect"); + } + /** SQL expression for an already-resolved external entity ID. */ default Field externalId(String id) { return DSL.inline(id, SQLDataType.VARCHAR); diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedFilterConverterTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedFilterConverterTest.java index d7d962a2f0..02dec21449 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedFilterConverterTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedFilterConverterTest.java @@ -7,11 +7,14 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.sql.Date; +import java.time.LocalDate; +import java.time.temporal.ChronoUnit; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.Set; import com.bakdata.conquery.models.datasets.ColumnType; import com.bakdata.conquery.sql.compiler.conversion.Converter; @@ -43,6 +46,15 @@ class ResolvedFilterConverterTest { private static final ResolvedColumn NUMBER_COLUMN = new ResolvedColumn( "score", TABLE, "score", ColumnType.DECIMAL, true ); + private static final ResolvedColumn DATE_COLUMN = new ResolvedColumn( + "start", TABLE, "start_date", ColumnType.DATE, true + ); + private static final ResolvedColumn FLAG_A = new ResolvedColumn( + "flag-a", TABLE, "flag_a", ColumnType.BOOLEAN, false + ); + private static final ResolvedColumn FLAG_B = new ResolvedColumn( + "flag-b", TABLE, "flag_b", ColumnType.BOOLEAN, false + ); private static final FilterConversionContext CONTEXT = new FilterConversionContext( new TestDialect(), new SqlNameGenerator(128), @@ -84,6 +96,42 @@ void shouldConvertNumericColumnRange() { assertEmptySelects(result.getSelects()); } + @Test + void shouldConvertDateDistanceRangeThroughDialect() { + BuiltInFilters.DateDistanceRange filter = new BuiltInFilters.DateDistanceRange( + "age", + DATE_COLUMN, + ChronoUnit.YEARS, + LocalDate.of(2025, 2, 1), + NumberRange.closed(18, 65) + ); + + SqlFilters result = converter.convert(filter, CONTEXT); + + assertEquals( + "(date_distance('years', \"analytics\".\"events\".\"start_date\", date '2025-02-01') >= 18 and date_distance('years', \"analytics\".\"events\".\"start_date\", date '2025-02-01') <= 65)", + render(result) + ); + } + + @Test + void shouldConvertSelectedFlags() { + BuiltInFilters.Flags filter = new BuiltInFilters.Flags( + "flags", + Map.of("A", FLAG_A, "B", FLAG_B), + Set.of("A", "B") + ); + + SqlFilters result = converter.convert(filter, CONTEXT); + + assertTrue( + Set.of( + "(\"analytics\".\"events\".\"flag_a\" = true or \"analytics\".\"events\".\"flag_b\" = true)", + "(\"analytics\".\"events\".\"flag_b\" = true or \"analytics\".\"events\".\"flag_a\" = true)" + ).contains(render(result)) + ); + } + @Test void shouldDispatchExtensionFilter() { Converter extension = new Converter<>() { @@ -139,6 +187,17 @@ public Field maximumDate() { return field(name("maximum_date"), Date.class); } + @Override + public Field dateDistance(ChronoUnit unit, Field startDate, LocalDate endDate) { + return DSL.function( + "date_distance", + Integer.class, + DSL.inline(unit.name()), + startDate, + DSL.inline(Date.valueOf(endDate)) + ); + } + @Override public Field anyValue(Field value) { return value; From 548a9aead997e0854e16f9fdfd1c66d4aaa52c8b Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 21 Sep 2026 07:52:59 +0200 Subject: [PATCH 62/81] docs: define behavior-preserving SQL connector extraction Co-authored-by: Codex GPT-6 --- ...6-09-21-sql-connector-extraction-design.md | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 docs/superpowers/specs/2026-09-21-sql-connector-extraction-design.md diff --git a/docs/superpowers/specs/2026-09-21-sql-connector-extraction-design.md b/docs/superpowers/specs/2026-09-21-sql-connector-extraction-design.md new file mode 100644 index 0000000000..35da95aedf --- /dev/null +++ b/docs/superpowers/specs/2026-09-21-sql-connector-extraction-design.md @@ -0,0 +1,57 @@ +# Complete the SQL connector extraction + +## Purpose and constraints + +Finish extracting the existing SQL connector so the productive Dropwizard backend uses a framework-neutral module. The extraction must be transferable onto a separate branch based on `develop`, without Quarkus sources, configuration, plugins, or build prerequisites. Quarkus integration is subsequent work. + +Preserve existing behavior, including quirks. Preserve recognizable class names, method structure, SQL expressions, and algorithm organization wherever the dependency boundary permits. Do not combine extraction with bug fixes, new validation rules, dependency upgrades, or general cleanup. + +## Current state + +The branch already contains `dataset-model`, resolved SQL input types, compiler contracts, query-step infrastructure, CTE composition, and condition/filter converters. The backend consumes extracted infrastructure, but `SqlCompiler` is only an interface: backend query conversion, aggregators, selects, forms, dialect implementations, and execution still contain substantive SQL responsibilities. + +Uncommitted aggregation conversion work is present and must be reviewed as part of the extraction. In particular, its distinct-sum path applies subtraction where the existing implementation ignores it. It also explicitly rejects duration `distinctBy`; compatibility must be checked against the existing path before accepting that rejection. Neither a cleaner implementation nor an existing new test establishes compatibility. + +## Approach + +Continue the existing extraction through small, independently reviewable moves. Prefer moving existing SQL algorithms and adapting their inputs over reimplementing them against the new model. Where input adaptation requires a split, leave model resolution in the backend and move SQL construction into the connector while retaining recognizable names and helper structure. + +Moving backend domain models wholesale would enlarge the shared dependency surface. Rewriting the compiler would increase behavioral and review risk. Neither is appropriate for this extraction. + +## Module boundary + +`dataset-model` contains only shared dataset vocabulary needed by the connector. It must not depend on either backend. Additional domain types are not moved merely to make compilation convenient. + +`sql-connector` owns backend-independent SQL construction, compilation orchestration, SQL dialect functions, and reusable SQL execution/reading mechanics. It may depend on `dataset-model`, jOOQ, and narrowly needed libraries, but not backend classes, Dropwizard lifecycle/configuration, Quarkus, or repository-based identifier resolution. + +The Dropwizard backend owns request deserialization, identifier and saved-query resolution, dataset access, permissions, application configuration, datasource lifecycle, execution tracking, and conversion to application-specific result/error types. Adapters must pass resolved data into the connector and translate results without duplicating SQL algorithms. + +Existing JDBC behavior, resource ownership, streaming, error translation, and result column ordering must be preserved when execution mechanics are separated. Application execution state and result metadata remain backend concerns. + +## Compatibility requirements + +Preserve SQL operations for concept queries, logical nodes, external IDs, date restrictions, secondary IDs, reused queries, table exports, and absolute/relative forms. Preserve dialect-specific behavior for the currently supported HANA and ClickHouse paths. + +Preserve numeric types, null handling, range bounds, distinctness, subtraction, quarter counting, interval packing, aliases, projection order, and result interpretation. Table-export operations must retain their row-level semantics rather than accidentally adopting grouped-query semantics. + +New resolved-model validation must not reject a previously supported request or change when application errors are reported. Backend adapters must preserve defaults and resolution semantics. Known quirks receive characterization coverage rather than fixes in this change. + +Remove superseded SQL implementations only after the production backend delegates to the extracted implementation and compatibility checks pass. Completion must not leave two independently maintained implementations of the same SQL algorithm. + +## Verification + +1. Establish the existing test baseline and characterize behavior-sensitive paths before replacing their implementation. Compare extracted output with the existing implementation, using equivalent resolved inputs and dialect settings. +2. Run connector tests and affected backend tests. Exercise aggregations, selects, forms, and export paths; include nulls, duplicate events, distinct sum with subtraction, date boundaries, and duration distinctness. +3. Use existing SQL integration fixtures for result-level verification on supported databases where infrastructure is available. SQL text checks alone do not establish result parity. Record unavailable integration infrastructure explicitly. +4. Verify module dependencies and source imports contain no backend or framework implementation dependencies in the connector. +5. Assemble the extraction in an isolated checkout based on the recorded local `develop` revision. Apply only extraction changes and required neutral module/build additions. Run the module and Dropwizard build/test checks with no Quarkus modules present. Do not infer portability solely from a selected-module build on the migration branch. + +## Review and portability + +Keep extraction changes grouped by responsibility: shared vocabulary/build wiring, compiler infrastructure, operations, orchestration/dialects, and backend integration/execution. Avoid formatting unrelated files. Document original-to-extracted class mappings where names or responsibility splits are unavoidable. + +Keep Quarkus changes out of extraction commits. Shared-file edits, especially the root POM, must be separable at the hunk level. Record the extraction commit/patch set and any application conflicts encountered during the `develop` verification. Do not rebase or rewrite the current migration branch to perform that check. + +## Completion criteria + +The Dropwizard backend uses the extracted SQL implementation for its supported paths; application-specific adapters are the remaining backend boundary. Compatibility checks pass without intended behavior changes, code remains recognizable, and an extraction-only checkout based on `develop` builds without knowledge of Quarkus. Any unverified database checks are reported as limitations rather than claimed successes. From 5d36a83edda6626945e968f1ff75a198d256ccc8 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 21 Sep 2026 15:26:37 +0200 Subject: [PATCH 63/81] refactor: delegate SQL aggregations to the connector Preserve develop SQL semantics for subtraction, distinct keys, numeric types, and date ranges. Record local compatibility coverage and the remaining extraction plan. Co-authored-by: Codex GPT-6 --- .../dialect/LegacyCompilerDialect.java | 30 ++ .../dialect/SqlFunctionProvider.java | 8 + .../ClickhouseFunctionProvider.java | 9 +- .../conversion/model/EntitySchemaAdapter.java | 14 + .../CountQuartersSqlAggregator.java | 114 +------ .../model/aggregator/CountSqlAggregator.java | 45 ++- .../aggregator/DurationSumSqlAggregator.java | 38 ++- .../ResolvedAggregationAdapter.java | 23 ++ .../model/aggregator/SumSqlAggregator.java | 205 ++----------- .../SqlExtractionAggregationTest.java | 149 +++++++++ .../conversion/SqlExtractionDialectTest.java | 26 ++ docs/sql-connector-extraction-verification.md | 37 +++ .../2026-09-21-sql-connector-extraction.md | 211 +++++++++++++ ...6-09-21-sql-connector-extraction-design.md | 14 +- sql-connector/README.md | 21 ++ .../AggregationConversionContext.java | 29 ++ .../BuiltInAggregationConverters.java | 91 ++++++ .../BuiltInAggregationFilterConverters.java | 76 +++++ .../operation/DateAggregationConverters.java | 115 +++++++ .../ResolvedAggregationConverter.java | 37 +++ .../operation/ResolvedFilterConverter.java | 10 +- .../operation/SumAggregationConverter.java | 121 ++++++++ .../sql/compiler/dialect/CompilerDialect.java | 38 +++ .../ir/concept/CommonAggregationSelect.java | 2 + .../ResolvedAggregationConverterTest.java | 285 ++++++++++++++++++ .../ResolvedFilterConverterTest.java | 137 ++++++++- 26 files changed, 1553 insertions(+), 332 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/ResolvedAggregationAdapter.java create mode 100644 backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionAggregationTest.java create mode 100644 backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionDialectTest.java create mode 100644 docs/sql-connector-extraction-verification.md create mode 100644 docs/superpowers/plans/2026-09-21-sql-connector-extraction.md create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/AggregationConversionContext.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInAggregationConverters.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInAggregationFilterConverters.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/DateAggregationConverters.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedAggregationConverter.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/SumAggregationConverter.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedAggregationConverterTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java index 0bcb7b511e..e85890f82d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java @@ -123,6 +123,36 @@ default Field dateDistance(ChronoUnit unit, Field startDate, Loca StratificationFunctions getStratificationFunctions(); + @Override + default Field dateDistance(ChronoUnit unit, Field startDate, Field endDate) { + return getFunctionProvider().dateDistance(unit, startDate, endDate); + } + + @Override + default Field addDays(Field date, Field days) { + return getFunctionProvider().addDays(date, days); + } + + @Override + default ColumnDateRange dateRange(Field start, Field inclusiveEnd) { + return getFunctionProvider().dateRange(start, inclusiveEnd); + } + + @Override + default Field yearQuarter(Field date) { + return getFunctionProvider().yearQuarter(date); + } + + @Override + default Field quarterStart(Field date) { + return getStratificationFunctions().jumpToQuarterStart(date); + } + + @Override + default Field nextQuarterStart(Field date) { + return getStratificationFunctions().jumpToNextQuarterStart(date); + } + SqlFunctionProvider getFunctionProvider(); List> getNodeConverters(DSLContext context); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index e291329a52..be44c90f97 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -142,6 +142,14 @@ default Condition isNotEmptyValidityDate(ValidityDate validityDate) { ColumnDateRange forArbitraryDateRange(DaterangeSelectOrFilter daterangeSelectOrFilter); + /** Converts inclusive date expressions while retaining the dialect's nullability semantics. */ + default ColumnDateRange dateRange(Field start, Field inclusiveEnd) { + return ColumnDateRange.of( + coalesce(start, getMinDateExpression()), + coalesce(addDays(inclusiveEnd, inline(1)), getMaxDateExpression()) + ); + } + /** * Aggregate columnDateRange into dateSpans of the grouping. */ diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java index 818fd629cb..03a85f02de 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java @@ -120,20 +120,25 @@ public Field arrayOut(List> fields) { } private ColumnDateRange ofStartAndEnd(String tableName, Column startColumn, Column endColumn) { + return dateRange(field(name(tableName, startColumn.getName()), Date.class), field(name(tableName, endColumn.getName()), Date.class)); + } + + @Override + public ColumnDateRange dateRange(Field start, Field inclusiveEnd) { // Since coalesce makes Clickhouse certain, that the field is not nullable, it will do silly stuff with it down the line: // missing values (for example in outer-joins) will be coerced to 0 = 01-01-1970, which is clearly not correct // Therefore we tag the values as Nullable again to make Clickhouse show some respect Field rangeStart = field("{0}::Nullable(Date32)", Date.class, coalesce( - field(name(tableName, startColumn.getName()), Date.class), + start, getMinDateExpression() ) ); // when aggregating date ranges, we want to treat the last day of the range as excluded, // so when using the date value of the end column, we add +1 day as end of the date range Field rangeEnd = field("{0}::Nullable(Date32)", Date.class, coalesce( - addDays(field(name(tableName, endColumn.getName()), Date.class), inline(1)), + addDays(inclusiveEnd, inline(1)), getMaxDateExpression() ) ); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/EntitySchemaAdapter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/EntitySchemaAdapter.java index 7a7b7aa66b..55073a3636 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/EntitySchemaAdapter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/EntitySchemaAdapter.java @@ -3,7 +3,10 @@ import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.models.config.IdColumnConfig; import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter; +import com.bakdata.conquery.sql.model.schema.DateColumns; import com.bakdata.conquery.sql.model.schema.EntitySchema; import com.bakdata.conquery.sql.model.schema.ResolvedColumn; import com.bakdata.conquery.sql.model.schema.SqlTable; @@ -34,4 +37,15 @@ public static EntitySchema from(IdColumnConfig idColumns) { public static SqlTable from(Table table) { return SqlTable.of(table.getId().toString(), table.getName()); } + + public static ResolvedColumn from(Column column) { + return new ResolvedColumn(column.getId().toString(), from(column.getTable()), column.getName(), + ColumnType.valueOf(column.getType().name()), true); + } + + public static DateColumns from(DaterangeSelectOrFilter dates) { + return dates.isSingleColumnDaterange() + ? new DateColumns.Single(from(dates.getColumn().resolve())) + : new DateColumns.Pair(from(dates.getStartColumn().resolve()), from(dates.getEndColumn().resolve())); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java index 0100eb56dc..3987ef2237 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java @@ -1,29 +1,20 @@ package com.bakdata.conquery.sql.conversion.model.aggregator; -import java.math.BigDecimal; -import java.sql.Date; -import java.time.temporal.ChronoUnit; - +import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; +import com.bakdata.conquery.sql.model.operation.BuiltInAggregations; import com.bakdata.conquery.models.common.Range; -import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.filters.specific.CountQuartersFilter; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.CountQuartersSelect; -import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; import com.bakdata.conquery.sql.compiler.ir.concept.CommonAggregationSelect; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; -import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; -import com.bakdata.conquery.sql.compiler.dialect.Interval; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.bakdata.conquery.sql.conversion.model.filter.LegacyInclusiveRangeCondition; import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; -import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import org.jooq.Condition; @@ -34,90 +25,17 @@ //TODO(FK): this needs a rework. Current implementation makes a sum of the quarters and doesn't take overlapping events into account. public class CountQuartersSqlAggregator implements SelectConverter, FilterConverter, SqlAggregator { - private static CommonAggregationSelect createSingleDateColumnAggregationSelect( - Column countColumn, - String alias, - SqlTables tables, - SqlFunctionProvider functionProvider) { - - ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), countColumn.getName(), Date.class); - - Field qualifiedRootSelect = rootSelect.qualify(tables.cteName(ConceptCteStep.PREPROCESSING)).select(); - FieldWrapper countQuartersAggregation = - new FieldWrapper<>(DSL.nullif(DSL.countDistinct(functionProvider.yearQuarter(qualifiedRootSelect)), 0).as(alias), countColumn.getName()); - - return CommonAggregationSelect.builder().rootSelect(rootSelect).groupBy(countQuartersAggregation).build(); - } - - private static CommonAggregationSelect createTwoDateColumnAggregationSelect( - Column startColumn, - Column endColumn, - String alias, - SqlTables tables, - SqlFunctionProvider functionProvider, - StratificationFunctions stratificationFunctions) { - - Field startDate = DSL.field(DSL.name(tables.getRootTable(), startColumn.getName()), Date.class); - Field endDate = DSL.field(DSL.name(tables.getRootTable(), endColumn.getName()), Date.class); - - Field quarterStart = stratificationFunctions.jumpToQuarterStart(startDate); - Field nextQuarterStart = stratificationFunctions.jumpToNextQuarterStart(endDate); - - return sumQuarterCount(quarterStart, nextQuarterStart, alias, tables, functionProvider); - } - - private static CommonAggregationSelect sumQuarterCount( - Field quarterStart, - Field nextQuarterStart, - String alias, - SqlTables tables, - SqlFunctionProvider functionProvider) { - Field quarterCount = calcQuarterCount(quarterStart, nextQuarterStart, alias, functionProvider); - FieldWrapper quarterCountWrapper = new FieldWrapper<>(quarterCount); - - Field qualifiedQuarterCount = quarterCountWrapper.qualify(tables.cteName(ConceptCteStep.PREPROCESSING)).select(); - FieldWrapper quarterCountAggregation = new FieldWrapper<>(DSL.nullif(DSL.sum(qualifiedQuarterCount), BigDecimal.ZERO).as(alias)); - - return CommonAggregationSelect.builder().rootSelect(quarterCountWrapper).groupBy(quarterCountAggregation).build(); - } - - private static Field calcQuarterCount(Field quarterStart, Field nextQuarterStart, String alias, SqlFunctionProvider functionProvider) { - return functionProvider.dateDistance(ChronoUnit.MONTHS, quarterStart, nextQuarterStart).divide(Interval.QUARTER_INTERVAL.getAmount()).as(alias); - } - - private static CommonAggregationSelect buildSqlSelect( - ColumnId column, - ColumnId startColumn, - ColumnId endColumn, - String alias, - SqlTables tables, - SqlFunctionProvider functionProvider, - StratificationFunctions stratificationFunctions) { - - if (column == null) { - return createTwoDateColumnAggregationSelect(startColumn.resolve(), endColumn.resolve(), alias, tables, functionProvider, stratificationFunctions); - } - - Column countColumn = column.resolve(); - - return createSingleDateColumnAggregationSelect(countColumn, alias, tables, functionProvider); - } - @Override public ConnectorSqlSelects connectorSelect(CountQuartersSelect countQuartersSelect, SelectContext selectContext) { - CommonAggregationSelect countAggregationSelect = - buildSqlSelect(countQuartersSelect.getColumn(), - countQuartersSelect.getStartColumn(), - countQuartersSelect.getEndColumn(), - selectContext.getNameGenerator().legacyOperationName(countQuartersSelect.getName()), - selectContext.getTables(), - selectContext.getFunctionProvider(), - selectContext.getCompilerDialect().getStratificationFunctions() - ); + CommonAggregationSelect countAggregationSelect = + ResolvedAggregationAdapter.convert( + new BuiltInAggregations.CountQuarters(EntitySchemaAdapter.from(countQuartersSelect)), + selectContext.getNameGenerator().legacyOperationName(countQuartersSelect.getName()), + selectContext.getIds(), selectContext.getTables(), selectContext); String finalPredecessor = selectContext.getTables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER); - ExtractingSqlSelect finalSelect = countAggregationSelect.getGroupBy().qualify(finalPredecessor); + ExtractingSqlSelect finalSelect = countAggregationSelect.getGroupBy().qualify(finalPredecessor); return ConnectorSqlSelects.builder() .preprocessingSelects(countAggregationSelect.getRootSelects()) @@ -129,15 +47,11 @@ public ConnectorSqlSelects connectorSelect(CountQuartersSelect countQuartersSele @Override public SqlFilters convertToSqlFilter(CountQuartersFilter countQuartersFilter, FilterContext filterContext) { - CommonAggregationSelect countAggregationSelect = - buildSqlSelect(countQuartersFilter.getColumn(), - countQuartersFilter.getStartColumn(), - countQuartersFilter.getEndColumn(), - filterContext.getNameGenerator().legacyOperationName(countQuartersFilter.getName()), - filterContext.getTables(), - filterContext.getFunctionProvider(), - filterContext.getCompilerDialect().getStratificationFunctions() - ); + CommonAggregationSelect countAggregationSelect = + ResolvedAggregationAdapter.convert( + new BuiltInAggregations.CountQuarters(EntitySchemaAdapter.from(countQuartersFilter)), + filterContext.getNameGenerator().legacyOperationName(countQuartersFilter.getName()), + filterContext.getIds(), filterContext.getTables(), filterContext); ConnectorSqlSelects selects = ConnectorSqlSelects.builder() .preprocessingSelects(countAggregationSelect.getRootSelects()) @@ -145,7 +59,7 @@ public SqlFilters convertToSqlFilter(CountQuartersFilter countQuartersFilter, Fi .build(); String predecessorTableName = filterContext.getTables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER); - Field qualifiedCountSelect = countAggregationSelect.getGroupBy().qualify(predecessorTableName).select(); + Field qualifiedCountSelect = countAggregationSelect.getGroupBy().qualify(predecessorTableName).select(); LegacyInclusiveRangeCondition countCondition = new LegacyInclusiveRangeCondition(qualifiedCountSelect, filterContext.getValue()); WhereClauses whereClauses = WhereClauses.builder().groupFilter(countCondition).build(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java index aa8831cb38..1a26302616 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java @@ -1,5 +1,12 @@ package com.bakdata.conquery.sql.conversion.model.aggregator; +import java.util.List; + +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.conversion.Context; +import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; +import com.bakdata.conquery.sql.model.operation.BuiltInAggregations; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.filters.specific.CountFilter; @@ -15,7 +22,6 @@ import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; -import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import lombok.NoArgsConstructor; @@ -35,10 +41,10 @@ public ConnectorSqlSelects connectorSelect(CountSelect countSelect, SelectContex Column countColumn = countSelect.getColumn().resolve(); String alias = selectContext.getNameGenerator().legacyOperationName(countSelect.getName()); - CommonAggregationSelect countAggregationSelect = createCountAggregationSelect(countColumn, distinct, alias, tables); + CommonAggregationSelect countAggregationSelect = createCountAggregationSelect(countColumn, distinct, alias, tables, selectContext.getIds(), selectContext); String finalPredecessor = tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER); - ExtractingSqlSelect finalSelect = countAggregationSelect.getGroupBy().qualify(finalPredecessor); + ExtractingSqlSelect finalSelect = countAggregationSelect.getGroupBy().qualify(finalPredecessor); return ConnectorSqlSelects.builder() .preprocessingSelects(countAggregationSelect.getRootSelects()) @@ -47,23 +53,6 @@ public ConnectorSqlSelects connectorSelect(CountSelect countSelect, SelectContex .build(); } - private CommonAggregationSelect createCountAggregationSelect(Column countColumn, boolean distinct, String alias, SqlTables tables) { - - ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), countColumn.getName(), Object.class); - - - Field qualifiedRootSelect = rootSelect.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT)).select(); - Field countField = distinct - ? DSL.countDistinct(qualifiedRootSelect) - : DSL.count(qualifiedRootSelect); - FieldWrapper countGroupBy = new FieldWrapper<>(DSL.nullif(countField, 0).as(alias), countColumn.getName()); - - return CommonAggregationSelect.builder() - .rootSelect(rootSelect) - .groupBy(countGroupBy) - .build(); - } - @Override public SqlFilters convertToSqlFilter(CountFilter countFilter, FilterContext filterContext) { @@ -72,13 +61,13 @@ public SqlFilters convertToSqlFilter(CountFilter countFilter, FilterContext countAggregationSelect = createCountAggregationSelect(countColumn, distinct, alias, tables); + CommonAggregationSelect countAggregationSelect = createCountAggregationSelect(countColumn, distinct, alias, tables, filterContext.getIds(), filterContext); ConnectorSqlSelects selects = ConnectorSqlSelects.builder() .preprocessingSelects(countAggregationSelect.getRootSelects()) .aggregationSelect(countAggregationSelect.getGroupBy()) .build(); - Field qualifiedCountSelect = countAggregationSelect.getGroupBy().qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)).select(); + Field qualifiedCountSelect = countAggregationSelect.getGroupBy().qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)).select(); LegacyInclusiveRangeCondition countCondition = new LegacyInclusiveRangeCondition(qualifiedCountSelect, filterContext.getValue()); WhereClauses whereClauses = WhereClauses.builder() .groupFilter(countCondition) @@ -92,7 +81,13 @@ public Condition convertForTableExport(CountFilter countFilter, FilterContext field = DSL.inline(1); // no grouping, count is always 1 per row return new LegacyInclusiveRangeCondition(field, filterContext.getValue()).condition(); } - - - + private static CommonAggregationSelect createCountAggregationSelect( + Column countColumn, boolean distinct, String alias, SqlTables tables, + SqlIdColumns ids, Context context + ) { + ResolvedColumn column = EntitySchemaAdapter.from(countColumn); + // Existing SQL COUNT DISTINCT uses the counted column, not distinctByColumn. + return ResolvedAggregationAdapter.convert(new BuiltInAggregations.Count(column, distinct ? List.of(column) : List.of()), + alias, ids, tables, context); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DurationSumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DurationSumSqlAggregator.java index 98f791508a..e85fc18f6e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DurationSumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DurationSumSqlAggregator.java @@ -4,6 +4,7 @@ import java.sql.Date; import java.time.temporal.ChronoUnit; +import java.util.List; import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.common.Range.LongRange; @@ -13,13 +14,14 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; +import com.bakdata.conquery.sql.model.operation.BuiltInAggregations; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.bakdata.conquery.sql.conversion.model.filter.LegacyInclusiveRangeCondition; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; -import com.bakdata.conquery.sql.conversion.model.select.DaterangeSelectUtil; -import com.bakdata.conquery.sql.conversion.model.select.DaterangeSelectUtil.AggregationFunction; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import org.jooq.Condition; @@ -30,16 +32,28 @@ public class DurationSumSqlAggregator implements SelectConverter selectContext) { - return DaterangeSelectUtil.createForSelect(select, durationSumSelectFunction(), selectContext); + String alias = selectContext.getNameGenerator().legacyOperationName(select.getName()); + var aggregation = ResolvedAggregationAdapter.convert( + new BuiltInAggregations.DurationSum(EntitySchemaAdapter.from(select), List.of()), + alias, selectContext.getIds(), selectContext.getTables(), selectContext); + return ConnectorSqlSelects.builder() + .preprocessingSelects(aggregation.getRootSelects()) + .additionalPredecessor(aggregation.getAdditionalPredecessor()) + .finalSelect(aggregation.getGroupBy().qualify(selectContext.getTables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER))) + .build(); } @Override public SqlFilters convertToSqlFilter(DurationSumFilter filter, FilterContext context) { - return DaterangeSelectUtil.createForFilter( - filter, - durationSumSelectFunction(), - (aggregationField -> new LegacyInclusiveRangeCondition(aggregationField, context.getValue())), - context); + String alias = context.getNameGenerator().legacyOperationName(filter.getName()); + var aggregation = ResolvedAggregationAdapter.convert( + new BuiltInAggregations.DurationSum(EntitySchemaAdapter.from(filter), List.of()), + alias, context.getIds(), context.getTables(), context); + Field field = aggregation.getGroupBy().qualify(context.getTables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER)).select(); + return new SqlFilters(ConnectorSqlSelects.builder() + .preprocessingSelects(aggregation.getRootSelects()) + .additionalPredecessor(aggregation.getAdditionalPredecessor()).build(), + WhereClauses.builder().groupFilter(new LegacyInclusiveRangeCondition(field, context.getValue())).build()); } @Override @@ -68,10 +82,4 @@ public Condition convertForTableExport(DurationSumFilter filter, FilterContext { - ColumnDateRange asDualColumn = functionProvider.toDualColumn(daterange); - return DaterangeSelectUtil.createDurationSumSqlSelect(alias, asDualColumn, functionProvider); - }; - } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/ResolvedAggregationAdapter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/ResolvedAggregationAdapter.java new file mode 100644 index 0000000000..25ea43271c --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/ResolvedAggregationAdapter.java @@ -0,0 +1,23 @@ +package com.bakdata.conquery.sql.conversion.model.aggregator; + +import com.bakdata.conquery.sql.compiler.conversion.operation.AggregationConversionContext; +import com.bakdata.conquery.sql.compiler.conversion.operation.ResolvedAggregationConverter; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.concept.CommonAggregationSelect; +import com.bakdata.conquery.sql.conversion.Context; +import com.bakdata.conquery.sql.model.operation.ResolvedAggregation; + +/** Passes resolved operations and the existing naming/table state to the shared compiler. */ +final class ResolvedAggregationAdapter { + + private static final ResolvedAggregationConverter CONVERTER = new ResolvedAggregationConverter(); + + private ResolvedAggregationAdapter() { + } + + static CommonAggregationSelect convert(ResolvedAggregation aggregation, String alias, SqlIdColumns ids, SqlTables tables, Context context) { + return CONVERTER.convert(aggregation, new AggregationConversionContext( + context.getCompilerDialect(), context.getNameGenerator(), tables, ids, alias)); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java index eb4d7281ea..a90990b13f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java @@ -1,10 +1,7 @@ package com.bakdata.conquery.sql.conversion.model.aggregator; -import java.math.BigDecimal; -import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; +import java.util.Optional; import com.bakdata.conquery.models.common.IRange; import com.bakdata.conquery.models.datasets.Column; @@ -15,11 +12,11 @@ import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; -import com.bakdata.conquery.sql.compiler.ir.CteStep; +import com.bakdata.conquery.sql.conversion.Context; +import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; +import com.bakdata.conquery.sql.model.operation.BuiltInAggregations; import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; import com.bakdata.conquery.sql.conversion.model.NumberMapUtil; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; @@ -28,12 +25,8 @@ import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; -import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; -import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; -import lombok.Getter; -import lombok.RequiredArgsConstructor; import org.jooq.Condition; import org.jooq.Field; import org.jooq.impl.DSL; @@ -82,9 +75,6 @@ public class SumSqlAggregator> impleme FilterConverter, RANGE>, SqlAggregator { - private static final String ROW_NUMBER_ALIAS = "row_number"; - private static final String SUM_DISTINCT_SUFFIX = "sum_distinct"; - @Override public ConnectorSqlSelects connectorSelect(SumSelect sumSelect, SelectContext selectContext) { @@ -98,12 +88,12 @@ public ConnectorSqlSelects connectorSelect(SumSelect sumSelect, SelectContext sumAggregationSelect; + CommonAggregationSelect sumAggregationSelect; if (!distinctByColumns.isEmpty()) { SqlIdColumns ids = selectContext.getIds(); - sumAggregationSelect = createDistinctSumAggregationSelect(sumColumn, distinctByColumns, alias, ids, tables, nameGenerator); - ExtractingSqlSelect finalSelect = createFinalSelect(sumAggregationSelect, tables); + sumAggregationSelect = createSumAggregationSelect(sumColumn, subtractColumn, distinctByColumns, alias, ids, tables, selectContext); + ExtractingSqlSelect finalSelect = createFinalSelect(sumAggregationSelect, tables); return ConnectorSqlSelects.builder() .preprocessingSelects(sumAggregationSelect.getRootSelects()) .additionalPredecessor(sumAggregationSelect.getAdditionalPredecessor()) @@ -111,8 +101,8 @@ public ConnectorSqlSelects connectorSelect(SumSelect sumSelect, SelectContext finalSelect = createFinalSelect(sumAggregationSelect, tables); + sumAggregationSelect = createSumAggregationSelect(sumColumn, subtractColumn, distinctByColumns, alias, selectContext.getIds(), tables, selectContext); + ExtractingSqlSelect finalSelect = createFinalSelect(sumAggregationSelect, tables); return ConnectorSqlSelects.builder() .preprocessingSelects(sumAggregationSelect.getRootSelects()) .aggregationSelect(sumAggregationSelect.getGroupBy()) @@ -121,154 +111,6 @@ public ConnectorSqlSelects connectorSelect(SumSelect sumSelect, SelectContext createDistinctSumAggregationSelect( - Column sumColumn, - List distinctByColumns, - String alias, - SqlIdColumns ids, - SqlTables tables, - SqlNameGenerator nameGenerator - ) { - List> preprocessingSelects = new ArrayList<>(); - - Class numberClass = NumberMapUtil.getType(sumColumn); - ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), sumColumn.getName(), numberClass); - preprocessingSelects.add(rootSelect); - - List> distinctByRootSelects = - distinctByColumns.stream() - .map(column -> new ExtractingSqlSelect<>(tables.getRootTable(), column.getName(), Object.class)) - .collect(Collectors.toList()); - preprocessingSelects.addAll(distinctByRootSelects); - - QueryStep rowNumberCte = createRowNumberCte(ids, rootSelect, distinctByRootSelects, alias, tables, nameGenerator); - Field rootSelectQualified = rootSelect.qualify(rowNumberCte.getCteName()).select(); - FieldWrapper sumGroupBy = new FieldWrapper<>(DSL.sum(DSL.coalesce(rootSelectQualified, DSL.inline(0))).as(alias)); - QueryStep rowNumberFilteredCte = createRowNumberFilteredCte(rowNumberCte, sumGroupBy, alias, nameGenerator); - - return CommonAggregationSelect.builder() - .rootSelects(preprocessingSelects) - .additionalPredecessor(rowNumberFilteredCte) - .groupBy(sumGroupBy) - .build(); - } - - private static ExtractingSqlSelect createFinalSelect(CommonAggregationSelect sumAggregationSelect, SqlTables tables) { - String finalPredecessor = tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER); - return sumAggregationSelect.getGroupBy().qualify(finalPredecessor); - } - - private CommonAggregationSelect createSumAggregationSelect(Column sumColumn, Column subtractColumn, String alias, SqlTables tables) { - - Class numberClass = NumberMapUtil.getType(sumColumn); - List> preprocessingSelects = new ArrayList<>(); - - ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), sumColumn.getName(), numberClass); - preprocessingSelects.add(rootSelect); - - String preprocessingCte = tables.cteName(ConceptCteStep.PREPROCESSING); - Field sumField = rootSelect.qualify(preprocessingCte).select(); - - FieldWrapper sumGroupBy; - - - if (subtractColumn != null) { - ExtractingSqlSelect subtractColumnRootSelect = new ExtractingSqlSelect<>( - tables.getRootTable(), - subtractColumn.getName(), - numberClass - ); - preprocessingSelects.add(subtractColumnRootSelect); - - Field subtractField = subtractColumnRootSelect.qualify(preprocessingCte).select(); - - - // This expression ensures that if there's any non-null field, we get a 0. But if there's only nulls we get a null: - // COALESCE would always result in 0 which is undesired to differentiate between missing-values and 0 sums. - Field zeroIfAnyNonNull = DSL.coalesce(sumField.multiply(0), subtractField.multiply(0)); - - sumGroupBy = new FieldWrapper<>(DSL.sum(DSL.coalesce(sumField, zeroIfAnyNonNull).minus(DSL.coalesce(subtractField, zeroIfAnyNonNull))).as(alias), - sumColumn.getName(), - subtractColumn.getName() - ); - } - else { - sumGroupBy = new FieldWrapper<>(DSL.sum(sumField).as(alias), sumColumn.getName()); - } - - return CommonAggregationSelect.builder() - .rootSelects(preprocessingSelects) - .groupBy(sumGroupBy) - .build(); - } - - /** - * Assigns row numbers for each partition over the pid and the distinct by columns. If the values per pid in the distinct by columns are duplicated, - * the row number will be incremented for each duplicated entry. - */ - private static QueryStep createRowNumberCte( - SqlIdColumns ids, - SingleColumnSqlSelect sumColumnRootSelect, - List> distinctByRootSelects, - String alias, - SqlTables connectorTables, - SqlNameGenerator nameGenerator - ) { - String predecessor = connectorTables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT); - SqlIdColumns qualifiedIds = ids.qualify(predecessor); - SingleColumnSqlSelect qualifiedSumRootSelect = sumColumnRootSelect.qualify(predecessor); - - List> partitioningFields = Stream.concat( - qualifiedIds.toFields().stream(), - distinctByRootSelects.stream().map(sqlSelect -> sqlSelect.qualify(predecessor).select()) - ) - .collect(Collectors.toList()); - FieldWrapper rowNumber = new FieldWrapper<>( - DSL.rowNumber().over(DSL.partitionBy(partitioningFields)).as(ROW_NUMBER_ALIAS), - partitioningFields.stream().map(Field::getName).toArray(String[]::new) - ); - - Selects rowNumberAssignedSelects = Selects.builder() - .ids(qualifiedIds) - .sqlSelects(List.of(qualifiedSumRootSelect, rowNumber)) - .build(); - - return QueryStep.builder() - .cteName(nameGenerator.cteStepName(SumDistinctCteStep.ROW_NUMBER_ASSIGNED, alias)) - .selects(rowNumberAssignedSelects) - .fromTable(QueryStep.toTableLike(predecessor)) - .build(); - } - - /** - * Sums up the sum column values but only those whose row number is 1. Thus, only unique entries will be summed up. - */ - private static QueryStep createRowNumberFilteredCte( - QueryStep rowNumberCte, - FieldWrapper sumSelect, - String alias, - SqlNameGenerator nameGenerator - ) { - SqlIdColumns ids = rowNumberCte.getQualifiedSelects().getIds(); - - Selects rowNumberFilteredSelects = Selects.builder() - .ids(ids) - .sqlSelects(List.of(sumSelect)) - .build(); - - Condition firstOccurrence = DSL.field(DSL.name(rowNumberCte.getCteName(), ROW_NUMBER_ALIAS)) - .eq(DSL.inline(1)); - - return QueryStep.builder() - .cteName(nameGenerator.cteStepName(SumDistinctCteStep.ROW_NUMBER_FILTERED, alias)) - .selects(rowNumberFilteredSelects) - .fromTable(QueryStep.toTableLike(rowNumberCte.getCteName())) - .conditions(List.of(firstOccurrence)) - .predecessors(List.of(rowNumberCte)) - .groupBy(ids.toFields()) - .build(); - } - @Override public SqlFilters convertToSqlFilter(SumFilter sumFilter, FilterContext filterContext) { @@ -278,19 +120,19 @@ public SqlFilters convertToSqlFilter(SumFilter sumFilter, FilterContext sumAggregationSelect; + CommonAggregationSelect sumAggregationSelect; ConnectorSqlSelects selects; if (!distinctByColumns.isEmpty()) { sumAggregationSelect = - createDistinctSumAggregationSelect(sumColumn, distinctByColumns, alias, filterContext.getIds(), tables, filterContext.getNameGenerator()); + createSumAggregationSelect(sumColumn, subtractColumn, distinctByColumns, alias, filterContext.getIds(), tables, filterContext); selects = ConnectorSqlSelects.builder() .preprocessingSelects(sumAggregationSelect.getRootSelects()) .additionalPredecessor(sumAggregationSelect.getAdditionalPredecessor()) .build(); } else { - sumAggregationSelect = createSumAggregationSelect(sumColumn, subtractColumn, alias, tables); + sumAggregationSelect = createSumAggregationSelect(sumColumn, subtractColumn, distinctByColumns, alias, filterContext.getIds(), tables, filterContext); selects = ConnectorSqlSelects.builder() .preprocessingSelects(sumAggregationSelect.getRootSelects()) .additionalPredecessor(sumAggregationSelect.getAdditionalPredecessor()) @@ -298,7 +140,7 @@ public SqlFilters convertToSqlFilter(SumFilter sumFilter, FilterContext qualifiedSumSelect = sumAggregationSelect.getGroupBy().qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)).select(); + Field qualifiedSumSelect = sumAggregationSelect.getGroupBy().qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)).select(); LegacyInclusiveRangeCondition sumCondition = new LegacyInclusiveRangeCondition(qualifiedSumSelect, filterContext.getValue()); WhereClauses whereClauses = WhereClauses.builder() .groupFilter(sumCondition) @@ -329,15 +171,18 @@ public Condition convertForTableExport(SumFilter filter, FilterContext createSumAggregationSelect( + Column sumColumn, Column subtractColumn, List distinctByColumns, + String alias, SqlIdColumns ids, SqlTables tables, Context context + ) { + return ResolvedAggregationAdapter.convert(new BuiltInAggregations.Sum( + EntitySchemaAdapter.from(sumColumn), + Optional.ofNullable(subtractColumn).map(EntitySchemaAdapter::from), + distinctByColumns.stream().map(EntitySchemaAdapter::from).toList() + ), alias, ids, tables, context); } + private static ExtractingSqlSelect createFinalSelect(CommonAggregationSelect aggregation, SqlTables tables) { + return aggregation.getGroupBy().qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); + } } diff --git a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionAggregationTest.java b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionAggregationTest.java new file mode 100644 index 0000000000..ce09012ba6 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionAggregationTest.java @@ -0,0 +1,149 @@ +package com.bakdata.conquery.sql.conversion; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.CountSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.CountQuartersSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DurationSumSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.SumSelect; +import com.bakdata.conquery.models.events.MajorTypeId; +import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; +import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCtePlan; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; +import com.bakdata.conquery.sql.conversion.dialect.hana.HanaDialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.clickhouse.ClickhouseDialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; +import com.bakdata.conquery.sql.conversion.model.aggregator.CountSqlAggregator; +import com.bakdata.conquery.sql.conversion.model.aggregator.CountQuartersSqlAggregator; +import com.bakdata.conquery.sql.conversion.model.aggregator.DurationSumSqlAggregator; +import com.bakdata.conquery.sql.conversion.model.aggregator.SumSqlAggregator; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; +import org.jooq.Field; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Test; + +/** Exercises application adapters without a repository or database connection. */ +class SqlExtractionAggregationTest { + + @Test + void shouldKeepDistinctSumSubtractionAndPredecessorBehavior() { + SumSelect select = new SumSelect(column("amount", MajorTypeId.INTEGER), column("discount", MajorTypeId.INTEGER)); + select.setName("total"); + select.setDistinctByColumn(List.of(column("key", MajorTypeId.STRING))); + var result = new SumSqlAggregator().connectorSelect(select, context()); + var predecessor = result.getAdditionalPredecessor().orElseThrow(); + + assertTrue(result.getAggregationSelects().isEmpty()); + assertEquals("total-1-row_number_filtered", predecessor.getCteName()); + assertEquals("sum(coalesce(\"total-1-row_number_assigned\".\"amount\", 0)) as \"total-1\"", + render(predecessor.getSelects().getSqlSelects().getFirst().toFields().getFirst())); + assertEquals(List.of("\"events\".\"amount\"", "\"events\".\"key\""), result.getPreprocessingSelects().stream() + .map(value -> render(value.toFields().getFirst())).toList()); + assertEquals(Integer.class, result.getPreprocessingSelects().getFirst().toFields().getFirst().getType()); + assertEquals("\"aggregation\".\"total-1\"", render(result.getFinalSelects().getFirst().toFields().getFirst())); + } + + @Test + void shouldKeepNullAwareOrdinarySumSubtraction() { + SumSelect select = new SumSelect(column("amount", MajorTypeId.MONEY), column("discount", MajorTypeId.MONEY)); + select.setName("total"); + var result = new SumSqlAggregator().connectorSelect(select, context()); + String zero = "coalesce((\"preprocessing\".\"amount\" * 0), (\"preprocessing\".\"discount\" * 0))"; + assertEquals("sum((coalesce(\"preprocessing\".\"amount\", " + zero + ") - coalesce(\"preprocessing\".\"discount\", " + zero + "))) as \"total-1\"", + render(result.getAggregationSelects().getFirst().toFields().getFirst())); + assertTrue(result.getAdditionalPredecessor().isEmpty()); + } + + @Test + void shouldCountDistinctColumnRatherThanConfiguredDistinctKeys() { + CountSelect select = new CountSelect(); + select.setName("count"); + select.setColumn(column("value", MajorTypeId.STRING)); + select.setDistinct(true); + select.setDistinctByColumn(List.of(column("key", MajorTypeId.STRING))); + var result = new CountSqlAggregator().connectorSelect(select, context()); + assertEquals("nullif(count(distinct \"preprocessing\".\"value\"), 0) as \"count-1\"", + render(result.getAggregationSelects().getFirst().toFields().getFirst())); + } + + private static SelectContext context() { + return context(new HanaDialectBundle()); + } + + @Test + void shouldSumQuartersPerEventForDatePairs() { + CountQuartersSelect select = new CountQuartersSelect(); + select.setName("quarters"); + select.setStartColumn(column("start", MajorTypeId.DATE)); + select.setEndColumn(column("end", MajorTypeId.DATE)); + var result = new CountQuartersSqlAggregator().connectorSelect(select, context()); + assertEquals("nullif(sum(\"preprocessing\".\"quarters-1\"), 0) as \"quarters-1\"", + render(result.getAggregationSelects().getFirst().toFields().getFirst())); + assertTrue(result.getAdditionalPredecessor().isEmpty()); + } + + @Test + void shouldPackDurationIntervalsWithoutApplyingDistinctKeys() { + DurationSumSelect select = new DurationSumSelect(); + select.setName("duration"); + select.setStartColumn(column("start", MajorTypeId.DATE)); + select.setEndColumn(column("end", MajorTypeId.DATE)); + select.setDistinctBy(List.of(column("key", MajorTypeId.STRING))); + var result = new DurationSumSqlAggregator().connectorSelect(select, context(new ClickhouseDialectBundle())); + assertEquals("duration-1-interval_packing_selects", result.getAdditionalPredecessor().orElseThrow().getCteName()); + assertTrue(result.getAggregationSelects().isEmpty()); + assertEquals(2, result.getPreprocessingSelects().size()); + assertTrue(render(result.getPreprocessingSelects().getFirst().toFields().getFirst()).contains("::nullable(date32)")); + assertEquals("\"aggregation\".\"duration-1\"", render(result.getFinalSelects().getFirst().toFields().getFirst())); + } + + private static SelectContext context(LegacyCompilerDialect dialect) { + var conversion = ConversionContext.builder() + .compilerDialect(dialect).stratificationFunctions(dialect.getStratificationFunctions()) + .nameGenerator(new SqlNameGenerator(127)).build(); + var tables = new SqlTables("events", + Map.of(ConceptCteStep.PREPROCESSING, "preprocessing", ConceptCteStep.AGGREGATION_SELECT, "aggregation"), + Map.of(ConceptCteStep.AGGREGATION_SELECT, ConceptCteStep.PREPROCESSING, + ConceptCteStep.AGGREGATION_FILTER, ConceptCteStep.AGGREGATION_SELECT)); + var plan = new ConnectorCtePlan("events", DSL.table(DSL.name("events")), tables, false, false); + return SelectContext.create(new SqlIdColumns(DSL.field(DSL.name("events", "id"), String.class)), + Optional.empty(), new ConnectorSqlTables(null, plan), conversion); + } + + private static ColumnId column(String name, MajorTypeId type) { + Table table = new Table(); + table.setName("events"); + table.setDataset(new DatasetId("test")); + Column column = new Column(); + column.setName(name); + column.setType(type); + column.setTable(table); + // Replace only repository access; conversion and SQL construction remain real. + return new ColumnId(table.getId(), name) { + @Override + public Column get() { + return column; + } + }; + } + + private static String render(Field field) { + return DSL.using(SQLDialect.POSTGRES).renderInlined(DSL.select(field)).substring("select ".length()).toLowerCase(Locale.ROOT); + } +} diff --git a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionDialectTest.java b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionDialectTest.java new file mode 100644 index 0000000000..4ee70cab08 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionDialectTest.java @@ -0,0 +1,26 @@ +package com.bakdata.conquery.sql.conversion; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.Date; +import java.util.Locale; + +import com.bakdata.conquery.sql.conversion.dialect.clickhouse.ClickhouseDialectBundle; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Test; + +class SqlExtractionDialectTest { + + @Test + void shouldKeepClickhouseRangeBoundsNullableAfterCoalesce() { + var dialect = new ClickhouseDialectBundle(); + var range = dialect.dateRange(DSL.field(DSL.name("events", "start"), Date.class), + DSL.field(DSL.name("events", "end"), Date.class)); + // ClickHouse otherwise replaces missing outer-join values with the epoch date. + for (var bound : range.toFields()) { + String sql = DSL.using(SQLDialect.CLICKHOUSE).renderInlined(bound).toLowerCase(Locale.ROOT); + assertTrue(sql.contains("::nullable(date32)"), sql); + } + } +} diff --git a/docs/sql-connector-extraction-verification.md b/docs/sql-connector-extraction-verification.md new file mode 100644 index 0000000000..4ede866afc --- /dev/null +++ b/docs/sql-connector-extraction-verification.md @@ -0,0 +1,37 @@ +# SQL connector extraction verification + +## Reference revision + +Fetched `origin/develop` on 2026-09-21: `353097249dd1ec4b79a02931996191c9fe39660d`. +The local `develop` branch remains at `93b4134e01ad804997303ad253fc0c9d4d64ac75`; it has not been moved. +Use the fetched revision for the extraction-only portability checkout. + +The SQL sum, count, and duration aggregator implementations have no differences between these two revisions. Compatibility assertions must be checked against this recorded `develop` source, not merely against the migration branch. + +## Aggregation coverage checked so far + +| Operation | SQL behavior on the reference revision | Existing fixture or evidence | Extraction obligation | +| --- | --- | --- | --- | +| Ordinary sum with `subtractColumn` | Subtracts, treating a single missing operand as zero and preserving NULL when both are missing | `aggregator/SUM_DIFF_AGGREGATOR`, `filter/DIFFSUM_INTEGER`, `filter/DIFFSUM_REAL`; `SumSqlAggregator.createSumAggregationSelect` | Preserve subtraction and input numeric types | +| Sum with `distinctByColumn` | Partitions by entity IDs and distinct keys, selects row number 1, and sums the values | `aggregator/SUM_DISTINCT_AGGREGATOR`; `SumSqlAggregator.createDistinctSumAggregationSelect` | Preserve the separate predecessor CTEs and grouping | +| Sum with both options | The SQL distinct branch does not pass `subtractColumn` into its helper; it sums the original column | Both `connectorSelect` and `convertToSqlFilter` in `SumSqlAggregator` | Preserve this combined-option SQL quirk; do not remove either independently supported option | +| In-memory sum with both options | Wraps the subtracting aggregator in `DistinctValuesWrapperAggregator` | `SumSelect.createAggregator/getAggregator` and `SumFilter.createFilterNode/getAggregator` | Not the SQL compiler baseline; leave these implementations unchanged | +| Duration sum with `distinctBy` | SQL date-range aggregation does not consume distinct keys | `aggregator/DURATION_SUM_DISTINCT_AGGREGATOR/DURATION_SUM.test.json` explicitly disables SQL testing with comment `distinctBy not yet implemented.` | Preserve the existing SQL path; do not introduce a rejection in compilation | +| Count with configured distinct keys | SQL checks `isDistinct` and counts distinct values of the counted column; it does not read `distinctByColumn` | `CountSqlAggregator.createCountAggregationSelect`; `filter/COUNT_DISTINCT_MULTI` exists but has not been executed here | Preserve SQL behavior separately from in-memory distinct-key behavior | + +Fixture paths in the table are relative to `backend/src/test/resources/tests/` on the reference revision. Source inspection establishes intended compatibility targets; it is not evidence that database integration tests have passed. + +## Local evidence + +- Initial connector baseline: 157 tests passed. +- Initial backend package with tests skipped: passed, including test compilation. +- Sum regressions reproduced before correction: subtraction incorrectly included in distinct sums; DECIMAL expressions incorrectly typed as BigDecimal. +- After correction: 158 connector tests passed. +- Duration distinct-key rejection reproduced and removed to match the existing SQL path. +- Backend public-entry-point characterization: five aggregation tests and one dialect test passed after delegating sum, count, quarter count, and duration sum to the connector. The selected connector aggregation tests also passed (13 tests). +- The ClickHouse nullable date-bound regression failed before the dialect bridge and passed afterward; preserving `Nullable(Date32)` prevents missing outer-join dates from becoming epoch dates. +- Final connector run for the numeric aggregation extraction: 158 tests passed (`./mvnw.cmd -o -B -ntp -pl sql-connector -am test`). HANA/ClickHouse connections were not opened. + +## Pending verification + +Extraction is still in progress. The remaining operation/dialect coverage, full extraction-only `develop` build, and HANA/ClickHouse result-level verification are pending. Database integration suites cannot run on this host; a final handoff will identify the tested extraction revision, commands, prerequisites, and required reports. diff --git a/docs/superpowers/plans/2026-09-21-sql-connector-extraction.md b/docs/superpowers/plans/2026-09-21-sql-connector-extraction.md new file mode 100644 index 0000000000..6447c293c1 --- /dev/null +++ b/docs/superpowers/plans/2026-09-21-sql-connector-extraction.md @@ -0,0 +1,211 @@ +# SQL Connector Extraction Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Complete a recognizable, behavior-preserving SQL connector extraction that the Dropwizard backend can use on a branch based on `develop` without Quarkus. + +**Architecture:** Move existing SQL algorithms into `sql-connector`; retain backend model resolution, configuration, lifecycle, and application result/error adaptation in `backend`. Reuse the existing resolved model and compiler infrastructure. Separate pure SQL and JDBC mechanics from application ownership without rewriting algorithms. + +**Tech Stack:** Java 21, Maven wrapper, jOOQ 3.20.2, Jakarta Bean Validation with Hibernate Validator, JUnit Jupiter, existing HANA and ClickHouse integration suites. + +**Spec:** `docs/superpowers/specs/2026-09-21-sql-connector-extraction-design.md` + +## Global Constraints + +- Preserve existing SQL behavior, including quirks. +- Preserve recognizable class names, method structure, SQL expressions, and algorithm organization wherever the dependency boundary permits. +- Strict validation at the resolved-model boundary is an explicit exception. +- No backend, Dropwizard, or Quarkus implementation dependencies in the connector. +- No dependency upgrades or unrelated cleanup. +- Preserve the current working changes; inspect them rather than resetting them. +- Work on the current migration branch; use an isolated checkout only for the later `develop` portability check. +- Do not run HANA or ClickHouse integration suites here. Prepare an external test handoff. +- Stage only the files belonging to each extraction step. Every commit must include the user-required `Co-authored-by: Codex ` trailer after a blank line. + +## Review Focus + +1. Integer, floating-point, and money inputs must retain their existing jOOQ field types and null arithmetic (Task 1). +2. Select/filter combinations must not recompute a distinct or interval-packed aggregate after joining its predecessor (Tasks 1–2). +3. Reused queries, secondary IDs, and multiple connector branches must preserve aliases and result ordering (Task 4). +4. Table-export filters operate on individual rows; form date windows and feature/outcome columns retain their existing meaning (Task 5). +5. JDBC SQL NULL, empty lists, date bounds, exceptions, and early stream closure must preserve existing decoding and resource ownership (Task 6). + +## File map and sequencing + +Paths below are repository-relative. Execute tasks in order because dialect, operation, and orchestration contracts overlap. + +- Shared compiler code: `sql-connector/src/main/java/com/bakdata/conquery/sql/`. +- Existing SQL implementation and backend adapters: `backend/src/main/java/com/bakdata/conquery/sql/`. +- Connector tests mirror the main package under `sql-connector/src/test/java/`. +- New backend compatibility tests live under `backend/src/test/java/com/bakdata/conquery/sql/` and use the `SqlExtraction*Test` prefix, allowing a safe explicit local test selection. +- Integration fixtures remain under `backend/src/test/resources/tests/`; retain expected outputs. +- `dataset-model` receives only shared vocabulary required by these moves. Do not relocate application domain models to avoid defining an adapter. +- Preserve existing public adapter signatures during each intermediate commit so the Dropwizard backend keeps compiling. + +## Local verification commands + +Run from the repository root in PowerShell. Inspect exit codes and Surefire reports; a zero-test run is not verification. + +```powershell +.\mvnw.cmd -B -ntp -pl sql-connector -am test +.\mvnw.cmd -B -ntp -pl backend -am '-DskipTests' package +.\mvnw.cmd -B -ntp -pl backend -am '-Dtest=SqlExtraction*Test' '-Dsurefire.failIfNoSpecifiedTests=false' test +git diff --check +``` + +The backend package command compiles tests but does not execute them. The explicit test selector executes only new local compatibility tests; do not replace it with an unrestricted backend test run. Run the third command only after those tests exist. Record any pre-existing build failures separately before changing code. + +### Task 1: Complete aggregation extraction with compatibility coverage + +**Files:** +- Modify the pending `compiler/conversion/operation/{ResolvedAggregationConverter,SumAggregationConverter,DateAggregationConverters,BuiltInAggregationConverters,BuiltInAggregationFilterConverters,AggregationConversionContext}.java` in the connector. +- Modify `compiler/conversion/operation/ResolvedFilterConverter.java` and its tests. +- Modify `compiler/conversion/operation/ResolvedAggregationConverterTest.java` in connector tests. +- Adapt backend `conversion/model/aggregator/{SumSqlAggregator,CountSqlAggregator,CountQuartersSqlAggregator,DurationSumSqlAggregator,DateDistanceSqlAggregator,FlagSqlAggregator}.java` as each operation moves. +- Create backend test `sql/conversion/SqlExtractionAggregationTest.java`. +- Update `sql-connector/README.md` to remove claims of intended behavior changes. + +**Interfaces:** Retain `ResolvedAggregationConverter.convert(ResolvedAggregation, AggregationConversionContext): CommonAggregationSelect`. Backend adapters resolve columns and construct operations; shared converters produce the existing aggregation IR. Preserve the additional-predecessor contract in `CommonAggregationSelect`. + +- [ ] Run the connector baseline and backend compilation commands. Record existing failures without attributing them to this task. +- [ ] Read each original aggregator alongside the pending implementation. Record numeric field classes from `NumberMapUtil`, projection ordering, date conversion, and table-export behavior before replacing any backend method. +- [ ] Replace the pending test that expects subtraction inside distinct sums with this characterization, using the existing test class's constants and helpers: + +```java +@Test +void shouldIgnoreSubtractionInsideDistinctSumLikeExistingBackend() { + var withoutSubtract = converter.convert( + new BuiltInAggregations.Sum(AMOUNT, Optional.empty(), List.of(PERSON)), CONTEXT); + var withSubtract = converter.convert( + new BuiltInAggregations.Sum(AMOUNT, Optional.of(DISCOUNT), List.of(PERSON)), CONTEXT); + assertEquals(renderRootSelects(withoutSubtract), renderRootSelects(withSubtract)); + assertEquals(render(withoutSubtract.getGroupBy().select()), render(withSubtract.getGroupBy().select())); +} +``` + +- [ ] Run `ResolvedAggregationConverterTest` via Maven with `-pl sql-connector -am`, `-Dtest=ResolvedAggregationConverterTest`, and `-Dsurefire.failIfNoSpecifiedTests=false`; confirm the new test fails because subtraction is included. +- [ ] Extract the original distinct-sum helper structure: project the sum and distinct-key columns, assign row numbers by IDs/keys, filter row number 1, and sum the original value with its original coalesce expression. Do not pass the subtraction column into this branch. +- [ ] Add compatibility cases for ordinary sum with zero/one/two null operands, INTEGER/REAL/DECIMAL/MONEY source types, count with multiple distinct keys, single-date versus date-pair quarter counts, and packed/unbounded durations. Compare rendered expressions and predecessor structure with expectations derived from the original backend implementation. +- [ ] Inspect the original duration `distinctBy` path. Preserve its SQL behavior for valid inputs; express any unsupported resolved-model combination as a declarative boundary constraint with a validation test instead of leaving an incidental exception deep in compilation. Document the chosen boundary contract in the README. +- [ ] Move remaining aggregation SQL with the original helper names where practical; keep column resolution and filter/select DTO adaptation in the backend. Verify that an additional predecessor supplies an already-computed aggregate and is not aggregated again. +- [ ] Run connector tests, the backend aggregation compatibility test, and backend compilation. Review the diff for unintended expression/type changes, then commit this task alone. + +### Task 2: Extract connector and concept selects + +**Files:** +- Move SQL bodies from backend `conversion/model/select/{ValueSelectUtil,RandomValueSelectConverter,LastValueSelectConverter,FirstValueSelectConverter,ExistsSelectConverter,EventDurationSumSelectConverter,EventDateUnionSelectConverter,DistinctSelectConverter,DateUnionSelectConverter,DaterangeSelectUtil,ConceptColumnSelectConverter}.java` into the connector's `compiler/conversion/operation/` package, preserving class/helper names where responsibilities match. +- Create connector `compiler/conversion/operation/ResolvedSelectConverter.java` and `SelectConversionContext.java`. +- Test in connector `compiler/conversion/operation/ResolvedSelectConverterTest.java` and backend `sql/conversion/SqlExtractionSelectTest.java`. + +**Interfaces:** `ResolvedSelectConverter.convert(ResolvedSelect, SelectConversionContext): ConnectorSqlSelects` for connector-level outputs. Keep concept-level selection composition separate where the existing backend uses a different result type; reuse its existing IR rather than forcing both paths into one shape. Reuse Task 1's aggregation converter. `SelectConversionContext` carries the resolved table/ID state, compiler dialect, name generator, and alias currently read by these select helpers, without backend DTOs. + +- [ ] Characterize every built-in select represented by `BuiltInSelects`: aggregation, values (distinct/first/last/random), date union/distance, concept values, event date union/duration, and exists. Pin each operation's SQL, required input columns, and final projection order before moving it. +- [ ] For first/last/random, retain existing ordering and tie behavior. For date distance, use the producer-frozen end date; do not introduce a compiler clock. For substring selection, preserve index/range conventions. +- [ ] Add failing cases for missing dispatch registrations and mixed aggregation/value selects with an additional predecessor. Use fixed aliases and fixed dates to make SQL comparisons deterministic; do not assert a particular database result for inherently random/tied choices. +- [ ] Move the original SQL expressions and helper bodies, replacing backend column resolution with `ResolvedColumn` inputs. Backend select adapters delegate to the moved code. +- [ ] Run connector select tests, the backend select compatibility test, and backend compilation. Commit only the select extraction and associated coverage. + +### Task 3: Move dialect SQL capabilities + +**Files:** +- Backend sources: `conversion/dialect/{SqlFunctionProvider,LegacyCompilerDialect,DialectBundle}.java`, `conversion/dialect/hana/{HanaSqlFunctionProvider,HanaStratificationFunctions,HanaDialectBundle}.java`, and `conversion/dialect/clickhouse/{ClickhouseFunctionProvider,ClickhouseStratificationFunctions,ClickhouseDistinctSelectConverter,ClickhouseDialectBundle}.java`. +- Move reusable providers and stratification functions into connector `compiler/dialect/`, retaining vendor subpackages and existing class names. +- Modify connector `compiler/dialect/CompilerDialect.java`. +- Create backend test `sql/conversion/SqlExtractionDialectTest.java`. + +**Interfaces:** `CompilerDialect` is the shared capability boundary. `DialectBundle` remains backend composition for configuration, datasource checks, result processors, and application type compatibility. Shared dialects must not register converters accepting backend `Visitable` or `Select` types. + +- [ ] Capture HANA/ClickHouse rendering expectations for date min/max, range construction, date arithmetic, quarters, arrays, concatenation, distinct selects, literal/no-op tables, and stratification expressions without opening a connection. +- [ ] Move provider methods with unchanged jOOQ expressions; adapt `CDateRange` and application enums at the backend boundary. Preserve vendor-specific native date-range behavior; do not substitute a generic dual-column implementation merely because the neutral model makes it easier. +- [ ] Remove backend references from the moved classes and wire backend dialect bundles to delegate. Avoid adding default implementations that silently substitute unsupported vendor behavior. +- [ ] Run rendering compatibility tests and backend compilation. Commit dialect moves separately from algorithm changes. + +### Task 4: Connect resolved concept queries to the shared compiler + +**Files:** +- Create connector `compiler/DefaultSqlCompiler.java` implementing existing `SqlCompiler`. +- Modify connector `compiler/conversion/` and `compiler/ir/concept/` to connect existing condition, filter, select, and CTE components. +- Create backend `conversion/ResolvedQueryAdapter.java`. +- Adapt backend `conversion/{SqlConverter,NodeConversions}.java`, `conversion/cqelement/` converters, `conversion/query/{ConceptQueryConverter,CQReusedQueryConverter,SecondaryIdQueryConverter}.java`, and `conversion/model/SqlQuery.java`. +- Create connector test `compiler/DefaultSqlCompilerTest.java` and backend test `sql/conversion/SqlExtractionQueryTest.java`. + +**Interfaces:** Preserve `SqlCompiler.compile(ResolvedQuery, CompilerDialect): CompiledQuery`. The backend adapter produces the existing `ResolvedQuery` from the already-resolved application query/context. Invoke `ResolvedQueryValidation.validate(ResolvedQuery)` once before compilation using the application's validator. Map ordered `CompiledColumn` metadata back to existing `ResultInfo` at the backend boundary. + +- [ ] Build local compatibility fixtures for all-entities, AND/OR/negation, external IDs, date restrictions, concepts with multiple connector branches, reused queries, and secondary IDs. Include repeated names, empty optional selections, and validity-date output. +- [ ] Assert ordered output IDs/types/aliases as well as rendered SQL. Add a validation-boundary test proving invalid nested operations are rejected before compiler invocation. +- [ ] Move traversal and composition into the connector using the existing logical/external/final/concept CTE helpers. Preserve the current name-generator lifetime and alias rules rather than creating a new naming convention. +- [ ] Implement backend adaptation: expand saved references, resolve physical columns and IDs, materialize defaults/date actions, freeze dates, and retain presentation metadata outside compiler internals. Do not query repositories during compilation. +- [ ] Route concept execution through the shared compiler. Retain backend wrappers needed by forms/export until Task 5 replaces their remaining SQL bodies; do not maintain a second complete concept compiler. +- [ ] Run connector tests and the explicit backend compatibility suite; compile the backend and commit this integration. + +### Task 5: Extract form and export SQL without changing their contracts + +**Files:** +- Backend sources: `conversion/query/{AbsoluteFormQueryConverter,RelativFormQueryConverter,FormConversionHelper,EntityDateQueryConverter,TableExportQueryConverter}.java` and `conversion/forms/`. +- Move pure form SQL helpers into connector `compiler/forms/` and export SQL helpers into `compiler/conversion/query/`, keeping the existing helper/class names. +- Add resolved input records adjacent to those helpers for the values currently read from application form/export DTOs; reuse `DateRange`, resolved columns, existing query-step IR, and moved form vocabulary. +- Create backend `sql/conversion/SqlExtractionFormTest.java` and `SqlExtractionExportTest.java`. + +**Interfaces:** Keep `ResolvedQuery` and `SqlCompiler` as the concept-query contract. Do not widen `ResolvedQuery` with unrelated optional form/export fields. Form helpers consume resolved dates, stratification settings, feature steps, and dialect functions; export helpers consume resolved columns and row predicates. Existing application converter entry points remain backend adapters. + +- [ ] Pin original absolute/relative form SQL and output order for empty windows, boundary dates, complete resolution, and feature/outcome scopes. Include two feature groups and secondary IDs to expose join/alias collisions. +- [ ] Pin table-export filter expressions separately from aggregation filters, including sum subtraction with null operands, duration distance, select filters, and date restrictions. Expected expressions come from the existing `convertForTableExport` methods. +- [ ] Move the original stratification, form join, and projection helper bodies. Replace application enum/config lookups with explicitly resolved values; keep the original algorithm and helper decomposition. +- [ ] Move row-level export expressions and route backend adapters to them. Do not reuse grouped filters where existing table exports use plain arithmetic or predicates. +- [ ] Run both compatibility classes, connector tests, and backend compilation. Commit form and export moves in separate commits when their diffs are independent. + +### Task 6: Extract reusable JDBC and result decoding mechanics + +**Files:** +- Backend sources: `execution/{SqlExecutionService,ResultSetProcessor,DefaultResultSetProcessor,SqlCDateSetParser,DefaultCDateSetParser,HanaSqlCDateSetParser,SqlEntityResult,SqlExecutionExecutionInfo}.java` and `conversion/dialect/clickhouse/ClickhouseResultSetProcessor.java`. +- Move reusable processor/parser/JDBC bodies into connector `execution/`; preserve existing names where a backend wrapper is unnecessary. +- Adapt backend `conquery/SqlExecutionManager.java` and dialect bundle wiring. +- Create backend test `sql/execution/SqlExtractionExecutionTest.java` and connector tests beside the moved execution classes. + +**Interfaces:** Keep application `ResultInfo`, `EntityResult`, `ExecutionState`, and `ConqueryError` in backend adapters. Retain the JDBC reader contract `T read(ResultSet, int) throws SQLException`. Shared execution consumes SQL and ordered readers and exposes raw row values; backend code owns conversion into application results and errors. Retain existing stream ownership and close behavior. + +- [ ] Characterize JDBC decoding using controlled `ResultSet`/connection doubles: SQL NULL versus zero, empty versus null lists, money precision, date sentinels, column offsets, and multiple result columns. +- [ ] Characterize statement/result-set closure on success, decoding failure, and SQL exceptions. Cover closing an execution stream before exhaustion. Preserve existing ownership of caller-supplied connections. +- [ ] Move decoding and execution loops without changing conversion expressions or resource scopes. Keep backend `readerForType` dispatch if it depends on application `ResultType`; delegate to shared reader methods rather than importing the application type into the module. +- [ ] Preserve backend error wrapping and execution state transitions. Run local execution tests, all connector tests, and backend compilation. Commit the extraction. + +### Task 7: Remove obsolete SQL implementations and verify the dependency boundary + +**Files:** Backend adapters touched in Tasks 1–6, `sql-connector/README.md`, `sql-connector/pom.xml`, `dataset-model/pom.xml`, `backend/pom.xml`, and root `pom.xml` only where necessary. + +- [ ] Search backend SQL classes for remaining jOOQ construction. Classify each remaining expression as application adaptation or reusable SQL; move remaining reusable bodies before declaring extraction complete. +- [ ] Remove superseded implementations after checking callers. Preserve public backend configuration and API schemas. +- [ ] Inspect all connector imports and Maven dependency trees for backend/framework dependencies. Check the resolved graph, not just direct POM dependencies: + +```powershell +.\mvnw.cmd -B -ntp -pl sql-connector -am dependency:tree +rg -n '^import .*?(quarkus|dropwizard|conquery\.(apiv1|models\.config|models\.identifiable|models\.query))' sql-connector/src/main/java +``` + +- [ ] Document each moved/split class and its backend adapter in the connector README. Document strict boundary validation and preserved SQL quirks, including distinct-sum subtraction. +- [ ] Run all local verification commands. Review the extraction diff for unrelated changes and Quarkus edits; commit cleanup and documentation. + +### Task 8: Verify extraction portability and prepare external tests + +**Files:** Create `docs/sql-connector-extraction-verification.md`; edit POM module lists only in the isolated extraction checkout as required to apply extraction-only changes. + +- [ ] Record `git rev-parse develop`, the migration branch revision, working-tree status, and extraction commit list. Never reset or rebase the migration checkout. +- [ ] Use the worktree skill to create an isolated checkout from the recorded `develop` revision. Apply the extraction commits/patches, including earlier extraction work already present before this plan. Inspect shared-file hunks so Quarkus module entries and migration-only dependencies are omitted. +- [ ] Resolve portability conflicts using the unchanged Dropwizard behavior as the reference. Record any prerequisite neutral changes explicitly. Do not copy the complete migration root POM. +- [ ] Confirm no Quarkus sources/modules/configuration are present. Run the connector suite, explicit backend compatibility suite, and backend package command from this checkout. Then package the existing Dropwizard reactor with tests skipped to catch downstream compilation failures. +- [ ] Write the exact verified local commands, revisions, results, and omitted integration checks in the verification document. Include reports under each module's `target/surefire-reports`. +- [ ] Prepare these external Linux commands for the extraction revision, after verifying the test selectors still match the repository. HANA container mode requires a suitable Linux/Docker host and the suite's `/tmp/data/hana` mount; ClickHouse requires Docker. Use the repository's existing provider configuration for remote database mode, and document required environment-variable names without recording credentials. + +```bash +USE_LOCAL_HANA_DB=true ./mvnw -B -ntp -pl backend -am \ + -Dtest=HanaSqlIntegrationTests -Dsurefire.failIfNoSpecifiedTests=false test +USE_LOCAL_CH_DB=true ./mvnw -B -ntp -pl backend -am \ + -Dtest=ClickhouseSqlIntegrationTests -Dsurefire.failIfNoSpecifiedTests=false test +``` + +- [ ] Ask the user to return the exact tested revision, command exit statuses, and Surefire reports from the other machine. Keep result-level compatibility marked pending until those results arrive; do not replace these databases with an in-memory database and claim parity. +- [ ] Commit the verification/handoff document. Report changes in logical review order, local verification evidence, extraction portability evidence, and pending external checks. + +## Plan self-review + +The tasks cover the spec's operations, dialects, orchestration, execution, strict validation, recognizable code, and `develop` portability. Review Focus cases are assigned to Tasks 1–6. The plan intentionally preserves the existing concept compiler signature and treats form/export helpers as separate contracts. External database verification remains a separate, explicit acceptance item; implementation can proceed without those databases locally. diff --git a/docs/superpowers/specs/2026-09-21-sql-connector-extraction-design.md b/docs/superpowers/specs/2026-09-21-sql-connector-extraction-design.md index 35da95aedf..87a3183617 100644 --- a/docs/superpowers/specs/2026-09-21-sql-connector-extraction-design.md +++ b/docs/superpowers/specs/2026-09-21-sql-connector-extraction-design.md @@ -4,13 +4,13 @@ Finish extracting the existing SQL connector so the productive Dropwizard backend uses a framework-neutral module. The extraction must be transferable onto a separate branch based on `develop`, without Quarkus sources, configuration, plugins, or build prerequisites. Quarkus integration is subsequent work. -Preserve existing behavior, including quirks. Preserve recognizable class names, method structure, SQL expressions, and algorithm organization wherever the dependency boundary permits. Do not combine extraction with bug fixes, new validation rules, dependency upgrades, or general cleanup. +Preserve existing SQL behavior, including quirks. Preserve recognizable class names, method structure, SQL expressions, and algorithm organization wherever the dependency boundary permits. Do not combine extraction with bug fixes, dependency upgrades, or general cleanup. Strict validation at the resolved-model boundary is an explicit exception: retain and extend the readable, declarative Jakarta Bean Validation constraints, using Hibernate Validator as the validation provider. ## Current state The branch already contains `dataset-model`, resolved SQL input types, compiler contracts, query-step infrastructure, CTE composition, and condition/filter converters. The backend consumes extracted infrastructure, but `SqlCompiler` is only an interface: backend query conversion, aggregators, selects, forms, dialect implementations, and execution still contain substantive SQL responsibilities. -Uncommitted aggregation conversion work is present and must be reviewed as part of the extraction. In particular, its distinct-sum path applies subtraction where the existing implementation ignores it. It also explicitly rejects duration `distinctBy`; compatibility must be checked against the existing path before accepting that rejection. Neither a cleaner implementation nor an existing new test establishes compatibility. +Uncommitted aggregation conversion work is unfinished. Resume by reviewing and completing it; do not treat its current behavior or tests as an approved design. In particular, its distinct-sum path applies subtraction where the existing implementation ignores it. It also explicitly rejects duration `distinctBy`; determine whether this belongs in the strict boundary contract or must retain the existing behavior. Neither a cleaner implementation nor an existing new test establishes SQL compatibility. ## Approach @@ -34,17 +34,17 @@ Preserve SQL operations for concept queries, logical nodes, external IDs, date r Preserve numeric types, null handling, range bounds, distinctness, subtraction, quarter counting, interval packing, aliases, projection order, and result interpretation. Table-export operations must retain their row-level semantics rather than accidentally adopting grouped-query semantics. -New resolved-model validation must not reject a previously supported request or change when application errors are reported. Backend adapters must preserve defaults and resolution semantics. Known quirks receive characterization coverage rather than fixes in this change. +Validate the complete resolved query graph at the boundary before compilation, using the existing `ResolvedQueryValidation` and declarative Jakarta Bean Validation constraints with Hibernate Validator. This boundary may strictly reject invalid resolved inputs, even if the previous path accepted them or failed later. Keep constraints readable and explicit; validation must not resolve identifiers, enrich data, or rewrite operations. Backend adapters must preserve defaults and resolution semantics and produce inputs conforming to this contract. For valid inputs, preserve existing SQL behavior; known SQL quirks receive characterization coverage rather than fixes in this change. Remove superseded SQL implementations only after the production backend delegates to the extracted implementation and compatibility checks pass. Completion must not leave two independently maintained implementations of the same SQL algorithm. ## Verification 1. Establish the existing test baseline and characterize behavior-sensitive paths before replacing their implementation. Compare extracted output with the existing implementation, using equivalent resolved inputs and dialect settings. -2. Run connector tests and affected backend tests. Exercise aggregations, selects, forms, and export paths; include nulls, duplicate events, distinct sum with subtraction, date boundaries, and duration distinctness. -3. Use existing SQL integration fixtures for result-level verification on supported databases where infrastructure is available. SQL text checks alone do not establish result parity. Record unavailable integration infrastructure explicitly. +2. Run connector tests and affected backend tests that do not require database instances. Exercise aggregations, selects, forms, and export paths; include nulls, duplicate events, distinct sum with subtraction, date boundaries, and duration distinctness. Test boundary constraint violations separately from SQL compatibility for valid inputs. +3. HANA and ClickHouse integration suites cannot run in this environment. Prepare a reproducible handoff for the user to run the existing SQL integration fixtures on a suitable machine: identify the exact extraction revision, prerequisites, commands, and expected reports. Local SQL rendering/comparison and unit tests provide partial evidence, not a substitute for result-level verification on those databases. Track external results separately and investigate failures when returned; lack of local database infrastructure does not block implementation or local verification. 4. Verify module dependencies and source imports contain no backend or framework implementation dependencies in the connector. -5. Assemble the extraction in an isolated checkout based on the recorded local `develop` revision. Apply only extraction changes and required neutral module/build additions. Run the module and Dropwizard build/test checks with no Quarkus modules present. Do not infer portability solely from a selected-module build on the migration branch. +5. Assemble the extraction in an isolated checkout based on the recorded local `develop` revision. Apply only extraction changes and required neutral module/build additions. Run the module and Dropwizard build/test checks that do not require database instances, with no Quarkus modules present. Include this extraction-only revision in the external integration-test handoff. Do not infer portability solely from a selected-module build on the migration branch. ## Review and portability @@ -54,4 +54,4 @@ Keep Quarkus changes out of extraction commits. Shared-file edits, especially th ## Completion criteria -The Dropwizard backend uses the extracted SQL implementation for its supported paths; application-specific adapters are the remaining backend boundary. Compatibility checks pass without intended behavior changes, code remains recognizable, and an extraction-only checkout based on `develop` builds without knowledge of Quarkus. Any unverified database checks are reported as limitations rather than claimed successes. +The Dropwizard backend uses the extracted SQL implementation for its supported paths; application-specific adapters are the remaining backend boundary. Local compatibility and boundary-validation checks pass, SQL behavior for valid inputs remains unchanged, code remains recognizable, and an extraction-only checkout based on `develop` builds without knowledge of Quarkus. Local delivery includes the external integration-test handoff and explicitly identifies HANA/ClickHouse verification as pending until results are available. Full database compatibility is not claimed before those external checks pass. diff --git a/sql-connector/README.md b/sql-connector/README.md index 62d49ee2eb..0138d98119 100644 --- a/sql-connector/README.md +++ b/sql-connector/README.md @@ -42,6 +42,27 @@ The `BuiltInFilters`, `BuiltInSelects`, `BuiltInConditions`, and `BuiltInAggrega operations understood by the standard compiler. They intentionally describe semantics rather than the configuration classes used by a particular query-producing application. +## Aggregation compilation + +`ResolvedAggregationConverter` compiles count, sum, quarter-count, and duration-sum operations. It accepts additional +converters for extension aggregation types. `ResolvedFilterConverter` uses this dispatcher for numeric +`BuiltInFilters.AggregationRange` filters, including custom numeric aggregations. + +Ordinary aggregates are projected in the connector's aggregation CTE. Distinct sums and interval-packed duration sums +are computed in separate predecessor CTEs; the range predicate reads their result after the branches are joined. +An aggregation returned with an `additionalPredecessor` must already be computed in that predecessor. + +Sum subtraction preserves missing values when both operands are null. Distinct sums preserve the existing behavior and +ignore the subtraction column. Numeric input expressions retain their existing integer, floating-point, or money types. +Quarter counts preserve the legacy behavior: distinct +year-quarter keys for single dates, and a sum of each event's quarter count for date pairs, including overlaps. +Duration sums pack overlapping intervals first and exclude unbounded durations. Duration `distinctBy` is ignored, +as in the existing backend SQL implementation. Physical date-range columns require dialect support for +extracting half-open bounds through `CompilerDialect.dateRangeColumn`. + +Flag aggregation for output selects and integration of resolved operations into the legacy backend remain separate +migration steps. Flag filters already compile as event predicates. + ## Validation The resolved model uses Jakarta Bean Validation annotations. The application supplies a validation provider, normally diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/AggregationConversionContext.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/AggregationConversionContext.java new file mode 100644 index 0000000000..0cc4e33b30 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/AggregationConversionContext.java @@ -0,0 +1,29 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import java.util.Objects; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; + +/** Compiler state available while converting a resolved aggregation into connector IR. */ +public record AggregationConversionContext( + CompilerDialect dialect, + SqlNameGenerator nameGenerator, + SqlTables tables, + SqlIdColumns ids, + String alias +) { + + public AggregationConversionContext { + Objects.requireNonNull(dialect, "dialect"); + Objects.requireNonNull(nameGenerator, "nameGenerator"); + Objects.requireNonNull(tables, "tables"); + Objects.requireNonNull(ids, "ids"); + Objects.requireNonNull(alias, "alias"); + if (alias.isBlank()) { + throw new IllegalArgumentException("alias must not be blank"); + } + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInAggregationConverters.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInAggregationConverters.java new file mode 100644 index 0000000000..6ef41116b0 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInAggregationConverters.java @@ -0,0 +1,91 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import java.util.List; +import java.util.function.BiFunction; +import java.util.stream.Stream; + +import com.bakdata.conquery.sql.compiler.conversion.Converter; +import com.bakdata.conquery.sql.compiler.ir.concept.CommonAggregationSelect; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; +import com.bakdata.conquery.sql.model.operation.BuiltInAggregations; +import com.bakdata.conquery.sql.model.operation.ResolvedAggregation; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import org.jooq.Field; +import org.jooq.impl.DSL; + +final class BuiltInAggregationConverters { + + private BuiltInAggregationConverters() { + } + + static List, AggregationConversionContext>> create() { + return List.of( + converter(BuiltInAggregations.Count.class, BuiltInAggregationConverters::convertCount), + converter(BuiltInAggregations.Sum.class, SumAggregationConverter::convert), + converter(BuiltInAggregations.CountQuarters.class, DateAggregationConverters::countQuarters), + converter(BuiltInAggregations.DurationSum.class, DateAggregationConverters::durationSum) + ); + } + + // TODO Why is this not in a separate file? + private static CommonAggregationSelect convertCount( + BuiltInAggregations.Count aggregation, + AggregationConversionContext context + ) { + ExtractingSqlSelect countSelect = rootSelect(aggregation.column(), context); + List> distinctSelects = aggregation.distinctBy().stream() + .map(column -> rootSelect(column, context)) + .toList(); + List rootSelects = Stream.concat(Stream.of(countSelect), distinctSelects.stream()) + .distinct() + .map(SingleColumnSqlSelect.class::cast) + .toList(); + + String aggregationSource = context.tables().getPredecessor(ConceptCteStep.AGGREGATION_SELECT); + Field count = distinctSelects.isEmpty() + ? DSL.count(countSelect.qualify(aggregationSource).select()) + : DSL.countDistinct(distinctSelects.stream() + .map(select -> select.qualify(aggregationSource).select()) + .toArray(Field[]::new)); + String[] requiredColumns = rootSelects.stream() + .flatMap(select -> select.requiredColumns().stream()) + .distinct() + .toArray(String[]::new); + FieldWrapper groupedCount = new FieldWrapper<>( + DSL.nullif(count, 0).as(context.alias()), + requiredColumns + ); + + return CommonAggregationSelect.builder() + .rootSelects(rootSelects) + .groupBy(groupedCount) + .build(); + } + + private static ExtractingSqlSelect rootSelect( + ResolvedColumn column, + AggregationConversionContext context + ) { + return new ExtractingSqlSelect<>(context.tables().getRootTable(), column.physicalName(), Object.class); + } + + private static Converter, AggregationConversionContext> converter( + Class type, + BiFunction> conversion + ) { + return new Converter<>() { + @Override + public Class getConversionClass() { + return type; + } + + @Override + public CommonAggregationSelect convert(I input, AggregationConversionContext context) { + return conversion.apply(input, context); + } + }; + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInAggregationFilterConverters.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInAggregationFilterConverters.java new file mode 100644 index 0000000000..beca035def --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInAggregationFilterConverters.java @@ -0,0 +1,76 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import java.math.BigDecimal; +import java.util.List; + +import com.bakdata.conquery.sql.compiler.conversion.Converter; +import com.bakdata.conquery.sql.compiler.ir.concept.CommonAggregationSelect; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; +import com.bakdata.conquery.sql.compiler.ir.condition.InclusiveRangeCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; +import com.bakdata.conquery.sql.model.operation.BuiltInFilters; +import com.bakdata.conquery.sql.model.operation.ResolvedFilter; +import org.jooq.Field; + +final class BuiltInAggregationFilterConverters { + + private BuiltInAggregationFilterConverters() { + } + + static List> create( + ResolvedAggregationConverter aggregationConverter + ) { + return List.of(new Converter() { + @Override + public Class getConversionClass() { + return BuiltInFilters.AggregationRange.class; + } + + @Override + public SqlFilters convert(BuiltInFilters.AggregationRange filter, FilterConversionContext context) { + return convertAggregationRange(filter, context, aggregationConverter); + } + }); + } + + // TODO Why is this not in a separate file? + private static SqlFilters convertAggregationRange( + BuiltInFilters.AggregationRange filter, + FilterConversionContext context, + ResolvedAggregationConverter aggregationConverter + ) { + String alias = context.nameGenerator().filterName(filter); + AggregationConversionContext aggregationContext = new AggregationConversionContext( + context.dialect(), + context.nameGenerator(), + context.tables(), + context.ids(), + alias + ); + CommonAggregationSelect aggregation = aggregationConverter.convert(filter.aggregation(), aggregationContext); + if (!Number.class.isAssignableFrom(aggregation.getGroupBy().select().getType())) { + throw new IllegalArgumentException("Aggregation range filters require a numeric aggregation result"); + } + + String groupFilterSource = context.tables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER); + Field aggregationField = aggregation.getGroupBy() + .qualify(groupFilterSource) + .select() + .coerce(BigDecimal.class); + ConnectorSqlSelects.ConnectorSqlSelectsBuilder selects = ConnectorSqlSelects.builder() + .preprocessingSelects(aggregation.getRootSelects()) + .additionalPredecessor(aggregation.getAdditionalPredecessor()); + if (aggregation.getAdditionalPredecessor().isEmpty()) { + selects.aggregationSelect(aggregation.getGroupBy()); + } + + return new SqlFilters( + selects.build(), + WhereClauses.builder() + .groupFilter(new InclusiveRangeCondition<>(aggregationField, filter.range())) + .build() + ); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/DateAggregationConverters.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/DateAggregationConverters.java new file mode 100644 index 0000000000..0ad846dae0 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/DateAggregationConverters.java @@ -0,0 +1,115 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import java.math.BigDecimal; +import java.sql.Date; +import java.time.temporal.ChronoUnit; +import java.util.List; + +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.concept.CommonAggregationSelect; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingSelectCompiler; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingSelectPreparation; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; +import com.bakdata.conquery.sql.model.operation.BuiltInAggregations; +import com.bakdata.conquery.sql.model.schema.DateColumns; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import lombok.experimental.UtilityClass; +import org.jooq.Field; +import org.jooq.impl.DSL; + +/** Date aggregations retain the legacy quarter-count and interval-packing semantics. */ +@UtilityClass +final class DateAggregationConverters { + + static CommonAggregationSelect countQuarters(BuiltInAggregations.CountQuarters aggregation, AggregationConversionContext context) { + CompilerDialect dialect = context.dialect(); + String source = context.tables().getPredecessor(ConceptCteStep.AGGREGATION_SELECT); + if (aggregation.dates() instanceof DateColumns.Single single && single.column().type() == ColumnType.DATE) { + ExtractingSqlSelect root = new ExtractingSqlSelect<>(context.tables().getRootTable(), single.column().physicalName(), Date.class); + Field count = DSL.nullif(DSL.countDistinct(dialect.yearQuarter(root.qualify(source).select())), 0); + return CommonAggregationSelect.builder() + .rootSelect(root) + .groupBy(new FieldWrapper<>(count.as(context.alias()), single.column().physicalName())) + .build(); + } + + ColumnDateRange inclusiveDates = inclusiveDates(aggregation.dates(), context); + Field firstQuarter = dialect.quarterStart(inclusiveDates.getStart()); + Field afterLastQuarter = dialect.nextQuarterStart(inclusiveDates.getEnd()); + Field quarters = dialect.dateDistance(ChronoUnit.MONTHS, firstQuarter, afterLastQuarter).divide(3); + FieldWrapper root = new FieldWrapper<>(quarters.as(context.alias()), requiredColumns(aggregation.dates())); + // Deliberately preserve the legacy sum of quarters per event, including overlapping events. + Field count = DSL.nullif(DSL.sum(root.qualify(source).select()), BigDecimal.ZERO); + return CommonAggregationSelect.builder() + .rootSelect(root) + .groupBy(new FieldWrapper<>(count.as(context.alias()), context.alias())) + .build(); + } + + static CommonAggregationSelect durationSum(BuiltInAggregations.DurationSum aggregation, AggregationConversionContext context) { + // Preserve the existing SQL path: intervals are packed without applying distinctBy. + CompilerDialect dialect = context.dialect(); + ColumnDateRange dates = dateRange(aggregation.dates(), context).as(context.alias()); + List roots = dates.toFields().stream() + .map(field -> new FieldWrapper<>(field, requiredColumns(aggregation.dates()))) + .toList(); + IntervalPackingSelectPreparation preparation = IntervalPackingSelectCompiler.prepareArbitrarySelect( + context.alias(), + context.tables().getPredecessor(ConceptCteStep.AGGREGATION_SELECT), + context.ids(), + dates, + dialect, + context.nameGenerator() + ); + ColumnDateRange packed = preparation.dateRange(); + Field days = dialect.dateDistance(ChronoUnit.DAYS, packed.getStart(), packed.getEnd()); + FieldWrapper sum = new FieldWrapper<>(DSL.sum( + DSL.when(packed.getStart().eq(dialect.minimumDate()).or(packed.getEnd().eq(dialect.maximumDate())), DSL.inline(null, Integer.class)) + .otherwise(days) + ).as(context.alias())); + QueryStep predecessor = IntervalPackingSelectCompiler.compileArbitrarySelect( + preparation.predecessor(), packed, sum, preparation.tables(), dialect + ); + return CommonAggregationSelect.builder() + .rootSelects(roots) + .groupBy(sum) + .additionalPredecessor(predecessor) + .build(); + } + + private static ColumnDateRange dateRange(DateColumns dates, AggregationConversionContext context) { + if (dates instanceof DateColumns.Single single && single.column().type() == ColumnType.DATE_RANGE) { + return context.dialect().dateRangeColumn(DSL.field(DSL.name(context.tables().getRootTable(), single.column().physicalName()))); + } + ColumnDateRange inclusive = inclusiveDates(dates, context); + return context.dialect().dateRange(inclusive.getStart(), inclusive.getEnd()); + } + + private static ColumnDateRange inclusiveDates(DateColumns dates, AggregationConversionContext context) { + return switch (dates) { + case DateColumns.Pair pair -> ColumnDateRange.of(dateField(pair.start(), context), dateField(pair.end(), context)); + case DateColumns.Single single -> { + if (single.column().type() == ColumnType.DATE_RANGE) { + ColumnDateRange range = dateRange(dates, context); + yield ColumnDateRange.of(range.getStart(), context.dialect().addDays(range.getEnd(), DSL.inline(-1))); + } + Field date = dateField(single.column(), context); + yield ColumnDateRange.of(date, date); + } + }; + } + + private static Field dateField(ResolvedColumn column, AggregationConversionContext context) { + return DSL.field(DSL.name(context.tables().getRootTable(), column.physicalName()), Date.class); + } + + private static String[] requiredColumns(DateColumns dates) { + return dates.columns().stream().map(ResolvedColumn::physicalName).distinct().toArray(String[]::new); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedAggregationConverter.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedAggregationConverter.java new file mode 100644 index 0000000000..4ee21511d1 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedAggregationConverter.java @@ -0,0 +1,37 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import java.util.ArrayList; +import java.util.List; + +import com.bakdata.conquery.sql.compiler.conversion.ConversionDispatcher; +import com.bakdata.conquery.sql.compiler.conversion.Converter; +import com.bakdata.conquery.sql.compiler.ir.concept.CommonAggregationSelect; +import com.bakdata.conquery.sql.model.operation.ResolvedAggregation; + +/** Converts resolved aggregation-model values into connector aggregation IR. */ +public final class ResolvedAggregationConverter { + + private final ConversionDispatcher, AggregationConversionContext> dispatcher; + + public ResolvedAggregationConverter() { + this(List.of()); + } + + /** + * Creates a converter with the built-in aggregation implementations and additional extension converters. + * + *

An extension must support a distinct runtime type; ambiguous converters are rejected by the dispatcher.

+ */ + public ResolvedAggregationConverter( + List, AggregationConversionContext>> extensions + ) { + List, AggregationConversionContext>> converters = + new ArrayList<>(BuiltInAggregationConverters.create()); + converters.addAll(extensions); + this.dispatcher = new ConversionDispatcher<>(converters); + } + + public CommonAggregationSelect convert(ResolvedAggregation aggregation, AggregationConversionContext context) { + return dispatcher.convert(aggregation, context); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedFilterConverter.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedFilterConverter.java index f3094071a2..c94fea8985 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedFilterConverter.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedFilterConverter.java @@ -14,7 +14,7 @@ public final class ResolvedFilterConverter { private final ConversionDispatcher dispatcher; public ResolvedFilterConverter() { - this(List.of()); + this(List.of(), new ResolvedAggregationConverter()); } /** @@ -24,10 +24,18 @@ public ResolvedFilterConverter() { */ public ResolvedFilterConverter( List> extensions + ) { + this(extensions, new ResolvedAggregationConverter()); + } + + public ResolvedFilterConverter( + List> extensions, + ResolvedAggregationConverter aggregationConverter ) { List> converters = new ArrayList<>( BuiltInColumnFilterConverters.create() ); + converters.addAll(BuiltInAggregationFilterConverters.create(aggregationConverter)); converters.addAll(extensions); this.dispatcher = new ConversionDispatcher<>(converters); } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/SumAggregationConverter.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/SumAggregationConverter.java new file mode 100644 index 0000000000..fde3806d44 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/SumAggregationConverter.java @@ -0,0 +1,121 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; + +import com.bakdata.conquery.sql.compiler.ir.CteStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.concept.CommonAggregationSelect; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; +import com.bakdata.conquery.sql.model.operation.BuiltInAggregations; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import lombok.experimental.UtilityClass; +import org.jooq.Field; +import org.jooq.impl.DSL; + +/** Builds sums, including the separate branch needed for distinct-by aggregation. */ +@UtilityClass +final class SumAggregationConverter { + + private static final String ROW_NUMBER = "row_number"; + + static CommonAggregationSelect convert(BuiltInAggregations.Sum aggregation, AggregationConversionContext context) { + String root = context.tables().getRootTable(); + String source = context.tables().getPredecessor(ConceptCteStep.AGGREGATION_SELECT); + List roots = new ArrayList<>(); + roots.add(new ExtractingSqlSelect<>(root, aggregation.column().physicalName(), numberClass(aggregation.column()))); + // The existing distinct-sum path ignores subtraction. + if (aggregation.distinctBy().isEmpty()) { + aggregation.subtractColumn().ifPresent(column -> roots.add(new ExtractingSqlSelect<>(root, column.physicalName(), numberClass(column)))); + } + aggregation.distinctBy().forEach(column -> roots.add(new ExtractingSqlSelect<>(root, column.physicalName(), Object.class))); + // Distinct keys may also be the sum/subtract columns; project each physical column only once. + List rootSelects = new ArrayList<>(); + for (SingleColumnSqlSelect select : roots) { + if (rootSelects.stream().noneMatch(existing -> existing.select().getName().equals(select.select().getName()))) { + rootSelects.add(select); + } + } + + if (aggregation.distinctBy().isEmpty()) { + return CommonAggregationSelect.builder() + .rootSelects(rootSelects) + .groupBy(new FieldWrapper<>(DSL.sum(value(aggregation, source)).as(context.alias()), + rootSelects.stream().map(select -> select.select().getName()).toArray(String[]::new))) + .build(); + } + + SqlIdColumns ids = context.ids().qualify(source); + List> partition = new ArrayList<>(ids.toFields()); + aggregation.distinctBy().stream().map(column -> DSL.field(DSL.name(source, column.physicalName()))).forEach(partition::add); + FieldWrapper rowNumber = new FieldWrapper<>( + DSL.rowNumber().over(DSL.partitionBy(partition)).as(ROW_NUMBER), + partition.stream().map(Field::getName).toArray(String[]::new) + ); + List values = new ArrayList<>(); + values.add(new ExtractingSqlSelect<>(source, aggregation.column().physicalName(), numberClass(aggregation.column()))); + values.add(rowNumber); + QueryStep numbered = QueryStep.builder() + .cteName(context.nameGenerator().cteStepName(SumStep.ROW_NUMBER_ASSIGNED, context.alias())) + .selects(Selects.builder().ids(ids).sqlSelects(values).build()) + .fromTable(QueryStep.toTableLike(source)) + .build(); + FieldWrapper sum = new FieldWrapper<>( + DSL.sum(DSL.coalesce(field(aggregation.column(), numbered.getCteName()), DSL.inline(0))).as(context.alias()) + ); + SqlIdColumns numberedIds = ids.qualify(numbered.getCteName()); + QueryStep summed = QueryStep.builder() + .cteName(context.nameGenerator().cteStepName(SumStep.ROW_NUMBER_FILTERED, context.alias())) + .selects(Selects.builder().ids(numberedIds).sqlSelect(sum).build()) + .fromTable(QueryStep.toTableLike(numbered.getCteName())) + .conditions(List.of(DSL.field(DSL.name(numbered.getCteName(), ROW_NUMBER), Integer.class).eq(DSL.inline(1)))) + .groupBy(numberedIds.toFields()) + .predecessor(numbered) + .build(); + return CommonAggregationSelect.builder() + .rootSelects(rootSelects) + .groupBy(sum) + .additionalPredecessor(summed) + .build(); + } + + private static Field value(BuiltInAggregations.Sum aggregation, String source) { + Field value = field(aggregation.column(), source); + if (aggregation.subtractColumn().isEmpty()) { + return value; + } + Field subtract = field(aggregation.subtractColumn().orElseThrow(), source); + // Preserve missing values when both inputs are null, but treat a single missing operand as zero. + Field zeroIfAnyNonNull = DSL.coalesce(value.multiply(0), subtract.multiply(0)); + return DSL.coalesce(value, zeroIfAnyNonNull).minus(DSL.coalesce(subtract, zeroIfAnyNonNull)); + } + + private static Field field(ResolvedColumn column, String source) { + return DSL.field(DSL.name(source, column.physicalName()), numberClass(column)); + } + + private static Class numberClass(ResolvedColumn column) { + return switch (column.type()) { + case INTEGER -> Integer.class; + case REAL, DECIMAL -> Double.class; + case MONEY -> BigDecimal.class; + default -> throw new IllegalArgumentException("Not a numeric column: " + column.type()); + }; + } + + private enum SumStep implements CteStep { + ROW_NUMBER_ASSIGNED, + ROW_NUMBER_FILTERED; + + @Override + public String getSuffix() { + return name().toLowerCase(java.util.Locale.ROOT); + } + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java index b28e1b50e5..3689136e00 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java @@ -72,6 +72,44 @@ default Field dateDistance(ChronoUnit unit, Field startDate, Loca throw new UnsupportedOperationException("Date-distance calculation is not implemented by this dialect"); } + /** Distance between two date expressions; used with half-open bounds for duration sums. */ + default Field dateDistance(ChronoUnit unit, Field startDate, Field endDate) { + throw new UnsupportedOperationException("Date-distance calculation is not implemented by this dialect"); + } + + /** Shift a date by a number of days. */ + default Field addDays(Field date, Field days) { + return date.add(days); + } + + /** Convert nullable, inclusive physical date bounds to the compiler's half-open range representation. */ + default ColumnDateRange dateRange(Field start, Field inclusiveEnd) { + return ColumnDateRange.of( + DSL.coalesce(start, minimumDate()), + DSL.coalesce(addDays(inclusiveEnd, DSL.inline(1)), maximumDate()) + ); + } + + /** Expand a physical range column into half-open date bounds. */ + default ColumnDateRange dateRangeColumn(Field range) { + throw new UnsupportedOperationException("Physical date-range columns are not implemented by this dialect"); + } + + /** A key identifying both the year and quarter of a date. */ + default Field yearQuarter(Field date) { + throw new UnsupportedOperationException("Year-quarter expressions are not implemented by this dialect"); + } + + /** First day of the quarter containing a date. */ + default Field quarterStart(Field date) { + throw new UnsupportedOperationException("Quarter boundaries are not implemented by this dialect"); + } + + /** First day of the quarter following a date. */ + default Field nextQuarterStart(Field date) { + throw new UnsupportedOperationException("Quarter boundaries are not implemented by this dialect"); + } + /** SQL expression for an already-resolved external entity ID. */ default Field externalId(String id) { return DSL.inline(id, SQLDataType.VARCHAR); diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/CommonAggregationSelect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/CommonAggregationSelect.java index 1c8e65d76f..4d70523b03 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/CommonAggregationSelect.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ir/concept/CommonAggregationSelect.java @@ -22,8 +22,10 @@ public class CommonAggregationSelect { @Singular List rootSelects; + /** Aggregate expression whose alias identifies the result, including results computed in a separate CTE. */ FieldWrapper groupBy; + /** When present, this CTE already computes groupBy; do not also add it to the main aggregation selects. */ QueryStep additionalPredecessor; public Optional getAdditionalPredecessor() { diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedAggregationConverterTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedAggregationConverterTest.java new file mode 100644 index 0000000000..ba411c050a --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedAggregationConverterTest.java @@ -0,0 +1,285 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.Date; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.concept.CommonAggregationSelect; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import com.bakdata.conquery.sql.model.operation.BuiltInAggregations; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.DateColumns; +import com.bakdata.conquery.sql.model.schema.SqlTable; +import org.jooq.Field; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Test; + +class ResolvedAggregationConverterTest { + + private static final SqlTable TABLE = SqlTable.of("events", "analytics", "events"); + private static final ResolvedColumn VALUE = column("value", ColumnType.STRING); + private static final ResolvedColumn PERSON = column("person_id", ColumnType.STRING); + private static final ResolvedColumn AMOUNT = column("amount", ColumnType.DECIMAL); + private static final ResolvedColumn DISCOUNT = column("discount", ColumnType.DECIMAL); + private static final ResolvedColumn START = column("start_date", ColumnType.DATE); + private static final ResolvedColumn END = column("end_date", ColumnType.DATE); + private static final SqlTables TABLES = new SqlTables( + "events", + Map.of(ConceptCteStep.PREPROCESSING, "preprocessing"), + Map.of(ConceptCteStep.AGGREGATION_SELECT, ConceptCteStep.PREPROCESSING) + ); + private static final AggregationConversionContext CONTEXT = new AggregationConversionContext( + new TestDialect(), + new SqlNameGenerator(128), + TABLES, + new SqlIdColumns(field(name("events", "id"), String.class)), + "event-count" + ); + + private final ResolvedAggregationConverter converter = new ResolvedAggregationConverter(); + + @Test + void shouldConvertCount() { + CommonAggregationSelect result = converter.convert( + new BuiltInAggregations.Count(VALUE, List.of()), + CONTEXT + ); + + assertEquals(List.of("\"events\".\"value\""), renderRootSelects(result)); + assertEquals( + "nullif(count(\"preprocessing\".\"value\"), 0) as \"event-count\"", + render(result.getGroupBy().select()) + ); + } + + @Test + void shouldCountDistinctValues() { + CommonAggregationSelect result = converter.convert( + new BuiltInAggregations.Count(VALUE, List.of(PERSON)), + CONTEXT + ); + + assertEquals(List.of("\"events\".\"value\"", "\"events\".\"person_id\""), renderRootSelects(result)); + assertEquals( + "nullif(count(distinct \"preprocessing\".\"person_id\"), 0) as \"event-count\"", + render(result.getGroupBy().select()) + ); + } + + private static List renderRootSelects(CommonAggregationSelect aggregation) { + return aggregation.getRootSelects().stream() + .map(select -> render(select.select())) + .toList(); + } + + @Test + void shouldConvertSumWithoutReplacingMissingValuesWithZero() { + CommonAggregationSelect result = converter.convert(new BuiltInAggregations.Sum(AMOUNT, Optional.empty(), List.of()), CONTEXT); + + assertEquals(List.of("\"events\".\"amount\""), renderRootSelects(result)); + assertEquals("sum(\"preprocessing\".\"amount\") as \"event-count\"", render(result.getGroupBy().select())); + assertTrue(result.getAdditionalPredecessor().isEmpty()); + } + + @Test + void shouldSubtractWithZeroOnlyWhenAnOperandIsPresent() { + CommonAggregationSelect result = converter.convert(new BuiltInAggregations.Sum(AMOUNT, Optional.of(DISCOUNT), List.of()), CONTEXT); + + assertEquals(List.of("\"events\".\"amount\"", "\"events\".\"discount\""), renderRootSelects(result)); + String zero = "coalesce((\"preprocessing\".\"amount\" * 0), (\"preprocessing\".\"discount\" * 0))"; + assertEquals( + "sum((coalesce(\"preprocessing\".\"amount\", " + zero + ") - coalesce(\"preprocessing\".\"discount\", " + zero + "))) as \"event-count\"", + render(result.getGroupBy().select()) + ); + } + + @Test + void shouldSumDistinctRowsInSeparateBranchPartitionedByAllIdsAndKeys() { + AggregationConversionContext context = new AggregationConversionContext( + new TestDialect(), new SqlNameGenerator(128), TABLES, + new SqlIdColumns(field(name("events", "id"), String.class), field(name("events", "secondary_id"), String.class)), "total" + ); + CommonAggregationSelect result = converter.convert(new BuiltInAggregations.Sum(AMOUNT, Optional.empty(), List.of(PERSON, AMOUNT)), context); + QueryStep summed = result.getAdditionalPredecessor().orElseThrow(); + QueryStep numbered = summed.getPredecessors().getFirst(); + + assertEquals(List.of("\"events\".\"amount\"", "\"events\".\"person_id\""), renderRootSelects(result)); + assertEquals( + "row_number() over (partition by \"preprocessing\".\"id\", \"preprocessing\".\"secondary_id\", \"preprocessing\".\"person_id\", \"preprocessing\".\"amount\") as \"row_number\"", + render(numbered.getSelects().getSqlSelects().getLast().toFields().getFirst()) + ); + assertEquals("\"total-row_number_assigned\".\"row_number\" = 1", DSL.using(SQLDialect.POSTGRES).renderInlined(summed.getConditions().getFirst())); + assertEquals(2, summed.getGroupBy().size()); + assertEquals("sum(coalesce(\"total-row_number_assigned\".\"amount\", 0)) as \"total\"", render(result.getGroupBy().select())); + assertEquals(List.of(result.getGroupBy()), summed.getSelects().getSqlSelects()); + } + + @Test + void shouldIgnoreSubtractionInsideDistinctSumLikeExistingBackend() { + CommonAggregationSelect result = converter.convert(new BuiltInAggregations.Sum(AMOUNT, Optional.of(DISCOUNT), List.of(PERSON)), CONTEXT); + QueryStep numbered = result.getAdditionalPredecessor().orElseThrow().getPredecessors().getFirst(); + + assertEquals(2, numbered.getSelects().getSqlSelects().size()); + assertEquals("sum(coalesce(\"event-count-row_number_assigned\".\"amount\", 0)) as \"event-count\"", render(result.getGroupBy().select())); + assertEquals(List.of("\"events\".\"amount\"", "\"events\".\"person_id\""), renderRootSelects(result)); + } + + @Test + void shouldPreserveNumericInputTypes() { + Map> types = Map.of( + ColumnType.INTEGER, Integer.class, + ColumnType.REAL, Double.class, + ColumnType.DECIMAL, Double.class, + ColumnType.MONEY, java.math.BigDecimal.class + ); + for (var entry : types.entrySet()) { + ResolvedColumn amount = column("amount", entry.getKey()); + ResolvedColumn discount = column("discount", entry.getKey()); + for (List distinctBy : List.of(List.of(), List.of(PERSON))) { + CommonAggregationSelect result = converter.convert( + new BuiltInAggregations.Sum(amount, Optional.of(discount), distinctBy), CONTEXT); + assertEquals(entry.getValue(), result.getRootSelects().getFirst().select().getType(), entry.getKey().name()); + } + } + } + + @Test + void shouldCountDistinctYearQuartersForSingleDates() { + CommonAggregationSelect result = converter.convert(new BuiltInAggregations.CountQuarters(new DateColumns.Single(START)), CONTEXT); + + assertEquals(List.of("\"events\".\"start_date\""), renderRootSelects(result)); + assertEquals("nullif(count(distinct year_quarter(\"preprocessing\".\"start_date\")), 0) as \"event-count\"", render(result.getGroupBy().select())); + } + + @Test + void shouldSumQuarterCountsForDatePairs() { + CommonAggregationSelect result = converter.convert(new BuiltInAggregations.CountQuarters(new DateColumns.Pair(START, END)), CONTEXT); + + assertEquals(List.of("(date_distance('months', quarter_start(\"events\".\"start_date\"), next_quarter_start(\"events\".\"end_date\")) / 3) as \"event-count\""), renderRootSelects(result)); + assertEquals("nullif(sum(\"preprocessing\".\"event-count\"), 0) as \"event-count\"", render(result.getGroupBy().select())); + } + + @Test + void shouldPackIntervalsBeforeSummingDurationsAndExcludeInfiniteBounds() { + CommonAggregationSelect result = converter.convert(new BuiltInAggregations.DurationSum(new DateColumns.Pair(START, END), List.of()), CONTEXT); + QueryStep sum = result.getAdditionalPredecessor().orElseThrow(); + + assertEquals(List.of( + "coalesce(\"events\".\"start_date\", \"minimum_date\") as \"event-count_start\"", + "coalesce(add_days(\"events\".\"end_date\", 1), \"maximum_date\") as \"event-count_end\"" + ), renderRootSelects(result)); + assertEquals("event-count-interval_packing_selects", sum.getCteName()); + assertEquals("event-count-interval_complete", sum.getPredecessors().getFirst().getCteName()); + assertEquals(List.of(result.getGroupBy()), sum.getSelects().getSqlSelects()); + String sql = render(result.getGroupBy().select()); + assertTrue(sql.contains("\"event-count-interval_complete\".\"event-count_start\" = \"minimum_date\""), sql); + assertTrue(sql.contains("\"event-count-interval_complete\".\"event-count_end\" = \"maximum_date\""), sql); + assertTrue(sql.contains("then null else date_distance('days'"), sql); + } + + @Test + void shouldTreatSingleDateAsOneInclusiveDayForDuration() { + CommonAggregationSelect result = converter.convert(new BuiltInAggregations.DurationSum(new DateColumns.Single(START), List.of()), CONTEXT); + assertEquals("coalesce(add_days(\"events\".\"start_date\", 1), \"maximum_date\") as \"event-count_end\"", renderRootSelects(result).getLast()); + } + + @Test + void shouldIgnoreDurationDistinctKeysLikeExistingBackend() { + CommonAggregationSelect result = converter.convert( + new BuiltInAggregations.DurationSum(new DateColumns.Pair(START, END), List.of(PERSON)), CONTEXT); + assertEquals(List.of( + "coalesce(\"events\".\"start_date\", \"minimum_date\") as \"event-count_start\"", + "coalesce(add_days(\"events\".\"end_date\", 1), \"maximum_date\") as \"event-count_end\"" + ), renderRootSelects(result)); + assertEquals("event-count-interval_packing_selects", result.getAdditionalPredecessor().orElseThrow().getCteName()); + } + + @Test + void shouldRequireDialectSupportForPhysicalRangeColumns() { + assertThrows(UnsupportedOperationException.class, () -> converter.convert( + new BuiltInAggregations.DurationSum(new DateColumns.Single(column("range", ColumnType.DATE_RANGE)), List.of()), CONTEXT + )); + } + + private static String render(Field field) { + return DSL.using(SQLDialect.POSTGRES).renderInlined(DSL.select(field)).substring("select ".length()).toLowerCase(Locale.ROOT); + } + + private static ResolvedColumn column(String name, ColumnType type) { + return new ResolvedColumn("events." + name, TABLE, name, type, true); + } + + private static final class TestDialect implements CompilerDialect { + + @Override + public Field dateDistance(ChronoUnit unit, Field start, Field end) { + return DSL.function("date_distance", Integer.class, DSL.inline(unit.name()), start, end); + } + + @Override + public Field addDays(Field date, Field days) { + return DSL.function("add_days", Date.class, date, days); + } + + @Override + public Field yearQuarter(Field date) { + return DSL.function("year_quarter", String.class, date); + } + + @Override + public Field quarterStart(Field date) { + return DSL.function("quarter_start", Date.class, date); + } + + @Override + public Field nextQuarterStart(Field date) { + return DSL.function("next_quarter_start", Date.class, date); + } + + @Override + public Field minimumDate() { + return field(name("minimum_date"), Date.class); + } + + @Override + public Field maximumDate() { + return field(name("maximum_date"), Date.class); + } + + @Override + public Field anyValue(Field value) { + return value; + } + + @Override + public Field renderDateRange(Field start, Field end) { + return field(name("rendered_range"), Object.class); + } + + @Override + public Field aggregateDateRanges(Field start, Field end) { + return field(name("aggregated_ranges"), Object.class); + } + + @Override + public int getNameMaxLength() { + return 128; + } + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedFilterConverterTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedFilterConverterTest.java index 02dec21449..21264c6b6f 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedFilterConverterTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedFilterConverterTest.java @@ -4,6 +4,8 @@ import static org.jooq.impl.DSL.field; import static org.jooq.impl.DSL.name; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.sql.Date; @@ -21,16 +23,24 @@ import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.concept.CommonAggregationSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.condition.ConditionWrappingWhereCondition; import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import com.bakdata.conquery.sql.model.operation.BuiltInAggregations; import com.bakdata.conquery.sql.model.operation.BuiltInFilters; import com.bakdata.conquery.sql.model.operation.ResolvedFilter; +import com.bakdata.conquery.sql.model.operation.ResolvedAggregation; import com.bakdata.conquery.sql.model.range.NumberRange; import com.bakdata.conquery.sql.model.range.SubstringRange; import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.DateColumns; import com.bakdata.conquery.sql.model.schema.SqlTable; import org.jooq.Field; import org.jooq.SQLDialect; @@ -55,10 +65,20 @@ class ResolvedFilterConverterTest { private static final ResolvedColumn FLAG_B = new ResolvedColumn( "flag-b", TABLE, "flag_b", ColumnType.BOOLEAN, false ); - private static final FilterConversionContext CONTEXT = new FilterConversionContext( + private final FilterConversionContext CONTEXT = new FilterConversionContext( new TestDialect(), new SqlNameGenerator(128), - new SqlTables("events", Map.of(), Map.of()), + new SqlTables( + "events", + Map.of( + ConceptCteStep.PREPROCESSING, "preprocessing", + ConceptCteStep.JOIN_BRANCHES, "join_branches" + ), + Map.of( + ConceptCteStep.AGGREGATION_SELECT, ConceptCteStep.PREPROCESSING, + ConceptCteStep.AGGREGATION_FILTER, ConceptCteStep.JOIN_BRANCHES + ) + ), new SqlIdColumns(field(name("analytics", "events", "id"), String.class)) ); @@ -132,6 +152,27 @@ void shouldConvertSelectedFlags() { ); } + @Test + void shouldConvertAggregationRange() { + BuiltInFilters.AggregationRange filter = new BuiltInFilters.AggregationRange( + "event count", + new BuiltInAggregations.Count(STRING_COLUMN, List.of()), + NumberRange.atLeast(2) + ); + + SqlFilters result = converter.convert(filter, CONTEXT); + + assertEquals("\"join_branches\".\"event_count-1\" >= 2", renderGroupFilter(result)); + assertEquals( + "nullif(count(\"preprocessing\".\"event_code\"), 0) as \"event_count-1\"", + renderField(result.getSelects().getAggregationSelects().getFirst().toFields().getFirst()) + ); + assertEquals( + "\"events\".\"event_code\"", + renderField(result.getSelects().getPreprocessingSelects().getFirst().toFields().getFirst()) + ); + } + @Test void shouldDispatchExtensionFilter() { Converter extension = new Converter<>() { @@ -155,6 +196,80 @@ public SqlFilters convert(ExtensionFilter input, FilterConversionContext context assertEquals("(extension_filter)", render(converterWithExtension.convert(new ExtensionFilter("extension"), CONTEXT))); } + @Test + void shouldFilterDistinctSumFromJoinedBranchWithoutRepeatingAggregation() { + SqlFilters result = converter.convert(new BuiltInFilters.AggregationRange( + "distinct total", new BuiltInAggregations.Sum(NUMBER_COLUMN, Optional.empty(), List.of(STRING_COLUMN)), NumberRange.closed(2, 8) + ), CONTEXT); + + assertTrue(result.getSelects().getAggregationSelects().isEmpty()); + assertEquals("distinct_total-1-row_number_filtered", result.getSelects().getAdditionalPredecessor().orElseThrow().getCteName()); + assertEquals("(\"join_branches\".\"distinct_total-1\" >= 2 and \"join_branches\".\"distinct_total-1\" <= 8)", renderGroupFilter(result)); + } + + @Test + void shouldFilterDurationFromIntervalPackingBranch() { + SqlFilters result = converter.convert(new BuiltInFilters.AggregationRange( + "duration", new BuiltInAggregations.DurationSum(new DateColumns.Single(DATE_COLUMN), List.of()), NumberRange.atMost(10) + ), CONTEXT); + + assertTrue(result.getSelects().getAggregationSelects().isEmpty()); + assertEquals("duration-1-interval_packing_selects", result.getSelects().getAdditionalPredecessor().orElseThrow().getCteName()); + assertEquals("\"join_branches\".\"duration-1\" <= 10", renderGroupFilter(result)); + } + + @Test + void shouldDispatchCustomAggregationInsideRangeFilter() { + FieldWrapper value = new FieldWrapper<>(DSL.inline(42).as("custom-1")); + QueryStep predecessor = QueryStep.builder().cteName("custom_result") + .selects(Selects.builder().ids(CONTEXT.ids()).sqlSelect(value).build()).build(); + ResolvedAggregationConverter custom = new ResolvedAggregationConverter(List.of( + new Converter, AggregationConversionContext>() { + @Override + public Class getConversionClass() { + return ExtensionAggregation.class; + } + + @Override + public CommonAggregationSelect convert(ExtensionAggregation input, AggregationConversionContext context) { + assertEquals("custom-1", context.alias()); + assertSame(CONTEXT.ids(), context.ids()); + return CommonAggregationSelect.builder().groupBy(value).additionalPredecessor(predecessor).build(); + } + } + )); + SqlFilters result = new ResolvedFilterConverter(List.of(), custom).convert( + new BuiltInFilters.AggregationRange("custom", new ExtensionAggregation(), NumberRange.atLeast(1)), CONTEXT + ); + + assertSame(predecessor, result.getSelects().getAdditionalPredecessor().orElseThrow()); + assertTrue(result.getSelects().getAggregationSelects().isEmpty()); + assertEquals("\"join_branches\".\"custom-1\" >= 1", renderGroupFilter(result)); + } + + @Test + void shouldRejectNonnumericAggregationInRangeFilter() { + ResolvedAggregationConverter custom = new ResolvedAggregationConverter(List.of( + new Converter, AggregationConversionContext>() { + @Override + public Class getConversionClass() { + return ExtensionAggregation.class; + } + + @Override + public CommonAggregationSelect convert(ExtensionAggregation input, AggregationConversionContext context) { + return CommonAggregationSelect.builder().groupBy(new FieldWrapper<>(DSL.inline("flag").as(context.alias()))).build(); + } + } + )); + assertThrows(IllegalArgumentException.class, () -> new ResolvedFilterConverter(List.of(), custom).convert( + new BuiltInFilters.AggregationRange("invalid", new ExtensionAggregation(), NumberRange.atLeast(1)), CONTEXT + )); + } + + private record ExtensionAggregation() implements ResolvedAggregation { + } + private static String render(SqlFilters filters) { assertEquals(1, filters.getWhereClauses().getEventFilters().size()); assertTrue(filters.getWhereClauses().getPreprocessingConditions().isEmpty()); @@ -164,6 +279,19 @@ private static String render(SqlFilters filters) { .toLowerCase(Locale.ROOT); } + private static String renderGroupFilter(SqlFilters filters) { + assertEquals(1, filters.getWhereClauses().getGroupFilters().size()); + assertTrue(filters.getWhereClauses().getPreprocessingConditions().isEmpty()); + assertTrue(filters.getWhereClauses().getEventFilters().isEmpty()); + return DSL.using(SQLDialect.POSTGRES) + .renderInlined(filters.getWhereClauses().getGroupFilters().getFirst().condition()) + .toLowerCase(Locale.ROOT); + } + + private static String renderField(Field field) { + return DSL.using(SQLDialect.POSTGRES).renderInlined(DSL.select(field)).substring("select ".length()).toLowerCase(Locale.ROOT); + } + private static void assertEmptySelects(ConnectorSqlSelects selects) { assertTrue(selects.getPreprocessingSelects().isEmpty()); assertTrue(selects.getAggregationSelects().isEmpty()); @@ -177,6 +305,11 @@ private record ExtensionFilter(String name) implements ResolvedFilter { private static final class TestDialect implements CompilerDialect { + @Override + public Field dateDistance(ChronoUnit unit, Field start, Field end) { + return DSL.function("date_distance", Integer.class, DSL.inline(unit.name()), start, end); + } + @Override public Field minimumDate() { return field(name("minimum_date"), Date.class); From ff05d9b5d74950bad50600d2e49e26f3bf6d4414 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 23 Sep 2026 10:29:50 +0200 Subject: [PATCH 64/81] fix(sql): integrate concept mapping with extracted pipeline Co-authored-by: Codex GPT-6 --- .../sql/conversion/cqelement/concept/CQConceptConverter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 634c8a3625..0f2ea9c8c0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -5,6 +5,7 @@ import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteInput; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteCompiler; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCteCompiler; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCtePipelineAssembler; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCtePipelineInput; @@ -130,7 +131,7 @@ private static SqlFilters collectConditionFilters( Connector connector = cqTable.getConnector().resolve(); if (connector.getColumn() != null) { - conditions.add(ConditionUtil.wrap(field(CTConditionContext.forConnector(connector, functionProvider).getConnectorColumn(), String.class).isNotNull())); + conditions.add(new ConditionWrappingWhereCondition(field(CTConditionContext.forConnector(connector, functionProvider).getConnectorColumn(), String.class).isNotNull())); } ValidityDate validityDate = cqTable.findValidityDate(); From aab5961e4bb28575aea69357309cfc503740b581 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 23 Sep 2026 13:50:26 +0200 Subject: [PATCH 65/81] refactor(sql): delegate output selects to connector Co-authored-by: Codex GPT-6 --- .../specific/MappableSingleColumnSelect.java | 32 +-- .../dialect/LegacyCompilerDialect.java | 48 ++++ .../clickhouse/ClickhouseDialectBundle.java | 10 + .../ClickhouseDistinctSelectConverter.java | 44 +-- .../aggregator/DateDistanceSqlAggregator.java | 23 +- .../model/aggregator/FlagSqlAggregator.java | 76 +----- .../select/ConceptColumnSelectConverter.java | 207 +++++--------- .../select/DateUnionSelectConverter.java | 7 +- .../model/select/DaterangeSelectUtil.java | 171 ------------ .../model/select/DistinctSelectConverter.java | 129 +-------- .../select/EventDateUnionSelectConverter.java | 43 +-- .../EventDurationSumSelectConverter.java | 62 +---- .../model/select/ExistsSelectConverter.java | 22 +- .../select/FirstValueSelectConverter.java | 13 +- .../select/LastValueSelectConverter.java | 10 +- .../select/RandomValueSelectConverter.java | 30 +-- .../model/select/ResolvedSelectAdapter.java | 89 ++++++ .../SqlExtractionAggregationTest.java | 28 +- .../SqlExtractionConceptSelectTest.java | 95 +++++++ .../SqlExtractionDateSelectTest.java | 99 +++++++ .../conversion/SqlExtractionSelectTest.java | 122 +++++++++ docs/sql-connector-extraction-verification.md | 9 +- sql-connector/README.md | 30 ++- .../BuiltInAggregationConverters.java | 1 + .../ClickhouseDistinctSelectConverter.java | 45 ++++ .../ConceptColumnSelectConverter.java | 154 +++++++++++ .../operation/DateAggregationConverters.java | 4 +- .../operation/DateDistanceSqlAggregator.java | 40 +++ .../operation/DaterangeSelectUtil.java | 87 ++++++ .../operation/DistinctSelectConverter.java | 136 ++++++++++ .../EventDateUnionSelectConverter.java | 53 ++++ .../EventDurationSumSelectConverter.java | 72 +++++ .../operation/FlagSqlAggregator.java | 81 ++++++ .../operation/RandomValueSelectConverter.java | 35 +++ .../operation/ResolvedSelectConverter.java | 91 +++++++ .../operation/SelectConversionContext.java | 58 ++++ .../conversion/operation/SubstringSelect.java | 34 +++ .../operation}/ValueSelectUtil.java | 40 +-- .../sql/compiler/dialect/CompilerDialect.java | 43 +++ .../ResolvedSelectConverterTest.java | 254 ++++++++++++++++++ .../ConnectorCtePipelineAssemblerTest.java | 27 ++ 41 files changed, 1874 insertions(+), 780 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ResolvedSelectAdapter.java create mode 100644 backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionConceptSelectTest.java create mode 100644 backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionDateSelectTest.java create mode 100644 backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionSelectTest.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ClickhouseDistinctSelectConverter.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ConceptColumnSelectConverter.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/DateDistanceSqlAggregator.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/DaterangeSelectUtil.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/DistinctSelectConverter.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/EventDateUnionSelectConverter.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/EventDurationSumSelectConverter.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/FlagSqlAggregator.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/RandomValueSelectConverter.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedSelectConverter.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/SelectConversionContext.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/SubstringSelect.java rename {backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation}/ValueSelectUtil.java (78%) create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedSelectConverterTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java index 18bf21b494..a98a50b7cc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java @@ -1,8 +1,11 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; +import com.bakdata.conquery.sql.conversion.model.select.ResolvedSelectAdapter; +import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; +import com.bakdata.conquery.sql.compiler.conversion.operation.SubstringSelect; import static com.bakdata.conquery.models.types.ResultType.Primitive.STRING; import static com.bakdata.conquery.models.types.ResultType.resolveResultType; -import static org.jooq.impl.DSL.*; + import java.util.Collections; import java.util.Set; @@ -26,14 +29,14 @@ import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; -import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; + import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; import lombok.Getter; -import org.jooq.Field; + @Getter public abstract class MappableSingleColumnSelect extends SingleColumnSelect { @@ -61,25 +64,10 @@ public static SingleColumnSqlSelect getSubstringSelect( Column column, Range.IntegerRange substringRange, SelectContext selectContext, String alias) { - Field field = field(name(selectContext.getTables().getRootTable(), column.getName()), String.class); - - if (substringRange != null && !substringRange.isAll()) { - if (substringRange.isAtLeast()) { - field = substring(field, 1 + substringRange.getMin()); - } - else if (substringRange.isAtMost()) { - field = substring(field, 1, substringRange.getMax()); - } - else { - field = substring(field, 1 + substringRange.getMin(), substringRange.getMax() - substringRange.getMin()); - } - } - - if (alias != null) { - field = field.as(name(alias)); - } - - return new FieldWrapper<>(field, column.getName()); + return SubstringSelect.getSubstringSelect( + EntitySchemaAdapter.from(column), + ResolvedSelectAdapter.substring(substringRange), + selectContext.getTables().getRootTable(), alias); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java index e85890f82d..fb06feb16f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java @@ -1,5 +1,10 @@ package com.bakdata.conquery.sql.conversion.dialect; +import org.jooq.DataType; +import org.jooq.SortField; +import org.jooq.OrderField; +import java.util.function.Function; +import java.util.Collection; import java.sql.Date; import java.time.LocalDate; import java.time.temporal.ChronoUnit; @@ -48,6 +53,42 @@ */ public interface LegacyCompilerDialect extends CompilerDialect { + @Override + default ColumnDateRange toDualColumn(ColumnDateRange range) { + return getFunctionProvider().toDualColumn(range); + } + + @Override + default Condition orAgg(Field field) { + return getFunctionProvider().orAgg(field); + } + + @Override + default Field arrayOut(List> fields) { + return getFunctionProvider().arrayOut(fields); + } + + @Override + default Field cast(Field field, DataType type) { + return getFunctionProvider().cast(field, type); + } + + @Override + default Field stringAggregation(Field field, Field delimiter, List> orderByFields) { + return getFunctionProvider().stringAggregation(field, delimiter, orderByFields); + } + + @Override + default Collection> orderByValidityDates( + Function, ? extends SortField> ordering, List> validityDateFields) { + return getFunctionProvider().orderByValidityDates(ordering, validityDateFields); + } + + @Override + default Field random(Field field) { + return getFunctionProvider().random(field); + } + @Override default Field minimumDate() { return getFunctionProvider().getMinDateExpression(); @@ -138,6 +179,13 @@ default ColumnDateRange dateRange(Field start, Field inclusiveEnd) { return getFunctionProvider().dateRange(start, inclusiveEnd); } + @Override + default ColumnDateRange dateRangeColumn(Field range) { + // Both existing SQL providers use a single physical column as inclusive start and end. + Field date = range.coerce(Date.class); + return dateRange(date, date); + } + @Override default Field yearQuarter(Field date) { return getFunctionProvider().yearQuarter(date); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java index e6a1db452b..0f318ae941 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java @@ -1,5 +1,8 @@ package com.bakdata.conquery.sql.conversion.dialect.clickhouse; +import com.bakdata.conquery.sql.compiler.conversion.operation.SelectConversionContext; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import java.util.List; import java.util.Map; @@ -24,6 +27,13 @@ @Slf4j public class ClickhouseDialectBundle implements DialectBundle { + @Override + public ConnectorSqlSelects distinctSelect( + BuiltInSelects.Values select, + SelectConversionContext context) { + return com.bakdata.conquery.sql.compiler.conversion.operation.ClickhouseDistinctSelectConverter.connectorSelect(select, context); + } + private final SqlFunctionProvider functionProvider; public ClickhouseDialectBundle() { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java index e4d15e5fd6..7d7968cd20 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java @@ -1,45 +1,7 @@ package com.bakdata.conquery.sql.conversion.dialect.clickhouse; -import static org.jooq.impl.DSL.field; - -import com.bakdata.conquery.models.datasets.concepts.select.connector.DistinctSelect; -import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; -import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; -import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; -import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; -import com.bakdata.conquery.sql.conversion.model.select.SelectContext; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; -import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; - -public class ClickhouseDistinctSelectConverter implements SelectConverter { - - - @Override - public ConnectorSqlSelects connectorSelect(DistinctSelect distinctSelect, SelectContext selectContext) { - - String alias = selectContext.getNameGenerator().legacyOperationName(distinctSelect.getName()); - - ConnectorSqlTables tables = selectContext.getTables(); - SingleColumnSqlSelect preprocessingSelect = - MappableSingleColumnSelect.getSubstringSelect(distinctSelect.getColumn().get(), distinctSelect.getSubstringRange(), selectContext, alias); - - String preprocessingTable = selectContext.getTables().cteName(ConceptCteStep.PREPROCESSING); - SingleColumnSqlSelect qualified = preprocessingSelect.qualify(preprocessingTable); - - FieldWrapper grouped = - new FieldWrapper<>(field("arrayFilter(x -> x <> '' and x is not null, groupUniqArray({0}))", Object.class, qualified.select()).as(alias), - qualified.select().getName() - ); - - SingleColumnSqlSelect finalSelect = grouped.qualify(tables.cteName(ConceptCteStep.AGGREGATION_SELECT)); - - return ConnectorSqlSelects.builder() - .preprocessingSelect(preprocessingSelect) - .aggregationSelect(grouped) - .finalSelect(finalSelect) - .build(); - } - +import com.bakdata.conquery.sql.conversion.model.select.DistinctSelectConverter; +/** Retained backend entry point; the dialect selects the shared ClickHouse implementation. */ +public class ClickhouseDistinctSelectConverter extends DistinctSelectConverter { } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java index 1359ff7ee4..689c65ea74 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java @@ -34,27 +34,10 @@ public class DateDistanceSqlAggregator implements SelectConverter selectContext) { - - Column column = select.getColumn().resolve(); - String alias = selectContext.getNameGenerator().legacyOperationName(select.getName()); - SqlTables tables = selectContext.getTables(); - - Field dateDistanceCalculation = createDateDistanceCalculation(column, select.getTimeUnit(), tables, selectContext.getConversionContext()); - FieldWrapper dateDistanceSelect = new FieldWrapper<>(dateDistanceCalculation.as(alias)); - - Field qualifiedDateDistance = dateDistanceSelect.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT)).select(); - FieldWrapper minDateDistance = new FieldWrapper<>(DSL.min(qualifiedDateDistance).as(alias)); - - String finalPredecessor = tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER); - ExtractingSqlSelect finalSelect = minDateDistance.qualify(finalPredecessor); - - return ConnectorSqlSelects.builder() - .preprocessingSelect(dateDistanceSelect) - .aggregationSelect(minDateDistance) - .finalSelect(finalSelect) - .build(); + return com.bakdata.conquery.sql.compiler.conversion.operation.DateDistanceSqlAggregator.connectorSelect( + com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter.from(select.getColumn().resolve()), select.getTimeUnit(), + getEndDate(selectContext.getConversionContext()), ResolvedSelectAdapter.context(select.getName(), selectContext)); } - @Override public SqlFilters convertToSqlFilter(DateDistanceFilter filter, FilterContext filterContext) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java index 619b2fef12..e5d765041d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java @@ -70,60 +70,6 @@ */ public class FlagSqlAggregator implements SelectConverter, FilterConverter>, SqlAggregator { - /** - * @return A mapping between a flags key and the corresponding {@link ExtractingSqlSelect} that will be created to reference the flag's column. - */ - private static Map createFlagRootSelectMap(FlagSelect flagSelect, String rootTable) { - return flagSelect.getFlags() - .entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> { - Column column = entry.getValue().resolve(); - Field field = field(name(rootTable, column.getName())); - return new FieldWrapper<>(field.as(column.getName()), column.getName() - ); - } - )); - } - - private static FieldWrapper createFlagSelect( - String alias, - SqlTables connectorTables, - SqlFunctionProvider functionProvider, - Map flagRootSelectMap - ) { - Map> flagFieldsMap = createRootSelectReferences(connectorTables, flagRootSelectMap); - - // we first aggregate each flag column - List> flagAggregations = new ArrayList<>(); - for (Map.Entry> entry : flagFieldsMap.entrySet()) { - Field boolColumn = entry.getValue(); - Condition anyTrue = functionProvider.orAgg(boolColumn); - - String flagName = entry.getKey(); - Field flag = when(anyTrue, inline(flagName)).otherwise(""); // else null is implicit in SQL - flagAggregations.add(flag); - } - - // and stuff them into 1 array field - Field flagsArray = functionProvider.arrayOut(flagAggregations).as(alias); - // we also need the references for all flag columns for the flag aggregation of multiple columns - String[] requiredColumns = flagFieldsMap.values().stream().map(Field::getName).toArray(String[]::new); - return new FieldWrapper<>(flagsArray, requiredColumns); - } - - private static Map> createRootSelectReferences( - SqlTables connectorTables, - Map flagRootSelectMap - ) { - return flagRootSelectMap.entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> (Field) entry.getValue().qualify(connectorTables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT)).select() - )); - } - /** * @return Columns names of a given flags map that match the selected flags of the filter value. */ @@ -136,24 +82,12 @@ private static List getRequiredColumns(Map flags, Set< @Override public ConnectorSqlSelects connectorSelect(FlagSelect flagSelect, SelectContext selectContext) { - - SqlFunctionProvider functionProvider = selectContext.getConversionContext().getCompilerDialect().getFunctionProvider(); - SqlTables connectorTables = selectContext.getTables(); - - Map rootSelects = createFlagRootSelectMap(flagSelect, connectorTables.getRootTable()); - - String alias = selectContext.getNameGenerator().legacyOperationName(flagSelect.getName()); - FieldWrapper flagAggregation = createFlagSelect(alias, connectorTables, functionProvider, rootSelects); - - ExtractingSqlSelect finalSelect = flagAggregation.qualify(connectorTables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); - - return ConnectorSqlSelects.builder() - .preprocessingSelects(rootSelects.values()) - .aggregationSelect(flagAggregation) - .finalSelect(finalSelect) - .build(); + var flags = flagSelect.getFlags().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, + entry -> com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter.from(entry.getValue().resolve()))); + return com.bakdata.conquery.sql.conversion.model.select.ResolvedSelectAdapter.connectorSelect( + new com.bakdata.conquery.sql.model.operation.BuiltInSelects.Aggregation(flagSelect.getName(), + new com.bakdata.conquery.sql.model.operation.BuiltInAggregations.Flags(flags)), flagSelect.getName(), selectContext); } - @Override public SqlFilters convertToSqlFilter(FlagFilter flagFilter, FilterContext> filterContext) { SqlTables connectorTables = filterContext.getTables(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java index 6c3fe3ccf3..671350ae87 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java @@ -1,184 +1,107 @@ package com.bakdata.conquery.sql.conversion.model.select; -import java.util.Collections; import java.util.List; -import java.util.Objects; import java.util.Optional; import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.select.concept.ConceptColumnSelect; import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; +import com.bakdata.conquery.sql.compiler.conversion.operation.SelectConversionContext; +import com.bakdata.conquery.sql.compiler.ir.SchemaSql; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptSqlSelects; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; -import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; -import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; -import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptIdMapping; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.compiler.ir.CteStep; -import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import com.bakdata.conquery.sql.execution.ResultSetProcessor; -import lombok.Getter; -import lombok.RequiredArgsConstructor; +import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import org.jooq.Condition; import org.jooq.Field; import org.jooq.Record; import org.jooq.Table; import org.jooq.TableLike; import org.jooq.impl.DSL; -import org.jooq.impl.SQLDataType; +/** Resolves concept connector columns and mapping-table sources before delegating SQL composition to the connector. */ public class ConceptColumnSelectConverter implements SelectConverter { - @Getter - @RequiredArgsConstructor - private enum CONCEPT_COLUMN_STEPS implements CteStep { - - UNIONED_COLUMNS("unioned_columns"), - STRING_AGG("concept_column_aggregated"); - - private final String suffix; - } - @Override - public ConnectorSqlSelects connectorSelect(ConceptColumnSelect select, SelectContext selectContext) { - Connector connector = selectContext.getTables().getConnector(); + public ConnectorSqlSelects connectorSelect(ConceptColumnSelect select, SelectContext context) { + Connector connector = context.getTables().getConnector(); if (connector.getColumn() == null) { return ConnectorSqlSelects.none(); } - if (select.isAsIds()) { - TreeConcept concept = (TreeConcept) select.getHolder().findConcept(); - ConceptIdMapping mapping = new ConceptIdMapping(concept, selectContext.getFunctionProvider()); - Field resolvedId = DSL.coalesce(mapping.resolvedId(), DSL.inline(concept.getLocalId())) - .as(connector.getColumn().getColumn()); - return ConnectorSqlSelects.builder() - .preprocessingSelect(new FieldWrapper<>(resolvedId)) - .build(); - } - ExtractingSqlSelect connectorColumn = new ExtractingSqlSelect<>(connector.resolveTableId().getTable(), connector.getColumn().getColumn(), Object.class); - return ConnectorSqlSelects.builder() - .preprocessingSelect(connectorColumn) - .build(); + ResolvedColumn column = EntitySchemaAdapter.from(connector.getColumn().resolve()); + BuiltInSelects.ConceptValues operation = new BuiltInSelects.ConceptValues(select.getName(), List.of(column)); + return ResolvedSelectAdapter.connectorSelect(operation, select.getName(), context, preparedSources(select, List.of(connector), context)); } @Override - public ConceptSqlSelects conceptSelect(ConceptColumnSelect select, SelectContext selectContext) { - - // we will do a union distinct on all Connector tables - List connectors; - if (isSingleConnector(selectContext.getTables())) { - // we union the Connector table with itself if there is only 1 Connector - Connector connector = selectContext.getTables().getConnectorTables().get(0).getConnector(); - connectors = List.of(connector, connector); - } - else { - connectors = selectContext.getTables().getConnectorTables().stream().map(ConnectorSqlTables::getConnector).toList(); - } - - SqlNameGenerator nameGenerator = selectContext.getNameGenerator(); - String alias = nameGenerator.legacyOperationName(select.getName()); - QueryStep unionStep = createUnionConnectorConnectorsStep(connectors, select, alias, selectContext); - - FieldWrapper conceptColumnSelect = createConnectorColumnStringAgg(selectContext, unionStep, alias); - Selects unionStepSelects = unionStep.getQualifiedSelects(); - Selects selects = Selects.builder() - .ids(unionStepSelects.getIds()) - .sqlSelect(conceptColumnSelect) - .build(); - - String stringAggCteName = nameGenerator.cteStepName(CONCEPT_COLUMN_STEPS.STRING_AGG, alias); - QueryStep stringAggStep = QueryStep.builder() - .cteName(stringAggCteName) - .selects(selects) - .fromTable(QueryStep.toTableLike(unionStep.getCteName())) - .groupBy(unionStepSelects.getIds().toFields()) - .predecessor(unionStep) - .build(); - - ExtractingSqlSelect finalSelect = conceptColumnSelect.qualify(stringAggStep.getCteName()); - - return ConceptSqlSelects.builder() - .additionalPredecessor(Optional.of(stringAggStep)) - .finalSelect(finalSelect) - .build(); + public ConceptSqlSelects conceptSelect(ConceptColumnSelect select, SelectContext context) { + List connectors = context.getTables().getConnectorTables().stream() + .map(ConnectorSqlTables::getConnector) + .toList(); + List columns = connectors.stream() + .map(Connector::getColumn) + .map(column -> EntitySchemaAdapter.from(column.resolve())) + .toList(); + BuiltInSelects.ConceptValues operation = new BuiltInSelects.ConceptValues(select.getName(), columns); + return ResolvedSelectAdapter.conceptSelect(operation, select.getName(), context, preparedSources(select, connectors, context)); } - private static boolean isSingleConnector(ConceptSqlTables tables) { - return tables.getConnectorTables().size() == 1; - } - - private static QueryStep createUnionConnectorConnectorsStep( - List connectors, + private static List preparedSources( ConceptColumnSelect select, - String alias, - SelectContext selectContext + List connectors, + SelectContext context ) { - List unionSteps = connectors.stream().map(connector -> createConnectorColumnSelectQuery(connector, select, alias, selectContext)).toList(); - String unionedColumnsCteName = selectContext.getNameGenerator().cteStepName(CONCEPT_COLUMN_STEPS.UNIONED_COLUMNS, alias); - return QueryStep.createUnionStep(unionSteps, unionedColumnsCteName, Collections.emptyList(), false); //TODO is false correct here? + if (!select.isAsIds()) { + return List.of(); + } + return connectors.stream() + .map(connector -> preparedSource(connector, (TreeConcept) select.getHolder().findConcept(), context)) + .toList(); } - private static QueryStep createConnectorColumnSelectQuery( + private static SelectConversionContext.ConceptColumnSource preparedSource( Connector connector, - ConceptColumnSelect select, - String alias, - SelectContext selectContext + TreeConcept concept, + SelectContext context ) { - // a ConceptColumn select uses all connectors a Concept has, even if they are not part of the CQConcept - // but if they are, we need to make sure we use the preprocessed and event-filtered table instead of the root table - Optional convertedConnector = selectContext.getTables() - .getConnectorTables() - .stream() - .filter(tables -> Objects.equals(tables.getRootTable(), connector.resolveTableId().getTable())) - .findFirst(); - String tableName = convertedConnector - .map(tables -> tables.cteName(ConceptCteStep.PREPROCESSING)) - .orElse(connector.resolveTableId().getTable()); - - Table connectorTable = DSL.table(DSL.name(tableName)); - SqlIdColumns ids = selectContext.getIds().qualify(connectorTable.getName()); - Field connectorColumn = DSL.field(DSL.name(connectorTable.getName(), connector.getColumn().resolve().getName())); - TableLike sourceTable = connectorTable; - List conditions = Collections.emptyList(); - - if (select.isAsIds() && convertedConnector.isEmpty()) { - TreeConcept concept = (TreeConcept) select.getHolder().findConcept(); - ConceptIdMapping mapping = new ConceptIdMapping(concept, selectContext.getFunctionProvider()); - sourceTable = selectContext.getFunctionProvider().leftJoin( - connectorTable, - mapping.table(), - List.of(mapping.joinCondition(connector)) + ResolvedColumn column = EntitySchemaAdapter.from(connector.getColumn().resolve()); + Optional convertedConnector = context.getTables() instanceof ConceptSqlTables tables + ? tables.getConnectorTables().stream() + .filter(connectorTables -> connectorTables.getConnector() != null) + .filter(connectorTables -> connectorTables.getRootTable().equals(connector.resolveTableId().getTable())) + .findFirst() + : Optional.empty(); + + if (convertedConnector.isPresent()) { + String tableName = convertedConnector.orElseThrow().cteName(ConceptCteStep.PREPROCESSING); + Table table = DSL.table(DSL.name(tableName)); + return new SelectConversionContext.ConceptColumnSource( + tableName, + table, + DSL.field(DSL.name(tableName, column.physicalName())), + List.of() ); - connectorColumn = DSL.coalesce(mapping.resolvedId(), DSL.inline(concept.getLocalId())); - Field rawConceptColumn = DSL.field(DSL.name(connectorTable.getName(), connector.getColumn().resolve().getName())); - conditions = List.of(rawConceptColumn.isNotNull()); } - Field casted = selectContext.getFunctionProvider().cast(connectorColumn, SQLDataType.VARCHAR).as(alias); - FieldWrapper connectorSelect = new FieldWrapper<>(casted); - - Selects selects = Selects.builder() - .ids(ids) - .sqlSelect(connectorSelect) - .build(); - - return QueryStep.builder() - .selects(selects) - .fromTable(sourceTable) - .conditions(conditions) - .build(); - } - - private static FieldWrapper createConnectorColumnStringAgg(SelectContext selectContext, QueryStep unionStep, String alias) { - SqlFunctionProvider functionProvider = selectContext.getFunctionProvider(); - Field unionedColumn = DSL.field(DSL.name(unionStep.getCteName(), alias), String.class); - return new FieldWrapper<>( - functionProvider.stringAggregation(unionedColumn, DSL.toChar(ResultSetProcessor.UNIT_SEPARATOR), List.of(unionedColumn)).as(alias) + ConceptIdMapping mapping = new ConceptIdMapping(concept, context.getFunctionProvider()); + Table connectorTable = SchemaSql.table(column.table()); + TableLike sourceTable = context.getFunctionProvider().leftJoin( + connectorTable, + mapping.table(), + List.of(mapping.joinCondition(connector)) + ); + Field resolvedId = DSL.coalesce(mapping.resolvedId(), DSL.inline(concept.getLocalId())); + Condition rawValuePresent = SchemaSql.field(column, Object.class).isNotNull(); + return new SelectConversionContext.ConceptColumnSource( + connectorTable.getName(), + sourceTable, + resolvedId, + List.of(rawValuePresent) ); } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java index 36c0eb093d..f27c6581d2 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java @@ -9,11 +9,8 @@ public class DateUnionSelectConverter implements SelectConverter selectContext) { - return DaterangeSelectUtil.createForSelect( - select, - (daterange, alias, functionProvider) -> new FieldWrapper<>(functionProvider.dateRangeAggregation(daterange).as(alias)), - selectContext - ); + return ResolvedSelectAdapter.connectorSelect(new com.bakdata.conquery.sql.model.operation.BuiltInSelects.DateUnion( + select.getName(), com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter.from(select)), select.getName(), selectContext); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java deleted file mode 100644 index 49c0cc0624..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java +++ /dev/null @@ -1,171 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model.select; - -import static com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep.PREPROCESSING; -import static org.jooq.impl.DSL.*; - -import java.math.BigDecimal; -import java.sql.Date; -import java.time.temporal.ChronoUnit; -import java.util.List; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; - -import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter; -import com.bakdata.conquery.models.datasets.concepts.select.Select; -import com.bakdata.conquery.models.identifiable.LabeledNamespaceIdentifiable; -import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; -import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; -import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; -import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; -import com.bakdata.conquery.sql.conversion.Context; -import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; -import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; -import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingSelectPreparation; -import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingSelectCompiler; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import com.bakdata.conquery.sql.compiler.ir.SqlTables; -import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; -import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; -import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; -import org.jooq.Condition; -import org.jooq.Field; - -public class DaterangeSelectUtil { - - /** - * Aggregates the daterange of a corresponding {@link DaterangeSelectOrFilter} and applies the respective converted aggregation via - * {@link IntervalPackingSelectCompiler} using an additional predecessor table. - */ - public static ConnectorSqlSelects createForSelect( - S select, - AggregationFunction aggregationFunction, - SelectContext context - ) { - String alias = context.getNameGenerator().legacyOperationName(select.getName()); - SqlFunctionProvider functionProvider = context.getFunctionProvider(); - - ColumnDateRange daterange = functionProvider.forArbitraryDateRange(select).as(alias); - List rootSelects = daterange.toFields().stream() - .map(FieldWrapper::new) - .collect(Collectors.toList()); - - IntervalPackingSelectPreparation preparation = prepareIntervalSelect( - alias, - daterange, - context.getIds(), - context.getTables(), - context - ); - ColumnDateRange qualified = preparation.dateRange(); - FieldWrapper aggregationField = aggregationFunction.apply(qualified, alias, functionProvider); - - QueryStep intervalPackingSelectsStep = IntervalPackingSelectCompiler.compileArbitrarySelect( - preparation.predecessor(), - qualified, - aggregationField, - preparation.tables(), - context.getCompilerDialect() - ); - - SqlTables tables = context.getTables(); - ExtractingSqlSelect finalSelect = aggregationField.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); - - return ConnectorSqlSelects.builder() - .preprocessingSelects(rootSelects) - .additionalPredecessor(Optional.of(intervalPackingSelectsStep)) - .finalSelect(finalSelect) - .build(); - } - - /** - * Aggregates the daterange of a corresponding {@link DaterangeSelectOrFilter} and applies the respective converted aggregation via - * {@link IntervalPackingSelectCompiler} using an additional predecessor table. Finally, the filter condition is created. - */ - public static SqlFilters createForFilter( - DaterangeSelectOrFilter filter, - AggregationFunction aggregationFunction, - Function, WhereCondition> filterFunction, - FilterContext context - ) { - String alias = context.getNameGenerator().legacyOperationName(((LabeledNamespaceIdentifiable) filter).getName()); - SqlFunctionProvider functionProvider = context.getCompilerDialect().getFunctionProvider(); - - ColumnDateRange daterange = functionProvider.forArbitraryDateRange(filter).as(alias); - List rootSelects = daterange.toFields().stream() - .map(FieldWrapper::new) - .collect(Collectors.toList()); - - IntervalPackingSelectPreparation preparation = prepareIntervalSelect( - alias, - daterange, - context.getIds(), - context.getTables(), - context - ); - ColumnDateRange qualified = preparation.dateRange(); - FieldWrapper aggregationField = aggregationFunction.apply(qualified, alias, functionProvider); - - QueryStep intervalPackingSelectsStep = IntervalPackingSelectCompiler.compileArbitrarySelect( - preparation.predecessor(), - qualified, - aggregationField, - preparation.tables(), - context.getCompilerDialect() - ); - - ConnectorSqlSelects sqlSelects = ConnectorSqlSelects.builder() - .preprocessingSelects(rootSelects) - .additionalPredecessor(Optional.of(intervalPackingSelectsStep)) - .build(); - - SqlTables tables = context.getTables(); - Field qualifiedAggregationField = aggregationField.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)).select(); - WhereClauses whereClauses = WhereClauses.builder().groupFilter(filterFunction.apply(qualifiedAggregationField)).build(); - - return new SqlFilters(sqlSelects, whereClauses); - } - - public static FieldWrapper createDurationSumSqlSelect(String alias, ColumnDateRange validityDate, SqlFunctionProvider functionProvider) { - Field dateDistanceInDays = functionProvider.dateDistance(ChronoUnit.DAYS, validityDate.getStart(), validityDate.getEnd()); - Field durationSum = sum(when(containsInfinityDate(validityDate, functionProvider), inline(null, Integer.class)) - .otherwise(dateDistanceInDays) - ) - .as(alias); - return new FieldWrapper<>(durationSum); - } - - private static Condition containsInfinityDate(ColumnDateRange validityDate, SqlFunctionProvider functionProvider) { - Field negativeInfinity = functionProvider.getMinDateExpression(); - Field positiveInfinity = functionProvider.getMaxDateExpression(); - - return validityDate.getStart().eq(negativeInfinity).or(validityDate.getEnd().eq(positiveInfinity)); - } - - private static IntervalPackingSelectPreparation prepareIntervalSelect( - String alias, - ColumnDateRange daterange, - SqlIdColumns ids, - SqlTables tables, - Context context - ) { - return IntervalPackingSelectCompiler.prepareArbitrarySelect( - alias, - tables.cteName(PREPROCESSING), - ids, - daterange, - context.getCompilerDialect(), - context.getNameGenerator() - ); - } - - @FunctionalInterface - public interface AggregationFunction { - FieldWrapper apply(ColumnDateRange daterange, String alias, SqlFunctionProvider functionProvider); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java index 75711c33e1..ab9488c38f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java @@ -1,136 +1,13 @@ package com.bakdata.conquery.sql.conversion.model.select; -import java.util.List; -import java.util.Optional; - import com.bakdata.conquery.models.datasets.concepts.select.connector.DistinctSelect; -import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; -import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; -import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; -import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.compiler.ir.CteStep; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import com.bakdata.conquery.sql.execution.ResultSetProcessor; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import org.jooq.Field; -import org.jooq.impl.DSL; -import org.jooq.impl.SQLDataType; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; -/** - * 
- *  The two additional CTEs this aggregator creates:
- * 	
    
- * 	    
  1. 
- * 	        Select distinct values of a column.
- *            {@code
- * 	        	"distinct" as (
- *     				select distinct "pid", "column"
- *     				from "preprocessing"
- *  			)
- *            }
- * 	    
    2. 
- * 	    
  2. 
- * 	        String agg all distinct values of the column.
- *            {@code
- * 	        "aggregated" as (
- *    			 select
- *    			   "select-1-distinct"."pid",
- *    			   string_agg(cast("column" as varchar), cast(' ' as varchar) ) as "select-1"
- *    			 from "distinct"
- *    			 group by "pid"
- *   			)
- *            }
- * 	    
    3. 
- * 	

- *
- */ public class DistinctSelectConverter implements SelectConverter { - - @Getter - @RequiredArgsConstructor - private enum DistinctSelectCteStep implements CteStep { - - DISTINCT_SELECT("distinct", null), - STRING_AGG("aggregated", DISTINCT_SELECT); - - private final String suffix; - private final DistinctSelectCteStep predecessor; - } - @Override - public ConnectorSqlSelects connectorSelect(DistinctSelect distinctSelect, SelectContext selectContext) { - - String alias = selectContext.getNameGenerator().legacyOperationName(distinctSelect.getName()); - - ConnectorSqlTables tables = selectContext.getTables(); - SingleColumnSqlSelect preprocessingSelect = MappableSingleColumnSelect.getSubstringSelect(distinctSelect.getColumn().get(), distinctSelect.getSubstringRange(), selectContext, alias); - - QueryStep distinctSelectCte = createDistinctSelectCte(preprocessingSelect, alias, selectContext); - QueryStep aggregatedCte = createAggregationCte(selectContext, preprocessingSelect, distinctSelectCte, alias); - - SingleColumnSqlSelect finalSelect = preprocessingSelect.qualify(tables.cteName(ConceptCteStep.AGGREGATION_FILTER)); - - return ConnectorSqlSelects.builder() - .preprocessingSelect(preprocessingSelect) - .additionalPredecessor(Optional.of(aggregatedCte)) - .finalSelect(finalSelect) - .build(); - } - - private static QueryStep createAggregationCte( - SelectContext selectContext, - SingleColumnSqlSelect preprocessingSelect, - QueryStep distinctSelectCte, - String alias - ) { - SqlFunctionProvider functionProvider = selectContext.getFunctionProvider(); - Field castedColumn = functionProvider.cast(preprocessingSelect.qualify(distinctSelectCte.getCteName()).select(), SQLDataType.VARCHAR); - Field aggregatedColumn = functionProvider.stringAggregation(castedColumn, DSL.toChar(ResultSetProcessor.UNIT_SEPARATOR), List.of(castedColumn)) - .as(alias); - - SqlIdColumns ids = distinctSelectCte.getQualifiedSelects().getIds(); - - Selects selects = Selects.builder() - .ids(ids) - .sqlSelect(new FieldWrapper<>(aggregatedColumn)) - .build(); - - return QueryStep.builder() - .cteName(selectContext.getNameGenerator().cteStepName(DistinctSelectCteStep.STRING_AGG, alias)) - .selects(selects) - .fromTable(QueryStep.toTableLike(distinctSelectCte.getCteName())) - .groupBy(ids.toFields()) - .predecessor(distinctSelectCte) - .build(); - } - - private static QueryStep createDistinctSelectCte( - SingleColumnSqlSelect preprocessingSelect, - String alias, - SelectContext selectContext - ) { - // values to aggregate must be event-filtered first - String preprocessingTable = selectContext.getTables().cteName(ConceptCteStep.PREPROCESSING); - SingleColumnSqlSelect qualified = preprocessingSelect.qualify(preprocessingTable); - - SqlIdColumns ids = selectContext.getIds().qualify(preprocessingTable); - - Selects selects = Selects.builder() - .ids(ids) - .sqlSelect(qualified) - .build(); - - return QueryStep.builder() - .cteName(selectContext.getNameGenerator().cteStepName(DistinctSelectCteStep.DISTINCT_SELECT, alias)) - .selectDistinct(true) - .selects(selects) - .fromTable(QueryStep.toTableLike(preprocessingTable)) - .build(); + public ConnectorSqlSelects connectorSelect(DistinctSelect select, SelectContext selectContext) { + return ResolvedSelectAdapter.values(select, BuiltInSelects.ValueOperation.DISTINCT, selectContext); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java index b6cfbe3ee1..89952c97bd 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java @@ -3,51 +3,18 @@ import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDateUnionSelect; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptSqlSelects; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; -import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; -import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; -import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.google.common.base.Preconditions; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; public class EventDateUnionSelectConverter implements SelectConverter { - @Override - public ConnectorSqlSelects connectorSelect(EventDateUnionSelect select, SelectContext selectContext) { - - FieldWrapper stringAggregation = createEventDateUnionAggregation(select, selectContext); - ExtractingSqlSelect finalSelect = stringAggregation.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); - - return ConnectorSqlSelects.builder() - .eventDateSelect(stringAggregation) - .finalSelect(finalSelect) - .build(); + public ConnectorSqlSelects connectorSelect(EventDateUnionSelect select, SelectContext context) { + return ResolvedSelectAdapter.connectorSelect(new BuiltInSelects.EventDateUnion(select.getName()), select.getName(), context); } @Override - public ConceptSqlSelects conceptSelect(EventDateUnionSelect select, SelectContext selectContext) { - - FieldWrapper stringAggregation = createEventDateUnionAggregation(select, selectContext); - ExtractingSqlSelect finalSelect = stringAggregation.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.UNIVERSAL_SELECTS)); - - return ConceptSqlSelects.builder() - .eventDateSelect(stringAggregation) - .finalSelect(finalSelect) - .build(); + public ConceptSqlSelects conceptSelect(EventDateUnionSelect select, SelectContext context) { + return ResolvedSelectAdapter.conceptSelect(new BuiltInSelects.EventDateUnion(select.getName()), select.getName(), context); } - - private static FieldWrapper createEventDateUnionAggregation(EventDateUnionSelect select, SelectContext selectContext) { - - Preconditions.checkArgument(selectContext.getValidityDate().isPresent(), "Can't convert an EventDateUnionSelect without a validity date being present"); - ColumnDateRange validityDate = selectContext.getValidityDate().get(); - - SqlFunctionProvider functionProvider = selectContext.getFunctionProvider(); - String alias = selectContext.getNameGenerator().legacyOperationName(select.getName()); - - ColumnDateRange qualified = validityDate.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.INTERVAL_PACKING_SELECTS)); - return new FieldWrapper<>(functionProvider.dateRangeAggregation(qualified).as(alias)); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java index 9421c33ad3..84e21254fb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java @@ -1,72 +1,20 @@ package com.bakdata.conquery.sql.conversion.model.select; -import java.math.BigDecimal; -import java.sql.Date; -import java.time.temporal.ChronoUnit; - import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDurationSumSelect; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptSqlSelects; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; -import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; -import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; -import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.google.common.base.Preconditions; -import org.jooq.Condition; -import org.jooq.Field; -import org.jooq.impl.DSL; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; public class EventDurationSumSelectConverter implements SelectConverter { - @Override - public ConnectorSqlSelects connectorSelect(EventDurationSumSelect select, SelectContext selectContext) { - - FieldWrapper stringAggregation = createEventDurationSumAggregation(select, selectContext); - ExtractingSqlSelect finalSelect = stringAggregation.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); - - return ConnectorSqlSelects.builder() - .eventDateSelect(stringAggregation) - .finalSelect(finalSelect) - .build(); + public ConnectorSqlSelects connectorSelect(EventDurationSumSelect select, SelectContext context) { + return ResolvedSelectAdapter.connectorSelect(new BuiltInSelects.EventDurationSum(select.getName()), select.getName(), context); } @Override - public ConceptSqlSelects conceptSelect(EventDurationSumSelect select, SelectContext selectContext) { - - FieldWrapper stringAggregation = createEventDurationSumAggregation(select, selectContext); - ExtractingSqlSelect finalSelect = stringAggregation.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.UNIVERSAL_SELECTS)); - - return ConceptSqlSelects.builder() - .eventDateSelect(stringAggregation) - .finalSelect(finalSelect) - .build(); - } - - private FieldWrapper createEventDurationSumAggregation(EventDurationSumSelect select, SelectContext selectContext) { - - Preconditions.checkArgument(selectContext.getValidityDate().isPresent(), "Can't convert an EventDateUnionSelect without a validity date being present"); - String predecessorCteName = selectContext.getTables().getPredecessor(ConceptCteStep.INTERVAL_PACKING_SELECTS); - ColumnDateRange qualified = selectContext.getValidityDate().get().qualify(predecessorCteName); - ColumnDateRange asDualColumn = selectContext.getFunctionProvider().toDualColumn(qualified); - - SqlFunctionProvider functionProvider = selectContext.getFunctionProvider(); - String alias = selectContext.getNameGenerator().legacyOperationName(select.getName()); - - Field durationSum = DSL.sum( - DSL.when(containsInfinityDate(asDualColumn, functionProvider), DSL.inline(null, Integer.class)) - .otherwise(functionProvider.dateDistance(ChronoUnit.DAYS, asDualColumn.getStart(), asDualColumn.getEnd())) - ) - .as(alias); - return new FieldWrapper<>(durationSum); + public ConceptSqlSelects conceptSelect(EventDurationSumSelect select, SelectContext context) { + return ResolvedSelectAdapter.conceptSelect(new BuiltInSelects.EventDurationSum(select.getName()), select.getName(), context); } - - private static Condition containsInfinityDate(ColumnDateRange validityDate, SqlFunctionProvider functionProvider) { - Field negativeInfinity = functionProvider.getMinDateExpression(); - Field positiveInfinity = functionProvider.getMaxDateExpression(); - return validityDate.getStart().eq(negativeInfinity).or(validityDate.getEnd().eq(positiveInfinity)); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java index 9d0c7049c8..77918d9555 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java @@ -3,30 +3,18 @@ import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.ExistsSelect; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptSqlSelects; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; -import com.bakdata.conquery.sql.compiler.ir.select.ExistsSqlSelect; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; public class ExistsSelectConverter implements SelectConverter { - @Override - public ConnectorSqlSelects connectorSelect(ExistsSelect select, SelectContext selectContext) { - ExistsSqlSelect existsSqlSelect = createExistsSelect(select, selectContext); - return ConnectorSqlSelects.builder() - .finalSelect(existsSqlSelect) - .build(); + public ConnectorSqlSelects connectorSelect(ExistsSelect select, SelectContext context) { + return ResolvedSelectAdapter.connectorSelect(new BuiltInSelects.Exists(select.getName()), select.getName(), context); } @Override - public ConceptSqlSelects conceptSelect(ExistsSelect select, SelectContext selectContext) { - ExistsSqlSelect existsSqlSelect = createExistsSelect(select, selectContext); - return ConceptSqlSelects.builder() - .finalSelect(existsSqlSelect) - .build(); - } - - private static ExistsSqlSelect createExistsSelect(ExistsSelect select, SelectContext selectContext) { - String alias = selectContext.getNameGenerator().legacyOperationName(select.getName()); - return ExistsSqlSelect.withAlias(alias); + public ConceptSqlSelects conceptSelect(ExistsSelect select, SelectContext context) { + return ResolvedSelectAdapter.conceptSelect(new BuiltInSelects.Exists(select.getName()), select.getName(), context); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java index 819bcf7337..905c6e9da9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java @@ -3,19 +3,12 @@ import com.bakdata.conquery.models.datasets.concepts.select.connector.FirstValueSelect; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; -import org.jooq.Field; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; public class FirstValueSelectConverter implements SelectConverter { @Override public ConnectorSqlSelects connectorSelect(FirstValueSelect select, SelectContext selectContext) { - return ValueSelectUtil.createValueSelect( - select.getColumn().resolve(), - selectContext.getNameGenerator().legacyOperationName(select.getName()), - Field::asc, - select.getSubstringRange(), - selectContext - ); + return ResolvedSelectAdapter.values(select, BuiltInSelects.ValueOperation.FIRST, selectContext); } - -} +} \ No newline at end of file diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java index 66b25d136a..188dc7c6fa 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java @@ -3,16 +3,12 @@ import com.bakdata.conquery.models.datasets.concepts.select.connector.LastValueSelect; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; -import org.jooq.Field; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; public class LastValueSelectConverter implements SelectConverter { @Override public ConnectorSqlSelects connectorSelect(LastValueSelect select, SelectContext selectContext) { - return ValueSelectUtil.createValueSelect( - select.getColumn().resolve(), - selectContext.getNameGenerator().legacyOperationName(select.getName()), - Field::desc, select.getSubstringRange(), selectContext - ); + return ResolvedSelectAdapter.values(select, BuiltInSelects.ValueOperation.LAST, selectContext); } -} +} \ No newline at end of file diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java index 0ab99f2952..d732ec3d18 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java @@ -2,34 +2,16 @@ import com.bakdata.conquery.models.datasets.concepts.select.connector.RandomValueSelect; import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; -import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; -import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; -import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; + + + import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; -import org.jooq.Field; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; public class RandomValueSelectConverter implements SelectConverter { @Override public ConnectorSqlSelects connectorSelect(RandomValueSelect select, SelectContext selectContext) { - - ConnectorSqlTables tables = selectContext.getTables(); - - String rootTableName = tables.getRootTable(); - String columnName = select.getColumn().getColumn(); - ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(rootTableName, columnName, Object.class); - - String alias = selectContext.getNameGenerator().legacyOperationName(select.getName()); - Field qualifiedRootSelect = rootSelect.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT)).select(); - Field firstAggregation = selectContext.getFunctionProvider().random(qualifiedRootSelect).as(alias); - FieldWrapper firstAggregationSqlSelect = new FieldWrapper<>(firstAggregation, columnName); - - ExtractingSqlSelect finalSelect = firstAggregationSqlSelect.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); - - return ConnectorSqlSelects.builder() - .preprocessingSelect(rootSelect) - .aggregationSelect(firstAggregationSqlSelect) - .finalSelect(finalSelect) - .build(); + return ResolvedSelectAdapter.values(select, BuiltInSelects.ValueOperation.RANDOM, selectContext); } -} +} \ No newline at end of file diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ResolvedSelectAdapter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ResolvedSelectAdapter.java new file mode 100644 index 0000000000..9c0b757df4 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ResolvedSelectAdapter.java @@ -0,0 +1,89 @@ +package com.bakdata.conquery.sql.conversion.model.select; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; +import com.bakdata.conquery.sql.compiler.conversion.operation.ResolvedSelectConverter; +import com.bakdata.conquery.sql.compiler.conversion.operation.SelectConversionContext; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; +import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; +import com.bakdata.conquery.sql.model.operation.ResolvedSelect; +import com.bakdata.conquery.sql.model.range.SubstringRange; + +/** Resolves application columns and ranges before invoking the shared select compiler. */ +public final class ResolvedSelectAdapter { + private static final ResolvedSelectConverter CONVERTER = new ResolvedSelectConverter(); + + private ResolvedSelectAdapter() { + } + + public static SelectConversionContext context(String name, SelectContext context) { + return context(name, context, List.of()); + } + + public static SelectConversionContext context( + String name, + SelectContext context, + List conceptColumnSources + ) { + Map conceptColumnTables = context.getTables() instanceof ConceptSqlTables tables + ? tables.getConnectorTables().stream() + .filter(connectorTables -> connectorTables.getConnector() != null) + .collect(Collectors.toMap( + connectorTables -> connectorTables.getConnector().resolveTableId().getTable(), + connectorTables -> connectorTables.cteName(ConceptCteStep.PREPROCESSING), + (first, ignored) -> first + )) + : Map.of(); + return new SelectConversionContext(context.getCompilerDialect(), context.getNameGenerator(), context.getTables(), context.getIds(), + context.getValidityDate(), context.getNameGenerator().legacyOperationName(name), conceptColumnTables, conceptColumnSources); + } + + public static ConnectorSqlSelects connectorSelect(ResolvedSelect select, String name, SelectContext context) { + return CONVERTER.convert(select, context(name, context)); + } + + public static ConceptSqlSelects conceptSelect(ResolvedSelect select, String name, SelectContext context) { + return CONVERTER.conceptSelect(select, context(name, context)); + } + + public static ConnectorSqlSelects connectorSelect( + ResolvedSelect select, + String name, + SelectContext context, + List conceptColumnSources + ) { + return CONVERTER.convert(select, context(name, context, conceptColumnSources)); + } + + public static ConceptSqlSelects conceptSelect( + ResolvedSelect select, + String name, + SelectContext context, + List conceptColumnSources + ) { + return CONVERTER.conceptSelect(select, context(name, context, conceptColumnSources)); + } + + static ConnectorSqlSelects values(MappableSingleColumnSelect select, BuiltInSelects.ValueOperation operation, SelectContext context) { + return CONVERTER.convert(new BuiltInSelects.Values(select.getName(), EntitySchemaAdapter.from(select.getColumn().resolve()), + operation, substring(select.getSubstringRange())), + new SelectConversionContext(context.getCompilerDialect(), context.getNameGenerator(), context.getTables(), context.getIds(), + context.getValidityDate(), context.getNameGenerator().legacyOperationName(select.getName()))); + } + + public static Optional substring(Range.IntegerRange range) { + if (range == null || range.isAll()) { + return Optional.empty(); + } + return Optional.of(new SubstringRange(range.getMin() == null ? 0 : range.getMin(), Optional.ofNullable(range.getMax()))); + } +} diff --git a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionAggregationTest.java b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionAggregationTest.java index ce09012ba6..1a30f056c3 100644 --- a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionAggregationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionAggregationTest.java @@ -82,7 +82,7 @@ void shouldCountDistinctColumnRatherThanConfiguredDistinctKeys() { render(result.getAggregationSelects().getFirst().toFields().getFirst())); } - private static SelectContext context() { + static SelectContext context() { return context(new HanaDialectBundle()); } @@ -98,6 +98,26 @@ void shouldSumQuartersPerEventForDatePairs() { assertTrue(result.getAdditionalPredecessor().isEmpty()); } + @Test + void shouldKeepSingleColumnQuarterSqlForDateRangeType() { + CountQuartersSelect select = new CountQuartersSelect(); + select.setName("quarters"); + select.setColumn(column("date", MajorTypeId.DATE_RANGE)); + var result = new CountQuartersSqlAggregator().connectorSelect(select, context()); + assertEquals(1, result.getPreprocessingSelects().size()); + assertTrue(render(result.getAggregationSelects().getFirst().toFields().getFirst()).contains("count(distinct")); + } + + @Test + void shouldKeepSingleColumnDurationSqlForDateRangeType() { + DurationSumSelect select = new DurationSumSelect(); + select.setName("duration"); + select.setColumn(column("date", MajorTypeId.DATE_RANGE)); + var result = new DurationSumSqlAggregator().connectorSelect(select, context()); + assertEquals(2, result.getPreprocessingSelects().size()); + assertTrue(render(result.getPreprocessingSelects().getFirst().toFields().getFirst()).startsWith("coalesce(\"events\".\"date\",")); + } + @Test void shouldPackDurationIntervalsWithoutApplyingDistinctKeys() { DurationSumSelect select = new DurationSumSelect(); @@ -113,7 +133,7 @@ void shouldPackDurationIntervalsWithoutApplyingDistinctKeys() { assertEquals("\"aggregation\".\"duration-1\"", render(result.getFinalSelects().getFirst().toFields().getFirst())); } - private static SelectContext context(LegacyCompilerDialect dialect) { + static SelectContext context(LegacyCompilerDialect dialect) { var conversion = ConversionContext.builder() .compilerDialect(dialect).stratificationFunctions(dialect.getStratificationFunctions()) .nameGenerator(new SqlNameGenerator(127)).build(); @@ -126,7 +146,7 @@ private static SelectContext context(LegacyCompilerDialect d Optional.empty(), new ConnectorSqlTables(null, plan), conversion); } - private static ColumnId column(String name, MajorTypeId type) { + static ColumnId column(String name, MajorTypeId type) { Table table = new Table(); table.setName("events"); table.setDataset(new DatasetId("test")); @@ -143,7 +163,7 @@ public Column get() { }; } - private static String render(Field field) { + static String render(Field field) { return DSL.using(SQLDialect.POSTGRES).renderInlined(DSL.select(field)).substring("select ".length()).toLowerCase(Locale.ROOT); } } diff --git a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionConceptSelectTest.java b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionConceptSelectTest.java new file mode 100644 index 0000000000..79d1d6ff05 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionConceptSelectTest.java @@ -0,0 +1,95 @@ +package com.bakdata.conquery.sql.conversion; + +import static com.bakdata.conquery.sql.conversion.SqlExtractionAggregationTest.*; +import static org.junit.jupiter.api.Assertions.*; + +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeConnector; +import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; +import com.bakdata.conquery.models.datasets.concepts.select.concept.ConceptColumnSelect; +import com.bakdata.conquery.models.events.MajorTypeId; +import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; +import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; +import com.bakdata.conquery.sql.conversion.model.select.ConceptColumnSelectConverter; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; +import org.junit.jupiter.api.Test; + +class SqlExtractionConceptSelectTest { + @Test + void shouldSelectResolvedConceptIdsFromPreparedMapping() { + var base = context(); + var connector = new ConceptTreeConnector(); + connector.setColumn(column("value", MajorTypeId.STRING)); + var concept = concept(connector); + var tables = new ConnectorSqlTables(connector, base.getTables().getPlan()); + var context = SelectContext.create(base.getIds(), Optional.empty(), tables, base.getConversionContext()); + var select = new ConceptColumnSelect(); + select.setName("concept"); + select.setAsIds(true); + select.setHolder(concept); + + var result = new ConceptColumnSelectConverter().connectorSelect(select, context); + + assertEquals("coalesce(\"test.concept_ids\".\"resolved_id\", 0) as \"value\"", + render(result.getPreprocessingSelects().getFirst().toFields().getFirst())); + } + + @Test + void shouldSelfUnionOneConnectorDistinctlyBeforeStringAggregation() { + var base = context(); + var connector = new ConceptTreeConnector(); + connector.setColumn(column("value", MajorTypeId.STRING)); + var concept = concept(connector); + var tables = new ConceptSqlTables(base.getTables(), List.of(new ConnectorSqlTables(connector, base.getTables().getPlan()))); + var context = SelectContext.create(base.getIds(), Optional.empty(), tables, base.getConversionContext()); + var select = new ConceptColumnSelect(); + select.setName("concept"); + select.setHolder(concept); + var result = new ConceptColumnSelectConverter().conceptSelect(select, context); + var aggregate = result.getAdditionalPredecessor().orElseThrow(); + var union = aggregate.getPredecessors().getFirst(); + assertEquals(1, union.getUnion().size()); + assertFalse(union.isUnionAll()); + assertTrue(render(union.getSelects().getSqlSelects().getFirst().toFields().getFirst()).contains("\"preprocessing\".\"value\"")); + assertEquals("concept-1-concept_column_aggregated", aggregate.getCteName()); + } + + @Test + void shouldUseOnlyConnectorsFromConvertedCqTables() { + var base = context(); + var convertedConnector = new ConceptTreeConnector(); + convertedConnector.setColumn(column("value", MajorTypeId.STRING)); + var unconvertedConnector = new ConceptTreeConnector(); + unconvertedConnector.setColumn(column("other_value", MajorTypeId.STRING)); + var concept = concept(convertedConnector, unconvertedConnector); + var tables = new ConceptSqlTables(base.getTables(), List.of(new ConnectorSqlTables(convertedConnector, base.getTables().getPlan()))); + var context = SelectContext.create(base.getIds(), Optional.empty(), tables, base.getConversionContext()); + var select = new ConceptColumnSelect(); + select.setName("concept"); + select.setHolder(concept); + + var result = new ConceptColumnSelectConverter().conceptSelect(select, context); + var union = result.getAdditionalPredecessor().orElseThrow().getPredecessors().getFirst(); + + assertFalse(render(union.getUnion().getFirst().getSelects().getSqlSelects().getFirst().toFields().getFirst()) + .contains("other_value")); + } + + private static TreeConcept concept(ConceptTreeConnector... connectors) { + TreeConcept concept = new TreeConcept() { + @Override + public ConceptId getId() { + return new ConceptId(new DatasetId("test"), "concept"); + } + }; + concept.setConnectors(List.of(connectors)); + for (ConceptTreeConnector connector : connectors) { + connector.setConcept(concept); + } + return concept; + } +} diff --git a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionDateSelectTest.java b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionDateSelectTest.java new file mode 100644 index 0000000000..81fb231ffa --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionDateSelectTest.java @@ -0,0 +1,99 @@ +package com.bakdata.conquery.sql.conversion; + +import static com.bakdata.conquery.sql.conversion.SqlExtractionAggregationTest.*; +import static org.junit.jupiter.api.Assertions.*; + +import java.sql.Date; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.Map; +import java.util.Optional; + +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DateUnionSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DateDistanceSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.FlagSelect; +import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDateUnionSelect; +import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDurationSumSelect; +import com.bakdata.conquery.models.events.MajorTypeId; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.conversion.model.aggregator.DateDistanceSqlAggregator; +import com.bakdata.conquery.sql.conversion.model.aggregator.FlagSqlAggregator; +import com.bakdata.conquery.sql.conversion.model.select.DateUnionSelectConverter; +import com.bakdata.conquery.sql.conversion.model.select.EventDateUnionSelectConverter; +import com.bakdata.conquery.sql.conversion.model.select.EventDurationSumSelectConverter; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Test; + +class SqlExtractionDateSelectTest { + @Test + void shouldUsePerRowStratificationEndForDistance() { + var base = context(); + var dates = ColumnDateRange.of(DSL.field(DSL.name("start"), Date.class), DSL.field(DSL.name("end"), Date.class)).as("window"); + var step = com.bakdata.conquery.sql.compiler.ir.QueryStep.builder().cteName("stratification") + .selects(com.bakdata.conquery.sql.compiler.ir.Selects.builder().ids(base.getIds()).stratificationDate(Optional.of(dates)).build()).build(); + var context = SelectContext.create(base.getIds(), base.getValidityDate(), base.getTables(), base.getConversionContext().withStratificationTable(step)); + var select = new DateDistanceSelect(column("date", MajorTypeId.DATE)); + select.setName("distance"); + var result = new DateDistanceSqlAggregator().connectorSelect(select, context); + String expression = render(result.getPreprocessingSelects().getFirst().toFields().getFirst()); + assertTrue(expression.contains("\"stratification\".\"window_end\"")); + assertTrue(expression.contains("-1")); + } + @Test + void shouldPackDateUnionBeforeAggregation() { + var select = new DateUnionSelect(); + select.setName("dates"); + select.setStartColumn(column("start", MajorTypeId.DATE)); + select.setEndColumn(column("end", MajorTypeId.DATE)); + var result = new DateUnionSelectConverter().connectorSelect(select, context()); + assertEquals(2, result.getPreprocessingSelects().size()); + assertEquals("dates-1-interval_packing_selects", result.getAdditionalPredecessor().orElseThrow().getCteName()); + assertTrue(result.getAggregationSelects().isEmpty()); + assertEquals("\"aggregation\".\"dates-1\"", render(result.getFinalSelects().getFirst().toFields().getFirst())); + } + + @Test + void shouldKeepEventDatesInEventAggregation() { + var base = context(); + var dates = ColumnDateRange.of(DSL.field(DSL.name("start"), Date.class), DSL.field(DSL.name("end"), Date.class)); + var context = SelectContext.create(base.getIds(), Optional.of(dates), base.getTables(), base.getConversionContext()); + var union = new EventDateUnionSelect(); + union.setName("union"); + var unionResult = new EventDateUnionSelectConverter().connectorSelect(union, context); + assertEquals(1, unionResult.getEventDateSelects().size()); + assertTrue(unionResult.getAggregationSelects().isEmpty()); + var duration = EventDurationSumSelect.create("duration"); + duration.setName("duration"); + var durationResult = new EventDurationSumSelectConverter().connectorSelect(duration, context); + assertEquals(1, durationResult.getEventDateSelects().size()); + String expression = render(durationResult.getEventDateSelects().getFirst().toFields().getFirst()); + assertTrue(expression.contains("sum(case when")); + assertTrue(expression.contains("then null")); + } + + @Test + void shouldCalculateMinimumDistanceToFrozenCurrentDate() { + var base = context(); + var conversion = base.getConversionContext().withClock(Clock.fixed(Instant.parse("2020-06-15T12:00:00Z"), ZoneOffset.UTC)); + var context = SelectContext.create(base.getIds(), base.getValidityDate(), base.getTables(), conversion); + var select = new DateDistanceSelect(column("date", MajorTypeId.DATE)); + select.setName("distance"); + var result = new DateDistanceSqlAggregator().connectorSelect(select, context); + assertTrue(render(result.getPreprocessingSelects().getFirst().toFields().getFirst()).contains("2020-06-15")); + assertEquals("min(\"preprocessing\".\"distance-1\") as \"distance-1\"", render(result.getAggregationSelects().getFirst().toFields().getFirst())); + } + + @Test + void shouldKeepFlagNamesAndEmptyStringFallback() { + var select = new FlagSelect(Map.of("A", column("a", MajorTypeId.BOOLEAN), "B", column("b", MajorTypeId.BOOLEAN))); + select.setName("flags"); + var result = new FlagSqlAggregator().connectorSelect(select, context()); + assertEquals(2, result.getPreprocessingSelects().size()); + String expression = render(result.getAggregationSelects().getFirst().toFields().getFirst()); + assertTrue(expression.contains("then 'a'")); + assertTrue(expression.contains("then 'b'")); + assertTrue(expression.contains("else ''")); + } +} diff --git a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionSelectTest.java b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionSelectTest.java new file mode 100644 index 0000000000..7ce260aa25 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionSelectTest.java @@ -0,0 +1,122 @@ +package com.bakdata.conquery.sql.conversion; + +import static com.bakdata.conquery.sql.conversion.SqlExtractionAggregationTest.*; +import static org.junit.jupiter.api.Assertions.*; + +import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.models.datasets.concepts.select.connector.FirstValueSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.LastValueSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.RandomValueSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.DistinctSelect; +import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.ExistsSelect; +import com.bakdata.conquery.models.events.MajorTypeId; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; +import com.bakdata.conquery.sql.conversion.model.select.FirstValueSelectConverter; +import com.bakdata.conquery.sql.conversion.model.select.LastValueSelectConverter; +import com.bakdata.conquery.sql.conversion.model.select.RandomValueSelectConverter; +import com.bakdata.conquery.sql.conversion.model.select.DistinctSelectConverter; +import com.bakdata.conquery.sql.conversion.model.select.ExistsSelectConverter; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; +import com.bakdata.conquery.sql.conversion.dialect.clickhouse.ClickhouseDistinctSelectConverter; +import com.bakdata.conquery.sql.conversion.dialect.clickhouse.ClickhouseDialectBundle; +import org.junit.jupiter.api.Test; + +class SqlExtractionSelectTest { + + @Test + void shouldDelegateExistsForConnectorAndConceptProjections() { + var select = new ExistsSelect(); + select.setName("exists"); + var connectorResult = new ExistsSelectConverter().connectorSelect(select, context()); + var base = context(); + var conceptContext = SelectContext.create(base.getIds(), java.util.Optional.empty(), + new ConceptSqlTables(base.getTables(), java.util.List.of()), base.getConversionContext()); + var conceptResult = new ExistsSelectConverter().conceptSelect(select, conceptContext); + + assertEquals("1 as \"exists-1\"", render(connectorResult.getFinalSelects().getFirst().toFields().getFirst())); + assertEquals("1 as \"exists-1\"", render(conceptResult.getFinalSelects().getFirst().toFields().getFirst())); + assertEquals(1, connectorResult.getFinalSelects().size()); + assertEquals(1, conceptResult.getFinalSelects().size()); + } + + @Test + void shouldOrderLastByBothValidityBoundsWithNullsLast() { + var base = context(); + var dates = com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange.of( + org.jooq.impl.DSL.field(org.jooq.impl.DSL.name("start"), java.sql.Date.class), + org.jooq.impl.DSL.field(org.jooq.impl.DSL.name("end"), java.sql.Date.class)).as("valid"); + var context = com.bakdata.conquery.sql.conversion.model.select.SelectContext.create( + base.getIds(), java.util.Optional.of(dates), base.getTables(), base.getConversionContext()); + var select = new LastValueSelect(column("value", MajorTypeId.STRING), null, null); + select.setName("value"); + var result = new LastValueSelectConverter().connectorSelect(select, context); + var rowNumber = result.getAdditionalPredecessor().orElseThrow().getPredecessors().getFirst(); + String expression = render(rowNumber.getSelects().getSqlSelects().getLast().toFields().getFirst()); + assertTrue(expression.contains("\"preprocessing\".\"valid_start\"")); + assertTrue(expression.contains("\"preprocessing\".\"valid_end\"")); + assertEquals(2, expression.split("desc nulls last", -1).length - 1); + } + + @Test + void shouldPreserveDistinctSeparatePredecessor() { + var select = new DistinctSelect(column("value", MajorTypeId.STRING), null, new Range.IntegerRange(2, 5)); + select.setName("value"); + var result = new DistinctSelectConverter().connectorSelect(select, context()); + var aggregated = result.getAdditionalPredecessor().orElseThrow(); + assertEquals("value-1-aggregated", aggregated.getCteName()); + assertTrue(aggregated.getPredecessors().getFirst().isSelectDistinct()); + assertTrue(result.getAggregationSelects().isEmpty()); + assertEquals("substring(\"events\".\"value\", 3, 3) as \"value-1\"", render(result.getPreprocessingSelects().getFirst().toFields().getFirst())); + assertTrue(render(aggregated.getSelects().getSqlSelects().getFirst().toFields().getFirst()).contains("string_agg(")); + } + + @Test + void shouldPreserveClickhouseDistinctFilteringAndGrouping() { + var select = new DistinctSelect(column("value", MajorTypeId.STRING), null, null); + select.setName("value"); + var result = new ClickhouseDistinctSelectConverter().connectorSelect(select, context(new ClickhouseDialectBundle())); + assertEquals("arrayfilter(x -> x <> '' and x is not null, groupuniqarray(\"preprocessing\".\"value-1\")) as \"value-1\"", + render(result.getAggregationSelects().getFirst().toFields().getFirst())); + assertEquals("\"aggregation\".\"value-1\"", render(result.getFinalSelects().getFirst().toFields().getFirst())); + assertTrue(result.getAdditionalPredecessor().isEmpty()); + } + + @Test + void shouldPreserveFirstSubstringAndFallbackOrdering() { + var select = new FirstValueSelect(column("value", MajorTypeId.STRING), null, new Range.IntegerRange(2, 5)); + select.setName("value"); + var result = new FirstValueSelectConverter().connectorSelect(select, context()); + assertEquals("substring(\"events\".\"value\", 3, 3) as \"value-1\"", render(result.getPreprocessingSelects().getFirst().toFields().getFirst())); + assertFallbackOrdering(result); + } + + @Test + void shouldPreserveLastFallbackOrderingWithoutValidityDate() { + var select = new LastValueSelect(column("value", MajorTypeId.STRING), null, null); + select.setName("value"); + var result = new LastValueSelectConverter().connectorSelect(select, context()); + assertFallbackOrdering(result); + } + + @Test + void shouldPreserveRandomIgnoringSubstring() { + var select = new RandomValueSelect(column("value", MajorTypeId.STRING), null, new Range.IntegerRange(2, 5)); + select.setName("value"); + var result = new RandomValueSelectConverter().connectorSelect(select, context()); + assertEquals("\"events\".\"value\"", render(result.getPreprocessingSelects().getFirst().toFields().getFirst())); + assertEquals("first_value(\"preprocessing\".\"value\" order by rand()) as \"value-1\"", render(result.getAggregationSelects().getFirst().toFields().getFirst())); + assertEquals("\"aggregation\".\"value-1\"", render(result.getFinalSelects().getFirst().toFields().getFirst())); + assertTrue(result.getAdditionalPredecessor().isEmpty()); + } + + private static void assertFallbackOrdering(ConnectorSqlSelects result) { + var rowFilter = result.getAdditionalPredecessor().orElseThrow(); + var rowNumber = rowFilter.getPredecessors().getFirst(); + assertEquals("value-1-value_select_first_row_step", rowFilter.getCteName()); + assertEquals("row_number() over (partition by \"preprocessing\".\"id\" order by \"preprocessing\".\"id\") as \"row-number\"", + render(rowNumber.getSelects().getSqlSelects().getLast().toFields().getFirst())); + assertTrue(result.getAggregationSelects().isEmpty()); + assertEquals(1, result.getFinalSelects().size()); + } +} diff --git a/docs/sql-connector-extraction-verification.md b/docs/sql-connector-extraction-verification.md index 4ede866afc..660390d59c 100644 --- a/docs/sql-connector-extraction-verification.md +++ b/docs/sql-connector-extraction-verification.md @@ -31,7 +31,14 @@ Fixture paths in the table are relative to `backend/src/test/resources/tests/` o - Backend public-entry-point characterization: five aggregation tests and one dialect test passed after delegating sum, count, quarter count, and duration sum to the connector. The selected connector aggregation tests also passed (13 tests). - The ClickHouse nullable date-bound regression failed before the dialect bridge and passed afterward; preserving `Nullable(Date32)` prevents missing outer-join dates from becoming epoch dates. - Final connector run for the numeric aggregation extraction: 158 tests passed (`./mvnw.cmd -o -B -ntp -pl sql-connector -am test`). HANA/ClickHouse connections were not opened. +- Output-select extraction: 174 connector tests passed. The selected backend compatibility suite passed 23 tests covering aggregation adapters, first/last/random/distinct selects, HANA and ClickHouse distinct rendering, date unions, frozen and stratified date distance, flags, event date/duration outputs, connector and concept exists projections, concept-value self-union, and concept-ID mapping. +- Direct comparison with `origin/develop` confirmed that concept-value aggregation enumerates only the connector tables represented by the CQ query. Prepared mapping sources remain ordered so repeated resolved columns from different connectors cannot overwrite one another. +- Direct comparison with `origin/develop` found that single `DATE_RANGE` columns are treated as the same inclusive physical start/end value by the existing quarter-count and duration SQL. Two backend regression tests failed before restoring that behavior and pass afterward. +- Spotless verification is unavailable offline on this host because the Spotless Maven plugin is not present in the local repository. `git diff --check`, compilation, and the test suites above pass. ## Pending verification -Extraction is still in progress. The remaining operation/dialect coverage, full extraction-only `develop` build, and HANA/ClickHouse result-level verification are pending. Database integration suites cannot run on this host; a final handoff will identify the tested extraction revision, commands, prerequisites, and required reports. +Extraction is still in progress. The remaining operation/dialect coverage, moving concept-ID mapping-table preparation +into the connector's reusable table-preparation lifecycle, the full extraction-only `develop` build, and HANA/ClickHouse +result-level verification are pending. Database integration suites cannot run on this host; a final handoff will identify +the tested extraction revision, commands, prerequisites, and required reports. diff --git a/sql-connector/README.md b/sql-connector/README.md index 0138d98119..060551946c 100644 --- a/sql-connector/README.md +++ b/sql-connector/README.md @@ -60,8 +60,34 @@ Duration sums pack overlapping intervals first and exclude unbounded durations. as in the existing backend SQL implementation. Physical date-range columns require dialect support for extracting half-open bounds through `CompilerDialect.dateRangeColumn`. -Flag aggregation for output selects and integration of resolved operations into the legacy backend remain separate -migration steps. Flag filters already compile as event predicates. +The backend's sum, count, quarter-count, duration-sum, and flag-output adapters delegate to the shared converters. +Flag filters compile separately as event predicates. + +## Output select extraction + +`ResolvedSelectConverter` handles aggregation, first/last/random/distinct values, date union/distance, flags, +event-date union/duration, exists, and concept values. Callers provide the existing generated alias, table graph, IDs, +optional validity date, resolved concept-column table mapping, and ordered prepared concept-value sources through +`SelectConversionContext`. + +| Original backend responsibility | Shared implementation | Backend responsibility retained | +| --- | --- | --- | +| `ValueSelectUtil` | `compiler.conversion.operation.ValueSelectUtil` | First/last select DTO adaptation | +| `RandomValueSelectConverter` SQL | Shared converter with the same name | Column resolution and alias allocation | +| `DistinctSelectConverter` SQL | Shared converter with the same name | Select DTO adaptation | +| `ClickhouseDistinctSelectConverter` SQL | Shared converter with the same name | Dialect selection | +| `MappableSingleColumnSelect.getSubstringSelect` SQL | `SubstringSelect.getSubstringSelect` | Mapping, result readers, range adaptation | +| `DaterangeSelectUtil` select SQL | Shared helper with the same name | Resolved date-column adaptation | +| `DateDistanceSqlAggregator` select SQL | Shared helper with the same name | Frozen/per-row end-date resolution and filter/export SQL | +| `FlagSqlAggregator` select SQL | Shared aggregation converter | Filter/export predicates | +| Event date/duration select SQL | Shared converters with the original names | Application select metadata | +| `ConceptColumnSelectConverter` SQL | Shared converter with the same name | Connector-column resolution and mapping-table preparation | + +First and last preserve the existing validity-date ordering and entity-ID fallback, without adding tie breakers. +Random selects continue to ignore substring bounds. Distinct selects retain separate HANA and ClickHouse algorithms, +including their existing ordering and empty/null handling. Substring bounds remain zero-based with an exclusive end. +Concept-ID mapping tables are currently prepared by the backend and passed to the connector as ordered SQL-resolved +sources. Their preparation belongs with the connector once the reusable table-preparation lifecycle is extracted. ## Validation diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInAggregationConverters.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInAggregationConverters.java index 6ef41116b0..6dc58369c6 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInAggregationConverters.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/BuiltInAggregationConverters.java @@ -25,6 +25,7 @@ static List, return List.of( converter(BuiltInAggregations.Count.class, BuiltInAggregationConverters::convertCount), converter(BuiltInAggregations.Sum.class, SumAggregationConverter::convert), + converter(BuiltInAggregations.Flags.class, FlagSqlAggregator::convert), converter(BuiltInAggregations.CountQuarters.class, DateAggregationConverters::countQuarters), converter(BuiltInAggregations.DurationSum.class, DateAggregationConverters::durationSum) ); diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ClickhouseDistinctSelectConverter.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ClickhouseDistinctSelectConverter.java new file mode 100644 index 0000000000..2c1a3b6dd4 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ClickhouseDistinctSelectConverter.java @@ -0,0 +1,45 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import static org.jooq.impl.DSL.field; + +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; + +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; + + +import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; + +public class ClickhouseDistinctSelectConverter { + + + + public static ConnectorSqlSelects connectorSelect(BuiltInSelects.Values distinctSelect, SelectConversionContext selectContext) { + + String alias = selectContext.alias(); + + SqlTables tables = selectContext.tables(); + SingleColumnSqlSelect preprocessingSelect = + SubstringSelect.getSubstringSelect(distinctSelect.column(), distinctSelect.substring(), selectContext.tables().getRootTable(), alias); + + String preprocessingTable = selectContext.tables().cteName(ConceptCteStep.PREPROCESSING); + SingleColumnSqlSelect qualified = preprocessingSelect.qualify(preprocessingTable); + + FieldWrapper grouped = + new FieldWrapper<>(field("arrayFilter(x -> x <> '' and x is not null, groupUniqArray({0}))", Object.class, qualified.select()).as(alias), + qualified.select().getName() + ); + + SingleColumnSqlSelect finalSelect = grouped.qualify(tables.cteName(ConceptCteStep.AGGREGATION_SELECT)); + + return ConnectorSqlSelects.builder() + .preprocessingSelect(preprocessingSelect) + .aggregationSelect(grouped) + .finalSelect(finalSelect) + .build(); + } + + +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ConceptColumnSelectConverter.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ConceptColumnSelectConverter.java new file mode 100644 index 0000000000..13f3794666 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ConceptColumnSelectConverter.java @@ -0,0 +1,154 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.IntStream; + +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; + + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.CteStep; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; + +public class ConceptColumnSelectConverter { + + @Getter + @RequiredArgsConstructor + private enum CONCEPT_COLUMN_STEPS implements CteStep { + + UNIONED_COLUMNS("unioned_columns"), + STRING_AGG("concept_column_aggregated"); + + private final String suffix; + } + + + public ConnectorSqlSelects connectorSelect(BuiltInSelects.ConceptValues select, SelectConversionContext context) { + return ConnectorSqlSelects.builder().preprocessingSelects(IntStream.range(0, select.columns().size()) + .mapToObj(index -> { + ResolvedColumn column = select.columns().get(index); + SelectConversionContext.ConceptColumnSource source = preparedSource(context, index); + if (source != null) { + return new FieldWrapper<>(source.value().as(column.physicalName())); + } + return new ExtractingSqlSelect<>(column.table().physicalName().getLast(), column.physicalName(), Object.class); + }) + .toList()).build(); + } + + public ConceptSqlSelects conceptSelect(BuiltInSelects.ConceptValues select, SelectConversionContext selectContext) { + + // we will do a union distinct on all Connector tables + List connectorIndexes; + if (select.columns().size() == 1) { + // Preserve the self-union for a single connector. + connectorIndexes = List.of(0, 0); + } + else { + connectorIndexes = IntStream.range(0, select.columns().size()).boxed().toList(); + } + SqlNameGenerator nameGenerator = selectContext.nameGenerator(); + String alias = selectContext.alias(); + QueryStep unionStep = createUnionConnectorConnectorsStep(select.columns(), connectorIndexes, alias, selectContext); + + FieldWrapper conceptColumnSelect = createConnectorColumnStringAgg(selectContext, unionStep, alias); + Selects unionStepSelects = unionStep.getQualifiedSelects(); + Selects selects = Selects.builder() + .ids(unionStepSelects.getIds()) + .sqlSelect(conceptColumnSelect) + .build(); + + String stringAggCteName = nameGenerator.cteStepName(CONCEPT_COLUMN_STEPS.STRING_AGG, alias); + QueryStep stringAggStep = QueryStep.builder() + .cteName(stringAggCteName) + .selects(selects) + .fromTable(QueryStep.toTableLike(unionStep.getCteName())) + .groupBy(unionStepSelects.getIds().toFields()) + .predecessor(unionStep) + .build(); + + ExtractingSqlSelect finalSelect = conceptColumnSelect.qualify(stringAggStep.getCteName()); + + return ConceptSqlSelects.builder() + .additionalPredecessor(Optional.of(stringAggStep)) + .finalSelect(finalSelect) + .build(); + } + + private static QueryStep createUnionConnectorConnectorsStep( + List connectors, + List connectorIndexes, + String alias, + SelectConversionContext selectContext + ) { + List unionSteps = connectorIndexes.stream() + .map(index -> createConnectorColumnSelectQuery(connectors.get(index), preparedSource(selectContext, index), alias, selectContext)) + .toList(); + String unionedColumnsCteName = selectContext.nameGenerator().cteStepName(CONCEPT_COLUMN_STEPS.UNIONED_COLUMNS, alias); + return QueryStep.createUnionStep(unionSteps, unionedColumnsCteName, Collections.emptyList(), false); //TODO is false correct here? + } + + private static QueryStep createConnectorColumnSelectQuery( + ResolvedColumn connector, + SelectConversionContext.ConceptColumnSource preparedSource, + String alias, + SelectConversionContext selectContext + ) { + // a ConceptColumn select uses all connectors a Concept has, even if they are not part of the CQConcept + // but if they are, we need to make sure we use the preprocessed and event-filtered table instead of the root table + String physicalTable = connector.table().physicalName().getLast(); + String tableName = preparedSource != null + ? preparedSource.qualifier() + : selectContext.conceptColumnTables().getOrDefault(physicalTable, physicalTable); + Table connectorTable = DSL.table(DSL.name(tableName)); + SqlIdColumns ids = selectContext.ids().qualify(connectorTable.getName()); + Field connectorColumn = preparedSource != null + ? preparedSource.value() + : DSL.field(DSL.name(connectorTable.getName(), connector.physicalName())); + Field casted = selectContext.dialect().cast(connectorColumn, SQLDataType.VARCHAR).as(alias); + FieldWrapper connectorSelect = new FieldWrapper<>(casted); + + Selects selects = Selects.builder() + .ids(ids) + .sqlSelect(connectorSelect) + .build(); + + return QueryStep.builder() + .selects(selects) + .fromTable(preparedSource != null ? preparedSource.table() : connectorTable) + .conditions(preparedSource != null ? preparedSource.conditions() : List.of()) + .build(); + } + + private static SelectConversionContext.ConceptColumnSource preparedSource(SelectConversionContext context, int index) { + return context.conceptColumnSources().isEmpty() ? null : context.conceptColumnSources().get(index); + } + + private static FieldWrapper createConnectorColumnStringAgg(SelectConversionContext selectContext, QueryStep unionStep, String alias) { + CompilerDialect functionProvider = selectContext.dialect(); + Field unionedColumn = DSL.field(DSL.name(unionStep.getCteName(), alias), String.class); + return new FieldWrapper<>( + functionProvider.stringAggregation(unionedColumn, DSL.toChar((char) 31), List.of(unionedColumn)).as(alias) + ); + } + +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/DateAggregationConverters.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/DateAggregationConverters.java index 0ad846dae0..965f0003a8 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/DateAggregationConverters.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/DateAggregationConverters.java @@ -30,7 +30,7 @@ final class DateAggregationConverters { static CommonAggregationSelect countQuarters(BuiltInAggregations.CountQuarters aggregation, AggregationConversionContext context) { CompilerDialect dialect = context.dialect(); String source = context.tables().getPredecessor(ConceptCteStep.AGGREGATION_SELECT); - if (aggregation.dates() instanceof DateColumns.Single single && single.column().type() == ColumnType.DATE) { + if (aggregation.dates() instanceof DateColumns.Single single) { ExtractingSqlSelect root = new ExtractingSqlSelect<>(context.tables().getRootTable(), single.column().physicalName(), Date.class); Field count = DSL.nullif(DSL.countDistinct(dialect.yearQuarter(root.qualify(source).select())), 0); return CommonAggregationSelect.builder() @@ -83,7 +83,7 @@ static CommonAggregationSelect durationSum(BuiltInAggregations.Durat .build(); } - private static ColumnDateRange dateRange(DateColumns dates, AggregationConversionContext context) { + static ColumnDateRange dateRange(DateColumns dates, AggregationConversionContext context) { if (dates instanceof DateColumns.Single single && single.column().type() == ColumnType.DATE_RANGE) { return context.dialect().dateRangeColumn(DSL.field(DSL.name(context.tables().getRootTable(), single.column().physicalName()))); } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/DateDistanceSqlAggregator.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/DateDistanceSqlAggregator.java new file mode 100644 index 0000000000..e9ae97d469 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/DateDistanceSqlAggregator.java @@ -0,0 +1,40 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import java.sql.Date; +import java.time.temporal.ChronoUnit; + +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import org.jooq.Field; +import org.jooq.impl.DSL; + +public final class DateDistanceSqlAggregator { + private DateDistanceSqlAggregator() { + } + + static ConnectorSqlSelects connectorSelect(BuiltInSelects.DateDistance select, SelectConversionContext context) { + Field startDate = DSL.field(DSL.name(context.tables().getRootTable(), select.column().physicalName()), Date.class); + return selects(context.dialect().dateDistance(select.unit(), startDate, select.endDate()), context); + } + + /** Supports a resolved per-row end date, as used by stratified forms. */ + public static ConnectorSqlSelects connectorSelect(ResolvedColumn column, ChronoUnit unit, Field endDate, SelectConversionContext context) { + Field startDate = DSL.field(DSL.name(context.tables().getRootTable(), column.physicalName()), Date.class); + return selects(context.dialect().dateDistance(unit, startDate, endDate), context); + } + + private static ConnectorSqlSelects selects(Field dateDistanceCalculation, SelectConversionContext context) { + String alias = context.alias(); + FieldWrapper dateDistanceSelect = new FieldWrapper<>(dateDistanceCalculation.as(alias)); + Field qualifiedDateDistance = dateDistanceSelect.qualify(context.tables().getPredecessor(ConceptCteStep.AGGREGATION_SELECT)).select(); + FieldWrapper minDateDistance = new FieldWrapper<>(DSL.min(qualifiedDateDistance).as(alias)); + return ConnectorSqlSelects.builder() + .preprocessingSelect(dateDistanceSelect) + .aggregationSelect(minDateDistance) + .finalSelect(minDateDistance.qualify(context.tables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER))) + .build(); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/DaterangeSelectUtil.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/DaterangeSelectUtil.java new file mode 100644 index 0000000000..9857c8380b --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/DaterangeSelectUtil.java @@ -0,0 +1,87 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import static com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep.PREPROCESSING; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingSelectCompiler; +import com.bakdata.conquery.sql.compiler.ir.interval.IntervalPackingSelectPreparation; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; +import com.bakdata.conquery.sql.model.schema.DateColumns; + +public class DaterangeSelectUtil { + public static ConnectorSqlSelects createForSelect( + DateColumns dates, + AggregationFunction aggregationFunction, + SelectConversionContext context + ) { + String alias = context.alias(); + CompilerDialect functionProvider = context.dialect(); + + ColumnDateRange daterange = DateAggregationConverters.dateRange(dates, new AggregationConversionContext(context.dialect(), context.nameGenerator(), context.tables(), context.ids(), alias)).as(alias); + List rootSelects = daterange.toFields().stream() + .map(FieldWrapper::new) + .collect(Collectors.toList()); + + IntervalPackingSelectPreparation preparation = prepareIntervalSelect( + alias, + daterange, + context.ids(), + context.tables(), + context + ); + ColumnDateRange qualified = preparation.dateRange(); + FieldWrapper aggregationField = aggregationFunction.apply(qualified, alias, functionProvider); + + QueryStep intervalPackingSelectsStep = IntervalPackingSelectCompiler.compileArbitrarySelect( + preparation.predecessor(), + qualified, + aggregationField, + preparation.tables(), + context.dialect() + ); + + SqlTables tables = context.tables(); + ExtractingSqlSelect finalSelect = aggregationField.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); + + return ConnectorSqlSelects.builder() + .preprocessingSelects(rootSelects) + .additionalPredecessor(Optional.of(intervalPackingSelectsStep)) + .finalSelect(finalSelect) + .build(); + } + + private static IntervalPackingSelectPreparation prepareIntervalSelect( + String alias, + ColumnDateRange daterange, + SqlIdColumns ids, + SqlTables tables, + SelectConversionContext context + ) { + return IntervalPackingSelectCompiler.prepareArbitrarySelect( + alias, + tables.cteName(PREPROCESSING), + ids, + daterange, + context.dialect(), + context.nameGenerator() + ); + } + + @FunctionalInterface + public interface AggregationFunction { + FieldWrapper apply(ColumnDateRange daterange, String alias, CompilerDialect functionProvider); + } + +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/DistinctSelectConverter.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/DistinctSelectConverter.java new file mode 100644 index 0000000000..4ccf7e58c1 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/DistinctSelectConverter.java @@ -0,0 +1,136 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; + +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.CteStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.jooq.Field; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; + +/** + * 
+ *  The two additional CTEs this aggregator creates:
+ * 	
    
+ * 	    
  1. 
+ * 	        Select distinct values of a column.
+ *            {@code
+ * 	        	"distinct" as (
+ *     				select distinct "pid", "column"
+ *     				from "preprocessing"
+ *  			)
+ *            }
+ * 	    
    2. 
+ * 	    
  2. 
+ * 	        String agg all distinct values of the column.
+ *            {@code
+ * 	        "aggregated" as (
+ *    			 select
+ *    			   "select-1-distinct"."pid",
+ *    			   string_agg(cast("column" as varchar), cast(' ' as varchar) ) as "select-1"
+ *    			 from "distinct"
+ *    			 group by "pid"
+ *   			)
+ *            }
+ * 	    
    3. 
+ * 	

+ *
+ */ +public class DistinctSelectConverter { + + @Getter + @RequiredArgsConstructor + private enum DistinctSelectCteStep implements CteStep { + + DISTINCT_SELECT("distinct", null), + STRING_AGG("aggregated", DISTINCT_SELECT); + + private final String suffix; + private final DistinctSelectCteStep predecessor; + } + + + public static ConnectorSqlSelects connectorSelect(BuiltInSelects.Values distinctSelect, SelectConversionContext selectContext) { + + String alias = selectContext.alias(); + + SqlTables tables = selectContext.tables(); + SingleColumnSqlSelect preprocessingSelect = SubstringSelect.getSubstringSelect(distinctSelect.column(), distinctSelect.substring(), selectContext.tables().getRootTable(), alias); + + QueryStep distinctSelectCte = createDistinctSelectCte(preprocessingSelect, alias, selectContext); + QueryStep aggregatedCte = createAggregationCte(selectContext, preprocessingSelect, distinctSelectCte, alias); + + SingleColumnSqlSelect finalSelect = preprocessingSelect.qualify(tables.cteName(ConceptCteStep.AGGREGATION_FILTER)); + + return ConnectorSqlSelects.builder() + .preprocessingSelect(preprocessingSelect) + .additionalPredecessor(Optional.of(aggregatedCte)) + .finalSelect(finalSelect) + .build(); + } + + private static QueryStep createAggregationCte( + SelectConversionContext selectContext, + SingleColumnSqlSelect preprocessingSelect, + QueryStep distinctSelectCte, + String alias + ) { + CompilerDialect functionProvider = selectContext.dialect(); + Field castedColumn = functionProvider.cast(preprocessingSelect.qualify(distinctSelectCte.getCteName()).select(), SQLDataType.VARCHAR); + Field aggregatedColumn = functionProvider.stringAggregation(castedColumn, DSL.toChar((char) 31), List.of(castedColumn)) + .as(alias); + + SqlIdColumns ids = distinctSelectCte.getQualifiedSelects().getIds(); + + Selects selects = Selects.builder() + .ids(ids) + .sqlSelect(new FieldWrapper<>(aggregatedColumn)) + .build(); + + return QueryStep.builder() + .cteName(selectContext.nameGenerator().cteStepName(DistinctSelectCteStep.STRING_AGG, alias)) + .selects(selects) + .fromTable(QueryStep.toTableLike(distinctSelectCte.getCteName())) + .groupBy(ids.toFields()) + .predecessor(distinctSelectCte) + .build(); + } + + private static QueryStep createDistinctSelectCte( + SingleColumnSqlSelect preprocessingSelect, + String alias, + SelectConversionContext selectContext + ) { + // values to aggregate must be event-filtered first + String preprocessingTable = selectContext.tables().cteName(ConceptCteStep.PREPROCESSING); + SingleColumnSqlSelect qualified = preprocessingSelect.qualify(preprocessingTable); + + SqlIdColumns ids = selectContext.ids().qualify(preprocessingTable); + + Selects selects = Selects.builder() + .ids(ids) + .sqlSelect(qualified) + .build(); + + return QueryStep.builder() + .cteName(selectContext.nameGenerator().cteStepName(DistinctSelectCteStep.DISTINCT_SELECT, alias)) + .selectDistinct(true) + .selects(selects) + .fromTable(QueryStep.toTableLike(preprocessingTable)) + .build(); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/EventDateUnionSelectConverter.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/EventDateUnionSelectConverter.java new file mode 100644 index 0000000000..1e2f0d539c --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/EventDateUnionSelectConverter.java @@ -0,0 +1,53 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; + + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; + + +public class EventDateUnionSelectConverter { + + + public ConnectorSqlSelects connectorSelect(BuiltInSelects.EventDateUnion select, SelectConversionContext selectContext) { + + FieldWrapper stringAggregation = createEventDateUnionAggregation(select, selectContext); + ExtractingSqlSelect finalSelect = stringAggregation.qualify(selectContext.tables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); + + return ConnectorSqlSelects.builder() + .eventDateSelect(stringAggregation) + .finalSelect(finalSelect) + .build(); + } + + + public ConceptSqlSelects conceptSelect(BuiltInSelects.EventDateUnion select, SelectConversionContext selectContext) { + + FieldWrapper stringAggregation = createEventDateUnionAggregation(select, selectContext); + ExtractingSqlSelect finalSelect = stringAggregation.qualify(selectContext.tables().getPredecessor(ConceptCteStep.UNIVERSAL_SELECTS)); + + return ConceptSqlSelects.builder() + .eventDateSelect(stringAggregation) + .finalSelect(finalSelect) + .build(); + } + + private static FieldWrapper createEventDateUnionAggregation(BuiltInSelects.EventDateUnion select, SelectConversionContext selectContext) { + + if (selectContext.validityDate().isEmpty()) { throw new IllegalArgumentException("Can't convert an EventDateUnionSelect without a validity date being present"); } + ColumnDateRange validityDate = selectContext.validityDate().get(); + + CompilerDialect functionProvider = selectContext.dialect(); + String alias = selectContext.alias(); + + ColumnDateRange qualified = validityDate.qualify(selectContext.tables().getPredecessor(ConceptCteStep.INTERVAL_PACKING_SELECTS)); + return new FieldWrapper<>(functionProvider.aggregateDateRanges(qualified.getStart(), qualified.getEnd()).as(alias)); + } + +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/EventDurationSumSelectConverter.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/EventDurationSumSelectConverter.java new file mode 100644 index 0000000000..e5fe7b1d98 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/EventDurationSumSelectConverter.java @@ -0,0 +1,72 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import java.math.BigDecimal; +import java.sql.Date; +import java.time.temporal.ChronoUnit; + +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; + + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; + +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.impl.DSL; + +public class EventDurationSumSelectConverter { + + + public ConnectorSqlSelects connectorSelect(BuiltInSelects.EventDurationSum select, SelectConversionContext selectContext) { + + FieldWrapper stringAggregation = createEventDurationSumAggregation(select, selectContext); + ExtractingSqlSelect finalSelect = stringAggregation.qualify(selectContext.tables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); + + return ConnectorSqlSelects.builder() + .eventDateSelect(stringAggregation) + .finalSelect(finalSelect) + .build(); + } + + + public ConceptSqlSelects conceptSelect(BuiltInSelects.EventDurationSum select, SelectConversionContext selectContext) { + + FieldWrapper stringAggregation = createEventDurationSumAggregation(select, selectContext); + ExtractingSqlSelect finalSelect = stringAggregation.qualify(selectContext.tables().getPredecessor(ConceptCteStep.UNIVERSAL_SELECTS)); + + return ConceptSqlSelects.builder() + .eventDateSelect(stringAggregation) + .finalSelect(finalSelect) + .build(); + } + + private FieldWrapper createEventDurationSumAggregation(BuiltInSelects.EventDurationSum select, SelectConversionContext selectContext) { + + if (selectContext.validityDate().isEmpty()) { throw new IllegalArgumentException("Can't convert an EventDateUnionSelect without a validity date being present"); } + String predecessorCteName = selectContext.tables().getPredecessor(ConceptCteStep.INTERVAL_PACKING_SELECTS); + ColumnDateRange qualified = selectContext.validityDate().get().qualify(predecessorCteName); + ColumnDateRange asDualColumn = selectContext.dialect().toDualColumn(qualified); + + CompilerDialect functionProvider = selectContext.dialect(); + String alias = selectContext.alias(); + + Field durationSum = DSL.sum( + DSL.when(containsInfinityDate(asDualColumn, functionProvider), DSL.inline(null, Integer.class)) + .otherwise(functionProvider.dateDistance(ChronoUnit.DAYS, asDualColumn.getStart(), asDualColumn.getEnd())) + ) + .as(alias); + return new FieldWrapper<>(durationSum); + } + + private static Condition containsInfinityDate(ColumnDateRange validityDate, CompilerDialect functionProvider) { + Field negativeInfinity = functionProvider.minimumDate(); + Field positiveInfinity = functionProvider.maximumDate(); + return validityDate.getStart().eq(negativeInfinity).or(validityDate.getEnd().eq(positiveInfinity)); + } + +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/FlagSqlAggregator.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/FlagSqlAggregator.java new file mode 100644 index 0000000000..105303628e --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/FlagSqlAggregator.java @@ -0,0 +1,81 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import static org.jooq.impl.DSL.*; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.concept.CommonAggregationSelect; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; +import com.bakdata.conquery.sql.model.operation.BuiltInAggregations; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import org.jooq.Condition; +import org.jooq.Field; + +final class FlagSqlAggregator { + static CommonAggregationSelect convert(BuiltInAggregations.Flags aggregation, AggregationConversionContext context) { + Map rootSelects = createFlagRootSelectMap(aggregation, context.tables().getRootTable()); + return aggregation(rootSelects, createFlagSelect(context.alias(), context.tables(), context.dialect(), rootSelects)); + } + + private static CommonAggregationSelect aggregation(Map roots, FieldWrapper grouped) { + return CommonAggregationSelect.builder().rootSelects(roots.values()).groupBy(grouped).build(); + } + private static Map createFlagRootSelectMap(BuiltInAggregations.Flags flagSelect, String rootTable) { + return flagSelect.columns() + .entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> { + ResolvedColumn column = entry.getValue(); + Field field = field(name(rootTable, column.physicalName())); + return new FieldWrapper<>(field.as(column.physicalName()), column.physicalName() + ); + } + )); + } + + private static FieldWrapper createFlagSelect( + String alias, + SqlTables connectorTables, + CompilerDialect functionProvider, + Map flagRootSelectMap + ) { + Map> flagFieldsMap = createRootSelectReferences(connectorTables, flagRootSelectMap); + + // we first aggregate each flag column + List> flagAggregations = new ArrayList<>(); + for (Map.Entry> entry : flagFieldsMap.entrySet()) { + Field boolColumn = entry.getValue(); + Condition anyTrue = functionProvider.orAgg(boolColumn); + + String flagName = entry.getKey(); + Field flag = when(anyTrue, inline(flagName)).otherwise(""); // else null is implicit in SQL + flagAggregations.add(flag); + } + + // and stuff them into 1 array field + Field flagsArray = functionProvider.arrayOut(flagAggregations).as(alias); + // we also need the references for all flag columns for the flag aggregation of multiple columns + String[] requiredColumns = flagFieldsMap.values().stream().map(Field::getName).toArray(String[]::new); + return new FieldWrapper<>(flagsArray, requiredColumns); + } + + private static Map> createRootSelectReferences( + SqlTables connectorTables, + Map flagRootSelectMap + ) { + return flagRootSelectMap.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> (Field) entry.getValue().qualify(connectorTables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT)).select() + )); + } + +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/RandomValueSelectConverter.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/RandomValueSelectConverter.java new file mode 100644 index 0000000000..e4b3cd6a71 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/RandomValueSelectConverter.java @@ -0,0 +1,35 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import org.jooq.Field; + +final class RandomValueSelectConverter { + + + static ConnectorSqlSelects connectorSelect(BuiltInSelects.Values select, SelectConversionContext selectContext) { + + SqlTables tables = selectContext.tables(); + + String rootTableName = tables.getRootTable(); + String columnName = select.column().physicalName(); + ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(rootTableName, columnName, Object.class); + + String alias = selectContext.alias(); + Field qualifiedRootSelect = rootSelect.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT)).select(); + Field firstAggregation = selectContext.dialect().random(qualifiedRootSelect).as(alias); + FieldWrapper firstAggregationSqlSelect = new FieldWrapper<>(firstAggregation, columnName); + + ExtractingSqlSelect finalSelect = firstAggregationSqlSelect.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); + + return ConnectorSqlSelects.builder() + .preprocessingSelect(rootSelect) + .aggregationSelect(firstAggregationSqlSelect) + .finalSelect(finalSelect) + .build(); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedSelectConverter.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedSelectConverter.java new file mode 100644 index 0000000000..efc4ac2099 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedSelectConverter.java @@ -0,0 +1,91 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiFunction; + +import com.bakdata.conquery.sql.compiler.conversion.ConversionDispatcher; +import com.bakdata.conquery.sql.compiler.conversion.Converter; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.select.ExistsSqlSelect; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptSqlSelects; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; +import com.bakdata.conquery.sql.model.operation.ResolvedSelect; +import org.jooq.Field; + +/** Dispatches resolved output operations to their SQL helpers. */ +public final class ResolvedSelectConverter { + + private final ConversionDispatcher dispatcher; + + public ResolvedSelectConverter() { + this(List.of()); + } + + public ResolvedSelectConverter(List> extensions) { + List> converters = new ArrayList<>(List.of( + converter(BuiltInSelects.Values.class, ResolvedSelectConverter::values), + converter(BuiltInSelects.ConceptValues.class, new ConceptColumnSelectConverter()::connectorSelect), + converter(BuiltInSelects.DateUnion.class, (select, context) -> DaterangeSelectUtil.createForSelect(select.dates(), + (dates, alias, dialect) -> new FieldWrapper<>(dialect.aggregateDateRanges(dates.getStart(), dates.getEnd()).as(alias)), context)), + converter(BuiltInSelects.DateDistance.class, DateDistanceSqlAggregator::connectorSelect), + converter(BuiltInSelects.EventDateUnion.class, new EventDateUnionSelectConverter()::connectorSelect), + converter(BuiltInSelects.EventDurationSum.class, new EventDurationSumSelectConverter()::connectorSelect), + converter(BuiltInSelects.Aggregation.class, ResolvedSelectConverter::aggregation), + converter(BuiltInSelects.Exists.class, (select, context) -> ConnectorSqlSelects.builder().finalSelect(ExistsSqlSelect.withAlias(context.alias())).build()) + )); + converters.addAll(extensions); + dispatcher = new ConversionDispatcher<>(converters); + } + + public ConnectorSqlSelects convert(ResolvedSelect select, SelectConversionContext context) { + return dispatcher.convert(select, context); + } + + public ConceptSqlSelects conceptSelect(ResolvedSelect select, SelectConversionContext context) { + if (select instanceof BuiltInSelects.ConceptValues values) { + return new ConceptColumnSelectConverter().conceptSelect(values, context); + } + if (select instanceof BuiltInSelects.EventDateUnion union) { + return new EventDateUnionSelectConverter().conceptSelect(union, context); + } + if (select instanceof BuiltInSelects.EventDurationSum duration) { + return new EventDurationSumSelectConverter().conceptSelect(duration, context); + } + if (select instanceof BuiltInSelects.Exists) { + return ConceptSqlSelects.builder().finalSelect(ExistsSqlSelect.withAlias(context.alias())).build(); + } + throw new IllegalStateException("No concept select converter found for " + select); + } + + private static ConnectorSqlSelects values(BuiltInSelects.Values select, SelectConversionContext context) { + return switch (select.operation()) { + case FIRST -> ValueSelectUtil.createValueSelect(select.column(), context.alias(), Field::asc, select.substring(), context); + case LAST -> ValueSelectUtil.createValueSelect(select.column(), context.alias(), Field::desc, select.substring(), context); + case RANDOM -> RandomValueSelectConverter.connectorSelect(select, context); + case DISTINCT -> context.dialect().distinctSelect(select, context); + }; + } + + private static ConnectorSqlSelects aggregation(BuiltInSelects.Aggregation select, SelectConversionContext context) { + var aggregation = new ResolvedAggregationConverter().convert(select.aggregation(), new AggregationConversionContext( + context.dialect(), context.nameGenerator(), context.tables(), context.ids(), context.alias())); + var result = ConnectorSqlSelects.builder().preprocessingSelects(aggregation.getRootSelects()) + .additionalPredecessor(aggregation.getAdditionalPredecessor()) + .finalSelect(aggregation.getGroupBy().qualify(context.tables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER))); + if (aggregation.getAdditionalPredecessor().isEmpty()) { + result.aggregationSelect(aggregation.getGroupBy()); + } + return result.build(); + } + + private static Converter converter( + Class type, BiFunction conversion) { + return new Converter<>() { + @Override public Class getConversionClass() { return type; } + @Override public ConnectorSqlSelects convert(I input, SelectConversionContext context) { return conversion.apply(input, context); } + }; + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/SelectConversionContext.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/SelectConversionContext.java new file mode 100644 index 0000000000..6d65237de1 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/SelectConversionContext.java @@ -0,0 +1,58 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.TableLike; + +/** Resolved state used by select SQL helpers; aliases are allocated by the caller. */ +public record SelectConversionContext( + CompilerDialect dialect, + SqlNameGenerator nameGenerator, + SqlTables tables, + SqlIdColumns ids, + Optional validityDate, + String alias, + Map conceptColumnTables, + List conceptColumnSources +) { + public SelectConversionContext(CompilerDialect dialect, SqlNameGenerator nameGenerator, SqlTables tables, SqlIdColumns ids, + Optional validityDate, String alias) { + this(dialect, nameGenerator, tables, ids, validityDate, alias, Map.of(), List.of()); + } + + public SelectConversionContext(CompilerDialect dialect, SqlNameGenerator nameGenerator, SqlTables tables, SqlIdColumns ids, + Optional validityDate, String alias, Map conceptColumnTables) { + this(dialect, nameGenerator, tables, ids, validityDate, alias, conceptColumnTables, List.of()); + } + + public SelectConversionContext { + conceptColumnTables = Map.copyOf(conceptColumnTables); + conceptColumnSources = List.copyOf(conceptColumnSources); + } + + /** SQL-resolved source for a concept-value column whose physical table needs preparation before selection. */ + public record ConceptColumnSource( + String qualifier, + TableLike table, + Field value, + List conditions + ) { + public ConceptColumnSource { + Objects.requireNonNull(qualifier, "qualifier"); + Objects.requireNonNull(table, "table"); + Objects.requireNonNull(value, "value"); + conditions = List.copyOf(conditions); + } + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/SubstringSelect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/SubstringSelect.java new file mode 100644 index 0000000000..e47f6264a1 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/SubstringSelect.java @@ -0,0 +1,34 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import static org.jooq.impl.DSL.*; + +import java.util.Optional; + +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; +import com.bakdata.conquery.sql.model.range.SubstringRange; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import org.jooq.Field; + +/** SQL portion of the backend's mappable single-column select. */ +public final class SubstringSelect { + private SubstringSelect() { + } + + public static SingleColumnSqlSelect getSubstringSelect(ResolvedColumn column, Optional substringRange, String rootTable, String alias) { + Field field = field(name(rootTable, column.physicalName()), String.class); + if (substringRange.isPresent()) { + SubstringRange range = substringRange.get(); + if (range.endExclusive().isEmpty()) { + field = substring(field, 1 + range.startInclusive()); + } + else { + field = substring(field, 1 + range.startInclusive(), range.endExclusive().get() - range.startInclusive()); + } + } + if (alias != null) { + field = field.as(name(alias)); + } + return new FieldWrapper<>(field, column.physicalName()); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ValueSelectUtil.java similarity index 78% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ValueSelectUtil.java index 2708cdca6e..55abd8da17 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ValueSelectUtil.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.model.select; +package com.bakdata.conquery.sql.compiler.conversion.operation; import static org.jooq.impl.DSL.*; @@ -6,16 +6,16 @@ import java.util.Optional; import java.util.function.Function; -import com.bakdata.conquery.models.common.Range; -import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; +import com.bakdata.conquery.sql.model.range.SubstringRange; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; + import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; import com.bakdata.conquery.sql.compiler.ir.select.SqlSelect; import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.CteStep; import com.bakdata.conquery.sql.compiler.ir.QueryStep; @@ -23,7 +23,7 @@ import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import lombok.Getter; import lombok.RequiredArgsConstructor; -import org.jetbrains.annotations.NotNull; + import org.jooq.Field; import org.jooq.Record; import org.jooq.SelectConditionStep; @@ -32,13 +32,13 @@ class ValueSelectUtil { public static ConnectorSqlSelects createValueSelect( - Column column, + ResolvedColumn column, String alias, Function, ? extends SortField> ordering, - Range.IntegerRange substringRange, SelectContext selectContext) { + Optional substringRange, SelectConversionContext selectContext) { - SingleColumnSqlSelect rootSelect = MappableSingleColumnSelect.getSubstringSelect(column, substringRange, selectContext, alias); + SingleColumnSqlSelect rootSelect = SubstringSelect.getSubstringSelect(column, substringRange, selectContext.tables().getRootTable(), alias); // create a CTE, that per row makes a window calculation to select for the rank of the validity date. // Further down below, we select the values with rank=1, which is FIRST/LAST depending on sort order supplied by the creator. @@ -46,12 +46,12 @@ public static ConnectorSqlSelects createValueSelect( QueryStep rowNumberStep = buildRowNumberStep(rootSelect, ordering, alias, selectContext); - QueryStep rowFilterStep = buildRowFilterStep(rowNumberStep, alias, selectContext.getIds()); + QueryStep rowFilterStep = buildRowFilterStep(rowNumberStep, alias, selectContext.ids()); SqlSelect finalSelect = rowFilterStep.getQualifiedSelects().getSqlSelects().getFirst(); FieldWrapper aggregationSelect = - new FieldWrapper<>(field(coalesce(finalSelect.qualify(ValueSelectCteStep.ROW_SELECT_STEP.cteName(alias)))).as(alias), column.getName()); + new FieldWrapper<>(field(coalesce(finalSelect.qualify(ValueSelectCteStep.ROW_SELECT_STEP.cteName(alias)))).as(alias), column.physicalName()); return ConnectorSqlSelects.builder() .additionalPredecessor(Optional.of(rowFilterStep)) @@ -62,20 +62,20 @@ public static ConnectorSqlSelects createValueSelect( private static QueryStep buildRowNumberStep( SingleColumnSqlSelect rootSelect, Function, ? extends SortField> ordering, String alias, - SelectContext selectContext) { + SelectConversionContext selectContext) { - SqlFunctionProvider functionProvider = selectContext.getFunctionProvider(); + CompilerDialect functionProvider = selectContext.dialect(); - String predecessor = selectContext.getTables().getPredecessor(ConceptCteStep.AGGREGATION_SELECT); + String predecessor = selectContext.tables().getPredecessor(ConceptCteStep.AGGREGATION_SELECT); Field qualifiedRootSelect = rootSelect.qualify(predecessor).select(); return QueryStep.builder() .selects(Selects.builder() - .ids(selectContext.getIds().qualify(null)) + .ids(selectContext.ids().qualify(null)) .sqlSelects(List.of( new FieldWrapper<>(qualifiedRootSelect.as(alias), qualifiedRootSelect.getName()), - rowNumberField(predecessor, selectContext.getValidityDate(), ordering, - selectContext.getIds(), + rowNumberField(predecessor, selectContext.validityDate(), ordering, + selectContext.ids(), functionProvider ) )) @@ -109,10 +109,10 @@ private static QueryStep buildRowFilterStep(QueryStep rowNumberStep, String alia .build(); } - @NotNull + private static FieldWrapper rowNumberField( String predecessor, Optional validityDate, Function, ? extends SortField> ordering, - SqlIdColumns idColumns, SqlFunctionProvider functionProvider) { + SqlIdColumns idColumns, CompilerDialect functionProvider) { List> qualifiedIds = idColumns.qualify(predecessor).toFields(); diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java index 3689136e00..99f824cda5 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java @@ -1,15 +1,24 @@ package com.bakdata.conquery.sql.compiler.dialect; +import com.bakdata.conquery.sql.compiler.conversion.operation.DistinctSelectConverter; +import com.bakdata.conquery.sql.compiler.conversion.operation.SelectConversionContext; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; import java.sql.Date; import java.time.LocalDate; import java.time.temporal.ChronoUnit; import java.util.List; +import java.util.Collection; +import java.util.function.Function; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.model.range.DateRange; import org.jooq.Condition; +import org.jooq.DataType; import org.jooq.Field; +import org.jooq.OrderField; +import org.jooq.SortField; import org.jooq.Record; import org.jooq.Table; import org.jooq.impl.DSL; @@ -27,6 +36,40 @@ */ public interface CompilerDialect { + default ColumnDateRange toDualColumn(ColumnDateRange range) { + return ColumnDateRange.of(range.getStart(), range.getEnd()); + } + + default Condition orAgg(Field field) { + throw new UnsupportedOperationException("Boolean aggregation is not implemented by this dialect"); + } + + default Field arrayOut(List> fields) { + throw new UnsupportedOperationException("Array output is not implemented by this dialect"); + } + + default ConnectorSqlSelects distinctSelect( + BuiltInSelects.Values select, + SelectConversionContext context) { + return DistinctSelectConverter.connectorSelect(select, context); + } + + default Field cast(Field field, DataType type) { + throw new UnsupportedOperationException("Casting is not implemented by this dialect"); + } + + default Field stringAggregation(Field field, Field delimiter, List> orderByFields) { + throw new UnsupportedOperationException("String aggregation is not implemented by this dialect"); + } + + default Collection> orderByValidityDates(Function, ? extends SortField> ordering, List> validityDateFields) { + throw new UnsupportedOperationException("Validity-date ordering is not implemented by this dialect"); + } + + default Field random(Field field) { + throw new UnsupportedOperationException("Random aggregation is not implemented by this dialect"); + } + /** Expression used as the database-specific lower infinity sentinel for dates. */ Field minimumDate(); diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedSelectConverterTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedSelectConverterTest.java new file mode 100644 index 0000000000..af04859385 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedSelectConverterTest.java @@ -0,0 +1,254 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import static org.junit.jupiter.api.Assertions.*; + +import java.sql.Date; +import java.time.LocalDate; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import com.bakdata.conquery.sql.model.operation.BuiltInAggregations; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; +import com.bakdata.conquery.sql.model.operation.ResolvedSelect; +import com.bakdata.conquery.sql.model.range.SubstringRange; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.SqlTable; +import com.bakdata.conquery.sql.model.schema.DateColumns; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import org.jooq.Field; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Test; + +class ResolvedSelectConverterTest { + + private static final ResolvedColumn VALUE = new ResolvedColumn("events.value", SqlTable.of("events", "events"), "value", ColumnType.STRING, true); + private final ResolvedSelectConverter converter = new ResolvedSelectConverter(); + + @Test + void shouldRejectMissingSelectRegistration() { + assertThrows(IllegalStateException.class, () -> converter.convert(new UnsupportedSelect("value"), context())); + } + + @Test + void shouldProjectExistsForConnectorAndConceptConversions() { + var connector = converter.convert(new BuiltInSelects.Exists("exists"), context()); + var concept = converter.conceptSelect(new BuiltInSelects.Exists("exists"), context()); + + assertEquals("1 as \"value\"", render(connector.getFinalSelects().getFirst().toFields().getFirst())); + assertEquals("1 as \"value\"", render(concept.getFinalSelects().getFirst().toFields().getFirst())); + assertEquals(List.of(), connector.getFinalSelects().getFirst().requiredColumns()); + assertEquals(1, connector.getFinalSelects().size()); + assertEquals(1, concept.getFinalSelects().size()); + assertTrue(connector.getPreprocessingSelects().isEmpty()); + assertTrue(connector.getAdditionalPredecessor().isEmpty()); + } + + @Test + void shouldSelfUnionOneConceptValueColumn() { + var result = converter.conceptSelect(new BuiltInSelects.ConceptValues("value", List.of(VALUE)), context()); + var union = result.getAdditionalPredecessor().orElseThrow().getPredecessors().getFirst(); + assertEquals(1, union.getUnion().size()); + assertFalse(union.isUnionAll()); + } + + @Test + void shouldUsePreparedConceptIdExpressionForConnectorPreprocessing() { + var source = new SelectConversionContext.ConceptColumnSource( + "events", + DSL.table(DSL.name("events")), + DSL.coalesce(DSL.field(DSL.name("concept_ids", "resolved_id"), Integer.class), DSL.inline(0)), + List.of() + ); + var result = converter.convert( + new BuiltInSelects.ConceptValues("value", List.of(VALUE)), + context(List.of(source)) + ); + + assertEquals("coalesce(\"concept_ids\".\"resolved_id\", 0) as \"value\"", + render(result.getPreprocessingSelects().getFirst().toFields().getFirst())); + } + + @Test + void shouldUsePreparedConceptIdJoinForUnconvertedConceptConnector() { + var events = DSL.table(DSL.name("events")); + var mapping = DSL.table(DSL.name("concept_ids")); + var sourceTable = events.leftJoin(mapping).on( + DSL.field(DSL.name("events", "value")).eq(DSL.field(DSL.name("concept_ids", "value"))) + ); + var rawValuePresent = DSL.field(DSL.name("events", "value")).isNotNull(); + var source = new SelectConversionContext.ConceptColumnSource( + "events", + sourceTable, + DSL.coalesce(DSL.field(DSL.name("concept_ids", "resolved_id"), Integer.class), DSL.inline(0)), + List.of(rawValuePresent) + ); + var result = converter.conceptSelect( + new BuiltInSelects.ConceptValues("value", List.of(VALUE)), + context(List.of(source)) + ); + var union = result.getAdditionalPredecessor().orElseThrow().getPredecessors().getFirst(); + + assertTrue(DSL.using(SQLDialect.POSTGRES).renderInlined(union.getFromTables().getFirst()).toLowerCase(Locale.ROOT) + .contains("left outer join \"concept_ids\"")); + assertEquals(List.of(rawValuePresent), union.getConditions()); + assertEquals("cast(coalesce(\"concept_ids\".\"resolved_id\", 0) as varchar) as \"value\"", + render(union.getSelects().getSqlSelects().getFirst().toFields().getFirst())); + } + + @Test + void shouldKeepPreparedSourcesDistinctForDuplicateConceptColumns() { + var firstSource = new SelectConversionContext.ConceptColumnSource( + "events", + DSL.table(DSL.name("events")).leftJoin(DSL.table(DSL.name("concept_ids_a"))).on(DSL.trueCondition()), + DSL.field(DSL.name("concept_ids_a", "resolved_id"), Integer.class), + List.of() + ); + var secondSource = new SelectConversionContext.ConceptColumnSource( + "events", + DSL.table(DSL.name("events")).leftJoin(DSL.table(DSL.name("concept_ids_b"))).on(DSL.trueCondition()), + DSL.field(DSL.name("concept_ids_b", "resolved_id"), Integer.class), + List.of() + ); + var result = converter.conceptSelect( + new BuiltInSelects.ConceptValues("value", List.of(VALUE, VALUE)), + context(List.of(firstSource, secondSource)) + ); + var union = result.getAdditionalPredecessor().orElseThrow().getPredecessors().getFirst(); + + assertTrue(DSL.using(SQLDialect.POSTGRES).renderInlined(union.getFromTables().getFirst()).contains("concept_ids_a")); + assertTrue(DSL.using(SQLDialect.POSTGRES).renderInlined(union.getUnion().getFirst().getFromTables().getFirst()).contains("concept_ids_b")); + } + + @Test + void shouldDispatchDateUnionToIntervalPacking() { + var date = new ResolvedColumn("events.date", VALUE.table(), "date", ColumnType.DATE, true); + var result = converter.convert(new BuiltInSelects.DateUnion("value", new DateColumns.Single(date)), context()); + assertEquals("value-interval_packing_selects", result.getAdditionalPredecessor().orElseThrow().getCteName()); + assertEquals(2, result.getPreprocessingSelects().size()); + assertTrue(result.getAggregationSelects().isEmpty()); + } + + @Test + void shouldKeepEventDatesInEventBranch() { + var base = context(); + var dates = ColumnDateRange.of(DSL.field(DSL.name("start"), Date.class), DSL.field(DSL.name("end"), Date.class)); + var context = new SelectConversionContext(base.dialect(), base.nameGenerator(), base.tables(), base.ids(), Optional.of(dates), base.alias()); + for (var operation : List.of(new BuiltInSelects.EventDateUnion("value"), new BuiltInSelects.EventDurationSum("value"))) { + var result = converter.convert(operation, context); + assertEquals(1, result.getEventDateSelects().size()); + assertTrue(result.getAggregationSelects().isEmpty()); + } + } + + @Test + void shouldCompileFlagsThroughAggregation() { + var flag = new ResolvedColumn("events.flag", VALUE.table(), "flag", ColumnType.BOOLEAN, true); + var result = converter.convert(new BuiltInSelects.Aggregation("value", new BuiltInAggregations.Flags(Map.of("A", flag))), context()); + assertEquals(1, result.getAggregationSelects().size()); + assertTrue(render(result.getAggregationSelects().getFirst().toFields().getFirst()).contains("then 'a'")); + } + + @Test + void shouldCompileDateDistanceMinimum() { + var date = new ResolvedColumn("events.date", VALUE.table(), "date", ColumnType.DATE, true); + var result = converter.convert(new BuiltInSelects.DateDistance("value", date, ChronoUnit.YEARS, LocalDate.of(2020, 6, 15)), context()); + assertEquals("min(\"preprocessing\".\"value\") as \"value\"", render(result.getAggregationSelects().getFirst().toFields().getFirst())); + } + + @Test + void shouldConvertDistinctWithEventFilteringAndOrderedStringAggregation() { + var result = converter.convert(new BuiltInSelects.Values("value", VALUE, BuiltInSelects.ValueOperation.DISTINCT, Optional.empty()), context()); + var aggregation = result.getAdditionalPredecessor().orElseThrow(); + var distinct = aggregation.getPredecessors().getFirst(); + assertTrue(distinct.isSelectDistinct()); + assertEquals("value-distinct", distinct.getCteName()); + assertEquals("\"preprocessing\".\"value\"", render(distinct.getSelects().getSqlSelects().getFirst().toFields().getFirst())); + assertTrue(result.getAggregationSelects().isEmpty()); + assertTrue(render(aggregation.getSelects().getSqlSelects().getFirst().toFields().getFirst()).contains("string_agg(")); + } + + @Test + void shouldConvertFirstAndLastWithSubstringAndSeparatePredecessor() { + for (var operation : List.of(BuiltInSelects.ValueOperation.FIRST, BuiltInSelects.ValueOperation.LAST)) { + var result = converter.convert(new BuiltInSelects.Values("value", VALUE, operation, Optional.of(SubstringRange.between(2, 5))), context()); + assertEquals("substring(\"events\".\"value\", 3, 3) as \"value\"", render(result.getPreprocessingSelects().getFirst().toFields().getFirst())); + assertEquals("value-value_select_first_row_step", result.getAdditionalPredecessor().orElseThrow().getCteName()); + assertTrue(result.getAggregationSelects().isEmpty()); + assertEquals(1, result.getFinalSelects().size()); + } + } + + @Test + void shouldPreserveOpenSubstringBounds() { + var result = converter.convert(new BuiltInSelects.Values("value", VALUE, BuiltInSelects.ValueOperation.FIRST, + Optional.of(SubstringRange.from(2))), context()); + assertEquals("substring(\"events\".\"value\", 3) as \"value\"", render(result.getPreprocessingSelects().getFirst().toFields().getFirst())); + } + + @Test + void shouldNotAggregateDistinctSumPredecessorAgain() { + var amount = new ResolvedColumn("events.amount", VALUE.table(), "amount", ColumnType.INTEGER, true); + var result = converter.convert(new BuiltInSelects.Aggregation("value", new BuiltInAggregations.Sum(amount, Optional.empty(), List.of(VALUE))), context()); + assertTrue(result.getAdditionalPredecessor().isPresent()); + assertTrue(result.getAggregationSelects().isEmpty()); + assertEquals("\"aggregation\".\"value\"", render(result.getFinalSelects().getFirst().toFields().getFirst())); + } + + @Test + void shouldConvertRandomWithoutApplyingSubstring() { + var result = converter.convert(new BuiltInSelects.Values("value", VALUE, BuiltInSelects.ValueOperation.RANDOM, + Optional.of(SubstringRange.from(2))), context()); + assertEquals("\"events\".\"value\"", render(result.getPreprocessingSelects().getFirst().toFields().getFirst())); + assertEquals("random_value(\"preprocessing\".\"value\") as \"value\"", render(result.getAggregationSelects().getFirst().toFields().getFirst())); + } + + private static SelectConversionContext context() { + return context(List.of()); + } + + private static SelectConversionContext context(List conceptColumnSources) { + return new SelectConversionContext(new TestDialect(), new SqlNameGenerator(127), + new SqlTables("events", Map.of(ConceptCteStep.PREPROCESSING, "preprocessing", ConceptCteStep.AGGREGATION_SELECT, "aggregation"), + Map.of(ConceptCteStep.AGGREGATION_SELECT, ConceptCteStep.PREPROCESSING, ConceptCteStep.AGGREGATION_FILTER, ConceptCteStep.AGGREGATION_SELECT)), + new SqlIdColumns(DSL.field(DSL.name("events", "id"), String.class)), Optional.empty(), "value", Map.of(), conceptColumnSources); + } + + private static String render(Field field) { + return DSL.using(SQLDialect.POSTGRES).renderInlined(DSL.select(field)).substring("select ".length()).toLowerCase(Locale.ROOT); + } + + private static class TestDialect implements CompilerDialect { + @Override public org.jooq.Condition orAgg(Field field) { return DSL.max(field.cast(Integer.class)).gt(0); } + @Override public Field arrayOut(List> fields) { return DSL.array(fields); } + @Override public Field dateDistance(ChronoUnit unit, Field start, Field end) { + return DSL.function("date_distance", Integer.class, start, end); + } + @Override public Field dateDistance(ChronoUnit unit, Field start, LocalDate end) { + return dateDistance(unit, start, DSL.inline(Date.valueOf(end))); + } + @Override public Field cast(Field field, org.jooq.DataType type) { return field.cast(type); } + @Override public Field stringAggregation(Field field, Field delimiter, List> order) { + return DSL.field("string_agg({0}, {1} {2})", String.class, field, delimiter, DSL.orderBy(order)); + } + @Override public Field minimumDate() { return DSL.field("minimum_date", Date.class); } + @Override public Field maximumDate() { return DSL.field("maximum_date", Date.class); } + @Override public Field anyValue(Field field) { return field; } + @Override public Field random(Field field) { return DSL.function("random_value", field.getDataType(), field); } + @Override public Field renderDateRange(Field start, Field end) { return start; } + @Override public Field aggregateDateRanges(Field start, Field end) { return start; } + @Override public int getNameMaxLength() { return 127; } + } + + private record UnsupportedSelect(String name) implements ResolvedSelect { + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineAssemblerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineAssemblerTest.java index 20870a5510..3071105854 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineAssemblerTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/ir/concept/ConnectorCtePipelineAssemblerTest.java @@ -97,6 +97,33 @@ void shouldCollectEventSelectsAndAllAdditionalPredecessors() { assertEquals(List.of(directPredecessor, filterPredecessor), input.additionalPredecessors()); } + @Test + void shouldKeepValueSelectBesideAggregationWithAdditionalPredecessor() { + FieldWrapper valueSelect = new FieldWrapper<>(field(name("events", "value"), String.class)); + FieldWrapper aggregationInput = new FieldWrapper<>(field(name("events", "amount"), Integer.class)); + QueryStep aggregationPredecessor = queryStep("distinct_sum"); + ConnectorSqlSelects values = ConnectorSqlSelects.builder() + .preprocessingSelect(valueSelect) + .build(); + ConnectorSqlSelects aggregation = ConnectorSqlSelects.builder() + .preprocessingSelect(aggregationInput) + .additionalPredecessor(Optional.of(aggregationPredecessor)) + .build(); + + ConnectorCtePipelineInput input = ConnectorCtePipelineAssembler.assemble( + plan(), + table(name("events")), + new SqlIdColumns(field(name("person"), String.class)), + ColumnDateRange.of(field(name("start"), Date.class), field(name("end"), Date.class)), + List.of(values, aggregation), + List.of(), + Optional.empty() + ); + + assertEquals(List.of(values, aggregation), input.preprocessing().sqlSelects()); + assertEquals(List.of(aggregationPredecessor), input.additionalPredecessors()); + } + private static ConnectorCtePlan plan() { SqlTables tables = new SqlTables( "events", From 810157e6055f0b9119f825ed838fe0844fc92721 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 24 Sep 2026 07:33:32 +0200 Subject: [PATCH 66/81] refactor(sql): move dialect capabilities to connector Co-authored-by: Codex GPT-6 --- .../dialect/LegacyCompilerDialect.java | 71 +++--- .../clickhouse/ClickhouseDialectBundle.java | 19 +- .../dialect/hana/HanaDialectBundle.java | 9 + .../conversion/SqlExtractionDialectTest.java | 20 ++ .../clickhouse/ClickhouseCompilerDialect.java | 201 ++++++++++++++++ .../dialect/hana/HanaCompilerDialect.java | 224 ++++++++++++++++++ .../dialect/VendorCompilerDialectTest.java | 85 +++++++ 7 files changed, 584 insertions(+), 45 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/clickhouse/ClickhouseCompilerDialect.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/hana/HanaCompilerDialect.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/dialect/VendorCompilerDialectTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java index fb06feb16f..4bf10840b8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java @@ -12,10 +12,12 @@ import java.util.List; import java.util.Map; -import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.conversion.operation.SelectConversionContext; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.rendering.QueryStepRenderer; @@ -53,130 +55,127 @@ */ public interface LegacyCompilerDialect extends CompilerDialect { + @Override + default ConnectorSqlSelects distinctSelect(BuiltInSelects.Values select, SelectConversionContext context) { + return getCompilerDialect().distinctSelect(select, context); + } + @Override default ColumnDateRange toDualColumn(ColumnDateRange range) { - return getFunctionProvider().toDualColumn(range); + return getCompilerDialect().toDualColumn(range); } @Override default Condition orAgg(Field field) { - return getFunctionProvider().orAgg(field); + return getCompilerDialect().orAgg(field); } @Override default Field arrayOut(List> fields) { - return getFunctionProvider().arrayOut(fields); + return getCompilerDialect().arrayOut(fields); } @Override default Field cast(Field field, DataType type) { - return getFunctionProvider().cast(field, type); + return getCompilerDialect().cast(field, type); } @Override default Field stringAggregation(Field field, Field delimiter, List> orderByFields) { - return getFunctionProvider().stringAggregation(field, delimiter, orderByFields); + return getCompilerDialect().stringAggregation(field, delimiter, orderByFields); } @Override default Collection> orderByValidityDates( Function, ? extends SortField> ordering, List> validityDateFields) { - return getFunctionProvider().orderByValidityDates(ordering, validityDateFields); + return getCompilerDialect().orderByValidityDates(ordering, validityDateFields); } @Override default Field random(Field field) { - return getFunctionProvider().random(field); + return getCompilerDialect().random(field); } @Override default Field minimumDate() { - return getFunctionProvider().getMinDateExpression(); + return getCompilerDialect().minimumDate(); } @Override default Field maximumDate() { - return getFunctionProvider().getMaxDateExpression(); + return getCompilerDialect().maximumDate(); } @Override default Field anyValue(Field field) { - return getFunctionProvider().anyValue(field); + return getCompilerDialect().anyValue(field); } @Override default Field renderDateRange(Field start, Field end) { - return getFunctionProvider().dateRangeToField(ColumnDateRange.of(start, end)); + return getCompilerDialect().renderDateRange(start, end); } @Override default Field aggregateDateRanges(Field start, Field end) { - return getFunctionProvider().dateRangeAggregation(ColumnDateRange.of(start, end)); + return getCompilerDialect().aggregateDateRanges(start, end); } @Override default Field externalId(String id) { - return getFunctionProvider().externalId(id); + return getCompilerDialect().externalId(id); } @Override default Field externalStringValues(List values) { - return getFunctionProvider().asArrayRepr(values); + return getCompilerDialect().externalStringValues(values); } @Override default Table literalSelectTable() { - return getFunctionProvider().getNoOpTable(); + return getCompilerDialect().literalSelectTable(); } @Override default ColumnDateRange dateRangeLiteral(DateRange dateRange) { - CDateRange legacyDateRange = CDateRange.of( - dateRange.startInclusive().orElse(null), - dateRange.endInclusive().orElse(null) - ); - return getFunctionProvider().forCDateRange(legacyDateRange); + return getCompilerDialect().dateRangeLiteral(dateRange); } @Override default QueryStep unnestDateRange(ColumnDateRange dateRange, QueryStep predecessor, String cteName) { - return getFunctionProvider().unnestDaterange(dateRange, predecessor, cteName); + return getCompilerDialect().unnestDateRange(dateRange, predecessor, cteName); } @Override default String regexAnyCharacters() { - return getFunctionProvider().getAnyCharRegex(); + return getCompilerDialect().regexAnyCharacters(); } @Override default Condition regexMatches(Field field, String pattern) { - return getFunctionProvider().likeRegex(field, pattern); + return getCompilerDialect().regexMatches(field, pattern); } @Override default Field dateDistance(ChronoUnit unit, Field startDate, LocalDate endDate) { - return getFunctionProvider().dateDistance( - unit, - startDate, - getFunctionProvider().toDateField(endDate.toString()) - ); + return getCompilerDialect().dateDistance(unit, startDate, endDate); } StratificationFunctions getStratificationFunctions(); @Override default Field dateDistance(ChronoUnit unit, Field startDate, Field endDate) { - return getFunctionProvider().dateDistance(unit, startDate, endDate); + return getCompilerDialect().dateDistance(unit, startDate, endDate); } @Override default Field addDays(Field date, Field days) { - return getFunctionProvider().addDays(date, days); + return getCompilerDialect().addDays(date, days); } @Override default ColumnDateRange dateRange(Field start, Field inclusiveEnd) { - return getFunctionProvider().dateRange(start, inclusiveEnd); + return getCompilerDialect().dateRange(start, inclusiveEnd); } @Override @@ -188,21 +187,23 @@ default ColumnDateRange dateRangeColumn(Field range) { @Override default Field yearQuarter(Field date) { - return getFunctionProvider().yearQuarter(date); + return getCompilerDialect().yearQuarter(date); } @Override default Field quarterStart(Field date) { - return getStratificationFunctions().jumpToQuarterStart(date); + return getCompilerDialect().quarterStart(date); } @Override default Field nextQuarterStart(Field date) { - return getStratificationFunctions().jumpToNextQuarterStart(date); + return getCompilerDialect().nextQuarterStart(date); } SqlFunctionProvider getFunctionProvider(); + CompilerDialect getCompilerDialect(); + List> getNodeConverters(DSLContext context); default List> getDefaultNodeConverters(DSLContext dslContext) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java index 0f318ae941..3b2959e09a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java @@ -1,8 +1,7 @@ package com.bakdata.conquery.sql.conversion.dialect.clickhouse; -import com.bakdata.conquery.sql.compiler.conversion.operation.SelectConversionContext; -import com.bakdata.conquery.sql.model.operation.BuiltInSelects; -import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.dialect.clickhouse.ClickhouseCompilerDialect; import java.util.List; import java.util.Map; @@ -27,17 +26,12 @@ @Slf4j public class ClickhouseDialectBundle implements DialectBundle { - @Override - public ConnectorSqlSelects distinctSelect( - BuiltInSelects.Values select, - SelectConversionContext context) { - return com.bakdata.conquery.sql.compiler.conversion.operation.ClickhouseDistinctSelectConverter.connectorSelect(select, context); - } - private final SqlFunctionProvider functionProvider; + private final CompilerDialect compilerDialect; public ClickhouseDialectBundle() { this.functionProvider = new ClickhouseFunctionProvider(); + this.compilerDialect = new ClickhouseCompilerDialect(); } @Override @@ -80,6 +74,11 @@ public SqlFunctionProvider getFunctionProvider() { return this.functionProvider; } + @Override + public CompilerDialect getCompilerDialect() { + return compilerDialect; + } + @Override public ResultSetProcessor getResultSetProcessor(ConqueryConfig config) { return new ClickhouseResultSetProcessor(config); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaDialectBundle.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaDialectBundle.java index 018012a139..5f27a82470 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaDialectBundle.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaDialectBundle.java @@ -10,6 +10,8 @@ import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.dialect.hana.HanaCompilerDialect; import com.bakdata.conquery.sql.execution.DefaultResultSetProcessor; import com.bakdata.conquery.sql.execution.HanaSqlCDateSetParser; import com.bakdata.conquery.sql.execution.ResultSetProcessor; @@ -21,10 +23,12 @@ public class HanaDialectBundle implements DialectBundle { private final SqlFunctionProvider functionProvider; + private final CompilerDialect compilerDialect; private final SqlCDateSetParser dateSetParser; public HanaDialectBundle() { this.functionProvider = new HanaSqlFunctionProvider(); + this.compilerDialect = new HanaCompilerDialect(); this.dateSetParser = new HanaSqlCDateSetParser(); } @@ -82,4 +86,9 @@ public SqlFunctionProvider getFunctionProvider() { return this.functionProvider; } + @Override + public CompilerDialect getCompilerDialect() { + return compilerDialect; + } + } diff --git a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionDialectTest.java b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionDialectTest.java index 4ee70cab08..ce8538516b 100644 --- a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionDialectTest.java +++ b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionDialectTest.java @@ -1,11 +1,16 @@ package com.bakdata.conquery.sql.conversion; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.sql.Date; import java.util.Locale; import com.bakdata.conquery.sql.conversion.dialect.clickhouse.ClickhouseDialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.hana.HanaDialectBundle; +import com.bakdata.conquery.sql.compiler.dialect.clickhouse.ClickhouseCompilerDialect; +import com.bakdata.conquery.sql.compiler.dialect.hana.HanaCompilerDialect; import org.jooq.SQLDialect; import org.jooq.impl.DSL; import org.junit.jupiter.api.Test; @@ -23,4 +28,19 @@ void shouldKeepClickhouseRangeBoundsNullableAfterCoalesce() { assertTrue(sql.contains("::nullable(date32)"), sql); } } + + @Test + void shouldUseFrameworkNeutralDialectCapabilitiesFromLegacyBundles() { + var clickhouse = new ClickhouseDialectBundle(); + var hana = new HanaDialectBundle(); + + assertInstanceOf(ClickhouseCompilerDialect.class, clickhouse.getCompilerDialect()); + assertInstanceOf(HanaCompilerDialect.class, hana.getCompilerDialect()); + assertEquals( + DSL.using(SQLDialect.CLICKHOUSE).renderInlined(clickhouse.getFunctionProvider().getMinDateExpression()), + DSL.using(SQLDialect.CLICKHOUSE).renderInlined(clickhouse.minimumDate())); + assertEquals( + DSL.using(SQLDialect.DEFAULT).renderInlined(hana.getFunctionProvider().getMaxDateExpression()), + DSL.using(SQLDialect.DEFAULT).renderInlined(hana.maximumDate())); + } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/clickhouse/ClickhouseCompilerDialect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/clickhouse/ClickhouseCompilerDialect.java new file mode 100644 index 0000000000..4d2f3ae270 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/clickhouse/ClickhouseCompilerDialect.java @@ -0,0 +1,201 @@ +package com.bakdata.conquery.sql.compiler.dialect.clickhouse; + +import static com.bakdata.conquery.sql.compiler.dialect.Interval.MONTHS_PER_QUARTER; +import static org.jooq.impl.DSL.*; + +import com.bakdata.conquery.sql.compiler.conversion.operation.SelectConversionContext; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; +import java.sql.Date; +import java.time.LocalDate; +import java.time.temporal.ChronoUnit; +import java.util.Collection; +import java.util.List; +import java.util.function.Function; +import org.jooq.Condition; +import org.jooq.DataType; +import org.jooq.Field; +import org.jooq.OrderField; +import org.jooq.SortField; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; + +/** ClickHouse SQL capabilities used by the framework-neutral compiler. */ +public class ClickhouseCompilerDialect implements CompilerDialect { + + public static final int MIN_DATE_VALUE = -25567; + public static final int MAX_DATE_VALUE = 24855; + + @Override + public int getNameMaxLength() { + return 64; + } + + @Override + public Field minimumDate() { + return field("toDate32({0})", Date.class, MIN_DATE_VALUE); + } + + @Override + public Field maximumDate() { + return field("toDate32({0})", Date.class, MAX_DATE_VALUE); + } + + @Override + public Field anyValue(Field value) { + return DSL.anyValue(value); + } + + @Override + public Condition orAgg(Field value) { + return condition(max(value.cast(Integer.class)).gt(0)); + } + + @Override + public Field externalStringValues(List values) { + return array(values.toArray()); + } + + @Override + public ConnectorSqlSelects distinctSelect( + BuiltInSelects.Values select, SelectConversionContext context) { + return com.bakdata.conquery.sql.compiler.conversion.operation.ClickhouseDistinctSelectConverter + .connectorSelect(select, context); + } + + @Override + public Field externalId(String id) { + return field("{0}::Nullable(String)", String.class, inline(id, String.class)); + } + + @Override + public Field arrayOut(List> fields) { + return field("arrayFilter(x -> x <> '', {0})", Object.class, array(fields)); + } + + @Override + public ColumnDateRange dateRange(Field start, Field inclusiveEnd) { + return ColumnDateRange.of( + field("{0}::Nullable(Date32)", Date.class, coalesce(start, minimumDate())), + field( + "{0}::Nullable(Date32)", + Date.class, coalesce(addDays(inclusiveEnd, inline(1)), maximumDate()))); + } + + @Override + public Field addDays(Field date, Field days) { + return function("addDays", Date.class, date, days); + } + + @Override + public Field stringAggregation( + Field value, Field delimiter, List> orderByFields) { + return field( + "{0}({1}, {2} {3})", + String.class, keyword("string_agg"), value, delimiter, orderBy(orderByFields)); + } + + @Override + public Collection> orderByValidityDates( + Function, ? extends SortField> ordering, List> fields) { + return List.of( + ordering.apply(nullif(fields.getFirst(), minimumDate())).nullsLast(), + ordering.apply(nullif(fields.getLast(), maximumDate())).nullsLast()); + } + + @Override + public Field cast(Field value, DataType type) { + if (type == SQLDataType.VARCHAR) return function("toString", type.getType(), value); + return function( + name("CAST"), type.getType(), field("{0} AS {1}", value, keyword(type.getName()))); + } + + public Field toDateField(String expression) { + return function("toDate", Date.class, inline(expression), inline("yyyy-mm-dd")); + } + + @Override + public Field dateDistance(ChronoUnit unit, Field start, LocalDate end) { + return dateDistance(unit, start, toDateField(end.toString())); + } + + @Override + public Field dateDistance(ChronoUnit unit, Field start, Field end) { + String unitName = + switch (unit) { + case DAYS -> "days"; + case MONTHS -> "months"; + case YEARS, DECADES, CENTURIES -> "years"; + default -> + throw new UnsupportedOperationException( + "Given ChronoUnit %s is not supported.".formatted(unit)); + }; + Field distance = function("age", Integer.class, inline(unitName), start, end); + distance = + switch (unit) { + case DECADES -> distance.divide(10); + case CENTURIES -> distance.divide(100); + default -> distance; + }; + return distance.cast(Integer.class); + } + + @Override + public Field random(Field value) { + return field("groupArraySample(1)({0})[1]", value.getType(), value); + } + + @Override + public Condition regexMatches(Field value, String pattern) { + return condition(function("match", Boolean.class, value, inline(pattern))); + } + + @Override + public Field yearQuarter(Field date) { + return field("formatDateTime({0}, '%Y-Q%Q')", String.class, date); + } + + @Override + public Field quarterStart(Field date) { + Field quarter = + cast(function("RIGHT", String.class, yearQuarter(date), inline(1)), SQLDataType.INTEGER); + return function( + "addMonths", + Date.class, + function("toStartOfYear", Date.class, date), + quarter.minus(1).times(MONTHS_PER_QUARTER)); + } + + @Override + public Field nextQuarterStart(Field date) { + Field quarter = + cast(function("RIGHT", String.class, yearQuarter(date), inline(1)), SQLDataType.INTEGER); + return function( + "addMonths", + Date.class, + function("toStartOfYear", Date.class, date), + quarter.times(MONTHS_PER_QUARTER)); + } + + @Override + public Field renderDateRange(Field start, Field end) { + return function( + "tuple", + Object.class, + field("{0}::Nullable(Integer)", Object.class, start), + field("{0}::Nullable(Integer)", Object.class, end)); + } + + @Override + public Field aggregateDateRanges(Field start, Field end) { + return field("groupArray({0})", Object[].class, renderDateRange(start, end)); + } + + @Override + public QueryStep unnestDateRange(ColumnDateRange range, QueryStep predecessor, String cteName) { + return predecessor; + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/hana/HanaCompilerDialect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/hana/HanaCompilerDialect.java new file mode 100644 index 0000000000..a1b1db3794 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/hana/HanaCompilerDialect.java @@ -0,0 +1,224 @@ +package com.bakdata.conquery.sql.compiler.dialect.hana; + +import static com.bakdata.conquery.sql.compiler.dialect.Interval.MONTHS_PER_QUARTER; +import static org.jooq.impl.DSL.*; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.model.range.DateRange; +import java.sql.Date; +import java.time.LocalDate; +import java.time.temporal.ChronoUnit; +import java.util.Collection; +import java.util.List; +import java.util.function.Function; +import org.jooq.Condition; +import org.jooq.DataType; +import org.jooq.Field; +import org.jooq.OrderField; +import org.jooq.Record; +import org.jooq.SortField; +import org.jooq.Table; +import org.jooq.impl.SQLDataType; + +/** HANA SQL capabilities used by the framework-neutral compiler. */ +public class HanaCompilerDialect implements CompilerDialect { + + public static final String MAX_DATE_VALUE = "9999-12-31"; + public static final String MIN_DATE_VALUE = "0001-01-01"; + public static final String DATERANGE_SEPARATOR = "/"; + private static final char DATE_SET_SEPARATOR = (char) 31; + + @Override + public int getNameMaxLength() { + return 127; + } + + @Override + public String regexAnyCharacters() { + return ".*"; + } + + @Override + public Condition regexMatches(Field value, String pattern) { + return condition("{0} {1} {2}", value, keyword("LIKE_REGEXPR"), pattern); + } + + @Override + public Table literalSelectTable() { + return table(name("DUMMY")); + } + + public Condition unconditionalJoinCondition() { + return inline(true).eq(inline(true)); + } + + public Field toDateField(String expression) { + return function("TO_DATE", Date.class, inline(expression), inline("yyyy-mm-dd")); + } + + @Override + public Field minimumDate() { + return toDateField(MIN_DATE_VALUE); + } + + @Override + public Field maximumDate() { + return toDateField(MAX_DATE_VALUE); + } + + @Override + public Field anyValue(Field value) { + return min(value); + } + + @Override + public Condition orAgg(Field value) { + return condition(max(value.cast(Integer.class)).gt(0)); + } + + @Override + public Field arrayOut(List> fields) { + return field( + fields.stream() + .map(Field::toString) + .collect(java.util.stream.Collectors.joining(" || '" + DATE_SET_SEPARATOR + "' || ")), + String.class); + } + + @Override + public Field externalStringValues(List values) { + return field( + values.stream() + .map(org.jooq.impl.DSL::inline) + .map(Field::toString) + .collect(java.util.stream.Collectors.joining(" || '" + DATE_SET_SEPARATOR + "' || ")), + Object.class); + } + + @Override + public Field cast(Field value, DataType type) { + if (type == SQLDataType.VARCHAR) return function("TO_VARCHAR", type.getType(), value); + return function( + unquotedName("CAST"), type.getType(), field("{0} AS {1}", value, keyword(type.getName()))); + } + + @Override + public Field stringAggregation( + Field value, Field delimiter, List> orderByFields) { + return field( + "{0}({1}, {2} {3})", + String.class, keyword("string_agg"), value, delimiter, orderBy(orderByFields)); + } + + @Override + public Collection> orderByValidityDates( + Function, ? extends SortField> ordering, List> fields) { + return List.of( + ordering.apply(nullif(fields.getFirst(), minimumDate())).nullsLast(), + ordering.apply(nullif(fields.getLast(), maximumDate())).nullsLast()); + } + + @Override + public Field dateDistance(ChronoUnit unit, Field start, LocalDate end) { + return dateDistance(unit, start, toDateField(end.toString())); + } + + @Override + public ColumnDateRange dateRangeLiteral(DateRange range) { + Field start = + range + .startInclusive() + .>map(value -> toDateField(value.toString())) + .orElseGet(this::minimumDate); + Field end = + range + .endInclusive() + .>map(value -> toDateField(value.plusDays(1).toString())) + .orElseGet(this::maximumDate); + return ColumnDateRange.of(start, end); + } + + @Override + public Field dateDistance(ChronoUnit unit, Field start, Field end) { + String functionName = + switch (unit) { + case DAYS -> "DAYS_BETWEEN"; + case MONTHS -> "MONTHS_BETWEEN"; + case YEARS, DECADES, CENTURIES -> "YEARS_BETWEEN"; + default -> + throw new UnsupportedOperationException( + "Given ChronoUnit %s is not supported.".formatted(unit)); + }; + Field distance = function(functionName, Integer.class, start, end); + distance = + switch (unit) { + case DECADES -> distance.divide(10); + case CENTURIES -> distance.divide(100); + default -> distance; + }; + return distance.cast(Integer.class); + } + + @Override + public Field addDays(Field date, Field days) { + return function("ADD_DAYS", Date.class, date, days); + } + + @Override + public Field random(Field value) { + return field( + "{0}({1} {2})", + value.getType(), keyword("FIRST_VALUE"), value, orderBy(function("RAND", Object.class))); + } + + @Override + public Field yearQuarter(Field date) { + return function("QUARTER", String.class, date); + } + + @Override + public Field quarterStart(Field date) { + Field quarter = + cast(function("RIGHT", String.class, yearQuarter(date), inline(1)), SQLDataType.INTEGER); + return function( + "ADD_MONTHS", Date.class, yearStart(date), quarter.minus(1).times(MONTHS_PER_QUARTER)); + } + + @Override + public Field nextQuarterStart(Field date) { + Field quarter = + cast(function("RIGHT", String.class, yearQuarter(date), inline(1)), SQLDataType.INTEGER); + return function("ADD_MONTHS", Date.class, yearStart(date), quarter.times(MONTHS_PER_QUARTER)); + } + + private static Field yearStart(Field date) { + return field( + "SERIES_ROUND({0}, {1}, {2})", + Date.class, date, inline("INTERVAL 1 YEAR"), keyword("ROUND_DOWN")); + } + + @Override + public Field renderDateRange(Field start, Field end) { + return field( + "'[' || {0} || {2} || {1} || ')'", + String.class, + cast(start, SQLDataType.VARCHAR), + cast(end, SQLDataType.VARCHAR), + DATERANGE_SEPARATOR); + } + + @Override + public Field aggregateDateRanges(Field start, Field end) { + return stringAggregation( + (Field) renderDateRange(start, end), + toChar(DATE_SET_SEPARATOR), + List.of(coalesce(start, minimumDate()))); + } + + @Override + public QueryStep unnestDateRange(ColumnDateRange range, QueryStep predecessor, String cteName) { + return predecessor; + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/dialect/VendorCompilerDialectTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/dialect/VendorCompilerDialectTest.java new file mode 100644 index 0000000000..b642acd143 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/dialect/VendorCompilerDialectTest.java @@ -0,0 +1,85 @@ +package com.bakdata.conquery.sql.compiler.dialect; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.bakdata.conquery.sql.compiler.dialect.clickhouse.ClickhouseCompilerDialect; +import com.bakdata.conquery.sql.compiler.dialect.hana.HanaCompilerDialect; +import com.bakdata.conquery.sql.model.range.DateRange; +import java.sql.Date; +import java.time.LocalDate; +import java.time.temporal.ChronoUnit; +import java.util.List; +import org.jooq.Field; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Test; + +class VendorCompilerDialectTest { + + private static final Field START = field(name("events", "start"), Date.class); + private static final Field END = field(name("events", "end"), Date.class); + + @Test + void shouldRenderHanaCapabilitiesWithoutBackendTypes() { + var dialect = new HanaCompilerDialect(); + + assertEquals("127", Integer.toString(dialect.getNameMaxLength())); + assertTrue(render(dialect.minimumDate(), SQLDialect.DEFAULT).contains("TO_DATE('0001-01-01'")); + assertTrue(render(dialect.maximumDate(), SQLDialect.DEFAULT).contains("TO_DATE('9999-12-31'")); + assertTrue( + render(dialect.dateDistance(ChronoUnit.DAYS, START, END), SQLDialect.DEFAULT) + .contains("DAYS_BETWEEN")); + assertTrue(render(dialect.quarterStart(START), SQLDialect.DEFAULT).contains("SERIES_ROUND")); + assertTrue(render(dialect.nextQuarterStart(START), SQLDialect.DEFAULT).contains("ADD_MONTHS")); + assertTrue( + render(dialect.externalStringValues(List.of("a", "b")), SQLDialect.DEFAULT) + .contains(Character.toString((char) 31))); + assertEquals("\"DUMMY\"", render(dialect.literalSelectTable(), SQLDialect.DEFAULT)); + assertFalse(dialect.supportsSingleColumnRanges()); + } + + @Test + void shouldRenderClickhouseCapabilitiesWithoutBackendTypes() { + var dialect = new ClickhouseCompilerDialect(); + + assertEquals(64, dialect.getNameMaxLength()); + assertTrue(render(dialect.minimumDate(), SQLDialect.CLICKHOUSE).contains("toDate32(-25567)")); + assertTrue(render(dialect.maximumDate(), SQLDialect.CLICKHOUSE).contains("toDate32(24855)")); + assertTrue( + render(dialect.dateRange(START, END).getStart(), SQLDialect.CLICKHOUSE) + .contains("::Nullable(Date32)")); + assertTrue( + render(dialect.dateDistance(ChronoUnit.MONTHS, START, END), SQLDialect.CLICKHOUSE) + .contains("age('months'")); + assertTrue( + render(dialect.quarterStart(START), SQLDialect.CLICKHOUSE).contains("toStartOfYear")); + assertTrue( + render(dialect.nextQuarterStart(START), SQLDialect.CLICKHOUSE).contains("addMonths")); + assertTrue( + render(dialect.externalStringValues(List.of("a", "b")), SQLDialect.CLICKHOUSE) + .contains("array")); + assertTrue( + render(dialect.externalId("id"), SQLDialect.CLICKHOUSE).contains("::Nullable(String)")); + } + + @Test + void shouldKeepInclusiveLiteralRangeSemantics() { + var range = + new HanaCompilerDialect() + .dateRangeLiteral( + new DateRange( + java.util.Optional.of(LocalDate.of(2020, 1, 1)), + java.util.Optional.of(LocalDate.of(2020, 1, 31)))); + + assertTrue(render(range.getStart(), SQLDialect.DEFAULT).contains("2020-01-01")); + assertTrue(render(range.getEnd(), SQLDialect.DEFAULT).contains("2020-02-01")); + } + + private static String render(org.jooq.QueryPart part, SQLDialect dialect) { + return DSL.using(dialect).renderInlined(part).replace('`', '"'); + } +} From ac2df44325cf9fd167574c1a7a9e97812a279726 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 28 Sep 2026 09:01:37 +0200 Subject: [PATCH 67/81] refactor(sql): add resolved query compiler Co-authored-by: Codex GPT-6 --- .../sql/conversion/ResolvedQueryAdapter.java | 44 ++++ .../sql/conversion/model/SqlQuery.java | 5 + .../conversion/SqlExtractionQueryTest.java | 83 +++++++ .../sql/compiler/DefaultSqlCompiler.java | 121 ++++++++++ .../sql/compiler/ResolvedConceptCompiler.java | 160 +++++++++++++ .../conquery/sql/model/node/ConceptNode.java | 9 + .../conquery/sql/model/node/NegationNode.java | 10 +- .../sql/compiler/DefaultSqlCompilerTest.java | 210 ++++++++++++++++++ .../conquery/sql/model/ResolvedQueryTest.java | 23 +- 9 files changed, 660 insertions(+), 5 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryAdapter.java create mode 100644 backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionQueryTest.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/DefaultSqlCompiler.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ResolvedConceptCompiler.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/DefaultSqlCompilerTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryAdapter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryAdapter.java new file mode 100644 index 0000000000..227719bc64 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryAdapter.java @@ -0,0 +1,44 @@ +package com.bakdata.conquery.sql.conversion; + +import java.util.List; + +import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.sql.compiler.ColumnRole; +import com.bakdata.conquery.sql.compiler.CompiledQuery; +import com.bakdata.conquery.sql.compiler.SqlCompiler; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.conversion.model.SqlQuery; +import com.bakdata.conquery.sql.model.ResolvedQuery; +import com.bakdata.conquery.sql.validation.ResolvedQueryValidation; +import jakarta.validation.Validator; + +/** Validates resolved application input and adapts connector output to the existing backend query contract. */ +public final class ResolvedQueryAdapter { + + private final ResolvedQueryValidation validation; + private final SqlCompiler compiler; + + public ResolvedQueryAdapter(Validator validator, SqlCompiler compiler) { + this.validation = new ResolvedQueryValidation(validator); + this.compiler = compiler; + } + + public SqlQuery compile( + ResolvedQuery resolvedQuery, + CompilerDialect dialect, + List resultInfos + ) { + validation.validate(resolvedQuery); + CompiledQuery compiled = compiler.compile(resolvedQuery, dialect); + long resultColumnCount = compiled.columns().stream() + .filter(column -> column.role() == ColumnRole.RESULT) + .count(); + if (resultColumnCount != resultInfos.size()) { + throw new IllegalStateException( + "Compiler returned %d result columns for %d backend result descriptors" + .formatted(resultColumnCount, resultInfos.size()) + ); + } + return SqlQuery.fromCompiled(compiled, resultInfos); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlQuery.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlQuery.java index 925f88efee..5f031684ff 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlQuery.java @@ -4,6 +4,7 @@ import java.util.List; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.sql.compiler.CompiledQuery; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; @@ -23,6 +24,10 @@ public SqlQuery(Select finalQuery, List resultInfos) { this.resultInfos = resultInfos; } + public static SqlQuery fromCompiled(CompiledQuery compiledQuery, List resultInfos) { + return new SqlQuery(compiledQuery.sql(), List.copyOf(resultInfos)); + } + /** * For testing purposes */ diff --git a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionQueryTest.java b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionQueryTest.java new file mode 100644 index 0000000000..8038d5b1f2 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionQueryTest.java @@ -0,0 +1,83 @@ +package com.bakdata.conquery.sql.conversion; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.sql.compiler.dialect.hana.HanaCompilerDialect; +import com.bakdata.conquery.sql.compiler.ColumnRole; +import com.bakdata.conquery.sql.compiler.CompiledColumn; +import com.bakdata.conquery.sql.compiler.CompiledQuery; +import com.bakdata.conquery.sql.model.result.ResultType; +import com.bakdata.conquery.sql.model.ResolvedQuery; +import com.bakdata.conquery.sql.model.node.AllEntitiesNode; +import com.bakdata.conquery.sql.model.schema.EntitySchema; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.SqlTable; +import jakarta.validation.ConstraintViolationException; +import jakarta.validation.Validation; +import org.junit.jupiter.api.Test; + +class SqlExtractionQueryTest { + + @Test + void shouldAdaptValidatedCompilerOutputToTheExistingSqlQueryContract() { + ResolvedQuery query = validAllEntitiesQuery(); + ResolvedQueryAdapter adapter = new ResolvedQueryAdapter( + Validation.buildDefaultValidatorFactory().getValidator(), + (resolved, dialect) -> new CompiledQuery( + "select entity_id from entities", + List.of(new CompiledColumn( + "entity-id", "entity_id", ResultType.Primitive.STRING, ColumnRole.ENTITY_ID + )) + ) + ); + + var adapted = adapter.compile(query, new HanaCompilerDialect(), List.of()); + + assertEquals("select entity_id from entities", adapted.getSql()); + assertEquals(List.of(), adapted.getResultInfos()); + } + + @Test + void shouldRejectInvalidResolvedInputBeforeInvokingTheCompiler() { + AtomicBoolean compilerInvoked = new AtomicBoolean(); + ResolvedQueryAdapter adapter = new ResolvedQueryAdapter( + Validation.buildDefaultValidatorFactory().getValidator(), + (query, dialect) -> { + compilerInvoked.set(true); + throw new AssertionError("compiler must not be invoked"); + } + ); + SqlTable entities = SqlTable.of("entities", "entities"); + ResolvedQuery invalid = new ResolvedQuery( + new EntitySchema(new ResolvedColumn( + "entity-id", entities, "id", ColumnType.INTEGER, false + )), + new AllEntitiesNode(), + false, + List.of() + ); + + assertThrows(ConstraintViolationException.class, () -> adapter.compile( + invalid, new HanaCompilerDialect(), List.of() + )); + assertFalse(compilerInvoked.get()); + } + + private static ResolvedQuery validAllEntitiesQuery() { + SqlTable entities = SqlTable.of("entities", "entities"); + return new ResolvedQuery( + new EntitySchema(new ResolvedColumn( + "entity-id", entities, "id", ColumnType.STRING, false + )), + new AllEntitiesNode(), + false, + List.of() + ); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/DefaultSqlCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/DefaultSqlCompiler.java new file mode 100644 index 0000000000..7b5651ff14 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/DefaultSqlCompiler.java @@ -0,0 +1,121 @@ +package com.bakdata.conquery.sql.compiler; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.models.query.DateAggregationAction; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.AllEntitiesQueryStepCompiler; +import com.bakdata.conquery.sql.compiler.ir.ExternalQueryStepCompiler; +import com.bakdata.conquery.sql.compiler.ir.ExternalQuerySteps; +import com.bakdata.conquery.sql.compiler.ir.FinalQueryStepComposer; +import com.bakdata.conquery.sql.compiler.ir.JoinMode; +import com.bakdata.conquery.sql.compiler.ir.LogicalQueryStepCompiler; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStepComposer; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import com.bakdata.conquery.sql.compiler.rendering.QueryStepRenderer; +import com.bakdata.conquery.sql.model.ResolvedQuery; +import com.bakdata.conquery.sql.model.node.AllEntitiesNode; +import com.bakdata.conquery.sql.model.node.AndNode; +import com.bakdata.conquery.sql.model.node.ConceptNode; +import com.bakdata.conquery.sql.model.node.DateRestrictionNode; +import com.bakdata.conquery.sql.model.node.ExternalNode; +import com.bakdata.conquery.sql.model.node.NegationNode; +import com.bakdata.conquery.sql.model.node.OrNode; +import com.bakdata.conquery.sql.model.node.QueryNode; +import com.bakdata.conquery.sql.model.range.DateRange; +import com.bakdata.conquery.sql.model.result.ResultColumn; +import com.bakdata.conquery.sql.model.result.ResultType; +import com.bakdata.conquery.sql.model.schema.EntitySchema; +import org.jooq.DSLContext; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Select; +import org.jooq.conf.ParamType; + +/** Default compiler for framework-neutral, fully resolved concept queries. */ +public final class DefaultSqlCompiler implements SqlCompiler { + + private final QueryStepRenderer renderer; + + public DefaultSqlCompiler(DSLContext dslContext) { + this.renderer = new QueryStepRenderer(dslContext); + } + + @Override + public CompiledQuery compile(ResolvedQuery query, CompilerDialect dialect) { + SqlNameGenerator names = new SqlNameGenerator(dialect.getNameMaxLength()); + CompiledStep compiled = compileNode(query.root(), query.entitySchema(), dialect, names, false, Optional.empty()); + QueryStep root = compiled.step(); + if (root.isNegate()) { + root = QueryStepComposer.antiJoinWithAllEntities( + root, query.entitySchema(), rootNegationAction(query.root()), dialect); + } + QueryStep finalStep = FinalQueryStepComposer.compose(root, compiled.externalValues(), query.includeValidityDate(), dialect); + Select rendered = renderer.toSelectQuery(finalStep, dialect); + return new CompiledQuery(rendered.getSQL(ParamType.INLINED), compiledColumns(query, rendered.getSelect(), finalStep)); + } + + private static DateAggregationAction rootNegationAction(QueryNode node) { + return switch (node) { + case NegationNode negation -> negation.dateAction(); + case DateRestrictionNode restriction -> rootNegationAction(restriction.child()); + default -> DateAggregationAction.BLOCK; + }; + } + + private static List compiledColumns(ResolvedQuery query, List> fields, QueryStep step) { + int idCount = 1; + if (fields.size() != idCount + query.resultColumns().size()) { + throw new IllegalStateException("Compiled projection does not match the declared result columns"); + } + List columns = new ArrayList<>(fields.size()); + for (int index = 0; index < idCount; index++) { + String outputId = index == 0 ? query.entitySchema().primaryId().logicalId() : "entity-id-%d".formatted(index + 1); + columns.add(new CompiledColumn(outputId, fields.get(index).getName(), ResultType.Primitive.STRING, ColumnRole.ENTITY_ID)); + } + for (int index = 0; index < query.resultColumns().size(); index++) { + ResultColumn result = query.resultColumns().get(index); + columns.add(new CompiledColumn(result.outputId(), fields.get(idCount + index).getName(), result.type(), ColumnRole.RESULT)); + } + return columns; + } + + private static CompiledStep compileNode(QueryNode node, EntitySchema schema, CompilerDialect dialect, + SqlNameGenerator names, boolean negate, Optional restriction) { + return switch (node) { + case AllEntitiesNode all -> new CompiledStep( + AllEntitiesQueryStepCompiler.compile(all, schema, dialect).toBuilder().negate(negate).build(), Optional.empty()); + case ExternalNode external -> { + ExternalQuerySteps steps = ExternalQueryStepCompiler.compile(external, negate, restriction, dialect); + yield new CompiledStep(steps.entities(), steps.values()); + } + case DateRestrictionNode restricted -> compileNode( + restricted.child(), schema, dialect, names, negate, Optional.of(restricted.dateRange())); + case NegationNode negation -> compileNode(negation.child(), schema, dialect, names, true, restriction); + case AndNode and -> compileLogical(and.children(), JoinMode.INNER, and.dateAction(), and.createExists(), + schema, dialect, names, negate, restriction); + case OrNode or -> compileLogical(or.children(), JoinMode.FULL_OUTER, or.dateAction(), or.createExists(), + schema, dialect, names, negate, restriction); + case ConceptNode concept -> new CompiledStep( + ResolvedConceptCompiler.compile(concept, schema, dialect, names, negate, restriction), Optional.empty()); + }; + } + + private static CompiledStep compileLogical(List children, JoinMode mode, DateAggregationAction action, + boolean createExists, EntitySchema schema, CompilerDialect dialect, SqlNameGenerator names, + boolean negate, Optional restriction) { + List compiled = children.stream() + .map(child -> compileNode(child, schema, dialect, names, negate, restriction)).toList(); + QueryStep step = LogicalQueryStepCompiler.compile(compiled.stream().map(CompiledStep::step).toList(), + mode, action, createExists, schema, dialect, names); + Optional external = compiled.stream().flatMap(value -> value.externalValues().stream()) + .reduce((first, second) -> second); + return new CompiledStep(step, external); + } + + private record CompiledStep(QueryStep step, Optional externalValues) { + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ResolvedConceptCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ResolvedConceptCompiler.java new file mode 100644 index 0000000000..667a68caf7 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ResolvedConceptCompiler.java @@ -0,0 +1,160 @@ +package com.bakdata.conquery.sql.compiler; + +import java.sql.Date; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import com.bakdata.conquery.models.query.DateAggregationAction; +import com.bakdata.conquery.sql.compiler.conversion.operation.FilterConversionContext; +import com.bakdata.conquery.sql.compiler.conversion.operation.ResolvedConditionConverter; +import com.bakdata.conquery.sql.compiler.conversion.operation.ResolvedFilterConverter; +import com.bakdata.conquery.sql.compiler.conversion.operation.ResolvedSelectConverter; +import com.bakdata.conquery.sql.compiler.conversion.operation.SelectConversionContext; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.JoinMode; +import com.bakdata.conquery.sql.compiler.ir.LogicalQueryStepCompiler; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.SchemaSql; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.SqlTables; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteCompiler; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteInput; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCtePlanner; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; +import com.bakdata.conquery.sql.compiler.ir.concept.ConceptSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCteCompiler; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCtePipelineAssembler; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCtePlan; +import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; +import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; +import com.bakdata.conquery.sql.compiler.ir.condition.DateRestrictionCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; +import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import com.bakdata.conquery.sql.model.node.ConceptNode; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; +import com.bakdata.conquery.sql.model.operation.ResolvedSelect; +import com.bakdata.conquery.sql.model.range.DateRange; +import com.bakdata.conquery.sql.model.schema.EntitySchema; +import com.bakdata.conquery.sql.model.schema.ResolvedConnector; +import com.bakdata.conquery.sql.model.schema.ResolvedValidityDate; +import lombok.experimental.UtilityClass; +import org.jooq.Condition; +import org.jooq.impl.DSL; + +/** Connects resolved concepts to the connector-owned CTE pipeline. */ +@UtilityClass +final class ResolvedConceptCompiler { + + static QueryStep compile(ConceptNode concept, EntitySchema entitySchema, CompilerDialect dialect, + SqlNameGenerator names, boolean negate, Optional restriction) { + ResolvedSelectConverter selectConverter = new ResolvedSelectConverter(); + List connectors = concept.connectors().stream() + .map(connector -> compileConnector(concept, connector, dialect, names, restriction, selectConverter)) + .flatMap(Optional::stream) + .toList(); + QueryStep joined = LogicalQueryStepCompiler.compile( + connectors.stream().map(ConnectorCompilation::step).toList(), JoinMode.FULL_OUTER, + DateAggregationAction.MERGE, false, entitySchema, dialect, names); + SqlTables tables = ConceptCtePlanner.planConcept(joined, names.conceptName(concept), + concept.selects().stream().anyMatch(ResolvedConceptCompiler::isEventDateSelect), dialect, names); + Selects joinedSelects = joined.getQualifiedSelects(); + Map connectorTables = new LinkedHashMap<>(); + connectors.forEach(item -> connectorTables.put(item.connector().table().physicalName().getLast(), + item.plan().tables().cteName(ConceptCteStep.PREPROCESSING))); + List selects = concept.selects().stream() + .map(select -> selectConverter.conceptSelect(select, new SelectConversionContext( + dialect, names, tables, joinedSelects.getIds(), joinedSelects.getValidityDate(), + names.selectName(select), connectorTables))) + .toList(); + return ConceptCteCompiler.compileConcept(new ConceptCteInput(joined, selects, tables, negate)); + } + + private static Optional compileConnector(ConceptNode concept, ResolvedConnector connector, + CompilerDialect dialect, SqlNameGenerator names, + Optional restriction, ResolvedSelectConverter selectConverter) { + String connectorName = names.conceptConnectorName(concept, connector); + ConnectorCtePlan plan = ConceptCtePlanner.planConnector(connector.table(), connectorName, + concept.dateAction() != DateAggregationAction.BLOCK, + connector.selects().stream().anyMatch(ResolvedConceptCompiler::isEventDateSelect), dialect, names); + SqlIdColumns ids = connector.secondaryId() + .map(secondary -> new SqlIdColumns(SchemaSql.field(connector.primaryId(), String.class), + SchemaSql.field(secondary, String.class))) + .orElseGet(() -> new SqlIdColumns(SchemaSql.field(connector.primaryId(), String.class))) + .withAlias(); + ColumnDateRange validityDate = validityDate(connector.validityDate(), restriction, dialect); + List filters = new ArrayList<>(); + ResolvedFilterConverter filterConverter = new ResolvedFilterConverter(); + FilterConversionContext filterContext = new FilterConversionContext(dialect, names, plan.tables(), ids); + connector.filters().stream().map(filter -> filterConverter.convert(filter, filterContext)).forEach(filters::add); + ResolvedConditionConverter conditionConverter = new ResolvedConditionConverter(); + List conditions = connector.conditions().stream() + .map(condition -> conditionConverter.convert(condition, dialect)) + .collect(Collectors.toCollection(ArrayList::new)); + validityPresentCondition(connector.validityDate()).ifPresent(condition -> conditions.add(() -> condition)); + filters.add(new SqlFilters(ConnectorSqlSelects.none(), + WhereClauses.builder().preprocessingConditions(conditions).build())); + restriction.ifPresent(range -> filters.add(new SqlFilters(ConnectorSqlSelects.none(), + WhereClauses.builder().eventFilter(new DateRestrictionCondition(dialect.dateRangeLiteral(range), validityDate)).build()))); + List selects = new ArrayList<>(); + for (ResolvedSelect select : concept.selects()) { + if (select instanceof BuiltInSelects.ConceptValues) { + selects.add(selectConverter.convert(select, selectContext(dialect, names, plan, ids, validityDate, + connectorName, names.selectName(select)))); + } + } + for (ResolvedSelect select : connector.selects()) { + selects.add(selectConverter.convert(select, selectContext(dialect, names, plan, ids, validityDate, + connectorName, names.selectName(select)))); + } + return ConnectorCteCompiler.compileConnector(ConnectorCtePipelineAssembler.assemble( + plan, plan.sourceTable(), ids, validityDate, selects, filters, Optional.empty())) + .map(step -> new ConnectorCompilation(connector, plan, step)); + } + + private static SelectConversionContext selectContext(CompilerDialect dialect, SqlNameGenerator names, + ConnectorCtePlan plan, SqlIdColumns ids, ColumnDateRange validityDate, String connectorName, String alias) { + return new SelectConversionContext(dialect, names, plan.tables(), ids, + Optional.of(validityDate.asValidityDateRange(connectorName)), alias); + } + + private static ColumnDateRange validityDate(ResolvedValidityDate value, Optional restriction, + CompilerDialect dialect) { + ColumnDateRange physical = switch (value) { + case ResolvedValidityDate.None ignored -> restriction.map(dialect::dateRangeLiteral).orElseGet(dialect::unboundedDateRange); + case ResolvedValidityDate.Point point -> dialect.dateRange(SchemaSql.field(point.column(), Date.class), + SchemaSql.field(point.column(), Date.class)); + case ResolvedValidityDate.Range range -> dialect.dateRange(SchemaSql.field(range.start(), Date.class), + SchemaSql.field(range.end(), Date.class)); + }; + if (restriction.isEmpty() || value instanceof ResolvedValidityDate.None) { + return physical; + } + ColumnDateRange bounds = dialect.dateRangeLiteral(restriction.orElseThrow()); + return ColumnDateRange.of( + DSL.when(physical.getStart().lessThan(bounds.getStart()), bounds.getStart()).otherwise(physical.getStart()), + DSL.when(physical.getEnd().greaterThan(bounds.getEnd()), bounds.getEnd()).otherwise(physical.getEnd())); + } + + private static Optional validityPresentCondition(ResolvedValidityDate value) { + return switch (value) { + case ResolvedValidityDate.None ignored -> Optional.empty(); + case ResolvedValidityDate.Point point -> Optional.of(SchemaSql.field(point.column(), Object.class).isNotNull()); + case ResolvedValidityDate.Range range -> Optional.of(SchemaSql.field(range.start(), Object.class).isNotNull() + .or(SchemaSql.field(range.end(), Object.class).isNotNull())); + }; + } + + private static boolean isEventDateSelect(ResolvedSelect select) { + return select instanceof BuiltInSelects.EventDateUnion || select instanceof BuiltInSelects.EventDurationSum; + } + + private record ConnectorCompilation(ResolvedConnector connector, ConnectorCtePlan plan, QueryStep step) { + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/ConceptNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/ConceptNode.java index 6a0a1a35c5..90389415c1 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/ConceptNode.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/ConceptNode.java @@ -7,6 +7,7 @@ import com.bakdata.conquery.sql.model.operation.ResolvedSelect; import com.bakdata.conquery.sql.model.schema.ResolvedConnector; import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; @@ -23,4 +24,12 @@ public record ConceptNode( connectors = ModelNormalization.immutableCopy(connectors); selects = ModelNormalization.immutableCopy(selects); } + + @AssertTrue(message = "connector logicalIds must be unique within a concept") + public boolean isConnectorLogicalIdsUnique() { + return connectors == null || connectors.stream() + .map(ResolvedConnector::logicalId) + .distinct() + .count() == connectors.size(); + } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/NegationNode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/NegationNode.java index 27538a6d3f..b06fdd5327 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/NegationNode.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/node/NegationNode.java @@ -1,8 +1,16 @@ package com.bakdata.conquery.sql.model.node; +import com.bakdata.conquery.models.query.DateAggregationAction; import jakarta.validation.Valid; import jakarta.validation.constraints.NotNull; /** Logical negation. */ -public record NegationNode(@NotNull @Valid QueryNode child) implements QueryNode { +public record NegationNode( + @NotNull @Valid QueryNode child, + @NotNull DateAggregationAction dateAction +) implements QueryNode { + + public NegationNode(QueryNode child) { + this(child, DateAggregationAction.BLOCK); + } } diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/DefaultSqlCompilerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/DefaultSqlCompilerTest.java new file mode 100644 index 0000000000..69c7071b0b --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/DefaultSqlCompilerTest.java @@ -0,0 +1,210 @@ +package com.bakdata.conquery.sql.compiler; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.models.query.DateAggregationAction; +import com.bakdata.conquery.sql.compiler.dialect.hana.HanaCompilerDialect; +import com.bakdata.conquery.sql.model.ResolvedQuery; +import com.bakdata.conquery.sql.model.node.AllEntitiesNode; +import com.bakdata.conquery.sql.model.node.ConceptNode; +import com.bakdata.conquery.sql.model.node.ExternalEntity; +import com.bakdata.conquery.sql.model.node.ExternalNode; +import com.bakdata.conquery.sql.model.node.NegationNode; +import com.bakdata.conquery.sql.model.node.OrNode; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; +import com.bakdata.conquery.sql.model.result.ResultColumn; +import com.bakdata.conquery.sql.model.result.ResultType; +import com.bakdata.conquery.sql.model.schema.EntitySchema; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.ResolvedConnector; +import com.bakdata.conquery.sql.model.schema.ResolvedValidityDate; +import com.bakdata.conquery.sql.model.schema.SqlTable; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Test; + +class DefaultSqlCompilerTest { + + private static final SqlTable ENTITIES = SqlTable.of("entities", "catalog", "entities"); + private static final ResolvedColumn ENTITY_ID = new ResolvedColumn( + "entity-id", ENTITIES, "person_id", ColumnType.STRING, false + ); + + @Test + void shouldCompileAllEntitiesWithOrderedColumnMetadata() { + ResolvedQuery query = new ResolvedQuery( + new EntitySchema(ENTITY_ID), + new AllEntitiesNode(), + false, + List.of() + ); + + CompiledQuery compiled = new DefaultSqlCompiler(DSL.using(SQLDialect.DEFAULT)) + .compile(query, new HanaCompilerDialect()); + + assertTrue(compiled.sql().contains("\"catalog\".\"entities\"")); + assertEquals(List.of(new CompiledColumn( + "entity-id", "person_id", ResultType.Primitive.STRING, ColumnRole.ENTITY_ID + )), compiled.columns()); + } + + @Test + void shouldCompileAResolvedConceptThroughTheConnectorPipeline() { + SqlTable events = SqlTable.of("events", "catalog", "events"); + ResolvedConnector connector = new ResolvedConnector( + "claims", + events, + new ResolvedColumn("events.entity-id", events, "person_id", ColumnType.STRING, false), + Optional.empty(), + new ResolvedValidityDate.None(), + List.of(), + List.of(), + List.of() + ); + ResolvedQuery query = new ResolvedQuery( + new EntitySchema(ENTITY_ID), + new ConceptNode( + "diagnosis", + List.of(connector), + List.of(new BuiltInSelects.Exists("Diagnosis exists")), + DateAggregationAction.BLOCK + ), + false, + List.of(new ResultColumn("diagnosis.exists", ResultType.Primitive.BOOLEAN)) + ); + + CompiledQuery compiled = new DefaultSqlCompiler(DSL.using(SQLDialect.DEFAULT)) + .compile(query, new HanaCompilerDialect()); + + assertTrue(compiled.sql().contains("concept_diagnosis_claims-0-preprocessing")); + assertEquals("diagnosis_exists-1", compiled.columns().get(1).sqlAlias()); + assertEquals("diagnosis.exists", compiled.columns().get(1).outputId()); + } + + @Test + void shouldExposeASecondaryIdAsTheFirstResultColumn() { + SqlTable events = SqlTable.of("events", "catalog", "events"); + ResolvedColumn secondaryId = new ResolvedColumn( + "events.insurance-id", events, "insurance_id", ColumnType.STRING, false + ); + ResolvedConnector connector = new ResolvedConnector( + "claims", events, + new ResolvedColumn("events.entity-id", events, "person_id", ColumnType.STRING, false), + Optional.of(secondaryId), new ResolvedValidityDate.None(), List.of(), List.of(), List.of() + ); + ResolvedQuery query = new ResolvedQuery( + new EntitySchema(ENTITY_ID), + new ConceptNode("diagnosis", List.of(connector), List.of(), DateAggregationAction.BLOCK), + false, + List.of(new ResultColumn("secondary-id", ResultType.Primitive.STRING)) + ); + + CompiledQuery compiled = new DefaultSqlCompiler(DSL.using(SQLDialect.DEFAULT)) + .compile(query, new HanaCompilerDialect()); + + assertEquals(ColumnRole.ENTITY_ID, compiled.columns().get(0).role()); + assertEquals(new CompiledColumn( + "secondary-id", "secondary_id", ResultType.Primitive.STRING, ColumnRole.RESULT + ), compiled.columns().get(1)); + } + + @Test + void shouldKeepNestedNegationStickyLikeTheExistingBackend() { + ResolvedQuery query = new ResolvedQuery( + new EntitySchema(ENTITY_ID), + new NegationNode(new NegationNode(new AllEntitiesNode())), + false, + List.of() + ); + + CompiledQuery compiled = new DefaultSqlCompiler(DSL.using(SQLDialect.DEFAULT)) + .compile(query, new HanaCompilerDialect()); + + assertTrue(compiled.sql().contains("all_ids_negated")); + } + + @Test + void shouldKeepExternalValuesInDeclaredOrderWhenJoiningLogicalBranches() { + ExternalNode external = new ExternalNode( + List.of(new ExternalEntity("person-1", List.of(), Map.of( + "first value", List.of("a"), + "second value", List.of("b") + ))), + List.of("second value", "first value") + ); + ResolvedQuery query = new ResolvedQuery( + new EntitySchema(ENTITY_ID), + new OrNode(List.of(new AllEntitiesNode(), external), DateAggregationAction.BLOCK, false), + false, + List.of( + new ResultColumn("second", ResultType.Primitive.STRING), + new ResultColumn("first", ResultType.Primitive.STRING) + ) + ); + + CompiledQuery compiled = new DefaultSqlCompiler(DSL.using(SQLDialect.DEFAULT)) + .compile(query, new HanaCompilerDialect()); + + assertTrue(compiled.sql().contains("external_extra")); + assertEquals(List.of("primary_id", "second_value", "first_value"), + compiled.columns().stream().map(CompiledColumn::sqlAlias).toList()); + assertEquals(List.of("entity-id", "second", "first"), + compiled.columns().stream().map(CompiledColumn::outputId).toList()); + } + + @Test + void shouldKeepValidityDateBeforeRepeatedConceptSelectsAcrossConnectors() { + SqlTable claims = SqlTable.of("claims", "catalog", "claims"); + SqlTable prescriptions = SqlTable.of("prescriptions", "catalog", "prescriptions"); + ResolvedConnector first = new ResolvedConnector( + "claims", claims, + new ResolvedColumn("claims.entity-id", claims, "person_id", ColumnType.STRING, false), + Optional.empty(), + new ResolvedValidityDate.Point(new ResolvedColumn( + "claims.date", claims, "event_date", ColumnType.DATE, true + )), + List.of(), List.of(), List.of() + ); + ResolvedConnector second = new ResolvedConnector( + "prescriptions", prescriptions, + new ResolvedColumn("prescriptions.entity-id", prescriptions, "person_id", ColumnType.STRING, false), + Optional.empty(), + new ResolvedValidityDate.Point(new ResolvedColumn( + "prescriptions.date", prescriptions, "event_date", ColumnType.DATE, true + )), + List.of(), List.of(), List.of() + ); + ResolvedQuery query = new ResolvedQuery( + new EntitySchema(ENTITY_ID), + new ConceptNode( + "events", + List.of(first, second), + List.of(new BuiltInSelects.Exists("found"), new BuiltInSelects.Exists("found")), + DateAggregationAction.MERGE + ), + true, + List.of( + new ResultColumn("validity", ResultType.Primitive.DATE_RANGE), + new ResultColumn("first-found", ResultType.Primitive.BOOLEAN), + new ResultColumn("second-found", ResultType.Primitive.BOOLEAN) + ) + ); + + CompiledQuery compiled = new DefaultSqlCompiler(DSL.using(SQLDialect.DEFAULT)) + .compile(query, new HanaCompilerDialect()); + + assertTrue(compiled.sql().contains("concept_events_claims-0-preprocessing")); + assertTrue(compiled.sql().contains("concept_events_prescriptions-0-preprocessing")); + assertEquals(List.of("primary_id", "dates", "found-1", "found-2"), + compiled.columns().stream().map(CompiledColumn::sqlAlias).toList()); + assertEquals(List.of("entity-id", "validity", "first-found", "second-found"), + compiled.columns().stream().map(CompiledColumn::outputId).toList()); + } + +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java index e33bd395f3..9d8a7333a4 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/model/ResolvedQueryTest.java @@ -157,13 +157,28 @@ void shouldRejectAmbiguousResultColumnIds() { assertInvalid(query); } + @Test + void shouldRejectAmbiguousConnectorIdsWithinAConcept() { + ResolvedConnector first = connector("concept.connector", EVENTS); + SqlTable otherTable = SqlTable.of("dataset.other", "analytics", "other"); + ResolvedConnector second = connector("concept.connector", otherTable); + + assertInvalid(new ConceptNode( + "concept", List.of(first, second), List.of(), DateAggregationAction.BLOCK + )); + } + private static ConceptNode conceptNode(SqlTable table) { - ResolvedColumn id = new ResolvedColumn("concept.id", table, "entity_id", ColumnType.STRING, false); - ResolvedConnector connector = new ResolvedConnector( - "concept.connector", table, id, Optional.empty(), new ResolvedValidityDate.None(), + ResolvedConnector connector = connector("concept.connector", table); + return new ConceptNode("concept", List.of(connector), List.of(), DateAggregationAction.BLOCK); + } + + private static ResolvedConnector connector(String logicalId, SqlTable table) { + ResolvedColumn id = new ResolvedColumn(logicalId + ".id", table, "entity_id", ColumnType.STRING, false); + return new ResolvedConnector( + logicalId, table, id, Optional.empty(), new ResolvedValidityDate.None(), List.of(), List.of(), List.of() ); - return new ConceptNode("concept", List.of(connector), List.of(), DateAggregationAction.BLOCK); } private static ResolvedColumn dateColumn(String name) { From c6f17b503aa1b39966a56fb10b015eab8b657e98 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 28 Sep 2026 09:16:22 +0200 Subject: [PATCH 68/81] refactor(sql): move concept mapping table lifecycle Co-authored-by: Codex GPT-6 --- .../sql/conquery/SqlMatchingStats.java | 66 +---------------- .../cqelement/concept/ConceptIdMapping.java | 19 +++-- .../sql/mapping/ConceptIdMappingTable.java | 47 ++++++++++++ .../mapping/ConceptIdMappingTableManager.java | 73 +++++++++++++++++++ .../ConceptIdMappingTableManagerTest.java | 68 +++++++++++++++++ 5 files changed, 202 insertions(+), 71 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingTable.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingTableManager.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingTableManagerTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java index eff787c82d..5281703be9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java @@ -19,6 +19,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptIdMapping; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.mapping.ConceptIdMappingTableManager; import com.bakdata.conquery.util.TablePrimaryColumnUtil; import com.google.common.base.Stopwatch; import com.google.common.util.concurrent.ListenableFuture; @@ -85,30 +86,7 @@ private static Select unionSelects(List> fields = createConceptIdsTable(tableName, mapping.tableFields()); - - insertConceptIdMappings(tableName, fields, mapping.getRows(), dslContext); - createConceptIdIndexes(mapping); - } - - private void createConceptIdIndexes(ConceptIdMapping mapping) { - if (dslContext.dialect().family() == SQLDialect.CLICKHOUSE) { - log.debug("Skipping secondary indexes for ClickHouse concept id table {}", mapping.getTableName()); - return; - } - - String indexToken = Integer.toUnsignedString(mapping.getTableName().hashCode(), 16); - if (!mapping.getKeyFields().isEmpty()) { - dslContext.createIndex(name("cq_map_%s_keys".formatted(indexToken))) - .on(mapping.table(), mapping.getKeyFields().stream().map(Field::sortDefault).toList()) - .execute(); - } - dslContext.createIndex(name("cq_map_%s_id".formatted(indexToken))) - .on(mapping.table(), mapping.resolvedId().sortDefault()) - .execute(); + new ConceptIdMappingTableManager(dslContext).recreate(mapping.mappingTable()); } @NotNull @@ -174,38 +152,6 @@ private Map, MatchingStats.Entry> readStats(TreeConcept conc return matchingStats; } - private void insertConceptIdMappings(Name tableName, List> fieldNames, List rows, DSLContext dsl) { - log.info("BEGIN inserting {} rows into {}", rows.size(), tableName); - Stopwatch stopwatch = Stopwatch.createStarted(); - - // We're using batching here because some DBMS don't allow mass inserts. - // There's a chance, we rework this to use a prepared statement with lots of bindings under the hood. But that needs to rework the entire stream of rows. - List> inserts = new ArrayList<>(rows.size()); - - for (RowN row : rows) { - inserts.add(dsl.insertInto(table(tableName)).columns(fieldNames).values(row)); - } - - dsl.batch(inserts).execute(); - - log.debug("DONE inserting into {} within {}", tableName, stopwatch); - } - - /** - * Create a table and its fields. Assumes the table has been dropped already. - */ - private List> createConceptIdsTable(Name tableName, List> fields) { - log.debug("Creating table {} with fields {}", tableName, fields); - - CreateTableElementListStep createTable = - dslContext.createTable(tableName) - .columns(fields); - - createTable.execute(); - - return fields; - } - public ListenableFuture collectMatchingStatsForConcept(TreeConcept concept, ListeningExecutorService executorService, int tries) { return executorService.submit(() -> { dslContext.connection(cfg -> { @@ -280,13 +226,7 @@ private SelectJoinStep createMatchingStatsStatement(TreeConcep public void deleteConceptIdJoinTable(ConceptId concept) { Name tableName = ConceptIdMapping.tableName(concept); log.debug("Trying to delete id-table {}", tableName); - - try { - dslContext.dropTable(tableName).execute(); - } catch (DataAccessException exception) { - // Likely it doesn't exist. Some DBMS just don't support drop-IfExists so this is the next best thing :^) - log.trace("Failed to drop table {}", tableName, exception); - } + new ConceptIdMappingTableManager(dslContext).delete(tableName); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptIdMapping.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptIdMapping.java index 998b283336..9226193905 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptIdMapping.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptIdMapping.java @@ -10,6 +10,7 @@ import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.mapping.ConceptIdMappingTable; import com.google.common.collect.Sets; import lombok.Data; import org.jooq.*; @@ -25,7 +26,7 @@ @Data public final class ConceptIdMapping { - public static final String RESOLVED_ID_COLUMN = "resolved_id"; + public static final String RESOLVED_ID_COLUMN = ConceptIdMappingTable.RESOLVED_ID_COLUMN; private final TreeConcept concept; private final SqlFunctionProvider functionProvider; @@ -155,24 +156,26 @@ private static List collectAllExpressions( } public Table table() { - return DSL.table(tableName); + return mappingTable().table(); } public Field resolvedId() { - return field(name(tableName, name(RESOLVED_ID_COLUMN)), Integer.class); + return mappingTable().resolvedId(); } public List> tableFields() { - List> fields = new ArrayList<>(keyFields.size() + 1); - fields.add(field(name(RESOLVED_ID_COLUMN), Integer.class)); - fields.addAll(keyFields); - return fields; + return mappingTable().tableFields(); + } + + public ConceptIdMappingTable mappingTable() { + return new ConceptIdMappingTable(tableName, keyFields, rows); } public Condition joinCondition(Connector connector) { CTConditionContext context = CTConditionContext.forConnector(connector, functionProvider); List expressions = collectAllExpressions(concept, null, context); Map> extractors = collectExtractors(expressions); + ConceptIdMappingTable mappingTable = mappingTable(); Condition condition = noCondition(); for (Field keyField : keyFields) { @@ -183,7 +186,7 @@ public Condition joinCondition(Connector connector) { .formatted(keyField.getName(), connector.getId()) ); } - Field mappingField = field(name(tableName, keyField.getUnqualifiedName()), keyField.getDataType()); + Field mappingField = mappingTable.mappingField(keyField); condition = condition.and(mappingField.eq((Field) extractor)); } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingTable.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingTable.java new file mode 100644 index 0000000000..6e9b9cea0f --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingTable.java @@ -0,0 +1,47 @@ +package com.bakdata.conquery.sql.mapping; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import org.jooq.Field; +import org.jooq.Name; +import org.jooq.Record; +import org.jooq.RowN; +import org.jooq.Table; +import org.jooq.impl.DSL; + +/** Resolved physical contents of a concept ID lookup table. */ +public record ConceptIdMappingTable( + Name tableName, + List> keyFields, + List rows +) { + + public static final String RESOLVED_ID_COLUMN = "resolved_id"; + + public ConceptIdMappingTable { + Objects.requireNonNull(tableName, "tableName"); + keyFields = List.copyOf(keyFields); + rows = List.copyOf(rows); + } + + public Table table() { + return DSL.table(tableName); + } + + public Field resolvedId() { + return DSL.field(DSL.name(tableName, DSL.name(RESOLVED_ID_COLUMN)), Integer.class); + } + + public Field mappingField(Field keyField) { + return DSL.field(DSL.name(tableName, keyField.getUnqualifiedName()), keyField.getDataType()); + } + + public List> tableFields() { + List> fields = new ArrayList<>(keyFields.size() + 1); + fields.add(DSL.field(DSL.name(RESOLVED_ID_COLUMN), Integer.class)); + fields.addAll(keyFields); + return List.copyOf(fields); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingTableManager.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingTableManager.java new file mode 100644 index 0000000000..903e0372f9 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingTableManager.java @@ -0,0 +1,73 @@ +package com.bakdata.conquery.sql.mapping; + +import java.util.ArrayList; +import java.util.List; + +import org.jooq.CreateTableElementListStep; +import org.jooq.DSLContext; +import org.jooq.Field; +import org.jooq.InsertValuesStepN; +import org.jooq.Name; +import org.jooq.RowN; +import org.jooq.SQLDialect; +import org.jooq.exception.DataAccessException; + +/** Creates and populates the physical lookup table used to resolve concept element IDs. */ +public final class ConceptIdMappingTableManager { + + private final DSLContext dslContext; + + public ConceptIdMappingTableManager(DSLContext dslContext) { + this.dslContext = dslContext; + } + + public void recreate(ConceptIdMappingTable mapping) { + delete(mapping.tableName()); + List> fields = createTable(mapping); + insertMappings(mapping.tableName(), fields, mapping.rows()); + createIndexes(mapping); + } + + public void delete(Name tableName) { + try { + dslContext.dropTable(tableName).execute(); + } + catch (DataAccessException ignored) { + // Some supported databases do not provide a portable DROP TABLE IF EXISTS variant. + } + } + + private List> createTable(ConceptIdMappingTable mapping) { + List> fields = mapping.tableFields(); + CreateTableElementListStep createTable = dslContext.createTable(mapping.tableName()).columns(fields); + createTable.execute(); + return fields; + } + + private void insertMappings(Name tableName, List> fields, List rows) { + List> inserts = new ArrayList<>(rows.size()); + for (RowN row : rows) { + inserts.add(dslContext.insertInto(mappingTable(tableName)).columns(fields).values(row)); + } + dslContext.batch(inserts).execute(); + } + + private void createIndexes(ConceptIdMappingTable mapping) { + if (dslContext.dialect().family() == SQLDialect.CLICKHOUSE) { + return; + } + String indexToken = Integer.toUnsignedString(mapping.tableName().hashCode(), 16); + if (!mapping.keyFields().isEmpty()) { + dslContext.createIndex(org.jooq.impl.DSL.name("cq_map_%s_keys".formatted(indexToken))) + .on(mapping.table(), mapping.keyFields().stream().map(Field::sortDefault).toList()) + .execute(); + } + dslContext.createIndex(org.jooq.impl.DSL.name("cq_map_%s_id".formatted(indexToken))) + .on(mapping.table(), mapping.resolvedId().sortDefault()) + .execute(); + } + + private static org.jooq.Table mappingTable(Name tableName) { + return org.jooq.impl.DSL.table(tableName); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingTableManagerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingTableManagerTest.java new file mode 100644 index 0000000000..79b079feaa --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingTableManagerTest.java @@ -0,0 +1,68 @@ +package com.bakdata.conquery.sql.mapping; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.jooq.DSLContext; +import org.jooq.Field; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.jooq.tools.jdbc.MockConnection; +import org.jooq.tools.jdbc.MockResult; +import org.junit.jupiter.api.Test; + +class ConceptIdMappingTableManagerTest { + + @Test + void shouldRecreateAndPopulateTheMappingTableInTheExistingOrder() { + List statements = new ArrayList<>(); + MockConnection connection = new MockConnection(context -> { + statements.addAll(Arrays.asList(context.batchSQL())); + return new MockResult[] {new MockResult(1, DSL.using(SQLDialect.POSTGRES).newResult())}; + }); + DSLContext dsl = DSL.using(connection, SQLDialect.POSTGRES); + Field resolvedId = DSL.field(DSL.name("resolved_id"), Integer.class); + Field code = DSL.field(DSL.name("code"), SQLDataType.VARCHAR(12)); + ConceptIdMappingTable mapping = new ConceptIdMappingTable( + DSL.name("diagnosis_ids"), + List.of(code), + List.of(DSL.row(List.of(DSL.val(7), DSL.val("A12")))) + ); + + new ConceptIdMappingTableManager(dsl).recreate(mapping); + + assertEquals(List.of( + "drop table \"diagnosis_ids\"", + "create table \"diagnosis_ids\" (\"resolved_id\" int, \"code\" varchar(12))", + "insert into \"diagnosis_ids\" (\"resolved_id\", \"code\") values (7, 'A12')", + "create index \"cq_map_44649089_keys\" on \"diagnosis_ids\"(\"code\")", + "create index \"cq_map_44649089_id\" on \"diagnosis_ids\"(\"resolved_id\")" + ), statements); + assertEquals(List.of(resolvedId, code), mapping.tableFields()); + } + + @Test + void shouldKeepSkippingIndexesForClickHouse() { + List statements = new ArrayList<>(); + MockConnection connection = new MockConnection(context -> { + statements.addAll(Arrays.asList(context.batchSQL())); + return new MockResult[] {new MockResult(1, DSL.using(SQLDialect.CLICKHOUSE).newResult())}; + }); + DSLContext dsl = DSL.using(connection, SQLDialect.CLICKHOUSE); + Field code = DSL.field(DSL.name("code"), SQLDataType.VARCHAR(12)); + ConceptIdMappingTable mapping = new ConceptIdMappingTable( + DSL.name("diagnosis_ids"), + List.of(code), + List.of(DSL.row(List.of(DSL.val(7), DSL.val("A12")))) + ); + + new ConceptIdMappingTableManager(dsl).recreate(mapping); + + assertFalse(statements.stream().anyMatch(statement -> statement.startsWith("create index"))); + } +} From aa2f94f0891fb5957ca4b04bf7d9df9d0c105150 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 28 Sep 2026 09:22:12 +0200 Subject: [PATCH 69/81] refactor(sql): resolve concept mapping rows in connector Co-authored-by: Codex GPT-6 --- .../cqelement/concept/ConceptIdMapping.java | 111 +++---------- .../sql/mapping/ConceptIdMappingResolver.java | 152 ++++++++++++++++++ .../mapping/ConceptIdMappingResolverTest.java | 68 ++++++++ 3 files changed, 240 insertions(+), 91 deletions(-) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingResolver.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingResolverTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptIdMapping.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptIdMapping.java index 9226193905..246cd6868c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptIdMapping.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptIdMapping.java @@ -10,15 +10,13 @@ import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.mapping.ConceptIdMappingResolver; import com.bakdata.conquery.sql.mapping.ConceptIdMappingTable; -import com.google.common.collect.Sets; import lombok.Data; import org.jooq.*; import org.jooq.Record; -import org.jooq.impl.DSL; import static org.jooq.impl.DSL.*; -import static org.jooq.impl.SQLDataType.VARCHAR; /** * Description of the physical lookup table that maps connector values to their most specific concept element. @@ -39,101 +37,32 @@ public ConceptIdMapping(TreeConcept concept, SqlFunctionProvider functionProvide this.functionProvider = functionProvider; CTConditionContext context = CTConditionContext.forJoinTables(functionProvider); List expressions = collectAllExpressions(concept, null, context); - this.keyFields = collectKeyFields(expressions); - this.rows = expressionsToRows(concept, expressions, keyFields); this.tableName = tableName(concept.getId()); + ConceptIdMappingTable mappingTable = ConceptIdMappingResolver.resolve( + concept.getId().toString(), tableName, resolveExpressions(expressions) + ); + this.keyFields = mappingTable.keyFields(); + this.rows = mappingTable.rows(); } public static Name tableName(ConceptId conceptId) { return name("%s_ids".formatted(conceptId)); } - private static List> collectKeyFields(List expressions) { - Map> fields = new TreeMap<>(); - for (CTCondition.ConceptConditions expression : expressions) { - for (Field field : expression.conditions().keySet()) { - fields.merge(field.getName(), field, ConceptIdMapping::mergeFieldType); - } - } - return List.copyOf(fields.values()); - } - - private static Field mergeFieldType(Field left, Field right) { - DataType leftType = left.getDataType(); - DataType rightType = right.getDataType(); - if (leftType.isString() && rightType.isString()) { - int length = Math.max(leftType.length(), rightType.length()); - return length > 0 ? field(left.getUnqualifiedName(), VARCHAR(length)) : field(left.getUnqualifiedName(), VARCHAR); - } - if (!leftType.getType().equals(rightType.getType())) { - throw new IllegalArgumentException( - "Concept mapping field `%s` is used with incompatible types %s and %s" - .formatted(left.getName(), leftType, rightType) - ); - } - return left; - } - - private static Map> collectExtractors(List expressions) { - Map> extractors = new LinkedHashMap<>(); - for (CTCondition.ConceptConditions expression : expressions) { - for (Map.Entry, CTCondition.FieldCondition> entry : expression.conditions().entrySet()) { - extractors.putIfAbsent(entry.getKey().getName(), entry.getValue().extractor()); - } - } - return extractors; - } - - private static List expressionsToRows( - TreeConcept concept, - List expressions, - List> keyFields + private static List resolveExpressions( + List expressions ) { - Map>, ConceptElement> resolvedMappings = new HashMap<>(); - - for (CTCondition.ConceptConditions expression : expressions) { - Map conditions = expression.conditions().entrySet().stream() - .collect(Collectors.toMap(entry -> entry.getKey().getName(), Map.Entry::getValue)); - List>> valuesByField = new ArrayList<>(); - for (Field keyField : keyFields) { - CTCondition.FieldCondition fieldCondition = conditions.get(keyField.getName()); - valuesByField.add(fieldCondition != null ? fieldCondition.params() : Set.of(defaultValue(keyField))); - } - - for (List> params : Sets.cartesianProduct(valuesByField)) { - ConceptElement previous = resolvedMappings.get(params); - ConceptElement current = expression.conceptElement(); - if (previous == null || previous.getDepth() < current.getDepth()) { - resolvedMappings.put(params, current); - continue; - } - if (previous.getDepth() == current.getDepth() && !previous.equals(current)) { - throw new IllegalArgumentException( - "Concept %s has overlapping sibling mappings for %s and %s on values %s" - .formatted(concept.getId(), previous.getId(), current.getId(), params) - ); - } - } - } - - return resolvedMappings.entrySet().stream() - .map(entry -> { - List> values = new ArrayList<>(entry.getKey().size() + 1); - values.add(val(entry.getValue().getLocalId())); - values.addAll(entry.getKey()); - return row(values); - }) - .toList(); - } - - private static Param defaultValue(Field field) { - if (field.getDataType().isBoolean()) { - return inline(false); - } - if (field.getDataType().isString()) { - return inline(null, String.class); - } - throw new IllegalStateException("Fields of type %s are not expected".formatted(field.getDataType())); + return expressions.stream().map(expression -> new ConceptIdMappingResolver.MappingExpression( + expression.conceptElement().getId().toString(), + expression.conceptElement().getLocalId(), + expression.conceptElement().getDepth(), + expression.conditions().entrySet().stream().collect(Collectors.toMap( + Map.Entry::getKey, + entry -> new ConceptIdMappingResolver.FieldCondition( + entry.getValue().extractor(), entry.getValue().params() + ) + )) + )).toList(); } private static List collectAllExpressions( @@ -174,7 +103,7 @@ public ConceptIdMappingTable mappingTable() { public Condition joinCondition(Connector connector) { CTConditionContext context = CTConditionContext.forConnector(connector, functionProvider); List expressions = collectAllExpressions(concept, null, context); - Map> extractors = collectExtractors(expressions); + Map> extractors = ConceptIdMappingResolver.collectExtractors(resolveExpressions(expressions)); ConceptIdMappingTable mappingTable = mappingTable(); Condition condition = noCondition(); diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingResolver.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingResolver.java new file mode 100644 index 0000000000..ad5052a644 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingResolver.java @@ -0,0 +1,152 @@ +package com.bakdata.conquery.sql.mapping; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import org.jooq.DataType; +import org.jooq.Field; +import org.jooq.Name; +import org.jooq.Param; +import org.jooq.RowN; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; + +/** Resolves concept condition expressions into the contents of a physical concept ID mapping table. */ +public final class ConceptIdMappingResolver { + + private ConceptIdMappingResolver() { + } + + public static ConceptIdMappingTable resolve( + String conceptId, + Name tableName, + List expressions + ) { + List> keyFields = collectKeyFields(expressions); + return new ConceptIdMappingTable(tableName, keyFields, expressionsToRows(conceptId, expressions, keyFields)); + } + + public static Map> collectExtractors(List expressions) { + Map> extractors = new LinkedHashMap<>(); + for (MappingExpression expression : expressions) { + for (Map.Entry, FieldCondition> entry : expression.conditions().entrySet()) { + extractors.putIfAbsent(entry.getKey().getName(), entry.getValue().extractor()); + } + } + return Map.copyOf(extractors); + } + + private static List> collectKeyFields(List expressions) { + Map> fields = new TreeMap<>(); + for (MappingExpression expression : expressions) { + for (Field field : expression.conditions().keySet()) { + fields.merge(field.getName(), field, ConceptIdMappingResolver::mergeFieldType); + } + } + return List.copyOf(fields.values()); + } + + private static Field mergeFieldType(Field left, Field right) { + DataType leftType = left.getDataType(); + DataType rightType = right.getDataType(); + if (leftType.isString() && rightType.isString()) { + int length = Math.max(leftType.length(), rightType.length()); + return length > 0 + ? DSL.field(left.getUnqualifiedName(), SQLDataType.VARCHAR(length)) + : DSL.field(left.getUnqualifiedName(), SQLDataType.VARCHAR); + } + if (!leftType.getType().equals(rightType.getType())) { + throw new IllegalArgumentException( + "Concept mapping field `%s` is used with incompatible types %s and %s" + .formatted(left.getName(), leftType, rightType) + ); + } + return left; + } + + private static List expressionsToRows( + String conceptId, + List expressions, + List> keyFields + ) { + Map>, MappingExpression> resolvedMappings = new HashMap<>(); + for (MappingExpression expression : expressions) { + Map conditions = new HashMap<>(); + expression.conditions().forEach((field, condition) -> conditions.put(field.getName(), condition)); + List>> valuesByField = new ArrayList<>(); + for (Field keyField : keyFields) { + FieldCondition condition = conditions.get(keyField.getName()); + valuesByField.add(condition == null ? List.of(defaultValue(keyField)) : List.copyOf(condition.params())); + } + for (List> params : cartesianProduct(valuesByField)) { + MappingExpression previous = resolvedMappings.get(params); + if (previous == null || previous.depth() < expression.depth()) { + resolvedMappings.put(params, expression); + continue; + } + if (previous.depth() == expression.depth() && !previous.elementId().equals(expression.elementId())) { + throw new IllegalArgumentException( + "Concept %s has overlapping sibling mappings for %s and %s on values %s" + .formatted(conceptId, previous.elementId(), expression.elementId(), params) + ); + } + } + } + return resolvedMappings.entrySet().stream().map(entry -> { + List> values = new ArrayList<>(entry.getKey().size() + 1); + values.add(DSL.val(entry.getValue().localId())); + values.addAll(entry.getKey()); + return DSL.row(values); + }).toList(); + } + + private static Param defaultValue(Field field) { + if (field.getDataType().isBoolean()) { + return DSL.inline(false); + } + if (field.getDataType().isString()) { + return DSL.inline(null, String.class); + } + throw new IllegalStateException("Fields of type %s are not expected".formatted(field.getDataType())); + } + + private static List>> cartesianProduct(List>> valuesByField) { + List>> product = new ArrayList<>(); + product.add(List.of()); + for (List> values : valuesByField) { + List>> next = new ArrayList<>(); + for (List> prefix : product) { + for (Param value : values) { + List> combined = new ArrayList<>(prefix.size() + 1); + combined.addAll(prefix); + combined.add(value); + next.add(List.copyOf(combined)); + } + } + product = next; + } + return product; + } + + public record FieldCondition(Field extractor, Set> params) { + public FieldCondition { + params = Set.copyOf(params); + } + } + + public record MappingExpression( + String elementId, + int localId, + int depth, + Map, FieldCondition> conditions + ) { + public MappingExpression { + conditions = Map.copyOf(conditions); + } + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingResolverTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingResolverTest.java new file mode 100644 index 0000000000..dffe42a1b6 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingResolverTest.java @@ -0,0 +1,68 @@ +package com.bakdata.conquery.sql.mapping; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.jooq.Field; +import org.jooq.Param; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.junit.jupiter.api.Test; + +class ConceptIdMappingResolverTest { + + @Test + void shouldResolveTheMostSpecificRowsWithExistingDefaultsAndFieldOrder() { + Field shortCode = DSL.field(DSL.name("code"), SQLDataType.VARCHAR(2)); + Field longCode = DSL.field(DSL.name("code"), SQLDataType.VARCHAR(5)); + Field flag = DSL.field(DSL.name("flag"), Boolean.class); + Field codeExtractor = DSL.field(DSL.name("events", "code"), String.class); + Field flagExtractor = DSL.field(DSL.name("events", "flag"), Boolean.class); + List expressions = List.of( + new ConceptIdMappingResolver.MappingExpression("root", 0, 0, Map.of()), + new ConceptIdMappingResolver.MappingExpression("parent", 1, 1, Map.of( + shortCode, new ConceptIdMappingResolver.FieldCondition(codeExtractor, Set.of(DSL.val("A"))) + )), + new ConceptIdMappingResolver.MappingExpression("child", 2, 2, Map.of( + longCode, new ConceptIdMappingResolver.FieldCondition(codeExtractor, Set.of(DSL.val("A"))), + flag, new ConceptIdMappingResolver.FieldCondition(flagExtractor, Set.of(DSL.val(true))) + )) + ); + + ConceptIdMappingTable mapping = ConceptIdMappingResolver.resolve( + "diagnosis", DSL.name("diagnosis_ids"), expressions + ); + + assertEquals(List.of("code", "flag"), mapping.keyFields().stream().map(Field::getName).toList()); + assertEquals(5, mapping.keyFields().getFirst().getDataType().length()); + assertEquals(Set.of( + Arrays.asList(0, null, false), + List.of(1, "A", false), + List.of(2, "A", true) + ), mapping.rows().stream().map(row -> Arrays.stream(row.fields()) + .map(field -> ((Param) field).getValue()).toList()).collect(java.util.stream.Collectors.toSet())); + } + + @Test + void shouldRejectOverlappingSiblingMappings() { + Field code = DSL.field(DSL.name("code"), String.class); + Field extractor = DSL.field(DSL.name("events", "code"), String.class); + ConceptIdMappingResolver.FieldCondition condition = new ConceptIdMappingResolver.FieldCondition( + extractor, Set.of(DSL.val("A")) + ); + + assertThrows(IllegalArgumentException.class, () -> ConceptIdMappingResolver.resolve( + "diagnosis", + DSL.name("diagnosis_ids"), + List.of( + new ConceptIdMappingResolver.MappingExpression("left", 1, 1, Map.of(code, condition)), + new ConceptIdMappingResolver.MappingExpression("right", 2, 1, Map.of(code, condition)) + ) + )); + } +} From 13ef751217309a00db418cae91108046f0670345 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:22:58 +0200 Subject: [PATCH 70/81] refactor(sql): route concept queries through shared compiler Co-authored-by: Codex GPT-6 --- .../mode/local/LocalNamespaceHandler.java | 10 +- .../conversion/ResolvedOperationAdapter.java | 210 ++++++++++++++++ .../sql/conversion/ResolvedQueryMapper.java | 227 ++++++++++++++++++ .../conquery/sql/conversion/SqlConverter.java | 33 ++- .../cqelement/concept/CQConceptConverter.java | 21 +- .../cqelement/concept/ConceptIdMapping.java | 22 +- .../dialect/LegacyCompilerDialect.java | 5 + .../select/ConceptColumnSelectConverter.java | 10 +- .../sql/conversion/SqlConverterTest.java | 90 +++++++ .../sql/compiler/ResolvedConceptCompiler.java | 48 +++- .../sql/compiler/dialect/CompilerDialect.java | 5 + .../dialect/hana/HanaCompilerDialect.java | 1 + .../mapping/ConceptIdMappingSelection.java | 33 +++ .../sql/mapping/ConceptIdMappingSource.java | 69 ++++++ .../sql/model/schema/ResolvedConnector.java | 17 +- .../sql/compiler/DefaultSqlCompilerTest.java | 54 +++++ .../mapping/ConceptIdMappingSourceTest.java | 79 ++++++ 17 files changed, 891 insertions(+), 43 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedOperationAdapter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java create mode 100644 backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlConverterTest.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingSelection.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingSource.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingSourceTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java index e458522d30..fca73bd91a 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java @@ -12,7 +12,10 @@ import com.bakdata.conquery.models.worker.LocalNamespace; import com.bakdata.conquery.sql.conquery.SqlExecutionManager; import com.bakdata.conquery.sql.conversion.NodeConversions; +import com.bakdata.conquery.sql.conversion.ResolvedQueryAdapter; +import com.bakdata.conquery.sql.conversion.ResolvedQueryMapper; import com.bakdata.conquery.sql.conversion.SqlConverter; +import com.bakdata.conquery.sql.compiler.DefaultSqlCompiler; import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.bakdata.conquery.sql.execution.SqlExecutionService; @@ -50,7 +53,12 @@ public LocalNamespace createNamespace( SqlExecutionService sqlExecutionService = new SqlExecutionService(dslContext, resultSetProcessor); NodeConversions nodeConversions = new NodeConversions(config.getIdColumns(), dialectBundle, dslContext, clock, connection.getConnection().getPrimaryColumn()); - SqlConverter sqlConverter = new SqlConverter(nodeConversions, config); + ResolvedQueryMapper resolvedQueryMapper = new ResolvedQueryMapper( + config.getIdColumns(), dialectBundle, clock, connection.getConnection().getPrimaryColumn()); + ResolvedQueryAdapter resolvedQueryAdapter = new ResolvedQueryAdapter( + environment.getValidator(), new DefaultSqlCompiler(dslContext)); + SqlConverter sqlConverter = new SqlConverter( + nodeConversions, config, resolvedQueryMapper, resolvedQueryAdapter, dialectBundle); ExecutionManager executionManager = new SqlExecutionManager(sqlConverter, sqlExecutionService, metaStorage, datasetRegistry, config); SqlStorageHandler sqlStorageHandler = new SqlStorageHandler(sqlExecutionService); SqlEntityResolver sqlEntityResolver = new SqlEntityResolver(config.getIdColumns(), dslContext, dialectBundle, sqlExecutionService); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedOperationAdapter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedOperationAdapter.java new file mode 100644 index 0000000000..83c7c5e0fa --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedOperationAdapter.java @@ -0,0 +1,210 @@ +package com.bakdata.conquery.sql.conversion; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; +import com.bakdata.conquery.models.common.IRange; +import com.bakdata.conquery.models.datasets.concepts.conditions.AndCondition; +import com.bakdata.conquery.models.datasets.concepts.conditions.ColumnEqualCondition; +import com.bakdata.conquery.models.datasets.concepts.conditions.CTCondition; +import com.bakdata.conquery.models.datasets.concepts.conditions.EqualCondition; +import com.bakdata.conquery.models.datasets.concepts.conditions.IsEmptyCondition; +import com.bakdata.conquery.models.datasets.concepts.conditions.IsPresentCondition; +import com.bakdata.conquery.models.datasets.concepts.conditions.NotCondition; +import com.bakdata.conquery.models.datasets.concepts.conditions.PrefixCondition; +import com.bakdata.conquery.models.datasets.concepts.conditions.PrefixRangeCondition; +import com.bakdata.conquery.models.datasets.concepts.filters.Filter; +import com.bakdata.conquery.models.datasets.concepts.filters.specific.CountFilter; +import com.bakdata.conquery.models.datasets.concepts.filters.specific.CountQuartersFilter; +import com.bakdata.conquery.models.datasets.concepts.filters.specific.DateDistanceFilter; +import com.bakdata.conquery.models.datasets.concepts.filters.specific.DurationSumFilter; +import com.bakdata.conquery.models.datasets.concepts.filters.specific.FlagFilter; +import com.bakdata.conquery.models.datasets.concepts.filters.specific.NumberFilter; +import com.bakdata.conquery.models.datasets.concepts.filters.specific.SelectFilter; +import com.bakdata.conquery.models.datasets.concepts.filters.specific.SumFilter; +import com.bakdata.conquery.models.datasets.concepts.select.Select; +import com.bakdata.conquery.models.datasets.concepts.select.concept.ConceptColumnSelect; +import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDateUnionSelect; +import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDurationSumSelect; +import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.ExistsSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.DistinctSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.FirstValueSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.LastValueSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.RandomValueSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.CountQuartersSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.CountSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DateDistanceSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DateUnionSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DurationSumSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.FlagSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.SumSelect; +import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.models.types.ResultType; +import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; +import com.bakdata.conquery.sql.conversion.model.select.ResolvedSelectAdapter; +import com.bakdata.conquery.sql.model.operation.BuiltInAggregations; +import com.bakdata.conquery.sql.model.operation.BuiltInConditions; +import com.bakdata.conquery.sql.model.operation.BuiltInFilters; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; +import com.bakdata.conquery.sql.model.operation.ResolvedAggregation; +import com.bakdata.conquery.sql.model.operation.ResolvedCondition; +import com.bakdata.conquery.sql.model.operation.ResolvedFilter; +import com.bakdata.conquery.sql.model.operation.ResolvedSelect; +import com.bakdata.conquery.sql.model.range.NumberRange; +import com.bakdata.conquery.sql.model.result.ResultColumn; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.SqlTable; +import lombok.experimental.UtilityClass; + +/** Converts already-resolved backend operations into the connector's immutable input model. */ +@UtilityClass +public class ResolvedOperationAdapter { + + public static ResolvedSelect select(Select select, List conceptColumns, LocalDate endDate) { + return switch (select) { + case ConceptColumnSelect value -> new BuiltInSelects.ConceptValues(value.getName(), conceptColumns); + case EventDateUnionSelect value -> new BuiltInSelects.EventDateUnion(value.getName()); + case EventDurationSumSelect value -> new BuiltInSelects.EventDurationSum(value.getName()); + case ExistsSelect value -> new BuiltInSelects.Exists(value.getName()); + case DistinctSelect value -> values(value, BuiltInSelects.ValueOperation.DISTINCT); + case FirstValueSelect value -> values(value, BuiltInSelects.ValueOperation.FIRST); + case LastValueSelect value -> values(value, BuiltInSelects.ValueOperation.LAST); + case RandomValueSelect value -> values(value, BuiltInSelects.ValueOperation.RANDOM); + case CountSelect value -> aggregation(value.getName(), count(value.getColumn().resolve(), value.isDistinct())); + case CountQuartersSelect value -> aggregation(value.getName(), new BuiltInAggregations.CountQuarters(EntitySchemaAdapter.from(value))); + case DurationSumSelect value -> aggregation(value.getName(), new BuiltInAggregations.DurationSum( + EntitySchemaAdapter.from(value), List.of())); + case SumSelect value -> aggregation(value.getName(), sum( + value.getColumn().resolve(), value.getSubtractColumn(), value.getDistinctByColumn())); + case FlagSelect value -> aggregation(value.getName(), flags(value.getFlags())); + case DateDistanceSelect value -> new BuiltInSelects.DateDistance( + value.getName(), EntitySchemaAdapter.from(value.getColumn().resolve()), value.getTimeUnit(), endDate); + case DateUnionSelect value -> new BuiltInSelects.DateUnion(value.getName(), EntitySchemaAdapter.from(value)); + default -> throw new UnsupportedOperationException("SQL resolved select is not implemented for " + select.getClass()); + }; + } + + public static ResolvedFilter filter(FilterValue filterValue, LocalDate endDate) { + Filter filter = filterValue.getFilter().resolve(); + Object value = filterValue.readValue(); + return switch (filter) { + case SelectFilter selectFilter -> new BuiltInFilters.StringValues( + filter.getName(), EntitySchemaAdapter.from(selectFilter.getColumn().resolve()), stringValues(value), + ResolvedSelectAdapter.substring(selectFilter.getSubstringRange())); + case NumberFilter numberFilter -> new BuiltInFilters.NumericColumnRange( + filter.getName(), EntitySchemaAdapter.from(numberFilter.getColumn().resolve()), numberRange((IRange) value)); + case CountFilter countFilter -> aggregationFilter(filter.getName(), + count(countFilter.getColumn().resolve(), countFilter.isDistinct()), value); + case CountQuartersFilter quartersFilter -> aggregationFilter(filter.getName(), + new BuiltInAggregations.CountQuarters(EntitySchemaAdapter.from(quartersFilter)), value); + case DurationSumFilter durationFilter -> aggregationFilter(filter.getName(), + new BuiltInAggregations.DurationSum(EntitySchemaAdapter.from(durationFilter), List.of()), value); + case SumFilter sumFilter -> aggregationFilter(filter.getName(), + sum(sumFilter.getColumn().resolve(), sumFilter.getSubtractColumn(), sumFilter.getDistinctByColumn()), value); + case FlagFilter flagFilter -> new BuiltInFilters.Flags(filter.getName(), flags(flagFilter.getFlags()), (Set) value); + case DateDistanceFilter distanceFilter -> new BuiltInFilters.DateDistanceRange( + filter.getName(), EntitySchemaAdapter.from(distanceFilter.getColumn().resolve()), distanceFilter.getTimeUnit(), + endDate, numberRange((IRange) value)); + default -> throw new UnsupportedOperationException("SQL resolved filter is not implemented for " + filter.getClass()); + }; + } + + public static ResolvedCondition condition(CTCondition condition, SqlTable table, Optional connectorColumn) { + return switch (condition) { + case AndCondition value -> new BuiltInConditions.AllOf(value.getConditions().stream() + .map(child -> condition(child, table, connectorColumn)).toList()); + case NotCondition value -> new BuiltInConditions.Not(condition(value.getCondition(), table, connectorColumn)); + case EqualCondition value -> new BuiltInConditions.StringValues(connectorColumn.orElseThrow(), value.getValues()); + case ColumnEqualCondition value -> new BuiltInConditions.StringValues(stringColumn(table, value.getColumn()), value.getValues()); + case IsPresentCondition value -> new BuiltInConditions.Presence(stringColumn(table, value.getColumn()), true); + case IsEmptyCondition value -> new BuiltInConditions.Presence(stringColumn(table, value.getColumn()), false); + case PrefixCondition value -> new BuiltInConditions.Prefixes(connectorColumn.orElseThrow(), List.of(value.getPrefixes())); + case PrefixRangeCondition value -> new BuiltInConditions.PrefixRange( + connectorColumn.orElseThrow(), value.getMin(), value.getMax()); + default -> throw new UnsupportedOperationException("SQL resolved condition is not implemented for " + condition.getClass()); + }; + } + + public static List resultColumns(List infos) { + return java.util.stream.IntStream.range(0, infos.size()) + .mapToObj(index -> new ResultColumn("result-%d".formatted(index), resultType(infos.get(index).getType()))) + .toList(); + } + + private static BuiltInSelects.Aggregation aggregation(String name, ResolvedAggregation aggregation) { + return new BuiltInSelects.Aggregation(name, aggregation); + } + + private static BuiltInFilters.AggregationRange aggregationFilter(String name, ResolvedAggregation aggregation, Object range) { + return new BuiltInFilters.AggregationRange(name, aggregation, numberRange((IRange) range)); + } + + private static BuiltInSelects.Values values(MappableSingleColumnSelect select, BuiltInSelects.ValueOperation operation) { + return new BuiltInSelects.Values(select.getName(), EntitySchemaAdapter.from(select.getColumn().resolve()), operation, + ResolvedSelectAdapter.substring(select.getSubstringRange())); + } + + private static BuiltInAggregations.Count count(com.bakdata.conquery.models.datasets.Column column, boolean distinct) { + ResolvedColumn resolved = EntitySchemaAdapter.from(column); + return new BuiltInAggregations.Count(resolved, distinct ? List.of(resolved) : List.of()); + } + + private static BuiltInAggregations.Sum sum( + com.bakdata.conquery.models.datasets.Column column, + com.bakdata.conquery.models.identifiable.ids.specific.ColumnId subtract, + List distinctBy + ) { + return new BuiltInAggregations.Sum(EntitySchemaAdapter.from(column), + Optional.ofNullable(subtract).map(id -> EntitySchemaAdapter.from(id.resolve())), columns(distinctBy)); + } + + private static BuiltInAggregations.Flags flags(Map flags) { + Map resolved = new LinkedHashMap<>(); + flags.forEach((name, column) -> resolved.put(name, EntitySchemaAdapter.from(column.resolve()))); + return new BuiltInAggregations.Flags(resolved); + } + + private static List columns(List columns) { + return Optional.ofNullable(columns).orElseGet(List::of).stream() + .map(id -> EntitySchemaAdapter.from(id.resolve())).toList(); + } + + private static Set stringValues(Object value) { + return value instanceof Set values + ? values.stream().map(String.class::cast).collect(java.util.stream.Collectors.toCollection(java.util.LinkedHashSet::new)) + : Set.of((String) value); + } + + private static NumberRange numberRange(IRange range) { + return new NumberRange( + Optional.ofNullable((Number) range.getMin()).map(ResolvedOperationAdapter::decimal), + Optional.ofNullable((Number) range.getMax()).map(ResolvedOperationAdapter::decimal)); + } + + private static BigDecimal decimal(Number value) { + return value instanceof BigDecimal decimal ? decimal : new BigDecimal(value.toString()); + } + + private static ResolvedColumn stringColumn(SqlTable table, String physicalName) { + return new ResolvedColumn(table.logicalId() + "." + physicalName, table, physicalName, + com.bakdata.conquery.models.datasets.ColumnType.STRING, true); + } + + private static com.bakdata.conquery.sql.model.result.ResultType resultType(ResultType type) { + if (type instanceof ResultType.ListT list) { + return new com.bakdata.conquery.sql.model.result.ResultType.ListType(primitive((ResultType.Primitive) list.getElementType())); + } + return primitive((ResultType.Primitive) type); + } + + private static com.bakdata.conquery.sql.model.result.ResultType.Primitive primitive(ResultType.Primitive type) { + return com.bakdata.conquery.sql.model.result.ResultType.Primitive.valueOf(type.name()); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java new file mode 100644 index 0000000000..5a579edee0 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java @@ -0,0 +1,227 @@ +package com.bakdata.conquery.sql.conversion; + +import java.time.Clock; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import com.bakdata.conquery.apiv1.query.CQElement; +import com.bakdata.conquery.apiv1.query.CQYes; +import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; +import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import com.bakdata.conquery.apiv1.query.concept.specific.CQDateRestriction; +import com.bakdata.conquery.apiv1.query.concept.specific.CQNegation; +import com.bakdata.conquery.apiv1.query.concept.specific.CQOr; +import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; +import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.config.IdColumnConfig; +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.datasets.SecondaryIdDescription; +import com.bakdata.conquery.models.datasets.concepts.ConceptElement; +import com.bakdata.conquery.models.datasets.concepts.Connector; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; +import com.bakdata.conquery.models.datasets.concepts.select.Select; +import com.bakdata.conquery.models.datasets.concepts.select.concept.ConceptColumnSelect; +import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; +import com.bakdata.conquery.models.identifiable.ids.specific.ConceptElementId; +import com.bakdata.conquery.models.identifiable.ids.specific.SelectId; +import com.bakdata.conquery.models.query.DateAggregationAction; +import com.bakdata.conquery.models.query.DateAggregationMode; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptIdMapping; +import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; +import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; +import com.bakdata.conquery.sql.mapping.ConceptIdMappingSelection; +import com.bakdata.conquery.sql.model.ResolvedQuery; +import com.bakdata.conquery.sql.model.node.AllEntitiesNode; +import com.bakdata.conquery.sql.model.node.AndNode; +import com.bakdata.conquery.sql.model.node.ConceptNode; +import com.bakdata.conquery.sql.model.node.DateRestrictionNode; +import com.bakdata.conquery.sql.model.node.ExternalEntity; +import com.bakdata.conquery.sql.model.node.ExternalNode; +import com.bakdata.conquery.sql.model.node.NegationNode; +import com.bakdata.conquery.sql.model.node.OrNode; +import com.bakdata.conquery.sql.model.node.QueryNode; +import com.bakdata.conquery.sql.model.operation.BuiltInConditions; +import com.bakdata.conquery.sql.model.operation.ResolvedCondition; +import com.bakdata.conquery.sql.model.operation.ResolvedSelect; +import com.bakdata.conquery.sql.model.range.DateRange; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.ResolvedConnector; +import com.bakdata.conquery.sql.model.schema.ResolvedValidityDate; +import com.bakdata.conquery.sql.model.schema.SqlTable; +import com.bakdata.conquery.util.TablePrimaryColumnUtil; + +/** Resolves the initialized backend concept-query tree into the SQL connector API. */ +public final class ResolvedQueryMapper { + + private final IdColumnConfig idColumns; + private final LegacyCompilerDialect dialect; + private final Clock clock; + private final String defaultPrimaryColumn; + + public ResolvedQueryMapper( + IdColumnConfig idColumns, + LegacyCompilerDialect dialect, + Clock clock, + String defaultPrimaryColumn + ) { + this.idColumns = idColumns; + this.dialect = dialect; + this.clock = clock; + this.defaultPrimaryColumn = defaultPrimaryColumn; + } + + public ResolvedQuery map(ConceptQuery query, Optional secondaryId) { + return map(query, secondaryId, query.getResultInfos()); + } + + public ResolvedQuery map( + ConceptQuery query, + Optional secondaryId, + List resultInfos + ) { + NodeContext context = new NodeContext(secondaryId, Optional.empty()); + return new ResolvedQuery( + EntitySchemaAdapter.from(idColumns), + mapNode(query.getRoot(), context), + query.getResolvedDateAggregationMode() != DateAggregationMode.NONE, + ResolvedOperationAdapter.resultColumns(resultInfos) + ); + } + + private QueryNode mapNode(CQElement node, NodeContext context) { + return switch (node) { + case CQYes ignored -> new AllEntitiesNode(); + case CQAnd and -> new AndNode(and.getChildren().stream().map(child -> mapNode(child, context)).toList(), + and.getDateAction(), and.getCreateExists().orElse(false)); + case CQOr or -> new OrNode(or.getChildren().stream().map(child -> mapNode(child, context)).toList(), + or.getDateAction(), or.getCreateExists().orElse(false)); + case CQNegation negation -> new NegationNode(mapNode(negation.getChild(), context), negation.getDateAction()); + case CQDateRestriction restriction -> { + DateRange range = new DateRange(Optional.ofNullable(restriction.getDateRange().getMin()), + Optional.ofNullable(restriction.getDateRange().getMax())); + yield new DateRestrictionNode(range, mapNode(restriction.getChild(), context.withDateRestriction(range))); + } + case CQExternal external -> external(external); + case CQReusedQuery reused -> mapNode(reused.getResolvedQuery().getReusableComponents(), + reused.isExcludeFromSecondaryId() ? context.withSecondaryId(Optional.empty()) : context); + case CQConcept concept -> concept(concept, context); + default -> throw new UnsupportedOperationException("SQL resolved query node is not implemented for " + node.getClass()); + }; + } + + private ConceptNode concept(CQConcept query, NodeContext context) { + TreeConcept concept = (TreeConcept) query.getConcept(); + List connectors = query.getTables().stream().map(table -> table.getConnector().resolve()).toList(); + List conceptColumns = connectors.stream() + .map(Connector::getColumn).filter(java.util.Objects::nonNull) + .map(id -> EntitySchemaAdapter.from(id.resolve())).toList(); + LocalDate endDate = context.dateRestriction().flatMap(DateRange::endInclusive).orElseGet(() -> LocalDate.now(clock)); + List conceptSelects = query.getSelects().stream().map(SelectId::resolve) + .map(select -> ResolvedOperationAdapter.select(select, conceptColumns, endDate)).toList(); + boolean resolveConceptIds = query.getSelects().stream().map(SelectId::resolve) + .filter(ConceptColumnSelect.class::isInstance).map(ConceptColumnSelect.class::cast) + .anyMatch(ConceptColumnSelect::isAsIds); + List> selected = query.getElements().stream().>map(ConceptElementId::resolve).toList(); + List resolvedConnectors = query.getTables().stream() + .map(table -> connector(query, table, concept, selected, resolveConceptIds, conceptColumns, endDate, context)) + .toList(); + return new ConceptNode( + query.userLabel(java.util.Locale.ROOT), + resolvedConnectors, + conceptSelects, + query.isAggregateEventDates() ? DateAggregationAction.MERGE : DateAggregationAction.BLOCK + ); + } + + private ResolvedConnector connector( + CQConcept query, + CQTable table, + TreeConcept concept, + List> selected, + boolean resolveConceptIds, + List conceptColumns, + LocalDate endDate, + NodeContext context + ) { + Connector connector = table.getConnector().resolve(); + SqlTable sqlTable = EntitySchemaAdapter.from(connector.getResolvedTable()); + org.jooq.Field primary = TablePrimaryColumnUtil.findPrimaryColumn(connector.getResolvedTable(), defaultPrimaryColumn); + ResolvedColumn primaryId = new ResolvedColumn( + sqlTable.logicalId() + ".primary-id", sqlTable, primary.getName(), + com.bakdata.conquery.models.datasets.ColumnType.STRING, false); + Optional secondaryId = Optional.empty(); + if (!query.isExcludeFromSecondaryId() && context.secondaryId().isPresent() + && table.hasSelectedSecondaryId(context.secondaryId().orElseThrow().getId())) { + Column column = connector.getResolvedTable().findSecondaryIdColumn(context.secondaryId().orElseThrow().getId()); + secondaryId = Optional.of(EntitySchemaAdapter.from(column)); + } + Optional connectorColumn = Optional.ofNullable(connector.getColumn()) + .map(id -> EntitySchemaAdapter.from(id.resolve())); + List conditions = new ArrayList<>(); + if (connector.getCondition() != null) { + conditions.add(ResolvedOperationAdapter.condition(connector.getCondition(), sqlTable, connectorColumn)); + } + connectorColumn.ifPresent(column -> conditions.add(new BuiltInConditions.Presence(column, true))); + List selects = table.getSelects().stream().map(SelectId::resolve) + .map(select -> ResolvedOperationAdapter.select(select, conceptColumns, endDate)).toList(); + ConceptIdMapping mapping = new ConceptIdMapping(concept, dialect.getFunctionProvider()); + ConceptIdMappingSelection mappingSelection = new ConceptIdMappingSelection( + mapping.source(connector), mapping.includedLocalIds(selected), mapping.includesRoot(selected), + resolveConceptIds, concept.getLocalId()); + return new ResolvedConnector( + connector.getName(), sqlTable, primaryId, secondaryId, validityDate(table.findValidityDate()), + table.getFilters().stream().map(filter -> ResolvedOperationAdapter.filter(filter, endDate)).toList(), + selects, conditions, Optional.of(mappingSelection) + ); + } + + private static ResolvedValidityDate validityDate(ValidityDate validityDate) { + if (validityDate == null) { + return new ResolvedValidityDate.None(); + } + if (validityDate.isSingleColumnDaterange()) { + return new ResolvedValidityDate.Point(EntitySchemaAdapter.from(validityDate.getColumn().resolve())); + } + return new ResolvedValidityDate.Range( + EntitySchemaAdapter.from(validityDate.getStartColumn().resolve()), + EntitySchemaAdapter.from(validityDate.getEndColumn().resolve())); + } + + private static ExternalNode external(CQExternal external) { + List entities = external.getValuesResolved().entrySet().stream() + .map(entry -> externalEntity(external, entry.getKey(), entry.getValue().asRanges())).toList(); + return new ExternalNode(entities, external.getExtraHeaders()); + } + + private static ExternalEntity externalEntity( + CQExternal external, + String entityId, + Collection validityDates + ) { + Map> values = new LinkedHashMap<>(); + external.getExtrasForId(entityId).forEach(entry -> values.put(entry.getKey(), entry.getValue())); + return new ExternalEntity(entityId, validityDates.stream().map(ResolvedQueryMapper::dateRange).toList(), values); + } + + private static DateRange dateRange(CDateRange range) { + return new DateRange(Optional.ofNullable(range.getMin()), Optional.ofNullable(range.getMax())); + } + + private record NodeContext(Optional secondaryId, Optional dateRestriction) { + private NodeContext withSecondaryId(Optional value) { + return new NodeContext(value, dateRestriction); + } + + private NodeContext withDateRestriction(DateRange value) { + return new NodeContext(secondaryId, Optional.of(value)); + } + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java index 3b7f22822b..bf346bda8e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java @@ -1,21 +1,52 @@ package com.bakdata.conquery.sql.conversion; +import java.util.Optional; + +import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.QueryDescription; +import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; +import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.model.SqlQuery; public class SqlConverter { private final NodeConversions nodeConversions; private final ConqueryConfig config; + private final ResolvedQueryMapper resolvedQueryMapper; + private final ResolvedQueryAdapter resolvedQueryAdapter; + private final LegacyCompilerDialect dialect; - public SqlConverter(NodeConversions nodeConversions, ConqueryConfig config) { + public SqlConverter( + NodeConversions nodeConversions, + ConqueryConfig config, + ResolvedQueryMapper resolvedQueryMapper, + ResolvedQueryAdapter resolvedQueryAdapter, + LegacyCompilerDialect dialect + ) { this.nodeConversions = nodeConversions; this.config = config; + this.resolvedQueryMapper = resolvedQueryMapper; + this.resolvedQueryAdapter = resolvedQueryAdapter; + this.dialect = dialect; } public SqlQuery convert(QueryDescription queryDescription) { + if (queryDescription instanceof ConceptQuery conceptQuery) { + return resolvedQueryAdapter.compile( + resolvedQueryMapper.map(conceptQuery, Optional.empty()), dialect, conceptQuery.getResultInfos()); + } + if (queryDescription instanceof SecondaryIdQuery secondaryIdQuery) { + return resolvedQueryAdapter.compile( + resolvedQueryMapper.map( + secondaryIdQuery.getQuery(), + Optional.of(secondaryIdQuery.getSecondaryId().resolve()), + secondaryIdQuery.getResultInfos()), + dialect, + secondaryIdQuery.getResultInfos() + ); + } ConversionContext converted = nodeConversions.convert(queryDescription, config); return converted.getFinalQuery(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 0f2ea9c8c0..d84deb9eae 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -256,7 +256,7 @@ private ConnectorCtePipelineInput createConnectorCteInput( conceptIdMapping, conceptElements, resolveConceptIds, - functionProvider + conversionContext.getCompilerDialect() ), ids, validityDateCalculation, @@ -271,22 +271,15 @@ private static org.jooq.Table createConceptSourceTable( ConceptIdMapping conceptIdMapping, List> selectedConceptElements, boolean resolveConceptIds, - SqlFunctionProvider functionProvider + com.bakdata.conquery.sql.compiler.dialect.CompilerDialect dialect ) { org.jooq.Table connectorTable = table(name(connectorTables.getPredecessor(ConceptCteStep.PREPROCESSING))); - - if (conceptIdMapping.includesRoot(selectedConceptElements) && !resolveConceptIds) { - return connectorTable; - } - - Condition conceptFilterCondition = selectedConceptElements.isEmpty() - ? noCondition() - : conceptIdMapping.resolvedId().in(conceptIdMapping.includedLocalIds(selectedConceptElements)); - - return functionProvider.innerJoin( + return (org.jooq.Table) conceptIdMapping.source(connectorTables.getConnector()).selectedConcepts( connectorTable, - conceptIdMapping.table(), - List.of(conceptIdMapping.joinCondition(connectorTables.getConnector()).and(conceptFilterCondition)) + conceptIdMapping.includedLocalIds(selectedConceptElements), + conceptIdMapping.includesRoot(selectedConceptElements), + resolveConceptIds, + dialect ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptIdMapping.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptIdMapping.java index 246cd6868c..cb147bba4a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptIdMapping.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptIdMapping.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.mapping.ConceptIdMappingResolver; +import com.bakdata.conquery.sql.mapping.ConceptIdMappingSource; import com.bakdata.conquery.sql.mapping.ConceptIdMappingTable; import lombok.Data; import org.jooq.*; @@ -100,26 +101,15 @@ public ConceptIdMappingTable mappingTable() { return new ConceptIdMappingTable(tableName, keyFields, rows); } - public Condition joinCondition(Connector connector) { + public ConceptIdMappingSource source(Connector connector) { CTConditionContext context = CTConditionContext.forConnector(connector, functionProvider); List expressions = collectAllExpressions(concept, null, context); Map> extractors = ConceptIdMappingResolver.collectExtractors(resolveExpressions(expressions)); - ConceptIdMappingTable mappingTable = mappingTable(); - - Condition condition = noCondition(); - for (Field keyField : keyFields) { - Field extractor = extractors.get(keyField.getName()); - if (extractor == null) { - throw new IllegalArgumentException( - "No connector expression for concept mapping field `%s` in connector %s" - .formatted(keyField.getName(), connector.getId()) - ); - } - Field mappingField = mappingTable.mappingField(keyField); - condition = condition.and(mappingField.eq((Field) extractor)); - } + return new ConceptIdMappingSource(mappingTable(), extractors); + } - return condition.equals(noCondition()) ? functionProvider.unconditionalJoinCondition() : condition; + public Condition joinCondition(Connector connector) { + return source(connector).joinCondition(functionProvider.unconditionalJoinCondition()); } /** diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java index 4bf10840b8..abbf6726ed 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java @@ -55,6 +55,11 @@ */ public interface LegacyCompilerDialect extends CompilerDialect { + @Override + default Condition unconditionalJoinCondition() { + return getCompilerDialect().unconditionalJoinCondition(); + } + @Override default ConnectorSqlSelects distinctSelect(BuiltInSelects.Values select, SelectConversionContext context) { return getCompilerDialect().distinctSelect(select, context); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java index 671350ae87..4c0def50bd 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java @@ -17,6 +17,7 @@ import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; import com.bakdata.conquery.sql.model.operation.BuiltInSelects; import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.mapping.ConceptIdMappingSource; import org.jooq.Condition; import org.jooq.Field; import org.jooq.Record; @@ -90,12 +91,11 @@ private static SelectConversionContext.ConceptColumnSource preparedSource( ConceptIdMapping mapping = new ConceptIdMapping(concept, context.getFunctionProvider()); Table connectorTable = SchemaSql.table(column.table()); - TableLike sourceTable = context.getFunctionProvider().leftJoin( - connectorTable, - mapping.table(), - List.of(mapping.joinCondition(connector)) + ConceptIdMappingSource mappingSource = mapping.source(connector); + TableLike sourceTable = mappingSource.resolvedConceptIds( + connectorTable, context.getCompilerDialect() ); - Field resolvedId = DSL.coalesce(mapping.resolvedId(), DSL.inline(concept.getLocalId())); + Field resolvedId = mappingSource.resolvedIdOrRoot(concept.getLocalId()); Condition rawValuePresent = SchemaSql.field(column, Object.class).isNotNull(); return new SelectConversionContext.ConceptColumnSource( connectorTable.getName(), diff --git a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlConverterTest.java b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlConverterTest.java new file mode 100644 index 0000000000..5ecd52b8fc --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlConverterTest.java @@ -0,0 +1,90 @@ +package com.bakdata.conquery.sql.conversion; + +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.apiv1.query.CQYes; +import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.apiv1.query.QueryDescription; +import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.datasets.SecondaryIdDescription; +import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; +import com.bakdata.conquery.models.query.DateAggregationMode; +import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; +import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; +import com.bakdata.conquery.sql.conversion.model.SqlQuery; +import com.bakdata.conquery.sql.model.ResolvedQuery; +import org.junit.jupiter.api.Test; + +class SqlConverterTest { + + private final NodeConversions nodeConversions = mock(NodeConversions.class); + private final ConqueryConfig config = mock(ConqueryConfig.class); + private final ResolvedQueryMapper mapper = mock(ResolvedQueryMapper.class); + private final ResolvedQueryAdapter adapter = mock(ResolvedQueryAdapter.class); + private final LegacyCompilerDialect dialect = mock(LegacyCompilerDialect.class); + private final SqlConverter converter = new SqlConverter(nodeConversions, config, mapper, adapter, dialect); + + @Test + void shouldRouteConceptQueriesThroughTheResolvedCompiler() { + ConceptQuery query = new ConceptQuery(new CQYes()); + query.setResolvedDateAggregationMode(DateAggregationMode.NONE); + ResolvedQuery resolved = mock(ResolvedQuery.class); + SqlQuery sql = mock(SqlQuery.class); + when(mapper.map(same(query), eq(Optional.empty()))).thenReturn(resolved); + when(adapter.compile(resolved, dialect, List.of())).thenReturn(sql); + + SqlQuery converted = converter.convert(query); + + assertSame(sql, converted); + verify(mapper).map(same(query), eq(Optional.empty())); + verify(adapter).compile(resolved, dialect, List.of()); + verifyNoInteractions(nodeConversions); + } + + @Test + void shouldRouteSecondaryIdQueriesThroughTheResolvedCompiler() { + SecondaryIdQuery query = mock(SecondaryIdQuery.class); + ConceptQuery nested = mock(ConceptQuery.class); + SecondaryIdDescriptionId secondaryId = mock(SecondaryIdDescriptionId.class); + SecondaryIdDescription resolvedSecondaryId = mock(SecondaryIdDescription.class); + ResolvedQuery resolved = mock(ResolvedQuery.class); + SqlQuery sql = mock(SqlQuery.class); + when(query.getQuery()).thenReturn(nested); + when(query.getSecondaryId()).thenReturn(secondaryId); + when(secondaryId.resolve()).thenReturn(resolvedSecondaryId); + when(query.getResultInfos()).thenReturn(List.of()); + when(mapper.map(nested, Optional.of(resolvedSecondaryId), List.of())).thenReturn(resolved); + when(adapter.compile(resolved, dialect, List.of())).thenReturn(sql); + + SqlQuery converted = converter.convert(query); + + assertSame(sql, converted); + verify(mapper).map(nested, Optional.of(resolvedSecondaryId), List.of()); + verify(adapter).compile(resolved, dialect, List.of()); + verifyNoInteractions(nodeConversions); + } + + @Test + void shouldKeepOtherQueryDescriptionsOnTheLegacyConversionPath() { + QueryDescription query = mock(QueryDescription.class); + ConversionContext context = mock(ConversionContext.class); + SqlQuery sql = mock(SqlQuery.class); + when(nodeConversions.convert(query, config)).thenReturn(context); + when(context.getFinalQuery()).thenReturn(sql); + + SqlQuery converted = converter.convert(query); + + assertSame(sql, converted); + verifyNoInteractions(mapper, adapter); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ResolvedConceptCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ResolvedConceptCompiler.java index 667a68caf7..2de6dd32d5 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ResolvedConceptCompiler.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ResolvedConceptCompiler.java @@ -42,8 +42,10 @@ import com.bakdata.conquery.sql.model.operation.ResolvedSelect; import com.bakdata.conquery.sql.model.range.DateRange; import com.bakdata.conquery.sql.model.schema.EntitySchema; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; import com.bakdata.conquery.sql.model.schema.ResolvedConnector; import com.bakdata.conquery.sql.model.schema.ResolvedValidityDate; +import com.bakdata.conquery.sql.mapping.ConceptIdMappingSelection; import lombok.experimental.UtilityClass; import org.jooq.Condition; import org.jooq.impl.DSL; @@ -61,7 +63,7 @@ static QueryStep compile(ConceptNode concept, EntitySchema entitySchema, Compile .toList(); QueryStep joined = LogicalQueryStepCompiler.compile( connectors.stream().map(ConnectorCompilation::step).toList(), JoinMode.FULL_OUTER, - DateAggregationAction.MERGE, false, entitySchema, dialect, names); + concept.dateAction(), false, entitySchema, dialect, names); SqlTables tables = ConceptCtePlanner.planConcept(joined, names.conceptName(concept), concept.selects().stream().anyMatch(ResolvedConceptCompiler::isEventDateSelect), dialect, names); Selects joinedSelects = joined.getQualifiedSelects(); @@ -104,26 +106,62 @@ private static Optional compileConnector(ConceptNode conce WhereClauses.builder().eventFilter(new DateRestrictionCondition(dialect.dateRangeLiteral(range), validityDate)).build()))); List selects = new ArrayList<>(); for (ResolvedSelect select : concept.selects()) { - if (select instanceof BuiltInSelects.ConceptValues) { - selects.add(selectConverter.convert(select, selectContext(dialect, names, plan, ids, validityDate, - connectorName, names.selectName(select)))); + if (select instanceof BuiltInSelects.ConceptValues values) { + values.columns().stream() + .filter(column -> column.table().equals(connector.table())) + .findFirst() + .ifPresent(column -> { + BuiltInSelects.ConceptValues connectorValues = new BuiltInSelects.ConceptValues( + values.name(), List.of(column)); + selects.add(selectConverter.convert(connectorValues, + selectContext(dialect, names, plan, ids, validityDate, connectorName, + names.selectName(select), conceptValueSources(column, connector, plan, dialect)))); + }); } } for (ResolvedSelect select : connector.selects()) { selects.add(selectConverter.convert(select, selectContext(dialect, names, plan, ids, validityDate, connectorName, names.selectName(select)))); } + org.jooq.Table sourceTable = connector.conceptIdMapping() + .map(mapping -> mapping.selectedConcepts(plan.sourceTable(), dialect)) + .orElse(plan.sourceTable()); return ConnectorCteCompiler.compileConnector(ConnectorCtePipelineAssembler.assemble( - plan, plan.sourceTable(), ids, validityDate, selects, filters, Optional.empty())) + plan, sourceTable, ids, validityDate, selects, filters, Optional.empty())) .map(step -> new ConnectorCompilation(connector, plan, step)); } + private static List conceptValueSources( + ResolvedColumn column, + ResolvedConnector connector, + ConnectorCtePlan plan, + CompilerDialect dialect + ) { + if (!connector.conceptIdMapping().map(ConceptIdMappingSelection::resolveConceptIds).orElse(false)) { + return List.of(); + } + ConceptIdMappingSelection mapping = connector.conceptIdMapping().orElseThrow(); + return List.of(new SelectConversionContext.ConceptColumnSource( + plan.sourceTable().getName(), + mapping.source().resolvedConceptIds(plan.sourceTable(), dialect), + mapping.source().resolvedIdOrRoot(mapping.rootLocalId()), + List.of(SchemaSql.field(column, Object.class).isNotNull()) + )); + } + private static SelectConversionContext selectContext(CompilerDialect dialect, SqlNameGenerator names, ConnectorCtePlan plan, SqlIdColumns ids, ColumnDateRange validityDate, String connectorName, String alias) { return new SelectConversionContext(dialect, names, plan.tables(), ids, Optional.of(validityDate.asValidityDateRange(connectorName)), alias); } + private static SelectConversionContext selectContext(CompilerDialect dialect, SqlNameGenerator names, + ConnectorCtePlan plan, SqlIdColumns ids, ColumnDateRange validityDate, String connectorName, String alias, + List conceptColumnSources) { + return new SelectConversionContext(dialect, names, plan.tables(), ids, + Optional.of(validityDate.asValidityDateRange(connectorName)), alias, Map.of(), conceptColumnSources); + } + private static ColumnDateRange validityDate(ResolvedValidityDate value, Optional restriction, CompilerDialect dialect) { ColumnDateRange physical = switch (value) { diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java index 99f824cda5..3f8aeed0ac 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java @@ -36,6 +36,11 @@ */ public interface CompilerDialect { + /** Any condition accepted by this database when a join has no mapping-key comparison. */ + default Condition unconditionalJoinCondition() { + return DSL.noCondition(); + } + default ColumnDateRange toDualColumn(ColumnDateRange range) { return ColumnDateRange.of(range.getStart(), range.getEnd()); } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/hana/HanaCompilerDialect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/hana/HanaCompilerDialect.java index a1b1db3794..32a8b4e4aa 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/hana/HanaCompilerDialect.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/hana/HanaCompilerDialect.java @@ -50,6 +50,7 @@ public Table literalSelectTable() { return table(name("DUMMY")); } + @Override public Condition unconditionalJoinCondition() { return inline(true).eq(inline(true)); } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingSelection.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingSelection.java new file mode 100644 index 0000000000..423cb22c2b --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingSelection.java @@ -0,0 +1,33 @@ +package com.bakdata.conquery.sql.mapping; + +import java.util.Objects; +import java.util.Set; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import org.jooq.Table; +import org.jooq.Record; + +/** Fully resolved concept-element selection applied to one connector source. */ +public record ConceptIdMappingSelection( + ConceptIdMappingSource source, + Set includedLocalIds, + boolean includesRoot, + boolean resolveConceptIds, + int rootLocalId +) { + + public ConceptIdMappingSelection { + Objects.requireNonNull(source, "source"); + includedLocalIds = Set.copyOf(includedLocalIds); + } + + public Table selectedConcepts(Table connectorTable, CompilerDialect dialect) { + return source.selectedConcepts( + connectorTable, + includedLocalIds, + includesRoot, + resolveConceptIds, + dialect + ); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingSource.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingSource.java new file mode 100644 index 0000000000..747850c1f1 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingSource.java @@ -0,0 +1,69 @@ +package com.bakdata.conquery.sql.mapping; + +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.TableLike; +import org.jooq.impl.DSL; + +/** SQL source built by joining connector values to their most-specific concept element. */ +public record ConceptIdMappingSource( + ConceptIdMappingTable mapping, + Map> extractors +) { + + public ConceptIdMappingSource { + Objects.requireNonNull(mapping, "mapping"); + extractors = Map.copyOf(extractors); + } + + public Table selectedConcepts( + Table connectorTable, + Set includedLocalIds, + boolean includesRoot, + boolean resolveConceptIds, + CompilerDialect dialect + ) { + if (includesRoot && !resolveConceptIds) { + return connectorTable; + } + + Condition conceptFilterCondition = includedLocalIds.isEmpty() + ? DSL.noCondition() + : mapping.resolvedId().in(includedLocalIds); + + return connectorTable.innerJoin(mapping.table()).on(joinCondition(dialect).and(conceptFilterCondition)); + } + + public TableLike resolvedConceptIds(Table connectorTable, CompilerDialect dialect) { + return connectorTable.leftJoin(mapping.table()).on(joinCondition(dialect)); + } + + public Field resolvedIdOrRoot(int rootLocalId) { + return DSL.coalesce(mapping.resolvedId(), DSL.inline(rootLocalId)); + } + + public Condition joinCondition(CompilerDialect dialect) { + return joinCondition(dialect.unconditionalJoinCondition()); + } + + public Condition joinCondition(Condition unconditionalJoinCondition) { + Condition condition = DSL.noCondition(); + for (Field keyField : mapping.keyFields()) { + Field extractor = extractors.get(keyField.getName()); + if (extractor == null) { + throw new IllegalArgumentException( + "No connector expression for concept mapping field `%s`".formatted(keyField.getName()) + ); + } + condition = condition.and(mapping.mappingField(keyField).eq((Field) extractor)); + } + return condition.equals(DSL.noCondition()) ? unconditionalJoinCondition : condition; + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/ResolvedConnector.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/ResolvedConnector.java index 4201426902..0c871989fc 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/ResolvedConnector.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/schema/ResolvedConnector.java @@ -7,6 +7,7 @@ import com.bakdata.conquery.sql.model.operation.ResolvedCondition; import com.bakdata.conquery.sql.model.operation.ResolvedFilter; import com.bakdata.conquery.sql.model.operation.ResolvedSelect; +import com.bakdata.conquery.sql.mapping.ConceptIdMappingSelection; import jakarta.validation.Valid; import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.NotBlank; @@ -21,9 +22,23 @@ public record ResolvedConnector( @NotNull @Valid ResolvedValidityDate validityDate, @NotNull List<@NotNull @Valid ResolvedFilter> filters, @NotNull List<@NotNull @Valid ResolvedSelect> selects, - @NotNull List<@NotNull @Valid ResolvedCondition> conditions + @NotNull List<@NotNull @Valid ResolvedCondition> conditions, + @NotNull Optional conceptIdMapping ) { + public ResolvedConnector( + String logicalId, + SqlTable table, + ResolvedColumn primaryId, + Optional secondaryId, + ResolvedValidityDate validityDate, + List filters, + List selects, + List conditions + ) { + this(logicalId, table, primaryId, secondaryId, validityDate, filters, selects, conditions, Optional.empty()); + } + public ResolvedConnector { filters = ModelNormalization.immutableCopy(filters); selects = ModelNormalization.immutableCopy(selects); diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/DefaultSqlCompilerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/DefaultSqlCompilerTest.java index 69c7071b0b..f7c8a408df 100644 --- a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/DefaultSqlCompilerTest.java +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/DefaultSqlCompilerTest.java @@ -6,10 +6,14 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import com.bakdata.conquery.models.datasets.ColumnType; import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.sql.compiler.dialect.hana.HanaCompilerDialect; +import com.bakdata.conquery.sql.mapping.ConceptIdMappingSelection; +import com.bakdata.conquery.sql.mapping.ConceptIdMappingSource; +import com.bakdata.conquery.sql.mapping.ConceptIdMappingTable; import com.bakdata.conquery.sql.model.ResolvedQuery; import com.bakdata.conquery.sql.model.node.AllEntitiesNode; import com.bakdata.conquery.sql.model.node.ConceptNode; @@ -27,6 +31,7 @@ import com.bakdata.conquery.sql.model.schema.SqlTable; import org.jooq.SQLDialect; import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; import org.junit.jupiter.api.Test; class DefaultSqlCompilerTest { @@ -87,6 +92,55 @@ void shouldCompileAResolvedConceptThroughTheConnectorPipeline() { assertEquals("diagnosis.exists", compiled.columns().get(1).outputId()); } + @Test + void shouldApplyConceptSelectionAndResolveConceptIdsInTheConnector() { + SqlTable events = SqlTable.of("events", "catalog", "events"); + ResolvedColumn conceptColumn = new ResolvedColumn( + "events.diagnosis", events, "diagnosis", ColumnType.STRING, true + ); + ConceptIdMappingSource mappingSource = new ConceptIdMappingSource( + new ConceptIdMappingTable( + DSL.name("diagnosis_ids"), + List.of(DSL.field(DSL.name("code"), SQLDataType.VARCHAR)), + List.of() + ), + Map.of("code", DSL.field(DSL.name("events", "diagnosis"), String.class)) + ); + ResolvedConnector connector = new ResolvedConnector( + "claims", + events, + new ResolvedColumn("events.entity-id", events, "person_id", ColumnType.STRING, false), + Optional.empty(), + new ResolvedValidityDate.None(), + List.of(), + List.of(), + List.of(), + Optional.of(new ConceptIdMappingSelection(mappingSource, Set.of(4, 7), false, true, 0)) + ); + ResolvedQuery query = new ResolvedQuery( + new EntitySchema(ENTITY_ID), + new ConceptNode( + "diagnosis", + List.of(connector), + List.of(new BuiltInSelects.ConceptValues("concept ids", List.of(conceptColumn))), + DateAggregationAction.BLOCK + ), + false, + List.of(new ResultColumn("concept-ids", new ResultType.ListType(ResultType.Primitive.STRING))) + ); + + CompiledQuery compiled = new DefaultSqlCompiler(DSL.using(SQLDialect.DEFAULT)) + .compile(query, new HanaCompilerDialect()); + + assertTrue( + compiled.sql().contains("\"diagnosis_ids\".\"resolved_id\" in (4, 7)") + || compiled.sql().contains("\"diagnosis_ids\".\"resolved_id\" in (7, 4)"), + compiled.sql() + ); + assertTrue(compiled.sql().contains("coalesce(\"diagnosis_ids\".\"resolved_id\", 0)"), compiled.sql()); + assertEquals("concept-ids", compiled.columns().get(1).outputId()); + } + @Test void shouldExposeASecondaryIdAsTheFirstResultColumn() { SqlTable events = SqlTable.of("events", "catalog", "events"); diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingSourceTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingSourceTest.java new file mode 100644 index 0000000000..07c5b45b17 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/mapping/ConceptIdMappingSourceTest.java @@ -0,0 +1,79 @@ +package com.bakdata.conquery.sql.mapping; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + +import com.bakdata.conquery.sql.compiler.dialect.hana.HanaCompilerDialect; +import org.jooq.Field; +import org.jooq.SQLDialect; +import org.jooq.Table; +import org.jooq.TableLike; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.junit.jupiter.api.Test; + +class ConceptIdMappingSourceTest { + + private static final Table EVENTS = DSL.table(DSL.name("events")); + private static final Field MAPPING_CODE = DSL.field(DSL.name("code"), SQLDataType.VARCHAR(12)); + private static final Field EVENT_CODE = DSL.field(DSL.name("events", "diagnosis"), String.class); + private static final ConceptIdMappingTable MAPPING = new ConceptIdMappingTable( + DSL.name("diagnosis_ids"), List.of(MAPPING_CODE), List.of() + ); + + @Test + void shouldFilterSelectedConceptIdsWithAnInnerJoin() { + ConceptIdMappingSource source = new ConceptIdMappingSource(MAPPING, Map.of("code", EVENT_CODE)); + + Table selected = source.selectedConcepts( + EVENTS, new LinkedHashSet<>(List.of(4, 7)), false, false, new HanaCompilerDialect() + ); + + assertEquals( + "\"events\" join \"diagnosis_ids\" on (\"diagnosis_ids\".\"code\" = \"events\".\"diagnosis\" and \"diagnosis_ids\".\"resolved_id\" in (4, 7))", + render(selected) + ); + } + + @Test + void shouldKeepTheConnectorTableWhenTheRootIsSelectedWithoutIdResolution() { + ConceptIdMappingSource source = new ConceptIdMappingSource(MAPPING, Map.of("code", EVENT_CODE)); + + Table selected = source.selectedConcepts( + EVENTS, Set.of(0, 4, 7), true, false, new HanaCompilerDialect() + ); + + assertEquals("\"events\"", render(selected)); + } + + @Test + void shouldLeftJoinAndFallBackToTheRootIdForConceptIdOutput() { + ConceptIdMappingSource source = new ConceptIdMappingSource(MAPPING, Map.of("code", EVENT_CODE)); + + TableLike resolved = source.resolvedConceptIds(EVENTS, new HanaCompilerDialect()); + + assertEquals( + "\"events\" left outer join \"diagnosis_ids\" on \"diagnosis_ids\".\"code\" = \"events\".\"diagnosis\"", + render(resolved) + ); + assertEquals("coalesce(\"diagnosis_ids\".\"resolved_id\", 3)", render(source.resolvedIdOrRoot(3))); + } + + @Test + void shouldUseTheHanaJoinConditionWhenTheMappingHasNoKeys() { + ConceptIdMappingTable rootOnly = new ConceptIdMappingTable(DSL.name("root_ids"), List.of(), List.of()); + ConceptIdMappingSource source = new ConceptIdMappingSource(rootOnly, Map.of()); + + TableLike resolved = source.resolvedConceptIds(EVENTS, new HanaCompilerDialect()); + + assertEquals("\"events\" left outer join \"root_ids\" on true = true", render(resolved)); + } + + private static String render(org.jooq.QueryPart part) { + return DSL.using(SQLDialect.POSTGRES).renderInlined(part); + } +} From 4458691fbd74e7d845091cf053da715618a20377 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 1 Oct 2026 16:47:32 +0200 Subject: [PATCH 71/81] refactor(sql): centralize resolved query adaptation Co-authored-by: Codex GPT-5.6-sol --- .../mode/local/LocalNamespaceHandler.java | 4 +- .../sql/conversion/ResolvedQueryAdapter.java | 32 +++++++- .../sql/conversion/ResolvedQueryMapper.java | 9 +-- .../conquery/sql/conversion/SqlConverter.java | 17 +--- .../sql/conversion/SqlConverterTest.java | 33 ++------ .../conversion/SqlExtractionQueryTest.java | 78 ++++++++++++++++++- 6 files changed, 119 insertions(+), 54 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java index fca73bd91a..b81069d10a 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java @@ -56,9 +56,9 @@ public LocalNamespace createNamespace( ResolvedQueryMapper resolvedQueryMapper = new ResolvedQueryMapper( config.getIdColumns(), dialectBundle, clock, connection.getConnection().getPrimaryColumn()); ResolvedQueryAdapter resolvedQueryAdapter = new ResolvedQueryAdapter( - environment.getValidator(), new DefaultSqlCompiler(dslContext)); + environment.getValidator(), new DefaultSqlCompiler(dslContext), resolvedQueryMapper); SqlConverter sqlConverter = new SqlConverter( - nodeConversions, config, resolvedQueryMapper, resolvedQueryAdapter, dialectBundle); + nodeConversions, config, resolvedQueryAdapter, dialectBundle); ExecutionManager executionManager = new SqlExecutionManager(sqlConverter, sqlExecutionService, metaStorage, datasetRegistry, config); SqlStorageHandler sqlStorageHandler = new SqlStorageHandler(sqlExecutionService); SqlEntityResolver sqlEntityResolver = new SqlEntityResolver(config.getIdColumns(), dslContext, dialectBundle, sqlExecutionService); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryAdapter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryAdapter.java index 227719bc64..43d1ab6aa9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryAdapter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryAdapter.java @@ -1,7 +1,11 @@ package com.bakdata.conquery.sql.conversion; import java.util.List; +import java.util.Optional; +import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; +import com.bakdata.conquery.models.datasets.SecondaryIdDescription; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.sql.compiler.ColumnRole; import com.bakdata.conquery.sql.compiler.CompiledQuery; @@ -12,15 +16,39 @@ import com.bakdata.conquery.sql.validation.ResolvedQueryValidation; import jakarta.validation.Validator; -/** Validates resolved application input and adapts connector output to the existing backend query contract. */ +/** Maps initialized backend queries, validates the resolved input, and adapts compiler output to the backend contract. */ public final class ResolvedQueryAdapter { private final ResolvedQueryValidation validation; private final SqlCompiler compiler; + private final ResolvedQueryMapper mapper; - public ResolvedQueryAdapter(Validator validator, SqlCompiler compiler) { + public ResolvedQueryAdapter(Validator validator, SqlCompiler compiler, ResolvedQueryMapper mapper) { this.validation = new ResolvedQueryValidation(validator); this.compiler = compiler; + this.mapper = mapper; + } + + public SqlQuery compile(ConceptQuery query, CompilerDialect dialect) { + return compile(query, Optional.empty(), query.getResultInfos(), dialect); + } + + public SqlQuery compile(SecondaryIdQuery query, CompilerDialect dialect) { + return compile( + query.getQuery(), + Optional.of(query.getSecondaryId().resolve()), + query.getResultInfos(), + dialect + ); + } + + private SqlQuery compile( + ConceptQuery query, + Optional secondaryId, + List resultInfos, + CompilerDialect dialect + ) { + return compile(mapper.map(query, secondaryId, resultInfos), dialect, resultInfos); } public SqlQuery compile( diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java index 5a579edee0..bea9cfc894 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java @@ -34,6 +34,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.SelectId; import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.models.query.DateAggregationMode; +import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptIdMapping; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; @@ -78,14 +79,10 @@ public ResolvedQueryMapper( this.defaultPrimaryColumn = defaultPrimaryColumn; } - public ResolvedQuery map(ConceptQuery query, Optional secondaryId) { - return map(query, secondaryId, query.getResultInfos()); - } - - public ResolvedQuery map( + ResolvedQuery map( ConceptQuery query, Optional secondaryId, - List resultInfos + List resultInfos ) { NodeContext context = new NodeContext(secondaryId, Optional.empty()); return new ResolvedQuery( diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java index bf346bda8e..c515fc85a2 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.sql.conversion; -import java.util.Optional; - import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; @@ -14,38 +12,27 @@ public class SqlConverter { private final NodeConversions nodeConversions; private final ConqueryConfig config; - private final ResolvedQueryMapper resolvedQueryMapper; private final ResolvedQueryAdapter resolvedQueryAdapter; private final LegacyCompilerDialect dialect; public SqlConverter( NodeConversions nodeConversions, ConqueryConfig config, - ResolvedQueryMapper resolvedQueryMapper, ResolvedQueryAdapter resolvedQueryAdapter, LegacyCompilerDialect dialect ) { this.nodeConversions = nodeConversions; this.config = config; - this.resolvedQueryMapper = resolvedQueryMapper; this.resolvedQueryAdapter = resolvedQueryAdapter; this.dialect = dialect; } public SqlQuery convert(QueryDescription queryDescription) { if (queryDescription instanceof ConceptQuery conceptQuery) { - return resolvedQueryAdapter.compile( - resolvedQueryMapper.map(conceptQuery, Optional.empty()), dialect, conceptQuery.getResultInfos()); + return resolvedQueryAdapter.compile(conceptQuery, dialect); } if (queryDescription instanceof SecondaryIdQuery secondaryIdQuery) { - return resolvedQueryAdapter.compile( - resolvedQueryMapper.map( - secondaryIdQuery.getQuery(), - Optional.of(secondaryIdQuery.getSecondaryId().resolve()), - secondaryIdQuery.getResultInfos()), - dialect, - secondaryIdQuery.getResultInfos() - ); + return resolvedQueryAdapter.compile(secondaryIdQuery, dialect); } ConversionContext converted = nodeConversions.convert(queryDescription, config); return converted.getFinalQuery(); diff --git a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlConverterTest.java b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlConverterTest.java index 5ecd52b8fc..f01192cd4f 100644 --- a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlConverterTest.java +++ b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlConverterTest.java @@ -1,76 +1,55 @@ package com.bakdata.conquery.sql.conversion; import static org.junit.jupiter.api.Assertions.assertSame; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; -import java.util.List; -import java.util.Optional; - import com.bakdata.conquery.apiv1.query.CQYes; import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.datasets.SecondaryIdDescription; -import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; import com.bakdata.conquery.models.query.DateAggregationMode; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.model.SqlQuery; -import com.bakdata.conquery.sql.model.ResolvedQuery; import org.junit.jupiter.api.Test; class SqlConverterTest { private final NodeConversions nodeConversions = mock(NodeConversions.class); private final ConqueryConfig config = mock(ConqueryConfig.class); - private final ResolvedQueryMapper mapper = mock(ResolvedQueryMapper.class); private final ResolvedQueryAdapter adapter = mock(ResolvedQueryAdapter.class); private final LegacyCompilerDialect dialect = mock(LegacyCompilerDialect.class); - private final SqlConverter converter = new SqlConverter(nodeConversions, config, mapper, adapter, dialect); + private final SqlConverter converter = new SqlConverter(nodeConversions, config, adapter, dialect); @Test void shouldRouteConceptQueriesThroughTheResolvedCompiler() { ConceptQuery query = new ConceptQuery(new CQYes()); query.setResolvedDateAggregationMode(DateAggregationMode.NONE); - ResolvedQuery resolved = mock(ResolvedQuery.class); SqlQuery sql = mock(SqlQuery.class); - when(mapper.map(same(query), eq(Optional.empty()))).thenReturn(resolved); - when(adapter.compile(resolved, dialect, List.of())).thenReturn(sql); + when(adapter.compile(same(query), same(dialect))).thenReturn(sql); SqlQuery converted = converter.convert(query); assertSame(sql, converted); - verify(mapper).map(same(query), eq(Optional.empty())); - verify(adapter).compile(resolved, dialect, List.of()); + verify(adapter).compile(same(query), same(dialect)); verifyNoInteractions(nodeConversions); } @Test void shouldRouteSecondaryIdQueriesThroughTheResolvedCompiler() { SecondaryIdQuery query = mock(SecondaryIdQuery.class); - ConceptQuery nested = mock(ConceptQuery.class); - SecondaryIdDescriptionId secondaryId = mock(SecondaryIdDescriptionId.class); - SecondaryIdDescription resolvedSecondaryId = mock(SecondaryIdDescription.class); - ResolvedQuery resolved = mock(ResolvedQuery.class); SqlQuery sql = mock(SqlQuery.class); - when(query.getQuery()).thenReturn(nested); - when(query.getSecondaryId()).thenReturn(secondaryId); - when(secondaryId.resolve()).thenReturn(resolvedSecondaryId); - when(query.getResultInfos()).thenReturn(List.of()); - when(mapper.map(nested, Optional.of(resolvedSecondaryId), List.of())).thenReturn(resolved); - when(adapter.compile(resolved, dialect, List.of())).thenReturn(sql); + when(adapter.compile(same(query), same(dialect))).thenReturn(sql); SqlQuery converted = converter.convert(query); assertSame(sql, converted); - verify(mapper).map(nested, Optional.of(resolvedSecondaryId), List.of()); - verify(adapter).compile(resolved, dialect, List.of()); + verify(adapter).compile(same(query), same(dialect)); verifyNoInteractions(nodeConversions); } @@ -85,6 +64,6 @@ void shouldKeepOtherQueryDescriptionsOnTheLegacyConversionPath() { SqlQuery converted = converter.convert(query); assertSame(sql, converted); - verifyNoInteractions(mapper, adapter); + verifyNoInteractions(adapter); } } diff --git a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionQueryTest.java b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionQueryTest.java index 8038d5b1f2..d018b8ee6c 100644 --- a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionQueryTest.java +++ b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionQueryTest.java @@ -3,11 +3,23 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.util.List; +import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; +import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; +import com.bakdata.conquery.models.datasets.SecondaryIdDescription; +import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.sql.compiler.dialect.hana.HanaCompilerDialect; import com.bakdata.conquery.sql.compiler.ColumnRole; import com.bakdata.conquery.sql.compiler.CompiledColumn; @@ -34,7 +46,8 @@ void shouldAdaptValidatedCompilerOutputToTheExistingSqlQueryContract() { List.of(new CompiledColumn( "entity-id", "entity_id", ResultType.Primitive.STRING, ColumnRole.ENTITY_ID )) - ) + ), + mock(ResolvedQueryMapper.class) ); var adapted = adapter.compile(query, new HanaCompilerDialect(), List.of()); @@ -51,7 +64,8 @@ void shouldRejectInvalidResolvedInputBeforeInvokingTheCompiler() { (query, dialect) -> { compilerInvoked.set(true); throw new AssertionError("compiler must not be invoked"); - } + }, + mock(ResolvedQueryMapper.class) ); SqlTable entities = SqlTable.of("entities", "entities"); ResolvedQuery invalid = new ResolvedQuery( @@ -69,6 +83,66 @@ invalid, new HanaCompilerDialect(), List.of() assertFalse(compilerInvoked.get()); } + @Test + void shouldMapAConceptQueryWithOneResultInfoSnapshot() { + ConceptQuery query = mock(ConceptQuery.class); + ResultInfo resultInfo = mock(ResultInfo.class); + List resultInfos = List.of(resultInfo); + ResolvedQuery resolved = validAllEntitiesQuery(); + ResolvedQueryMapper mapper = mock(ResolvedQueryMapper.class); + when(query.getResultInfos()).thenReturn(resultInfos); + when(mapper.map(same(query), eq(Optional.empty()), same(resultInfos))).thenReturn(resolved); + ResolvedQueryAdapter adapter = new ResolvedQueryAdapter( + Validation.buildDefaultValidatorFactory().getValidator(), + (ignored, dialect) -> compiledWithOneResult(), + mapper + ); + + var adapted = adapter.compile(query, new HanaCompilerDialect()); + + assertEquals(resultInfos, adapted.getResultInfos()); + verify(query, times(1)).getResultInfos(); + verify(mapper).map(same(query), eq(Optional.empty()), same(resultInfos)); + } + + @Test + void shouldMapASecondaryIdQueryWithItsOuterResultInfos() { + SecondaryIdQuery query = mock(SecondaryIdQuery.class); + ConceptQuery nested = mock(ConceptQuery.class); + SecondaryIdDescriptionId secondaryId = mock(SecondaryIdDescriptionId.class); + SecondaryIdDescription resolvedSecondaryId = mock(SecondaryIdDescription.class); + ResultInfo resultInfo = mock(ResultInfo.class); + List resultInfos = List.of(resultInfo); + ResolvedQuery resolved = validAllEntitiesQuery(); + ResolvedQueryMapper mapper = mock(ResolvedQueryMapper.class); + when(query.getQuery()).thenReturn(nested); + when(query.getSecondaryId()).thenReturn(secondaryId); + when(secondaryId.resolve()).thenReturn(resolvedSecondaryId); + when(query.getResultInfos()).thenReturn(resultInfos); + when(mapper.map(same(nested), eq(Optional.of(resolvedSecondaryId)), same(resultInfos))).thenReturn(resolved); + ResolvedQueryAdapter adapter = new ResolvedQueryAdapter( + Validation.buildDefaultValidatorFactory().getValidator(), + (ignored, dialect) -> compiledWithOneResult(), + mapper + ); + + var adapted = adapter.compile(query, new HanaCompilerDialect()); + + assertEquals(resultInfos, adapted.getResultInfos()); + verify(query, times(1)).getResultInfos(); + verify(mapper).map(same(nested), eq(Optional.of(resolvedSecondaryId)), same(resultInfos)); + } + + private static CompiledQuery compiledWithOneResult() { + return new CompiledQuery( + "select entity_id, result from entities", + List.of( + new CompiledColumn("entity-id", "entity_id", ResultType.Primitive.STRING, ColumnRole.ENTITY_ID), + new CompiledColumn("result", "result", ResultType.Primitive.STRING, ColumnRole.RESULT) + ) + ); + } + private static ResolvedQuery validAllEntitiesQuery() { SqlTable entities = SqlTable.of("entities", "entities"); return new ResolvedQuery( From 3158b3fb41a174d8beb9edc4d2d66176310becef Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Fri, 2 Oct 2026 11:12:43 +0200 Subject: [PATCH 72/81] refactor(sql): extract table export compilation Co-authored-by: Codex GPT-6 --- .../mode/local/LocalNamespaceHandler.java | 8 +- .../sql/conversion/ResolvedQueryMapper.java | 2 +- .../ResolvedTableExportAdapter.java | 24 ++++ .../conversion/ResolvedTableExportMapper.java | 85 ++++++++++++++ .../conquery/sql/conversion/SqlConverter.java | 7 ++ .../dialect/LegacyCompilerDialect.java | 4 +- .../sql/conversion/SqlConverterTest.java | 19 ++- .../sql/compiler/CompiledQueryColumns.java | 29 +++++ .../sql/compiler/DefaultSqlCompiler.java | 91 +-------------- .../sql/compiler/ResolvedConceptCompiler.java | 10 +- .../compiler/ResolvedQueryStepCompiler.java | 105 +++++++++++++++++ .../sql/compiler/TableExportSqlCompiler.java | 109 ++++++++++++++++++ .../ResolvedTableExportFilterConverter.java | 94 +++++++++++++++ .../sql/model/export/ResolvedExportTable.java | 26 +++++ .../export/ResolvedTableExportQuery.java | 20 ++++ ...esolvedTableExportFilterConverterTest.java | 53 +++++++++ 16 files changed, 588 insertions(+), 98 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedTableExportAdapter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedTableExportMapper.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/CompiledQueryColumns.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ResolvedQueryStepCompiler.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/TableExportSqlCompiler.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedTableExportFilterConverter.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/model/export/ResolvedExportTable.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/model/export/ResolvedTableExportQuery.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedTableExportFilterConverterTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java index b81069d10a..824e621c97 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java @@ -14,8 +14,11 @@ import com.bakdata.conquery.sql.conversion.NodeConversions; import com.bakdata.conquery.sql.conversion.ResolvedQueryAdapter; import com.bakdata.conquery.sql.conversion.ResolvedQueryMapper; +import com.bakdata.conquery.sql.conversion.ResolvedTableExportAdapter; +import com.bakdata.conquery.sql.conversion.ResolvedTableExportMapper; import com.bakdata.conquery.sql.conversion.SqlConverter; import com.bakdata.conquery.sql.compiler.DefaultSqlCompiler; +import com.bakdata.conquery.sql.compiler.TableExportSqlCompiler; import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.bakdata.conquery.sql.execution.SqlExecutionService; @@ -57,8 +60,11 @@ public LocalNamespace createNamespace( config.getIdColumns(), dialectBundle, clock, connection.getConnection().getPrimaryColumn()); ResolvedQueryAdapter resolvedQueryAdapter = new ResolvedQueryAdapter( environment.getValidator(), new DefaultSqlCompiler(dslContext), resolvedQueryMapper); + ResolvedTableExportAdapter tableExportAdapter = new ResolvedTableExportAdapter( + new ResolvedTableExportMapper(resolvedQueryMapper, clock, connection.getConnection().getPrimaryColumn()), + new TableExportSqlCompiler(dslContext)); SqlConverter sqlConverter = new SqlConverter( - nodeConversions, config, resolvedQueryAdapter, dialectBundle); + nodeConversions, config, resolvedQueryAdapter, tableExportAdapter, dialectBundle); ExecutionManager executionManager = new SqlExecutionManager(sqlConverter, sqlExecutionService, metaStorage, datasetRegistry, config); SqlStorageHandler sqlStorageHandler = new SqlStorageHandler(sqlExecutionService); SqlEntityResolver sqlEntityResolver = new SqlEntityResolver(config.getIdColumns(), dslContext, dialectBundle, sqlExecutionService); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java index bea9cfc894..9b5842b73f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java @@ -180,7 +180,7 @@ private ResolvedConnector connector( ); } - private static ResolvedValidityDate validityDate(ValidityDate validityDate) { + static ResolvedValidityDate validityDate(ValidityDate validityDate) { if (validityDate == null) { return new ResolvedValidityDate.None(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedTableExportAdapter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedTableExportAdapter.java new file mode 100644 index 0000000000..b9a035b760 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedTableExportAdapter.java @@ -0,0 +1,24 @@ +package com.bakdata.conquery.sql.conversion; + +import com.bakdata.conquery.apiv1.query.TableExportQuery; +import com.bakdata.conquery.sql.compiler.CompiledQuery; +import com.bakdata.conquery.sql.compiler.TableExportSqlCompiler; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.conversion.model.SqlQuery; + +/** Backend boundary for resolved row-level table exports. */ +public final class ResolvedTableExportAdapter { + + private final ResolvedTableExportMapper mapper; + private final TableExportSqlCompiler compiler; + + public ResolvedTableExportAdapter(ResolvedTableExportMapper mapper, TableExportSqlCompiler compiler) { + this.mapper = mapper; + this.compiler = compiler; + } + + public SqlQuery compile(TableExportQuery query, CompilerDialect dialect) { + CompiledQuery compiled = compiler.compile(mapper.map(query, dialect), dialect); + return SqlQuery.fromCompiled(compiled, query.getResultInfos()); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedTableExportMapper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedTableExportMapper.java new file mode 100644 index 0000000000..370d661eb8 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedTableExportMapper.java @@ -0,0 +1,85 @@ +package com.bakdata.conquery.sql.conversion; + +import java.time.Clock; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.apiv1.query.TableExportQuery; +import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; +import com.bakdata.conquery.sql.model.export.ResolvedExportTable; +import com.bakdata.conquery.sql.model.export.ResolvedTableExportQuery; +import com.bakdata.conquery.sql.model.range.DateRange; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.SqlTable; +import com.bakdata.conquery.util.TablePrimaryColumnUtil; + +/** Resolves backend table-export DTOs into the connector-owned export model. */ +public final class ResolvedTableExportMapper { + + private final ResolvedQueryMapper queryMapper; + private final Clock clock; + private final String defaultPrimaryColumn; + + public ResolvedTableExportMapper(ResolvedQueryMapper queryMapper, Clock clock, String defaultPrimaryColumn) { + this.queryMapper = queryMapper; + this.clock = clock; + this.defaultPrimaryColumn = defaultPrimaryColumn; + } + + public ResolvedTableExportQuery map(TableExportQuery query, CompilerDialect dialect) { + if (!(query.getQuery() instanceof ConceptQuery prerequisite)) { + throw new UnsupportedOperationException("Resolved table exports currently require a concept-query prerequisite"); + } + DateRange restriction = new DateRange(Optional.ofNullable(query.getDateRange().getMin()), + Optional.ofNullable(query.getDateRange().getMax())); + LocalDate endDate = restriction.endInclusive().orElseGet(() -> LocalDate.now(clock)); + List tables = query.getConcepts().stream() + .flatMap(concept -> concept.getTables().stream().map(table -> table( + concept, table, query.getPositions(), endDate))) + .toList(); + return new ResolvedTableExportQuery( + queryMapper.map(prerequisite, Optional.empty(), prerequisite.getResultInfos()), + restriction, + tables, + ResolvedOperationAdapter.resultColumns(query.getResultInfos()) + ); + } + + private ResolvedExportTable table(CQConcept concept, CQTable table, Map positions, + LocalDate endDate) { + com.bakdata.conquery.models.datasets.concepts.Connector connector = table.getConnector().resolve(); + SqlTable sqlTable = EntitySchemaAdapter.from(connector.getResolvedTable()); + org.jooq.Field primary = TablePrimaryColumnUtil.findPrimaryColumn( + connector.getResolvedTable(), defaultPrimaryColumn); + ResolvedColumn primaryId = new ResolvedColumn(sqlTable.logicalId() + ".primary-id", sqlTable, + primary.getName(), com.bakdata.conquery.models.datasets.ColumnType.STRING, false); + int width = TableExportQuery.calculateWidth(positions) - 1; + List> output = new ArrayList<>(java.util.Collections.nCopies(width, Optional.empty())); + for (Column column : connector.getResolvedTable().getColumns()) { + Integer position = positions.get(column.getId()); + if (position != null) { + output.set(position - 1, Optional.of(EntitySchemaAdapter.from(column))); + } + } + return new ResolvedExportTable( + concept.userLabel(Locale.ROOT), + connector.getName(), + sqlTable, + primaryId, + ResolvedQueryMapper.validityDate(table.findValidityDate()), + connector.getResolvedTable().getName(), + output, + table.getFilters().stream().map(filter -> ResolvedOperationAdapter.filter(filter, endDate)).toList() + ); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java index c515fc85a2..dd17d1487a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java @@ -3,6 +3,7 @@ import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; +import com.bakdata.conquery.apiv1.query.TableExportQuery; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; @@ -13,17 +14,20 @@ public class SqlConverter { private final NodeConversions nodeConversions; private final ConqueryConfig config; private final ResolvedQueryAdapter resolvedQueryAdapter; + private final ResolvedTableExportAdapter tableExportAdapter; private final LegacyCompilerDialect dialect; public SqlConverter( NodeConversions nodeConversions, ConqueryConfig config, ResolvedQueryAdapter resolvedQueryAdapter, + ResolvedTableExportAdapter tableExportAdapter, LegacyCompilerDialect dialect ) { this.nodeConversions = nodeConversions; this.config = config; this.resolvedQueryAdapter = resolvedQueryAdapter; + this.tableExportAdapter = tableExportAdapter; this.dialect = dialect; } @@ -34,6 +38,9 @@ public SqlQuery convert(QueryDescription queryDescription) { if (queryDescription instanceof SecondaryIdQuery secondaryIdQuery) { return resolvedQueryAdapter.compile(secondaryIdQuery, dialect); } + if (queryDescription instanceof TableExportQuery tableExportQuery) { + return tableExportAdapter.compile(tableExportQuery, dialect); + } ConversionContext converted = nodeConversions.convert(queryDescription, config); return converted.getFinalQuery(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java index abbf6726ed..af26838380 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java @@ -39,7 +39,6 @@ import com.bakdata.conquery.sql.conversion.query.FormConversionHelper; import com.bakdata.conquery.sql.conversion.query.RelativFormQueryConverter; import com.bakdata.conquery.sql.conversion.query.SecondaryIdQueryConverter; -import com.bakdata.conquery.sql.conversion.query.TableExportQueryConverter; import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.Field; @@ -229,8 +228,7 @@ default List> getDefaultNodeConverters(DSLCon new SecondaryIdQueryConverter(), new AbsoluteFormQueryConverter(formConversionUtil), new EntityDateQueryConverter(formConversionUtil), - new RelativFormQueryConverter(formConversionUtil), - new TableExportQueryConverter(queryStepRenderer) + new RelativFormQueryConverter(formConversionUtil) ); } diff --git a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlConverterTest.java b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlConverterTest.java index f01192cd4f..90afc2ae05 100644 --- a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlConverterTest.java +++ b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlConverterTest.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; +import com.bakdata.conquery.apiv1.query.TableExportQuery; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.query.DateAggregationMode; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; @@ -23,8 +24,9 @@ class SqlConverterTest { private final NodeConversions nodeConversions = mock(NodeConversions.class); private final ConqueryConfig config = mock(ConqueryConfig.class); private final ResolvedQueryAdapter adapter = mock(ResolvedQueryAdapter.class); + private final ResolvedTableExportAdapter tableExportAdapter = mock(ResolvedTableExportAdapter.class); private final LegacyCompilerDialect dialect = mock(LegacyCompilerDialect.class); - private final SqlConverter converter = new SqlConverter(nodeConversions, config, adapter, dialect); + private final SqlConverter converter = new SqlConverter(nodeConversions, config, adapter, tableExportAdapter, dialect); @Test void shouldRouteConceptQueriesThroughTheResolvedCompiler() { @@ -53,6 +55,19 @@ void shouldRouteSecondaryIdQueriesThroughTheResolvedCompiler() { verifyNoInteractions(nodeConversions); } + @Test + void shouldRouteTableExportsThroughTheResolvedCompiler() { + TableExportQuery query = mock(TableExportQuery.class); + SqlQuery sql = mock(SqlQuery.class); + when(tableExportAdapter.compile(same(query), same(dialect))).thenReturn(sql); + + SqlQuery converted = converter.convert(query); + + assertSame(sql, converted); + verify(tableExportAdapter).compile(same(query), same(dialect)); + verifyNoInteractions(nodeConversions); + } + @Test void shouldKeepOtherQueryDescriptionsOnTheLegacyConversionPath() { QueryDescription query = mock(QueryDescription.class); @@ -64,6 +79,6 @@ void shouldKeepOtherQueryDescriptionsOnTheLegacyConversionPath() { SqlQuery converted = converter.convert(query); assertSame(sql, converted); - verifyNoInteractions(adapter); + verifyNoInteractions(adapter, tableExportAdapter); } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/CompiledQueryColumns.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/CompiledQueryColumns.java new file mode 100644 index 0000000000..73a203ccb2 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/CompiledQueryColumns.java @@ -0,0 +1,29 @@ +package com.bakdata.conquery.sql.compiler; + +import java.util.ArrayList; +import java.util.List; + +import com.bakdata.conquery.sql.model.result.ResultColumn; +import com.bakdata.conquery.sql.model.result.ResultType; +import com.bakdata.conquery.sql.model.schema.EntitySchema; +import org.jooq.Field; + +final class CompiledQueryColumns { + + private CompiledQueryColumns() { + } + + static List create(EntitySchema schema, List results, List> fields) { + if (fields.size() != 1 + results.size()) { + throw new IllegalStateException("Compiled projection does not match the declared result columns"); + } + List columns = new ArrayList<>(fields.size()); + columns.add(new CompiledColumn(schema.primaryId().logicalId(), fields.getFirst().getName(), + ResultType.Primitive.STRING, ColumnRole.ENTITY_ID)); + for (int index = 0; index < results.size(); index++) { + ResultColumn result = results.get(index); + columns.add(new CompiledColumn(result.outputId(), fields.get(index + 1).getName(), result.type(), ColumnRole.RESULT)); + } + return columns; + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/DefaultSqlCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/DefaultSqlCompiler.java index 7b5651ff14..d854c07ebe 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/DefaultSqlCompiler.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/DefaultSqlCompiler.java @@ -1,34 +1,13 @@ package com.bakdata.conquery.sql.compiler; -import java.util.ArrayList; -import java.util.List; import java.util.Optional; -import com.bakdata.conquery.models.query.DateAggregationAction; import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; -import com.bakdata.conquery.sql.compiler.ir.AllEntitiesQueryStepCompiler; -import com.bakdata.conquery.sql.compiler.ir.ExternalQueryStepCompiler; -import com.bakdata.conquery.sql.compiler.ir.ExternalQuerySteps; import com.bakdata.conquery.sql.compiler.ir.FinalQueryStepComposer; -import com.bakdata.conquery.sql.compiler.ir.JoinMode; -import com.bakdata.conquery.sql.compiler.ir.LogicalQueryStepCompiler; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.QueryStepComposer; import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; import com.bakdata.conquery.sql.compiler.rendering.QueryStepRenderer; import com.bakdata.conquery.sql.model.ResolvedQuery; -import com.bakdata.conquery.sql.model.node.AllEntitiesNode; -import com.bakdata.conquery.sql.model.node.AndNode; -import com.bakdata.conquery.sql.model.node.ConceptNode; -import com.bakdata.conquery.sql.model.node.DateRestrictionNode; -import com.bakdata.conquery.sql.model.node.ExternalNode; -import com.bakdata.conquery.sql.model.node.NegationNode; -import com.bakdata.conquery.sql.model.node.OrNode; -import com.bakdata.conquery.sql.model.node.QueryNode; -import com.bakdata.conquery.sql.model.range.DateRange; -import com.bakdata.conquery.sql.model.result.ResultColumn; -import com.bakdata.conquery.sql.model.result.ResultType; -import com.bakdata.conquery.sql.model.schema.EntitySchema; import org.jooq.DSLContext; import org.jooq.Field; import org.jooq.Record; @@ -39,6 +18,7 @@ public final class DefaultSqlCompiler implements SqlCompiler { private final QueryStepRenderer renderer; + private final ResolvedQueryStepCompiler stepCompiler = new ResolvedQueryStepCompiler(); public DefaultSqlCompiler(DSLContext dslContext) { this.renderer = new QueryStepRenderer(dslContext); @@ -47,75 +27,12 @@ public DefaultSqlCompiler(DSLContext dslContext) { @Override public CompiledQuery compile(ResolvedQuery query, CompilerDialect dialect) { SqlNameGenerator names = new SqlNameGenerator(dialect.getNameMaxLength()); - CompiledStep compiled = compileNode(query.root(), query.entitySchema(), dialect, names, false, Optional.empty()); + ResolvedQueryStepCompiler.CompiledQuerySteps compiled = stepCompiler.compile(query, dialect, names, Optional.empty()); QueryStep root = compiled.step(); - if (root.isNegate()) { - root = QueryStepComposer.antiJoinWithAllEntities( - root, query.entitySchema(), rootNegationAction(query.root()), dialect); - } QueryStep finalStep = FinalQueryStepComposer.compose(root, compiled.externalValues(), query.includeValidityDate(), dialect); Select rendered = renderer.toSelectQuery(finalStep, dialect); - return new CompiledQuery(rendered.getSQL(ParamType.INLINED), compiledColumns(query, rendered.getSelect(), finalStep)); + return new CompiledQuery(rendered.getSQL(ParamType.INLINED), + CompiledQueryColumns.create(query.entitySchema(), query.resultColumns(), rendered.getSelect())); } - private static DateAggregationAction rootNegationAction(QueryNode node) { - return switch (node) { - case NegationNode negation -> negation.dateAction(); - case DateRestrictionNode restriction -> rootNegationAction(restriction.child()); - default -> DateAggregationAction.BLOCK; - }; - } - - private static List compiledColumns(ResolvedQuery query, List> fields, QueryStep step) { - int idCount = 1; - if (fields.size() != idCount + query.resultColumns().size()) { - throw new IllegalStateException("Compiled projection does not match the declared result columns"); - } - List columns = new ArrayList<>(fields.size()); - for (int index = 0; index < idCount; index++) { - String outputId = index == 0 ? query.entitySchema().primaryId().logicalId() : "entity-id-%d".formatted(index + 1); - columns.add(new CompiledColumn(outputId, fields.get(index).getName(), ResultType.Primitive.STRING, ColumnRole.ENTITY_ID)); - } - for (int index = 0; index < query.resultColumns().size(); index++) { - ResultColumn result = query.resultColumns().get(index); - columns.add(new CompiledColumn(result.outputId(), fields.get(idCount + index).getName(), result.type(), ColumnRole.RESULT)); - } - return columns; - } - - private static CompiledStep compileNode(QueryNode node, EntitySchema schema, CompilerDialect dialect, - SqlNameGenerator names, boolean negate, Optional restriction) { - return switch (node) { - case AllEntitiesNode all -> new CompiledStep( - AllEntitiesQueryStepCompiler.compile(all, schema, dialect).toBuilder().negate(negate).build(), Optional.empty()); - case ExternalNode external -> { - ExternalQuerySteps steps = ExternalQueryStepCompiler.compile(external, negate, restriction, dialect); - yield new CompiledStep(steps.entities(), steps.values()); - } - case DateRestrictionNode restricted -> compileNode( - restricted.child(), schema, dialect, names, negate, Optional.of(restricted.dateRange())); - case NegationNode negation -> compileNode(negation.child(), schema, dialect, names, true, restriction); - case AndNode and -> compileLogical(and.children(), JoinMode.INNER, and.dateAction(), and.createExists(), - schema, dialect, names, negate, restriction); - case OrNode or -> compileLogical(or.children(), JoinMode.FULL_OUTER, or.dateAction(), or.createExists(), - schema, dialect, names, negate, restriction); - case ConceptNode concept -> new CompiledStep( - ResolvedConceptCompiler.compile(concept, schema, dialect, names, negate, restriction), Optional.empty()); - }; - } - - private static CompiledStep compileLogical(List children, JoinMode mode, DateAggregationAction action, - boolean createExists, EntitySchema schema, CompilerDialect dialect, SqlNameGenerator names, - boolean negate, Optional restriction) { - List compiled = children.stream() - .map(child -> compileNode(child, schema, dialect, names, negate, restriction)).toList(); - QueryStep step = LogicalQueryStepCompiler.compile(compiled.stream().map(CompiledStep::step).toList(), - mode, action, createExists, schema, dialect, names); - Optional external = compiled.stream().flatMap(value -> value.externalValues().stream()) - .reduce((first, second) -> second); - return new CompiledStep(step, external); - } - - private record CompiledStep(QueryStep step, Optional externalValues) { - } } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ResolvedConceptCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ResolvedConceptCompiler.java index 2de6dd32d5..d03c1379df 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ResolvedConceptCompiler.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ResolvedConceptCompiler.java @@ -55,10 +55,11 @@ final class ResolvedConceptCompiler { static QueryStep compile(ConceptNode concept, EntitySchema entitySchema, CompilerDialect dialect, - SqlNameGenerator names, boolean negate, Optional restriction) { + SqlNameGenerator names, boolean negate, Optional restriction, + Optional stratificationTable) { ResolvedSelectConverter selectConverter = new ResolvedSelectConverter(); List connectors = concept.connectors().stream() - .map(connector -> compileConnector(concept, connector, dialect, names, restriction, selectConverter)) + .map(connector -> compileConnector(concept, connector, dialect, names, restriction, selectConverter, stratificationTable)) .flatMap(Optional::stream) .toList(); QueryStep joined = LogicalQueryStepCompiler.compile( @@ -80,7 +81,8 @@ static QueryStep compile(ConceptNode concept, EntitySchema entitySchema, Compile private static Optional compileConnector(ConceptNode concept, ResolvedConnector connector, CompilerDialect dialect, SqlNameGenerator names, - Optional restriction, ResolvedSelectConverter selectConverter) { + Optional restriction, ResolvedSelectConverter selectConverter, + Optional stratificationTable) { String connectorName = names.conceptConnectorName(concept, connector); ConnectorCtePlan plan = ConceptCtePlanner.planConnector(connector.table(), connectorName, concept.dateAction() != DateAggregationAction.BLOCK, @@ -127,7 +129,7 @@ private static Optional compileConnector(ConceptNode conce .map(mapping -> mapping.selectedConcepts(plan.sourceTable(), dialect)) .orElse(plan.sourceTable()); return ConnectorCteCompiler.compileConnector(ConnectorCtePipelineAssembler.assemble( - plan, sourceTable, ids, validityDate, selects, filters, Optional.empty())) + plan, sourceTable, ids, validityDate, selects, filters, stratificationTable)) .map(step -> new ConnectorCompilation(connector, plan, step)); } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ResolvedQueryStepCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ResolvedQueryStepCompiler.java new file mode 100644 index 0000000000..dc80c86f48 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/ResolvedQueryStepCompiler.java @@ -0,0 +1,105 @@ +package com.bakdata.conquery.sql.compiler; + +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.models.query.DateAggregationAction; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.AllEntitiesQueryStepCompiler; +import com.bakdata.conquery.sql.compiler.ir.ExternalQueryStepCompiler; +import com.bakdata.conquery.sql.compiler.ir.ExternalQuerySteps; +import com.bakdata.conquery.sql.compiler.ir.JoinMode; +import com.bakdata.conquery.sql.compiler.ir.LogicalQueryStepCompiler; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStepComposer; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import com.bakdata.conquery.sql.model.ResolvedQuery; +import com.bakdata.conquery.sql.model.node.AllEntitiesNode; +import com.bakdata.conquery.sql.model.node.AndNode; +import com.bakdata.conquery.sql.model.node.ConceptNode; +import com.bakdata.conquery.sql.model.node.DateRestrictionNode; +import com.bakdata.conquery.sql.model.node.ExternalNode; +import com.bakdata.conquery.sql.model.node.NegationNode; +import com.bakdata.conquery.sql.model.node.OrNode; +import com.bakdata.conquery.sql.model.node.QueryNode; +import com.bakdata.conquery.sql.model.range.DateRange; + +/** Compiles a resolved query graph to reusable query-step IR without rendering a final projection. */ +public final class ResolvedQueryStepCompiler { + + public CompiledQuerySteps compile( + ResolvedQuery query, + CompilerDialect dialect, + SqlNameGenerator names, + Optional stratificationTable + ) { + CompiledQuerySteps compiled = compileNode(query.root(), query, dialect, names, false, Optional.empty(), stratificationTable); + if (!compiled.step().isNegate()) { + return compiled; + } + QueryStep root = QueryStepComposer.antiJoinWithAllEntities( + compiled.step(), query.entitySchema(), rootNegationAction(query.root()), dialect); + return new CompiledQuerySteps(root, compiled.externalValues()); + } + + private static DateAggregationAction rootNegationAction(QueryNode node) { + return switch (node) { + case NegationNode negation -> negation.dateAction(); + case DateRestrictionNode restriction -> rootNegationAction(restriction.child()); + default -> DateAggregationAction.BLOCK; + }; + } + + private static CompiledQuerySteps compileNode( + QueryNode node, + ResolvedQuery query, + CompilerDialect dialect, + SqlNameGenerator names, + boolean negate, + Optional restriction, + Optional stratificationTable + ) { + return switch (node) { + case AllEntitiesNode all -> new CompiledQuerySteps( + AllEntitiesQueryStepCompiler.compile(all, query.entitySchema(), dialect).toBuilder().negate(negate).build(), Optional.empty()); + case ExternalNode external -> { + ExternalQuerySteps steps = ExternalQueryStepCompiler.compile(external, negate, restriction, dialect); + yield new CompiledQuerySteps(steps.entities(), steps.values()); + } + case DateRestrictionNode restricted -> compileNode( + restricted.child(), query, dialect, names, negate, Optional.of(restricted.dateRange()), stratificationTable); + case NegationNode negation -> compileNode(negation.child(), query, dialect, names, true, restriction, stratificationTable); + case AndNode and -> compileLogical(and.children(), JoinMode.INNER, and.dateAction(), and.createExists(), + query, dialect, names, negate, restriction, stratificationTable); + case OrNode or -> compileLogical(or.children(), JoinMode.FULL_OUTER, or.dateAction(), or.createExists(), + query, dialect, names, negate, restriction, stratificationTable); + case ConceptNode concept -> new CompiledQuerySteps( + ResolvedConceptCompiler.compile(concept, query.entitySchema(), dialect, names, negate, restriction, stratificationTable), + Optional.empty()); + }; + } + + private static CompiledQuerySteps compileLogical( + List children, + JoinMode mode, + DateAggregationAction action, + boolean createExists, + ResolvedQuery query, + CompilerDialect dialect, + SqlNameGenerator names, + boolean negate, + Optional restriction, + Optional stratificationTable + ) { + List compiled = children.stream() + .map(child -> compileNode(child, query, dialect, names, negate, restriction, stratificationTable)).toList(); + QueryStep step = LogicalQueryStepCompiler.compile(compiled.stream().map(CompiledQuerySteps::step).toList(), + mode, action, createExists, query.entitySchema(), dialect, names); + Optional external = compiled.stream().flatMap(value -> value.externalValues().stream()) + .reduce((first, second) -> second); + return new CompiledQuerySteps(step, external); + } + + public record CompiledQuerySteps(QueryStep step, Optional externalValues) { + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/TableExportSqlCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/TableExportSqlCompiler.java new file mode 100644 index 0000000000..10149b832b --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/TableExportSqlCompiler.java @@ -0,0 +1,109 @@ +package com.bakdata.conquery.sql.compiler; + +import static org.jooq.impl.DSL.inline; + +import java.sql.Date; +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.sql.compiler.conversion.operation.ResolvedTableExportFilterConverter; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.ProjectionMode; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.SchemaSql; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SharedAliases; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.condition.DateRestrictionCondition; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import com.bakdata.conquery.sql.compiler.rendering.QueryStepRenderer; +import com.bakdata.conquery.sql.model.export.ResolvedExportTable; +import com.bakdata.conquery.sql.model.export.ResolvedTableExportQuery; +import com.bakdata.conquery.sql.model.schema.ResolvedValidityDate; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Select; +import org.jooq.Table; +import org.jooq.conf.ParamType; + +/** Compiler for row-level table exports. */ +public final class TableExportSqlCompiler { + + private final QueryStepRenderer renderer; + private final ResolvedQueryStepCompiler stepCompiler = new ResolvedQueryStepCompiler(); + private final ResolvedTableExportFilterConverter filterConverter = new ResolvedTableExportFilterConverter(); + + public TableExportSqlCompiler(DSLContext dslContext) { + this.renderer = new QueryStepRenderer(dslContext); + } + + public CompiledQuery compile(ResolvedTableExportQuery query, CompilerDialect dialect) { + SqlNameGenerator names = new SqlNameGenerator(dialect.getNameMaxLength()); + QueryStep prerequisite = extractIds(stepCompiler.compile( + query.prerequisite(), dialect, names, Optional.empty()).step()); + List tables = query.tables().stream() + .map(table -> compileTable(table, query, prerequisite, dialect, names)).toList(); + QueryStep union = QueryStep.createUnionAllStep(tables, null, List.of(prerequisite), false); + Select rendered = renderer.toSelectQuery(union, dialect); + return new CompiledQuery(rendered.getSQL(ParamType.INLINED), CompiledQueryColumns.create( + query.prerequisite().entitySchema(), query.resultColumns(), rendered.getSelect())); + } + + private static QueryStep extractIds(QueryStep prerequisite) { + Selects selects = Selects.builder() + .ids(new SqlIdColumns(prerequisite.getQualifiedSelects().getIds().getPrimaryColumn())) + .build(); + return QueryStep.builder() + .cteName("extract_ids") + .selects(selects) + .fromTable(QueryStep.toTableLike(prerequisite.getCteName())) + .groupBy(selects.getIds().toFields()) + .predecessors(List.of(prerequisite)) + .build(); + } + + private QueryStep compileTable(ResolvedExportTable value, ResolvedTableExportQuery query, + QueryStep prerequisite, CompilerDialect dialect, SqlNameGenerator names) { + SqlIdColumns ids = new SqlIdColumns(SchemaSql.field(value.primaryId(), String.class)); + ColumnDateRange validity = validityDate(value.validityDate(), dialect); + List> output = new java.util.ArrayList<>(); + output.add(new FieldWrapper<>(inline(value.source()).as(SharedAliases.SOURCE.getAlias()))); + for (int index = 1; index < value.outputColumns().size(); index++) { + int outputIndex = index; + Optional column = value.outputColumns().get(index); + Field field = column.>map(resolved -> SchemaSql.field(resolved, Object.class) + .as(resolved.physicalName() + "-" + outputIndex)) + .orElseGet(() -> inline(null, Object.class).as("null-" + outputIndex)); + output.add(new FieldWrapper<>(field)); + } + Selects selects = Selects.builder().ids(ids).validityDate(Optional.of(validity)).sqlSelects(output).build(); + Table source = SchemaSql.table(value.table()); + Table prerequisiteTable = org.jooq.impl.DSL.table(org.jooq.impl.DSL.name(prerequisite.getCteName())); + List joins = new java.util.ArrayList<>(ids.join(prerequisite.getQualifiedSelects().getIds())); + joins.add(new DateRestrictionCondition(dialect.dateRangeLiteral(query.dateRestriction()), validity).condition()); + Table joined = source.innerJoin(prerequisiteTable).on(joins.toArray(Condition[]::new)); + return QueryStep.builder() + .cteName(names.legacyConceptConnectorName(value.conceptName(), value.connectorName())) + .selects(selects) + .fromTable(joined) + .conditions(value.filters().stream().map(filter -> filterConverter.convert(filter, dialect)).toList()) + .projectionMode(ProjectionMode.INDIVIDUAL) + .build(); + } + + private static ColumnDateRange validityDate(ResolvedValidityDate value, CompilerDialect dialect) { + return switch (value) { + case ResolvedValidityDate.None ignored -> dialect.emptyDateRange(); + case ResolvedValidityDate.Point point -> { + Field field = SchemaSql.field(point.column(), Date.class); + yield dialect.dateRange(field, field); + } + case ResolvedValidityDate.Range range -> dialect.dateRange( + SchemaSql.field(range.start(), Date.class), SchemaSql.field(range.end(), Date.class)); + }; + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedTableExportFilterConverter.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedTableExportFilterConverter.java new file mode 100644 index 0000000000..52dc5e803e --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedTableExportFilterConverter.java @@ -0,0 +1,94 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import java.math.BigDecimal; +import java.sql.Date; +import java.util.List; + +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.SchemaSql; +import com.bakdata.conquery.sql.compiler.ir.condition.FlagCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.InclusiveRangeCondition; +import com.bakdata.conquery.sql.compiler.ir.condition.StringValuesCondition; +import com.bakdata.conquery.sql.model.operation.BuiltInAggregations; +import com.bakdata.conquery.sql.model.operation.BuiltInFilters; +import com.bakdata.conquery.sql.model.operation.ResolvedAggregation; +import com.bakdata.conquery.sql.model.operation.ResolvedFilter; +import com.bakdata.conquery.sql.model.range.SubstringRange; +import com.bakdata.conquery.sql.model.schema.DateColumns; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.impl.DSL; + +/** Converts resolved filters to predicates evaluated on individual table-export rows. */ +public final class ResolvedTableExportFilterConverter { + + public Condition convert(ResolvedFilter filter, CompilerDialect dialect) { + return switch (filter) { + case BuiltInFilters.StringValues values -> stringValues(values); + case BuiltInFilters.NumericColumnRange range -> inclusive( + SchemaSql.field(range.column(), BigDecimal.class), range.range()); + case BuiltInFilters.DateDistanceRange distance -> inclusive( + dialect.dateDistance(distance.unit(), SchemaSql.field(distance.column(), Date.class), distance.endDate()) + .coerce(BigDecimal.class), distance.range()); + case BuiltInFilters.Flags flags -> flags(flags); + case BuiltInFilters.AggregationRange aggregation -> inclusive( + rowValue(aggregation.aggregation(), dialect).coerce(BigDecimal.class), aggregation.range()); + default -> throw new UnsupportedOperationException( + "Table-export filter is not implemented for " + filter.getClass()); + }; + } + + private static Condition stringValues(BuiltInFilters.StringValues filter) { + Field field = SchemaSql.field(filter.column(), String.class); + if (filter.substring().isPresent()) { + field = substring(field, filter.substring().orElseThrow()); + } + return new StringValuesCondition(field, filter.values().toArray(String[]::new)).condition(); + } + + private static Condition flags(BuiltInFilters.Flags filter) { + List> fields = filter.selectedFlags().stream() + .map(filter.availableFlags().columns()::get) + .map(column -> SchemaSql.field(column, Boolean.class)) + .toList(); + return new FlagCondition(fields).condition(); + } + + private static Field rowValue(ResolvedAggregation aggregation, CompilerDialect dialect) { + return switch (aggregation) { + case BuiltInAggregations.Count ignored -> DSL.inline(1); + case BuiltInAggregations.CountQuarters ignored -> DSL.inline(1); + case BuiltInAggregations.Sum sum -> sum.subtractColumn() + .>map(subtract -> SchemaSql.field(sum.column(), BigDecimal.class) + .minus(SchemaSql.field(subtract, BigDecimal.class))) + .orElseGet(() -> SchemaSql.field(sum.column(), BigDecimal.class)); + case BuiltInAggregations.DurationSum duration -> duration(duration.dates(), dialect); + case BuiltInAggregations.Flags ignored -> throw new IllegalArgumentException( + "Flag filters use the dedicated resolved flag operation"); + default -> throw new UnsupportedOperationException( + "Table-export aggregation is not implemented for " + aggregation.getClass()); + }; + } + + private static Field duration(DateColumns dates, CompilerDialect dialect) { + return switch (dates) { + case DateColumns.Single single -> { + Field date = SchemaSql.field(single.column(), Date.class); + yield dialect.dateDistance(java.time.temporal.ChronoUnit.DAYS, date, date); + } + case DateColumns.Pair pair -> dialect.dateDistance(java.time.temporal.ChronoUnit.DAYS, + SchemaSql.field(pair.start(), Date.class), SchemaSql.field(pair.end(), Date.class)); + }; + } + + private static Condition inclusive(Field field, com.bakdata.conquery.sql.model.range.NumberRange range) { + return new InclusiveRangeCondition<>(field, range).condition(); + } + + private static Field substring(Field field, SubstringRange range) { + int start = range.startInclusive() + 1; + return range.endExclusive() + .map(end -> DSL.substring(field, start, end - range.startInclusive())) + .orElseGet(() -> DSL.substring(field, start)); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/export/ResolvedExportTable.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/export/ResolvedExportTable.java new file mode 100644 index 0000000000..f9768c175e --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/export/ResolvedExportTable.java @@ -0,0 +1,26 @@ +package com.bakdata.conquery.sql.model.export; + +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.sql.model.operation.ResolvedFilter; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.ResolvedValidityDate; +import com.bakdata.conquery.sql.model.schema.SqlTable; + +/** One physical connector table participating in a row-level export. */ +public record ResolvedExportTable( + String conceptName, + String connectorName, + SqlTable table, + ResolvedColumn primaryId, + ResolvedValidityDate validityDate, + String source, + List> outputColumns, + List filters +) { + public ResolvedExportTable { + outputColumns = List.copyOf(outputColumns); + filters = List.copyOf(filters); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/export/ResolvedTableExportQuery.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/export/ResolvedTableExportQuery.java new file mode 100644 index 0000000000..1ce8761df0 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/export/ResolvedTableExportQuery.java @@ -0,0 +1,20 @@ +package com.bakdata.conquery.sql.model.export; + +import java.util.List; + +import com.bakdata.conquery.sql.model.ResolvedQuery; +import com.bakdata.conquery.sql.model.range.DateRange; +import com.bakdata.conquery.sql.model.result.ResultColumn; + +/** Fully resolved input for exporting individual physical rows. */ +public record ResolvedTableExportQuery( + ResolvedQuery prerequisite, + DateRange dateRestriction, + List tables, + List resultColumns +) { + public ResolvedTableExportQuery { + tables = List.copyOf(tables); + resultColumns = List.copyOf(resultColumns); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedTableExportFilterConverterTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedTableExportFilterConverterTest.java new file mode 100644 index 0000000000..19e96b380e --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/conversion/operation/ResolvedTableExportFilterConverterTest.java @@ -0,0 +1,53 @@ +package com.bakdata.conquery.sql.compiler.conversion.operation; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.math.BigDecimal; +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.sql.compiler.dialect.hana.HanaCompilerDialect; +import com.bakdata.conquery.sql.model.operation.BuiltInAggregations; +import com.bakdata.conquery.sql.model.operation.BuiltInFilters; +import com.bakdata.conquery.sql.model.range.NumberRange; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.SqlTable; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Test; + +class ResolvedTableExportFilterConverterTest { + + private static final SqlTable TABLE = SqlTable.of("table", "events"); + private static final ResolvedColumn VALUE = new ResolvedColumn( + "value", TABLE, "value", ColumnType.DECIMAL, true); + private static final ResolvedColumn SUBTRACT = new ResolvedColumn( + "subtract", TABLE, "subtract", ColumnType.DECIMAL, true); + private static final NumberRange RANGE = new NumberRange( + Optional.of(BigDecimal.ONE), Optional.of(BigDecimal.TEN)); + private final ResolvedTableExportFilterConverter converter = new ResolvedTableExportFilterConverter(); + + @Test + void shouldApplySubtractionToIndividualRows() { + var filter = new BuiltInFilters.AggregationRange("sum", + new BuiltInAggregations.Sum(VALUE, Optional.of(SUBTRACT), List.of()), RANGE); + + String sql = DSL.using(SQLDialect.DEFAULT).renderInlined( + converter.convert(filter, new HanaCompilerDialect())); + + assertEquals("((\"events\".\"value\" - \"events\".\"subtract\") >= 1 and " + + "(\"events\".\"value\" - \"events\".\"subtract\") <= 10)", sql); + } + + @Test + void shouldTreatCountAsOneForEachExportedRow() { + var filter = new BuiltInFilters.AggregationRange("count", + new BuiltInAggregations.Count(VALUE, List.of()), RANGE); + + String sql = DSL.using(SQLDialect.DEFAULT).renderInlined( + converter.convert(filter, new HanaCompilerDialect())); + + assertEquals("(1 >= 1 and 1 <= 10)", sql); + } +} From d2c79c4f9d7024f405e27ea31c88617a4924ff95 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:37:28 +0200 Subject: [PATCH 73/81] refactor(sql): extract form compilation Co-authored-by: Codex GPT-6 --- .../mode/local/LocalNamespaceHandler.java | 7 +- .../sql/conversion/NodeConversions.java | 1 - .../sql/conversion/ResolvedFormAdapter.java | 37 ++ .../sql/conversion/ResolvedFormMapper.java | 94 ++++ .../sql/conversion/ResolvedQueryMapper.java | 12 + .../conquery/sql/conversion/SqlConverter.java | 15 + .../cqelement/ConversionContext.java | 3 - .../dialect/LegacyCompilerDialect.java | 32 +- .../clickhouse/ClickhouseDialectBundle.java | 6 - .../dialect/hana/HanaDialectBundle.java | 6 - .../CombinationNotSupportedException.java | 25 - .../sql/conversion/forms/FormType.java | 7 - .../conquery/sql/conversion/forms/README.md | 428 ------------------ .../query/AbsoluteFormQueryConverter.java | 78 ---- .../query/EntityDateQueryConverter.java | 97 ---- .../query/FormConversionHelper.java | 169 ------- .../query/RelativFormQueryConverter.java | 37 -- .../query/TableExportQueryConverter.java | 225 --------- .../sql/conversion/SqlConverterTest.java | 31 +- .../SqlExtractionAggregationTest.java | 2 +- .../conversion/SqlExtractionDialectTest.java | 15 + .../sql/conversion/SqlExtractionFormTest.java | 120 +++++ .../sql/compiler/FormSqlCompiler.java | 200 ++++++++ .../sql/compiler/dialect/CompilerDialect.java | 19 + .../clickhouse/ClickhouseCompilerDialect.java | 12 + .../ClickhouseStratificationFunctions.java | 25 +- .../dialect/hana/HanaCompilerDialect.java | 5 + .../hana/HanaStratificationFunctions.java | 24 +- .../forms/AbsoluteStratification.java | 32 +- .../CombinationNotSupportedException.java | 25 + .../sql/compiler}/forms/FormCteStep.java | 8 +- .../forms/RelativeStratification.java | 115 ++--- .../forms/StratificationFunctions.java | 53 ++- .../forms/StratificationTableFactory.java | 36 +- .../sql/model/form/FormAlignment.java | 3 + .../sql/model/form/FormCalendarUnit.java | 3 + .../sql/model/form/FormIndexPlacement.java | 3 + .../sql/model/form/FormIndexSelector.java | 3 + .../sql/model/form/FormResolution.java | 3 + .../sql/model/form/RelativeFormSettings.java | 20 + .../model/form/ResolutionAndAlignment.java | 6 + .../sql/model/form/ResolvedFormMode.java | 16 + .../sql/model/form/ResolvedFormQuery.java | 18 + .../sql/compiler/FormSqlCompilerTest.java | 155 +++++++ 44 files changed, 988 insertions(+), 1243 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedFormAdapter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedFormMapper.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/CombinationNotSupportedException.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormType.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/README.md delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java create mode 100644 backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionFormTest.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/FormSqlCompiler.java rename {backend/src/main/java/com/bakdata/conquery/sql/conversion => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler}/dialect/clickhouse/ClickhouseStratificationFunctions.java (84%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler}/dialect/hana/HanaStratificationFunctions.java (87%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler}/forms/AbsoluteStratification.java (85%) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/forms/CombinationNotSupportedException.java rename {backend/src/main/java/com/bakdata/conquery/sql/conversion => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler}/forms/FormCteStep.java (83%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler}/forms/RelativeStratification.java (75%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler}/forms/StratificationFunctions.java (83%) rename {backend/src/main/java/com/bakdata/conquery/sql/conversion => sql-connector/src/main/java/com/bakdata/conquery/sql/compiler}/forms/StratificationTableFactory.java (56%) create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/FormAlignment.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/FormCalendarUnit.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/FormIndexPlacement.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/FormIndexSelector.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/FormResolution.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/RelativeFormSettings.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/ResolutionAndAlignment.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/ResolvedFormMode.java create mode 100644 sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/ResolvedFormQuery.java create mode 100644 sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/FormSqlCompilerTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java index 824e621c97..9110c0c30e 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java @@ -14,10 +14,13 @@ import com.bakdata.conquery.sql.conversion.NodeConversions; import com.bakdata.conquery.sql.conversion.ResolvedQueryAdapter; import com.bakdata.conquery.sql.conversion.ResolvedQueryMapper; +import com.bakdata.conquery.sql.conversion.ResolvedFormAdapter; +import com.bakdata.conquery.sql.conversion.ResolvedFormMapper; import com.bakdata.conquery.sql.conversion.ResolvedTableExportAdapter; import com.bakdata.conquery.sql.conversion.ResolvedTableExportMapper; import com.bakdata.conquery.sql.conversion.SqlConverter; import com.bakdata.conquery.sql.compiler.DefaultSqlCompiler; +import com.bakdata.conquery.sql.compiler.FormSqlCompiler; import com.bakdata.conquery.sql.compiler.TableExportSqlCompiler; import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; import com.bakdata.conquery.sql.execution.ResultSetProcessor; @@ -63,8 +66,10 @@ public LocalNamespace createNamespace( ResolvedTableExportAdapter tableExportAdapter = new ResolvedTableExportAdapter( new ResolvedTableExportMapper(resolvedQueryMapper, clock, connection.getConnection().getPrimaryColumn()), new TableExportSqlCompiler(dslContext)); + ResolvedFormAdapter formAdapter = new ResolvedFormAdapter( + new ResolvedFormMapper(resolvedQueryMapper), new FormSqlCompiler(dslContext)); SqlConverter sqlConverter = new SqlConverter( - nodeConversions, config, resolvedQueryAdapter, tableExportAdapter, dialectBundle); + nodeConversions, config, resolvedQueryAdapter, tableExportAdapter, formAdapter, dialectBundle); ExecutionManager executionManager = new SqlExecutionManager(sqlConverter, sqlExecutionService, metaStorage, datasetRegistry, config); SqlStorageHandler sqlStorageHandler = new SqlStorageHandler(sqlExecutionService); SqlEntityResolver sqlEntityResolver = new SqlEntityResolver(config.getIdColumns(), dslContext, dialectBundle, sqlExecutionService); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java index ab44c944a4..bac5216ec5 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java @@ -55,7 +55,6 @@ public ConversionContext convert(QueryDescription queryDescription, ConqueryConf .nodeConversions(this) .clock(clock) .defaultPrimaryColumn(this.defaultPrimaryColumn) - .stratificationFunctions(dialect.getStratificationFunctions()) .compilerDialect(dialect) .build(); return convert(queryDescription, initialCtx); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedFormAdapter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedFormAdapter.java new file mode 100644 index 0000000000..b7fc65393b --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedFormAdapter.java @@ -0,0 +1,37 @@ +package com.bakdata.conquery.sql.conversion; + +import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery; +import com.bakdata.conquery.models.forms.managed.EntityDateQuery; +import com.bakdata.conquery.models.forms.managed.RelativeFormQuery; +import com.bakdata.conquery.sql.compiler.CompiledQuery; +import com.bakdata.conquery.sql.compiler.FormSqlCompiler; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.conversion.model.SqlQuery; + +/** Backend boundary for resolved form compilation. */ +public final class ResolvedFormAdapter { + + private final ResolvedFormMapper mapper; + private final FormSqlCompiler compiler; + + public ResolvedFormAdapter(ResolvedFormMapper mapper, FormSqlCompiler compiler) { + this.mapper = mapper; + this.compiler = compiler; + } + + public SqlQuery compile(AbsoluteFormQuery query, CompilerDialect dialect) { + return adapt(compiler.compile(mapper.map(query), dialect), query); + } + + public SqlQuery compile(RelativeFormQuery query, CompilerDialect dialect) { + return adapt(compiler.compile(mapper.map(query), dialect), query); + } + + public SqlQuery compile(EntityDateQuery query, CompilerDialect dialect) { + return adapt(compiler.compile(mapper.map(query), dialect), query); + } + + private static SqlQuery adapt(CompiledQuery compiled, com.bakdata.conquery.apiv1.query.Query query) { + return SqlQuery.fromCompiled(compiled, query.getResultInfos()); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedFormMapper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedFormMapper.java new file mode 100644 index 0000000000..5f8a9e0690 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedFormMapper.java @@ -0,0 +1,94 @@ +package com.bakdata.conquery.sql.conversion; + +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; +import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery; +import com.bakdata.conquery.models.forms.managed.EntityDateQuery; +import com.bakdata.conquery.models.forms.managed.RelativeFormQuery; +import com.bakdata.conquery.sql.model.ResolvedQuery; +import com.bakdata.conquery.sql.model.form.FormAlignment; +import com.bakdata.conquery.sql.model.form.FormCalendarUnit; +import com.bakdata.conquery.sql.model.form.FormIndexPlacement; +import com.bakdata.conquery.sql.model.form.FormIndexSelector; +import com.bakdata.conquery.sql.model.form.FormResolution; +import com.bakdata.conquery.sql.model.form.RelativeFormSettings; +import com.bakdata.conquery.sql.model.form.ResolutionAndAlignment; +import com.bakdata.conquery.sql.model.form.ResolvedFormMode; +import com.bakdata.conquery.sql.model.form.ResolvedFormQuery; +import com.bakdata.conquery.sql.model.range.DateRange; + +/** Resolves initialized backend form DTOs into the connector-owned form model. */ +public final class ResolvedFormMapper { + + private final ResolvedQueryMapper queryMapper; + + public ResolvedFormMapper(ResolvedQueryMapper queryMapper) { + this.queryMapper = queryMapper; + } + + public ResolvedFormQuery map(AbsoluteFormQuery form) { + return create( + form.getQuery(), + form.getFeatures().getChildQueries(), + new ResolvedFormMode.Absolute( + new DateRange(Optional.ofNullable(form.getDateRange().getMin()), + Optional.ofNullable(form.getDateRange().getMax())), + resolutions(form.getResolutionsAndAlignmentMap())), + form.getResultInfos() + ); + } + + public ResolvedFormQuery map(RelativeFormQuery form) { + RelativeFormSettings settings = new RelativeFormSettings( + FormIndexSelector.valueOf(form.getIndexSelector().name()), + FormIndexPlacement.valueOf(form.getIndexPlacement().name()), + form.getTimeCountBefore(), + form.getTimeCountAfter(), + FormCalendarUnit.valueOf(form.getTimeUnit().name()), + resolutions(form.getResolutionsAndAlignmentMap()) + ); + return create(form.getQuery(), form.getFeatures().getChildQueries(), + new ResolvedFormMode.Relative(settings), form.getResultInfos()); + } + + public ResolvedFormQuery map(EntityDateQuery form) { + Optional bounds = Optional.ofNullable(form.getDateRange()).map(ResolvedFormMapper::dateRange); + return create(form.getQuery(), form.getFeatures().getChildQueries(), + new ResolvedFormMode.EntityDate(bounds, resolutions(form.getResolutionsAndAlignments())), + form.getResultInfos()); + } + + private ResolvedFormQuery create( + com.bakdata.conquery.apiv1.query.Query prerequisite, + List featureQueries, + ResolvedFormMode mode, + List resultInfos + ) { + ResolvedQuery resolvedPrerequisite = queryMapper.map(prerequisite, List.of()); + List features = featureQueries.stream() + .map(feature -> queryMapper.map(feature, Optional.empty(), feature.getResultInfos())) + .toList(); + return new ResolvedFormQuery( + resolvedPrerequisite, + features, + mode, + ResolvedOperationAdapter.resultColumns(resultInfos) + ); + } + + private static List resolutions( + List values + ) { + return values.stream().map(value -> new ResolutionAndAlignment( + FormResolution.valueOf(value.getResolution().name()), + FormAlignment.valueOf(value.getAlignment().name()))).toList(); + } + + private static DateRange dateRange(CDateRange value) { + return new DateRange(Optional.ofNullable(value.getMin()), Optional.ofNullable(value.getMax())); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java index 9b5842b73f..59ccc91609 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java @@ -12,6 +12,8 @@ import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.apiv1.query.CQYes; import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.apiv1.query.Query; +import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; @@ -93,6 +95,16 @@ ResolvedQuery map( ); } + ResolvedQuery map(Query query, List resultInfos) { + return switch (query) { + case ConceptQuery concept -> map(concept, Optional.empty(), resultInfos); + case SecondaryIdQuery secondary -> map( + secondary.getQuery(), Optional.of(secondary.getSecondaryId().resolve()), resultInfos); + default -> throw new UnsupportedOperationException( + "Resolved SQL query mapping is not implemented for " + query.getClass()); + }; + } + private QueryNode mapNode(CQElement node, NodeContext context) { return switch (node) { case CQYes ignored -> new AllEntitiesNode(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java index dd17d1487a..638a7f0c0f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java @@ -5,6 +5,9 @@ import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; import com.bakdata.conquery.apiv1.query.TableExportQuery; import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery; +import com.bakdata.conquery.models.forms.managed.EntityDateQuery; +import com.bakdata.conquery.models.forms.managed.RelativeFormQuery; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; import com.bakdata.conquery.sql.conversion.model.SqlQuery; @@ -15,6 +18,7 @@ public class SqlConverter { private final ConqueryConfig config; private final ResolvedQueryAdapter resolvedQueryAdapter; private final ResolvedTableExportAdapter tableExportAdapter; + private final ResolvedFormAdapter formAdapter; private final LegacyCompilerDialect dialect; public SqlConverter( @@ -22,12 +26,14 @@ public SqlConverter( ConqueryConfig config, ResolvedQueryAdapter resolvedQueryAdapter, ResolvedTableExportAdapter tableExportAdapter, + ResolvedFormAdapter formAdapter, LegacyCompilerDialect dialect ) { this.nodeConversions = nodeConversions; this.config = config; this.resolvedQueryAdapter = resolvedQueryAdapter; this.tableExportAdapter = tableExportAdapter; + this.formAdapter = formAdapter; this.dialect = dialect; } @@ -41,6 +47,15 @@ public SqlQuery convert(QueryDescription queryDescription) { if (queryDescription instanceof TableExportQuery tableExportQuery) { return tableExportAdapter.compile(tableExportQuery, dialect); } + if (queryDescription instanceof AbsoluteFormQuery absoluteForm) { + return formAdapter.compile(absoluteForm, dialect); + } + if (queryDescription instanceof RelativeFormQuery relativeForm) { + return formAdapter.compile(relativeForm, dialect); + } + if (queryDescription instanceof EntityDateQuery entityDateForm) { + return formAdapter.compile(entityDateForm, dialect); + } ConversionContext converted = nodeConversions.convert(queryDescription, config); return converted.getFinalQuery(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java index 0a22bd78a1..de859cb831 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java @@ -13,7 +13,6 @@ import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.NodeConversionDispatcher; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; -import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlQuery; @@ -32,8 +31,6 @@ public class ConversionContext implements Context { IdColumnConfig idColumns; Clock clock; - @NonNull - StratificationFunctions stratificationFunctions; String defaultPrimaryColumn; PrintSettings sqlPrintSettings; NodeConversionDispatcher nodeConversions; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java index af26838380..6b70ae35fc 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java @@ -30,14 +30,9 @@ import com.bakdata.conquery.sql.conversion.cqelement.CQOrConverter; import com.bakdata.conquery.sql.conversion.cqelement.CQYesConverter; import com.bakdata.conquery.sql.conversion.cqelement.concept.CQConceptConverter; -import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; -import com.bakdata.conquery.sql.conversion.query.AbsoluteFormQueryConverter; import com.bakdata.conquery.sql.conversion.query.CQReusedQueryConverter; import com.bakdata.conquery.sql.conversion.query.ConceptQueryConverter; -import com.bakdata.conquery.sql.conversion.query.EntityDateQueryConverter; -import com.bakdata.conquery.sql.conversion.query.FormConversionHelper; -import com.bakdata.conquery.sql.conversion.query.RelativFormQueryConverter; import com.bakdata.conquery.sql.conversion.query.SecondaryIdQueryConverter; import org.jooq.Condition; import org.jooq.DSLContext; @@ -165,7 +160,25 @@ default Field dateDistance(ChronoUnit unit, Field startDate, Loca return getCompilerDialect().dateDistance(unit, startDate, endDate); } - StratificationFunctions getStratificationFunctions(); + @Override + default com.bakdata.conquery.sql.compiler.forms.StratificationFunctions stratificationFunctions() { + return getCompilerDialect().stratificationFunctions(); + } + + @Override + default ColumnDateRange intersection(ColumnDateRange left, ColumnDateRange right) { + return getCompilerDialect().intersection(left, right); + } + + @Override + default Condition isNotEmptyDateRange(ColumnDateRange range) { + return getCompilerDialect().isNotEmptyDateRange(range); + } + + @Override + default boolean supportsSingleColumnRanges() { + return getCompilerDialect().supportsSingleColumnRanges(); + } @Override default Field dateDistance(ChronoUnit unit, Field startDate, Field endDate) { @@ -213,8 +226,6 @@ default Field nextQuarterStart(Field date) { default List> getDefaultNodeConverters(DSLContext dslContext) { QueryStepRenderer queryStepRenderer = new QueryStepRenderer(dslContext); - FormConversionHelper formConversionUtil = new FormConversionHelper(queryStepRenderer); - return List.of( new CQDateRestrictionConverter(), new CQAndConverter(), @@ -225,10 +236,7 @@ default List> getDefaultNodeConverters(DSLCon new CQExternalConverter(), new CQReusedQueryConverter(), new ConceptQueryConverter(queryStepRenderer), - new SecondaryIdQueryConverter(), - new AbsoluteFormQueryConverter(formConversionUtil), - new EntityDateQueryConverter(formConversionUtil), - new RelativFormQueryConverter(formConversionUtil) + new SecondaryIdQueryConverter() ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java index 3b2959e09a..b408a08947 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java @@ -14,7 +14,6 @@ import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.bakdata.conquery.sql.execution.SqlCDateSetParser; @@ -59,11 +58,6 @@ public List> getNodeConverters(DSLContext dsl return getDefaultNodeConverters(dslContext); } - @Override - public StratificationFunctions getStratificationFunctions() { - return new ClickhouseStratificationFunctions(getFunctionProvider()); - } - @Override public boolean isTypeCompatible(Field field, MajorTypeId type) { return true; //TODO CLickhouse integration is terrible here. We always receive just Object. diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaDialectBundle.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaDialectBundle.java index 5f27a82470..9d56891167 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaDialectBundle.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaDialectBundle.java @@ -9,7 +9,6 @@ import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import com.bakdata.conquery.sql.compiler.dialect.hana.HanaCompilerDialect; import com.bakdata.conquery.sql.execution.DefaultResultSetProcessor; @@ -62,11 +61,6 @@ public List> getNodeConverters(DSLContext dsl return getDefaultNodeConverters(dslContext); } - @Override - public StratificationFunctions getStratificationFunctions() { - return new HanaStratificationFunctions((HanaSqlFunctionProvider) getFunctionProvider()); - } - @Override public boolean isTypeCompatible(Field field, MajorTypeId type) { return switch (type) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/CombinationNotSupportedException.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/CombinationNotSupportedException.java deleted file mode 100644 index a29f4999c5..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/CombinationNotSupportedException.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.bakdata.conquery.sql.conversion.forms; - -import com.bakdata.conquery.apiv1.forms.IndexPlacement; -import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; -import com.bakdata.conquery.models.forms.util.CalendarUnit; -import com.bakdata.conquery.models.forms.util.Resolution; - -class CombinationNotSupportedException extends RuntimeException { - - public CombinationNotSupportedException(ExportForm.ResolutionAndAlignment resolutionAndAlignment) { - super("Alignment %s does not fit the resolution %s.".formatted( - resolutionAndAlignment.getAlignment(), - resolutionAndAlignment.getResolution() - )); - } - - public CombinationNotSupportedException(IndexPlacement indexPlacement, CalendarUnit timeUnit) { - super("Combination of index placement %s and time unit %s not supported".formatted(indexPlacement, timeUnit)); - } - - public CombinationNotSupportedException(CalendarUnit timeUnit, Resolution resolution) { - super("Combination of time unit %s and resolution %s not supported".formatted(timeUnit, resolution)); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormType.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormType.java deleted file mode 100644 index d448a6ba33..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormType.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.bakdata.conquery.sql.conversion.forms; - -public enum FormType { - ABSOLUTE, - ENTITY_DATE, - RELATIVE -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/README.md b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/README.md deleted file mode 100644 index ba92a64bba..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/README.md +++ /dev/null @@ -1,428 +0,0 @@ -# Form conversion - how to apply stratification in SQL? - -This document outlines the procedure to apply stratification within SQL in the context of the form conversion process. -The process or to be more precise, specific functions used while creating stratification tables differ from dialect -to dialect, but the overall process is the same. This document is using Postgres dialect for the SQL examples. - -# Table of Contents - -1. [Prerequisite conversion](#prerequisite-conversion) -2. [Absolute stratification](#absolute-stratification-for-absolute-forms-and-entity-date-forms) - 1. [For absolute forms](#absolute-forms) - 2. [For entity date queries](#entity-date) - 3. [Stratification tables](#stratification-tables) -3. [Relative stratification](#relative-stratification) -4. [Full stratification table](#full-stratification-table) -5. [Feature conversion](#feature-conversion) -6. [Left-join converted features](#left-join-converted-features-with-the-full-stratification-table-for-the-final-select) -7. [Full export form](#full-export-form) - -## Prerequisite conversion - -The prerequisite query conversion produces a CTE, which will contain the IDs of those subjects relevant for the form. -Because this could be any kind of Query, the CTE might also contain a validity date and converted Selects. -Take this CTE representing a converted CQExternal as an example: - -**CTE:** `external` - -```sql -select '1' "primary_id", - daterange('2001-12-01', '2016-12-02', '[)') as "validity_date" -from "DUMMY"; -- "DUMMY" is SAP HANAs built-in no-op table -``` - -## Absolute stratification (for absolute forms and entity date forms) - -### Absolute forms - -For an absolute form, we only care for the primary ID, so we extract the primary IDs and discard the validitiy date. For entity date queries, it's kept. The `stratification_bounds` represent the absolute forms date -range. They define the required complete stratification window. We group by primary ID (and validity date, if present) -to keep only distinct entries for each entity and discard any duplicated entries which, for example, might occur due -to a preceding secondary id query. - -**CTE:** `extract_ids` - -```sql -select "primary_id", - "validity_date", -- the validity date is only kept in case we convert an entity date query - daterange('2012-06-01', '2012-09-30', '[]') as "stratification_bounds" -from "external" -group by "primary_id", "validity_date" -``` - -### Entity date - -For an entity date form, we only care for the primary ID and the validity date. We group by primary ID and validity -date to keep only distinct entries for each entity and discard any duplicated entries which, for example, might occur -due to a preceding secondary id query. - -**CTE:** `extract_ids` - -```sql -select "primary_id", - "validity_date" -from "external" -group by "primary_id", "validity_date" -``` - -**CTE:** `overwrite_bounds` - -We create an additional CTE which intersects the entities validity dates with the given forms date range (if there is -one). The intersection defines the required complete stratification window. Besides this, there is no difference in the -following conversion process between absolute forms and entity date queries. - -```sql -select "primary_id", - -- the validity date is a multirange, so we unnest first - daterange('2012-06-01', '2012-09-30', '[]') * unnest("validity_date") as "stratification_bounds" -from "extract_ids" -``` - -### Stratification tables - -Now, we want to create a resolution table for each resolution (`COMPLETE`, `YEAR`, `QUARTER`). - -**CTE:** `complete` - -```sql -select "primary_id", - 'COMPLETE' "resolution", - 1 "index", - "stratification_bounds" -from "extract_ids" -- or `overwrite_bounds` if it is an entity date query -``` - -A complete range shall have a `null` index, because it spans the complete range, but we set it to 1 to ensure we can -join tables on index. - -### Calculating index start dates - -For finer resolutions (`YEAR`, `QUARTER`, `DAY`), the approach will be the following: at first, we will calculate -the required year and quarter start and end dates for all possible resolution and alignment combos. We will use these -later to calculate date distances easily. - -We will create the following fields: - -- `year_start`: This field is calculated by truncating the lower boundary of `stratification_bounds` to the start of the - corresponding year, providing the earliest date of the year in which the stratification begins. -- `quarter_start`: This field is derived by truncating the lower boundary of `stratification_bounds` to the start of its - year and then adjusting for the number of complete quarters passed since the beginning of that year, marking the start - of the quarter in which the stratification starts. -- `year_end`: This field captures the last day of the year for the upper boundary of `stratification_bounds`. It is - calculated by truncating the upper boundary to the start of its year, advancing to the start of the following year, - and then subtracting one day, thus marking the end of the year in which the stratification ends. -- `year_end_quarter_aligned`: This field aligns the end of a calendar year of the upper boundary - of `stratification_bounds` - with the quarter of the starting date. This edge case is required for the `YEAR` resolution, `QUARTER` alignment - combo. -- `quarter_end`: This field indicates the last day of the quarter for the upper boundary of `stratification_bounds`. It - is calculated by truncating the upper boundary to the start of its year, advancing to the start of the next quarter - after the quarter in which the stratification ends, and subtracting one day to find the close of that quarter. - -```sql -select "primary_id", - "stratification_bounds", - lower("stratification_bounds") as "index_start", - date_trunc('year', lower("stratification_bounds")) as "year_start", - date_trunc('year', upper("stratification_bounds")) + interval '1 year' as "year_end", - date_trunc('year', upper("stratification_bounds")) + interval '1 year' - + (extract(quarter from lower("stratification_bounds")) - 1) - * interval '3 months' as "year_end_quarter_aligned", - date_trunc('year', lower("stratification_bounds")) - + (extract(quarter from lower("stratification_bounds")) - 1) - * interval '3 months' as "quarter_start", - date_trunc('year', upper("stratification_bounds")) - + (extract(quarter from upper("stratification_bounds"))) - * interval '3 months' as "quarter_end" -from "extract_ids" -``` - -| primary\_id | stratification\_bounds | index\_start | year\_start | year\_end | year\_end\_quarter\_aligned | quarter\_start | quarter\_end | -|:------------|:--------------------------|:-------------|:------------|:-----------|:----------------------------|:---------------|:-------------| -| 1 | \[2012-01-16,2012-12-18\) | 2012-01-16 | 2012-01-01 | 2013-01-01 | 2013-01-01 | 2012-01-01 | 2013-01-01 | - -### Calculating resolution counts - -For each required resolution and alignment, we will create a `counts` CTE. The calculation slightly differs for each -valid resolution and alignment combination, but it all comes down to calculating date diffs. - -For example, take the `YEAR` resolution and `QUARTER` alignment as an example. We calculate the date diff in years from -the quarter end (of the upper stratification bound) and the quarter start (of the lower stratification bound) that we -calculated in the previous step. - -```sql -select "primary_id", - "stratification_bounds", - "index_start", - "year_start", - "quarter_start", - (extract(year from age("quarter_end", "quarter_start"))) as "quarter_aligned_count" -from "index_start"; -``` - -For this example, the `"year_aligned_count"` is 3. - -### Calculating the actual stratification ranges - -The key idea of the resolution count is to calculate a start and end date for all required resolution windows by adding -a date interval times an index until the index reaches the count. For example, if we consider the `quarter_start` -`2012-04-01` as out starting point, the resolution interval to be 1 year and the resolution count to be 3, we can -calculate the resolution range with the following approach: - -```sql -select "primary_id", - 'YEARS' as "resolution", - row_number() over (partition by "primary_id") as "index", - daterange( - ("quarter_start" + interval '1 year' * ("index" - 1))::date, - ("quarter_start" + interval '1 year' * ("index" - 0))::date, - '[)' - ) * "stratification_bounds" as "stratification_bounds" -from "year_counts", - generate_series(1, 10000) as "index" -where "index" <= "quarter_aligned_count" -``` - -By cross-joining the `year_counts` CTE with the integer series, we can create a set of ranges for each primary ID. -Intervals are multiplied and added depending on the respective index to the starting date, which is the `quarter_start` -in our case. The result looks like this: - -| primary\_id | resolution | index | stratification\_bounds | -|:------------|:-----------|:------|:--------------------------| -| 1 | YEARS | 1 | \[2012-06-16,2013-04-01\) | -| 1 | YEARS | 2 | \[2013-04-01,2014-04-01\) | -| 1 | YEARS | 3 | \[2014-04-01,2014-12-18\) | - -## Relative stratification - -Like for entity date queries, we need to extract the primary ID and the corresponding validity date for each distinct -entity. - -**CTE:** `extract_ids` - -```sql -select "primary_id", - unnest("validity_date") as "validity_date" -- unnesting is only required for dialects with multiranges -from "external" -group by "primary_id", "validity_date" -``` - -Next, we need to find the index selector date: For each validity date of an entity, we calculate either: - -- the `EARLIEST` date of the given range -- the `LATEST` date of the given range -- or a `RANDOM` date within the given range - depending on the relative forms - [index selector](../../../apiv1/query/concept/specific/temporal/TemporalSamplerFactory.java). - -**CTE:** `index_selector` - -```sql -select "primary_id", - min(lower("dates")) as "index_selector" -- example for EARLIEST -from "extract_ids" -group by "primary_id" -``` - -Using the index selector date, we can now define the index start dates from where the feature and/or outcome ranges of -the relative form start. Their exact calculation depends on the -[index placement](../../../apiv1/forms/IndexPlacement.java) of the relative form. - -For the `BEFORE` and `AFTER` placement, the positive start (outcome range) and negative start (feature range) is the -same. Only for the `NEUTRAL` placement, the start dates differ. - -We take the `BEFORE` placement with the time unit `QUARTERS` as an example: We jump to the next quarters start of the -index selector date. From these index start dates, we will start in the next step when calculating the stratification -windows of the feature and outcome range. - -**CTE:** `index_start` - -```sql -select "primary_id", - "index_selector", - (date_trunc('year', "index_selector") + - interval '3 months' * (extract(quarter from ("index_selector" + -1)) + 0))::date as "index_start_positive", - (date_trunc('year', "index_selector") + - interval '3 months' * (extract(quarter from ("index_selector" + -1)) + 0))::date as "index_start_negative" -from "index_selector" -``` - -The last step before calculating the actual stratification windows is to calculate the min and max date of the -stratification for each entity, which is basically the lower bound of the feature range and the upper bound of the -outcome range. Assuming a time count before of 6 quarters and a time count after of 2 quarters, the calculation looks -like the following: - -**CTE:** `total_bounds` - -```sql -select "primary_id", - daterange( - ("index_start_negative" + interval '3 months' * (-6 + 0))::date, - ("index_start_positive" + interval '3 months' * (2 + 0))::date, - '[)' - ) as "stratification_bounds", - "index_selector", - "index_start_positive", - "index_start_negative" -from "index_start" -``` - -We will intersect this range with the calculated ranges in the next step, because calculated ranges always span over -whole intervals (`YEARS`, `QUARTERS`), but must be ultimately bound by the complete min and max dates. In the following -example, we take a look at how the `YEARS` resolution is calculated. We still assume a time count before of 6 quarters -and a time count after of 2 quarters. This means we need to jump 2 years back for the feature range and 1 year forward -for the outcome range. Similar to absolute stratification, we create a set of date ranges by manipulating the start -dates by adding positive or negative time intervals times an index. The index is again taken from a generated integer -series. - -**CTE:** `years` - -```sql --- feature range -select "primary_id", - 'YEARS' as "resolution", - "index", - "index_selector", - daterange( - ("index_start_negative" + interval '1 year' * ("index" + 0))::date, - ("index_start_negative" + interval '1 year' * ("index" + 1))::date, - '[)' - ) * "stratification_bounds" as "stratification_bounds" -from "total_bounds", - generate_series(-2, -1) as "index" -- -2 -> we jump 2 years back -union all --- outcome range -select "primary_id", - 'YEARS' as "resolution", - "index", - "index_selector", - daterange( - ("index_start_positive" + interval '1 year' * ("index" + -1))::date, - ("index_start_positive" + interval '1 year' * ("index" + 0))::date, - '[)' - ) * "stratification_bounds" as "stratification_bounds" -from "total_bounds", - generate_series(1, 1) as "index" -- 1 -> we jump 1 year forward -``` - -Using the `stratification_bounds`, representing the min and max stratification date calculated in the previous step, -we intersect this range with our calculated time frame to generate stratification windows which are correctly bound. - -## Full stratification table - -Now, we union all the resolution tables to obtain the full stratification table. - -```sql -select "complete"."primary_id", - "complete"."resolution", - "complete"."index", - "complete"."stratification_bounds" -from "complete" -union all -select "years"."primary_id", - "years"."resolution", - "years"."index", - "years"."stratification_bounds" -from "years" -union all -select "quarters"."primary_id", - "quarters"."resolution", - "quarters"."index", - "quarters"."stratification_bounds" -from "quarters" -``` - -| primary\_id | resolution | index | stratification\_bounds | -|:------------|:-----------|:------|:--------------------------| -| 1 | COMPLETE | 1 | \[2012-06-16,2014-12-18\) | -| 1 | YEARS | 1 | \[2012-06-16,2013-04-01\) | -| 1 | YEARS | 2 | \[2013-04-01,2014-04-01\) | -| 1 | YEARS | 3 | \[2014-04-01,2014-12-18\) | -| 1 | QUARTERS | 1 | \[2012-06-16,2012-07-01\) | -| 1 | QUARTERS | 2 | \[2012-07-01,2012-10-01\) | -| 1 | QUARTERS | 3 | \[2012-10-01,2013-01-01\) | -| 1 | QUARTERS | 4 | \[2013-01-01,2013-04-01\) | -| 1 | QUARTERS | 5 | \[2013-04-01,2013-07-01\) | -| 1 | QUARTERS | 6 | \[2013-07-01,2013-10-01\) | -| 1 | QUARTERS | 7 | \[2013-10-01,2014-01-01\) | -| 1 | QUARTERS | 8 | \[2014-01-01,2014-04-01\) | -| 1 | QUARTERS | 9 | \[2014-04-01,2014-07-01\) | -| 1 | QUARTERS | 10 | \[2014-07-01,2014-10-01\) | -| 1 | QUARTERS | 11 | \[2014-10-01,2014-12-18\) | - -## Feature conversion - -After we got our full stratification table, containing all stratification windows for each ID, we want to convert all -the features of the form, while using our stratification table as a starting point: -When converting a concept and creating the `PREPROCESSING` CTE, which is the starting point of each concept -conversion, we join the concepts or respectively the connectors table with the stratification table for all IDs from -the stratification table. In the same step, we filter all entries where the stratification range and the subject's -validity date do not overlap. This is important because we only want to compute aggregations for overlapping ranges. - -**CTE:** `preprocessing` - -```sql -select "full_stratification"."primary_id", - "full_stratification"."resolution", - "full_stratification"."index", - "full_stratification"."stratification_range_start", - "full_stratification"."stratification_range_end", - "vers_stamm"."date_start" "validity_date_start", - ADD_DAYS("vers_stamm"."date_end", 1) "validity_date_end", - "vers_stamm"."date_of_birth" -from "vers_stamm" - join "full_stratification" - on "full_stratification"."primary_id" = "vers_stamm"."pid" -where "full_stratification"."stratification_range_start" < ADD_DAYS("vers_stamm"."date_end", 1) - and "full_stratification"."stratification_range_end" > "vers_stamm"."date_start" -``` - -Besides grouping by ID, resolution, index and stratification range, the remaining concept conversion CTE process -remains as usual. If we have multiple features, we'll join the respective converted concept queries via an OR. - -## Left-join converted features with the full stratification table for the final select - -For an absolute form, we expect the final result to contain all stratification ranges for each ID of the respective -chosen resolutions. Because we filter all entries where stratification range and validity date do not overlap during -each concept conversion's preprocessing step, the converted feature(s) table might not contain all stratification ranges. -Thus, we left-join the table with the converted feature(s) back with the full stratification table. - -## Full export form - -When converting full export forms, we must ensure the columns in the final select query have the right order. Because -the `TableExportQuery` of an `FullExportForm` contains already the mapping from each required column to the position -in the final select, we just export each required table by selecting all the required columns. For each column, we -check if the current table contains the respective column. If so, we select the column, otherwise we just select null. -This way, we can easily union all export tables without the need to apply additional logic to obtain the correct column -order. - -```sql -with "external" as (select 1 as "primary_id"), - "extract_ids" as (select "primary_id" - from "external" - group by "primary_id") -(select "table"."pid", - "validity_date"::varchar as "validity_date", - 'table' as "source", -- name of the table we select from - "table"."second_id" as "second_id-2", - "table"."column" as "column-3", - null as "date_end-4", - null as "geburtsdatum-5", - null as "geschlecht-6" - from "table" - join "extract_ids" - on "table"."pid" = "extract_ids"."primary_id" - where "table"."column" in ('A')) - union all -(select "vers_stamm"."pid", - "vers_stamm"."validity_date"::varchar as "validity_date", - 'vers_stamm' as "source", - null as "second_id-2", - null as "column-3", - "vers_stamm"."date_end" as "date_end-4", - "vers_stamm"."geburtsdatum" as "geburtsdatum-5", - "vers_stamm"."geschlecht" as "geschlecht-6" - from "vers_stamm" - join "extract_ids" - on "vers_stamm"."pid" = "extract_ids"."primary_id") -``` diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java deleted file mode 100644 index 0070516eec..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java +++ /dev/null @@ -1,78 +0,0 @@ -package com.bakdata.conquery.sql.conversion.query; - -import java.util.List; -import java.util.Optional; - -import com.bakdata.conquery.apiv1.query.Query; -import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery; -import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.compiler.ir.SharedAliases; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.forms.FormCteStep; -import com.bakdata.conquery.sql.conversion.forms.FormType; -import com.bakdata.conquery.sql.conversion.forms.StratificationTableFactory; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import com.google.common.base.Preconditions; -import lombok.RequiredArgsConstructor; - -@RequiredArgsConstructor -public class AbsoluteFormQueryConverter implements NodeConverter { - - private final FormConversionHelper formHelper; - - @Override - public Class getConversionClass() { - return AbsoluteFormQuery.class; - } - - @Override - public ConversionContext convert(AbsoluteFormQuery form, ConversionContext context) { - - QueryStep convertedPrerequisite = convertPrerequisite(form, context); - StratificationTableFactory tableFactory = new StratificationTableFactory(convertedPrerequisite, context); - QueryStep stratificationTable = tableFactory.createAbsoluteStratificationTable(form.getResolutionsAndAlignmentMap(), context); - - return formHelper.convertForm( - FormType.ABSOLUTE, - stratificationTable, - form.getFeatures(), - form.getResultInfos(), - context - ); - } - - /** - * Converts the given {@link Query} and creates another {@link QueryStep} on top which extracts only the primary id. The form's date range is set - * as stratification range. - */ - private static QueryStep convertPrerequisite(AbsoluteFormQuery absoluteForm, ConversionContext context) { - - ConversionContext withConvertedPrerequisite = context.getNodeConversions().convert(absoluteForm.getQuery(), context); - Preconditions.checkArgument(withConvertedPrerequisite.getQuerySteps().size() == 1, "Base query conversion should produce exactly 1 QueryStep"); - QueryStep convertedPrerequisite = withConvertedPrerequisite.getLastConvertedStep(); - - ColumnDateRange bounds = context.getCompilerDialect() - .getFunctionProvider() - .forCDateRange(CDateRange.of(absoluteForm.getDateRange())).as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); - - Selects prerequisiteSelects = convertedPrerequisite.getQualifiedSelects(); - // we only keep the primary column for the upcoming form - Selects selects = Selects.builder() - .ids(new SqlIdColumns(prerequisiteSelects.getIds().getPrimaryColumn())) - .stratificationDate(Optional.of(bounds)) - .build(); - - return QueryStep.builder() - .cteName(FormCteStep.EXTRACT_IDS.getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(convertedPrerequisite.getCteName())) - .groupBy(selects.getIds().toFields()) // group by primary column to ensure max. 1 entry per subject - .predecessors(List.of(convertedPrerequisite)) - .build(); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java deleted file mode 100644 index 11d2d0ed5a..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java +++ /dev/null @@ -1,97 +0,0 @@ -package com.bakdata.conquery.sql.conversion.query; - -import java.util.List; -import java.util.Optional; - -import com.bakdata.conquery.models.forms.managed.EntityDateQuery; -import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.compiler.ir.SharedAliases; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.forms.FormCteStep; -import com.bakdata.conquery.sql.conversion.forms.FormType; -import com.bakdata.conquery.sql.conversion.forms.StratificationTableFactory; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.google.common.base.Preconditions; -import lombok.RequiredArgsConstructor; - -@RequiredArgsConstructor -public class EntityDateQueryConverter implements NodeConverter { - - private final FormConversionHelper formHelper; - - @Override - public Class getConversionClass() { - return EntityDateQuery.class; - } - - @Override - public ConversionContext convert(EntityDateQuery entityDateQuery, ConversionContext context) { - - QueryStep prerequisite = formHelper.convertPrerequisite(entityDateQuery.getQuery(), context); - QueryStep withOverwrittenValidityDateBounds = overwriteBounds(prerequisite, entityDateQuery, context); - StratificationTableFactory tableFactory = new StratificationTableFactory(withOverwrittenValidityDateBounds, context); - QueryStep stratificationTable = tableFactory.createAbsoluteStratificationTable(entityDateQuery.getResolutionsAndAlignments(), context); - - return formHelper.convertForm( - FormType.ENTITY_DATE, - stratificationTable, - entityDateQuery.getFeatures(), - entityDateQuery.getResultInfos(), - context - ); - } - - /** - * Computes the intersection of the entity date and the entity date query's daterange. - */ - private static QueryStep overwriteBounds(QueryStep prerequisite, EntityDateQuery entityDateQuery, ConversionContext context) { - - Preconditions.checkArgument( - prerequisite.getQualifiedSelects().getValidityDate().isPresent(), - "Expecting the prerequisite step's Selects to contain a validity date" - ); - - SqlFunctionProvider functionProvider = context.getFunctionProvider(); - // we want to create a stratification for each distinct validity date range of an entity, - // so we first need to unnest the validity date in case it is a multirange - String unnestCteName = FormCteStep.UNNEST_ENTITY_DATE_CTE.getSuffix(); - QueryStep unnestedEntityDate = functionProvider.unnestDaterange(prerequisite.getSelects().getValidityDate().get(), prerequisite, unnestCteName); - Selects unnestedSelects = unnestedEntityDate.getQualifiedSelects(); - - ColumnDateRange withOverwrittenBounds; - if (entityDateQuery.getDateRange() == null) { - withOverwrittenBounds = unnestedSelects.getValidityDate().get(); - } - else { - ColumnDateRange formDateRange = functionProvider.forCDateRange(entityDateQuery.getDateRange()); - ColumnDateRange entityDate = unnestedSelects.getValidityDate().get(); - withOverwrittenBounds = functionProvider.intersection(formDateRange, entityDate) - .as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); - } - - Selects selects = Selects.builder() - .ids(unnestedSelects.getIds()) - .stratificationDate(Optional.of(withOverwrittenBounds)) - .build(); - - List predecessors; - // unnest step is optional depending on the dialect - if (unnestedEntityDate == prerequisite) { - predecessors = List.of(prerequisite); - } - else { - predecessors = List.of(prerequisite, unnestedEntityDate); - } - - return QueryStep.builder() - .cteName(FormCteStep.OVERWRITE_BOUNDS.getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(unnestedEntityDate.getCteName())) - .predecessors(predecessors) - .build(); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java deleted file mode 100644 index 69fd11ac22..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java +++ /dev/null @@ -1,169 +0,0 @@ -package com.bakdata.conquery.sql.conversion.query; - -import com.bakdata.conquery.sql.compiler.ir.JoinMode; -import com.bakdata.conquery.sql.compiler.ir.ProjectionMode; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.QueryStepJoiner; -import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.bakdata.conquery.sql.compiler.rendering.QueryStepRenderer; -import com.bakdata.conquery.apiv1.forms.FeatureGroup; -import com.bakdata.conquery.apiv1.query.ArrayConceptQuery; -import com.bakdata.conquery.apiv1.query.ConceptQuery; -import com.bakdata.conquery.apiv1.query.Query; -import com.bakdata.conquery.models.forms.util.Resolution; -import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; -import com.bakdata.conquery.sql.compiler.ir.SharedAliases; -import com.bakdata.conquery.models.query.DateAggregationAction; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.forms.FormCteStep; -import com.bakdata.conquery.sql.conversion.forms.FormType; -import com.bakdata.conquery.sql.conversion.model.*; -import com.google.common.base.Preconditions; -import lombok.RequiredArgsConstructor; -import org.jooq.*; -import org.jooq.Record; - -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.jooq.impl.DSL.*; - -@RequiredArgsConstructor -public class FormConversionHelper { - - private final QueryStepRenderer queryStepRenderer; - - /** - * Selects the ID, resolution, index and date range from stratification table plus all explicit selects from the converted features step. - * For {@link FormType#RELATIVE}, the {@link FeatureGroup} will be set too. - */ - private static Selects getFinalSelects( - FormType formType, - QueryStep stratificationTable, - QueryStep convertedFeatures - ) { - Selects preFinalSelects = convertedFeatures.getQualifiedSelects(); - Selects stratificationSelects = stratificationTable.getQualifiedSelects(); - - SqlIdColumns ids = stratificationSelects.getIds().forFinalSelect(Resolution.COMPLETE.name()); - - Selects.SelectsBuilder selects = Selects.builder() - .ids(ids) - .validityDate(preFinalSelects.getValidityDate()) - .stratificationDate(stratificationSelects.getStratificationDate()); - - // relative forms have FeatureGroup information after the stratification date and before all other selects - if (formType.equals(FormType.RELATIVE)) { - Field indexField = field(name(stratificationTable.getCteName(), SharedAliases.INDEX.getAlias()), Integer.class); - Field scope = when(indexField.isNull().or(indexField.lessThan(0)), inline(FeatureGroup.FEATURE.toString())) - .otherwise(inline(FeatureGroup.OUTCOME.toString())) - .as(SharedAliases.OBSERVATION_SCOPE.getAlias()); - - selects = selects.sqlSelect(new FieldWrapper<>(scope)); - } - - return selects.sqlSelects(preFinalSelects.getSqlSelects()).build(); - - } - - /** - * Converts the given {@link Query} and creates another {@link QueryStep} on top which extracts only the primary id and the validity dates. - */ - public QueryStep convertPrerequisite(Query query, ConversionContext context) { - - ConversionContext withConvertedPrerequisite = context.getNodeConversions().convert(query, context); - Preconditions.checkArgument(withConvertedPrerequisite.getQuerySteps().size() == 1, "Base query conversion should produce exactly 1 QueryStep"); - QueryStep convertedPrerequisite = withConvertedPrerequisite.getLastConvertedStep(); - - Selects prerequisiteSelects = convertedPrerequisite.getQualifiedSelects(); - // we keep the primary column and the validity date - Selects selects = Selects.builder() - .ids(new SqlIdColumns(prerequisiteSelects.getIds().getPrimaryColumn())) - .validityDate(prerequisiteSelects.getValidityDate()) - .build(); - - // we want to keep each primary column and the corresponding distinct validity date ranges - List> groupByFields = Stream.concat( - Stream.of(prerequisiteSelects.getIds().getPrimaryColumn()), - prerequisiteSelects.getValidityDate().stream().flatMap(validityDate -> validityDate.toFields().stream()) - ) - .collect(Collectors.toList()); - - // filter out entries with a null validity date - Optional columnDateRange = prerequisiteSelects.getValidityDate(); - Condition dateNotNullCondition; - - if (columnDateRange.isPresent()) { - dateNotNullCondition = context.getFunctionProvider().isNotEmptyDateRange(columnDateRange.get()); - } else { - dateNotNullCondition = falseCondition(); - } - - return QueryStep.builder() - .cteName(FormCteStep.EXTRACT_IDS.getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(convertedPrerequisite.getCteName())) - .conditions(List.of(dateNotNullCondition)) - .groupBy(groupByFields) - .predecessors(List.of(convertedPrerequisite)) - .build(); - } - - public ConversionContext convertForm( - FormType formType, - QueryStep stratificationTable, - ArrayConceptQuery features, - List resultInfos, - ConversionContext context - ) { - // feature conversion - ConversionContext childContext = context.createChildContext().withStratificationTable(stratificationTable); - for (ConceptQuery conceptQuery : features.getChildQueries()) { - childContext = context.getNodeConversions().convert(conceptQuery, childContext); - } - - // child context contains the converted feature's QuerySteps - List queriesToJoin = childContext.getQuerySteps(); - QueryStep joinedFeatures = QueryStepComposer.joinSteps(queriesToJoin, JoinMode.FULL_OUTER, DateAggregationAction.BLOCK, context); - return createFinalSelect(formType, stratificationTable, joinedFeatures, resultInfos, context); - } - - /** - * Left-joins the full stratification table back with the converted feature tables to keep all the resolutions. - *

- * When converting features, we filter out rows where a subjects validity date and the stratification date do not overlap. - * Thus, the pre-final step might not contain an entry for each expected stratification window. That's why we need to left-join the converted - * features with the full stratification table again. - */ - private ConversionContext createFinalSelect( - FormType formType, - QueryStep stratificationTable, - QueryStep convertedFeatures, - List resultInfos, - ConversionContext context - ) { - Preconditions.checkArgument( - stratificationTable.getSelects().getStratificationDate().isPresent() && convertedFeatures.getSelects().getStratificationDate().isPresent(), - "Expecting stratification table and converted features table to contain a stratification date" - ); - - List queriesToJoin = List.of(stratificationTable, convertedFeatures); - TableLike joinedTable = QueryStepJoiner.join(queriesToJoin, JoinMode.LEFT); - QueryStep finalStep = QueryStep.builder() - .cteName(null) // the final QueryStep won't be converted to a CTE - .projectionMode(ProjectionMode.AGGREGATED) - .selects(getFinalSelects(formType, stratificationTable, convertedFeatures)) - .fromTable(joinedTable) - .predecessors(queriesToJoin) - .build(); - - Select selectQuery = queryStepRenderer.toSelectQuery(finalStep, context.getCompilerDialect()); - return context.withFinalQuery(new SqlQuery(selectQuery, resultInfos)); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java deleted file mode 100644 index 68b54a37a5..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.bakdata.conquery.sql.conversion.query; - -import com.bakdata.conquery.models.forms.managed.RelativeFormQuery; -import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.forms.FormType; -import com.bakdata.conquery.sql.conversion.forms.StratificationTableFactory; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import lombok.RequiredArgsConstructor; - -@RequiredArgsConstructor -public class RelativFormQueryConverter implements NodeConverter { - - private final FormConversionHelper formHelper; - - @Override - public Class getConversionClass() { - return RelativeFormQuery.class; - } - - @Override - public ConversionContext convert(RelativeFormQuery form, ConversionContext context) { - - QueryStep convertedPrerequisite = formHelper.convertPrerequisite(form.getQuery(), context); - StratificationTableFactory tableFactory = new StratificationTableFactory(convertedPrerequisite, context); - QueryStep stratificationTable = tableFactory.createRelativeStratificationTable(form, context); - - return formHelper.convertForm( - FormType.RELATIVE, - stratificationTable, - form.getFeatures(), - form.getResultInfos(), - context - ); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java deleted file mode 100644 index 9193a185c8..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java +++ /dev/null @@ -1,225 +0,0 @@ -package com.bakdata.conquery.sql.conversion.query; - -import com.bakdata.conquery.sql.compiler.ir.ProjectionMode; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.bakdata.conquery.sql.compiler.rendering.QueryStepRenderer; -import com.bakdata.conquery.apiv1.query.Query; -import com.bakdata.conquery.apiv1.query.TableExportQuery; -import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; -import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; -import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; -import com.bakdata.conquery.sql.compiler.ir.condition.DateRestrictionCondition; -import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; -import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.compiler.ir.SharedAliases; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.forms.FormCteStep; -import com.bakdata.conquery.sql.conversion.model.*; -import com.bakdata.conquery.util.TablePrimaryColumnUtil; -import com.google.common.base.Preconditions; -import lombok.RequiredArgsConstructor; -import org.jooq.*; -import org.jooq.Record; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.jooq.impl.DSL.*; - -@RequiredArgsConstructor -public class TableExportQueryConverter implements NodeConverter { - - /** - * Validity date is part of positions, but not when converting to SQL because it might have multiple columns and not just one. - * Thus, we need to apply an offset to the positions. - */ - private static final int POSITION_OFFSET = 1; - - private final QueryStepRenderer queryStepRenderer; - - /** - * Converts the {@link Query} of the given {@link TableExportQuery} and creates another {@link QueryStep} on top which extracts only the primary id. - */ - private static QueryStep convertPrerequisite(TableExportQuery exportQuery, ConversionContext context) { - - final ConversionContext withConvertedPrerequisite = context.getNodeConversions().convert(exportQuery.getQuery(), context); - Preconditions.checkArgument(withConvertedPrerequisite.getQuerySteps().size() == 1, "Base query conversion should produce exactly 1 QueryStep"); - final QueryStep convertedPrerequisite = withConvertedPrerequisite.getLastConvertedStep(); - - final Selects prerequisiteSelects = convertedPrerequisite.getQualifiedSelects(); - final Selects selects = Selects.builder() - .ids(new SqlIdColumns(prerequisiteSelects.getIds().getPrimaryColumn())) - .build(); - - return QueryStep.builder() - .cteName(FormCteStep.EXTRACT_IDS.getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(convertedPrerequisite.getCteName())) - .groupBy(selects.getIds().toFields()) // group by primary column to ensure max. 1 entry per subject - .predecessors(List.of(convertedPrerequisite)) - .build(); - } - - /** - * Create a CTE selecting all positions of a {@link TableExportQuery} from the given {@link CQTable} properly ordered. For all columns, which are not - * linked to the given table, we just do a null select. - */ - private static QueryStep convertTable( - CQTable cqTable, - CQConcept concept, - CDateRange dateRestriction, - QueryStep convertedPrerequisite, - Map positions, - ConversionContext context - ) { - final Field primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(cqTable.getConnector().resolve().getResolvedTable(), - context.getDefaultPrimaryColumn() - ); - final SqlIdColumns ids = new SqlIdColumns(primaryColumn); - final String conceptConnectorName = context.getNameGenerator().legacyConceptConnectorName( - concept.userLabel(context.getSqlPrintSettings().getLocale()), - cqTable.getConnector().resolve().getName() - ); - final Optional validityDate = convertTablesValidityDate(cqTable, conceptConnectorName, context); - - final List> exportColumns = initializeFields(cqTable, positions); - - final Selects selects = Selects.builder() - .ids(ids) - .validityDate(validityDate) - .sqlSelects(exportColumns) - .build(); - - final List filters = cqTable.getFilters().stream().map(filterValue -> filterValue.convertForTableExport(ids, context)).toList(); - final Table joinedTable = joinConnectorTableWithPrerequisite(cqTable, ids, convertedPrerequisite, dateRestriction, context); - - return QueryStep.builder() - .cteName(conceptConnectorName) - .selects(selects) - .fromTable(joinedTable) - .conditions(filters) - .projectionMode(ProjectionMode.INDIVIDUAL) - .build(); - } - - private static Optional convertTablesValidityDate(CQTable table, String alias, ConversionContext context) { - - final SqlFunctionProvider functionProvider = context.getFunctionProvider(); - - if (table.findValidityDate() == null) { - return Optional.of(functionProvider.emptyColumnDateRange()); - } - - final ColumnDateRange validityDateField = functionProvider.forValidityDate(table.findValidityDate()); - - return Optional.of(validityDateField); - } - - private static List> initializeFields(CQTable cqTable, Map positions) { - - final Field[] exportColumns = createPlaceholders(positions); - - exportColumns[0] = createSourceInfoSelect(cqTable); - - for (Column column : cqTable.getConnector().resolve().getResolvedTable().getColumns()) { - // e.g. date column(s) are handled separately and not part of positions - if (!positions.containsKey(column.getId())) { - continue; - } - final int position = positions.get(column.getId()) - POSITION_OFFSET; - exportColumns[position] = createColumnSelect(column, position); - } - - return Arrays.stream(exportColumns).map(FieldWrapper::new).collect(Collectors.toList()); - } - - private static Table joinConnectorTableWithPrerequisite( - CQTable cqTable, - SqlIdColumns ids, - QueryStep convertedPrerequisite, - CDateRange dateRestriction, - ConversionContext context - ) { - final SqlFunctionProvider functionProvider = context.getFunctionProvider(); - final Table connectorTable = table(name(cqTable.getConnector().resolve().resolveTableId().getTable())); - final Table convertedPrerequisiteTable = table(name(convertedPrerequisite.getCteName())); - - final ColumnDateRange validityDate = functionProvider.forValidityDate(cqTable.findValidityDate()); - final List joinConditions = Stream.concat( - ids.join(convertedPrerequisite.getQualifiedSelects().getIds()).stream(), - Stream.of(new DateRestrictionCondition(functionProvider.forCDateRange(dateRestriction), validityDate).condition()) - ).toList(); - - return connectorTable.innerJoin(convertedPrerequisiteTable).on(joinConditions.toArray(Condition[]::new)); - } - - private static Field[] createPlaceholders(Map positions) { - - final int size = TableExportQuery.calculateWidth(positions) - POSITION_OFFSET; - final Field[] exportColumns = new Field[size]; - - // if columns have the same computed position, they can share a common name because they will be unioned over multiple tables anyway - for (int index = 0; index < exportColumns.length; index++) { - final Field columnSelect = inline(null, Object.class).as("null-%d".formatted(index)); - exportColumns[index] = columnSelect; - } - - return exportColumns; - } - - private static Field createSourceInfoSelect(CQTable cqTable) { - final String tableName = cqTable.getConnector().resolve().resolveTableId().getTable(); - return inline(tableName).as(SharedAliases.SOURCE.getAlias()); - } - - private static Field createColumnSelect(Column column, int position) { - final String columnName = "%s-%s".formatted(column.getName(), position); - return field(name(column.getTable().getName(), column.getName())) - .as(columnName); - } - - @Override - public Class getConversionClass() { - return TableExportQuery.class; - } - - @Override - public ConversionContext convert(TableExportQuery tableExportQuery, ConversionContext context) { - - final QueryStep convertedPrerequisite = convertPrerequisite(tableExportQuery, context); - final Map positions = tableExportQuery.getPositions(); - final CDateRange dateRestriction = CDateRange.of(tableExportQuery.getDateRange()); - - final List convertedTables = tableExportQuery.getConcepts().stream() - .flatMap(concept -> concept.getTables().stream().map(table -> convertTable( - table, - concept, - dateRestriction, - convertedPrerequisite, - positions, - context - ))) - .toList(); - - final QueryStep unionedTables = QueryStep.createUnionAllStep( - convertedTables, - null, // no CTE name required as this step will be the final select - List.of(convertedPrerequisite), - context.isNegation() - ); - final Select selectQuery = queryStepRenderer.toSelectQuery(unionedTables, context.getCompilerDialect()); - - return context.withFinalQuery(new SqlQuery(selectQuery, tableExportQuery.getResultInfos())); - } - -} diff --git a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlConverterTest.java b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlConverterTest.java index 90afc2ae05..6516eefa32 100644 --- a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlConverterTest.java +++ b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlConverterTest.java @@ -13,6 +13,9 @@ import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; import com.bakdata.conquery.apiv1.query.TableExportQuery; import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery; +import com.bakdata.conquery.models.forms.managed.EntityDateQuery; +import com.bakdata.conquery.models.forms.managed.RelativeFormQuery; import com.bakdata.conquery.models.query.DateAggregationMode; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; @@ -25,8 +28,10 @@ class SqlConverterTest { private final ConqueryConfig config = mock(ConqueryConfig.class); private final ResolvedQueryAdapter adapter = mock(ResolvedQueryAdapter.class); private final ResolvedTableExportAdapter tableExportAdapter = mock(ResolvedTableExportAdapter.class); + private final ResolvedFormAdapter formAdapter = mock(ResolvedFormAdapter.class); private final LegacyCompilerDialect dialect = mock(LegacyCompilerDialect.class); - private final SqlConverter converter = new SqlConverter(nodeConversions, config, adapter, tableExportAdapter, dialect); + private final SqlConverter converter = new SqlConverter( + nodeConversions, config, adapter, tableExportAdapter, formAdapter, dialect); @Test void shouldRouteConceptQueriesThroughTheResolvedCompiler() { @@ -68,6 +73,28 @@ void shouldRouteTableExportsThroughTheResolvedCompiler() { verifyNoInteractions(nodeConversions); } + @Test + void shouldRouteAllFormTypesThroughTheResolvedCompiler() { + AbsoluteFormQuery absolute = mock(AbsoluteFormQuery.class); + RelativeFormQuery relative = mock(RelativeFormQuery.class); + EntityDateQuery entityDate = mock(EntityDateQuery.class); + SqlQuery absoluteSql = mock(SqlQuery.class); + SqlQuery relativeSql = mock(SqlQuery.class); + SqlQuery entityDateSql = mock(SqlQuery.class); + when(formAdapter.compile(same(absolute), same(dialect))).thenReturn(absoluteSql); + when(formAdapter.compile(same(relative), same(dialect))).thenReturn(relativeSql); + when(formAdapter.compile(same(entityDate), same(dialect))).thenReturn(entityDateSql); + + assertSame(absoluteSql, converter.convert(absolute)); + assertSame(relativeSql, converter.convert(relative)); + assertSame(entityDateSql, converter.convert(entityDate)); + + verify(formAdapter).compile(same(absolute), same(dialect)); + verify(formAdapter).compile(same(relative), same(dialect)); + verify(formAdapter).compile(same(entityDate), same(dialect)); + verifyNoInteractions(nodeConversions); + } + @Test void shouldKeepOtherQueryDescriptionsOnTheLegacyConversionPath() { QueryDescription query = mock(QueryDescription.class); @@ -79,6 +106,6 @@ void shouldKeepOtherQueryDescriptionsOnTheLegacyConversionPath() { SqlQuery converted = converter.convert(query); assertSame(sql, converted); - verifyNoInteractions(adapter, tableExportAdapter); + verifyNoInteractions(adapter, tableExportAdapter, formAdapter); } } diff --git a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionAggregationTest.java b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionAggregationTest.java index 1a30f056c3..0c0f300d60 100644 --- a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionAggregationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionAggregationTest.java @@ -135,7 +135,7 @@ void shouldPackDurationIntervalsWithoutApplyingDistinctKeys() { static SelectContext context(LegacyCompilerDialect dialect) { var conversion = ConversionContext.builder() - .compilerDialect(dialect).stratificationFunctions(dialect.getStratificationFunctions()) + .compilerDialect(dialect) .nameGenerator(new SqlNameGenerator(127)).build(); var tables = new SqlTables("events", Map.of(ConceptCteStep.PREPROCESSING, "preprocessing", ConceptCteStep.AGGREGATION_SELECT, "aggregation"), diff --git a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionDialectTest.java b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionDialectTest.java index ce8538516b..af7fb0a490 100644 --- a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionDialectTest.java +++ b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionDialectTest.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.sql.conversion.dialect.hana.HanaDialectBundle; import com.bakdata.conquery.sql.compiler.dialect.clickhouse.ClickhouseCompilerDialect; import com.bakdata.conquery.sql.compiler.dialect.hana.HanaCompilerDialect; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import org.jooq.SQLDialect; import org.jooq.impl.DSL; import org.junit.jupiter.api.Test; @@ -43,4 +44,18 @@ void shouldUseFrameworkNeutralDialectCapabilitiesFromLegacyBundles() { DSL.using(SQLDialect.DEFAULT).renderInlined(hana.getFunctionProvider().getMaxDateExpression()), DSL.using(SQLDialect.DEFAULT).renderInlined(hana.maximumDate())); } + + @Test + void shouldPreserveClickhouseFormDateEmptinessCondition() { + var dialect = new ClickhouseDialectBundle(); + ColumnDateRange range = ColumnDateRange.of( + DSL.field(DSL.name("events", "start"), Date.class), + DSL.field(DSL.name("events", "end"), Date.class)); + + assertEquals( + DSL.using(SQLDialect.CLICKHOUSE).renderInlined( + dialect.getFunctionProvider().isNotEmptyDateRange(range)), + DSL.using(SQLDialect.CLICKHOUSE).renderInlined(dialect.isNotEmptyDateRange(range)) + ); + } } diff --git a/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionFormTest.java b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionFormTest.java new file mode 100644 index 0000000000..e6a3a2bad3 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionFormTest.java @@ -0,0 +1,120 @@ +package com.bakdata.conquery.sql.conversion; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.time.LocalDate; +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.apiv1.forms.IndexPlacement; +import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; +import com.bakdata.conquery.apiv1.query.ArrayConceptQuery; +import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.apiv1.query.Query; +import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; +import com.bakdata.conquery.apiv1.query.TemporalSamplerFactory; +import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery; +import com.bakdata.conquery.models.forms.managed.EntityDateQuery; +import com.bakdata.conquery.models.forms.managed.RelativeFormQuery; +import com.bakdata.conquery.models.forms.util.Alignment; +import com.bakdata.conquery.models.forms.util.CalendarUnit; +import com.bakdata.conquery.models.forms.util.Resolution; +import com.bakdata.conquery.sql.model.ResolvedQuery; +import com.bakdata.conquery.sql.model.form.FormAlignment; +import com.bakdata.conquery.sql.model.form.FormCalendarUnit; +import com.bakdata.conquery.sql.model.form.FormIndexPlacement; +import com.bakdata.conquery.sql.model.form.FormIndexSelector; +import com.bakdata.conquery.sql.model.form.FormResolution; +import com.bakdata.conquery.sql.model.form.ResolvedFormMode; +import com.bakdata.conquery.sql.model.form.ResolvedFormQuery; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class SqlExtractionFormTest { + + private final ResolvedQueryMapper queryMapper = mock(ResolvedQueryMapper.class); + private final Query prerequisite = mock(SecondaryIdQuery.class); + private final ConceptQuery feature = mock(ConceptQuery.class); + private final ResolvedQuery resolvedPrerequisite = mock(ResolvedQuery.class); + private final ResolvedQuery resolvedFeature = mock(ResolvedQuery.class); + private final ArrayConceptQuery features = mock(ArrayConceptQuery.class); + private final ResolvedFormMapper mapper = new ResolvedFormMapper(queryMapper); + + @BeforeEach + void setUp() { + when(features.getChildQueries()).thenReturn(List.of(feature)); + when(feature.getResultInfos()).thenReturn(List.of()); + when(queryMapper.map(same(prerequisite), eq(List.of()))).thenReturn(resolvedPrerequisite); + when(queryMapper.map(same(feature), eq(Optional.empty()), anyList())).thenReturn(resolvedFeature); + } + + @Test + void shouldMapAbsoluteAndEntityDateBoundsWithoutChangingResolutionSettings() { + ExportForm.ResolutionAndAlignment resolution = ExportForm.ResolutionAndAlignment.of( + Resolution.COMPLETE, Alignment.NO_ALIGN); + AbsoluteFormQuery absolute = mock(AbsoluteFormQuery.class); + when(absolute.getQuery()).thenReturn(prerequisite); + when(absolute.getFeatures()).thenReturn(features); + when(absolute.getDateRange()).thenReturn(Range.of( + LocalDate.of(2020, 1, 1), LocalDate.of(2020, 12, 31))); + when(absolute.getResolutionsAndAlignmentMap()).thenReturn(List.of(resolution)); + EntityDateQuery entityDate = mock(EntityDateQuery.class); + when(entityDate.getQuery()).thenReturn(prerequisite); + when(entityDate.getFeatures()).thenReturn(features); + when(entityDate.getDateRange()).thenReturn(CDateRange.atLeast(LocalDate.of(2021, 1, 1))); + when(entityDate.getResolutionsAndAlignments()).thenReturn(List.of(resolution)); + + ResolvedFormQuery absoluteResolved = mapper.map(absolute); + ResolvedFormQuery entityDateResolved = mapper.map(entityDate); + + ResolvedFormMode.Absolute absoluteMode = assertInstanceOf( + ResolvedFormMode.Absolute.class, absoluteResolved.mode()); + assertEquals(Optional.of(LocalDate.of(2020, 1, 1)), absoluteMode.bounds().startInclusive()); + assertEquals(Optional.of(LocalDate.of(2020, 12, 31)), absoluteMode.bounds().endInclusive()); + assertEquals(FormResolution.COMPLETE, absoluteMode.resolutions().getFirst().resolution()); + assertEquals(FormAlignment.NO_ALIGN, absoluteMode.resolutions().getFirst().alignment()); + ResolvedFormMode.EntityDate entityMode = assertInstanceOf( + ResolvedFormMode.EntityDate.class, entityDateResolved.mode()); + assertEquals(Optional.of(LocalDate.of(2021, 1, 1)), entityMode.bounds().orElseThrow().startInclusive()); + assertEquals(Optional.empty(), entityMode.bounds().orElseThrow().endInclusive()); + assertSame(resolvedPrerequisite, absoluteResolved.prerequisite()); + assertEquals(List.of(resolvedFeature), absoluteResolved.features()); + } + + @Test + void shouldMapRelativeFeatureAndOutcomeSettings() { + RelativeFormQuery relative = mock(RelativeFormQuery.class); + when(relative.getQuery()).thenReturn(prerequisite); + when(relative.getFeatures()).thenReturn(features); + when(relative.getIndexSelector()).thenReturn(TemporalSamplerFactory.LATEST); + when(relative.getIndexPlacement()).thenReturn(IndexPlacement.NEUTRAL); + when(relative.getTimeCountBefore()).thenReturn(4); + when(relative.getTimeCountAfter()).thenReturn(2); + when(relative.getTimeUnit()).thenReturn(CalendarUnit.QUARTERS); + when(relative.getResolutionsAndAlignmentMap()).thenReturn(List.of( + ExportForm.ResolutionAndAlignment.of(Resolution.YEARS, Alignment.QUARTER))); + + ResolvedFormMode.Relative mode = assertInstanceOf( + ResolvedFormMode.Relative.class, mapper.map(relative).mode()); + + assertEquals(FormIndexSelector.LATEST, mode.settings().indexSelector()); + assertEquals(FormIndexPlacement.NEUTRAL, mode.settings().indexPlacement()); + assertEquals(FormCalendarUnit.QUARTERS, mode.settings().timeUnit()); + assertEquals(4, mode.settings().timeCountBefore()); + assertEquals(2, mode.settings().timeCountAfter()); + assertEquals(FormResolution.YEARS, mode.settings().resolutions().getFirst().resolution()); + assertEquals(FormAlignment.QUARTER, mode.settings().resolutions().getFirst().alignment()); + verify(queryMapper).map(same(prerequisite), eq(List.of())); + verify(queryMapper).map(same(feature), eq(Optional.empty()), eq(List.of())); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/FormSqlCompiler.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/FormSqlCompiler.java new file mode 100644 index 0000000000..3eb34c06b8 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/FormSqlCompiler.java @@ -0,0 +1,200 @@ +package com.bakdata.conquery.sql.compiler; + +import static org.jooq.impl.DSL.falseCondition; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.inline; +import static org.jooq.impl.DSL.name; +import static org.jooq.impl.DSL.when; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import com.bakdata.conquery.models.query.DateAggregationAction; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; +import com.bakdata.conquery.sql.compiler.forms.FormCteStep; +import com.bakdata.conquery.sql.compiler.forms.StratificationTableFactory; +import com.bakdata.conquery.sql.compiler.ir.JoinMode; +import com.bakdata.conquery.sql.compiler.ir.ProjectionMode; +import com.bakdata.conquery.sql.compiler.ir.QueryStep; +import com.bakdata.conquery.sql.compiler.ir.QueryStepComposer; +import com.bakdata.conquery.sql.compiler.ir.QueryStepJoiner; +import com.bakdata.conquery.sql.compiler.ir.Selects; +import com.bakdata.conquery.sql.compiler.ir.SharedAliases; +import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; +import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; +import com.bakdata.conquery.sql.compiler.rendering.QueryStepRenderer; +import com.bakdata.conquery.sql.model.ResolvedQuery; +import com.bakdata.conquery.sql.model.form.FormResolution; +import com.bakdata.conquery.sql.model.form.ResolvedFormMode; +import com.bakdata.conquery.sql.model.form.ResolvedFormQuery; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Select; +import org.jooq.TableLike; +import org.jooq.conf.ParamType; + +/** Compiles fully resolved absolute, relative, and entity-date forms. */ +public final class FormSqlCompiler { + + private final QueryStepRenderer renderer; + private final ResolvedQueryStepCompiler stepCompiler = new ResolvedQueryStepCompiler(); + + public FormSqlCompiler(DSLContext dslContext) { + this.renderer = new QueryStepRenderer(dslContext); + } + + public CompiledQuery compile(ResolvedFormQuery query, CompilerDialect dialect) { + SqlNameGenerator names = new SqlNameGenerator(dialect.getNameMaxLength()); + QueryStep prerequisiteRoot = stepCompiler.compile( + query.prerequisite(), dialect, names, Optional.empty()).step(); + QueryStep formBase = createFormBase(prerequisiteRoot, query.mode(), dialect); + StratificationTableFactory tableFactory = new StratificationTableFactory(formBase, dialect); + QueryStep stratification = switch (query.mode()) { + case ResolvedFormMode.Absolute absolute -> tableFactory.createAbsoluteStratificationTable(absolute.resolutions(), false); + case ResolvedFormMode.Relative relative -> tableFactory.createRelativeStratificationTable(relative.settings(), false); + case ResolvedFormMode.EntityDate entityDate -> tableFactory.createAbsoluteStratificationTable(entityDate.resolutions(), false); + }; + + List features = query.features().stream() + .map(feature -> stepCompiler.compile(feature, dialect, names, Optional.of(stratification)).step()) + .toList(); + if (features.isEmpty()) { + throw new IllegalArgumentException("A form must contain at least one feature query"); + } + QueryStep joinedFeatures = QueryStepComposer.joinSteps( + features, + JoinMode.FULL_OUTER, + DateAggregationAction.BLOCK, + query.prerequisite().entitySchema(), + dialect, + names + ); + QueryStep finalStep = createFinalStep(query.mode(), stratification, joinedFeatures); + Select rendered = renderer.toSelectQuery(finalStep, dialect); + return new CompiledQuery(rendered.getSQL(ParamType.INLINED), CompiledQueryColumns.create( + query.prerequisite().entitySchema(), query.resultColumns(), rendered.getSelect())); + } + + private static QueryStep createFormBase(QueryStep prerequisite, ResolvedFormMode mode, CompilerDialect dialect) { + return switch (mode) { + case ResolvedFormMode.Absolute absolute -> createAbsoluteBase(prerequisite, absolute, dialect); + case ResolvedFormMode.Relative ignored -> extractIdsAndValidity(prerequisite, dialect); + case ResolvedFormMode.EntityDate entityDate -> createEntityDateBase( + extractIdsAndValidity(prerequisite, dialect), entityDate, dialect); + }; + } + + private static QueryStep createAbsoluteBase( + QueryStep prerequisite, + ResolvedFormMode.Absolute form, + CompilerDialect dialect + ) { + Selects prerequisiteSelects = prerequisite.getQualifiedSelects(); + ColumnDateRange bounds = dialect.dateRangeLiteral(form.bounds()) + .as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); + Selects selects = Selects.builder() + .ids(new SqlIdColumns(prerequisiteSelects.getIds().getPrimaryColumn())) + .stratificationDate(Optional.of(bounds)) + .build(); + return QueryStep.builder() + .cteName(FormCteStep.EXTRACT_IDS.getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(prerequisite.getCteName())) + .groupBy(selects.getIds().toFields()) + .predecessor(prerequisite) + .build(); + } + + private static QueryStep extractIdsAndValidity(QueryStep prerequisite, CompilerDialect dialect) { + Selects prerequisiteSelects = prerequisite.getQualifiedSelects(); + Optional validity = prerequisiteSelects.getValidityDate(); + Selects selects = Selects.builder() + .ids(new SqlIdColumns(prerequisiteSelects.getIds().getPrimaryColumn())) + .validityDate(validity) + .build(); + List> groupBy = Stream.concat( + Stream.of(prerequisiteSelects.getIds().getPrimaryColumn()), + validity.stream().flatMap(range -> range.toFields().stream()) + ).toList(); + Condition nonEmpty = validity.map(dialect::isNotEmptyDateRange).orElseGet(() -> falseCondition()); + return QueryStep.builder() + .cteName(FormCteStep.EXTRACT_IDS.getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(prerequisite.getCteName())) + .conditions(List.of(nonEmpty)) + .groupBy(groupBy) + .predecessor(prerequisite) + .build(); + } + + private static QueryStep createEntityDateBase( + QueryStep prerequisite, + ResolvedFormMode.EntityDate form, + CompilerDialect dialect + ) { + ColumnDateRange validity = prerequisite.getSelects().getValidityDate() + .orElseThrow(() -> new IllegalArgumentException("Entity-date forms require a validity date")); + QueryStep unnested = dialect.supportsSingleColumnRanges() + ? dialect.unnestDateRange(validity, prerequisite, FormCteStep.UNNEST_ENTITY_DATE_CTE.getSuffix()) + : prerequisite; + Selects unnestedSelects = unnested.getQualifiedSelects(); + ColumnDateRange entityDate = unnestedSelects.getValidityDate().orElseThrow(); + ColumnDateRange bounds = form.bounds() + .map(dialect::dateRangeLiteral) + .map(formRange -> dialect.intersection(formRange, entityDate)) + .orElse(entityDate) + .as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); + Selects selects = Selects.builder() + .ids(unnestedSelects.getIds()) + .stratificationDate(Optional.of(bounds)) + .build(); + List predecessors = unnested == prerequisite + ? List.of(prerequisite) + : List.of(prerequisite, unnested); + return QueryStep.builder() + .cteName(FormCteStep.OVERWRITE_BOUNDS.getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(unnested.getCteName())) + .predecessors(predecessors) + .build(); + } + + private static QueryStep createFinalStep( + ResolvedFormMode mode, + QueryStep stratification, + QueryStep joinedFeatures + ) { + if (stratification.getSelects().getStratificationDate().isEmpty() + || joinedFeatures.getSelects().getStratificationDate().isEmpty()) { + throw new IllegalArgumentException("Form stratification and features must contain stratification dates"); + } + Selects featureSelects = joinedFeatures.getQualifiedSelects(); + Selects stratificationSelects = stratification.getQualifiedSelects(); + Selects.SelectsBuilder selects = Selects.builder() + .ids(stratificationSelects.getIds().forFinalSelect(FormResolution.COMPLETE.name())) + .validityDate(featureSelects.getValidityDate()) + .stratificationDate(stratificationSelects.getStratificationDate()); + if (mode instanceof ResolvedFormMode.Relative) { + Field index = field(name(stratification.getCteName(), SharedAliases.INDEX.getAlias()), Integer.class); + Field scope = when(index.isNull().or(index.lessThan(0)), inline("FEATURE")) + .otherwise(inline("OUTCOME")) + .as(SharedAliases.OBSERVATION_SCOPE.getAlias()); + selects.sqlSelect(new FieldWrapper<>(scope)); + } + selects.sqlSelects(featureSelects.getSqlSelects()); + List inputs = List.of(stratification, joinedFeatures); + TableLike joined = QueryStepJoiner.join(inputs, JoinMode.LEFT); + return QueryStep.builder() + .cteName(null) + .projectionMode(ProjectionMode.AGGREGATED) + .selects(selects.build()) + .fromTable(joined) + .predecessors(inputs) + .build(); + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java index 3f8aeed0ac..b053e9b154 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/CompilerDialect.java @@ -13,6 +13,7 @@ import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.compiler.forms.StratificationFunctions; import com.bakdata.conquery.sql.model.range.DateRange; import org.jooq.Condition; import org.jooq.DataType; @@ -101,6 +102,24 @@ default ColumnDateRange conditionalUnboundedDateRange(Condition condition) { ); } + /** Form stratification expressions for this database. */ + default StratificationFunctions stratificationFunctions() { + throw new UnsupportedOperationException("Form stratification is not implemented by this dialect"); + } + + /** Intersect two half-open date ranges. */ + default ColumnDateRange intersection(ColumnDateRange left, ColumnDateRange right) { + return ColumnDateRange.of( + DSL.greatest(left.getStart(), right.getStart()), + DSL.least(left.getEnd(), right.getEnd()) + ); + } + + /** Test whether a date range contains a concrete bound. */ + default Condition isNotEmptyDateRange(ColumnDateRange range) { + return range.getStart().notEqual(minimumDate()).or(range.getEnd().notEqual(maximumDate())); + } + /** Regex fragment matching any number of characters in this dialect. */ default String regexAnyCharacters() { return ".*"; diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/clickhouse/ClickhouseCompilerDialect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/clickhouse/ClickhouseCompilerDialect.java index 4d2f3ae270..cc74e12dbd 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/clickhouse/ClickhouseCompilerDialect.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/clickhouse/ClickhouseCompilerDialect.java @@ -26,6 +26,11 @@ /** ClickHouse SQL capabilities used by the framework-neutral compiler. */ public class ClickhouseCompilerDialect implements CompilerDialect { + @Override + public ClickhouseStratificationFunctions stratificationFunctions() { + return new ClickhouseStratificationFunctions(this); + } + public static final int MIN_DATE_VALUE = -25567; public static final int MAX_DATE_VALUE = 24855; @@ -44,6 +49,13 @@ public Field maximumDate() { return field("toDate32({0})", Date.class, MAX_DATE_VALUE); } + @Override + public Condition isNotEmptyDateRange(ColumnDateRange range) { + Condition startNotMin = range.getStart().notEqual(minimumDate()); + Condition endNotMax = range.getEnd().notEqual(maximumDate()); + return condition(startNotMin.and(endNotMax).neg()); + } + @Override public Field anyValue(Field value) { return DSL.anyValue(value); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/clickhouse/ClickhouseStratificationFunctions.java similarity index 84% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/clickhouse/ClickhouseStratificationFunctions.java index 17a6cab018..3097edfd8c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/clickhouse/ClickhouseStratificationFunctions.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.dialect.clickhouse; +package com.bakdata.conquery.sql.compiler.dialect.clickhouse; import static com.bakdata.conquery.sql.compiler.dialect.Interval.MONTHS_PER_QUARTER; import static com.bakdata.conquery.sql.compiler.ir.form.FormConstants.SERIES_INDEX; @@ -8,12 +8,11 @@ import java.sql.Date; import java.time.temporal.ChronoUnit; -import com.bakdata.conquery.apiv1.query.TemporalSamplerFactory; import com.bakdata.conquery.sql.compiler.dialect.Interval; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.compiler.forms.StratificationFunctions; import com.bakdata.conquery.sql.compiler.ir.form.Offset; -import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.model.form.FormIndexSelector; import lombok.Getter; import lombok.RequiredArgsConstructor; import org.jooq.Field; @@ -25,7 +24,7 @@ @RequiredArgsConstructor public class ClickhouseStratificationFunctions extends StratificationFunctions { - private final SqlFunctionProvider functionProvider; + private final ClickhouseCompilerDialect dialect; private static Field addMonths(Field yearStart, Field amount) { return function("addMonths", Date.class, yearStart, amount); @@ -46,7 +45,7 @@ public Field lower(ColumnDateRange dateRange) { @Override protected Field inclusiveUpper(ColumnDateRange dateRange) { - return functionProvider.addDays(exclusiveUpper(dateRange), inline(-1)); + return dialect.addDays(exclusiveUpper(dateRange), inline(-1)); } @Override @@ -126,19 +125,19 @@ public Table generateIntSeries(int start, int end) { } @Override - public Field indexSelectorField(TemporalSamplerFactory indexSelector, ColumnDateRange validityDate) { + public Field indexSelectorField(FormIndexSelector indexSelector, ColumnDateRange validityDate) { return switch (indexSelector) { case EARLIEST -> min(validityDate.getStart()); case LATEST -> max(inclusiveUpper(validityDate)); case RANDOM -> { // we calculate a random int which is in range of the date distance between upper and lower bound - Field dateDistanceInDays = functionProvider.dateDistance(ChronoUnit.DAYS, validityDate.getStart(), validityDate.getEnd()); + Field dateDistanceInDays = dialect.dateDistance(ChronoUnit.DAYS, validityDate.getStart(), validityDate.getEnd()); Field randomAmountOfDays = function("RAND", Double.class).times(dateDistanceInDays); - Field flooredAsInt = functionProvider.cast(floor(randomAmountOfDays), SQLDataType.INTEGER); + Field flooredAsInt = dialect.cast(floor(randomAmountOfDays), SQLDataType.INTEGER); // then we add this random amount (of days) to the start date - Field randomDateInRange = functionProvider.addDays(lower(validityDate), flooredAsInt); + Field randomDateInRange = dialect.addDays(lower(validityDate), flooredAsInt); // finally, we handle multiple ranges by randomizing which range we use to select a random date from - yield functionProvider.random(randomDateInRange); + yield dialect.random(randomDateInRange); } }; } @@ -168,9 +167,9 @@ private Field calcDate(Field start, Interval interval, Offset offset } private Field getQuartersInMonths(Field date, Offset offset) { - Field quarterExpression = functionProvider.yearQuarter(date); + Field quarterExpression = dialect.yearQuarter(date); Field rightMostCharacter = function("RIGHT", String.class, quarterExpression, inline(1)); - Field amountOfQuarters = functionProvider.cast(rightMostCharacter, SQLDataType.INTEGER) + Field amountOfQuarters = dialect.cast(rightMostCharacter, SQLDataType.INTEGER) .plus(offset.getOffset()); return amountOfQuarters.times(MONTHS_PER_QUARTER); } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/hana/HanaCompilerDialect.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/hana/HanaCompilerDialect.java index 32a8b4e4aa..ce1083f024 100644 --- a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/hana/HanaCompilerDialect.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/hana/HanaCompilerDialect.java @@ -25,6 +25,11 @@ /** HANA SQL capabilities used by the framework-neutral compiler. */ public class HanaCompilerDialect implements CompilerDialect { + @Override + public HanaStratificationFunctions stratificationFunctions() { + return new HanaStratificationFunctions(this); + } + public static final String MAX_DATE_VALUE = "9999-12-31"; public static final String MIN_DATE_VALUE = "0001-01-01"; public static final String DATERANGE_SEPARATOR = "/"; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/hana/HanaStratificationFunctions.java similarity index 87% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/hana/HanaStratificationFunctions.java index 7ef2a9e12b..3bc2900bc8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/dialect/hana/HanaStratificationFunctions.java @@ -1,15 +1,15 @@ -package com.bakdata.conquery.sql.conversion.dialect.hana; +package com.bakdata.conquery.sql.compiler.dialect.hana; import static com.bakdata.conquery.sql.compiler.dialect.Interval.MONTHS_PER_QUARTER; import java.sql.Date; import java.time.temporal.ChronoUnit; -import com.bakdata.conquery.apiv1.query.TemporalSamplerFactory; +import com.bakdata.conquery.sql.compiler.forms.StratificationFunctions; import com.bakdata.conquery.sql.compiler.dialect.Interval; import com.bakdata.conquery.sql.compiler.ir.form.Offset; -import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; +import com.bakdata.conquery.sql.model.form.FormIndexSelector; import lombok.Getter; import lombok.RequiredArgsConstructor; import org.jooq.Field; @@ -30,7 +30,7 @@ public class HanaStratificationFunctions extends StratificationFunctions { */ private static final String GENERATED_PERIOD_END = "GENERATED_PERIOD_END"; - private final HanaSqlFunctionProvider functionProvider; + private final HanaCompilerDialect dialect; @Override public Field lower(ColumnDateRange dateRange) { @@ -40,7 +40,7 @@ public Field lower(ColumnDateRange dateRange) { @Override protected Field inclusiveUpper(ColumnDateRange dateRange) { - return functionProvider.addDays(exclusiveUpper(dateRange), DSL.inline(-1)); + return dialect.addDays(exclusiveUpper(dateRange), DSL.inline(-1)); } @Override @@ -129,19 +129,19 @@ public Table generateIntSeries(int start, int end) { } @Override - public Field indexSelectorField(TemporalSamplerFactory indexSelector, ColumnDateRange validityDate) { + public Field indexSelectorField(FormIndexSelector indexSelector, ColumnDateRange validityDate) { return switch (indexSelector) { case EARLIEST -> DSL.min(validityDate.getStart()); case LATEST -> DSL.max(inclusiveUpper(validityDate)); case RANDOM -> { // we calculate a random int which is in range of the date distance between upper and lower bound - Field dateDistanceInDays = functionProvider.dateDistance(ChronoUnit.DAYS, validityDate.getStart(), validityDate.getEnd()); + Field dateDistanceInDays = dialect.dateDistance(ChronoUnit.DAYS, validityDate.getStart(), validityDate.getEnd()); Field randomAmountOfDays = DSL.function("RAND", Double.class).times(dateDistanceInDays); - Field flooredAsInt = functionProvider.cast(DSL.floor(randomAmountOfDays), SQLDataType.INTEGER); + Field flooredAsInt = dialect.cast(DSL.floor(randomAmountOfDays), SQLDataType.INTEGER); // then we add this random amount (of days) to the start date - Field randomDateInRange = functionProvider.addDays(lower(validityDate), flooredAsInt); + Field randomDateInRange = dialect.addDays(lower(validityDate), flooredAsInt); // finally, we handle multiple ranges by randomizing which range we use to select a random date from - yield functionProvider.random(randomDateInRange); + yield dialect.random(randomDateInRange); } }; } @@ -189,9 +189,9 @@ private static Field jumpToYearStart(Field date) { } private Field getQuartersInMonths(Field date, Offset offset) { - Field quarterExpression = functionProvider.yearQuarter(date); + Field quarterExpression = dialect.yearQuarter(date); Field rightMostCharacter = DSL.function("RIGHT", String.class, quarterExpression, DSL.inline(1)); - Field amountOfQuarters = functionProvider.cast(rightMostCharacter, SQLDataType.INTEGER) + Field amountOfQuarters = dialect.cast(rightMostCharacter, SQLDataType.INTEGER) .plus(offset.getOffset()); return amountOfQuarters.times(MONTHS_PER_QUARTER); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/forms/AbsoluteStratification.java similarity index 85% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/forms/AbsoluteStratification.java index e7eab81037..6da5d72478 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/forms/AbsoluteStratification.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.forms; +package com.bakdata.conquery.sql.compiler.forms; import static org.jooq.impl.DSL.*; @@ -7,16 +7,14 @@ import java.util.Optional; import java.util.stream.Stream; -import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; -import com.bakdata.conquery.models.forms.util.Resolution; import com.bakdata.conquery.sql.compiler.ir.SharedAliases; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; -import com.google.common.base.Preconditions; +import com.bakdata.conquery.sql.model.form.FormResolution; +import com.bakdata.conquery.sql.model.form.ResolutionAndAlignment; import lombok.RequiredArgsConstructor; import org.jooq.Condition; import org.jooq.Field; @@ -32,7 +30,7 @@ class AbsoluteStratification { private final QueryStep baseStep; private final StratificationFunctions stratificationFunctions; - public QueryStep createStratificationTable(List resolutionAndAlignments, ConversionContext context) { + public QueryStep createStratificationTable(List resolutionAndAlignments, boolean negate) { QueryStep intSeriesStep = createIntSeriesStep(); QueryStep indexStartStep = createIndexStartStep(); @@ -42,7 +40,7 @@ public QueryStep createStratificationTable(List predecessors = List.of(baseStep, intSeriesStep, indexStartStep); - return StratificationTableFactory.unionResolutionTables(resolutionTables, predecessors, context); + return StratificationTableFactory.unionResolutionTables(resolutionTables, predecessors, negate); } private QueryStep createIntSeriesStep() { @@ -70,7 +68,9 @@ private QueryStep createIntSeriesStep() { private QueryStep createIndexStartStep() { Selects baseStepSelects = baseStep.getQualifiedSelects(); - Preconditions.checkArgument(baseStepSelects.getStratificationDate().isPresent(), "The base step must have a stratification date set"); + if (baseStepSelects.getStratificationDate().isEmpty()) { + throw new IllegalArgumentException("The base step must have a stratification date set"); + } ColumnDateRange bounds = baseStepSelects.getStratificationDate().get(); Field indexStart = stratificationFunctions.absoluteIndexStartDate(bounds).as(SharedAliases.INDEX_START.getAlias()); @@ -104,7 +104,7 @@ private QueryStep createIndexStartStep() { .build(); } - private QueryStep createResolutionTable(QueryStep indexStartStep, ExportForm.ResolutionAndAlignment resolutionAndAlignment) { + private QueryStep createResolutionTable(QueryStep indexStartStep, ResolutionAndAlignment resolutionAndAlignment) { return switch (resolutionAndAlignment.getResolution()) { case COMPLETE -> createCompleteTable(); case YEARS, QUARTERS, DAYS -> createIntervalTable(indexStartStep, resolutionAndAlignment); @@ -118,7 +118,7 @@ private QueryStep createCompleteTable() { // complete range shall have a null index because it spans the complete range, but we set it to 1 to ensure we can join tables on index, // because a condition involving null in a join (e.g., null = some_value or null = null) always evaluates to false Field index = field(inline(1)).as(SharedAliases.INDEX.getAlias()); - SqlIdColumns ids = baseStepSelects.getIds().withStratification(Resolution.COMPLETE.name(), index); + SqlIdColumns ids = baseStepSelects.getIds().withStratification(FormResolution.COMPLETE.name(), index); ColumnDateRange completeRange = baseStepSelects.getStratificationDate().get(); @@ -134,10 +134,12 @@ private QueryStep createCompleteTable() { .build(); } - private QueryStep createIntervalTable(QueryStep indexStartStep, ExportForm.ResolutionAndAlignment resolutionAndAlignment) { + private QueryStep createIntervalTable(QueryStep indexStartStep, ResolutionAndAlignment resolutionAndAlignment) { QueryStep countsCte = createCountsCte(indexStartStep, resolutionAndAlignment); - Preconditions.checkArgument(countsCte.getSelects().getStratificationDate().isPresent(), "The countsCte must have a stratification date set"); + if (countsCte.getSelects().getStratificationDate().isEmpty()) { + throw new IllegalArgumentException("The countsCte must have a stratification date set"); + } Selects countsCteSelects = countsCte.getQualifiedSelects(); ColumnDateRange stratificationRange = stratificationFunctions.createStratificationRange( @@ -165,10 +167,12 @@ private QueryStep createIntervalTable(QueryStep indexStartStep, ExportForm.Resol .build(); } - private QueryStep createCountsCte(QueryStep indexStartStep, ExportForm.ResolutionAndAlignment resolutionAndAlignment) { + private QueryStep createCountsCte(QueryStep indexStartStep, ResolutionAndAlignment resolutionAndAlignment) { Selects indexStartSelects = indexStartStep.getQualifiedSelects(); - Preconditions.checkArgument(indexStartSelects.getStratificationDate().isPresent(), "The indexStartStep must have a stratification date set"); + if (indexStartSelects.getStratificationDate().isEmpty()) { + throw new IllegalArgumentException("The indexStartStep must have a stratification date set"); + } Field resolutionWindowCount = stratificationFunctions.calculateResolutionWindowCount( resolutionAndAlignment, diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/forms/CombinationNotSupportedException.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/forms/CombinationNotSupportedException.java new file mode 100644 index 0000000000..5b4a7bb835 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/forms/CombinationNotSupportedException.java @@ -0,0 +1,25 @@ +package com.bakdata.conquery.sql.compiler.forms; + +import com.bakdata.conquery.sql.model.form.FormCalendarUnit; +import com.bakdata.conquery.sql.model.form.FormIndexPlacement; +import com.bakdata.conquery.sql.model.form.FormResolution; +import com.bakdata.conquery.sql.model.form.ResolutionAndAlignment; + +class CombinationNotSupportedException extends RuntimeException { + + public CombinationNotSupportedException(ResolutionAndAlignment resolutionAndAlignment) { + super("Alignment %s does not fit the resolution %s.".formatted( + resolutionAndAlignment.getAlignment(), + resolutionAndAlignment.getResolution() + )); + } + + public CombinationNotSupportedException(FormIndexPlacement indexPlacement, FormCalendarUnit timeUnit) { + super("Combination of index placement %s and time unit %s not supported".formatted(indexPlacement, timeUnit)); + } + + public CombinationNotSupportedException(FormCalendarUnit timeUnit, FormResolution resolution) { + super("Combination of time unit %s and resolution %s not supported".formatted(timeUnit, resolution)); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/forms/FormCteStep.java similarity index 83% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/forms/FormCteStep.java index 76fab89fa0..511f31811e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/forms/FormCteStep.java @@ -1,6 +1,6 @@ -package com.bakdata.conquery.sql.conversion.forms; +package com.bakdata.conquery.sql.compiler.forms; -import com.bakdata.conquery.models.forms.util.Resolution; +import com.bakdata.conquery.sql.model.form.FormResolution; import com.bakdata.conquery.sql.compiler.ir.CteStep; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -35,7 +35,7 @@ public enum FormCteStep implements CteStep { private final String suffix; - public static FormCteStep countsCte(Resolution resolution) { + public static FormCteStep countsCte(FormResolution resolution) { return switch (resolution) { case COMPLETE -> throw new UnsupportedOperationException("COMPLETE resolution does not require a counts CTE"); case YEARS -> YEAR_COUNTS; @@ -44,7 +44,7 @@ public static FormCteStep countsCte(Resolution resolution) { }; } - public static FormCteStep stratificationCte(Resolution resolution) { + public static FormCteStep stratificationCte(FormResolution resolution) { return switch (resolution) { case COMPLETE -> FormCteStep.COMPLETE; case YEARS -> FormCteStep.YEARS; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/forms/RelativeStratification.java similarity index 75% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/forms/RelativeStratification.java index b3c4423053..e28bfeb5c2 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/forms/RelativeStratification.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.forms; +package com.bakdata.conquery.sql.compiler.forms; import static com.bakdata.conquery.sql.compiler.ir.form.FormConstants.INDEX_SELECTOR; import static com.bakdata.conquery.sql.compiler.ir.form.FormConstants.INDEX_START_NEGATIVE; @@ -11,25 +11,20 @@ import java.util.Optional; import java.util.stream.Stream; -import com.bakdata.conquery.apiv1.forms.IndexPlacement; -import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; -import com.bakdata.conquery.apiv1.query.TemporalSamplerFactory; -import com.bakdata.conquery.models.common.Range; -import com.bakdata.conquery.models.forms.managed.RelativeFormQuery; -import com.bakdata.conquery.models.forms.util.CalendarUnit; -import com.bakdata.conquery.models.forms.util.Resolution; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.compiler.ir.form.FormConstants; import com.bakdata.conquery.sql.compiler.ir.form.Offset; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.compiler.dialect.Interval; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.QueryStep; import com.bakdata.conquery.sql.compiler.ir.Selects; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; import com.bakdata.conquery.sql.compiler.ir.select.FieldWrapper; -import com.google.common.base.Preconditions; +import com.bakdata.conquery.sql.model.form.FormCalendarUnit; +import com.bakdata.conquery.sql.model.form.FormResolution; +import com.bakdata.conquery.sql.model.form.RelativeFormSettings; +import com.bakdata.conquery.sql.model.form.ResolutionAndAlignment; import lombok.RequiredArgsConstructor; import org.jooq.Field; import org.jooq.Record; @@ -41,25 +36,29 @@ class RelativeStratification { private final QueryStep baseStep; private final StratificationFunctions stratificationFunctions; - private final SqlFunctionProvider functionProvider; + private final CompilerDialect dialect; - public QueryStep createRelativeStratificationTable(RelativeFormQuery form, ConversionContext context) { + public QueryStep createRelativeStratificationTable(RelativeFormSettings form, boolean negate) { //TODO why does this not have a conversion context? // we want to create the stratification for each distinct validity date range of an entity, // so we first need to unnest the validity date in case it is a multirange - Preconditions.checkArgument(baseStep.getSelects().getValidityDate().isPresent(), "Base step must contain a validity date"); + if (baseStep.getSelects().getValidityDate().isEmpty()) { + throw new IllegalArgumentException("Base step must contain a validity date"); + } String unnestCteName = FormCteStep.UNNEST_DATES.getSuffix(); - QueryStep withUnnestedValidityDate = functionProvider.unnestDaterange(baseStep.getSelects().getValidityDate().get(), baseStep, unnestCteName); + QueryStep withUnnestedValidityDate = dialect.supportsSingleColumnRanges() + ? dialect.unnestDateRange(baseStep.getSelects().getValidityDate().get(), baseStep, unnestCteName) + : baseStep; QueryStep indexSelectorStep = createIndexSelectorStep(form, withUnnestedValidityDate); QueryStep indexStartStep = createIndexStartStep(form, indexSelectorStep); QueryStep totalBoundsStep = createTotalBoundsStep(form, indexStartStep); List tables = form.getResolutionsAndAlignmentMap().stream() - .map(ExportForm.ResolutionAndAlignment::getResolution) - .map(resolution -> createResolutionTable(totalBoundsStep, resolution, form, context)) + .map(ResolutionAndAlignment::getResolution) + .map(resolution -> createResolutionTable(totalBoundsStep, resolution, form, negate)) .toList(); List predecessors = new ArrayList<>(); @@ -69,13 +68,13 @@ public QueryStep createRelativeStratificationTable(RelativeFormQuery form, Conve } predecessors.addAll(List.of(indexSelectorStep, indexStartStep, totalBoundsStep)); - return StratificationTableFactory.unionResolutionTables(tables, predecessors, context); + return StratificationTableFactory.unionResolutionTables(tables, predecessors, negate); } /** - * Creates {@link QueryStep} containing the date select for the corresponding {@link TemporalSamplerFactory} of the relative form. + * Creates a {@link QueryStep} containing the selected index date of the relative form. */ - private QueryStep createIndexSelectorStep(RelativeFormQuery form, QueryStep prerequisite) { + private QueryStep createIndexSelectorStep(RelativeFormSettings form, QueryStep prerequisite) { Selects predecessorSelects = prerequisite.getQualifiedSelects(); ColumnDateRange validityDate = predecessorSelects.getValidityDate() @@ -98,9 +97,9 @@ private QueryStep createIndexSelectorStep(RelativeFormQuery form, QueryStep prer /** * Creates {@link QueryStep} containing the start date selects ({@link FormConstants#INDEX_START_POSITIVE} and {@link FormConstants#INDEX_START_NEGATIVE}) - * from where the feature and/or outcome ranges of the relative form start. Their placement depends on the relative forms {@link IndexPlacement}. + * from where the feature and/or outcome ranges of the relative form start. */ - private QueryStep createIndexStartStep(RelativeFormQuery form, QueryStep indexSelectorStep) { + private QueryStep createIndexStartStep(RelativeFormSettings form, QueryStep indexSelectorStep) { List> indexStartFields = stratificationFunctions.indexStartFields(form.getIndexPlacement(), form.getTimeUnit()).stream() .map(FieldWrapper::new) @@ -122,13 +121,13 @@ private QueryStep createIndexStartStep(RelativeFormQuery form, QueryStep indexSe /** * Creates a {@link QueryStep} containing the minimum and maximum stratification date for each entity. */ - private QueryStep createTotalBoundsStep(RelativeFormQuery form, QueryStep indexStartStep) { + private QueryStep createTotalBoundsStep(RelativeFormSettings form, QueryStep indexStartStep) { - Interval interval = getInterval(form.getTimeUnit(), Resolution.COMPLETE); - Range intRange = toGenerateSeriesBounds(form, Resolution.COMPLETE); + Interval interval = getInterval(form.getTimeUnit(), FormResolution.COMPLETE); + IntRange intRange = toGenerateSeriesBounds(form, FormResolution.COMPLETE); - Field minStratificationDate = stratificationFunctions.shiftByInterval(INDEX_START_NEGATIVE, interval, DSL.inline(intRange.getMin()), Offset.NONE); - Field maxStratificationDate = stratificationFunctions.shiftByInterval(INDEX_START_POSITIVE, interval, DSL.inline(intRange.getMax()), Offset.NONE); + Field minStratificationDate = stratificationFunctions.shiftByInterval(INDEX_START_NEGATIVE, interval, DSL.inline(intRange.min()), Offset.NONE); + Field maxStratificationDate = stratificationFunctions.shiftByInterval(INDEX_START_POSITIVE, interval, DSL.inline(intRange.max()), Offset.NONE); ColumnDateRange minAndMaxStratificationDate = stratificationFunctions.ofStartAndEnd(minStratificationDate, maxStratificationDate) .as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); @@ -145,18 +144,18 @@ private QueryStep createTotalBoundsStep(RelativeFormQuery form, QueryStep indexS .build(); } - private QueryStep createResolutionTable(QueryStep indexStartStep, Resolution resolution, RelativeFormQuery form, ConversionContext context) { + private QueryStep createResolutionTable(QueryStep indexStartStep, FormResolution resolution, RelativeFormSettings form, boolean negate) { return switch (resolution) { - case COMPLETE -> createCompleteTable(indexStartStep, form, context); - case YEARS, QUARTERS, DAYS -> createIntervalTable(indexStartStep, resolution, form, context); + case COMPLETE -> createCompleteTable(indexStartStep, form, negate); + case YEARS, QUARTERS, DAYS -> createIntervalTable(indexStartStep, resolution, form, negate); }; } - private QueryStep createCompleteTable(QueryStep totalBoundsStep, RelativeFormQuery form, ConversionContext context) { + private QueryStep createCompleteTable(QueryStep totalBoundsStep, RelativeFormSettings form, boolean negate) { Selects predecessorSelects = totalBoundsStep.getQualifiedSelects(); - Interval interval = getInterval(form.getTimeUnit(), Resolution.COMPLETE); - Range intRange = toGenerateSeriesBounds(form, Resolution.COMPLETE); + Interval interval = getInterval(form.getTimeUnit(), FormResolution.COMPLETE); + IntRange intRange = toGenerateSeriesBounds(form, FormResolution.COMPLETE); QueryStep featureTable = form.getTimeCountBefore() > 0 ? createCompleteFeatureTable(predecessorSelects, interval, intRange, totalBoundsStep) : null; QueryStep outcomeTable = form.getTimeCountAfter() > 0 ? createCompleteOutcomeTable(predecessorSelects, interval, intRange, totalBoundsStep) : null; @@ -165,31 +164,31 @@ private QueryStep createCompleteTable(QueryStep totalBoundsStep, RelativeFormQue Stream.concat(Stream.ofNullable(outcomeTable), Stream.ofNullable(featureTable)).toList(), FormCteStep.COMPLETE.getSuffix(), Collections.emptyList(), - context.isNegation() + negate ); } - private QueryStep createCompleteFeatureTable(Selects predecessorSelects, Interval interval, Range intRange, QueryStep totalBoundsStep) { + private QueryStep createCompleteFeatureTable(Selects predecessorSelects, Interval interval, IntRange intRange, QueryStep totalBoundsStep) { Field featureIndex = DSL.field(DSL.inline(-1)).as(SharedAliases.INDEX.getAlias()); - SqlIdColumns featureIds = predecessorSelects.getIds().withStratification(Resolution.COMPLETE.name(), featureIndex, INDEX_SELECTOR); - Field rangeStart = stratificationFunctions.shiftByInterval(INDEX_START_NEGATIVE, interval, DSL.inline(intRange.getMin()), Offset.NONE); + SqlIdColumns featureIds = predecessorSelects.getIds().withStratification(FormResolution.COMPLETE.name(), featureIndex, INDEX_SELECTOR); + Field rangeStart = stratificationFunctions.shiftByInterval(INDEX_START_NEGATIVE, interval, DSL.inline(intRange.min()), Offset.NONE); return createIntervalStep(featureIds, rangeStart, INDEX_START_NEGATIVE, Optional.empty(), totalBoundsStep); } - private QueryStep createCompleteOutcomeTable(Selects predecessorSelects, Interval interval, Range intRange, QueryStep totalBoundsStep) { + private QueryStep createCompleteOutcomeTable(Selects predecessorSelects, Interval interval, IntRange intRange, QueryStep totalBoundsStep) { Field outcomeIndex = DSL.field(DSL.inline(1)).as(SharedAliases.INDEX.getAlias()); - SqlIdColumns outcomeIds = predecessorSelects.getIds().withStratification(Resolution.COMPLETE.name(), outcomeIndex, INDEX_SELECTOR); - Field rangeEnd = stratificationFunctions.shiftByInterval(INDEX_START_POSITIVE, interval, DSL.inline(intRange.getMax()), Offset.NONE); + SqlIdColumns outcomeIds = predecessorSelects.getIds().withStratification(FormResolution.COMPLETE.name(), outcomeIndex, INDEX_SELECTOR); + Field rangeEnd = stratificationFunctions.shiftByInterval(INDEX_START_POSITIVE, interval, DSL.inline(intRange.max()), Offset.NONE); return createIntervalStep(outcomeIds, INDEX_START_POSITIVE, rangeEnd, Optional.empty(), totalBoundsStep); } - private QueryStep createIntervalTable(QueryStep totalBoundsStep, Resolution resolution, RelativeFormQuery form, ConversionContext context) { + private QueryStep createIntervalTable(QueryStep totalBoundsStep, FormResolution resolution, RelativeFormSettings form, boolean negate) { Field seriesIndex = stratificationFunctions.intSeriesField(); Selects predecessorSelects = totalBoundsStep.getQualifiedSelects(); SqlIdColumns ids = predecessorSelects.getIds().withStratification(resolution.name(), seriesIndex, INDEX_SELECTOR); Interval interval = getInterval(form.getTimeUnit(), resolution); - Range bounds = toGenerateSeriesBounds(form, resolution); + IntRange bounds = toGenerateSeriesBounds(form, resolution); QueryStep timeBeforeStep = createFeatureTable(totalBoundsStep, interval, seriesIndex, bounds, ids); QueryStep timeAfterStep = createOutcomeTable(totalBoundsStep, interval, seriesIndex, bounds, ids); @@ -198,21 +197,21 @@ private QueryStep createIntervalTable(QueryStep totalBoundsStep, Resolution reso List.of(timeBeforeStep, timeAfterStep), FormCteStep.stratificationCte(resolution).getSuffix(), Collections.emptyList(), - context.isNegation() + negate ); } - private QueryStep createOutcomeTable(QueryStep totalBoundsStep, Interval interval, Field seriesIndex, Range bounds, SqlIdColumns ids) { + private QueryStep createOutcomeTable(QueryStep totalBoundsStep, Interval interval, Field seriesIndex, IntRange bounds, SqlIdColumns ids) { Field outcomeRangeStart = stratificationFunctions.shiftByInterval(INDEX_START_POSITIVE, interval, seriesIndex, Offset.MINUS_ONE); Field outcomeRangeEnd = stratificationFunctions.shiftByInterval(INDEX_START_POSITIVE, interval, seriesIndex, Offset.NONE); - Table outcomeSeries = stratificationFunctions.generateIntSeries(1, bounds.getMax()).as(SharedAliases.INDEX.getAlias()); + Table outcomeSeries = stratificationFunctions.generateIntSeries(1, bounds.max()).as(SharedAliases.INDEX.getAlias()); return createIntervalStep(ids, outcomeRangeStart, outcomeRangeEnd, Optional.of(outcomeSeries), totalBoundsStep); } - private QueryStep createFeatureTable(QueryStep totalBoundsStep, Interval interval, Field seriesIndex, Range bounds, SqlIdColumns ids) { + private QueryStep createFeatureTable(QueryStep totalBoundsStep, Interval interval, Field seriesIndex, IntRange bounds, SqlIdColumns ids) { Field featureRangeStart = stratificationFunctions.shiftByInterval(INDEX_START_NEGATIVE, interval, seriesIndex, Offset.NONE); Field featureRangeEnd = stratificationFunctions.shiftByInterval(INDEX_START_NEGATIVE, interval, seriesIndex, Offset.ONE); - Table featureSeries = stratificationFunctions.generateIntSeries(bounds.getMin(), -1).as(SharedAliases.INDEX.getAlias()); + Table featureSeries = stratificationFunctions.generateIntSeries(bounds.min(), -1).as(SharedAliases.INDEX.getAlias()); return createIntervalStep(ids, featureRangeStart, featureRangeEnd, Optional.of(featureSeries), totalBoundsStep); } @@ -223,11 +222,10 @@ private QueryStep createIntervalStep( Optional> seriesTable, QueryStep predecessor ) { - Preconditions.checkArgument( - predecessor.getSelects().getStratificationDate().isPresent(), - "Expecting %s to contain a stratification date representing the min and max stratification bounds" - ); - ColumnDateRange finalRange = functionProvider.intersection( + if (predecessor.getSelects().getStratificationDate().isEmpty()) { + throw new IllegalArgumentException("Expecting predecessor to contain the min and max stratification bounds"); + } + ColumnDateRange finalRange = dialect.intersection( stratificationFunctions.ofStartAndEnd(rangeStart, rangeEnd), predecessor.getQualifiedSelects().getStratificationDate().get() ) @@ -247,18 +245,18 @@ private QueryStep createIntervalStep( } /** - * Adjusts the {@link RelativeFormQuery#getTimeCountBefore()} && {@link RelativeFormQuery#getTimeCountAfter()} bounds, so they fit the SQL approach. + * Adjusts the before and after bounds so they fit the SQL approach. * Take time unit QUARTERS and Resolution YEARS as an example: If the time counts are not divisible by 4 (because 1 year == 4 quarters), we need to round * up for each starting year. 5 Quarters mean 2 years we have to consider when creating the stratification. */ - private static Range toGenerateSeriesBounds(RelativeFormQuery relativeForm, Resolution resolution) { + private static IntRange toGenerateSeriesBounds(RelativeFormSettings relativeForm, FormResolution resolution) { int timeCountBefore; int timeCountAfter; switch (relativeForm.getTimeUnit()) { case QUARTERS -> { - if (resolution == Resolution.YEARS) { + if (resolution == FormResolution.YEARS) { timeCountBefore = divideAndRoundUp(relativeForm.getTimeCountBefore(), 4); timeCountAfter = divideAndRoundUp(relativeForm.getTimeCountAfter(), 4); } @@ -287,7 +285,7 @@ private static Range toGenerateSeriesBounds(RelativeFormQuery relativeF default -> throw new CombinationNotSupportedException(relativeForm.getTimeUnit(), resolution); } - return Range.of( + return new IntRange( -timeCountBefore, timeCountAfter ); @@ -304,7 +302,7 @@ private static int divideAndRoundUp(int numerator, int denominator) { * @return The interval expression which will be multiplied by the {@link StratificationFunctions#intSeriesField()} and added to the * {@link SharedAliases#INDEX_START_NEGATIVE} or {@link SharedAliases#INDEX_START_POSITIVE}. */ - private static Interval getInterval(CalendarUnit timeUnit, Resolution resolution) { + private static Interval getInterval(FormCalendarUnit timeUnit, FormResolution resolution) { return switch (timeUnit) { case QUARTERS -> switch (resolution) { case COMPLETE, QUARTERS -> Interval.QUARTER_INTERVAL; @@ -320,4 +318,7 @@ private static Interval getInterval(CalendarUnit timeUnit, Resolution resolution }; } + private record IntRange(int min, int max) { + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/forms/StratificationFunctions.java similarity index 83% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/forms/StratificationFunctions.java index 4c96d70440..a6f1d33257 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/forms/StratificationFunctions.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.forms; +package com.bakdata.conquery.sql.compiler.forms; import static com.bakdata.conquery.sql.compiler.dialect.Interval.DAYS_PER_QUARTER; import static com.bakdata.conquery.sql.compiler.dialect.Interval.DAYS_PER_YEAR; @@ -21,17 +21,16 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import com.bakdata.conquery.ConqueryConstants; -import com.bakdata.conquery.apiv1.forms.IndexPlacement; -import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; -import com.bakdata.conquery.apiv1.query.TemporalSamplerFactory; -import com.bakdata.conquery.models.forms.util.CalendarUnit; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import com.bakdata.conquery.sql.compiler.ir.SharedAliases; import com.bakdata.conquery.sql.compiler.ir.form.Offset; import com.bakdata.conquery.sql.compiler.dialect.Interval; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; +import com.bakdata.conquery.sql.model.form.FormCalendarUnit; +import com.bakdata.conquery.sql.model.form.FormIndexPlacement; +import com.bakdata.conquery.sql.model.form.FormIndexSelector; +import com.bakdata.conquery.sql.model.form.ResolutionAndAlignment; import org.jooq.Condition; import org.jooq.Field; import org.jooq.Record; @@ -44,7 +43,7 @@ public ColumnDateRange ofStartAndEnd(Field start, Field end) { return ColumnDateRange.of(start, end); // needs to be overwritten for dialects that support single-column ranges } - protected abstract SqlFunctionProvider getFunctionProvider(); + protected abstract CompilerDialect getDialect(); /** * Extract the lower bounds from a given daterange. @@ -118,9 +117,9 @@ public ColumnDateRange ofStartAndEnd(Field start, Field end) { public abstract Table generateIntSeries(int start, int end); /** - * Generates a date field representing the {@link TemporalSamplerFactory} using the given validity date range. + * Generates a date field representing the selected index strategy using the given validity date range. */ - public abstract Field indexSelectorField(TemporalSamplerFactory indexSelector, ColumnDateRange validityDate); + public abstract Field indexSelectorField(FormIndexSelector indexSelector, ColumnDateRange validityDate); /** * Shift's a start date by an interval times an amount. The offset will we added to the amount before multiplying. @@ -128,9 +127,9 @@ public ColumnDateRange ofStartAndEnd(Field start, Field end) { public abstract Field shiftByInterval(Field startDate, Interval interval, Field amount, Offset offset); /** - * Generates the start and end field for the respective {@link IndexPlacement} and {@link CalendarUnit timeUnit}. + * Generates the start and end field for the respective index placement and time unit. */ - public List> indexStartFields(IndexPlacement indexPlacement, CalendarUnit timeUnit) { + public List> indexStartFields(FormIndexPlacement indexPlacement, FormCalendarUnit timeUnit) { Field positiveStart; Field negativeStart; @@ -162,7 +161,7 @@ public List> indexStartFields(IndexPlacement indexPlacement, Calenda negativeStart = INDEX_SELECTOR; } case NEUTRAL -> { - positiveStart = getFunctionProvider().addDays(INDEX_SELECTOR, DSL.inline(1)); + positiveStart = getDialect().addDays(INDEX_SELECTOR, DSL.inline(1)); negativeStart = INDEX_SELECTOR; } default -> throw new CombinationNotSupportedException(indexPlacement, timeUnit); @@ -183,8 +182,8 @@ public List> indexStartFields(IndexPlacement indexPlacement, Calenda * @return A {@link Field} representing the count of resolution windows. * @throws CombinationNotSupportedException if the combination of resolution and alignment is not supported. */ - public Field calculateResolutionWindowCount(ExportForm.ResolutionAndAlignment resolutionAndAlignment, ColumnDateRange bounds) { - SqlFunctionProvider functionProvider = getFunctionProvider(); + public Field calculateResolutionWindowCount(ResolutionAndAlignment resolutionAndAlignment, ColumnDateRange bounds) { + CompilerDialect functionProvider = getDialect(); return switch (resolutionAndAlignment.getResolution()) { case COMPLETE -> DSL.inline(1); case YEARS -> calculateResolutionWindowForYearResolution(resolutionAndAlignment, bounds, functionProvider); @@ -197,9 +196,9 @@ public Field calculateResolutionWindowCount(ExportForm.ResolutionAndAli /** * Determines the stratification range based on resolution and alignment parameters. The created stratification range is bound by the given range. */ - public ColumnDateRange createStratificationRange(ExportForm.ResolutionAndAlignment resolutionAndAlignment, ColumnDateRange bounds) { + public ColumnDateRange createStratificationRange(ResolutionAndAlignment resolutionAndAlignment, ColumnDateRange bounds) { - SqlFunctionProvider functionProvider = getFunctionProvider(); + CompilerDialect functionProvider = getDialect(); ColumnDateRange stratificationRange = switch (resolutionAndAlignment.getResolution()) { case COMPLETE -> bounds; @@ -212,7 +211,7 @@ public ColumnDateRange createStratificationRange(ExportForm.ResolutionAndAlignme } /** - * The index field for the corresponding resolution index {@link ConqueryConstants#CONTEXT_INDEX_INFO}. + * The index field for the corresponding resolution index. */ public Field index(SqlIdColumns ids, Optional stratificationBounds) { @@ -232,7 +231,7 @@ public Field index(SqlIdColumns ids, Optional stratifi * Generates a condition to limit the resolution window count in a query. * This method applies a check to ensure the series index does not exceed the maximum window count based on the given resolution and alignment. */ - public Condition stopOnMaxResolutionWindowCount(ExportForm.ResolutionAndAlignment resolutionAndAlignment) { + public Condition stopOnMaxResolutionWindowCount(ResolutionAndAlignment resolutionAndAlignment) { Field seriesIndex = intSeriesField(); return switch (resolutionAndAlignment.getResolution()) { case COMPLETE -> DSL.noCondition(); @@ -243,9 +242,9 @@ public Condition stopOnMaxResolutionWindowCount(ExportForm.ResolutionAndAlignmen } private Field calculateResolutionWindowForQuarterResolution( - ExportForm.ResolutionAndAlignment resolutionAndAlignment, + ResolutionAndAlignment resolutionAndAlignment, ColumnDateRange bounds, - SqlFunctionProvider functionProvider + CompilerDialect functionProvider ) { return switch (resolutionAndAlignment.getAlignment()) { case QUARTER -> functionProvider.dateDistance(ChronoUnit.MONTHS, QUARTER_START, QUARTER_END) @@ -260,9 +259,9 @@ private Field calculateResolutionWindowForQuarterResolution( } private Field calculateResolutionWindowForYearResolution( - ExportForm.ResolutionAndAlignment resolutionAndAlignment, + ResolutionAndAlignment resolutionAndAlignment, ColumnDateRange bounds, - SqlFunctionProvider functionProvider + CompilerDialect functionProvider ) { return switch (resolutionAndAlignment.getAlignment()) { case YEAR -> functionProvider.dateDistance(ChronoUnit.YEARS, YEAR_START, YEAR_END) @@ -277,7 +276,7 @@ private Field calculateResolutionWindowForYearResolution( }; } - private ColumnDateRange createStratificationRangeForQuarterResolution(ExportForm.ResolutionAndAlignment resolutionAndAlignment) { + private ColumnDateRange createStratificationRangeForQuarterResolution(ResolutionAndAlignment resolutionAndAlignment) { return switch (resolutionAndAlignment.getAlignment()) { case QUARTER -> calcRange(QUARTER_START, Interval.QUARTER_INTERVAL); case DAY -> calcRange(INDEX_START, Interval.NINETY_DAYS_INTERVAL); @@ -285,7 +284,7 @@ private ColumnDateRange createStratificationRangeForQuarterResolution(ExportForm }; } - private ColumnDateRange createStratificationRangeForYearResolution(ExportForm.ResolutionAndAlignment resolutionAndAlignment) { + private ColumnDateRange createStratificationRangeForYearResolution(ResolutionAndAlignment resolutionAndAlignment) { return switch (resolutionAndAlignment.getAlignment()) { case YEAR -> calcRange(YEAR_START, Interval.ONE_YEAR_INTERVAL); case QUARTER -> calcRange(QUARTER_START, Interval.ONE_YEAR_INTERVAL); @@ -294,7 +293,7 @@ private ColumnDateRange createStratificationRangeForYearResolution(ExportForm.Re }; } - private static Condition windowCountForQuarterResolution(ExportForm.ResolutionAndAlignment resolutionAndAlignment, Field seriesIndex) { + private static Condition windowCountForQuarterResolution(ResolutionAndAlignment resolutionAndAlignment, Field seriesIndex) { return switch (resolutionAndAlignment.getAlignment()) { case QUARTER -> seriesIndex.lessOrEqual(QUARTER_ALIGNED_COUNT); case DAY -> seriesIndex.lessOrEqual(DAY_ALIGNED_COUNT); @@ -302,7 +301,7 @@ private static Condition windowCountForQuarterResolution(ExportForm.ResolutionAn }; } - private static Condition windowCountForYearResolution(ExportForm.ResolutionAndAlignment resolutionAndAlignment, Field seriesIndex) { + private static Condition windowCountForYearResolution(ResolutionAndAlignment resolutionAndAlignment, Field seriesIndex) { return switch (resolutionAndAlignment.getAlignment()) { case YEAR -> seriesIndex.lessOrEqual(YEAR_ALIGNED_COUNT); case QUARTER -> seriesIndex.lessOrEqual(QUARTER_ALIGNED_COUNT); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/forms/StratificationTableFactory.java similarity index 56% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java rename to sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/forms/StratificationTableFactory.java index 81f80138ce..6a6b53ebbe 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/compiler/forms/StratificationTableFactory.java @@ -1,14 +1,12 @@ -package com.bakdata.conquery.sql.conversion.forms; +package com.bakdata.conquery.sql.compiler.forms; import java.util.List; import java.util.stream.Stream; -import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; -import com.bakdata.conquery.models.forms.managed.RelativeFormQuery; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.google.common.base.Preconditions; +import com.bakdata.conquery.sql.model.form.RelativeFormSettings; +import com.bakdata.conquery.sql.model.form.ResolutionAndAlignment; import lombok.AccessLevel; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -22,27 +20,29 @@ public class StratificationTableFactory { private final QueryStep baseStep; private final StratificationFunctions stratificationFunctions; - private final SqlFunctionProvider functionProvider; + private final CompilerDialect dialect; - public StratificationTableFactory(QueryStep baseStep, ConversionContext context) { + public StratificationTableFactory(QueryStep baseStep, CompilerDialect dialect) { this.baseStep = baseStep; - this.stratificationFunctions = context.getStratificationFunctions(); - this.functionProvider = context.getFunctionProvider(); + this.stratificationFunctions = dialect.stratificationFunctions(); + this.dialect = dialect; } - public QueryStep createRelativeStratificationTable(RelativeFormQuery form, ConversionContext context) { - RelativeStratification relativeStratification = new RelativeStratification(baseStep, stratificationFunctions, functionProvider); - return relativeStratification.createRelativeStratificationTable(form, context); + public QueryStep createRelativeStratificationTable(RelativeFormSettings form, boolean negate) { + RelativeStratification relativeStratification = new RelativeStratification(baseStep, stratificationFunctions, dialect); + return relativeStratification.createRelativeStratificationTable(form, negate); } - public QueryStep createAbsoluteStratificationTable(List resolutionAndAlignments, ConversionContext context) { + public QueryStep createAbsoluteStratificationTable(List resolutionAndAlignments, boolean negate) { AbsoluteStratification absoluteStratification = new AbsoluteStratification(baseStep, stratificationFunctions); - return absoluteStratification.createStratificationTable(resolutionAndAlignments, context); + return absoluteStratification.createStratificationTable(resolutionAndAlignments, negate); } - protected static QueryStep unionResolutionTables(List unionSteps, List predecessors, ConversionContext context) { + protected static QueryStep unionResolutionTables(List unionSteps, List predecessors, boolean negate) { - Preconditions.checkArgument(!unionSteps.isEmpty(), "Expecting at least 1 resolution table"); + if (unionSteps.isEmpty()) { + throw new IllegalArgumentException("Expecting at least 1 resolution table"); + } List withQualifiedSelects = unionSteps.stream() .map(queryStep -> QueryStep.builder() @@ -55,7 +55,7 @@ protected static QueryStep unionResolutionTables(List unionSteps, Lis withQualifiedSelects, FormCteStep.FULL_STRATIFICATION.getSuffix(), Stream.concat(predecessors.stream(), unionSteps.stream()).toList(), - context.isNegation() + negate ); } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/FormAlignment.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/FormAlignment.java new file mode 100644 index 0000000000..86287e7a96 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/FormAlignment.java @@ -0,0 +1,3 @@ +package com.bakdata.conquery.sql.model.form; + +public enum FormAlignment { NO_ALIGN, YEAR, QUARTER, DAY } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/FormCalendarUnit.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/FormCalendarUnit.java new file mode 100644 index 0000000000..bc347da30a --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/FormCalendarUnit.java @@ -0,0 +1,3 @@ +package com.bakdata.conquery.sql.model.form; + +public enum FormCalendarUnit { DAYS, QUARTERS, YEARS } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/FormIndexPlacement.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/FormIndexPlacement.java new file mode 100644 index 0000000000..c6d468aa3e --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/FormIndexPlacement.java @@ -0,0 +1,3 @@ +package com.bakdata.conquery.sql.model.form; + +public enum FormIndexPlacement { BEFORE, NEUTRAL, AFTER } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/FormIndexSelector.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/FormIndexSelector.java new file mode 100644 index 0000000000..9f94abd9e7 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/FormIndexSelector.java @@ -0,0 +1,3 @@ +package com.bakdata.conquery.sql.model.form; + +public enum FormIndexSelector { EARLIEST, LATEST, RANDOM } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/FormResolution.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/FormResolution.java new file mode 100644 index 0000000000..603cab60d6 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/FormResolution.java @@ -0,0 +1,3 @@ +package com.bakdata.conquery.sql.model.form; + +public enum FormResolution { COMPLETE, YEARS, QUARTERS, DAYS } diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/RelativeFormSettings.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/RelativeFormSettings.java new file mode 100644 index 0000000000..d51fdf3171 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/RelativeFormSettings.java @@ -0,0 +1,20 @@ +package com.bakdata.conquery.sql.model.form; + +import java.util.List; + +public record RelativeFormSettings( + FormIndexSelector indexSelector, + FormIndexPlacement indexPlacement, + int timeCountBefore, + int timeCountAfter, + FormCalendarUnit timeUnit, + List resolutions +) { + public RelativeFormSettings { resolutions = List.copyOf(resolutions); } + public FormIndexSelector getIndexSelector() { return indexSelector; } + public FormIndexPlacement getIndexPlacement() { return indexPlacement; } + public int getTimeCountBefore() { return timeCountBefore; } + public int getTimeCountAfter() { return timeCountAfter; } + public FormCalendarUnit getTimeUnit() { return timeUnit; } + public List getResolutionsAndAlignmentMap() { return resolutions; } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/ResolutionAndAlignment.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/ResolutionAndAlignment.java new file mode 100644 index 0000000000..5b1b4d4207 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/ResolutionAndAlignment.java @@ -0,0 +1,6 @@ +package com.bakdata.conquery.sql.model.form; + +public record ResolutionAndAlignment(FormResolution resolution, FormAlignment alignment) { + public FormResolution getResolution() { return resolution; } + public FormAlignment getAlignment() { return alignment; } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/ResolvedFormMode.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/ResolvedFormMode.java new file mode 100644 index 0000000000..8079c60be3 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/ResolvedFormMode.java @@ -0,0 +1,16 @@ +package com.bakdata.conquery.sql.model.form; + +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.sql.model.range.DateRange; + +public sealed interface ResolvedFormMode { + record Absolute(DateRange bounds, List resolutions) implements ResolvedFormMode { + public Absolute { resolutions = List.copyOf(resolutions); } + } + record Relative(RelativeFormSettings settings) implements ResolvedFormMode {} + record EntityDate(Optional bounds, List resolutions) implements ResolvedFormMode { + public EntityDate { resolutions = List.copyOf(resolutions); } + } +} diff --git a/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/ResolvedFormQuery.java b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/ResolvedFormQuery.java new file mode 100644 index 0000000000..6d1b9825e0 --- /dev/null +++ b/sql-connector/src/main/java/com/bakdata/conquery/sql/model/form/ResolvedFormQuery.java @@ -0,0 +1,18 @@ +package com.bakdata.conquery.sql.model.form; + +import java.util.List; + +import com.bakdata.conquery.sql.model.ResolvedQuery; +import com.bakdata.conquery.sql.model.result.ResultColumn; + +public record ResolvedFormQuery( + ResolvedQuery prerequisite, + List features, + ResolvedFormMode mode, + List resultColumns +) { + public ResolvedFormQuery { + features = List.copyOf(features); + resultColumns = List.copyOf(resultColumns); + } +} diff --git a/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/FormSqlCompilerTest.java b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/FormSqlCompilerTest.java new file mode 100644 index 0000000000..69978806a5 --- /dev/null +++ b/sql-connector/src/test/java/com/bakdata/conquery/sql/compiler/FormSqlCompilerTest.java @@ -0,0 +1,155 @@ +package com.bakdata.conquery.sql.compiler; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.LocalDate; +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.models.datasets.ColumnType; +import com.bakdata.conquery.models.query.DateAggregationAction; +import com.bakdata.conquery.sql.compiler.dialect.hana.HanaCompilerDialect; +import com.bakdata.conquery.sql.model.ResolvedQuery; +import com.bakdata.conquery.sql.model.form.FormAlignment; +import com.bakdata.conquery.sql.model.form.FormCalendarUnit; +import com.bakdata.conquery.sql.model.form.FormIndexPlacement; +import com.bakdata.conquery.sql.model.form.FormIndexSelector; +import com.bakdata.conquery.sql.model.form.FormResolution; +import com.bakdata.conquery.sql.model.form.RelativeFormSettings; +import com.bakdata.conquery.sql.model.form.ResolutionAndAlignment; +import com.bakdata.conquery.sql.model.form.ResolvedFormMode; +import com.bakdata.conquery.sql.model.form.ResolvedFormQuery; +import com.bakdata.conquery.sql.model.node.AllEntitiesNode; +import com.bakdata.conquery.sql.model.node.ConceptNode; +import com.bakdata.conquery.sql.model.operation.BuiltInSelects; +import com.bakdata.conquery.sql.model.range.DateRange; +import com.bakdata.conquery.sql.model.result.ResultColumn; +import com.bakdata.conquery.sql.model.result.ResultType; +import com.bakdata.conquery.sql.model.schema.EntitySchema; +import com.bakdata.conquery.sql.model.schema.ResolvedColumn; +import com.bakdata.conquery.sql.model.schema.ResolvedConnector; +import com.bakdata.conquery.sql.model.schema.ResolvedValidityDate; +import com.bakdata.conquery.sql.model.schema.SqlTable; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Test; + +class FormSqlCompilerTest { + + private static final SqlTable ENTITIES = SqlTable.of("entities", "catalog", "entities"); + private static final ResolvedColumn ENTITY_ID = new ResolvedColumn( + "entity-id", ENTITIES, "person_id", ColumnType.STRING, false); + private static final EntitySchema ENTITY_SCHEMA = new EntitySchema(ENTITY_ID); + + @Test + void shouldCompileAnAbsoluteFormThroughTheConnectorPipeline() { + ResolvedFormQuery form = new ResolvedFormQuery( + new ResolvedQuery(ENTITY_SCHEMA, new AllEntitiesNode(), false, List.of()), + List.of(feature()), + new ResolvedFormMode.Absolute( + new DateRange(Optional.of(LocalDate.of(2020, 1, 1)), Optional.of(LocalDate.of(2020, 12, 31))), + List.of(new ResolutionAndAlignment(FormResolution.COMPLETE, FormAlignment.NO_ALIGN))), + List.of( + new ResultColumn("resolution", ResultType.Primitive.STRING), + new ResultColumn("index", ResultType.Primitive.INTEGER), + new ResultColumn("range", ResultType.Primitive.DATE_RANGE), + new ResultColumn("feature", ResultType.Primitive.BOOLEAN)) + ); + + CompiledQuery compiled = new FormSqlCompiler(DSL.using(SQLDialect.DEFAULT)) + .compile(form, new HanaCompilerDialect()); + + assertTrue(compiled.sql().contains("full_stratification"), compiled.sql()); + assertTrue(compiled.sql().contains("left outer join"), compiled.sql()); + assertEquals(List.of("entity-id", "resolution", "index", "range", "feature"), + compiled.columns().stream().map(CompiledColumn::outputId).toList()); + } + + @Test + void shouldCompileRelativeAndEntityDateForms() { + ResolvedQuery datedPrerequisite = datedQuery("prerequisite", false); + List features = List.of(feature("feature"), feature("outcome")); + FormSqlCompiler compiler = new FormSqlCompiler(DSL.using(SQLDialect.DEFAULT)); + HanaCompilerDialect dialect = new HanaCompilerDialect(); + ResolvedFormQuery relative = new ResolvedFormQuery( + datedPrerequisite, + features, + new ResolvedFormMode.Relative(new RelativeFormSettings( + FormIndexSelector.EARLIEST, + FormIndexPlacement.NEUTRAL, + 1, + 1, + FormCalendarUnit.DAYS, + List.of(new ResolutionAndAlignment(FormResolution.COMPLETE, FormAlignment.NO_ALIGN)))), + formResults(true) + ); + ResolvedFormQuery entityDate = new ResolvedFormQuery( + datedPrerequisite, + features, + new ResolvedFormMode.EntityDate( + Optional.of(new DateRange(Optional.of(LocalDate.of(2020, 1, 1)), Optional.empty())), + List.of(new ResolutionAndAlignment(FormResolution.COMPLETE, FormAlignment.NO_ALIGN))), + formResults(false) + ); + + CompiledQuery relativeCompiled = compiler.compile(relative, dialect); + CompiledQuery entityDateCompiled = compiler.compile(entityDate, dialect); + + assertTrue(relativeCompiled.sql().contains("index_selector"), relativeCompiled.sql()); + assertTrue(relativeCompiled.sql().contains("\"scope\""), relativeCompiled.sql()); + assertTrue(entityDateCompiled.sql().contains("overwrite_bounds"), entityDateCompiled.sql()); + assertEquals(formResults(true).stream().map(ResultColumn::outputId).toList(), + relativeCompiled.columns().stream().skip(1).map(CompiledColumn::outputId).toList()); + assertEquals(formResults(false).stream().map(ResultColumn::outputId).toList(), + entityDateCompiled.columns().stream().skip(1).map(CompiledColumn::outputId).toList()); + } + + private static ResolvedQuery feature() { + return feature("feature"); + } + + private static ResolvedQuery feature(String name) { + return datedQuery(name, true); + } + + private static ResolvedQuery datedQuery(String name, boolean withSelect) { + SqlTable events = SqlTable.of("events", "catalog", "events"); + ResolvedConnector connector = new ResolvedConnector( + "claims", + events, + new ResolvedColumn("events.entity-id", events, "person_id", ColumnType.STRING, false), + Optional.empty(), + new ResolvedValidityDate.Point(new ResolvedColumn( + "events.date", events, "event_date", ColumnType.DATE, true)), + List.of(), + List.of(), + List.of() + ); + return new ResolvedQuery( + ENTITY_SCHEMA, + new ConceptNode(name, List.of(connector), + withSelect ? List.of(new BuiltInSelects.Exists(name)) : List.of(), + withSelect ? DateAggregationAction.BLOCK : DateAggregationAction.MERGE), + !withSelect, + withSelect ? List.of(new ResultColumn(name, ResultType.Primitive.BOOLEAN)) : List.of() + ); + } + + private static List formResults(boolean relative) { + java.util.ArrayList results = new java.util.ArrayList<>(List.of( + new ResultColumn("resolution", ResultType.Primitive.STRING), + new ResultColumn("index", ResultType.Primitive.INTEGER) + )); + if (relative) { + results.add(new ResultColumn("event-date", ResultType.Primitive.DATE)); + } + results.add(new ResultColumn("range", ResultType.Primitive.DATE_RANGE)); + if (relative) { + results.add(new ResultColumn("scope", ResultType.Primitive.STRING)); + } + results.add(new ResultColumn("feature", ResultType.Primitive.BOOLEAN)); + results.add(new ResultColumn("outcome", ResultType.Primitive.BOOLEAN)); + return results; + } +} From 8a099ae65dac1af27a5d35dec475de6fbdc18d45 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 6 Oct 2026 08:35:55 +0200 Subject: [PATCH 74/81] refactor(sql): remove legacy compiler path Co-authored-by: Codex GPT-6 --- .../query/concept/filter/FilterValue.java | 20 -- .../mode/local/LocalNamespaceHandler.java | 7 +- .../datasets/concepts/filters/Filter.java | 6 - .../specific/BigMultiSelectFilter.java | 7 - .../filters/specific/CountFilter.java | 6 - .../filters/specific/CountQuartersFilter.java | 6 - .../filters/specific/DateDistanceFilter.java | 6 - .../filters/specific/DurationSumFilter.java | 8 - .../concepts/filters/specific/FlagFilter.java | 6 - .../filters/specific/MultiSelectFilter.java | 6 - .../filters/specific/NumberFilter.java | 8 - .../filters/specific/SingleSelectFilter.java | 6 - .../concepts/filters/specific/SumFilter.java | 7 - .../datasets/concepts/select/Select.java | 6 - .../select/concept/ConceptColumnSelect.java | 8 - .../specific/EventDateUnionSelect.java | 7 - .../specific/EventDurationSumSelect.java | 7 - .../select/concept/specific/ExistsSelect.java | 7 - .../select/connector/DistinctSelect.java | 7 - .../select/connector/FirstValueSelect.java | 7 - .../select/connector/LastValueSelect.java | 6 - .../select/connector/RandomValueSelect.java | 6 - .../specific/CountQuartersSelect.java | 6 - .../connector/specific/CountSelect.java | 7 - .../specific/DateDistanceSelect.java | 7 - .../connector/specific/DateUnionSelect.java | 6 - .../connector/specific/DurationSumSelect.java | 7 - .../select/connector/specific/FlagSelect.java | 7 - .../specific/MappableSingleColumnSelect.java | 18 -- .../select/connector/specific/SumSelect.java | 7 - .../conquery/sql/conversion/Context.java | 25 -- .../conversion/NodeConversionDispatcher.java | 10 - .../sql/conversion/NodeConversions.java | 63 ---- .../sql/conversion/NodeConverter.java | 18 -- .../conversion/ResolvedOperationAdapter.java | 14 +- .../sql/conversion/ResolvedQueryMapper.java | 10 +- .../conquery/sql/conversion/SqlConverter.java | 18 +- .../conversion/cqelement/CQAndConverter.java | 39 --- .../cqelement/CQDateRestrictionConverter.java | 20 -- .../cqelement/CQExternalConverter.java | 66 ---- .../cqelement/CQNegationConverter.java | 21 -- .../conversion/cqelement/CQOrConverter.java | 39 --- .../conversion/cqelement/CQYesConverter.java | 29 -- .../cqelement/ConversionContext.java | 100 ------ .../cqelement/concept/CQConceptConverter.java | 286 ------------------ .../cqelement/concept/ConceptSqlTables.java | 21 -- .../cqelement/concept/ConnectorSqlTables.java | 31 -- .../cqelement/concept/FilterContext.java | 52 ---- .../cqelement/concept/TablePath.java | 80 ----- .../sql/conversion/dialect/DialectBundle.java | 18 +- .../dialect/LegacyCompilerDialect.java | 256 ---------------- .../clickhouse/ClickhouseDialectBundle.java | 23 -- .../ClickhouseDistinctSelectConverter.java | 7 - .../dialect/hana/HanaDialectBundle.java | 15 - .../sql/conversion/model/NumberMapUtil.java | 22 -- .../conversion/model/QueryStepComposer.java | 63 ---- .../CountQuartersSqlAggregator.java | 75 ----- .../model/aggregator/CountSqlAggregator.java | 93 ------ .../aggregator/DateDistanceSqlAggregator.java | 110 ------- .../aggregator/DurationSumSqlAggregator.java | 85 ------ .../model/aggregator/FlagSqlAggregator.java | 124 -------- .../ResolvedAggregationAdapter.java | 23 -- .../model/aggregator/SqlAggregator.java | 13 - .../model/aggregator/SumSqlAggregator.java | 188 ------------ .../filter/AbstractSelectFilterConverter.java | 51 ---- .../model/filter/FilterConverter.java | 14 - .../filter/LegacyInclusiveRangeCondition.java | 26 -- .../filter/MultiSelectFilterConverter.java | 14 - .../model/filter/NumberFilterConverter.java | 48 --- .../filter/SingleSelectFilterConverter.java | 12 - .../select/ConceptColumnSelectConverter.java | 107 ------- .../select/DateUnionSelectConverter.java | 16 - .../model/select/DistinctSelectConverter.java | 13 - .../select/EventDateUnionSelectConverter.java | 20 -- .../EventDurationSumSelectConverter.java | 20 -- .../model/select/ExistsSelectConverter.java | 20 -- .../select/FirstValueSelectConverter.java | 14 - .../select/LastValueSelectConverter.java | 14 - .../select/RandomValueSelectConverter.java | 17 -- .../model/select/ResolvedSelectAdapter.java | 89 ------ .../model/select/SelectContext.java | 59 ---- .../model/select/SelectConverter.java | 19 -- .../query/CQReusedQueryConverter.java | 25 -- .../query/ConceptQueryConverter.java | 54 ---- .../query/SecondaryIdQueryConverter.java | 30 -- .../sql/conversion/SqlConverterTest.java | 24 +- .../SqlExtractionAggregationTest.java | 169 ----------- .../SqlExtractionConceptSelectTest.java | 95 ------ .../SqlExtractionDateSelectTest.java | 99 ------ .../conversion/SqlExtractionDialectTest.java | 16 +- .../conversion/SqlExtractionSelectTest.java | 122 -------- 91 files changed, 47 insertions(+), 3442 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/Context.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversionDispatcher.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQDateRestrictionConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NumberMapUtil.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepComposer.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DurationSumSqlAggregator.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/ResolvedAggregationAdapter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SqlAggregator.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/FilterConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/LegacyInclusiveRangeCondition.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/MultiSelectFilterConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/SingleSelectFilterConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ResolvedSelectAdapter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/query/CQReusedQueryConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdQueryConverter.java delete mode 100644 backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionAggregationTest.java delete mode 100644 backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionConceptSelectTest.java delete mode 100644 backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionDateSelectTest.java delete mode 100644 backend/src/test/java/com/bakdata/conquery/sql/conversion/SqlExtractionSelectTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java index 3f82b387fb..4c0297f89d 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java @@ -12,14 +12,8 @@ import com.bakdata.conquery.models.common.Range.LongRange; import com.bakdata.conquery.models.common.Range.MoneyRange; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.identifiable.ids.specific.FilterId; import com.bakdata.conquery.models.query.QueryResolveContext; -import com.bakdata.conquery.sql.compiler.ir.SqlTables; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; -import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -31,7 +25,6 @@ import lombok.Setter; import lombok.ToString; import org.jetbrains.annotations.TestOnly; -import org.jooq.Condition; /** * @implNote The {@link JsonCreator} annos are necessary. Otherwise, Jackson will deserilaize all values as generic objects. @@ -58,19 +51,6 @@ public VALUE readValue() { return ((VALUE) value); } - public SqlFilters convertToSqlFilter(SqlIdColumns ids, ConversionContext context, SqlTables tables) { - FilterContext filterContext = FilterContext.forConceptConversion(ids, readValue(), context, tables); - final Filter resolve = (Filter) filter.resolve(); - SqlFilters sqlFilters = resolve.createConverter().convertToSqlFilter(resolve, filterContext); - return sqlFilters; - } - - public Condition convertForTableExport(SqlIdColumns ids, ConversionContext context) { - FilterContext filterContext = FilterContext.forTableExport(ids, readValue(), context); - final Filter resolve = (Filter) filter.resolve(); - return resolve.createConverter().convertForTableExport(resolve, filterContext); - } - @CPSType(id = FrontendFilterType.Fields.MULTI_SELECT, base = FilterValue.class) public static class CQMultiSelectFilter extends FilterValue> { @JsonCreator diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java index 9110c0c30e..ebde381559 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java @@ -11,7 +11,6 @@ import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.LocalNamespace; import com.bakdata.conquery.sql.conquery.SqlExecutionManager; -import com.bakdata.conquery.sql.conversion.NodeConversions; import com.bakdata.conquery.sql.conversion.ResolvedQueryAdapter; import com.bakdata.conquery.sql.conversion.ResolvedQueryMapper; import com.bakdata.conquery.sql.conversion.ResolvedFormAdapter; @@ -58,9 +57,9 @@ public LocalNamespace createNamespace( ResultSetProcessor resultSetProcessor = dialectBundle.getResultSetProcessor(config); SqlExecutionService sqlExecutionService = new SqlExecutionService(dslContext, resultSetProcessor); - NodeConversions nodeConversions = new NodeConversions(config.getIdColumns(), dialectBundle, dslContext, clock, connection.getConnection().getPrimaryColumn()); ResolvedQueryMapper resolvedQueryMapper = new ResolvedQueryMapper( - config.getIdColumns(), dialectBundle, clock, connection.getConnection().getPrimaryColumn()); + config.getIdColumns(), dialectBundle.getFunctionProvider(), clock, + connection.getConnection().getPrimaryColumn()); ResolvedQueryAdapter resolvedQueryAdapter = new ResolvedQueryAdapter( environment.getValidator(), new DefaultSqlCompiler(dslContext), resolvedQueryMapper); ResolvedTableExportAdapter tableExportAdapter = new ResolvedTableExportAdapter( @@ -69,7 +68,7 @@ public LocalNamespace createNamespace( ResolvedFormAdapter formAdapter = new ResolvedFormAdapter( new ResolvedFormMapper(resolvedQueryMapper), new FormSqlCompiler(dslContext)); SqlConverter sqlConverter = new SqlConverter( - nodeConversions, config, resolvedQueryAdapter, tableExportAdapter, formAdapter, dialectBundle); + resolvedQueryAdapter, tableExportAdapter, formAdapter, dialectBundle.getCompilerDialect()); ExecutionManager executionManager = new SqlExecutionManager(sqlConverter, sqlExecutionService, metaStorage, datasetRegistry, config); SqlStorageHandler sqlStorageHandler = new SqlStorageHandler(sqlExecutionService); SqlEntityResolver sqlEntityResolver = new SqlEntityResolver(config.getIdColumns(), dslContext, dialectBundle, sqlExecutionService); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/Filter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/Filter.java index 5f9ee9f52e..030a917a5d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/Filter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/Filter.java @@ -13,7 +13,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.FilterId; import com.bakdata.conquery.models.identifiable.ids.specific.TableId; import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonBackReference; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -96,11 +95,6 @@ public boolean isForConnectorsTable() { @JsonIgnore public abstract List getRequiredColumns(); - @JsonIgnore - public > FilterConverter createConverter() { - throw new UnsupportedOperationException("No converter implemented for Filter %s".formatted(getClass())); - } - @Override public FilterId createId() { return new FilterId(connector.getId(), getName()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java index d5328474a7..cdd834bb31 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java @@ -8,8 +8,6 @@ import com.bakdata.conquery.models.query.filter.event.MultiSelectFilterNode; import com.bakdata.conquery.models.query.filter.event.SubstringMultiSelectFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.EventFilterNode; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; -import com.bakdata.conquery.sql.conversion.model.filter.MultiSelectFilterConverter; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.Setter; @@ -40,9 +38,4 @@ public EventFilterNode> createFilterNode(Set value) { return new MultiSelectFilterNode(getColumn().resolve(), value); } - @Override - public FilterConverter> createConverter() { - //TODO (FK) Converter for Substring - return new MultiSelectFilterConverter(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java index d73f3b9e01..981b00acad 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java @@ -17,8 +17,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.DistinctValuesWrapperAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.CountAggregator; import com.bakdata.conquery.models.query.queryplan.filter.AggregationFilterNode; -import com.bakdata.conquery.sql.conversion.model.aggregator.CountSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; @@ -69,8 +67,4 @@ public List getRequiredColumns() { return out; } - @Override - public FilterConverter createConverter() { - return new CountSqlAggregator(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountQuartersFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountQuartersFilter.java index a985f4e046..446cc6f222 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountQuartersFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountQuartersFilter.java @@ -19,8 +19,6 @@ import com.bakdata.conquery.models.query.filter.RangeFilterNode; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.CountQuartersOfDatesAggregator; import com.bakdata.conquery.models.query.queryplan.filter.AggregationFilterNode; -import com.bakdata.conquery.sql.conversion.model.aggregator.CountQuartersSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; import lombok.Getter; @@ -61,8 +59,4 @@ public List getRequiredColumns() { return new RangeFilterNode(value, new CountQuartersOfDatesAggregator(column)); } - @Override - public FilterConverter createConverter() { - return new CountQuartersSqlAggregator(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java index 3418d6d0ca..7722b4c6f3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java @@ -13,8 +13,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; import com.bakdata.conquery.models.query.filter.event.DateDistanceFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.EventFilterNode; -import com.bakdata.conquery.sql.conversion.model.aggregator.DateDistanceSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; import jakarta.validation.Valid; @@ -69,8 +67,4 @@ public EventFilterNode createFilterNode(Range.LongRange value) { return new DateDistanceFilterNode(getColumn().resolve(), timeUnit, value); } - @Override - public FilterConverter createConverter() { - return new DateDistanceSqlAggregator(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DurationSumFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DurationSumFilter.java index a57964a527..fc3da7d596 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DurationSumFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DurationSumFilter.java @@ -26,8 +26,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.specific.DurationSumAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.TwoColumnDurationSumAggregator; import com.bakdata.conquery.models.query.queryplan.filter.AggregationFilterNode; -import com.bakdata.conquery.sql.conversion.model.aggregator.DurationSumSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; import lombok.Data; @@ -92,10 +90,4 @@ private boolean hasDistinct() { return new RangeFilterNode(value, aggregator); } - @Override - public FilterConverter createConverter() { - return new DurationSumSqlAggregator(); - } - - } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java index 0da9a4d251..e9e56d7d1e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java @@ -17,8 +17,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; import com.bakdata.conquery.models.query.filter.event.FlagColumnsFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.EventFilterNode; -import com.bakdata.conquery.sql.conversion.model.aggregator.FlagSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; @@ -95,8 +93,4 @@ public boolean isAllColumnsBoolean() { return valid; } - @Override - public FilterConverter> createConverter() { - return new FlagSqlAggregator(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/MultiSelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/MultiSelectFilter.java index 8bd5e85a7e..6be0672de2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/MultiSelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/MultiSelectFilter.java @@ -8,8 +8,6 @@ import com.bakdata.conquery.models.query.filter.event.MultiSelectFilterNode; import com.bakdata.conquery.models.query.filter.event.SubstringMultiSelectFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.EventFilterNode; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; -import com.bakdata.conquery.sql.conversion.model.filter.MultiSelectFilterConverter; import com.fasterxml.jackson.annotation.JsonIgnore; /** @@ -39,8 +37,4 @@ public EventFilterNode> createFilterNode(Set value) { return new MultiSelectFilterNode(getColumn().resolve(), value); } - @Override - public FilterConverter> createConverter() { - return new MultiSelectFilterConverter(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java index ccbd0f7866..982006bff0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java @@ -24,8 +24,6 @@ import com.bakdata.conquery.models.query.filter.event.number.MoneyFilterNode; import com.bakdata.conquery.models.query.filter.event.number.RealFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.EventFilterNode; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; -import com.bakdata.conquery.sql.conversion.model.filter.NumberFilterConverter; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.OptBoolean; @@ -87,12 +85,6 @@ public EventFilterNode createFilterNode(RANGE value) { }; } - @Override - public FilterConverter, RANGE> createConverter() { - return new NumberFilterConverter<>(); - } - - @JsonIgnore @ValidationMethod(message = "Columns do not match required Type.") public boolean isValidColumnType() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SingleSelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SingleSelectFilter.java index cb19b38917..7c97ce71c0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SingleSelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SingleSelectFilter.java @@ -8,8 +8,6 @@ import com.bakdata.conquery.models.query.filter.event.MultiSelectFilterNode; import com.bakdata.conquery.models.query.filter.event.SubstringMultiSelectFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.EventFilterNode; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; -import com.bakdata.conquery.sql.conversion.model.filter.SingleSelectFilterConverter; import net.minidev.json.annotate.JsonIgnore; /** @@ -35,8 +33,4 @@ public String getFilterType() { return FrontendFilterType.Fields.SELECT; } - @Override - public FilterConverter createConverter() { - return new SingleSelectFilterConverter(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java index 10e87702ac..fb0480112d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java @@ -25,8 +25,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.specific.sum.MoneySumAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.sum.RealSumAggregator; import com.bakdata.conquery.models.query.queryplan.filter.AggregationFilterNode; -import com.bakdata.conquery.sql.conversion.model.aggregator.SumSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; import jakarta.validation.constraints.NotNull; @@ -107,11 +105,6 @@ public List getRequiredColumns() { return new RangeFilterNode(range, getAggregator()); } - @Override - public FilterConverter, RANGE> createConverter() { - return new SumSqlAggregator<>(); - } - @JsonIgnore private ColumnAggregator getAggregator() { final Column resolvedColumn = getColumn().resolve(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java index 4a1828699e..2d6ddab496 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java @@ -20,7 +20,6 @@ import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonBackReference; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -108,11 +107,6 @@ public boolean isForConnectorsTable() { @JsonIgnore public abstract List getRequiredColumns(); - @JsonIgnore - public SelectConverter createConverter() { - throw new UnsupportedOperationException("No converter implemented for Select %s".formatted(getClass())); - } - @JsonIgnore public boolean isEventDateSelect() { return false; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java index 0e285e8202..097c4f0e4b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java @@ -19,8 +19,6 @@ import com.bakdata.conquery.models.query.resultinfo.printers.common.ConceptIdPrinter; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; -import com.bakdata.conquery.sql.conversion.model.select.ConceptColumnSelectConverter; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; @@ -77,12 +75,6 @@ public ResultType getResultType() { return new ResultType.ListT<>(ResultType.Primitive.STRING); } - @Override - public SelectConverter createConverter() { - //TODO bind Select to converter here - return new ConceptColumnSelectConverter(); - } - @Override public ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor) { return isAsIds() ? processor::getIntegerList : processor::getStringList; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java index 1b7d1f43a4..cfa327ddfd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java @@ -13,8 +13,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.EventDateUnionAggregator; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.select.EventDateUnionSelectConverter; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; /** @@ -43,11 +41,6 @@ public boolean isEventDateSelect() { return true; } - @Override - public SelectConverter createConverter() { - return new EventDateUnionSelectConverter(); - } - @Override public ResultType getResultType() { return new ResultType.ListT<>(ResultType.Primitive.DATE_RANGE); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java index d5bb67b5c5..8133709d54 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java @@ -9,8 +9,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.EventDurationSumAggregator; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.select.EventDurationSumSelectConverter; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.google.common.base.Preconditions; import lombok.AccessLevel; @@ -44,11 +42,6 @@ public ResultSetProcessor.Reader createResultSetReader(ResultSetProcess return processor::getInteger; } - @Override - public SelectConverter createConverter() { - return new EventDurationSumSelectConverter(); - } - @Override public ResultType getResultType() { return ResultType.Primitive.INTEGER; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java index f02a41fd63..7d3ba50958 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java @@ -12,8 +12,6 @@ import com.bakdata.conquery.models.datasets.concepts.select.concept.UniversalSelect; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.ExistsAggregator; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.select.ExistsSelectConverter; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; @@ -35,11 +33,6 @@ public ResultSetProcessor.Reader createResultSetReader(ResultSetProcess return processor::getBoolean; } - @Override - public SelectConverter createConverter() { - return new ExistsSelectConverter(); - } - @Override public ResultType getResultType() { return ResultType.Primitive.BOOLEAN; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java index a014283044..fb5e792758 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java @@ -17,8 +17,6 @@ import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.query.resultinfo.printers.common.OneToManyMappingPrinter; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.select.DistinctSelectConverter; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; @@ -35,11 +33,6 @@ public Aggregator createAggregator() { return new AllValuesAggregator(getColumn().resolve(), getSubstringRange()); } - @Override - public SelectConverter createConverter() { - return new DistinctSelectConverter(); - } - @Override public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { if (getMapping() == null) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java index aa43ede5ae..7188cdce7a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java @@ -10,8 +10,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.InternToExternMapperId; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.FirstValueAggregator; -import com.bakdata.conquery.sql.conversion.model.select.FirstValueSelectConverter; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; @@ -34,9 +32,4 @@ public Aggregator createAggregator() { - @Override - public SelectConverter createConverter() { - return new FirstValueSelectConverter(); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/LastValueSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/LastValueSelect.java index e5c1fd01a4..8e97baf806 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/LastValueSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/LastValueSelect.java @@ -8,8 +8,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.InternToExternMapperId; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.LastValueAggregator; -import com.bakdata.conquery.sql.conversion.model.select.LastValueSelectConverter; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.fasterxml.jackson.annotation.JsonCreator; @CPSType(id = "LAST", base = Select.class) @@ -28,8 +26,4 @@ public Aggregator createAggregator() { return new LastValueAggregator<>(getColumn().resolve(), getSubstringRange()); } - @Override - public SelectConverter createConverter() { - return new LastValueSelectConverter(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/RandomValueSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/RandomValueSelect.java index 1bc92c9d7a..631b491daf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/RandomValueSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/RandomValueSelect.java @@ -8,8 +8,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.InternToExternMapperId; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.RandomValueAggregator; -import com.bakdata.conquery.sql.conversion.model.select.RandomValueSelectConverter; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.fasterxml.jackson.annotation.JsonCreator; @CPSType(id = "RANDOM", base = Select.class) @@ -27,8 +25,4 @@ public Aggregator createAggregator() { return new RandomValueAggregator<>(getColumn().resolve(), getSubstringRange()); } - @Override - public SelectConverter createConverter() { - return new RandomValueSelectConverter(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountQuartersSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountQuartersSelect.java index 0eabfac564..1e833ef045 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountQuartersSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountQuartersSelect.java @@ -8,8 +8,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.CountQuartersOfDatesAggregator; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.aggregator.CountQuartersSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; import lombok.Getter; @@ -61,8 +59,4 @@ public ResultSetProcessor.Reader createResultSetReader(ResultSetProcess return processor::getInteger; } - @Override - public SelectConverter createConverter() { - return new CountQuartersSqlAggregator(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountSelect.java index 6812eddb0a..198877a797 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountSelect.java @@ -13,8 +13,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.DistinctValuesWrapperAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.CountAggregator; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.aggregator.CountSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import lombok.Data; import lombok.NoArgsConstructor; @@ -61,11 +59,6 @@ public List getRequiredColumns() { return out; } - @Override - public SelectConverter createConverter() { - return new CountSqlAggregator(); - } - @Override public ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor) { return processor::getInteger; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateDistanceSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateDistanceSelect.java index 76bb598163..e23cd8d366 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateDistanceSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateDistanceSelect.java @@ -14,8 +14,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.DateDistanceAggregator; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.aggregator.DateDistanceSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; import lombok.Getter; @@ -44,11 +42,6 @@ public Aggregator createAggregator() { return new DateDistanceAggregator(getColumn().resolve(), getTimeUnit()); } - @Override - public SelectConverter createConverter() { - return new DateDistanceSqlAggregator(); - } - @Override public ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor) { return processor::getInteger; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java index 177134a31d..ff9ddfded8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java @@ -12,8 +12,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.DateUnionAggregator; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.select.DateUnionSelectConverter; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -59,8 +57,4 @@ public ResultSetProcessor.Reader>> createResultSetReader(Resu return processor::getDateRangeList; } - @Override - public SelectConverter createConverter() { - return new DateUnionSelectConverter(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java index ff74ef6435..31aa2046f8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java @@ -16,8 +16,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.specific.DurationSumAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.TwoColumnDurationSumAggregator; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.aggregator.DurationSumSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -82,9 +80,4 @@ public ResultType getResultType() { return ResultType.Primitive.INTEGER; } - @Override - public SelectConverter createConverter() { - //TODO apply distinctBy (though needs to be done once other branches are merged) - return new DurationSumSqlAggregator(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java index 2679f75364..8e9c6fcc99 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java @@ -15,8 +15,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.FlagsAggregator; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.aggregator.FlagSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -69,11 +67,6 @@ public boolean isAllColumnsBoolean() { return flags.values().stream().map(ColumnId::resolve).map(Column::getType).allMatch(MajorTypeId.BOOLEAN::equals); } - @Override - public SelectConverter createConverter() { - return new FlagSqlAggregator(); - } - @Override public ResultType getResultType() { return new ResultType.ListT<>(ResultType.Primitive.STRING); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java index a98a50b7cc..a2679994e4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java @@ -1,8 +1,5 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; -import com.bakdata.conquery.sql.conversion.model.select.ResolvedSelectAdapter; -import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; -import com.bakdata.conquery.sql.compiler.conversion.operation.SubstringSelect; import static com.bakdata.conquery.models.types.ResultType.Primitive.STRING; import static com.bakdata.conquery.models.types.ResultType.resolveResultType; @@ -16,7 +13,6 @@ import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider; import com.bakdata.conquery.models.common.Range; -import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.select.connector.SingleColumnSelect; import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; @@ -28,10 +24,6 @@ import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; - -import com.bakdata.conquery.sql.conversion.model.select.SelectContext; -import com.bakdata.conquery.sql.compiler.ir.select.SingleColumnSqlSelect; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; @@ -60,16 +52,6 @@ public MappableSingleColumnSelect(ColumnId column, @Nullable InternToExternMappe this.substringRange = substringRange; } - public static SingleColumnSqlSelect getSubstringSelect( - Column column, Range.IntegerRange substringRange, SelectContext selectContext, - String alias) { - - return SubstringSelect.getSubstringSelect( - EntitySchemaAdapter.from(column), - ResolvedSelectAdapter.substring(substringRange), - selectContext.getTables().getRootTable(), alias); - } - @Override public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { if (mapping == null) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java index c620a327a8..294acbdbda 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java @@ -25,8 +25,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.specific.sum.MoneySumAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.sum.RealSumAggregator; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.aggregator.SumSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -118,11 +116,6 @@ public ResultSetProcessor.Reader createResultSetReader(ResultSetProcess return processor::getInteger; } - @Override - public SelectConverter createConverter() { - return new SumSqlAggregator<>(); - } - @Override public ResultType getResultType() { return ResultType.resolveResultType(getColumn().resolve().getType()); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Context.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/Context.java deleted file mode 100644 index 26856c9ead..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Context.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.bakdata.conquery.sql.conversion; - -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; - -/** - * Marker for a conversion context. - */ -public interface Context { - - ConversionContext getConversionContext(); - - LegacyCompilerDialect getCompilerDialect(); - - default SqlNameGenerator getNameGenerator() { - return getConversionContext().getNameGenerator(); - } - - default SqlFunctionProvider getFunctionProvider(){ - return getConversionContext().getCompilerDialect().getFunctionProvider(); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversionDispatcher.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversionDispatcher.java deleted file mode 100644 index 64c1749fe9..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversionDispatcher.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.bakdata.conquery.sql.conversion; - -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; - -/** Recursively dispatches a query node to its matching SQL converter. */ -@FunctionalInterface -public interface NodeConversionDispatcher { - - ConversionContext convert(Object node, ConversionContext context); -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java deleted file mode 100644 index bac5216ec5..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java +++ /dev/null @@ -1,63 +0,0 @@ -package com.bakdata.conquery.sql.conversion; - -import com.bakdata.conquery.apiv1.query.QueryDescription; -import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.config.IdColumnConfig; -import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.sql.compiler.conversion.ConversionDispatcher; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; -import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; -import lombok.NonNull; -import org.jooq.DSLContext; - -import java.time.Clock; -import java.util.Locale; - -/** - * Entry point for converting {@link QueryDescription} to an SQL query. - */ -public class NodeConversions implements NodeConversionDispatcher { - - private final ConversionDispatcher dispatcher; - private final IdColumnConfig idColumns; - private final LegacyCompilerDialect dialect; - private final SqlNameGenerator nameGenerator; - private final Clock clock; - @NonNull - private final String defaultPrimaryColumn; - - public NodeConversions( - IdColumnConfig idColumns, - LegacyCompilerDialect compilerDialect, - DSLContext dslContext, - Clock clock, - String defaultPrimaryColumn - ) { - this.dispatcher = new ConversionDispatcher<>(compilerDialect.getNodeConverters(dslContext)); - this.idColumns = idColumns; - this.dialect = compilerDialect; - this.nameGenerator = new SqlNameGenerator(compilerDialect.getNameMaxLength()); - this.clock = clock; - this.defaultPrimaryColumn = defaultPrimaryColumn; - } - - @Override - public ConversionContext convert(Object node, ConversionContext context) { - return dispatcher.convert(node, context); - } - - public ConversionContext convert(QueryDescription queryDescription, ConqueryConfig conqueryConfig) { - ConversionContext initialCtx = ConversionContext.builder() - .idColumns(idColumns) - .sqlPrintSettings(new PrintSettings(false, Locale.ROOT, conqueryConfig, null, null)) - .nameGenerator(nameGenerator) - .nodeConversions(this) - .clock(clock) - .defaultPrimaryColumn(this.defaultPrimaryColumn) - .compilerDialect(dialect) - .build(); - return convert(queryDescription, initialCtx); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverter.java deleted file mode 100644 index 514c7190d0..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverter.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.bakdata.conquery.sql.conversion; - -import com.bakdata.conquery.models.query.Visitable; -import com.bakdata.conquery.sql.compiler.conversion.Converter; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; - -/** - * Interface for converters that implement the translation of a ConQuery query to an SQL query. - * - *

- * A ConQuery is a graph that has a {@link com.bakdata.conquery.apiv1.query.QueryDescription} as its root. - * The children of the root are of type {@link com.bakdata.conquery.apiv1.query.CQElement}. - * - * @param type of the node to convert - */ -public interface NodeConverter extends Converter { - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedOperationAdapter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedOperationAdapter.java index 83c7c5e0fa..a2aa16024f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedOperationAdapter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedOperationAdapter.java @@ -10,6 +10,7 @@ import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; import com.bakdata.conquery.models.common.IRange; +import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.datasets.concepts.conditions.AndCondition; import com.bakdata.conquery.models.datasets.concepts.conditions.ColumnEqualCondition; import com.bakdata.conquery.models.datasets.concepts.conditions.CTCondition; @@ -48,7 +49,6 @@ import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; -import com.bakdata.conquery.sql.conversion.model.select.ResolvedSelectAdapter; import com.bakdata.conquery.sql.model.operation.BuiltInAggregations; import com.bakdata.conquery.sql.model.operation.BuiltInConditions; import com.bakdata.conquery.sql.model.operation.BuiltInFilters; @@ -58,6 +58,7 @@ import com.bakdata.conquery.sql.model.operation.ResolvedFilter; import com.bakdata.conquery.sql.model.operation.ResolvedSelect; import com.bakdata.conquery.sql.model.range.NumberRange; +import com.bakdata.conquery.sql.model.range.SubstringRange; import com.bakdata.conquery.sql.model.result.ResultColumn; import com.bakdata.conquery.sql.model.schema.ResolvedColumn; import com.bakdata.conquery.sql.model.schema.SqlTable; @@ -97,7 +98,7 @@ public static ResolvedFilter filter(FilterValue filterValue, LocalDate endDat return switch (filter) { case SelectFilter selectFilter -> new BuiltInFilters.StringValues( filter.getName(), EntitySchemaAdapter.from(selectFilter.getColumn().resolve()), stringValues(value), - ResolvedSelectAdapter.substring(selectFilter.getSubstringRange())); + substring(selectFilter.getSubstringRange())); case NumberFilter numberFilter -> new BuiltInFilters.NumericColumnRange( filter.getName(), EntitySchemaAdapter.from(numberFilter.getColumn().resolve()), numberRange((IRange) value)); case CountFilter countFilter -> aggregationFilter(filter.getName(), @@ -148,7 +149,14 @@ private static BuiltInFilters.AggregationRange aggregationFilter(String name, Re private static BuiltInSelects.Values values(MappableSingleColumnSelect select, BuiltInSelects.ValueOperation operation) { return new BuiltInSelects.Values(select.getName(), EntitySchemaAdapter.from(select.getColumn().resolve()), operation, - ResolvedSelectAdapter.substring(select.getSubstringRange())); + substring(select.getSubstringRange())); + } + + private static Optional substring(Range.IntegerRange range) { + if (range == null || range.isAll()) { + return Optional.empty(); + } + return Optional.of(new SubstringRange(range.getMin() == null ? 0 : range.getMin(), Optional.ofNullable(range.getMax()))); } private static BuiltInAggregations.Count count(com.bakdata.conquery.models.datasets.Column column, boolean distinct) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java index 59ccc91609..61885cc2ef 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ResolvedQueryMapper.java @@ -38,7 +38,7 @@ import com.bakdata.conquery.models.query.DateAggregationMode; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptIdMapping; -import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; import com.bakdata.conquery.sql.mapping.ConceptIdMappingSelection; import com.bakdata.conquery.sql.model.ResolvedQuery; @@ -65,18 +65,18 @@ public final class ResolvedQueryMapper { private final IdColumnConfig idColumns; - private final LegacyCompilerDialect dialect; + private final SqlFunctionProvider functionProvider; private final Clock clock; private final String defaultPrimaryColumn; public ResolvedQueryMapper( IdColumnConfig idColumns, - LegacyCompilerDialect dialect, + SqlFunctionProvider functionProvider, Clock clock, String defaultPrimaryColumn ) { this.idColumns = idColumns; - this.dialect = dialect; + this.functionProvider = functionProvider; this.clock = clock; this.defaultPrimaryColumn = defaultPrimaryColumn; } @@ -181,7 +181,7 @@ private ResolvedConnector connector( connectorColumn.ifPresent(column -> conditions.add(new BuiltInConditions.Presence(column, true))); List selects = table.getSelects().stream().map(SelectId::resolve) .map(select -> ResolvedOperationAdapter.select(select, conceptColumns, endDate)).toList(); - ConceptIdMapping mapping = new ConceptIdMapping(concept, dialect.getFunctionProvider()); + ConceptIdMapping mapping = new ConceptIdMapping(concept, functionProvider); ConceptIdMappingSelection mappingSelection = new ConceptIdMappingSelection( mapping.source(connector), mapping.includedLocalIds(selected), mapping.includesRoot(selected), resolveConceptIds, concept.getLocalId()); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java index 638a7f0c0f..8e3e164a7f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java @@ -4,33 +4,25 @@ import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; import com.bakdata.conquery.apiv1.query.TableExportQuery; -import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery; import com.bakdata.conquery.models.forms.managed.EntityDateQuery; import com.bakdata.conquery.models.forms.managed.RelativeFormQuery; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import com.bakdata.conquery.sql.conversion.model.SqlQuery; public class SqlConverter { - private final NodeConversions nodeConversions; - private final ConqueryConfig config; private final ResolvedQueryAdapter resolvedQueryAdapter; private final ResolvedTableExportAdapter tableExportAdapter; private final ResolvedFormAdapter formAdapter; - private final LegacyCompilerDialect dialect; + private final CompilerDialect dialect; public SqlConverter( - NodeConversions nodeConversions, - ConqueryConfig config, ResolvedQueryAdapter resolvedQueryAdapter, ResolvedTableExportAdapter tableExportAdapter, ResolvedFormAdapter formAdapter, - LegacyCompilerDialect dialect + CompilerDialect dialect ) { - this.nodeConversions = nodeConversions; - this.config = config; this.resolvedQueryAdapter = resolvedQueryAdapter; this.tableExportAdapter = tableExportAdapter; this.formAdapter = formAdapter; @@ -56,7 +48,7 @@ public SqlQuery convert(QueryDescription queryDescription) { if (queryDescription instanceof EntityDateQuery entityDateForm) { return formAdapter.compile(entityDateForm, dialect); } - ConversionContext converted = nodeConversions.convert(queryDescription, config); - return converted.getFinalQuery(); + throw new UnsupportedOperationException( + "SQL compilation is not implemented for " + queryDescription.getClass()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java deleted file mode 100644 index 5379b282f3..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement; - -import java.util.List; - -import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; -import com.bakdata.conquery.sql.compiler.ir.JoinMode; -import com.bakdata.conquery.sql.compiler.ir.LogicalQueryStepCompiler; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; -import com.bakdata.conquery.sql.conversion.model.QueryStepComposer; - -/** - * Adapts a resolved legacy conjunction to connector-owned logical query-step compilation. - * - *

TODO Remove this adapter once backend query resolution produces connector query nodes directly.

- */ -public class CQAndConverter implements NodeConverter { - - @Override - public Class getConversionClass() { - return CQAnd.class; - } - - @Override - public ConversionContext convert(CQAnd andNode, ConversionContext context) { - List children = QueryStepComposer.convertChildren(andNode.getChildren(), context); - QueryStep joined = LogicalQueryStepCompiler.compile( - children, - JoinMode.INNER, - andNode.getDateAction(), - andNode.getCreateExists().orElse(false), - EntitySchemaAdapter.from(context.getIdColumns()), - context.getCompilerDialect(), - context.getNameGenerator() - ); - return context.withQueryStep(joined); - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQDateRestrictionConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQDateRestrictionConverter.java deleted file mode 100644 index 5ce01a3634..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQDateRestrictionConverter.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement; - -import com.bakdata.conquery.apiv1.query.concept.specific.CQDateRestriction; -import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.sql.conversion.NodeConverter; - -public class CQDateRestrictionConverter implements NodeConverter { - - @Override - public ConversionContext convert(CQDateRestriction dateRestrictionNode, ConversionContext context) { - ConversionContext childContext = context.withDateRestrictionRange(CDateRange.of(dateRestrictionNode.getDateRange())); - return context.getNodeConversions().convert(dateRestrictionNode.getChild(), childContext).withDateRestrictionRange(null); - } - - @Override - public Class getConversionClass() { - return CQDateRestriction.class; - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java deleted file mode 100644 index 469ce34225..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement; - -import java.util.Collection; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; -import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.sql.compiler.ir.ExternalQueryStepCompiler; -import com.bakdata.conquery.sql.compiler.ir.ExternalQuerySteps; -import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.model.node.ExternalEntity; -import com.bakdata.conquery.sql.model.node.ExternalNode; -import com.bakdata.conquery.sql.model.range.DateRange; - -/** - * Adapts the resolved legacy external-query payload to connector-owned compilation. - * - *

TODO Remove this adapter once backend query resolution produces {@link ExternalNode} directly.

- */ -public class CQExternalConverter implements NodeConverter { - - @Override - public Class getConversionClass() { - return CQExternal.class; - } - - @Override - public ConversionContext convert(CQExternal external, ConversionContext context) { - ExternalQuerySteps steps = ExternalQueryStepCompiler.compile( - toExternalNode(external), - context.isNegation(), - Optional.ofNullable(context.getDateRestrictionRange()).map(CQExternalConverter::toDateRange), - context.getCompilerDialect() - ); - ConversionContext converted = context.withQueryStep(steps.entities()); - return steps.values().map(converted::withExternalExtras).orElse(converted); - } - - private static ExternalNode toExternalNode(CQExternal external) { - List entities = external.getValuesResolved().entrySet().stream() - .map(entry -> toExternalEntity(external, entry.getKey(), entry.getValue().asRanges())) - .toList(); - return new ExternalNode(entities, external.getExtraHeaders()); - } - - private static ExternalEntity toExternalEntity( - CQExternal external, - String entityId, - Collection validityDates - ) { - Map> values = new LinkedHashMap<>(); - external.getExtrasForId(entityId).forEach(entry -> values.put(entry.getKey(), entry.getValue())); - return new ExternalEntity( - entityId, - validityDates.stream().map(CQExternalConverter::toDateRange).toList(), - values - ); - } - - private static DateRange toDateRange(CDateRange range) { - return new DateRange(Optional.ofNullable(range.getMin()), Optional.ofNullable(range.getMax())); - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java deleted file mode 100644 index 2bb90a6d62..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement; - -import com.bakdata.conquery.apiv1.query.concept.specific.CQNegation; -import com.bakdata.conquery.sql.conversion.NodeConverter; - -public class CQNegationConverter implements NodeConverter { - - @Override - public Class getConversionClass() { - return CQNegation.class; - } - - - @Override - public ConversionContext convert(CQNegation negationNode, ConversionContext context) { - return context.getNodeConversions() - .convert(negationNode.getChild(), context.withNegation(true)) - .withNegation(false); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java deleted file mode 100644 index 009fbae701..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement; - -import java.util.List; - -import com.bakdata.conquery.apiv1.query.concept.specific.CQOr; -import com.bakdata.conquery.sql.compiler.ir.JoinMode; -import com.bakdata.conquery.sql.compiler.ir.LogicalQueryStepCompiler; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; -import com.bakdata.conquery.sql.conversion.model.QueryStepComposer; - -/** - * Adapts a resolved legacy disjunction to connector-owned logical query-step compilation. - * - *

TODO Remove this adapter once backend query resolution produces connector query nodes directly.

- */ -public class CQOrConverter implements NodeConverter { - - @Override - public Class getConversionClass() { - return CQOr.class; - } - - @Override - public ConversionContext convert(CQOr orNode, ConversionContext context) { - List children = QueryStepComposer.convertChildren(orNode.getChildren(), context); - QueryStep joined = LogicalQueryStepCompiler.compile( - children, - JoinMode.FULL_OUTER, - orNode.getDateAction(), - orNode.getCreateExists().orElse(false), - EntitySchemaAdapter.from(context.getIdColumns()), - context.getCompilerDialect(), - context.getNameGenerator() - ); - return context.withQueryStep(joined); - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java deleted file mode 100644 index 3627dfa7fa..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement; - -import com.bakdata.conquery.apiv1.query.CQYes; -import com.bakdata.conquery.sql.compiler.ir.AllEntitiesQueryStepCompiler; -import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; -import com.bakdata.conquery.sql.model.node.AllEntitiesNode; - -/** - * Adapts the legacy all-entities query node to connector-owned compilation. - * - *

TODO Remove this adapter once backend query resolution produces the connector's all-entities node directly.

- */ -public class CQYesConverter implements NodeConverter { - - @Override - public Class getConversionClass() { - return CQYes.class; - } - - @Override - public ConversionContext convert(CQYes cqYes, ConversionContext context) { - return context.withQueryStep(AllEntitiesQueryStepCompiler.compile( - new AllEntitiesNode(), - EntitySchemaAdapter.from(context.getIdColumns()), - context.getCompilerDialect() - )); - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java deleted file mode 100644 index de859cb831..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java +++ /dev/null @@ -1,100 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement; - -import java.time.Clock; -import java.util.List; -import javax.annotation.Nullable; - -import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; -import com.bakdata.conquery.apiv1.query.concept.specific.CQDateRestriction; -import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.models.config.IdColumnConfig; -import com.bakdata.conquery.models.datasets.SecondaryIdDescription; -import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.sql.conversion.Context; -import com.bakdata.conquery.sql.conversion.NodeConversionDispatcher; -import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; -import com.bakdata.conquery.sql.compiler.naming.SqlNameGenerator; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.conversion.model.SqlQuery; -import lombok.Builder; -import lombok.NonNull; -import lombok.Singular; -import lombok.Value; -import lombok.With; - -//TODO this class is less context and more state. It's also incredibly dangerous because it hides a lot of moving parts and creates indirections. -@Value -@With -@Builder(toBuilder = true) -public class ConversionContext implements Context { - - IdColumnConfig idColumns; - - Clock clock; - String defaultPrimaryColumn; - PrintSettings sqlPrintSettings; - NodeConversionDispatcher nodeConversions; - LegacyCompilerDialect compilerDialect; - SqlNameGenerator nameGenerator; - @Singular - List querySteps; - @Nullable - SqlQuery finalQuery; - @Nullable - QueryStep stratificationTable; - @Nullable - QueryStep externalExtras; - - /** - * An optional date restriction range. Is set when converting a {@link CQDateRestriction}. - */ - @Nullable - CDateRange dateRestrictionRange; - - /** - * An optional secondary id to group results by in addition to the primary id. Only set when converting {@link SecondaryIdQuery}s. - */ - @Nullable - SecondaryIdDescription secondaryIdDescription; - - boolean negation; - - boolean isGroupBy; - - public boolean dateRestrictionActive() { - return this.dateRestrictionRange != null; - } - - public boolean isWithStratification() { - return this.stratificationTable != null; - } - - - /** - * Adds a query step to the list of {@link QueryStep} of this context. - */ - public ConversionContext withQueryStep(QueryStep queryStep) { - return this.toBuilder().queryStep(queryStep).build(); - } - - /** - * @return Creates a child context that keeps the meta information of this context, but empties the list of already converted querySteps. - */ - public ConversionContext createChildContext() { - return this.toBuilder().clearQuerySteps().build(); - } - - @Override - public ConversionContext getConversionContext() { - return this; - } - - - /** - * Get the last query {@link QueryStep} that has been added to this context query steps. - */ - public QueryStep getLastConvertedStep() { - return this.querySteps.getLast(); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java deleted file mode 100644 index d84deb9eae..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ /dev/null @@ -1,286 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import com.bakdata.conquery.sql.compiler.ir.JoinMode; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.Selects; -import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteInput; -import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteCompiler; -import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCteStep; -import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCteCompiler; -import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCtePipelineAssembler; -import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCtePipelineInput; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.stream.Stream; - -import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; -import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; -import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.models.datasets.Table; -import com.bakdata.conquery.models.datasets.concepts.ConceptElement; -import com.bakdata.conquery.models.datasets.concepts.Connector; -import com.bakdata.conquery.models.datasets.concepts.ValidityDate; -import com.bakdata.conquery.models.datasets.concepts.select.Select; -import com.bakdata.conquery.models.datasets.concepts.select.concept.ConceptColumnSelect; -import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; -import com.bakdata.conquery.models.identifiable.ids.specific.ConceptElementId; -import com.bakdata.conquery.models.identifiable.ids.specific.SelectId; -import com.bakdata.conquery.sql.compiler.ir.condition.ConditionWrappingWhereCondition; -import com.bakdata.conquery.sql.compiler.ir.condition.DateRestrictionCondition; -import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.models.query.DateAggregationAction; -import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.*; -import com.bakdata.conquery.sql.compiler.ir.concept.SqlFilters; -import com.bakdata.conquery.sql.compiler.ir.condition.WhereClauses; -import com.bakdata.conquery.sql.compiler.ir.condition.WhereCondition; -import com.bakdata.conquery.sql.compiler.ir.concept.ConceptSqlSelects; -import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; -import com.bakdata.conquery.sql.conversion.model.select.SelectContext; -import com.bakdata.conquery.util.TablePrimaryColumnUtil; -import com.google.common.base.Preconditions; -import org.jooq.Condition; -import org.jooq.Field; -import org.jooq.Record; - -import static org.jooq.impl.DSL.*; - -public class CQConceptConverter implements NodeConverter { - - private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcept cqConcept, TablePath tablePath, ConversionContext context) { - - ConceptSqlTables universalTables = tablePath.createConceptTables(predecessor); - Selects predecessorSelects = predecessor.getQualifiedSelects(); - Optional validityDate = predecessorSelects.getValidityDate(); - SqlIdColumns ids = predecessorSelects.getIds(); - - SelectContext selectContext = SelectContext.create(ids, validityDate, universalTables, context); - List converted = cqConcept.getSelects().stream() - .map(selectId -> { - Select select = selectId.resolve(); - return context.getCompilerDialect().getSelectConverter(select).conceptSelect(select, selectContext); - }) - .toList(); - - return ConceptCteCompiler.compileConcept(new ConceptCteInput( - predecessor, - converted, - universalTables, - context.isNegation() - )); - } - - public static SqlIdColumns convertIds(CQConcept cqConcept, CQTable cqTable, ConversionContext conversionContext) { - - Table table = cqTable.getConnector().resolve().getResolvedTable(); - Field primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(table, conversionContext.getDefaultPrimaryColumn()); - - if (cqConcept.isExcludeFromSecondaryId() - || conversionContext.getSecondaryIdDescription() == null - || !cqTable.hasSelectedSecondaryId(conversionContext.getSecondaryIdDescription().getId()) - ) { - return new SqlIdColumns(primaryColumn).withAlias(); - } - - Column secondaryIdColumn = table.findSecondaryIdColumn(conversionContext.getSecondaryIdDescription().getId()); - - Preconditions.checkArgument( - secondaryIdColumn != null, - "Expecting Table %s to have a matching secondary id for %s".formatted( - table, - conversionContext.getSecondaryIdDescription() - ) - ); - - Field secondaryId = field(name(table.getName(), secondaryIdColumn.getName()), String.class); - return new SqlIdColumns(primaryColumn, secondaryId).withAlias(); - } - - private static ColumnDateRange convertValidityDate(ConversionContext context, ValidityDate validityDate) { - SqlFunctionProvider functionProvider = context.getFunctionProvider(); - ColumnDateRange sqlValidityDate; - - boolean hasValidityDate = validityDate != null; - boolean hasDateRestriction = context.getDateRestrictionRange() != null; - - if (hasValidityDate) { - if (hasDateRestriction) { - sqlValidityDate = functionProvider.forValidityDate(validityDate, context.getDateRestrictionRange()); - } else { - sqlValidityDate = functionProvider.forValidityDate(validityDate); - } - } else { - if (hasDateRestriction) { - sqlValidityDate = functionProvider.forCDateRange(context.getDateRestrictionRange()); - } else { - sqlValidityDate = context.getCompilerDialect().unboundedDateRange(); - } - } - - return sqlValidityDate; - } - - private static SqlFilters collectConditionFilters( - CQTable cqTable, SqlFunctionProvider functionProvider) { - List conditions = new ArrayList<>(); - convertConnectorCondition(cqTable, functionProvider).ifPresent(conditions::add); - - Connector connector = cqTable.getConnector().resolve(); - if (connector.getColumn() != null) { - conditions.add(new ConditionWrappingWhereCondition(field(CTConditionContext.forConnector(connector, functionProvider).getConnectorColumn(), String.class).isNotNull())); - } - - ValidityDate validityDate = cqTable.findValidityDate(); - Condition validityDateFilter = noCondition(); - if (validityDate != null) { - validityDateFilter = functionProvider.isNotEmptyValidityDate(validityDate); - } - - return new SqlFilters(ConnectorSqlSelects.none(), - WhereClauses.builder() - .preprocessingConditions(conditions) - .preprocessingCondition(new ConditionWrappingWhereCondition(validityDateFilter)) - .build() - ); - } - - private static Optional convertConnectorCondition(CQTable cqTable, SqlFunctionProvider functionProvider) { - final Connector connector = cqTable.getConnector().resolve(); - - return Optional.ofNullable(connector.getCondition()) - .map(condition -> condition.convertToSqlCondition(CTConditionContext.forConnector(connector, functionProvider))); - } - - private static SqlFilters dateRestrictionFilter(ConversionContext context, ColumnDateRange validityDate) { - - List conditions = new ArrayList<>(); - SqlFunctionProvider functionProvider = context.getFunctionProvider(); - - if (context.getDateRestrictionRange() != null) { - ColumnDateRange dateRestriction = functionProvider.forCDateRange(context.getDateRestrictionRange()); - conditions.add(new DateRestrictionCondition(dateRestriction, validityDate)); - } - - return new SqlFilters( - ConnectorSqlSelects.none(), - WhereClauses.builder() - .eventFilters(conditions) - .build() - ); - } - - private static ConnectorSqlSelects createConceptColumnConnectorSqlSelects(CQConcept cqConcept, SelectContext selectContext) { - - - return cqConcept.getSelects().stream() - .map(SelectId::resolve) - .filter(select -> select instanceof ConceptColumnSelect) - .findFirst() - .map(select -> selectContext.getCompilerDialect().getSelectConverter(select).connectorSelect(select, selectContext)) - .orElse(ConnectorSqlSelects.none()); - } - - @Override - public Class getConversionClass() { - return CQConcept.class; - } - - @Override - public ConversionContext convert(CQConcept cqConcept, ConversionContext context) { - - TablePath tablePath = new TablePath(cqConcept, context); - List convertedCQTables = cqConcept.getTables().stream() - .flatMap(cqTable -> convertCqTable(tablePath, cqConcept, cqTable, context).stream()) - .toList(); - - QueryStep joinedStep = QueryStepComposer.joinSteps(convertedCQTables, JoinMode.FULL_OUTER, DateAggregationAction.MERGE, context); - QueryStep lastConceptStep = finishConceptConversion(joinedStep, cqConcept, tablePath, context); - return context.withQueryStep(lastConceptStep); - } - - private Optional convertCqTable(TablePath tablePath, CQConcept cqConcept, CQTable cqTable, ConversionContext context) { - ConnectorCtePipelineInput input = createConnectorCteInput(tablePath, cqConcept, cqTable, context); - return ConnectorCteCompiler.compileConnector(input); - } - - private ConnectorCtePipelineInput createConnectorCteInput( - TablePath tablePath, - CQConcept cqConcept, - CQTable cqTable, - ConversionContext conversionContext - ) { - - SqlIdColumns ids = convertIds(cqConcept, cqTable, conversionContext); - ConnectorSqlTables connectorTables = tablePath.getConnectorTables(cqTable); - ColumnDateRange validityDateCalculation = convertValidityDate(conversionContext, cqTable.findValidityDate()); - - // convert filters - SqlFunctionProvider functionProvider = conversionContext.getFunctionProvider(); - List allSqlFiltersForTable = new ArrayList<>(); - - cqTable.getFilters().stream() - .map(filterValue -> filterValue.convertToSqlFilter(ids, conversionContext, connectorTables)) - .forEach(allSqlFiltersForTable::add); - - List> conceptElements = cqConcept.getElements().stream().>map(ConceptElementId::resolve).toList(); - TreeConcept concept = (TreeConcept) cqConcept.getConcept(); - ConceptIdMapping conceptIdMapping = new ConceptIdMapping(concept, functionProvider); - boolean resolveConceptIds = cqConcept.getSelects().stream() - .map(SelectId::resolve) - .filter(ConceptColumnSelect.class::isInstance) - .map(ConceptColumnSelect.class::cast) - .anyMatch(ConceptColumnSelect::isAsIds); - allSqlFiltersForTable.add(collectConditionFilters(cqTable, functionProvider)); - - allSqlFiltersForTable.add(dateRestrictionFilter(conversionContext, validityDateCalculation)); - - // convert selects - SelectContext selectContext = SelectContext.create(ids, Optional.of(validityDateCalculation.asValidityDateRange(connectorTables.getName())), connectorTables, conversionContext); - List allSelectsForTable = new ArrayList<>(); - ConnectorSqlSelects conceptColumnSelect = createConceptColumnConnectorSqlSelects(cqConcept, selectContext); - allSelectsForTable.add(conceptColumnSelect); - cqTable.getSelects() - .stream() - .map(SelectId::resolve) - .map(select -> selectContext.getCompilerDialect().getSelectConverter(select).connectorSelect(select, selectContext)) - .forEach(allSelectsForTable::add); - - return ConnectorCtePipelineAssembler.assemble( - connectorTables.getPlan(), - createConceptSourceTable( - connectorTables, - conceptIdMapping, - conceptElements, - resolveConceptIds, - conversionContext.getCompilerDialect() - ), - ids, - validityDateCalculation, - allSelectsForTable, - allSqlFiltersForTable, - Optional.ofNullable(conversionContext.getStratificationTable()) - ); - } - - private static org.jooq.Table createConceptSourceTable( - ConnectorSqlTables connectorTables, - ConceptIdMapping conceptIdMapping, - List> selectedConceptElements, - boolean resolveConceptIds, - com.bakdata.conquery.sql.compiler.dialect.CompilerDialect dialect - ) { - org.jooq.Table connectorTable = table(name(connectorTables.getPredecessor(ConceptCteStep.PREPROCESSING))); - return (org.jooq.Table) conceptIdMapping.source(connectorTables.getConnector()).selectedConcepts( - connectorTable, - conceptIdMapping.includedLocalIds(selectedConceptElements), - conceptIdMapping.includesRoot(selectedConceptElements), - resolveConceptIds, - dialect - ); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java deleted file mode 100644 index 335fc419f6..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import java.util.List; - -import com.bakdata.conquery.sql.compiler.ir.SqlTables; -import lombok.Getter; - -@Getter -public class ConceptSqlTables extends SqlTables { - - private final List connectorTables; - - public ConceptSqlTables( - SqlTables tables, - List connectorTables - ) { - super(tables); - this.connectorTables = connectorTables; - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java deleted file mode 100644 index 5934294a87..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import com.bakdata.conquery.models.datasets.concepts.Connector; -import com.bakdata.conquery.sql.compiler.ir.SqlTables; -import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCtePlan; -import lombok.Getter; - -@Getter -public class ConnectorSqlTables extends SqlTables { - - /** - * Corresponding {@link Connector} of these {@link SqlTables}. - */ - private final Connector connector; - private final ConnectorCtePlan plan; - - public ConnectorSqlTables( - Connector connector, - ConnectorCtePlan plan - ) { - super(plan.tables()); - this.connector = connector; - this.plan = plan; - } - - /** A unique label for these tables to create unique SQL CTE/select names. */ - public String getName() { - return plan.connectorName(); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java deleted file mode 100644 index 004306ab90..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import javax.annotation.Nullable; - -import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; -import com.bakdata.conquery.sql.conversion.Context; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.LegacyCompilerDialect; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.compiler.ir.SqlIdColumns; -import com.bakdata.conquery.sql.compiler.ir.SqlTables; -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Value; - -@Value -@AllArgsConstructor(access = AccessLevel.PRIVATE) -public class FilterContext implements Context { - - SqlIdColumns ids; - - /** - * A filter value ({@link FilterValue#getValue()}) - */ - V value; - - ConversionContext conversionContext; - - /** - * Not present if this context is for table export. - */ - @Nullable - SqlTables tables; - - public static FilterContext forConceptConversion(SqlIdColumns ids, V value, ConversionContext conversionContext, SqlTables tables) { - return new FilterContext<>(ids, value, conversionContext, tables); - } - - public static FilterContext forTableExport(SqlIdColumns ids, V value, ConversionContext conversionContext) { - return new FilterContext<>(ids, value, conversionContext, null); - } - - @Override - public LegacyCompilerDialect getCompilerDialect() { - return getConversionContext().getCompilerDialect(); - } - - public SqlFunctionProvider getFunctionProvider() { - return getCompilerDialect().getFunctionProvider(); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java deleted file mode 100644 index 5a627fd8a5..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java +++ /dev/null @@ -1,80 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; -import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; -import com.bakdata.conquery.models.datasets.concepts.Connector; -import com.bakdata.conquery.models.datasets.concepts.select.Select; -import com.bakdata.conquery.models.identifiable.ids.specific.SelectId; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.SqlTables; -import com.bakdata.conquery.sql.compiler.ir.concept.ConceptCtePlanner; -import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorCtePlan; -import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.model.EntitySchemaAdapter; -import lombok.Getter; - -/** - * Resolves legacy concept models and attaches their backend metadata to the connector-owned CTE plans. - */ -class TablePath { - - private final Map connectorTableMap = new HashMap<>(); - - @Getter - private final CQConcept cqConcept; - - @Getter - private final ConversionContext context; - - public TablePath(CQConcept cqConcept, ConversionContext context) { - this.cqConcept = cqConcept; - this.context = context; - cqConcept.getTables().forEach(cqTable -> this.connectorTableMap.put(cqTable, createConnectorTables(cqConcept, cqTable, context))); - } - - public ConnectorSqlTables getConnectorTables(CQTable cqTable) { - return connectorTableMap.get(cqTable); - } - - private static ConnectorSqlTables createConnectorTables(CQConcept cqConcept, CQTable cqTable, ConversionContext context) { - Connector connector = cqTable.getConnector().resolve(); - String connectorName = context.getNameGenerator().legacyConceptConnectorName( - cqConcept.userLabel(context.getSqlPrintSettings().getLocale()), - connector.getName() - ); - boolean eventDateSelectsPresent = cqTable.getSelects().stream() - .map(SelectId::resolve) - .anyMatch(Select::isEventDateSelect); - ConnectorCtePlan plan = ConceptCtePlanner.planConnector( - EntitySchemaAdapter.from(connector.getResolvedTable()), - connectorName, - cqConcept.isAggregateEventDates(), - eventDateSelectsPresent, - context.getCompilerDialect(), - context.getNameGenerator() - ); - - return new ConnectorSqlTables(connector, plan); - } - - public ConceptSqlTables createConceptTables(QueryStep predecessor) { - String conceptName = context.getNameGenerator().legacyConceptName(cqConcept.userLabel(context.getSqlPrintSettings().getLocale())); - boolean eventDateSelectsPresent = cqConcept.getSelects().stream() - .map(SelectId::resolve) - .anyMatch(Select::isEventDateSelect); - SqlTables tables = ConceptCtePlanner.planConcept( - predecessor, - conceptName, - eventDateSelectsPresent, - context.getCompilerDialect(), - context.getNameGenerator() - ); - List connectorSqlTables = this.connectorTableMap.values().stream().toList(); - - return new ConceptSqlTables(tables, connectorSqlTables); - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java index 1e38a3ab88..b146ebbbeb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java @@ -3,22 +3,24 @@ import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.Dialect; import com.bakdata.conquery.models.events.MajorTypeId; +import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import org.jooq.Field; import org.jooq.SQLDialect; /** - * Compatibility aggregate used when wiring all database-specific services for a backend namespace. + * Aggregate used when wiring database-specific services for a backend namespace. * - *

The bundle exposes both the legacy compiler adapter inherited from {@link LegacyCompilerDialect} and the runtime - * methods declared here. Legacy compiler code must depend on {@code LegacyCompilerDialect}, not on this aggregate. - * Runtime wiring may retain the bundle when it needs to construct both compiler and execution services for the same - * database.

+ *

The compiler and SQL-function capabilities are exposed as separate services. Runtime wiring may retain the + * bundle when it needs to construct compiler and execution services for the same database.

* - *

This interface keeps the existing HANA and ClickHouse integrations source-compatible while their compiler and - * execution capabilities are separated. It is an application composition boundary, not a compiler dependency.

+ *

This interface is an application composition boundary, not a compiler dependency.

*/ -public interface DialectBundle extends LegacyCompilerDialect { +public interface DialectBundle { + + CompilerDialect getCompilerDialect(); + + SqlFunctionProvider getFunctionProvider(); ResultSetProcessor getResultSetProcessor(ConqueryConfig config); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java deleted file mode 100644 index 6b70ae35fc..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/LegacyCompilerDialect.java +++ /dev/null @@ -1,256 +0,0 @@ -package com.bakdata.conquery.sql.conversion.dialect; - -import org.jooq.DataType; -import org.jooq.SortField; -import org.jooq.OrderField; -import java.util.function.Function; -import java.util.Collection; -import java.sql.Date; -import java.time.LocalDate; -import java.time.temporal.ChronoUnit; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import com.bakdata.conquery.models.datasets.concepts.select.Select; -import com.bakdata.conquery.models.query.Visitable; -import com.bakdata.conquery.sql.compiler.dialect.CompilerDialect; -import com.bakdata.conquery.sql.compiler.conversion.operation.SelectConversionContext; -import com.bakdata.conquery.sql.compiler.ir.concept.ConnectorSqlSelects; -import com.bakdata.conquery.sql.model.operation.BuiltInSelects; -import com.bakdata.conquery.sql.compiler.ir.QueryStep; -import com.bakdata.conquery.sql.compiler.ir.select.ColumnDateRange; -import com.bakdata.conquery.sql.compiler.rendering.QueryStepRenderer; -import com.bakdata.conquery.sql.model.range.DateRange; -import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.cqelement.CQAndConverter; -import com.bakdata.conquery.sql.conversion.cqelement.CQDateRestrictionConverter; -import com.bakdata.conquery.sql.conversion.cqelement.CQExternalConverter; -import com.bakdata.conquery.sql.conversion.cqelement.CQNegationConverter; -import com.bakdata.conquery.sql.conversion.cqelement.CQOrConverter; -import com.bakdata.conquery.sql.conversion.cqelement.CQYesConverter; -import com.bakdata.conquery.sql.conversion.cqelement.concept.CQConceptConverter; -import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; -import com.bakdata.conquery.sql.conversion.query.CQReusedQueryConverter; -import com.bakdata.conquery.sql.conversion.query.ConceptQueryConverter; -import com.bakdata.conquery.sql.conversion.query.SecondaryIdQueryConverter; -import org.jooq.Condition; -import org.jooq.DSLContext; -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Table; - -/** - * Temporary backend adapter exposing services required by the legacy SQL compiler. - * - *

The framework-neutral dialect contract lives in {@link CompilerDialect}. This adapter retains dependencies on - * backend query DTOs, converter registries, and legacy compiler services until those implementations move into the SQL - * connector. New connector code must not depend on this interface.

- */ -public interface LegacyCompilerDialect extends CompilerDialect { - - @Override - default Condition unconditionalJoinCondition() { - return getCompilerDialect().unconditionalJoinCondition(); - } - - @Override - default ConnectorSqlSelects distinctSelect(BuiltInSelects.Values select, SelectConversionContext context) { - return getCompilerDialect().distinctSelect(select, context); - } - - @Override - default ColumnDateRange toDualColumn(ColumnDateRange range) { - return getCompilerDialect().toDualColumn(range); - } - - @Override - default Condition orAgg(Field field) { - return getCompilerDialect().orAgg(field); - } - - @Override - default Field arrayOut(List> fields) { - return getCompilerDialect().arrayOut(fields); - } - - @Override - default Field cast(Field field, DataType type) { - return getCompilerDialect().cast(field, type); - } - - @Override - default Field stringAggregation(Field field, Field delimiter, List> orderByFields) { - return getCompilerDialect().stringAggregation(field, delimiter, orderByFields); - } - - @Override - default Collection> orderByValidityDates( - Function, ? extends SortField> ordering, List> validityDateFields) { - return getCompilerDialect().orderByValidityDates(ordering, validityDateFields); - } - - @Override - default Field random(Field field) { - return getCompilerDialect().random(field); - } - - @Override - default Field minimumDate() { - return getCompilerDialect().minimumDate(); - } - - @Override - default Field maximumDate() { - return getCompilerDialect().maximumDate(); - } - - @Override - default Field anyValue(Field field) { - return getCompilerDialect().anyValue(field); - } - - @Override - default Field renderDateRange(Field start, Field end) { - return getCompilerDialect().renderDateRange(start, end); - } - - @Override - default Field aggregateDateRanges(Field start, Field end) { - return getCompilerDialect().aggregateDateRanges(start, end); - } - - @Override - default Field externalId(String id) { - return getCompilerDialect().externalId(id); - } - - @Override - default Field externalStringValues(List values) { - return getCompilerDialect().externalStringValues(values); - } - - @Override - default Table literalSelectTable() { - return getCompilerDialect().literalSelectTable(); - } - - @Override - default ColumnDateRange dateRangeLiteral(DateRange dateRange) { - return getCompilerDialect().dateRangeLiteral(dateRange); - } - - @Override - default QueryStep unnestDateRange(ColumnDateRange dateRange, QueryStep predecessor, String cteName) { - return getCompilerDialect().unnestDateRange(dateRange, predecessor, cteName); - } - - @Override - default String regexAnyCharacters() { - return getCompilerDialect().regexAnyCharacters(); - } - - @Override - default Condition regexMatches(Field field, String pattern) { - return getCompilerDialect().regexMatches(field, pattern); - } - - @Override - default Field dateDistance(ChronoUnit unit, Field startDate, LocalDate endDate) { - return getCompilerDialect().dateDistance(unit, startDate, endDate); - } - - @Override - default com.bakdata.conquery.sql.compiler.forms.StratificationFunctions stratificationFunctions() { - return getCompilerDialect().stratificationFunctions(); - } - - @Override - default ColumnDateRange intersection(ColumnDateRange left, ColumnDateRange right) { - return getCompilerDialect().intersection(left, right); - } - - @Override - default Condition isNotEmptyDateRange(ColumnDateRange range) { - return getCompilerDialect().isNotEmptyDateRange(range); - } - - @Override - default boolean supportsSingleColumnRanges() { - return getCompilerDialect().supportsSingleColumnRanges(); - } - - @Override - default Field dateDistance(ChronoUnit unit, Field startDate, Field endDate) { - return getCompilerDialect().dateDistance(unit, startDate, endDate); - } - - @Override - default Field addDays(Field date, Field days) { - return getCompilerDialect().addDays(date, days); - } - - @Override - default ColumnDateRange dateRange(Field start, Field inclusiveEnd) { - return getCompilerDialect().dateRange(start, inclusiveEnd); - } - - @Override - default ColumnDateRange dateRangeColumn(Field range) { - // Both existing SQL providers use a single physical column as inclusive start and end. - Field date = range.coerce(Date.class); - return dateRange(date, date); - } - - @Override - default Field yearQuarter(Field date) { - return getCompilerDialect().yearQuarter(date); - } - - @Override - default Field quarterStart(Field date) { - return getCompilerDialect().quarterStart(date); - } - - @Override - default Field nextQuarterStart(Field date) { - return getCompilerDialect().nextQuarterStart(date); - } - - SqlFunctionProvider getFunctionProvider(); - - CompilerDialect getCompilerDialect(); - - List> getNodeConverters(DSLContext context); - - default List> getDefaultNodeConverters(DSLContext dslContext) { - - QueryStepRenderer queryStepRenderer = new QueryStepRenderer(dslContext); - return List.of( - new CQDateRestrictionConverter(), - new CQAndConverter(), - new CQOrConverter(), - new CQNegationConverter(), - new CQYesConverter(), - new CQConceptConverter(), - new CQExternalConverter(), - new CQReusedQueryConverter(), - new ConceptQueryConverter(queryStepRenderer), - new SecondaryIdQueryConverter() - ); - } - - default Map, ? extends SelectConverter> getSelectConverterOverrides() { - return Collections.emptyMap(); - } - - default SelectConverter maybeOverride = (SelectConverter