diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java index 6a92f8c30c..0ab1b4260b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java @@ -25,7 +25,6 @@ import org.jooq.*; import org.jooq.Record; import org.jooq.exception.DataAccessException; -import org.jooq.impl.SQLDataType; @Slf4j @Data @@ -282,7 +281,9 @@ private SelectJoinStep> createMatchin SelectJoinStep> records = dslContext.with(unioned).select(unioned.field(CONCEPT_ID_FIELD), PID_FIELD, // The infinities are intentionally swapped - functionProvider.cast(nullif(unioned.field(LB_FIELD), positiveInfinity), SQLDataType.INTEGER).as(LB_FIELD), functionProvider.cast(nullif(unioned.field(UB_FIELD), negativeInfinity), SQLDataType.INTEGER).as(UB_FIELD)).from(ct_name); + functionProvider.dateToInteger(nullif(unioned.field(LB_FIELD), positiveInfinity)).as(LB_FIELD), + functionProvider.dateToInteger(nullif(unioned.field(UB_FIELD), negativeInfinity)).as(UB_FIELD) + ).from(ct_name); return records; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index e1d9c5591c..6fdf74fdd4 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -21,7 +21,6 @@ import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import org.jooq.Condition; -import org.jooq.DataType; import org.jooq.Field; import org.jooq.OrderField; import org.jooq.Record; @@ -68,7 +67,9 @@ Collection> orderByValidityDates( */ Field getMaxDateExpression(); - Field cast(Field field, DataType type); + Field castToString(Field field); + + Field castToInteger(Field field); /** * @return The regex that matches any char repeated any times (including 0), for example: @@ -318,4 +319,8 @@ default Condition unconditionalJoinCondition(){ default Field isNull(Field field){ return field.isNull(); } + + default Field dateToInteger(Field field) { + return castToInteger(field); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java index b7225a4f74..13ee2c6516 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java @@ -17,7 +17,6 @@ import com.bakdata.conquery.sql.conversion.model.QueryStep; import org.jetbrains.annotations.NotNull; import org.jooq.Condition; -import org.jooq.DataType; import org.jooq.Field; import org.jooq.OrderField; import org.jooq.SortField; @@ -271,14 +270,16 @@ public Field dateRangeToField(ColumnDateRange columnDateRange) { } @Override - public Field cast(Field field, DataType type) { - if (type == SQLDataType.VARCHAR) { - return function("toString", type.getType(), field); - } + public Field castToString(Field field) { + return function("toString", String.class, field); + } + + @Override + public Field castToInteger(Field field) { return function( name("CAST"), - type.getType(), - field("{0} AS Nullable({1})", field, keyword(type.getName())) + Integer.class, + field("{0} AS Nullable({1})", field, keyword(SQLDataType.INTEGER.getName())) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseResultSetProcessor.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseResultSetProcessor.java index 1abe779086..51acaa39f8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseResultSetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseResultSetProcessor.java @@ -2,16 +2,14 @@ import java.math.BigDecimal; import java.sql.Array; -import java.sql.Date; import java.sql.ResultSet; import java.sql.SQLException; -import java.time.LocalDate; +import java.sql.Struct; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; -import com.bakdata.conquery.models.common.CDate; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.sql.execution.DefaultResultSetProcessor; @@ -19,73 +17,62 @@ public class ClickhouseResultSetProcessor extends DefaultResultSetProcessor { - public static final String DATE_SEPARATOR = ","; - public ClickhouseResultSetProcessor(ConqueryConfig config) { super(config, null); } @Override public List getDateRange(ResultSet resultSet, int columnIndex) throws SQLException { - String daterange = resultSet.getString(columnIndex); + Object daterange = resultSet.getObject(columnIndex); if (daterange == null) { return Collections.emptyList(); } - String[] dates = daterange.split(DATE_SEPARATOR); - Preconditions.checkArgument(dates.length == 2, "Dateranges must have a start and end."); - - String startDateExpression = dates[0]; - int startDate; - if (startDateExpression.equals(ClickhouseFunctionProvider.MIN_DATE_VALUE)) { - startDate = CDateRange.NEGATIVE_INFINITY; - } - else { - LocalDate dateValue = Date.valueOf(startDateExpression).toLocalDate(); - startDate = CDate.ofLocalDate(dateValue); - } - - String endDateExpression = dates[1]; - int endDate; - if (endDateExpression.equals(ClickhouseFunctionProvider.MAX_DATE_VALUE)) { - endDate = CDateRange.POSITIVE_INFINITY; - } - else { - LocalDate dateValue = Date.valueOf(endDateExpression).toLocalDate(); - endDate = CDate.ofLocalDate(dateValue); - } - - return List.of(startDate, endDate); + Object[] rawTuple = daterange instanceof Struct struct ? struct.getAttributes() : (Object[]) daterange; + return toEpochDayRange(rawTuple); } public List> toEpochDayRangeList(List raw) { + if (raw == null) { + return null; + } + List> out = new ArrayList<>(); for (Object[] rawTuple : raw) { - - if (rawTuple[0] == null) { - assert rawTuple[1] == null; - continue; + List dateRange = toEpochDayRange(rawTuple); + if (!dateRange.isEmpty()) { + out.add(dateRange); } + } - int begin = (Integer) rawTuple[0]; - int end = (Integer) rawTuple[1]; + return out; + } - if (begin == ClickhouseFunctionProvider.MIN_DATE_VALUE) { - begin = Integer.MIN_VALUE; - } - if (end == ClickhouseFunctionProvider.MAX_DATE_VALUE) { - end = Integer.MAX_VALUE; - } - else { - end--; - } + private List toEpochDayRange(Object[] rawTuple) { + Preconditions.checkArgument(rawTuple.length == 2, "Dateranges must have a start and end."); + + if (rawTuple[0] == null && rawTuple[1] == null) { + return Collections.emptyList(); + } + Preconditions.checkArgument(rawTuple[0] != null && rawTuple[1] != null, "Dateranges must have a start and end."); + Preconditions.checkArgument(rawTuple[0] instanceof Number && rawTuple[1] instanceof Number, "Dateranges must contain integer values."); - out.add(List.of(begin, end)); + int begin = ((Number) rawTuple[0]).intValue(); + int end = ((Number) rawTuple[1]).intValue(); + + if (begin == ClickhouseFunctionProvider.MIN_DATE_VALUE) { + begin = CDateRange.NEGATIVE_INFINITY; + } + if (end == ClickhouseFunctionProvider.MAX_DATE_VALUE) { + end = CDateRange.POSITIVE_INFINITY; + } + else { + end--; } - return out; + return List.of(begin, end); } private List sort(List list, Comparator sort) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java index 3126dcae20..bad5d9f927 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java @@ -19,7 +19,6 @@ import org.jooq.Field; import org.jooq.Record; import org.jooq.Table; -import org.jooq.impl.SQLDataType; @Getter @RequiredArgsConstructor @@ -120,9 +119,16 @@ public Field intSeriesField() { } @Override - public Table generateIntSeries(int start, int end) { - //TODO this only supports ungigned values. Probably need to handle negatives manually Q_Q - return table("generate_series({0}, {1}, 1)", start, end); + public Table generateIntSeries(int start, int end) { + Field startInclusive = functionProvider.castToInteger(inline(start)); + Field endExclusive = functionProvider.castToInteger(inline(end).plus(inline(1))); + Field seriesIndex = field( + "arrayJoin(range({0}, {1}))", + Integer.class, + startInclusive, + endExclusive + ).as(SERIES_INDEX.getName()); + return select(seriesIndex).asTable(); } @Override @@ -134,7 +140,7 @@ public Field indexSelectorField(TemporalSamplerFactory indexSelector, Colu // we calculate a random int which is in range of the date distance between upper and lower bound Field dateDistanceInDays = functionProvider.dateDistance(ChronoUnit.DAYS, validityDate.getStart(), validityDate.getEnd()); Field randomAmountOfDays = function("RAND", Double.class).times(dateDistanceInDays); - Field flooredAsInt = functionProvider.cast(floor(randomAmountOfDays), SQLDataType.INTEGER); + Field flooredAsInt = functionProvider.castToInteger(floor(randomAmountOfDays)); // then we add this random amount (of days) to the start date Field randomDateInRange = functionProvider.addDays(lower(validityDate), flooredAsInt); // finally, we handle multiple ranges by randomizing which range we use to select a random date from @@ -170,7 +176,7 @@ private Field calcDate(Field start, Interval interval, Offset offset private Field getQuartersInMonths(Field date, Offset offset) { Field quarterExpression = functionProvider.yearQuarter(date); Field rightMostCharacter = function("RIGHT", String.class, quarterExpression, inline(1)); - Field amountOfQuarters = functionProvider.cast(rightMostCharacter, SQLDataType.INTEGER) + Field amountOfQuarters = functionProvider.castToInteger(rightMostCharacter) .plus(offset.getOffset()); return amountOfQuarters.times(MONTHS_PER_QUARTER); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java index 87d60f3997..27c6428eb2 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java @@ -20,102 +20,100 @@ import com.bakdata.conquery.sql.conversion.model.QueryStep; import org.jetbrains.annotations.NotNull; import org.jooq.Condition; -import org.jooq.DataType; import org.jooq.Field; import org.jooq.OrderField; import org.jooq.Record; import org.jooq.SortField; import org.jooq.Table; import org.jooq.impl.DSL; -import org.jooq.impl.SQLDataType; public class HanaSqlFunctionProvider implements SqlFunctionProvider { - public static final String MAX_DATE_VALUE = "9999-12-31"; - public static final String MIN_DATE_VALUE = "0001-01-01"; - public static final String DATERANGE_SEPARATOR = "/"; - - public static final char DATE_SET_SEPARATOR = UNIT_SEPARATOR; - private static final String ANY_CHAR_REGEX = ".*"; - private static final String NOP_TABLE = "DUMMY"; - - @Override - public String getAnyCharRegex() { - return ANY_CHAR_REGEX; - } - - - @Override - public Table getNoOpTable() { - // see https://help.sap.com/docs/SAP_DATA_HUB/e8d3e271a4554a35a5a6136d3d6af3f8/4d4b939b37b84bea8b2aa2ada640c392.html - return table(name(NOP_TABLE)); - } - - @Override - public Condition unconditionalJoinCondition() { - // Hana requires a specific syntax structure, this is the minimal solution. - return inline(true).eq(inline(true)); - } - - @Override - public Condition dateRestriction(ColumnDateRange dateRestriction, ColumnDateRange daterange) { - Condition dateRestrictionStartsBeforeDate = dateRestriction.getStart().lessThan(daterange.getEnd()); - Condition dateRestrictionEndsAfterDate = dateRestriction.getEnd().greaterThan(daterange.getStart()); - - return condition(dateRestrictionStartsBeforeDate.and(dateRestrictionEndsAfterDate)); - } - - - @Override - public ColumnDateRange forCDateRange(CDateRange daterange) { - - String startDateExpression = MIN_DATE_VALUE; - String endDateExpression = MAX_DATE_VALUE; - - if (daterange.hasLowerBound()) { - startDateExpression = daterange.getMin().toString(); - } - if (daterange.hasUpperBound()) { - // end date is expected to be handled as exclusive, but if it's already the maximum date, we can't add +1 day - if (Objects.equals(daterange.getMax(), LocalDate.ofEpochDay(CDateRange.POSITIVE_INFINITY))) { - throw new UnsupportedOperationException( - "Given daterange has an upper bound of CDateRange.POSITIVE_INFINITY, which is not supported by ConQuery's HANA dialect."); - } - LocalDate exclusiveMaxDate = daterange.getMax().plusDays(1); - endDateExpression = exclusiveMaxDate.toString(); - } - - return ColumnDateRange.of(toDateField(startDateExpression), toDateField(endDateExpression)); - } - - @Override - public Field toDateField(String dateExpression) { - return function( - "TO_DATE", - Date.class, - inline(dateExpression), - inline(DEFAULT_DATE_FORMAT) - ); - } - - @Override - public Condition isNotEmptyDateRange(ColumnDateRange columnDateRange) { - return columnDateRange.getStart().notEqual(getMinDateExpression()).or(columnDateRange.getEnd().notEqual(getMaxDateExpression())); - - } - - - @Override + public static final String MAX_DATE_VALUE = "9999-12-31"; + public static final String MIN_DATE_VALUE = "0001-01-01"; + public static final String DATERANGE_SEPARATOR = "/"; + + public static final char DATE_SET_SEPARATOR = UNIT_SEPARATOR; + private static final String ANY_CHAR_REGEX = ".*"; + private static final String NOP_TABLE = "DUMMY"; + + @Override + public String getAnyCharRegex() { + return ANY_CHAR_REGEX; + } + + + @Override + public Table getNoOpTable() { + // see https://help.sap.com/docs/SAP_DATA_HUB/e8d3e271a4554a35a5a6136d3d6af3f8/4d4b939b37b84bea8b2aa2ada640c392.html + return table(name(NOP_TABLE)); + } + + @Override + public Condition unconditionalJoinCondition() { + // Hana requires a specific syntax structure, this is the minimal solution. + return inline(true).eq(inline(true)); + } + + @Override + public Condition dateRestriction(ColumnDateRange dateRestriction, ColumnDateRange daterange) { + Condition dateRestrictionStartsBeforeDate = dateRestriction.getStart().lessThan(daterange.getEnd()); + Condition dateRestrictionEndsAfterDate = dateRestriction.getEnd().greaterThan(daterange.getStart()); + + return condition(dateRestrictionStartsBeforeDate.and(dateRestrictionEndsAfterDate)); + } + + + @Override + public ColumnDateRange forCDateRange(CDateRange daterange) { + + String startDateExpression = MIN_DATE_VALUE; + String endDateExpression = MAX_DATE_VALUE; + + if (daterange.hasLowerBound()) { + startDateExpression = daterange.getMin().toString(); + } + if (daterange.hasUpperBound()) { + // end date is expected to be handled as exclusive, but if it's already the maximum date, we can't add +1 day + if (Objects.equals(daterange.getMax(), LocalDate.ofEpochDay(CDateRange.POSITIVE_INFINITY))) { + throw new UnsupportedOperationException( + "Given daterange has an upper bound of CDateRange.POSITIVE_INFINITY, which is not supported by ConQuery's HANA dialect."); + } + LocalDate exclusiveMaxDate = daterange.getMax().plusDays(1); + endDateExpression = exclusiveMaxDate.toString(); + } + + return ColumnDateRange.of(toDateField(startDateExpression), toDateField(endDateExpression)); + } + + @Override + public Field toDateField(String dateExpression) { + return function( + "TO_DATE", + Date.class, + inline(dateExpression), + inline(DEFAULT_DATE_FORMAT) + ); + } + + @Override + public Condition isNotEmptyDateRange(ColumnDateRange columnDateRange) { + return columnDateRange.getStart().notEqual(getMinDateExpression()).or(columnDateRange.getEnd().notEqual(getMaxDateExpression())); + + } + + + @Override public ColumnDateRange forValidityDate(ValidityDate validityDate) { return toColumnDateRange(validityDate); } - @Override - public ColumnDateRange allRange() { - return ColumnDateRange.of(getMinDateExpression(), getMaxDateExpression()); - } + @Override + public ColumnDateRange allRange() { + return ColumnDateRange.of(getMinDateExpression(), getMaxDateExpression()); + } - @Override + @Override public Field anyValue(Field field) { // Hana does not have any_value return DSL.min(field); @@ -123,244 +121,245 @@ public Field anyValue(Field field) { private ColumnDateRange toColumnDateRange(ValidityDate validityDate) { - String tableName = validityDate.getConnector().resolveTableId().getTable(); - - Column startColumn; - Column endColumn; - - // if no end column is present, the only existing column is both start and end of the date range - if (validityDate.isSingleColumnDaterange()) { - Column column = validityDate.getColumn().resolve(); - startColumn = column; - endColumn = column; - } else { - startColumn = validityDate.getStartColumn().resolve(); - endColumn = validityDate.getEndColumn().resolve(); - } - - return ofStartAndEnd(tableName, startColumn, endColumn); - } - - private ColumnDateRange ofStartAndEnd(String tableName, Column startColumn, Column endColumn) { - - Field rangeStart = coalesce( - field(name(tableName, startColumn.getName()), Date.class), - getMinDateExpression() - ); - // when aggregating date ranges, we want to treat the last day of the range as excluded, - // so when using the date value of the end column, we add +1 day as end of the date range - Field rangeEnd = coalesce( - addDays(field(name(tableName, endColumn.getName()), Date.class), inline(1)), - getMaxDateExpression() - ); - - return ColumnDateRange.of(rangeStart, rangeEnd); - } - - @Override - public Field addDays(Field dateColumn, Field amountOfDays) { - return function( - "ADD_DAYS", - Date.class, - dateColumn, - amountOfDays - ); - } - - @Override - public ColumnDateRange allRangeIf(Condition condition) { - return ColumnDateRange.of( - when(condition.isTrue(), - getMinDateExpression() - ), - when(condition.isTrue(), - getMaxDateExpression() - ) - ); - } - - - @Override - public ColumnDateRange forValidityDate(ValidityDate validityDate, CDateRange dateRestriction) { - - ColumnDateRange validityDateRange = toColumnDateRange(validityDate); - ColumnDateRange restriction = toColumnDateRange(dateRestriction); - - Field lowerBound = when(validityDateRange.getStart().lessThan(restriction.getStart()), restriction.getStart()) - .otherwise(validityDateRange.getStart()); - - Field maxDate = toDateField(MAX_DATE_VALUE); // we want to add +1 day to the end date - except when it's the max date already - Field restrictionUpperBound = when(restriction.getEnd().eq(maxDate), maxDate).otherwise(addDays(restriction.getEnd(), inline(1))); - Field upperBound = when(validityDateRange.getEnd().greaterThan(restriction.getEnd()), restrictionUpperBound) - .otherwise(validityDateRange.getEnd()); - - return ColumnDateRange.of(lowerBound, upperBound); - } - - private ColumnDateRange toColumnDateRange(CDateRange dateRestriction) { - - String startDateExpression = MIN_DATE_VALUE; - String endDateExpression = MAX_DATE_VALUE; - - if (dateRestriction.hasLowerBound()) { - startDateExpression = dateRestriction.getMin().toString(); - } - if (dateRestriction.hasUpperBound()) { - endDateExpression = dateRestriction.getMax().toString(); - } - - return ColumnDateRange.of(toDateField(startDateExpression), toDateField(endDateExpression)); - } - - - @NotNull - @Override - public Collection> orderByValidityDates( - Function, ? extends SortField> ordering, - List> validityDateFields) { - - return List.of( - ordering.apply(nullif(validityDateFields.getFirst(), getMinDateExpression())).nullsLast(), - ordering.apply(nullif(validityDateFields.getLast(), getMaxDateExpression())).nullsLast() - ); - } - - @Override - public Field getMinDateExpression() { - return toDateField(MIN_DATE_VALUE); - } - - @Override - public Field getMaxDateExpression() { - return toDateField(MAX_DATE_VALUE); - } - - @Override - public ColumnDateRange forArbitraryDateRange(DaterangeSelectOrFilter daterangeSelectOrFilter) { - String tableName = daterangeSelectOrFilter.getTable().getName(); - if (daterangeSelectOrFilter.isSingleColumnDaterange()) { - Column column = daterangeSelectOrFilter.getColumn().resolve(); - return ofStartAndEnd(tableName, column, column); - } else { - return ofStartAndEnd(tableName, daterangeSelectOrFilter.getStartColumn().resolve(), daterangeSelectOrFilter.getEndColumn().resolve()); - } - } - - @Override - public ColumnDateRange aggregated(ColumnDateRange columnDateRange) { - return ColumnDateRange.of( - min(columnDateRange.getStart()), - max(columnDateRange.getEnd()) - ) - .as(columnDateRange.getAlias()); - } - - @Override - public ColumnDateRange toDualColumn(ColumnDateRange columnDateRange) { - // HANA does not support single column ranges - return ColumnDateRange.of(columnDateRange.getStart(), columnDateRange.getEnd()); - } - - @Override - public ColumnDateRange intersection(ColumnDateRange left, ColumnDateRange right) { - Field greatest = DSL.greatest(left.getStart(), right.getStart()); - Field least = DSL.least(left.getEnd(), right.getEnd()); - return ColumnDateRange.of(greatest, least); - } - - @Override - public QueryStep unnestDaterange(ColumnDateRange nested, QueryStep predecessor, String cteName) { - // HANA does not support single column datemultiranges - return predecessor; - } - - @Override - public Field dateRangeAggregation(ColumnDateRange columnDateRange) { - - Field stringAggregation = stringAggregation( - dateRangeToField(columnDateRange), - toChar(DATE_SET_SEPARATOR), - // The coalesce is necessary so that hana isn't upset about a potential `order by null` which happens for empty date-ranges + String tableName = validityDate.getConnector().resolveTableId().getTable(); + + Column startColumn; + Column endColumn; + + // if no end column is present, the only existing column is both start and end of the date range + if (validityDate.isSingleColumnDaterange()) { + Column column = validityDate.getColumn().resolve(); + startColumn = column; + endColumn = column; + } else { + startColumn = validityDate.getStartColumn().resolve(); + endColumn = validityDate.getEndColumn().resolve(); + } + + return ofStartAndEnd(tableName, startColumn, endColumn); + } + + private ColumnDateRange ofStartAndEnd(String tableName, Column startColumn, Column endColumn) { + + Field rangeStart = coalesce( + field(name(tableName, startColumn.getName()), Date.class), + getMinDateExpression() + ); + // when aggregating date ranges, we want to treat the last day of the range as excluded, + // so when using the date value of the end column, we add +1 day as end of the date range + Field rangeEnd = coalesce( + addDays(field(name(tableName, endColumn.getName()), Date.class), inline(1)), + getMaxDateExpression() + ); + + return ColumnDateRange.of(rangeStart, rangeEnd); + } + + @Override + public Field addDays(Field dateColumn, Field amountOfDays) { + return function( + "ADD_DAYS", + Date.class, + dateColumn, + amountOfDays + ); + } + + @Override + public ColumnDateRange allRangeIf(Condition condition) { + return ColumnDateRange.of( + when(condition.isTrue(), + getMinDateExpression() + ), + when(condition.isTrue(), + getMaxDateExpression() + ) + ); + } + + + @Override + public ColumnDateRange forValidityDate(ValidityDate validityDate, CDateRange dateRestriction) { + + ColumnDateRange validityDateRange = toColumnDateRange(validityDate); + ColumnDateRange restriction = toColumnDateRange(dateRestriction); + + Field lowerBound = when(validityDateRange.getStart().lessThan(restriction.getStart()), restriction.getStart()) + .otherwise(validityDateRange.getStart()); + + Field maxDate = toDateField(MAX_DATE_VALUE); // we want to add +1 day to the end date - except when it's the max date already + Field restrictionUpperBound = when(restriction.getEnd().eq(maxDate), maxDate).otherwise(addDays(restriction.getEnd(), inline(1))); + Field upperBound = when(validityDateRange.getEnd().greaterThan(restriction.getEnd()), restrictionUpperBound) + .otherwise(validityDateRange.getEnd()); + + return ColumnDateRange.of(lowerBound, upperBound); + } + + private ColumnDateRange toColumnDateRange(CDateRange dateRestriction) { + + String startDateExpression = MIN_DATE_VALUE; + String endDateExpression = MAX_DATE_VALUE; + + if (dateRestriction.hasLowerBound()) { + startDateExpression = dateRestriction.getMin().toString(); + } + if (dateRestriction.hasUpperBound()) { + endDateExpression = dateRestriction.getMax().toString(); + } + + return ColumnDateRange.of(toDateField(startDateExpression), toDateField(endDateExpression)); + } + + + @NotNull + @Override + public Collection> orderByValidityDates( + Function, ? extends SortField> ordering, + List> validityDateFields) { + + return List.of( + ordering.apply(nullif(validityDateFields.getFirst(), getMinDateExpression())).nullsLast(), + ordering.apply(nullif(validityDateFields.getLast(), getMaxDateExpression())).nullsLast() + ); + } + + @Override + public Field getMinDateExpression() { + return toDateField(MIN_DATE_VALUE); + } + + @Override + public Field getMaxDateExpression() { + return toDateField(MAX_DATE_VALUE); + } + + @Override + public ColumnDateRange forArbitraryDateRange(DaterangeSelectOrFilter daterangeSelectOrFilter) { + String tableName = daterangeSelectOrFilter.getTable().getName(); + if (daterangeSelectOrFilter.isSingleColumnDaterange()) { + Column column = daterangeSelectOrFilter.getColumn().resolve(); + return ofStartAndEnd(tableName, column, column); + } else { + return ofStartAndEnd(tableName, daterangeSelectOrFilter.getStartColumn().resolve(), daterangeSelectOrFilter.getEndColumn().resolve()); + } + } + + @Override + public ColumnDateRange aggregated(ColumnDateRange columnDateRange) { + return ColumnDateRange.of( + min(columnDateRange.getStart()), + max(columnDateRange.getEnd()) + ) + .as(columnDateRange.getAlias()); + } + + @Override + public ColumnDateRange toDualColumn(ColumnDateRange columnDateRange) { + // HANA does not support single column ranges + return ColumnDateRange.of(columnDateRange.getStart(), columnDateRange.getEnd()); + } + + @Override + public ColumnDateRange intersection(ColumnDateRange left, ColumnDateRange right) { + Field greatest = DSL.greatest(left.getStart(), right.getStart()); + Field least = DSL.least(left.getEnd(), right.getEnd()); + return ColumnDateRange.of(greatest, least); + } + + @Override + public QueryStep unnestDaterange(ColumnDateRange nested, QueryStep predecessor, String cteName) { + // HANA does not support single column datemultiranges + return predecessor; + } + + @Override + public Field dateRangeAggregation(ColumnDateRange columnDateRange) { + + Field stringAggregation = stringAggregation( + dateRangeToField(columnDateRange), + toChar(DATE_SET_SEPARATOR), + // The coalesce is necessary so that hana isn't upset about a potential `order by null` which happens for empty date-ranges List.of(coalesce(columnDateRange.getStart(), getMinDateExpression())) - ); - - // encapsulate all ranges (including empty ranges) within curly braces - return stringAggregation; - } - - @Override - public Field dateRangeToField(ColumnDateRange columnDateRange) { - // translation is handled in printer - return field("'[' || {0} || {2} || {1} || ')'", String.class, - cast(columnDateRange.getStart(), SQLDataType.VARCHAR), - cast(columnDateRange.getEnd(), SQLDataType.VARCHAR), - DATERANGE_SEPARATOR - ); - } - - @Override - public Field cast(Field field, DataType type) { - // HANA would require an explicit length param when using CAST with varchar type, TO_VARCHAR does not require this - if (type == SQLDataType.VARCHAR) { - return function("TO_VARCHAR", type.getType(), field); - } - return function( - // Needs to be explicitly unquoted, otherwise Hana is angry when Jooq quotes it on occasion. - unquotedName("CAST"), - type.getType(), - field("{0} AS {1}", field, keyword(type.getName())) - ); - } - - @Override - public Field dateDistance(ChronoUnit datePart, Field startDate, Field endDate) { - - String betweenFunction = switch (datePart) { - case DAYS -> "DAYS_BETWEEN"; - case MONTHS -> "MONTHS_BETWEEN"; - case YEARS, DECADES, CENTURIES -> "YEARS_BETWEEN"; - default -> throw new UnsupportedOperationException("Given ChronoUnit %s is not supported."); - }; - - Field dateDistance = function(betweenFunction, Integer.class, startDate, endDate); - - // HANA does not support decades or centuries directly - dateDistance = switch (datePart) { - case DECADES -> dateDistance.divide(10); - case CENTURIES -> dateDistance.divide(100); - default -> dateDistance; - }; - - // otherwise HANA would return floating point numbers for date distances - return dateDistance.cast(Integer.class); - } - - - @Override - public Field random(Field column) { - return field( - "{0}({1} {2})", - column.getType(), - keyword("FIRST_VALUE"), - column, - orderBy(function("RAND", Object.class)) - ); - } - - @Override - public Condition likeRegex(Field field, String pattern) { - return condition("{0} {1} {2}", field, keyword("LIKE_REGEXPR"), pattern); - } - - - @Override - public Field yearQuarter(Field dateField) { - return function("QUARTER", String.class, dateField); - } - - @Override - public Field isNull(Field field) { - // DSl.isNull does not work in some cases for Hana. This accomplishes the same thing with extra steps (: - return DSL.function("IFNULL", Boolean.class, field, inline(true)); - } + ); + + // encapsulate all ranges (including empty ranges) within curly braces + return stringAggregation; + } + + @Override + public Field dateRangeToField(ColumnDateRange columnDateRange) { + // translation is handled in printer + return field("'[' || {0} || {2} || {1} || ')'", String.class, + castToString(columnDateRange.getStart()), + castToString(columnDateRange.getEnd()), + DATERANGE_SEPARATOR + ); + } + + @Override + public Field castToString(Field field) { + return function(unquotedName("TO_VARCHAR"), String.class, field); + } + + @Override + public Field dateToInteger(Field field) { + return function("DAYS_BETWEEN", Integer.class, toDateField("1970-01-01"), field); + } + + @Override + public Field castToInteger(Field field) { + return function(unquotedName("TO_INTEGER"), Integer.class, field); + } + + @Override + public Field dateDistance(ChronoUnit datePart, Field startDate, Field endDate) { + + String betweenFunction = switch (datePart) { + case DAYS -> "DAYS_BETWEEN"; + case MONTHS -> "MONTHS_BETWEEN"; + case YEARS, DECADES, CENTURIES -> "YEARS_BETWEEN"; + default -> throw new UnsupportedOperationException("Given ChronoUnit %s is not supported."); + }; + + Field dateDistance = function(betweenFunction, Integer.class, startDate, endDate); + + // HANA does not support decades or centuries directly + dateDistance = switch (datePart) { + case DECADES -> dateDistance.divide(10); + case CENTURIES -> dateDistance.divide(100); + default -> dateDistance; + }; + + // otherwise HANA would return floating point numbers for date distances + return dateDistance.cast(Integer.class); + } + + + @Override + public Field random(Field column) { + return field( + "{0}({1} {2})", + column.getType(), + keyword("FIRST_VALUE"), + column, + orderBy(function("RAND", Object.class)) + ); + } + + @Override + public Condition likeRegex(Field field, String pattern) { + return condition("{0} {1} {2}", field, keyword("LIKE_REGEXPR"), pattern); + } + + + @Override + public Field yearQuarter(Field dateField) { + return function("QUARTER", String.class, dateField); + } + + @Override + public Field isNull(Field field) { + // DSl.isNull does not work in some cases for Hana. This accomplishes the same thing with extra steps (: + return DSL.function("IFNULL", Boolean.class, field, inline(true)); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java index 2bae80d371..dc1ac9abc9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java @@ -16,7 +16,6 @@ import org.jooq.Record; import org.jooq.Table; import org.jooq.impl.DSL; -import org.jooq.impl.SQLDataType; @Getter @RequiredArgsConstructor @@ -137,7 +136,7 @@ public Field indexSelectorField(TemporalSamplerFactory indexSelector, Colu // we calculate a random int which is in range of the date distance between upper and lower bound Field dateDistanceInDays = functionProvider.dateDistance(ChronoUnit.DAYS, validityDate.getStart(), validityDate.getEnd()); Field randomAmountOfDays = DSL.function("RAND", Double.class).times(dateDistanceInDays); - Field flooredAsInt = functionProvider.cast(DSL.floor(randomAmountOfDays), SQLDataType.INTEGER); + Field flooredAsInt = functionProvider.castToInteger(DSL.floor(randomAmountOfDays)); // then we add this random amount (of days) to the start date Field randomDateInRange = functionProvider.addDays(lower(validityDate), flooredAsInt); // finally, we handle multiple ranges by randomizing which range we use to select a random date from @@ -191,7 +190,7 @@ private static Field jumpToYearStart(Field date) { private Field getQuartersInMonths(Field date, Offset offset) { Field quarterExpression = functionProvider.yearQuarter(date); Field rightMostCharacter = DSL.function("RIGHT", String.class, quarterExpression, DSL.inline(1)); - Field amountOfQuarters = functionProvider.cast(rightMostCharacter, SQLDataType.INTEGER) + Field amountOfQuarters = functionProvider.castToInteger(rightMostCharacter) .plus(offset.getOffset()); return amountOfQuarters.times(MONTHS_PER_QUARTER); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java index 190aa2c512..70eae535e0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java @@ -58,7 +58,7 @@ private QueryStep createIntSeriesStep() { .sqlSelect(seriesIndex) .build(); - Table seriesTable = stratificationFunctions.generateIntSeries(INDEX_START, INDEX_END).asTable(SharedAliases.INDEX.getAlias()); + Table seriesTable = stratificationFunctions.generateIntSeries(INDEX_START, INDEX_END); return QueryStep.builder() .cteName(FormCteStep.INT_SERIES.getSuffix()) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java index d06755cbdc..cd55a58bbf 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java @@ -203,14 +203,14 @@ private QueryStep createIntervalTable(QueryStep totalBoundsStep, Resolution reso private QueryStep createOutcomeTable(QueryStep totalBoundsStep, Interval interval, Field seriesIndex, Range bounds, SqlIdColumns ids) { Field outcomeRangeStart = stratificationFunctions.shiftByInterval(INDEX_START_POSITIVE, interval, seriesIndex, Offset.MINUS_ONE); Field outcomeRangeEnd = stratificationFunctions.shiftByInterval(INDEX_START_POSITIVE, interval, seriesIndex, Offset.NONE); - Table outcomeSeries = stratificationFunctions.generateIntSeries(1, bounds.getMax()).as(SharedAliases.INDEX.getAlias()); + Table outcomeSeries = stratificationFunctions.generateIntSeries(1, bounds.getMax()); return createIntervalStep(ids, outcomeRangeStart, outcomeRangeEnd, Optional.of(outcomeSeries), totalBoundsStep); } private QueryStep createFeatureTable(QueryStep totalBoundsStep, Interval interval, Field seriesIndex, Range bounds, SqlIdColumns ids) { Field featureRangeStart = stratificationFunctions.shiftByInterval(INDEX_START_NEGATIVE, interval, seriesIndex, Offset.NONE); Field featureRangeEnd = stratificationFunctions.shiftByInterval(INDEX_START_NEGATIVE, interval, seriesIndex, Offset.ONE); - Table featureSeries = stratificationFunctions.generateIntSeries(bounds.getMin(), -1).as(SharedAliases.INDEX.getAlias()); + Table featureSeries = stratificationFunctions.generateIntSeries(bounds.getMin(), -1); return createIntervalStep(ids, featureRangeStart, featureRangeEnd, Optional.of(featureSeries), totalBoundsStep); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java index 1a2478b4f1..f4bd6fff20 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java @@ -112,9 +112,9 @@ public ColumnDateRange ofStartAndEnd(Field start, Field end) { public abstract Field intSeriesField(); /** - * Generates a series of integers from the start value to the exclusive end value. + * Generates a series of integers from the start value to the inclusive end value. */ - public abstract Table generateIntSeries(int start, int end); + public abstract Table generateIntSeries(int start, int end); /** * Generates a date field representing the {@link TemporalSamplerFactory} using the given validity date range. diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java index 56b5301cbe..9fd5d115b7 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java @@ -26,7 +26,6 @@ import org.jooq.Table; import org.jooq.TableLike; import org.jooq.impl.DSL; -import org.jooq.impl.SQLDataType; public class ConceptColumnSelectConverter implements SelectConverter { @@ -154,7 +153,7 @@ private static QueryStep createConnectorColumnSelectQuery( conditions = List.of(rawConceptColumn.isNotNull()); } - Field casted = selectContext.getFunctionProvider().cast(connectorColumn, SQLDataType.VARCHAR).as(alias); + Field casted = selectContext.getFunctionProvider().castToString(connectorColumn).as(alias); FieldWrapper connectorSelect = new FieldWrapper<>(casted); Selects selects = Selects.builder() diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java index 834832cefc..b36bc872e0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java @@ -19,7 +19,6 @@ import lombok.RequiredArgsConstructor; import org.jooq.Field; import org.jooq.impl.DSL; -import org.jooq.impl.SQLDataType; /** *
@@ -89,7 +88,7 @@ private static QueryStep createAggregationCte(
 			String alias
 	) {
 		SqlFunctionProvider functionProvider = selectContext.getFunctionProvider();
-		Field castedColumn = functionProvider.cast(preprocessingSelect.qualify(distinctSelectCte.getCteName()).select(), SQLDataType.VARCHAR);
+		Field castedColumn = functionProvider.castToString(preprocessingSelect.qualify(distinctSelectCte.getCteName()).select());
 		Field aggregatedColumn = functionProvider.stringAggregation(castedColumn, DSL.toChar(ResultSetProcessor.UNIT_SEPARATOR), List.of(castedColumn))
 														 .as(alias);
 
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java
index e4a9beec78..925b02867b 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java
@@ -119,11 +119,16 @@ public List jsonTests() {
 	@SneakyThrows
 	public Stream sqlProgrammaticTests(DatabaseConnectionConfig databaseConfig, SqlConnectorConfig sqlConfig, TestDataImporter testDataImporter) {
 		this.config.setSqlConnectorConfig(sqlConfig);
-		return programmaticTests(testDataImporter, StandaloneSupport.Mode.SQL);
+		return programmaticTests(testDataImporter, StandaloneSupport.Mode.SQL, databaseConfig.getDialect());
 	}
 
 	@SneakyThrows
 	public Stream programmaticTests(TestDataImporter testImporter, StandaloneSupport.Mode mode) {
+		return programmaticTests(testImporter, mode, null);
+	}
+
+	@SneakyThrows
+	public Stream programmaticTests(TestDataImporter testImporter, StandaloneSupport.Mode mode, Dialect dialect) {
 		String regexFilter = System.getenv(TestTags.TEST_PROGRAMMATIC_REGEX_FILTER);
 		List> programmatic =
 				CPSTypeIdResolver.SCAN_RESULT.getClassesImplementing(ProgrammaticIntegrationTest.class.getName())
@@ -154,7 +159,7 @@ public Stream programmaticTests(TestDataImporter testImporter, Stan
 						throw new RuntimeException(e);
 					}
 				})
-				.filter(test -> test.isEnabled(mode))
+				.filter(test -> test.isEnabled(mode, dialect))
 				.map(programmaticIntegrationTest -> createDynamicProgrammaticTestNode(programmaticIntegrationTest, testImporter));
 	}
 
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ClickhouseMatchingStatsTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ClickhouseMatchingStatsTest.java
new file mode 100644
index 0000000000..8521ed042e
--- /dev/null
+++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ClickhouseMatchingStatsTest.java
@@ -0,0 +1,68 @@
+package com.bakdata.conquery.integration.tests;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.time.LocalDate;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import com.bakdata.conquery.integration.IntegrationTest;
+import com.bakdata.conquery.integration.common.LoadingUtil;
+import com.bakdata.conquery.integration.json.ConqueryTestSpec;
+import com.bakdata.conquery.models.common.daterange.CDateRange;
+import com.bakdata.conquery.models.config.Dialect;
+import com.bakdata.conquery.models.datasets.concepts.Concept;
+import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept;
+import com.bakdata.conquery.models.exceptions.ValidatorHelper;
+import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId;
+import com.bakdata.conquery.util.support.StandaloneSupport;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class ClickhouseMatchingStatsTest extends IntegrationTest.Simple implements ProgrammaticIntegrationTest {
+
+	@Override
+	public Set forModes() {
+		return Set.of(StandaloneSupport.Mode.SQL);
+	}
+
+	@Override
+	public Set forDialects() {
+		return Set.of(Dialect.CLICKHOUSE);
+	}
+
+	@Override
+	public void execute(StandaloneSupport conquery) throws Exception {
+		String testJson = LoadingUtil.readResource("/tests/query/SIMPLE_TREECONCEPT_QUERY/SIMPLE_TREECONCEPT_Query.test.json");
+
+		DatasetId dataset = conquery.getDataset();
+
+		ConqueryTestSpec test = ConqueryTestSpec.readJson(dataset, testJson);
+		ValidatorHelper.failOnError(log, conquery.getValidator().validate(test));
+
+		test.importRequiredData(conquery);
+		LoadingUtil.updateMatchingStats(conquery);
+		conquery.waitUntilWorkDone();
+
+		TreeConcept concept;
+		try (Stream> allConcepts = conquery.getNamespace().getStorage().getAllConcepts()) {
+			concept = (TreeConcept) allConcepts.findFirst().orElseThrow();
+		}
+
+		assertThat(concept.getMatchingStats()).isNotNull();
+		assertThat(concept.getMatchingStats().countEvents()).isEqualTo(4);
+		assertThat(concept.getMatchingStats().countEntities()).isEqualTo(4);
+		assertThat(concept.getChildren()).allSatisfy(child -> {
+			assertThat(child.getMatchingStats()).isNotNull();
+			assertThat(child.getMatchingStats().countEvents()).isEqualTo(2);
+			assertThat(child.getMatchingStats().countEntities()).isEqualTo(2);
+		});
+
+		assertThat(concept.getMatchingStats().spanEvents())
+				.isEqualTo(CDateRange.of(LocalDate.parse("2010-07-15"), LocalDate.parse("2013-11-10")));
+		assertThat(concept.getChildren().get(0).getMatchingStats().spanEvents())
+				.isEqualTo(CDateRange.of(LocalDate.parse("2012-01-01"), LocalDate.parse("2013-11-10")));
+		assertThat(concept.getChildren().get(1).getMatchingStats().spanEvents())
+				.isEqualTo(CDateRange.of(LocalDate.parse("2010-07-15"), LocalDate.parse("2012-11-11")));
+	}
+}
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/HanaMatchingStatsTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/HanaMatchingStatsTest.java
new file mode 100644
index 0000000000..40e586e886
--- /dev/null
+++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/HanaMatchingStatsTest.java
@@ -0,0 +1,68 @@
+package com.bakdata.conquery.integration.tests;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.time.LocalDate;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import com.bakdata.conquery.integration.IntegrationTest;
+import com.bakdata.conquery.integration.common.LoadingUtil;
+import com.bakdata.conquery.integration.json.ConqueryTestSpec;
+import com.bakdata.conquery.models.common.daterange.CDateRange;
+import com.bakdata.conquery.models.config.Dialect;
+import com.bakdata.conquery.models.datasets.concepts.Concept;
+import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept;
+import com.bakdata.conquery.models.exceptions.ValidatorHelper;
+import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId;
+import com.bakdata.conquery.util.support.StandaloneSupport;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class HanaMatchingStatsTest extends IntegrationTest.Simple implements ProgrammaticIntegrationTest {
+
+	@Override
+	public Set forModes() {
+		return Set.of(StandaloneSupport.Mode.SQL);
+	}
+
+	@Override
+	public Set forDialects() {
+		return Set.of(Dialect.HANA);
+	}
+
+	@Override
+	public void execute(StandaloneSupport conquery) throws Exception {
+		String testJson = LoadingUtil.readResource("/tests/query/SIMPLE_TREECONCEPT_QUERY/SIMPLE_TREECONCEPT_Query.test.json");
+
+		DatasetId dataset = conquery.getDataset();
+
+		ConqueryTestSpec test = ConqueryTestSpec.readJson(dataset, testJson);
+		ValidatorHelper.failOnError(log, conquery.getValidator().validate(test));
+
+		test.importRequiredData(conquery);
+		LoadingUtil.updateMatchingStats(conquery);
+		conquery.waitUntilWorkDone();
+
+		TreeConcept concept;
+		try (Stream> allConcepts = conquery.getNamespace().getStorage().getAllConcepts()) {
+			concept = (TreeConcept) allConcepts.findFirst().orElseThrow();
+		}
+
+		assertThat(concept.getMatchingStats()).isNotNull();
+		assertThat(concept.getMatchingStats().countEvents()).isEqualTo(4);
+		assertThat(concept.getMatchingStats().countEntities()).isEqualTo(4);
+		assertThat(concept.getChildren()).allSatisfy(child -> {
+			assertThat(child.getMatchingStats()).isNotNull();
+			assertThat(child.getMatchingStats().countEvents()).isEqualTo(2);
+			assertThat(child.getMatchingStats().countEntities()).isEqualTo(2);
+		});
+
+		assertThat(concept.getMatchingStats().spanEvents())
+				.isEqualTo(CDateRange.of(LocalDate.parse("2010-07-15"), LocalDate.parse("2013-11-10")));
+		assertThat(concept.getChildren().get(0).getMatchingStats().spanEvents())
+				.isEqualTo(CDateRange.of(LocalDate.parse("2012-01-01"), LocalDate.parse("2013-11-10")));
+		assertThat(concept.getChildren().get(1).getMatchingStats().spanEvents())
+				.isEqualTo(CDateRange.of(LocalDate.parse("2010-07-15"), LocalDate.parse("2012-11-11")));
+	}
+}
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ProgrammaticIntegrationTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ProgrammaticIntegrationTest.java
index 03f7a049c1..489e641629 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ProgrammaticIntegrationTest.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ProgrammaticIntegrationTest.java
@@ -1,14 +1,16 @@
 package com.bakdata.conquery.integration.tests;
 
 import java.util.Set;
+import javax.annotation.Nullable;
 
 import com.bakdata.conquery.integration.IntegrationTest;
+import com.bakdata.conquery.models.config.Dialect;
 import com.bakdata.conquery.util.support.StandaloneSupport;
 
 public interface ProgrammaticIntegrationTest extends IntegrationTest {
 
-	default boolean isEnabled(StandaloneSupport.Mode mode) {
-		return forModes().contains(mode);
+	default boolean isEnabled(StandaloneSupport.Mode mode, @Nullable Dialect dialect) {
+		return forModes().contains(mode) && (dialect == null || forDialects().contains(dialect));
 	}
 
 	default Set forModes() {
@@ -16,4 +18,8 @@ default Set forModes() {
 		return Set.of(StandaloneSupport.Mode.WORKER);
 	}
 
+	default Set forDialects() {
+		return Set.of(Dialect.values());
+	}
+
 }
diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/EVENT_DATE_SELECT/EVENT_DATE_SELECT.test.json b/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/EVENT_DATE_SELECT/EVENT_DATE_SELECT.test.json
index 92baa36297..9de9ea6e27 100644
--- a/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/EVENT_DATE_SELECT/EVENT_DATE_SELECT.test.json
+++ b/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/EVENT_DATE_SELECT/EVENT_DATE_SELECT.test.json
@@ -4,7 +4,8 @@
   "sqlSpec": {
     "isEnabled": true,
     "supportedDialects": [
-      "HANA"
+      "HANA",
+      "CLICKHOUSE"
     ]
   },
   "expectedCsv": {
diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SECONDARY_ID/ABS_EXPORT_FORM_SECONDARY_ID.test.json b/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SECONDARY_ID/ABS_EXPORT_FORM_SECONDARY_ID.test.json
index 475eacbb8c..7cf70e39ca 100644
--- a/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SECONDARY_ID/ABS_EXPORT_FORM_SECONDARY_ID.test.json
+++ b/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SECONDARY_ID/ABS_EXPORT_FORM_SECONDARY_ID.test.json
@@ -4,7 +4,8 @@
     "sqlSpec": {
     "isEnabled": true,
     "supportedDialects": [
-       "HANA"
+       "HANA",
+       "CLICKHOUSE"
     ]
   },
   "expectedCsv": {
@@ -82,4 +83,4 @@
       }
     ]
   }
-}
\ No newline at end of file
+}
diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/ABS_EXPORT_FORM.test.json b/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/ABS_EXPORT_FORM.test.json
index e31e708cca..80407ca477 100644
--- a/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/ABS_EXPORT_FORM.test.json
+++ b/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/ABS_EXPORT_FORM.test.json
@@ -4,7 +4,8 @@
     "sqlSpec": {
     "isEnabled": true,
     "supportedDialects": [
-       "HANA"
+       "HANA",
+       "CLICKHOUSE"
     ]
   },
   "expectedCsv": {
@@ -48,4 +49,4 @@
       "tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/query_results_1.csv"
     ]
   }
-}
\ No newline at end of file
+}
diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/ABS_EXPORT_FORM_WITH_SELECT.test.json b/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/ABS_EXPORT_FORM_WITH_SELECT.test.json
index ae77d2de61..69e536611b 100644
--- a/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/ABS_EXPORT_FORM_WITH_SELECT.test.json
+++ b/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/ABS_EXPORT_FORM_WITH_SELECT.test.json
@@ -4,7 +4,8 @@
     "sqlSpec": {
     "isEnabled": true,
     "supportedDialects": [
-       "HANA"
+       "HANA",
+       "CLICKHOUSE"
     ]
   },
   "expectedCsv": {
@@ -53,4 +54,4 @@
       "tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/query_results_1.csv"
     ]
   }
-}
\ No newline at end of file
+}
diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE_DATE_RESTRICTION/AND_DR.test.json b/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE_DATE_RESTRICTION/AND_DR.test.json
index cf3ab51687..68e0ef6bad 100644
--- a/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE_DATE_RESTRICTION/AND_DR.test.json
+++ b/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE_DATE_RESTRICTION/AND_DR.test.json
@@ -4,7 +4,8 @@
     "sqlSpec": {
     "isEnabled": true,
     "supportedDialects": [
-       "HANA"
+       "HANA",
+       "CLICKHOUSE"
     ]
   },
 	"expectedCsv": {
@@ -58,4 +59,4 @@
 			"tests/form/EXPORT_FORM/ABSOLUT/SIMPLE_DATE_RESTRICTION/query_results_1.csv"
 		]
 	}
-}
\ No newline at end of file
+}
diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE_DATE_RESTRICTION/DR_AND.test.json b/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE_DATE_RESTRICTION/DR_AND.test.json
index 15f37b404c..9052e4ad04 100644
--- a/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE_DATE_RESTRICTION/DR_AND.test.json
+++ b/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE_DATE_RESTRICTION/DR_AND.test.json
@@ -4,7 +4,8 @@
     "sqlSpec": {
     "isEnabled": true,
     "supportedDialects": [
-       "HANA"
+       "HANA",
+       "CLICKHOUSE"
     ]
   },
 	"expectedCsv": {
@@ -58,4 +59,4 @@
 			"tests/form/EXPORT_FORM/ABSOLUT/SIMPLE_DATE_RESTRICTION/query_results_1.csv"
 		]
 	}
-}
\ No newline at end of file
+}
diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/LOGICAL/ABS_EXPORT_FORM.test.json b/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/LOGICAL/ABS_EXPORT_FORM.test.json
index 99518f3849..e9630c4bf0 100644
--- a/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/LOGICAL/ABS_EXPORT_FORM.test.json
+++ b/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/LOGICAL/ABS_EXPORT_FORM.test.json
@@ -4,7 +4,8 @@
     "sqlSpec": {
     "isEnabled": true,
     "supportedDialects": [
-       "HANA"
+       "HANA",
+       "CLICKHOUSE"
     ]
   },
 	"expectedCsv": {
@@ -81,4 +82,4 @@
 			}
 		}
 	}
-}
\ No newline at end of file
+}
diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/LOGICAL_WITH_EXCLUDE/ABS_EXPORT_FORM.test.json b/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/LOGICAL_WITH_EXCLUDE/ABS_EXPORT_FORM.test.json
index 7a9e931bef..37265acdbc 100644
--- a/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/LOGICAL_WITH_EXCLUDE/ABS_EXPORT_FORM.test.json
+++ b/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/LOGICAL_WITH_EXCLUDE/ABS_EXPORT_FORM.test.json
@@ -4,7 +4,8 @@
     "sqlSpec": {
     "isEnabled": true,
     "supportedDialects": [
-       "HANA"
+       "HANA",
+       "CLICKHOUSE"
     ]
   },
 	"expectedCsv": {
@@ -82,4 +83,4 @@
 			}
 		}
 	}
-}
\ No newline at end of file
+}
diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/SECONDARY_ID/SECONDARY_ID.test.json b/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/SECONDARY_ID/SECONDARY_ID.test.json
index a46fb4137d..a57befd6a0 100644
--- a/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/SECONDARY_ID/SECONDARY_ID.test.json
+++ b/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/SECONDARY_ID/SECONDARY_ID.test.json
@@ -4,7 +4,8 @@
     "sqlSpec": {
     "isEnabled": true,
     "supportedDialects": [
-       "HANA"
+       "HANA",
+       "CLICKHOUSE"
     ]
   },
 	"expectedCsv": {
@@ -73,4 +74,4 @@
 			}
 		]
 	}
-}
\ No newline at end of file
+}
diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/SIMPLE/ABS_EXPORT_FORM.test.json b/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/SIMPLE/ABS_EXPORT_FORM.test.json
index ae1a315959..01bc16b2d0 100644
--- a/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/SIMPLE/ABS_EXPORT_FORM.test.json
+++ b/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/SIMPLE/ABS_EXPORT_FORM.test.json
@@ -4,7 +4,8 @@
     "sqlSpec": {
     "isEnabled": true,
     "supportedDialects": [
-       "HANA"
+       "HANA",
+       "CLICKHOUSE"
     ]
   },
 	"expectedCsv": {
@@ -48,4 +49,4 @@
 			"tests/form/EXPORT_FORM/ENTITY_DATE/SIMPLE/query_results_1.csv"
 		]
 	}
-}
\ No newline at end of file
+}
diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/SIMPLE/ABS_EXPORT_FORM_WITH_SELECT.test.json b/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/SIMPLE/ABS_EXPORT_FORM_WITH_SELECT.test.json
index 238e5f938b..3cb72be67f 100644
--- a/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/SIMPLE/ABS_EXPORT_FORM_WITH_SELECT.test.json
+++ b/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/SIMPLE/ABS_EXPORT_FORM_WITH_SELECT.test.json
@@ -4,7 +4,8 @@
     "sqlSpec": {
     "isEnabled": true,
     "supportedDialects": [
-       "HANA"
+       "HANA",
+       "CLICKHOUSE"
     ]
   },
 	"expectedCsv": {
@@ -52,4 +53,4 @@
 			"tests/form/EXPORT_FORM/ENTITY_DATE/SIMPLE/query_results_1.csv"
 		]
 	}
-}
\ No newline at end of file
+}
diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/SECONDARY_ID/SECONDARY_ID.test.json b/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/SECONDARY_ID/SECONDARY_ID.test.json
index 10aaeef8db..c0b93de733 100644
--- a/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/SECONDARY_ID/SECONDARY_ID.test.json
+++ b/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/SECONDARY_ID/SECONDARY_ID.test.json
@@ -4,7 +4,8 @@
     "sqlSpec": {
     "isEnabled": true,
     "supportedDialects": [
-       "HANA"
+       "HANA",
+       "CLICKHOUSE"
     ]
   },
 	"expectedCsv": {
@@ -77,4 +78,4 @@
 			}
 		]
 	}
-}
\ No newline at end of file
+}
diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/SIMPLE/REL_EXPORT_FORM.test.json b/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/SIMPLE/REL_EXPORT_FORM.test.json
index ffe6e0e3be..ed0d5b73b0 100644
--- a/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/SIMPLE/REL_EXPORT_FORM.test.json
+++ b/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/SIMPLE/REL_EXPORT_FORM.test.json
@@ -4,7 +4,8 @@
     "sqlSpec": {
     "isEnabled": true,
     "supportedDialects": [
-       "HANA"
+       "HANA",
+       "CLICKHOUSE"
     ]
   },
 	"expectedCsv": {