Skip to content

Enable ClickHouse export form queries #4113

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
awildturtok wants to merge 3 commits into from
Open

Enable ClickHouse export form queries #4113

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b2ba45b
Changes casting to explicit usage of types.
awildturtok Oct 19, 2026
a150533
test(sql): add dialect-specific matching stats integration tests
awildturtok Oct 19, 2026
4abcfc7
Enable ClickHouse export form queries
awildturtok Oct 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import org.jooq.*;
import org.jooq.Record;
import org.jooq.exception.DataAccessException;
import org.jooq.impl.SQLDataType;

@Slf4j
@Data
Expand Down Expand Up @@ -282,7 +281,9 @@ private SelectJoinStep<Record4<Integer, String, Integer, Integer>> createMatchin

SelectJoinStep<Record4<Integer, String, Integer, Integer>> records = dslContext.with(unioned).select(unioned.field(CONCEPT_ID_FIELD), PID_FIELD,
// The infinities are intentionally swapped
functionProvider.cast(nullif(unioned.field(LB_FIELD), positiveInfinity), SQLDataType.INTEGER).as(LB_FIELD), functionProvider.cast(nullif(unioned.field(UB_FIELD), negativeInfinity), SQLDataType.INTEGER).as(UB_FIELD)).from(ct_name);
functionProvider.dateToInteger(nullif(unioned.field(LB_FIELD), positiveInfinity)).as(LB_FIELD),
functionProvider.dateToInteger(nullif(unioned.field(UB_FIELD), negativeInfinity)).as(UB_FIELD)
).from(ct_name);


return records;
Expand Down
9 changes: 7 additions & 2 deletions backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import com.bakdata.conquery.sql.conversion.model.QueryStep;
import com.bakdata.conquery.sql.execution.ResultSetProcessor;
import org.jooq.Condition;
import org.jooq.DataType;
import org.jooq.Field;
import org.jooq.OrderField;
import org.jooq.Record;
Expand Down Expand Up @@ -68,7 +67,9 @@ Collection<? extends OrderField<?>> orderByValidityDates(
*/
Field<Date> getMaxDateExpression();

<T> Field<T> cast(Field<?> field, DataType<T> type);
Field<String> castToString(Field<?> field);

Field<Integer> castToInteger(Field<?> field);

/**
* @return The regex that matches any char repeated any times (including 0), for example:
Expand Down Expand Up @@ -318,4 +319,8 @@ default Condition unconditionalJoinCondition(){
default Field<Boolean> isNull(Field<?> field){
return field.isNull();
}

default Field<Integer> dateToInteger(Field<Date> field) {
return castToInteger(field);
}
}
15 changes: 8 additions & 7 deletions ...va/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
import com.bakdata.conquery.sql.conversion.model.QueryStep;
import org.jetbrains.annotations.NotNull;
import org.jooq.Condition;
import org.jooq.DataType;
import org.jooq.Field;
import org.jooq.OrderField;
import org.jooq.SortField;
Expand Down Expand Up @@ -271,14 +270,16 @@ public Field<Object> dateRangeToField(ColumnDateRange columnDateRange) {
}

@Override
public <T> Field<T> cast(Field<?> field, DataType<T> type) {
if (type == SQLDataType.VARCHAR) {
return function("toString", type.getType(), field);
}
public Field<String> castToString(Field<?> field) {
return function("toString", String.class, field);
}

@Override
public Field<Integer> castToInteger(Field<?> field) {
return function(
name("CAST"),
type.getType(),
field("{0} AS Nullable({1})", field, keyword(type.getName()))
Integer.class,
field("{0} AS Nullable({1})", field, keyword(SQLDataType.INTEGER.getName()))
);
}

Expand Down
81 changes: 34 additions & 47 deletions .../com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseResultSetProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,90 +2,77 @@

import java.math.BigDecimal;
import java.sql.Array;
import java.sql.Date;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.LocalDate;
import java.sql.Struct;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;

import com.bakdata.conquery.models.common.CDate;
import com.bakdata.conquery.models.common.daterange.CDateRange;
import com.bakdata.conquery.models.config.ConqueryConfig;
import com.bakdata.conquery.sql.execution.DefaultResultSetProcessor;
import com.google.common.base.Preconditions;

public class ClickhouseResultSetProcessor extends DefaultResultSetProcessor {

public static final String DATE_SEPARATOR = ",";

public ClickhouseResultSetProcessor(ConqueryConfig config) {
super(config, null);
}

@Override
public List<Integer> getDateRange(ResultSet resultSet, int columnIndex) throws SQLException {
String daterange = resultSet.getString(columnIndex);
Object daterange = resultSet.getObject(columnIndex);

if (daterange == null) {
return Collections.emptyList();
}

String[] dates = daterange.split(DATE_SEPARATOR);
Preconditions.checkArgument(dates.length == 2, "Dateranges must have a start and end.");

String startDateExpression = dates[0];
int startDate;
if (startDateExpression.equals(ClickhouseFunctionProvider.MIN_DATE_VALUE)) {
startDate = CDateRange.NEGATIVE_INFINITY;
}
else {
LocalDate dateValue = Date.valueOf(startDateExpression).toLocalDate();
startDate = CDate.ofLocalDate(dateValue);
}

String endDateExpression = dates[1];
int endDate;
if (endDateExpression.equals(ClickhouseFunctionProvider.MAX_DATE_VALUE)) {
endDate = CDateRange.POSITIVE_INFINITY;
}
else {
LocalDate dateValue = Date.valueOf(endDateExpression).toLocalDate();
endDate = CDate.ofLocalDate(dateValue);
}

return List.of(startDate, endDate);
Object[] rawTuple = daterange instanceof Struct struct ? struct.getAttributes() : (Object[]) daterange;
return toEpochDayRange(rawTuple);
}

public List<List<Integer>> toEpochDayRangeList(List<Object[]> raw) {
if (raw == null) {
return null;
}

List<List<Integer>> out = new ArrayList<>();

for (Object[] rawTuple : raw) {

if (rawTuple[0] == null) {
assert rawTuple[1] == null;
continue;
List<Integer> dateRange = toEpochDayRange(rawTuple);
if (!dateRange.isEmpty()) {
out.add(dateRange);
}
}

int begin = (Integer) rawTuple[0];
int end = (Integer) rawTuple[1];
return out;
}

if (begin == ClickhouseFunctionProvider.MIN_DATE_VALUE) {
begin = Integer.MIN_VALUE;
}
if (end == ClickhouseFunctionProvider.MAX_DATE_VALUE) {
end = Integer.MAX_VALUE;
}
else {
end--;
}
private List<Integer> toEpochDayRange(Object[] rawTuple) {
Preconditions.checkArgument(rawTuple.length == 2, "Dateranges must have a start and end.");

if (rawTuple[0] == null && rawTuple[1] == null) {
return Collections.emptyList();
}
Preconditions.checkArgument(rawTuple[0] != null && rawTuple[1] != null, "Dateranges must have a start and end.");
Preconditions.checkArgument(rawTuple[0] instanceof Number && rawTuple[1] instanceof Number, "Dateranges must contain integer values.");

out.add(List.of(begin, end));
int begin = ((Number) rawTuple[0]).intValue();
int end = ((Number) rawTuple[1]).intValue();

if (begin == ClickhouseFunctionProvider.MIN_DATE_VALUE) {
begin = CDateRange.NEGATIVE_INFINITY;
}
if (end == ClickhouseFunctionProvider.MAX_DATE_VALUE) {
end = CDateRange.POSITIVE_INFINITY;
}
else {
end--;
}

return out;
return List.of(begin, end);
}

private <T> List<T> sort(List<T> list, Comparator<T> sort) {
Expand Down
18 changes: 12 additions & 6 deletions ...bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import org.jooq.Field;
import org.jooq.Record;
import org.jooq.Table;
import org.jooq.impl.SQLDataType;

@Getter
@RequiredArgsConstructor
Expand Down Expand Up @@ -120,9 +119,16 @@ public Field<Integer> intSeriesField() {
}

@Override
public Table<Record> generateIntSeries(int start, int end) {
//TODO this only supports ungigned values. Probably need to handle negatives manually Q_Q
return table("generate_series({0}, {1}, 1)", start, end);
public Table<? extends Record> generateIntSeries(int start, int end) {
Field<Integer> startInclusive = functionProvider.castToInteger(inline(start));
Field<Integer> endExclusive = functionProvider.castToInteger(inline(end).plus(inline(1)));
Field<Integer> seriesIndex = field(
"arrayJoin(range({0}, {1}))",
Integer.class,
startInclusive,
endExclusive
).as(SERIES_INDEX.getName());
return select(seriesIndex).asTable();
}

@Override
Expand All @@ -134,7 +140,7 @@ public Field<Date> indexSelectorField(TemporalSamplerFactory indexSelector, Colu
// we calculate a random int which is in range of the date distance between upper and lower bound
Field<Integer> dateDistanceInDays = functionProvider.dateDistance(ChronoUnit.DAYS, validityDate.getStart(), validityDate.getEnd());
Field<Double> randomAmountOfDays = function("RAND", Double.class).times(dateDistanceInDays);
Field<Integer> flooredAsInt = functionProvider.cast(floor(randomAmountOfDays), SQLDataType.INTEGER);
Field<Integer> flooredAsInt = functionProvider.castToInteger(floor(randomAmountOfDays));
// then we add this random amount (of days) to the start date
Field<Date> randomDateInRange = functionProvider.addDays(lower(validityDate), flooredAsInt);
// finally, we handle multiple ranges by randomizing which range we use to select a random date from
Expand Down Expand Up @@ -170,7 +176,7 @@ private Field<Date> calcDate(Field<Date> start, Interval interval, Offset offset
private Field<Integer> getQuartersInMonths(Field<Date> date, Offset offset) {
Field<String> quarterExpression = functionProvider.yearQuarter(date);
Field<String> rightMostCharacter = function("RIGHT", String.class, quarterExpression, inline(1));
Field<Integer> amountOfQuarters = functionProvider.cast(rightMostCharacter, SQLDataType.INTEGER)
Field<Integer> amountOfQuarters = functionProvider.castToInteger(rightMostCharacter)
.plus(offset.getOffset());
return amountOfQuarters.times(MONTHS_PER_QUARTER);
}
Expand Down
Loading
Loading