Skip to content

Reduce ClickHouse lookup memory use and scan time #966

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Intelli merged 4 commits into from
Sep 3, 2026
Merged

Reduce ClickHouse lookup memory use and scan time #966

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a09fd89
Optimize ClickHouse lookup queries
Telesphoreo Aug 29, 2026
bc0606e
Separate ClickHouse inspector queries
Telesphoreo Aug 29, 2026
89c1678
Keep inspector queries on compatibility views
Telesphoreo Aug 29, 2026
b25c51b
Fix WorldEdit bulk change logging
Telesphoreo Aug 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<skipTests>true</skipTests>
<maven.compiler.release>11</maven.compiler.release>
<clickhouse.version>0.9.8</clickhouse.version>
<clickhouse.version>0.10.0</clickhouse.version>
<duckdb.version>1.4.5.0</duckdb.version>
</properties>
<build>
Expand Down
22 changes: 11 additions & 11 deletions src/main/java/net/coreprotect/database/LookupRaw.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1014,8 +1014,8 @@ else if (ConfigHandler.databaseType.isSQLite()) {
commandQuery = restrictSource(commandQuery, pageRows, 1);
String chatTable = sourceTable(statement, "chat", locationWorldId, sourceBounds, entityContext, false, pageRows);
String commandTable = sourceTable(statement, "command", locationWorldId, sourceBounds, entityContext, false, pageRows);
query = unionSelect + "SELECT '0' as tbl," + rows + " FROM " + chatTable + " WHERE" + chatQuery + unionLimit + ") UNION ALL ";
query += unionSelect + "SELECT '1' as tbl," + rows + " FROM " + commandTable + " WHERE" + commandQuery + unionLimit + ")";
query = unionSelect + "SELECT 0 as tbl," + rows + " FROM " + chatTable + " WHERE" + chatQuery + unionLimit + ") UNION ALL ";
query += unionSelect + "SELECT 1 as tbl," + rows + " FROM " + commandTable + " WHERE" + commandQuery + unionLimit + ")";
if (!count) {
queryOrder = " ORDER BY time DESC, tbl DESC, id DESC";
}
Expand Down Expand Up @@ -1052,7 +1052,7 @@ else if (actionList.contains(LookupActions.SIGN)) {

String sourceQuery = restrictSource(baseQuery, pageRows, InventorySources.BLOCK);
String sourceTable = sourceTable(statement, "block", locationWorldId, sourceBounds, entityContext, entitySpawnLocation, pageRows);
query = unionSelect + "SELECT " + "'0' as tbl," + rows + " FROM " + sourceTable + " " + index + "WHERE" + sourceQuery + unionLimit + ") UNION ALL ";
query = unionSelect + "SELECT 0 as tbl," + rows + " FROM " + sourceTable + " " + index + "WHERE" + sourceQuery + unionLimit + ") UNION ALL ";
itemLookup = true;
}

Expand All @@ -1062,14 +1062,14 @@ else if (actionList.contains(LookupActions.SIGN)) {
}
String containerSourceQuery = restrictSource(queryNonBlock, pageRows, InventorySources.CONTAINER);
String containerTable = sourceTable(statement, "container", locationWorldId, sourceBounds, entityContext, false, pageRows);
query = query + unionSelect + "SELECT " + "'1' as tbl," + rows + " FROM " + containerTable + " WHERE" + containerSourceQuery + unionLimit + ") UNION ALL ";
query = query + unionSelect + "SELECT 1 as tbl," + rows + " FROM " + containerTable + " WHERE" + containerSourceQuery + unionLimit + ") UNION ALL ";

if (!count && !selectPageRows) {
rows = "rowid as id,time," + userColumn + ",wid,x,y,z,type,metadata,data,amount,action,rolled_back,entity_spawn_rowid";
}
String entityContainerSourceQuery = restrictSource(queryEntityContainer, pageRows, InventorySources.ENTITY_CONTAINER);
String entityContainerTable = sourceTable(statement, "entity_container", locationWorldId, sourceBounds, entityContext, entityContainerLocation, pageRows, entityContainerId);
query = query + unionSelect + "SELECT '" + InventorySources.ENTITY_CONTAINER + "' as tbl," + rows + " FROM " + entityContainerTable + " WHERE" + entityContainerSourceQuery + unionLimit + ") UNION ALL ";
query = query + unionSelect + "SELECT " + InventorySources.ENTITY_CONTAINER + " as tbl," + rows + " FROM " + entityContainerTable + " WHERE" + entityContainerSourceQuery + unionLimit + ") UNION ALL ";

if (!count && !selectPageRows) {
rows = "rowid as id,time," + userColumn + ",wid,x,y,z,type,data as metadata,0 as data,amount,action,rolled_back,0 as entity_spawn_rowid";
Expand All @@ -1084,7 +1084,7 @@ else if (actionList.contains(LookupActions.SIGN)) {

String itemSourceQuery = restrictSource(queryNonBlock, pageRows, InventorySources.ITEM);
String itemTable = sourceTable(statement, "item", locationWorldId, sourceBounds, entityContext, false, pageRows);
query = query + unionSelect + "SELECT " + "'2' as tbl," + rows + " FROM " + itemTable + " WHERE" + itemSourceQuery + unionLimit + ")";
query = query + unionSelect + "SELECT 2 as tbl," + rows + " FROM " + itemTable + " WHERE" + itemSourceQuery + unionLimit + ")";
}

if (!itemLookup && (actionList.contains(LookupActions.CONTAINER) || actionList.contains(5))) {
Expand All @@ -1094,7 +1094,7 @@ else if (actionList.contains(LookupActions.SIGN)) {
if (entityContainerId == null) {
String sourceQuery = restrictSource(queryNonBlock, pageRows, 0);
String sourceTable = sourceTable(statement, "container", locationWorldId, sourceBounds, entityContext, false, pageRows);
query = unionSelect + "SELECT '0' as tbl," + rows + " FROM " + sourceTable + " WHERE" + sourceQuery + unionLimit + ")";
query = unionSelect + "SELECT 0 as tbl," + rows + " FROM " + sourceTable + " WHERE" + sourceQuery + unionLimit + ")";
}
if (includeEntityContainers) {
if (!query.isEmpty()) {
Expand All @@ -1105,7 +1105,7 @@ else if (actionList.contains(LookupActions.SIGN)) {
}
String sourceQuery = restrictSource(queryEntityContainer, pageRows, InventorySources.ENTITY_CONTAINER);
String sourceTable = sourceTable(statement, "entity_container", locationWorldId, sourceBounds, entityContext, entityContainerLocation, pageRows, entityContainerId);
query += unionSelect + "SELECT '" + InventorySources.ENTITY_CONTAINER + "' as tbl," + rows + " FROM " + sourceTable + " WHERE" + sourceQuery + unionLimit + ")";
query += unionSelect + "SELECT " + InventorySources.ENTITY_CONTAINER + " as tbl," + rows + " FROM " + sourceTable + " WHERE" + sourceQuery + unionLimit + ")";
}
if (!count) {
queryOrder = " ORDER BY time DESC, tbl DESC, id DESC";
Expand All @@ -1119,15 +1119,15 @@ else if (actionList.contains(LookupActions.SIGN)) {
}
String sourceQuery = restrictSource(blockSourceQuery, pageRows, InventorySources.BLOCK);
String sourceTable = sourceTable(statement, "block", locationWorldId, sourceBounds, entityContext, entitySpawnLocation, pageRows);
query = unionSelect + "SELECT '0' as tbl," + rows + " FROM " + sourceTable + " " + index + "WHERE" + sourceQuery + unionLimit + ")";
query = unionSelect + "SELECT 0 as tbl," + rows + " FROM " + sourceTable + " " + index + "WHERE" + sourceQuery + unionLimit + ")";
}

if (!count && !selectPageRows) {
rows = "rowid as id,time," + userColumn + ",wid,x,y,z,type,metadata,action as data,-1 as amount," + LookupActions.INTERACTION + " as action,rolled_back,entity_spawn_rowid";
}
String sourceQuery = restrictSource(queryEntityInteraction, pageRows, InventorySources.ENTITY_INTERACTION);
String sourceTable = sourceTable(statement, "entity_interaction", locationWorldId, sourceBounds, entityContext, entityInteractionLocation, pageRows);
query += " UNION ALL " + unionSelect + "SELECT '" + InventorySources.ENTITY_INTERACTION + "' as tbl," + rows + " FROM " + sourceTable + " WHERE" + sourceQuery + unionLimit + ")";
query += " UNION ALL " + unionSelect + "SELECT " + InventorySources.ENTITY_INTERACTION + " as tbl," + rows + " FROM " + sourceTable + " WHERE" + sourceQuery + unionLimit + ")";
if (!count) {
queryOrder = " ORDER BY time DESC, tbl DESC, id DESC";
}
Expand All @@ -1144,7 +1144,7 @@ else if (actionList.contains(LookupActions.SIGN)) {
baseQuery = restrictSource(baseQuery, pageRows, 0);
Integer exactEntitySpawnRowId = queryTable.equals("entity_container") ? entityContainerId : null;
String sourceTable = sourceTable(statement, queryTable, locationWorldId, sourceBounds, entityContext, entityFallback, pageRows, exactEntitySpawnRowId);
query = "SELECT " + "'0' as tbl," + rows + " FROM " + sourceTable + " " + index + "WHERE" + baseQuery;
query = "SELECT 0 as tbl," + rows + " FROM " + sourceTable + " " + index + "WHERE" + baseQuery;
}

if (selectPageRows) {
Expand Down
3 changes: 3 additions & 0 deletions src/main/java/net/coreprotect/database/clickhouse/ClickHouseJdbcConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import java.net.URISyntaxException;
import java.util.Properties;

import com.clickhouse.client.api.ClientConfigProperties;

public final class ClickHouseJdbcConfig {

private static final String SERVER_SETTING_PREFIX = "clickhouse_setting_";
Expand Down Expand Up @@ -49,6 +51,7 @@ public ClickHouseJdbcConfig(String host, int port, String database, String usern
properties.setProperty(SERVER_SETTING_PREFIX + "prefer_column_name_to_alias", "1");
properties.setProperty("ssl", Boolean.toString(tls));
properties.setProperty("jdbc_ignore_unsupported_values", "false");
properties.setProperty(ClientConfigProperties.BINARY_STRING_SUPPORT.getKey(), "true");
}

public String getJdbcUrl() {
Expand Down
8 changes: 4 additions & 4 deletions src/main/java/net/coreprotect/database/clickhouse/ClickHouseSchema.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -335,13 +335,13 @@ private static String entitySpawnView(Names names) {
}

static String binary(String value, String alias) {
String presentValue = "ifNull(" + value + ",'')";
String bytes = "arrayMap(i -> reinterpretAsInt8(substring(" + presentValue + ",i,1)),range(1,length(" + presentValue + ")+1))";
return "if(isNull(" + value + "),CAST([], 'Array(Int8)'),arrayConcat([toInt8(0)]," + bytes + ")) AS " + alias;
return value + " AS " + alias;
}

private static String location(String column) {
return "if(e." + column + "_present=1,e." + column + ",NULL) AS " + column;
// Every compatibility family exposing a location stores this key. Project it directly so
// predicates on wid/x/z can be pushed through the view into the MergeTree sorting key.
return "e." + column + " AS " + column;
}

private static String events(Names names, ClickHouseFamily family) {
Expand Down
50 changes: 19 additions & 31 deletions src/main/java/net/coreprotect/worldedit/CoreProtectLogger.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,13 @@
import com.sk89q.worldedit.extension.platform.Actor;
import com.sk89q.worldedit.extent.AbstractDelegateExtent;
import com.sk89q.worldedit.extent.Extent;
import com.sk89q.worldedit.function.RegionFunction;
import com.sk89q.worldedit.function.RegionMaskingFilter;
import com.sk89q.worldedit.function.block.BlockReplace;
import com.sk89q.worldedit.function.mask.Mask;
import com.sk89q.worldedit.function.operation.Operations;
import com.sk89q.worldedit.function.pattern.Pattern;
import com.sk89q.worldedit.function.visitor.RegionVisitor;
import com.sk89q.worldedit.math.BlockVector3;
import com.sk89q.worldedit.regions.Region;
import com.sk89q.worldedit.world.World;
Expand Down Expand Up @@ -85,8 +90,9 @@ public int replaceBlocks(final Region region, final Mask mask, final Pattern pat
if (!Config.getConfig(world).WORLDEDIT) {
return eventExtent.replaceBlocks(region, mask, pattern);
}
processPatternToBlocks(world, region, pattern);
return eventExtent.replaceBlocks(region, mask, pattern);

BlockReplace replace = new BlockReplace(this, pattern);
return processBlocks(region, new RegionMaskingFilter(mask, replace));
}

@Override
Expand All @@ -100,8 +106,8 @@ public int setBlocks(Region region, Pattern pattern) throws MaxChangedBlocksExce
if (!Config.getConfig(world).WORLDEDIT) {
return eventExtent.setBlocks(region, pattern);
}
processPatternToBlocks(world, region, pattern);
return eventExtent.setBlocks(region, pattern);

return processBlocks(region, new BlockReplace(this, pattern));
}

@Override
Expand All @@ -110,37 +116,19 @@ public int setBlocks(Set<BlockVector3> vset, Pattern pattern) {
if (!Config.getConfig(world).WORLDEDIT) {
return eventExtent.setBlocks(vset, pattern);
}
processPatternToBlocks(world, vset, pattern);
return eventExtent.setBlocks(vset, pattern);
}

private void processPatternToBlocks(org.bukkit.World world, Set<BlockVector3> vset, Pattern pattern) {
int count = 0;
for (BlockVector3 position : vset) {
BlockState oldBlock = eventExtent.getBlock(position);
Material oldType = BukkitAdapter.adapt(oldBlock.getBlockType());
Location location = new Location(world, position.getBlockX(), position.getBlockY(), position.getBlockZ());
BaseBlock baseBlock = WorldEditLogger.getBaseBlock(eventExtent, position, location, oldType, oldBlock);

// No clear way to get container content data from within the WorldEdit API
// Data may be available by converting oldBlock.toBaseBlock().getNbtData()
// e.g. BaseBlock block = eventWorld.getBlock(position);
ItemStack[] containerData = CoreProtectEditSessionEvent.isFAWE() ? null : ItemUtils.getContainerContents(oldType, null, location);
WorldEditLogger.postProcess(eventExtent, eventActor, position, location, pattern.applyBlock(position), baseBlock, oldType, oldBlock, containerData);
if (pattern.apply(this, position, position)) {
count++;
}
}
return count;
}

private void processPatternToBlocks(org.bukkit.World world, Region region, Pattern pattern) {
for (BlockVector3 position : region.clone()) {
BlockState oldBlock = eventExtent.getBlock(position);
Material oldType = BukkitAdapter.adapt(oldBlock.getBlockType());
Location location = new Location(world, position.getBlockX(), position.getBlockY(), position.getBlockZ());
BaseBlock baseBlock = WorldEditLogger.getBaseBlock(eventExtent, position, location, oldType, oldBlock);

// No clear way to get container content data from within the WorldEdit API
// Data may be available by converting oldBlock.toBaseBlock().getNbtData()
// e.g. BaseBlock block = eventWorld.getBlock(position);
ItemStack[] containerData = CoreProtectEditSessionEvent.isFAWE() ? null : ItemUtils.getContainerContents(oldType, null, location);
WorldEditLogger.postProcess(eventExtent, eventActor, position, location, pattern.applyBlock(position), baseBlock, oldType, oldBlock, containerData);
}
private int processBlocks(Region region, RegionFunction function) {
RegionVisitor visitor = new RegionVisitor(region, function);
Operations.completeLegacy(visitor);
return visitor.getAffected();
}
}
Loading