Skip to content

[AMORO-4273] Add support for Lance format on Hive metastore #4450

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zhangwl9 wants to merge 2 commits into
base: master
Choose a base branch
from
+679 −180
Open

[AMORO-4273] Add support for Lance format on Hive metastore #4450

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d929afd
add lance-hive catalog
zhangwl9 Sep 3, 2026
1327f60
fixup
zhangwl9 Oct 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ public class CatalogBuilder {
TableFormat.MIXED_ICEBERG,
TableFormat.MIXED_HIVE,
TableFormat.PAIMON,
TableFormat.HUDI),
TableFormat.HUDI,
TableFormat.LANCE),
CATALOG_TYPE_AMS,
Sets.newHashSet(TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG));

Expand All @@ -86,7 +87,7 @@ public static ServerCatalog buildServerCatalog(
"Table format %s is not supported for metastore type: %s",
tableFormats,
type);
checkRestLanceExclusivity(type, tableFormats);
checkLanceExclusivity(type, tableFormats);

switch (type) {
case CATALOG_TYPE_HADOOP:
Expand All @@ -104,13 +105,14 @@ public static ServerCatalog buildServerCatalog(
}

/** Enforced on both catalog creation and metadata updates. */
static void checkRestLanceExclusivity(String type, Set<TableFormat> tableFormats) {
static void checkLanceExclusivity(String type, Set<TableFormat> tableFormats) {
boolean lanceExclusiveType = CATALOG_TYPE_REST.equals(type) || CATALOG_TYPE_HIVE.equals(type);
Preconditions.checkState(
!(CATALOG_TYPE_REST.equals(type)
!(lanceExclusiveType
&& tableFormats.contains(TableFormat.LANCE)
&& tableFormats.size() > 1),
"REST catalog serves a single protocol per uri,"
+ " Lance cannot be combined with other table formats: %s",
"Lance cannot be combined with other table formats in a %s catalog: %s",
type,
tableFormats);
}
}
5 changes: 2 additions & 3 deletions amoro-ams/src/main/java/org/apache/amoro/server/catalog/DefaultCatalogManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,11 +276,10 @@ private void validateCatalogUpdate(CatalogMeta oldMeta, CatalogMeta newMeta) {
throw new IllegalMetadataException("Cannot update catalog type");
}

// A REST catalog serves a single protocol per uri, so Lance cannot be combined with other
// table formats. Reject the update before it is persisted.
// Lance cannot mix with other table formats in REST/Hive catalogs; reject before persisting.
String type = newMeta.getCatalogType();
Set<TableFormat> tableFormats = CatalogUtil.tableFormats(newMeta);
CatalogBuilder.checkRestLanceExclusivity(type, tableFormats);
CatalogBuilder.checkLanceExclusivity(type, tableFormats);
}

private void disposeCatalog(String name) {
Expand Down
7 changes: 7 additions & 0 deletions amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,13 @@ public class CatalogController {
CatalogDescriptor.of(CATALOG_TYPE_REST, STORAGE_CONFIGS_VALUE_TYPE_HADOOP, LANCE));
VALIDATE_CATALOGS.add(
CatalogDescriptor.of(CATALOG_TYPE_REST, STORAGE_CONFIGS_VALUE_TYPE_OSS, LANCE));

VALIDATE_CATALOGS.add(
CatalogDescriptor.of(CATALOG_TYPE_HIVE, STORAGE_CONFIGS_VALUE_TYPE_S3, LANCE));
VALIDATE_CATALOGS.add(
CatalogDescriptor.of(CATALOG_TYPE_HIVE, STORAGE_CONFIGS_VALUE_TYPE_OSS, LANCE));
VALIDATE_CATALOGS.add(
CatalogDescriptor.of(CATALOG_TYPE_HIVE, STORAGE_CONFIGS_VALUE_TYPE_LOCAL, LANCE));
}

private final PlatformFileManager platformFileInfoService;
Expand Down
28 changes: 21 additions & 7 deletions amoro-common/src/main/java/org/apache/amoro/utils/CatalogUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,11 +121,9 @@ public static Map<String, String> mergeCatalogPropertiesToTable(

/** Build {@link TableMetaStore} from catalog meta. */
public static TableMetaStore buildMetaStore(CatalogMeta catalogMeta) {
// REST + Lance manages storage/auth server-side, so a client-side TableMetaStore is
// unnecessary. Set a local configuration so build() returns a disabled-auth store.
if (CatalogMetaProperties.CATALOG_TYPE_REST.equalsIgnoreCase(catalogMeta.getCatalogType())
&& isLanceOnly(catalogMeta)) {
return TableMetaStore.builder().withConfiguration(new Configuration()).build();
// Lance over REST/Hive catalog authenticates on its own; EMPTY is a disabled-auth store.
if (isLanceManagedExternally(catalogMeta)) {
return TableMetaStore.EMPTY;
}

// load storage configs
Expand Down Expand Up @@ -222,9 +220,25 @@ && isLanceOnly(catalogMeta)) {
return builder.build();
}

private static boolean isLanceOnly(CatalogMeta catalogMeta) {
private static boolean isLanceManagedExternally(CatalogMeta catalogMeta) {
Set<TableFormat> formats = tableFormats(catalogMeta);
return formats.size() == 1 && formats.contains(TableFormat.LANCE);
if (formats.size() != 1 || !formats.contains(TableFormat.LANCE)) {
return false;
}

String type = catalogMeta.getCatalogType();
Map<String, String> authConfigs = catalogMeta.getAuthConfigs();
if (CatalogMetaProperties.CATALOG_TYPE_HIVE.equalsIgnoreCase(type)
&& authConfigs != null
&& CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_KERBEROS.equalsIgnoreCase(
authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_TYPE))) {
throw new IllegalArgumentException(
"Kerberos auth is not supported for Lance catalogs backed by a Hive metastore: "
+ catalogMeta.getCatalogName());
}

return CatalogMetaProperties.CATALOG_TYPE_REST.equalsIgnoreCase(type)
|| CatalogMetaProperties.CATALOG_TYPE_HIVE.equalsIgnoreCase(type);
}

public static TableIdentifier tableId(TableMeta tableMeta) {
Expand Down
57 changes: 57 additions & 0 deletions amoro-format-lance/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,41 @@
<artifactId>lance-namespace-apache-client</artifactId>
<version>0.8.0</version>
</dependency>

<!-- Lance namespace Hive 3.x metastore implementation -->
<dependency>
<groupId>org.lance</groupId>
<artifactId>lance-namespace-hive3</artifactId>
<version>0.4.1</version>
<exclusions>
<exclusion>
<groupId>org.lance</groupId>
<artifactId>lance-namespace-core</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
</exclusion>
<!-- amoro-common already provides orc 1.9.7 (nohive) and hadoop-client-api, whose
unshaded 1.5.1 / 2.x counterparts only add duplicate org.apache.hadoop classes. -->
<exclusion>
<groupId>org.apache.orc</groupId>
<artifactId>orc-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-aws</artifactId>
Expand All @@ -79,6 +114,28 @@
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-aliyun</artifactId>
</dependency>

<!-- HMSMockServer and the derby schema used by the hive3 namespace tests -->
<dependency>
<groupId>org.apache.amoro</groupId>
<artifactId>amoro-common</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derby</artifactId>
<scope>test</scope>
</dependency>
<!-- arrow-memory-netty brings netty-buffer 4.1.129 while hive3 pulls netty-common
4.1.119; align them on the test classpath only. -->
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
<version>${netty.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

</project>
Loading
Loading