From d9ec640e21d4c691b84bb48b34fe15fcf5f1bf99 Mon Sep 17 00:00:00 2001 From: Lukas Bindreiter Date: Tue, 27 Oct 2026 10:27:20 +0000 Subject: [PATCH 1/4] Add dataset-local supporting type definitions Add an optional descriptor set to dataset create/update schemas for custom enum definitions and future message types. Document fully qualified references and name-based response resolution. Amp-Thread-ID: https://ampcode.com/threads/T-01a15368-905e-70db-a93b-31a2b835d742 Co-authored-by: Amp --- apis/datasets/v1/dataset_type.proto | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/apis/datasets/v1/dataset_type.proto b/apis/datasets/v1/dataset_type.proto index b0eb609..f7421be 100644 --- a/apis/datasets/v1/dataset_type.proto +++ b/apis/datasets/v1/dataset_type.proto @@ -16,8 +16,9 @@ option features.field_presence = IMPLICIT; // with its Tilebox annotation, matching the information represented separately by AnnotatedType. message Field { // The descriptor contains the name of the field, the type, optional labels (e.g. repeated) and other information. - // If the type is TYPE_MESSAGE, then the type_name must be a fully qualified name to a well known type, e.g. - // `datasets.v1.Vec3` or `google.protobuf.Timestamp`. + // For TYPE_MESSAGE and TYPE_ENUM, type_name references an allowed built-in type or a type declared in + // DatasetType.type_definitions. Use a fully qualified name with a leading dot, e.g. `.datasets.v1.Vec3` + // or `.google.protobuf.Timestamp`. google.protobuf.FieldDescriptorProto descriptor = 1 [(buf.validate.field).required = true]; // Additional documentation, source mapping, and query metadata for the field. @@ -86,13 +87,21 @@ message DatasetType { // A list of fields that this dataset consists of. repeated Field fields = 2; + + // Dataset-local supporting type definitions referenced by fields through descriptor.type_name. + // Initial support is limited to custom enums; the descriptor set can also represent message definitions + // for future support. It must not contain the dataset's datapoint message, which the server builds from fields. + // Include any custom dependencies needed to resolve these definitions. File paths and fully qualified type + // names must be unique and must not redefine built-in types. Definitions are resolved by name, not list order. + google.protobuf.FileDescriptorSet type_definitions = 3; } // AnnotatedType is the resolved representation returned on Dataset messages, including GetDataset responses. // It combines the generated protobuf descriptors with the corresponding Tilebox field annotations. message AnnotatedType { + // Descriptors for the dataset's datapoint message and its dataset-local supporting types. google.protobuf.FileDescriptorSet descriptor_set = 1; - // the url of the type, one of the types defined in the descriptor + // Identifies the datapoint message in descriptor_set by name, independent of file or declaration order. string type_url = 2; reserved 3; // Annotations corresponding by index to the FieldDescriptorProto entries of the message selected by type_url. From 14828c4baab056630b51e0200b7d80f97e184ffb Mon Sep 17 00:00:00 2001 From: Lukas Bindreiter Date: Tue, 27 Oct 2026 10:54:17 +0000 Subject: [PATCH 2/4] Document custom enum creation and filtering contracts Keep full enum descriptors in create/update requests and define reversible public values. Specify server-side string literal resolution, exact enum symbols, supported operators, and null behavior without changing filter message fields. Amp-Thread-ID: T-01a15368-905e-70db-a93b-31a2b835d742 --- apis/datasets/v1/data_access.proto | 14 ++++++++++++-- apis/datasets/v1/dataset_type.proto | 10 ++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/apis/datasets/v1/data_access.proto b/apis/datasets/v1/data_access.proto index cadfa7e..d9257c0 100644 --- a/apis/datasets/v1/data_access.proto +++ b/apis/datasets/v1/data_access.proto @@ -107,6 +107,8 @@ enum LogicalOperator { // Every comparison evaluates to null (unknown) when the field is absent or explicitly null, including NOT_EQUAL. // Consequently, both `field != value` and `NOT(field == value)` require the field to be present. To include absent or // null fields, explicitly combine the comparison with FieldNullCheck using OR. +// Enum fields support only EQUAL and NOT_EQUAL. Use a LogicalExpression with OR over equality comparisons to match +// any of several members. Unknown enum values, numeric enum literals, and ordering comparisons are invalid arguments. message FieldComparison { // Name of a top-level field in the dataset schema. string field_name = 1 [(buf.validate.field).string = { @@ -119,7 +121,8 @@ message FieldComparison { defined_only: true not_in: [0] }]; - // Literal value. Its kind must match the field's protobuf descriptor. + // Literal value, interpreted using the field's protobuf descriptor. Enum fields accept enum_name for an exact + // member name; dataset-local custom enums also accept string_value for a public value as defined by DatasetType. FieldQueryValue value = 3 [(buf.validate.field).required = true]; } @@ -168,10 +171,17 @@ message FieldQueryValue { int64 int64_value = 2 [features.field_presence = EXPLICIT]; uint64 uint64_value = 3 [features.field_presence = EXPLICIT]; double double_value = 4 [features.field_presence = EXPLICIT]; + // Literal text for a string field, or a public value for a dataset-local custom enum field. + // For an enum field, the server prepends the field name uppercased followed by an underscore and resolves the + // resulting member name to its number, e.g. product_type and L1 resolve to PRODUCT_TYPE_L1. Matching is exact and + // case-sensitive, with no normalization or fallback to an exact member name. UNSPECIFIED is not a public value; + // use enum_name to compare the zero member, or FieldNullCheck to match an absent field. string string_value = 5 [features.field_presence = EXPLICIT]; google.protobuf.Timestamp timestamp_value = 6; google.protobuf.Duration duration_value = 7; - // Symbolic value from the field's enum descriptor. + // Exact, case-sensitive member name from the field's enum descriptor, e.g. PRODUCT_TYPE_L1. + // This is neither a short public value nor a fully qualified type name. The server resolves it to its number + // using the referenced field's enum descriptor. Use string_value to query a custom enum with a public value. string enum_name = 8 [features.field_presence = EXPLICIT]; bytes bytes_value = 9 [features.field_presence = EXPLICIT]; } diff --git a/apis/datasets/v1/dataset_type.proto b/apis/datasets/v1/dataset_type.proto index f7421be..57a25ad 100644 --- a/apis/datasets/v1/dataset_type.proto +++ b/apis/datasets/v1/dataset_type.proto @@ -93,6 +93,16 @@ message DatasetType { // for future support. It must not contain the dataset's datapoint message, which the server builds from fields. // Include any custom dependencies needed to resolve these definitions. File paths and fully qualified type // names must be unique and must not redefine built-in types. Definitions are resolved by name, not list order. + // + // Clients supply complete enum descriptors, including member names and numbers; the server does not generate + // or rewrite them. Each custom enum belongs to one field. Its type name is the field name in PascalCase, and + // its member prefix is the field name uppercased followed by an underscore. For product_type, clients declare + // ProductType with PRODUCT_TYPE_UNSPECIFIED = 0, PRODUCT_TYPE_L1 = 1, PRODUCT_TYPE_L2SP = 2, etc. + // Public values such as L1 must match [A-Z][A-Z0-9_]* and are appended unchanged to the prefix; UNSPECIFIED is + // reserved for the zero member and is not a public vocabulary value. Member numbers must be unique and positive + // except for that zero member. Existing member names and numbers must be preserved when extending a vocabulary; + // reordering values must not renumber them. Clients expose public values by removing the exact prefix once. + // Enum fields may be marked queryable without counting toward the queryable string field limit. google.protobuf.FileDescriptorSet type_definitions = 3; } From 4431d1b04b93e300574c5a4e8d8c0d7b5280754b Mon Sep 17 00:00:00 2001 From: Lukas Bindreiter Date: Tue, 27 Oct 2026 11:25:06 +0000 Subject: [PATCH 3/4] Define custom enum values through field annotations Replace supporting descriptor input with a string-only EnumValue message. Document server-owned enum generation, append-only updates, response ordering, and public-value filtering. Amp-Thread-ID: https://ampcode.com/threads/T-01a15368-905e-70db-a93b-31a2b835d742 Co-authored-by: Amp --- apis/datasets/v1/data_access.proto | 18 ++++++------ apis/datasets/v1/dataset_type.proto | 43 ++++++++++++++--------------- 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/apis/datasets/v1/data_access.proto b/apis/datasets/v1/data_access.proto index d9257c0..efc1ae5 100644 --- a/apis/datasets/v1/data_access.proto +++ b/apis/datasets/v1/data_access.proto @@ -121,8 +121,8 @@ message FieldComparison { defined_only: true not_in: [0] }]; - // Literal value, interpreted using the field's protobuf descriptor. Enum fields accept enum_name for an exact - // member name; dataset-local custom enums also accept string_value for a public value as defined by DatasetType. + // Literal value, interpreted using the field's schema. For custom enums, use string_value with a value from + // FieldAnnotation.enum_values. Use enum_name to compare an exact protobuf member name. FieldQueryValue value = 3 [(buf.validate.field).required = true]; } @@ -171,17 +171,15 @@ message FieldQueryValue { int64 int64_value = 2 [features.field_presence = EXPLICIT]; uint64 uint64_value = 3 [features.field_presence = EXPLICIT]; double double_value = 4 [features.field_presence = EXPLICIT]; - // Literal text for a string field, or a public value for a dataset-local custom enum field. - // For an enum field, the server prepends the field name uppercased followed by an underscore and resolves the - // resulting member name to its number, e.g. product_type and L1 resolve to PRODUCT_TYPE_L1. Matching is exact and - // case-sensitive, with no normalization or fallback to an exact member name. UNSPECIFIED is not a public value; - // use enum_name to compare the zero member, or FieldNullCheck to match an absent field. + // Text for a string field, or a value from FieldAnnotation.enum_values for a custom enum, e.g. "L1". + // The server resolves enum values to their numbers. Matching is case-sensitive; unknown values are rejected. + // Generated protobuf names are not accepted here unless they are also declared public values. string string_value = 5 [features.field_presence = EXPLICIT]; google.protobuf.Timestamp timestamp_value = 6; google.protobuf.Duration duration_value = 7; - // Exact, case-sensitive member name from the field's enum descriptor, e.g. PRODUCT_TYPE_L1. - // This is neither a short public value nor a fully qualified type name. The server resolves it to its number - // using the referenced field's enum descriptor. Use string_value to query a custom enum with a public value. + // Exact, case-sensitive member name from the field's enum descriptor. Prefer string_value for custom enums + // so callers do not need to know generated names. This also allows comparing the generated zero member; + // use FieldNullCheck instead to match an absent field. string enum_name = 8 [features.field_presence = EXPLICIT]; bytes bytes_value = 9 [features.field_presence = EXPLICIT]; } diff --git a/apis/datasets/v1/dataset_type.proto b/apis/datasets/v1/dataset_type.proto index 57a25ad..5f42857 100644 --- a/apis/datasets/v1/dataset_type.proto +++ b/apis/datasets/v1/dataset_type.proto @@ -16,12 +16,11 @@ option features.field_presence = IMPLICIT; // with its Tilebox annotation, matching the information represented separately by AnnotatedType. message Field { // The descriptor contains the name of the field, the type, optional labels (e.g. repeated) and other information. - // For TYPE_MESSAGE and TYPE_ENUM, type_name references an allowed built-in type or a type declared in - // DatasetType.type_definitions. Use a fully qualified name with a leading dot, e.g. `.datasets.v1.Vec3` - // or `.google.protobuf.Timestamp`. + // For built-in message and enum types, set type_name to the full name with a leading dot, e.g. `.datasets.v1.Vec3`. + // For a custom enum, set TYPE_ENUM, leave type_name empty, and provide annotation.enum_values. google.protobuf.FieldDescriptorProto descriptor = 1 [(buf.validate.field).required = true]; - // Additional documentation, source mapping, and query metadata for the field. + // Documentation, source mapping, query metadata, and custom enum values for the field. FieldAnnotation annotation = 2; } @@ -48,6 +47,12 @@ enum FieldRole { FIELD_ROLE_PRIMARY_TITLE = 1; } +// EnumValue is one allowed value of a custom enum. +message EnumValue { + // The value users ingest, query, and receive, e.g. "L1". + string value = 1 [(buf.validate.field).string.min_len = 1]; +} + // FieldAnnotation contains additional information about a field in a dataset. message FieldAnnotation { string description = 1; @@ -59,6 +64,7 @@ message FieldAnnotation { // Whether the field is projected into query storage and may be used in server-side filter expressions. // Queryable fields are not necessarily backed by a secondary database index. + // Enum fields do not count toward the queryable string field limit. bool queryable = 4; // Optional JSON Schema reference for this field when it corresponds to a well known schema. @@ -74,6 +80,16 @@ message FieldAnnotation { not_in: [0] } ]; + + // Allowed values for a custom enum field. Values must be unique; leave this empty for other field types. + // The server generates the protobuf enum names and numbers. Clients do not need to know its naming rules. + // Updates may append values without changing existing numbers. Removing, renaming, or reordering values + // is not supported yet. + // + // In responses, entries match the enum descriptor's members in order, excluding the generated zero member. + // Pair each entry with the corresponding member's number to encode and decode datapoints. List positions + // are not enum numbers. The zero member is not a public value; missing fields remain distinct from zero. + repeated EnumValue enum_values = 7; } // DatasetType is the request representation of a dataset type, used when creating or updating a dataset. @@ -87,29 +103,12 @@ message DatasetType { // A list of fields that this dataset consists of. repeated Field fields = 2; - - // Dataset-local supporting type definitions referenced by fields through descriptor.type_name. - // Initial support is limited to custom enums; the descriptor set can also represent message definitions - // for future support. It must not contain the dataset's datapoint message, which the server builds from fields. - // Include any custom dependencies needed to resolve these definitions. File paths and fully qualified type - // names must be unique and must not redefine built-in types. Definitions are resolved by name, not list order. - // - // Clients supply complete enum descriptors, including member names and numbers; the server does not generate - // or rewrite them. Each custom enum belongs to one field. Its type name is the field name in PascalCase, and - // its member prefix is the field name uppercased followed by an underscore. For product_type, clients declare - // ProductType with PRODUCT_TYPE_UNSPECIFIED = 0, PRODUCT_TYPE_L1 = 1, PRODUCT_TYPE_L2SP = 2, etc. - // Public values such as L1 must match [A-Z][A-Z0-9_]* and are appended unchanged to the prefix; UNSPECIFIED is - // reserved for the zero member and is not a public vocabulary value. Member numbers must be unique and positive - // except for that zero member. Existing member names and numbers must be preserved when extending a vocabulary; - // reordering values must not renumber them. Clients expose public values by removing the exact prefix once. - // Enum fields may be marked queryable without counting toward the queryable string field limit. - google.protobuf.FileDescriptorSet type_definitions = 3; } // AnnotatedType is the resolved representation returned on Dataset messages, including GetDataset responses. // It combines the generated protobuf descriptors with the corresponding Tilebox field annotations. message AnnotatedType { - // Descriptors for the dataset's datapoint message and its dataset-local supporting types. + // Descriptors for the dataset's datapoint message, including its generated enums. google.protobuf.FileDescriptorSet descriptor_set = 1; // Identifies the datapoint message in descriptor_set by name, independent of file or declaration order. string type_url = 2; From 038d5ea5fbb1fb61d4290ab90d93c92f55c817b5 Mon Sep 17 00:00:00 2001 From: Lukas Bindreiter Date: Tue, 27 Oct 2026 13:14:31 +0000 Subject: [PATCH 4/4] Remove unsupported enum-name query literals Use public string values as the only custom enum query representation. Reserve the old enum_name field name and tag to prevent accidental reuse. Amp-Thread-ID: https://ampcode.com/threads/T-01a15368-905e-70db-a93b-31a2b835d742 Co-authored-by: Amp --- apis/datasets/v1/data_access.proto | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/apis/datasets/v1/data_access.proto b/apis/datasets/v1/data_access.proto index efc1ae5..171668e 100644 --- a/apis/datasets/v1/data_access.proto +++ b/apis/datasets/v1/data_access.proto @@ -122,7 +122,7 @@ message FieldComparison { not_in: [0] }]; // Literal value, interpreted using the field's schema. For custom enums, use string_value with a value from - // FieldAnnotation.enum_values. Use enum_name to compare an exact protobuf member name. + // FieldAnnotation.enum_values. FieldQueryValue value = 3 [(buf.validate.field).required = true]; } @@ -160,12 +160,14 @@ message FieldQueryValue { "string_value", "timestamp_value", "duration_value", - "enum_name", "bytes_value" ] required: true }; + reserved 8; + reserved enum_name; + // Exactly one typed literal must be set. Explicit presence preserves zero and empty values. bool bool_value = 1 [features.field_presence = EXPLICIT]; int64 int64_value = 2 [features.field_presence = EXPLICIT]; @@ -177,10 +179,6 @@ message FieldQueryValue { string string_value = 5 [features.field_presence = EXPLICIT]; google.protobuf.Timestamp timestamp_value = 6; google.protobuf.Duration duration_value = 7; - // Exact, case-sensitive member name from the field's enum descriptor. Prefer string_value for custom enums - // so callers do not need to know generated names. This also allows comparing the generated zero member; - // use FieldNullCheck instead to match an absent field. - string enum_name = 8 [features.field_presence = EXPLICIT]; bytes bytes_value = 9 [features.field_presence = EXPLICIT]; }