Dataset enum field support

CHANGELOG.md

@@ -7,8 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.68.0] - 2026-10-29
+
 ### Added
 
+- `tilebox-datasets`: Support scalar and repeated custom enums in schema definitions with `"type": "enum"`,
+  `enum_values`, and optional `"repeated": True`. Ingest and query case-sensitive public strings through
+  sync and async clients, including pandas and xarray inputs.
+- `tilebox-datasets`: Add `field(...).contains()`, `.contains_all()`, and `.contains_any()` filters for
+  queryable repeated enums. Empty `contains_all([])` matches every row; `contains_any([])` matches none.
+  Negated membership includes untagged rows. Requires a backend supporting enum membership predicates.
 - `tilebox-workflows`: Add `workflow` selection by reference or object to job submission and automation creation,
   and `workflows` filters to job queries. Omitting scope preserves unscoped direct-runner behavior.
 - `tilebox-workflows`: Preserve task workflow IDs and inherit the executing parent's scope for children.
@@ -18,6 +26,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- `tilebox-datasets`: Return repeated enums, including built-ins such as SAR polarizations, as one
+  `list[str]` per row in xarray object columns, preserving the returned order. Remove their extra
+  `n_<field>` dimension and retain empty values as `[]`, even when the entire returned column is empty.
+  Update code that indexes the former array dimension to access the per-row list instead.
+  Other repeated fields keep their padded arrays.
 - `tilebox-workflows`: Always create scoped-only workflows with legacy task aliases disabled.
 - `tilebox-workflows`: Use `UnboundTaskIdentifier` for task metadata, runner registries, and release contents;
   keep execution scope in `TaskIdentifier` and resolve each future task's scope before merging submissions.
@@ -716,7 +729,8 @@ the first client that does not cache data (since it's already on the local file
 - Released under the [MIT](https://opensource.org/license/mit) license.
 - Released packages: `tilebox-datasets`, `tilebox-workflows`, `tilebox-storage`, `tilebox-grpc`
 
-[Unreleased]: https://github.com/tilebox/tilebox-python/compare/v0.67.0...HEAD
+[Unreleased]: https://github.com/tilebox/tilebox-python/compare/v0.68.0...HEAD
+[0.68.0]: https://github.com/tilebox/tilebox-python/compare/v0.67.0...v0.68.0
 [0.67.0]: https://github.com/tilebox/tilebox-python/compare/v0.66.0...v0.67.0
 [0.65.0]: https://github.com/tilebox/tilebox-python/compare/v0.64.0...v0.65.0
 [0.64.0]: https://github.com/tilebox/tilebox-python/compare/v0.63.2...v0.64.0

tilebox-datasets/README.md

@@ -87,6 +87,97 @@ results = s2a_l1c.query(
 print(f"Found {results.sizes['time']} datapoints")  # Found 979 datapoints
 ```
 
+## Enum fields
+
+Define a scalar enum with an ordered list of public strings:
+
+```python
+from datetime import UTC, datetime
+
+from tilebox.datasets import field
+from tilebox.datasets.data.datasets import DatasetKind
+
+dataset = client.create_or_update_dataset(
+    DatasetKind.TEMPORAL,
+    "scenes",
+    fields=[
+        {"name": "processing_level", "type": "enum", "enum_values": ["L1", "L2-SP", "雪"], "queryable": True},
+    ],
+)
+collection = dataset.get_or_create_collection("example")
+collection.ingest(
+    [
+        {"time": datetime(2026, 10, 28, 12, tzinfo=UTC), "processing_level": "L2-SP"},
+        {"time": datetime(2026, 10, 28, 13, tzinfo=UTC)},
+    ]
+)
+
+results = collection.query(
+    temporal_extent=("2026-10-28", "2026-10-29"),
+    filter=field("processing_level") == "L2-SP",
+)
+assert results.processing_level.to_numpy().tolist() == ["L2-SP"]
+```
+
+Ingestion accepts public strings in records, columns, pandas DataFrames, and xarray Datasets.
+Query results contain public strings in object arrays. Missing values remain missing; use xarray's
+`isnull()` to identify them. You do not need a Python enum class to create or read a dataset.
+Numbers and generated protobuf symbols are not accepted as substitutes for public values.
+
+Scalar enum filters support `==` and `!=`. Combine equality expressions with `|` to match several values.
+The server validates values case-sensitively. Comparisons exclude missing values, including `!=`;
+include them explicitly with `(field("processing_level") != "L1") | field("processing_level").is_null()`.
+
+Vocabularies contain 1–100 unique, nonempty strings. Empty datasets allow replacing, removing,
+renaming, or reordering values. Once a dataset contains data, updates may only append values.
+Supply the full field schema when updating a dataset, and re-fetch the dataset after another client
+updates its vocabulary. The asynchronous client provides the same enum behavior.
+
+### Repeated enums
+
+Add `"repeated": True` to define lists of enum values, including queryable tags:
+
+```python
+fields = [
+    {
+        "name": "use_cases",
+        "type": "enum",
+        "enum_values": ["agriculture", "defense", "wind-power"],
+        "repeated": True,
+        "queryable": True,
+    },
+]
+dataset = client.create_or_update_dataset(DatasetKind.TEMPORAL, "tagged_scenes", fields)
+collection = dataset.get_or_create_collection("example")
+collection.ingest(
+    [
+        {"time": datetime(2026, 10, 28, 12, tzinfo=UTC), "use_cases": ["defense", "agriculture", "defense"]},
+    ]
+)
+results = collection.query(
+    temporal_extent=("2026-10-28", "2026-10-29"),
+    filter=field("use_cases").contains("defense"),
+)
+
+# Alternative membership filters:
+all_tags = field("use_cases").contains_all(["defense", "agriculture"])
+any_tag = field("use_cases").contains_any(["defense", "wind-power"])
+without_defense = ~field("use_cases").contains("defense")
+```
+
+The backend removes duplicates during ingestion, preserving first-occurrence order. The example
+stores `["defense", "agriculture"]`. Ingest lists of public strings through records, columns, or pandas;
+xarray results contain one `list[str]` per row in an object column, `use_cases(time)`, without an
+extra array dimension. Empty values are `[]`, including when every returned value is empty.
+Lists preserve the returned order and can be ingested again without changing enum ordering.
+Other repeated fields retain their existing padded array layout.
+
+Repeated enums use membership filters, not `==`, `!=`, or null checks. Combine predicates with `&`, `|`,
+and `~`. Membership treats omitted and empty lists as the empty set and never returns null.
+Consequently, `without_defense` includes untagged rows. `contains_all([])` matches every row;
+`contains_any([])` matches none. Each predicate accepts up to 100 nonempty public strings, counting
+duplicates toward that limit. The server checks vocabulary membership and whether the field is queryable.
+
 ## Documentation
 
 Check out the [Tilebox Datasets documentation](https://docs.tilebox.com/datasets/introduction) for more information.

tilebox-datasets/tests/data/datasets.py

@@ -46,7 +46,10 @@ def field_annotations(draw: DrawFn) -> FieldAnnotation:
     queryable = draw(booleans())
     json_schema_ref = draw(text(alphabet=string.ascii_letters + string.digits + "#/_-", max_size=25) | none())
     roles = draw(lists(sampled_from(FieldRole), unique=True))
-    return FieldAnnotation(description, example_value, source_json_pointer, queryable, json_schema_ref, roles)
+    enum_values = draw(lists(text(min_size=1, max_size=20), unique=True, max_size=5))
+    return FieldAnnotation(
+        description, example_value, source_json_pointer, queryable, json_schema_ref, roles, enum_values
+    )
 
 
 @composite

tilebox-datasets/tests/data/test_datasets.py

@@ -1,6 +1,6 @@
 from collections.abc import Callable
 from dataclasses import replace
-from typing import Any, Literal
+from typing import Any, Literal, cast
 from unittest.mock import MagicMock, patch
 from uuid import uuid4
 
@@ -128,6 +128,35 @@ def test_message_types_can_define_dataset_fields(message_type: type, message_nam
     assert repeated.descriptor.label == FieldDescriptorProto.Label.REPEATED
 
 
+@pytest.mark.parametrize(
+    "definition",
+    [
+        {"type": "enum"},
+        {"type": "enum", "enum_values": []},
+        {"type": "enum", "enum_values": [""]},
+        {"type": "enum", "enum_values": ["L1", "L1"]},
+        {"type": "enum", "enum_values": [1]},
+        {"type": "enum", "enum_values": "L1"},
+        {"type": "enum", "enum_values": [str(i) for i in range(101)]},
+        {"type": str, "repeated": True},
+        {"type": str, "enum_values": []},
+    ],
+)
+def test_invalid_enum_definitions(definition: dict[str, Any]) -> None:
+    with pytest.raises(ValueError, match="Field 'level'"):
+        Field.from_dict(cast(FieldDict, {"name": "level", **definition}))
+
+
+@pytest.mark.parametrize("repeated", [False, True])
+def test_enum_vocabulary_limit(repeated: bool) -> None:
+    values = [str(i) for i in range(100)]
+    definition: FieldDict = {"name": "level", "type": "enum", "enum_values": values, "repeated": repeated}
+    assert Field.from_dict(definition).annotation.enum_values == values
+    values.append("100")
+    with pytest.raises(ValueError, match="at most 100"):
+        Field.from_dict(definition)
+
+
 @given(fields())
 def test_fields_to_message_and_back(field: Field) -> None:
     assert Field.from_message(field.to_message()) == field
@@ -159,7 +188,8 @@ def test_list_datasets_responses_to_message_and_back(response: ListDatasetsRespo
 
 
 @pytest.mark.parametrize("operation", ["create", "update"])
-def test_create_and_update_dataset_include_field_annotations(operation: str) -> None:
+@pytest.mark.parametrize("repeated", [False, True])
+def test_create_and_update_dataset_include_field_annotations(operation: str, repeated: bool) -> None:
     """Public dataset writes send required and custom field annotations to native clients."""
     dataset_service = MagicMock()
     if operation == "create":
@@ -172,7 +202,14 @@ def test_create_and_update_dataset_include_field_annotations(operation: str) ->
             "queryable": True,
             "json_schema_ref": "https://example.com/schema.json#/title",
             "roles": [FieldRole.PRIMARY_TITLE],
-        }
+        },
+        {
+            "name": "level",
+            "type": "enum",
+            "enum_values": ["L1", "L2-SP", "雪"],
+            "queryable": True,
+            "repeated": repeated,
+        },
     ]
 
     with (
@@ -195,9 +232,20 @@ def test_create_and_update_dataset_include_field_annotations(operation: str) ->
     assert annotations["geometry"].source_json_pointer == "/geometry"
     assert annotations["geometry"].json_schema_ref == "https://geojson.org/schema/Geometry.json"
     assert all(
-        not annotation.queryable and not annotation.roles for name, annotation in annotations.items() if name != "title"
+        not annotation.queryable and not annotation.roles
+        for name, annotation in annotations.items()
+        if name not in {"title", "level"}
     )
     assert annotations["title"].source_json_pointer == "/properties/title"
     assert annotations["title"].queryable
     assert annotations["title"].json_schema_ref == "https://example.com/schema.json#/title"
     assert list(annotations["title"].roles) == [FieldRole.PRIMARY_TITLE.value]
+    enum_field = request.type.fields[-1]
+    assert enum_field.descriptor.type == FieldDescriptorProto.Type.ENUM
+    assert enum_field.descriptor.label == (
+        FieldDescriptorProto.Label.REPEATED if repeated else FieldDescriptorProto.Label.OPTIONAL
+    )
+    assert not enum_field.descriptor.type_name
+    assert enum_field.annotation.queryable
+    assert [entry.value for entry in enum_field.annotation.enum_values] == ["L1", "L2-SP", "雪"]
+    assert all(field.descriptor.options is None for field in request.type.fields)

tilebox-datasets/tests/protobuf_conversion/test_protobuf_xarray.py

@@ -1,4 +1,5 @@
 from enum import Enum
+from typing import Any
 from uuid import UUID
 
 import numpy as np
@@ -343,7 +344,7 @@ class Polarization(Enum):
     message_type = get_message_type(
         AnnotatedType(FileDescriptorSet(file=[file_descriptor]), "tests.protobuf_conversion.enums.EnumDatapoint", [])
     )
-    messages = [
+    messages: list[Any] = [
         message_type(primary_polarization=1, polarizations=[0, 1]),
         message_type(primary_polarization=2, polarizations=[2]),
         message_type(),
@@ -357,11 +358,8 @@ class Polarization(Enum):
     assert dataset.primary_polarization[:2].to_numpy().tolist() == ["HH", "VV"]
     assert pd.isna(dataset.primary_polarization[2].item())
     assert dataset.polarizations.dtype == object
-    assert dataset.polarizations[0].to_numpy().tolist() == ["UNSPECIFIED", "HH"]
-    assert dataset.polarizations[1, 0].item() == "VV"
-    assert pd.isna(dataset.polarizations[1, 1].item())
-    assert pd.isna(dataset.polarizations[2].to_numpy()).all()
-    assert dataset.polarizations.dims == ("time", "n_polarizations")
+    assert dataset.polarizations.to_numpy().tolist() == [["UNSPECIFIED", "HH"], ["VV"], []]
+    assert dataset.polarizations.dims == ("time",)
     assert dataset.primary_polarization.attrs == {}
     assert dataset.polarizations.attrs == {}
     assert to_messages(dataset, message_type) == messages
@@ -387,10 +385,8 @@ def test_convert_sar_polarizations_with_short_names_in_both_directions() -> None
 
     dataset = converter.finalize("item")
 
-    assert dataset.polarizations[0].to_numpy().tolist() == ["HH", "VV"]
-    assert dataset.polarizations[1, 0].item() == "VV"
-    assert pd.isna(dataset.polarizations[1, 1].item())
-    assert pd.isna(dataset.polarizations[2].to_numpy()).all()
+    assert dataset.polarizations.dims == ("item",)
+    assert dataset.polarizations.to_numpy().tolist() == [["HH", "VV"], ["VV"], []]
 
     other_fields = [field.name for field in SARProperties.desc().fields if field.name != "polarizations"]
     assert to_messages(dataset, SARProperties, ignore_fields=other_fields) == messages

tilebox-datasets/tests/protobuf_conversion/test_to_protobuf.py

@@ -105,6 +105,19 @@ def test_record_oriented_data_preserves_absent_fields_and_filters_missing_values
     assert converted[1].some_int == 0
 
 
+@pytest.mark.parametrize(
+    ("values", "expected"), [([], []), ([np.nan], []), ([None, np.nan], []), ([3, np.nan, 7], [3, 7])]
+)
+@pytest.mark.parametrize("as_array", [False, True])
+def test_required_repeated_field_is_not_a_missing_scalar(
+    values: list[float | None], expected: list[int], as_array: bool
+) -> None:
+    value = np.array(values) if as_array else values
+    converted = to_messages([{"some_repeated_int": value}], ExampleDatapoint, required_fields=["some_repeated_int"])
+
+    assert converted[0].some_repeated_int == expected
+
+
 def test_record_oriented_data_requires_every_record_to_have_required_fields() -> None:
     with pytest.raises(ValueError, match=r"Record 1: Missing required field.*time"):
         to_messages(