Migrate from grpc / google protobuf to connect/buf protobuf-py

.docs/packages.d2

@@ -16,14 +16,34 @@ private: {
   style.fill: '#F5EFED'
 }
 
-public.tilebox-datasets -> private.tilebox-grpc: {
+public.tilebox-core: {
+  label: tilebox-core\ntilebox.query · tilebox.errors
+}
+
+private._tilebox: {
+  label: _tilebox\nOwned by tilebox-core
+}
+
+public.tilebox-core -> private._tilebox: {
+  label: implements
+}
+
+public.tilebox-datasets -> public.tilebox-core: {
   label: depends on
 }
 
 public.tilebox-storage -> public.tilebox-datasets: {
   label: enhances
 }
 
-public.tilebox-workflows -> private.tilebox-grpc: {
+public.tilebox-workflows -> public.tilebox-core: {
   label: depends on
 }
+
+public.tilebox-storage -> public.tilebox-core: {
+  label: depends on
+}
+
+public.tilebox-grpc -> public.tilebox-core: {
+  label: compatibility dependency only
+}

.github/workflows/main.yml

@@ -104,10 +104,10 @@ jobs:
         # Select the SDK packages, not the root's unbounded lint/build tooling.
         run: >-
           uv sync --resolution lowest-direct --upgrade
-          --package tilebox-grpc --package tilebox-datasets
+          --package tilebox-core --package tilebox-datasets
           --package tilebox-storage --package tilebox-workflows
       - name: Run tests without replacing the minimum-version environment
         run: |
-          for package in tilebox-grpc tilebox-datasets tilebox-storage tilebox-workflows; do
+          for package in tilebox-core tilebox-datasets tilebox-storage tilebox-workflows; do
             uv run --no-sync pytest "$package" -Wall -Werror --import-mode=importlib
           done

.github/workflows/publish_tilebox_core.yml

@@ -0,0 +1,33 @@
+name: Publish Tilebox Core
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  release:
+    name: Build and Publish Tilebox Core
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v10.1.0
+        with:
+          python-version: "3.13"
+          enable-cache: true
+          cache-python: true
+          cache-dependency-glob: "uv.lock"
+      - name: Print version
+        run: git describe --tags
+      - name: Build
+        run: mkdir dist && uv run python -m build tilebox-core && mv tilebox-core/dist/* dist/
+      - name: Publish to PyPi.org
+        if: github.event_name == 'release'
+        # Configure this workflow as a trusted publisher for tilebox-core on PyPI before releasing.
+        uses: pypa/gh-action-pypi-publish@release/v1

AGENTS.md

@@ -6,19 +6,20 @@ This repository is the Python monorepo for Tilebox SDK packages. It is organized
 
 ## Monorepo Architecture
 
-The workspace is split into four packages:
+The workspace contains four implementation packages and a compatibility package:
 
-- `tilebox-grpc`: Shared low-level gRPC/protobuf plumbing (`_tilebox.grpc`), sync and async channel handling, error mapping, pagination, replay, and async-to-sync helpers.
+- `tilebox-core`: Public `tilebox.query` value types and `tilebox.errors` exceptions; private shared infrastructure, generated messages, conversions, transports, pagination, replay, and concurrency helpers under `_tilebox`. Only this distribution packages `_tilebox`.
+- `tilebox-grpc`: Empty compatibility package depending on `tilebox-core>=0.67.0`, retained for one release.
 - `tilebox-datasets`: Datasets SDK (`tilebox.datasets`) with sync + async clients, query/data model helpers, protobuf conversion, and generated datasets-related protobuf stubs.
 - `tilebox-workflows`: Workflows SDK (`tilebox.workflows`) with job/cluster/automation clients, task model + runner, observability/tracing, and generated workflows protobuf stubs.
 - `tilebox-storage`: Storage SDK (`tilebox.storage`) for provider-specific payload downloads (ASF, Copernicus, Umbra, Landsat, local FS), built mostly on async internals with sync wrappers.
 
 Design choices to keep in mind:
 
 - Namespace package layout: multiple packages contribute modules under shared top-level namespaces like `tilebox.*`.
-- Layering pattern: generated protobuf/gRPC types -> service wrappers -> data/domain classes -> user-facing clients.
-- Sync + async parity: many APIs exist in both forms; sync wrappers are often derived from async implementations.
-- External API communication is gRPC-first, with protobuf-generated contracts.
+- Public clients call generated Connect clients directly and convert protobuf messages to domain objects.
+- Sync + async parity: datasets has native blocking and async clients with shared conversion helpers.
+- External API communication uses native Connect RPC clients and protobuf-py messages.
 
 ## Developer Tooling
 
@@ -27,7 +28,7 @@ Use `uv` for all local development.
 ### Setup
 
 ```bash
-uv sync
+uv sync --all-packages
 ```
 
 ### Test Commands
@@ -40,7 +41,7 @@ uv sync
 uv run --package tilebox-datasets pytest tilebox-datasets
 uv run --package tilebox-workflows pytest tilebox-workflows
 uv run --package tilebox-storage pytest tilebox-storage
-uv run --package tilebox-grpc pytest tilebox-grpc
+uv run --no-sync pytest tilebox-core
 ```
 
 ### Lint / Format / Type Check
@@ -65,7 +66,7 @@ Pre-commit hooks include YAML checks, EOF fixer, `sync-with-uv`, Ruff, and `ty`.
 - Linting/formatting is Ruff-based and fairly strict (`select = ["ALL"]`) with explicit ignores configured in root `pyproject.toml`.
 - Type hints are used broadly across public APIs and internals.
 - Prefer dataclasses and explicit domain objects for request/response translation.
-- Service modules generally wrap generated gRPC stubs and convert to internal Pythonic types.
+- Keep transport and error translation in `_tilebox`; request construction and response conversion belong in clients.
 - Workflows uses a lightweight structured facade over stdlib logging, with separate internal and task channels;
 - Tests use `pytest`, with async coverage (`pytest-asyncio`) and property-based testing (`hypothesis`) in multiple packages.
 
@@ -81,21 +82,24 @@ Pre-commit hooks include YAML checks, EOF fixer, `sync-with-uv`, Ruff, and `ty`.
 
 Generated files live under paths such as:
 
-- `tilebox-datasets/tilebox/datasets/datasets/v1/*_pb2*.py`
-- `tilebox-datasets/tilebox/datasets/tilebox/v1/*_pb2*.py`
-- `tilebox-workflows/tilebox/workflows/workflows/v1/*_pb2*.py`
+- `tilebox-datasets/tilebox/datasets/v1/*_pb.py`
+- `tilebox-datasets/tilebox/datasets/stac/v1/*_pb.py`
+- `tilebox-core/_tilebox/proto/tilebox/v1/*_pb.py`
+- `tilebox-workflows/tilebox/workflows/v1/*_pb.py`
+- `tilebox-workflows/tilebox/workflows/_otel/opentelemetry/proto/**`
 
 Regenerate protobuf code with:
 
 ```bash
-uv run generate-protobuf
+./tools/generate-protobuf.sh
 ```
 
-This uses `buf.gen.datasets.yaml` and `buf.gen.workflows.yaml`, then runs import-fixup logic via `tools/generate_protobuf.py`.
+This uses `buf.gen.yaml` with pinned local plugins and import mappings.
+The script strips upstream trailing whitespace with Ruff and places the generated modules in their owning packages.
 
 ### Important Boundary: API Schema Source Of Truth
 
 - Many protobuf contracts are sourced from the separate `api` repository/module (`buf.build/tilebox/api`, with optional local `../api` input in buf configs).
-- Do **not** hand-edit generated `*_pb2.py`, `*_pb2.pyi`, or `*_pb2_grpc.py` files.
+- Do **not** hand-edit generated `*_pb.py` or `*_connect.py` files.
 - If a requested change requires modifying protobuf schema/contracts, that change must be made in the `api` repo first.
 - Whenever schema/proto edits are needed, ask the developer to point you to the `api` repo so those changes can be implemented there and then regenerated here.

CHANGELOG.md

@@ -7,6 +7,46 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- `tilebox-core`: New package to own shared Connect RPC infrastructure and protobuf-py messages, public
+  `tilebox.query` interval/pagination types, and `tilebox.errors` exceptions with a common `TileboxError` base.
+- `tilebox-grpc`: Retain an empty compatibility package depending on `tilebox-core>=0.67.0` for one release.
+- `tilebox-datasets`, `tilebox-workflows`, and `tilebox-storage`: Depend explicitly on
+  `tilebox-core`; datasets retains its scientific and LZ4 dependencies.
+- `tilebox-workflows`: Remove the runtime dependency on datasets. Existing public datasets imports
+  of shared intervals, pagination, and conversions remain compatible. Task interval serialization is unchanged.
+- `tilebox-workflows`: `GoogleStorageCache` now accepts only bucket names and uses obstore with google-auth.
+  Remove support for Google Storage SDK `Bucket` objects.
+- Parse ordinary Python datetimes without pandas. Strings and scientific datetime values still use pandas;
+  workflows-only users need to install pandas explicitly for those inputs. Dataset installations are unchanged.
+- Raise `tilebox.errors.RequestCanceledError` for remote request cancellation instead of `KeyboardInterrupt`.
+- Require Tilebox sibling dependencies version 0.67.0 or newer. When upgrading from older releases,
+  upgrade all installed Tilebox packages together; older releases do not enforce this minimum.
+- Use Connect over HTTP/2 by default. Keep `transport="http1"` for proxies that block HTTP/2,
+  and accept `transport="grpc"` as a silent alias for `"http2"`.
+- Replace generated `_pb2` modules with native `_pb` modules. User-defined protobuf task
+  arguments use protobuf-py messages; existing stored binary task payloads remain compatible.
+- Resolve dynamic dataset schemas by descriptor content and compile scientific conversion
+  plans per schema, including projected metadata-only queries.
+- Bundle native OTEL response types in workflows and convert them directly into log/span
+  objects. Keep the official OpenTelemetry SDK and HTTP exporters for telemetry export.
+- Serve managed workers over Connect HTTP/1 with independent log streams and task calls.
+  Managed workers require a CLI with Connect worker support; that CLI still supports older workers.
+
+### Deprecated
+
+- `tilebox-datasets`: Deprecate the root `TimeseriesDataset` and `TimeseriesCollection` aliases.
+  They still refer to asynchronous clients. Use `tilebox.datasets.aio.DatasetClient` and
+  `CollectionClient` for the same classes, or the root `DatasetClient` and `CollectionClient`
+  for synchronous code.
+
+### Fixed
+
+- `tilebox-workflows`: Flush and close client-owned tracing exporters when closing a client,
+  without shutting down shared exporters. Release closed clients from the tracing registry.
+- `tilebox-datasets`: Count each page's actual datapoints in synchronous query progress bars.
+
 ## [0.66.0] - 2026-10-14
 
 ### Added

CONTRIBUTING.md

@@ -7,6 +7,12 @@ Welcome! We're excited that you're interested in contributing to Tilebox Python.
 This repository contains multiple python packages. We use python [namespace packages](https://packaging.python.org/en/latest/guides/packaging-namespace-packages/) to
 organize our code into multiple packages that then share the common `tilebox` python module name.
 
+`tilebox-core` owns the public `tilebox.query` and `tilebox.errors` APIs and the private
+`_tilebox` infrastructure namespace. Datasets, workflows, and storage depend on core;
+scientific dependencies stay in datasets. Core and workflows have no runtime dependency
+on datasets. Their cross-SDK tests run in the all-packages workspace environment.
+`tilebox-grpc` is a dependency-only compatibility package retained for the 0.67 release.
+
 ## Required tooling
 
 Before starting to work on this repository, make sure you have uv installed. [Install instructions](https://docs.astral.sh/uv/getting-started/installation/).
@@ -23,7 +29,7 @@ them.
 ### Initial setup
 
 ```bash
-uv sync
+uv sync --all-packages
 ```
 
 ### Running tests
@@ -33,6 +39,7 @@ uv sync
 
 # testing a single package
 uv run --package tilebox-datasets pytest tilebox-datasets
+uv run --no-sync pytest tilebox-core
 ```
 
 ### Running code analysis
@@ -60,17 +67,28 @@ uv add --package tilebox-datasets "numpy>=2"
 ## Protobuf usage
 
 We use [protobuf](https://protobuf.dev/) for defining our API contracts. The protobuf files are generated from the messages
-defined in the [Tilebox Go repository](https://github.com/tilebox/tilebox-go/tree/main/apis).
+defined in [Tilebox API](https://buf.build/tilebox/api). `buf.gen.yaml` defines schema inputs, import mappings, and pinned native Python plugins.
 
 ### Updating protobuf files
 
 ```bash
-uv run generate-protobuf
+./tools/generate-protobuf.sh
 ```
 
+The script generates into a temporary directory, strips trailing whitespace with Ruff, and replaces the generated
+modules in each package. API types live in `tilebox.datasets.v1`, `tilebox.datasets.stac.v1`, and `tilebox.workflows.v1`;
+shared generated types live in `_tilebox.proto.tilebox.v1`. Workflows bundles its API telemetry types privately under
+`tilebox.workflows._otel`. The OpenTelemetry SDK and exporters continue to use their own generated types.
+
 ## Releasing and deploying a new version
 
 There is a Github CI workflow set up for building the packages and pushing them to the internal `pypi`.
+Sibling-package dependencies require at least version 0.67.0 for the Connect migration.
+Raise these minimums only when a package needs a newer sibling. Make core available before its dependents,
+and datasets before storage. The release workflows run independently; do not announce installation availability
+until all packages have published successfully.
+Before the first core release, configure PyPI trusted publishing for `tilebox-core` using this repository's
+`publish_tilebox_core.yml` workflow. The compatibility `tilebox-grpc` wheel must contain no Python modules.
 All that is needed is to create a github release for a given tag. This can be done in the github web UI or on the
 command line.
 

README.md

@@ -48,6 +48,12 @@ pip install tilebox
 
 Documentation is available at [docs.tilebox.com](https://docs.tilebox.com).
 
+The SDKs share types and native Connect RPC infrastructure in `tilebox-core`, installed
+automatically with datasets, workflows, and storage. Use `tilebox.query` for shared
+intervals and pagination, and `tilebox.errors` for SDK exceptions. Infrastructure under
+`_tilebox` is private. Core and workflows do not require the datasets scientific stack.
+`tilebox-grpc` is a dependency-only compatibility package for the 0.67 release.
+
 ## Getting started
 
 ### Tilebox Datasets

buf.gen.yaml

@@ -0,0 +1,30 @@
+# yaml-language-server: $schema=https://www.schemastore.org/buf.gen.json
+version: v2
+plugins:
+  - local: [uv, tool, run, --from, protoc-gen-py==0.6.0, --with, protobuf-py==0.6.0, protoc-gen-py]
+    out: .
+    opt:
+      - init_files=false
+      - map_imports=datasets/**:tilebox
+      - map_imports=workflows/**:tilebox
+      - map_imports=tilebox/**:_tilebox.proto
+      - map_imports=opentelemetry/**:tilebox.workflows._otel
+  - local: [uv, tool, run, --from, protoc-gen-connectrpc==0.12.1, --with, protobuf-py==0.6.0, protoc-gen-connectrpc]
+    out: .
+    opt:
+      - map_imports=datasets/**:tilebox
+      - map_imports=workflows/**:tilebox
+      - map_imports=tilebox/**:_tilebox.proto
+      - map_imports=opentelemetry/**:tilebox.workflows._otel
+inputs:
+  # For local schema development, replace module with directory: ../api.
+  - module: buf.build/tilebox/api
+    paths: [datasets, tilebox, workflows]
+    # Server-side annotations are not needed by the SDK.
+    exclude_types: ["buf.validate.**", "google.api.**"]
+  - module: buf.build/opentelemetry/opentelemetry:5f2c7d4f740541589805e0816dad4bb0
+    paths:
+      - opentelemetry/proto/common/v1/common.proto
+      - opentelemetry/proto/resource/v1/resource.proto
+      - opentelemetry/proto/logs/v1/logs.proto
+      - opentelemetry/proto/trace/v1/trace.proto