From 91dcf43cc314f29d24c8a2c766453e5e4508786d Mon Sep 17 00:00:00 2001 From: Gian Date: Wed, 14 Oct 2026 17:07:33 +0800 Subject: [PATCH 01/15] Add versioned native Iceberg storage path codecs --- ...34-access-iceberg-native-storage-layout.md | 324 ++++++++ doc/backlog/backlog.md | 8 +- .../design-crowdb-iceberg-upload-flow.md | 41 +- .../iceberge/design-crowdb-iceberg.md | 188 ++++- doc/doc_index.md | 4 +- doc/working/plan-iceberg-storage-path.md | 744 ++++++++++++++++++ .../src/catalog/storage.rs | 27 +- lib/crowdb-access-iceberg/src/file.rs | 3 + .../src/file/native_location.rs | 99 +++ lib/crowdb-access-iceberg/src/key.rs | 6 + lib/crowdb-access-iceberg/src/key/storage.rs | 47 ++ .../src/key/storage_path.rs | 74 ++ .../src/key/table_storage.rs | 153 ++++ .../tests/native_storage_path_test.rs | 225 ++++++ 14 files changed, 1907 insertions(+), 36 deletions(-) create mode 100644 doc/backlog/R234-access-iceberg-native-storage-layout.md create mode 100644 doc/working/plan-iceberg-storage-path.md create mode 100644 lib/crowdb-access-iceberg/src/file/native_location.rs create mode 100644 lib/crowdb-access-iceberg/src/key/storage.rs create mode 100644 lib/crowdb-access-iceberg/src/key/storage_path.rs create mode 100644 lib/crowdb-access-iceberg/src/key/table_storage.rs create mode 100644 lib/crowdb-access-iceberg/tests/native_storage_path_test.rs diff --git a/doc/backlog/R234-access-iceberg-native-storage-layout.md b/doc/backlog/R234-access-iceberg-native-storage-layout.md new file mode 100644 index 000000000..eac7fff5a --- /dev/null +++ b/doc/backlog/R234-access-iceberg-native-storage-layout.md @@ -0,0 +1,324 @@ + + + +### R234: access server / Iceberg — Native storage path and SDK-compatible file access + +## Problem + +Iceberg table files currently use a mixed location model: table identity is +encoded in one location form while file authority and parquet data can be +resolved through unrelated keys or directories. This makes the KV layout hard +to inspect, leaves rename and cross-namespace move invariants implicit, and +adds avoidable metadata work before a native file read. The root design is +[`design-crowdb-iceberg.md`](../design/access-server/iceberge/design-crowdb-iceberg.md) +and the execution details are in +[`plan-iceberg-storage-path.md`](../working/plan-iceberg-storage-path.md). + +The public contract cannot add a second chunk-location field: Iceberg clients +consume one `data_file.file_path` URL and third-party SDKs cannot be changed. +The URL must remain S3-shaped for compatibility while the bytes remain owned +by native `ChunkType::IcebergTable` storage. A client or server must also not +be able to list the whole Iceberg bucket by omitting a prefix. + +## Solution + +The requirement establishes these invariants: + +- **ICEBERG-PATH-1 — Immutable identity path:** every table-owned path starts + with `///`. The namespace ID is + captured at table creation. Rename and cross-namespace move update the live + name mapping only; they do not move file bytes or rewrite file URLs. +- **ICEBERG-PATH-2 — Fixed compatibility bucket:** every public URI uses the + fixed virtual bucket `iceberg`; the three IDs are object-key segments. A + table ID is never used as a bucket. +- **ICEBERG-PATH-3 — One public file URL:** Iceberg `file_path` is the only + SDK-visible file location. It uses the fixed bucket and immutable table + prefix. Native injection can issue a direct locator after durable upload, + before constructing manifests. Unmodified SDK writers keep their chosen + logical URL and resolve its native file record. Native injection determines + the final URL before manifest construction; readers need no additional field. +- **ICEBERG-PATH-4 — One file record:** each published file has exactly one + `file/` record under the immutable table prefix. + The filename is also the public URL suffix and list key suffix. Native + injection can use `f/v1/..parquet`; the authenticated + locator includes a unique file identity, so repeated readable stems do not + collide. The value contains file ID, length, integrity metadata, format, + native descriptor and lifecycle state. No path/public/id secondary file + indexes are created. List, inspection, recovery and GC use this same record. +- **ICEBERG-PATH-5 — Table-scoped list:** ListObjectsV2 accepts only a complete + table prefix `///` belonging to + the authorized request. Empty, catalog-only, namespace-only, foreign-table, + and bucket-wide prefixes are rejected. `prefix`, `delimiter`, `max-keys`, + continuation, start-after, URL encoding, common prefixes, and truncation + fields retain S3 behavior. +- **ICEBERG-PATH-6 — Native lifecycle:** file publication binds one immutable + file identity to one canonical public URL. Logical deletion removes the file + from the selected index/list and schedules delayed GC. A holder of its direct + URL with a valid grant may still read retained bytes until physical reclamation. + Direct reads do not query a file tombstone or location mapping. + Retained chunk IDs and offsets remain stable until deletion and reclamation. +- **ICEBERG-PATH-7 — SDK write compatibility:** ordinary SDK location providers + choose names locally before PUT; PUT admission cannot change that URL to + contain locations allocated later. Preserve the original URL for these + files. An internal injection operation may return one final direct URL after + publication; consumers put that URL into manifests using standard fields. +- **ICEBERG-PATH-8 — Locator classes:** logical URLs use the public-key file + index; direct URLs carry a versioned authenticated native descriptor or a + chunk-resident directory root. A token resolved through another KV mapping + is not a direct locator. Both classes bind the full table identity. Locator + authenticity is separate from access authorization: the persistent URL has + no credential expiry, while each HTTP request requires a valid grant. +- **ICEBERG-PATH-9 — No hidden rewrite:** metadata and manifest references are + kept byte-for-byte at the SDK-supplied URL. Commit publication does not scan + or rewrite manifests. Direct URLs are chosen before the first manifest is + constructed. Existing data-file names, position-delete paths, deletion-vector + references and historical snapshots are never silently converted to aliases. +- **ICEBERG-PATH-10 — Versioned migration:** the new slash-component layout is + encoded as a new key-layout version. Existing v1 binary keys remain readable + by the legacy decoder and are never reinterpreted as v2. New catalogs use v2; + existing catalogs retain v1 reads, writes and URLs. In-place conversion is + outside this requirement; activation must reject attempts to relabel a v1 + catalog as v2. Never rewrite historical metadata just to change its keys. +- **ICEBERG-PATH-11 — Control-plane isolation:** `iceberg` is one virtual + bucket, not a general bucket authority. HeadBucket and region/endpoint + discovery are satisfied by the Iceberg listener and table grant; bucket + creation/deletion and unscoped ListBuckets/ListObjects are rejected. Object + operations require a complete table prefix and a grant for that immutable + storage path. + +The model is defined in the permanent Iceberg design; the working plan tracks +implementation and verification. Documentation agreement is not evidence that +the current runtime already implements the new layout or direct-read path. + +The request path is: + +```text +Iceberg file_path URL + -> fixed bucket and three-ID table scope + -> validated native file locator + -> chunk read or table-scoped KV index operation + -> IcebergTable chunk storage +``` + +1. Add one canonical identity/path codec for catalog ID, storage namespace ID, + table ID, and safe relative file keys. Use it for table metadata, file + authorities, file-by-path indexes, multipart records, GC records, grants, + and continuation tokens. Reject traversal, malformed encodings, aliases, + and identity mismatches. +2. Assign the immutable storage path during normal and staged table creation. + Persist the fixed path in the table head and initial metadata. Keep the + live `(namespace, name)` mapping separate so rename, move, drop, and + recreate cannot alias retained files. +3. Move all table-owned KV records below the canonical identity prefix. Publish + one `file/` record with a create-only CAS after + native bytes are durable. Preserve the selected record on collision; + replay the same upload identity/result, and reject conflicting uploads. + Carry the canonical filename in lifecycle tasks rather than adding an ID + index. Bound every scan to the table prefix. +4. Keep the native Iceberg chunk type and `FileBlockStore` as the byte owner. + In native injection, finalize bytes and a bounded read descriptor, durably + publish the file, then return its one direct URL. Ordinary PUT/multipart + publishes a descriptor under its original URL. Implement exact GET, HEAD, + Range GET and cleanup for both classes without silently switching names. +5. Validate the fixed bucket and complete table prefix at the HTTP/FileIO + boundary. Do not expose bucket-wide `ListBuckets` or unscoped object list + results. For a missing exact object, support the SDK fallback list of + `/` without widening the authorization scope. +6. Return list results from the native file index with public `file_path` keys, + not by scanning chunk placement. Preserve Arrow/PyArrow directory behavior: + a `FileSelector` base directory becomes a key prefix with one trailing `/`, + while an exact file first uses HEAD and may list `/` only as a directory + fallback. Arrow's S3 implementation derives the bucket from the URI host + and the key from the URI path; it does not infer catalog or namespace from + the bucket. +7. Align the loader and container injection flow with the returned canonical + URI. Any optional general-S3 mirror remains a separate demo artifact and is + not an Iceberg file authority. +8. Reconcile the permanent Iceberg design and upload-flow documents with this + contract. In particular, replace the older bucket/table-prefix examples, + state that commit does not rewrite manifest paths, and define the direct + chunk-read API as separate from Iceberg file-record lookup. Do not mark R234 + complete while those documents disagree with the implemented contract. + +Defined edge behavior: + +- Catalog/name mappings and table head are sufficient to find the complete + storage identity; no table-ID reverse index is required. One file-record + type covers metadata JSON, manifest-list, manifest, data/delete and statistics + files. Snapshots and refs remain standard JSON fields, including the v1 + direct-manifest compatibility branch. No extra snapshot authority is added. +- Large metadata is a native chunk-backed object. Head and file values stay + bounded; the full snapshot array belongs in the canonical metadata JSON. + Existing validated generation projections may accelerate REFS reads but are + disposable. ALL responses and format-version requirements stay intact. +- Public `metadata/` and `statistics/` prefixes map directly to the equivalent + bounded `file/` KV ranges. List pages never decode those object payloads, + accumulate the complete table or scan unrelated file kinds. Snapshot + enumeration inside JSON is separate from file listing. Metadata/parser/ + response budgets reject oversize input explicitly, never truncate history. +- Head publication follows durable standard files and validates the prior + generation. Retained snapshots, branches and tags protect shared files from + GC. Removing an entry from the current snapshot alone proves no deletion right. + +- The three-ID path is the payload inside Iceberg storage; omit a redundant + `iceberg/` component. Catalog/namespace/name control records use separate + scopes. Live name lookup returns both table ID and storage namespace ID. + Rename may update live identity/epoch fields, but never the head key or file + locations. Removing an old namespace cannot delete a moved table's files. +- Keys retain the existing 1,024-byte bound and relative UTF-8 rules; HTTP + percent decoding occurs only at the transport boundary. Reserved `f/` paths + are issued only by native publication. Direct directory roots handle files + with many extents without an unbounded URL. Locator signatures retain their + verification keys for the file lifetime and never substitute for HTTP grants. +- List enumerates selected immutable public objects, including files awaiting + commit, not snapshot reachability. It excludes unfinished/losing/deleted + records and never emits an extra alias for a direct file. Return object keys, + not full URLs. Retain existing bounded pages, live forward traversal and + grant-bound continuation tokens; a refreshed credential starts a new scan. +- Preserve conditional HEAD/GET, single-range reads and create-only PUT. + Identical retry requires content identity, not merely equal length. Multipart + completion selects one ordered part set and replays after response loss; + abort cannot remove a published file. Explicit cleanup grants and retained + reference checks govern single/batch deletion. Snapshot expiration alone + does not delete physical bytes. Copy and bucket mutations remain unsupported. +- Existing table-bound grants survive rename/move until normal expiry or + revocation. New grants resolve the live name. Branches, tags, time travel, + rollback and compaction preserve existing file URLs and retained native + extents. Recreated names and reused physical offsets cannot revive old URLs. + +## Dependencies + +- Depends on the existing native Iceberg table, FileIO, chunk, and KV + authority designs listed in the working plan, plus a chunk read operation + that accepts a validated complete native locator without resolving a file + record through the Iceberg KV server. +- Changes direct-file deletion semantics explicitly: metadata/list visibility + ends at logical deletion, while retained bytes remain addressable until GC. + Existing logical-URL resolution may still reject its tombstone. Direct GET, + HEAD and Range GET must not use that resolver or add per-file state queries. + Grant signature/scope/expiry validation remains independent of file lookup. + The native read API must avoid chunk-KV location resolution on the direct + path; GC and storage reads must prevent reclaimed extents from aliasing new + data, including after restart and physical space reuse. +- Uses the standard Iceberg URI requirement and Arrow S3 behavior as + compatibility references, not as a requirement to introduce an external S3 + store: [Iceberg specification](https://iceberg.apache.org/spec/), + [Arrow S3 filesystem](https://raw.githubusercontent.com/apache/arrow/apache-arrow-25.0.0/cpp/src/arrow/filesystem/s3fs.cc). + +## Acceptance + +- Given a table created in the default namespace and a named namespace, when + their table heads and file indexes are inspected, assert every table-owned + key starts with the catalog, storage namespace, and table IDs in that order. + Invariant: ICEBERG-PATH-1. Unit test. +- Given a table URL such as + `s3://iceberg/cat/ns/table/f/file.parquet`, when Arrow parses it, assert the + bucket is `iceberg` and the object key starts with `cat/ns/table/`. Assert a + table ID is never parsed as the bucket. Invariants: ICEBERG-PATH-2 and + ICEBERG-PATH-3. Integration test. +- Given a file published before a same-namespace rename or cross-namespace + move, when the file is loaded through the new table name, assert its single + `file_path` and native chunk ownership are unchanged. Invariants: + ICEBERG-PATH-1 and ICEBERG-PATH-3. E2E test. +- Given a published parquet record, when a client performs HEAD, full GET, and + footer Range GET using only `file_path`, assert the server can serve the + request without requiring a second public location field and routes bytes to + native chunk storage. Invariants: ICEBERG-PATH-3 and ICEBERG-PATH-6. + Integration test. +- Given a third-party SDK PUT to its locally chosen URL and a native injection, + when both files are published, assert the SDK file keeps its original URL + while injection returns one direct URL before manifest construction. Neither + requires a second field or subsequent manifest rewrite. Invariants: + ICEBERG-PATH-7 and ICEBERG-PATH-9. Integration test. +- Given a direct locator token, when its version, table ID, file ID, length, + or chunk-root binding is changed, or its HTTP grant expires, assert the + request is rejected before chunk access. Given a valid token, assert direct + reads perform neither file-location nor file-lifecycle KV queries. + Invariants: ICEBERG-PATH-8 and ICEBERG-PATH-6. Integration test. +- Given a file key and its native file index record, when metadata and recovery + inspect the record, assert its key suffix is the public filename and its + value contains length, integrity and native locator metadata, with no + additional path/public/id file records. Invariant: + ICEBERG-PATH-4. Unit test. +- Given an authorized table list with recursive and non-recursive selectors, + when the server receives ListObjectsV2 requests, assert the prefix is the + complete table prefix, delimiter and pagination are honored, and public + object keys are returned. Invariant: ICEBERG-PATH-5. Integration test. +- Given an empty, catalog-only, namespace-only, foreign-table, or bucket-wide + list prefix, when a caller requests ListObjectsV2, assert the request is + rejected without revealing keys from another table. Invariant: + ICEBERG-PATH-5. E2E test. +- Given an exact missing file request, when Arrow performs HEAD followed by its + directory fallback list, assert only `/` is considered and no + broader prefix is queried. Invariant: ICEBERG-PATH-5. Integration test. +- Given a logical file delete followed by delayed physical GC, when the old URL + is read with a valid grant, assert list excludes the file while its direct URL + still reads retained bytes. After GC, the URL fails and cannot read replacement + bytes at a reused extent. Retained locators never change. Invariant: + ICEBERG-PATH-6. E2E test. +- Given a dropped table and a recreated table with the same name, when both + paths are inspected, assert the recreated table receives a new table ID and + cannot read the old table's records or chunks. Invariants: ICEBERG-PATH-1 and + ICEBERG-PATH-6. E2E test. +- Given a v1 catalog, when v2 layout activation is requested, assert activation + is rejected while legacy reads and writes remain available. Given a new v2 + catalog, assert v1 keys cannot decode as v2 and scans never merge layouts. + Invariant: ICEBERG-PATH-10. Integration test. +- Given HeadBucket, ListBuckets, CreateBucket, DeleteBucket, and a list without + a complete table prefix, assert the fixed virtual bucket returns only the + documented compatibility response and never reveals another table's keys. + Invariant: ICEBERG-PATH-11. Integration test. +- Given logical names and public keys with different ordering, when paginating + a table list with delimiters, assert ordered public keys, no aliases and + correct continuation through empty pages. Invariants: ICEBERG-PATH-4 and + ICEBERG-PATH-5. Integration test. +- Given a crash before or after the single file-record CAS, when recovery retries, + assert only selected records become visible and GC preserves the selected + bytes. Invariants: ICEBERG-PATH-4 and ICEBERG-PATH-6. Integration test. +- Given two injections with the same readable name and a retry of one upload, + when publication finishes, assert distinct files have distinct locator-bearing + filenames, the retry returns its original filename, and a conflicting exact + key never overwrites the winner. Invariant: ICEBERG-PATH-4. Integration test. +- Given a retained snapshot, new HTTP credentials and rotated signing keys, + when reading its direct URL, assert the URL is unchanged and still readable; + an expired grant remains rejected. Invariant: ICEBERG-PATH-8. Integration test. +- Given a moved table and its old namespace, when that namespace is dropped + and recreated, assert live lookup still returns the frozen storage path and + the files survive. Invariant: ICEBERG-PATH-1. E2E test. +- Given files with multiple extents, when reading footer and boundary ranges, + assert correct bytes, bounded URLs and zero file-location, deletion-state + and chunk-KV resolution RPCs on the direct path. Invariants: ICEBERG-PATH-3 and + ICEBERG-PATH-8. Integration test. +- Given concurrent multipart completion/abort, same-length conflicting PUTs + and conditional/range reads, when requests retry after response loss, assert + immutable selected bytes, one completion result and documented HTTP errors. + Invariants: ICEBERG-PATH-6 and ICEBERG-PATH-7. Integration test. +- Given branches, tags, position deletes and historical snapshots, when moving, + compacting and rolling back the table, assert original paths and delete + applicability remain intact. Invariants: ICEBERG-PATH-1 and ICEBERG-PATH-9. + E2E test. + + +- Given metadata exceeding the inline threshold with many snapshots, when + publishing and reloading it through ALL/REFS, assert chunk-backed canonical + bytes, bounded head/file values and identical required table semantics with + projections present, missing or corrupt. Invariants: ICEBERG-PATH-4 and + ICEBERG-PATH-9; permanent ICE-I2 and ICE-I13. Integration test. +- Given many metadata/manifest files mixed with data files, when listing the + table, metadata directory and a filename prefix over multiple pages, assert + correct public keys, bounded per-page work, no payload reads and no scans + outside the chosen prefix. Invariant: ICEBERG-PATH-5. Integration test. +- Given v1/v2/v3 fixtures and two concurrent commits, when clients reload the + selected head and historical snapshots, assert the standard reference graph, + version-specific fields and a single complete winning generation; losing + candidates remain invisible to table load. Invariants: ICEBERG-PATH-9 and + permanent ICE-I3/ICE-I7. E2E test. + +Required implementation gates (documentation editing does not run these): + +- `pixi run -- cargo test -p crowdb-access-iceberg --all-targets` +- `pixi run -- cargo test -p crowdb-access-server --all-targets` +- `pixi run -- cargo test -p crowdb-chunk-client --all-targets` +- `pixi run -- cargo clippy -p crowdb-access-iceberg -p crowdb-access-server -p crowdb-chunk-client --all-targets -- -D warnings` +- `pixi run -- cargo fmt --all -- --check` +- `pixi run rs-lint` diff --git a/doc/backlog/backlog.md b/doc/backlog/backlog.md index ebfe9e860..6e487f84e 100644 --- a/doc/backlog/backlog.md +++ b/doc/backlog/backlog.md @@ -9,7 +9,7 @@ requirements and their temporary execution plans are removed during the final live in the linked document; implementation follows [`implement-requirement`](../../.agents/skills/implement-requirement/SKILL.md). -**Next R number: R231** — R221 is reserved by another workstream. +**Next R number: R235** — R231 through R233 are reserved by other workstreams. ## Dataset and access @@ -41,6 +41,12 @@ live in the linked document; implementation follows - **[R196](R196-access-upload-benchmark-regression.md)** — S3 and Iceberg HTTP upload benchmark regression. **Area:** access server / CLI / benchmarks. **Complexity:** Medium. **Dependencies:** completed upload correctness paths. +- **[R234](R234-access-iceberg-native-storage-layout.md)** — native Iceberg + ID directory, single file record, injection direct URL, large metadata objects + and bounded table/metadata prefix listing. + **Area:** access server / Iceberg / Chunk-KV. **Complexity:** High. + **Status:** Implementation started; native identity/URI/key codecs and storage + envelope dispatch verified. Runtime publication and direct reads pending. ## Chunk, Chunk-KV, and recovery diff --git a/doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md b/doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md index db86e40f3..db0c3bc05 100644 --- a/doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md +++ b/doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md @@ -3,7 +3,7 @@ # CROWDB - Design: Iceberg File Upload Flow -This document defines the FileIO upload path used by Iceberg clients and records +This document defines native injection and the FileIO upload path used by Iceberg clients and records measurements from the single-node profile. It covers file publication, not table snapshot publication. @@ -19,6 +19,38 @@ Depends on: [Native Iceberg Storage](design-crowdb-iceberg.md) and ## 1. Client and server flow +### Native injection + +Native injection owns both byte ingestion and the subsequent construction of +Iceberg file references: + +1. Finish durable IcebergTable chunk writes and seal a complete native + descriptor or bounded chunk-resident directory root. +2. Form `f/v1/..parquet` under the immutable + `///` prefix. Bind the locator's + unique file identity, length, descriptor and exact filename by signature. +3. Publish one `file/` record with create-only CAS. Return the one + `s3://iceberg//` URL after publication. A retry uses + the same operation/file identity and returns the selected URL; conflicts + never overwrite published content. +4. Construct standard manifests using that URL, write manifest lists and + metadata JSON, then publish the table head by its normal atomic commit. + Failed table commits leave unreferenced durable files for the existing + safe cleanup flow; they do not change a previously returned URL. + +Direct HEAD/GET/range requests validate cached permissions and the locator, +then read native storage without file-record or chunk-KV location lookup. +List and cleanup still use the single file record. Logical deletion removes +the file from list; its known direct URL may read retained bytes until physical +GC. Reclaimed identities cannot alias newly allocated bytes. + +### Standard SDK uploads + +SDK location providers choose logical filenames before PUT. These uploads +retain that filename and resolve the single file record for reads. They do +not require a pre-upload reservation API, a second SDK location field, or +post-upload rewriting of manifests and path-based delete references. + The TPC loader hashes each local Parquet file through bounded reads, checks that its unique target is absent, then sends a known-size S3 request using the vended native credentials. Below 256 MiB it uses a single PUT; larger files @@ -121,6 +153,13 @@ also counts repair-driven rotations, replayed bytes, and rotation time. ## 3. Integrity and durable publication +Large metadata JSON, manifest lists and manifests follow the same native +object durability rules. Each keeps one bounded file descriptor; complete +snapshot arrays and file bytes are not copied into TableHead or an unbounded +KV value. Public `metadata/` and `statistics/` prefixes remain available for +bounded listing without opening object payloads. Small inline metadata and +large chunk-backed metadata return identical canonical FileIO bytes. + The per-object OpenSSL digest state computes MD5 over ordered logical payload views. Small objects update it synchronously in the receive task; large objects use a checksum worker. It also computes SHA-256 when a signed payload requires it. It never diff --git a/doc/design/access-server/iceberge/design-crowdb-iceberg.md b/doc/design/access-server/iceberge/design-crowdb-iceberg.md index 945a47feb..352a2ae2f 100644 --- a/doc/design/access-server/iceberge/design-crowdb-iceberg.md +++ b/doc/design/access-server/iceberge/design-crowdb-iceberg.md @@ -62,6 +62,82 @@ head publication. No cross-generation cache or deduplication is implied. Iceberg metadata stores bounded logical records and opaque data references. Physical chunk placement and storage topology remain below the access boundary. +### Identity directory and file authority + +The Iceberg key codec carries its storage family and layout version separately +from the directory payload. A table-owned payload begins with +`///`; it has no redundant +`iceberg`, `namespace` or `table` labels before those IDs. The storage namespace +is frozen at table creation, including staged creation. The default namespace +has a durable ID under the same rules as named namespaces. + +```text +/ +├── head +├── namespace/name/ -> namespace-id +└── / + ├── head + ├── name/ -> (storage-namespace-id, table-id) + └── / + ├── head + ├── file/ + ├── projection// + ├── operation//... + ├── multipart//... + └── gc//... +``` + +Catalog and namespace control records have their own typed scopes. Namespace +names use the existing unambiguous multipart-name encoding. Table-name lookup +returns the complete storage identity; internal tasks carry it as well, so no +table-ID reverse index is needed. A move updates live name mappings and the +head's live identity/epoch while retaining the storage path. Deleting the old +namespace cannot cascade into moved tables, and IDs are never reassigned. + +Each file has one `file/` record. The suffix is also +its public object-key suffix. Its value contains FileId, canonical length, +format, integrity metadata, native descriptor/root and lifecycle state. FileId +is a value/locator binding, not another file index. One create-only CAS publishes +the complete record after durable bytes; exact-key conflicts cannot overwrite +selected content. Retries preserve the selected file identity and URL. Workflow +records exist only where recovery or multipart/GC progress requires them, not +as three mandatory records for every file. + +### Standard metadata and bounded scale + +TableHead selects an immutable standard metadata JSON file. Its `snapshots` +array contains snapshot records and its `refs` map selects branches/tags; +snapshot-log and metadata-log retain their standard meanings. Snapshot records +reference manifest-list files, which reference manifest files, which describe +data/delete files. Reading v1 also supports its direct `manifests` alternative. +The backed-up v1/v2/v3 specification determines fields, sequence inheritance, +schema/partition evolution, delete matching and row lineage. No authoritative +snapshot or manifest KV index substitutes for those standard file contents. + +All of these file kinds use the same single file-record model. Large metadata +JSON, manifests and manifest lists are native chunk-backed objects. KV values +and TableHead never embed an unbounded metadata document or snapshot array. +Small eligible metadata may use the bounded inline representation; larger +objects and large extent sets use chunk payloads and bounded directory roots. +Changing storage representation preserves the canonical bytes returned by +FileIO. Table publication selects a completely durable metadata generation by +CAS; a losing commit cannot expose a partially written generation. + +Metadata files use useful public relative prefixes such as +`metadata/.metadata.json`, `metadata/.avro` and +`statistics/.puffin`. These are defaults, not restrictions on safe +client-chosen paths. The optional generation-bound projections described above +accelerate metadata reads; they are not duplicate file authorities. Parser, +collection and response budgets remain explicit even for chunk-backed JSON. +An unsupported oversize request fails clearly rather than silently truncating +snapshots. ALL and REFS keep their standard response semantics; internal +projection paging does not introduce private REST pagination fields. + +New catalogs use the identity-directory layout version. Existing legacy +catalogs retain their own decoder, keys, URLs and read/write behavior. Layout +selection is catalog-bound; relabeling or mixing key versions in one scan is +rejected. In-place migration is not implicit in decoding. + ### Catalog foundation One active root selects a random stable CatalogId and activation epoch. Display @@ -254,20 +330,32 @@ Iceberg FileIO uses reserved S3-shaped locations so existing Iceberg clients can address immutable metadata and data files. The shape is a compatibility contract, not delegation to the general S3 authority. File publication, immutability, authorization, and deletion remain under Iceberg control. +Public URIs use the fixed virtual bucket `iceberg`: +`s3://iceberg////`. Typed locations use lower-case unpadded base32 catalog IDs and lower-case hex -table IDs. Relative UTF-8 object keys preserve case, literal percent signs, plus +namespace and table IDs. Relative UTF-8 object keys preserve case, literal percent signs, plus signs and repeated internal slashes; the whole object key is bounded to 1,024 bytes. Dot traversal, leading slash, backslash, controls, query and fragment delimiters are rejected rather than normalized. HTTP percent decoding belongs only at the transport boundary, not in stored S3-shaped locations. Native `ListObjectsV2` retains this address model. A list request must specify -an explicit `t//` prefix and a table-bound delegated list capability; +the complete `///` prefix or a +descendant prefix and a table-bound delegated list capability; catalog-wide discovery and general S3 credentials are rejected. Reader and writer FileIO credentials include listing; cleanup-only credentials do not. Exact-object HEAD/GET/PUT remain independently usable. PyArrow missing-file probes may issue listing to distinguish a missing object from a directory. +Empty, catalog-only, namespace-only and foreign-table prefixes fail before a +scan. HeadBucket/GetBucketLocation expose only the configured virtual bucket +and region to a valid grant; ListBuckets returns AccessDenied, and bucket +creation/deletion is unsupported. The URI host determines the bucket. Arrow +FileSelector adds a trailing slash to its key prefix and uses `/` as delimiter +for nonrecursive listing; an exact HEAD miss may probe `/` without +widening scope. These behaviors follow the +[Arrow S3 implementation](https://raw.githubusercontent.com/apache/arrow/apache-arrow-25.0.0/cpp/src/arrow/filesystem/s3fs.cc). + Listing enumerates selected, fully published immutable file-location records, including files awaiting a table commit or later reclamation. It excludes incomplete uploads, unselected draft/losing candidates and logical deletion @@ -290,6 +378,15 @@ catalog epoch, table, prefix, delimiter and encoding. Their fixed expiry is no later than the originating grant, and invalid or foreign tokens fail before storage access. A credential refresh requires starting a new traversal. +List operates directly on `file/` records in public-key order. For example, +the public prefix `/metadata/` maps to the KV prefix +`/file/metadata/`; filename prefixes narrow that same range. +Pagination uses encoded prefix bounds and the last examined key, including +filtered tombstones and sparse pages. It never materializes or sorts the whole +table, opens metadata objects, scans unrelated file prefixes or enumerates +snapshots inside JSON. Returned keys omit the internal `file/` component. +The same limits apply whether a table has ten or millions of metadata files. + Native file records bind FileId to exact location, kind, format and canonical length. Legacy tree and inline records also bind a whole-file SHA-256 digest; streamed records bind a bounded array of complete Chunk locations and an HTTP ETag, and @@ -315,6 +412,45 @@ The isolated native HTTP surface exposes signed immutable object reads/writes, multipart operations and separately authorized single/batch cleanup, without a general S3 bucket authority. +### Native injection and direct reads + +Native injection finishes chunk writes and its bounded descriptor, publishes +one file record, then returns one canonical URL before constructing manifests. +For parquet, the public filename is +`f/v1/..parquet`. The URL-safe, dot-free locator carries +a unique FileId, length, integrity information and complete stable native +location or chunk-resident directory root. Its versioned signature binds the +full table identity, readable stem and unsigned locator fields; the signature +bytes are excluded from that signing input. Repeated readable stems are harmless; +an identical upload retry returns the original selected filename. Locator and +stem together must fit the object-key limit. + +Ordinary SDK writers choose logical filenames before S3 PUT and keep those +URLs. A PUT-time allocation does not change a client's chosen path. These +files resolve their one file record; no response field or manifest rewrite +converts them into direct locators. Metadata/Avro files may retain useful +logical prefixes regardless of their native object size. Position-delete and +deletion-vector path references always use the original canonical URL. + +Direct HEAD/GET/Range GET validate request permissions and locator authenticity +and read the native descriptor without file-location, file-tombstone or +chunk-KV resolution queries. A KV-resolved opaque token is not a direct +locator. Large files use bounded chunk-resident directory traversal, not an +unbounded URL. Native chunk routing preserves repair/failover without embedding +transient replica addresses in URLs. Reclaimed identities and reused extents +must never make an old locator return a different file's bytes. + +Authorization uses bounded validated grant/context caches and signature, +scope and expiry checks. An authorization cache miss may load authority; it +must not become a per-file location/state lookup. Expired entries are not +accepted during refresh failure. Cache entries bind principal, catalog epoch, +full storage identity and operation rights, with validity bounded by the grant +and applicable authority lease. Fresh credential issuance still checks live +authority. Persistent locators have no credential expiry; retain signing-key +verification material for their files' lifetime. A locator is not a bearer +permission. Cold/warm authorization costs and storage resolution counts are +measured separately. + Legacy chunk-backed files use bounded leaf blocks and immutable chunk-resident directory pages, with at most 256 children per page and eight directory levels. Each page binds its catalog, table and file identity, child heights and covered byte count. @@ -503,7 +639,9 @@ Delegation tokens carry catalog activation epoch, table, principal fingerprint, nonce, exact operation set, issue/expiry times and independent request/file byte limits. Domain-separated HMAC authenticates bounded claims and derives per-grant S3 credential material without a mutable credential registry. Verification requires -a freshly checked Ready context. Ordinary FileIO grants exclude DELETE; +the operation's authorized context: writes and credential issuance check Ready +authority, while direct reads may reuse the bounded validated permission cache +described above. Ordinary FileIO grants exclude DELETE; explicit cleanup grants include the separate delete operation. These token primitives feed native request-signature verification through a request-local credential provider. Only the shared SigV4 algorithm is reused; general S3 @@ -513,7 +651,7 @@ expiry remains exact even when signature timestamps allow clock skew. Table credential issuance requires a matching Ready catalog authority and rejects lifetimes above its persisted delegated-access bound, independently of the signer's configured maximum. A zero persisted delegation bound disables issuance. -Callers still must freshly authorize the root and exact live table or draft; +Credential-issuance callers must freshly authorize the root and exact live table or draft; the serialization primitive does not perform those reads. The credential endpoint checks the current namespace and published table or exact unbound draft. Draft vending requires its original writer principal and a table-ID query selector; @@ -653,20 +791,24 @@ authorizes deletion. Committed files remain readable when new chunk allocation fails. Progress resumes after capacity is restored through the normal storage flow. Shared-chunk ranges remain pending while range deletion is unsupported. -Request entry checks catalog and table or draft authority without request-level -pins. Selected context and records flow through normal publication and reads; -PUT and GET do not perform deletion-fence or GC-claim GETs. Physical reclamation observes a -minimum retention interval that covers admitted request lifetimes. Once a file's canonical -logical deletion has started, its location-key tombstone rejects resolution -until cleanup state permits reuse, even if physical range reclamation is deferred. Legacy live tasks are retired without +Publication and logical-URL resolution check catalog and table or draft +authority without request-level pins. Direct reads instead use the validated +permission cache and authenticated locator. Physical reclamation observes the +configured retention interval. Logical deletion hides the file from listing; +its tombstone rejects logical-URL resolution but is not consulted by direct +HEAD/GET/Range GET. With a valid grant, an already known direct URL can still +read retained bytes until physical reclamation. Logical deletion is not an +instantaneous direct-URL revocation guarantee. Legacy live tasks are retired without further deletion, releasing an owned table fence. Retained and deferred candidates remain durable work for later inactive passes. An already authorized FileIO GET or HEAD can continue within the configured -retention interval after a logical deletion. A concurrent transition to -`Reclaiming` rejects new admission. Uploads and new table credentials require -a Ready table. Logical drop preserves admitted file reads until their bounded -response lifetime ends. +retention interval after logical deletion. Native storage arbitrates direct +read/reclamation races, returning the original bytes or a storage error, never +bytes from a reused allocation. Logical-URL admission rejects a reclaiming +record. Uploads and new credentials require a Ready table. After logical drop, +direct URLs remain readable under still-valid grants while bytes are retained; +physical purge and grant expiry bound that access. Retired catalog recovery scans system retry and management ledgers before file deletion and after the final file rescan. Pending or retained bindings stop the @@ -788,9 +930,11 @@ The native file endpoint exposes separately authorized S3 `DeleteObject` and credentials endpoint with `cleanup=true`; ordinary FileIO credentials cannot delete objects. The server scans retained table metadata and refuses a path that is referenced or whose reference status is uncertain. Successful deletion -records a logical tombstone, then queues bounded physical reclamation after -20 minutes. A new PUT may reuse the path only after the old file's GC candidate -is durable. Batch deletion returns individual errors inside HTTP 200, so callers +records a logical tombstone and removes list visibility, then queues bounded +physical reclamation after 20 minutes. Direct URLs remain readable until their +bytes are reclaimed, subject to grant validity. Their FileIds/locators are never +reused. A new logical-path PUT may reuse its path only after the old file's GC +candidate is durable. Batch deletion returns individual errors inside HTTP 200, so callers must inspect every item. > **Warning:** Before deleting, the client must stop and resolve every writer @@ -821,3 +965,13 @@ must inspect every item. reclaimed solely because no retained snapshot refers to it; active cleanup handles expired multipart remnants and explicitly deleted files with durable owner checks. +- **ICE-I10 — Stable identity directory:** rename and cross-namespace move + preserve storage namespace, table path and every published file URL. +- **ICE-I11 — Single file authority:** one filename-keyed record selects each + published file; list and lifecycle use that record without secondary file indexes. +- **ICE-I12 — Direct native read:** a direct URL supplies its immutable native + descriptor without file-location, file-state or chunk-KV resolution queries. + Logical deletion removes list visibility; physical GC ends byte availability. +- **ICE-I13 — Bounded metadata scale:** large metadata bytes live in native + objects, descriptors/head remain bounded, prefix listing reads bounded KV + pages, and optional projections cannot replace standard metadata authority. diff --git a/doc/doc_index.md b/doc/doc_index.md index a99bfec07..23029f563 100644 --- a/doc/doc_index.md +++ b/doc/doc_index.md @@ -42,7 +42,7 @@ Temporary plans live under `doc/working/`; flow analyses live under | `doc/design/kv/kv-write-flow-analysis.md` | KV write flow and optimization evidence. | | `doc/design/chunkio/chunkio-write-flow-analysis.md` | Chunk I/O large-write flow and benchmarks. | | `doc/design/chunkio/chunkio-small-io-flow-analysis.md` | Chunk I/O small-I/O flow and benchmarks. | -| `doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md` | Iceberg FileIO upload path and container measurements. | +| `doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md` | Native injection, SDK upload and container measurements. | | `doc/design/rpc/rpc-flow-analysis.md` | RPC flow, benchmarks, and performance history. | ## Dev Environment (`doc/dev/`) @@ -119,7 +119,7 @@ Temporary plans live under `doc/working/`; flow analyses live under | `doc/design/rpc/design-crowdb-rpc-diskdb-migration.md` | Diskdb RPC migration, rollout, connection lifetime. | | `doc/design/access-server/s3/design-crowdb-access-s3.md` | S3 namespace, publication, integrity, list, delete. | | `doc/design/access-server/s3/design-crowdb-access-s3-rdma.md` | Optional RDMA and direct-to-GPU data planes. | -| `doc/design/access-server/iceberge/design-crowdb-iceberg.md` | Native Iceberg authority, commits, files, and recovery. | +| `doc/design/access-server/iceberge/design-crowdb-iceberg.md` | Iceberg ID paths, direct files and metadata scale. | | `doc/design/access-server/iceberge/iceberg-table-spec-1.11.0.md` | Backed-up normative Iceberg table format specification. | | `doc/design/access-server/iceberge/iceberg-rest-catalog-open-api-1.11.0.yaml` | Backed-up normative Iceberg REST Catalog OpenAPI. | | `doc/design/access-server/dataset/design-crowdb-access-dataset.md` | Dataset HTTP, native topology access, and GPU delivery. | diff --git a/doc/working/plan-iceberg-storage-path.md b/doc/working/plan-iceberg-storage-path.md new file mode 100644 index 000000000..f1b0c08f0 --- /dev/null +++ b/doc/working/plan-iceberg-storage-path.md @@ -0,0 +1,744 @@ + + + +# Iceberg Storage Path Plan + +Requirement: [R234](../backlog/R234-access-iceberg-native-storage-layout.md) + +Upstream references: + +- [Iceberg table specification](../design/access-server/iceberge/iceberg-table-spec-1.11.0.md) +- [CROWDB Iceberg design](../design/access-server/iceberge/design-crowdb-iceberg.md) +- [CROWDB Iceberg upload flow](../design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md) + +Goal: make catalog, namespace, and table identities explicit in the Iceberg +KV and FileIO path model, keep file bytes in the native CROWDB Iceberg storage, +and preserve file locations across table rename and cross-namespace move. + +Status: implementation started. Immutable storage identity, native URI codec and +version-isolated table-key codec and routed-store dispatch are implemented; runtime publication and direct +reads are not switched to the new layout yet. Verification is tracked below. + +Verified foundation gates: + +- `pixi run -- cargo test -p crowdb-access-iceberg --all-targets` passed, + including seven native storage-path cases and existing legacy/lifecycle tests. +- `pixi run -- cargo fmt --all -- --check` passed. +- `pixi run rs-lint` passed, including workspace all-target Clippy. + +## Target structure + +Iceberg storage owns the logical directory. Native chunk placement remains +opaque and is still selected by the Iceberg chunk type, allocator, and +`FileBlockStore`; the directory below is the authority path and the stable +name used to resolve a file. The namespace component is the storage namespace +ID captured when the table is created: + +```text +/// +├── metadata/ +│ ├── .metadata.json +│ ├── .avro +│ └── .avro +├── data/ +│ ├── /data-.parquet +│ ├── /delete-.parquet +│ └── /delete-vector-.puffin +└── statistics/ + └── .puffin +``` + +`metadata/`, `data/`, and `statistics/` are conventional defaults, not a new +Iceberg semantic requirement. The server must accept any safe relative key +below the immutable table prefix so ordinary Iceberg clients can choose their +own file names and subdirectories. Every metadata, manifest, data, delete, +Puffin, and statistics reference generated by CROWDB must remain under this +prefix. Opaque client manifests are not recursively validated by normal commit; +FileIO enforces scope whenever an actual reference is accessed. Importing +external files requires explicit native ingestion, not an external authority. + +This is the logical Iceberg file namespace. The single public `file_path` may +render a native locator such as `f/.parquet` under the same +table prefix. The readable stem may be included in the public filename itself; +there is no second internal filename index or second public location field. + +Native injection returns a versioned locator file component after bytes and +their descriptor are durable, before creating any referencing manifest: +`s3://iceberg////f/v1/..parquet`. +Ordinary SDK writes retain their chosen `data/...` or other safe relative key; +their location provider runs in the client and cannot be assumed to call a +CROWDB reservation API. Both forms have exactly one public URL per file. The +fixed `iceberg` compatibility bucket is always the URI bucket; the +object key carries the three ID segments. A table ID is never promoted to a +bucket. The `s3://` wrapper exists for SDK compatibility; the object is still +read from native Iceberg authority and chunk storage. + +The KV layout uses the same identity tuple as a slash-separated logical path; +the serialized key must still version and validate each fixed ID segment: + +```text +/ +├── head +├── namespace/name/ -> namespace-id +└── / + ├── head + ├── name/ -> (storage-namespace-id, table-id) + └── / + ├── head + ├── file/ + ├── projection// + ├── operation/ + ├── multipart/ + └── gc/ +``` + +The namespace ID in a table's directory is always its storage namespace ID. +The name mapping may live in a different namespace after a move. There is no +table-ID reverse index: callers and recovery tasks carry the full identity. +`projection/` represents the existing optional, rebuildable metadata read +acceleration, not another file record or authoritative snapshot store. + +The first three path segments are parsed by fixed identity encodings and +positions. The canonical encodings remain lower-case unpadded base32 for the +catalog ID and lower-case fixed-width hexadecimal for namespace and table IDs; +labels such as `namespace` and `table` are rejected rather than accepted as +aliases. + +The displayed paths are inside Iceberg storage. Neither the bucket name nor +`iceberg/` is repeated in the KV payload; a binary storage-family/version header +may precede it. Catalog/namespace control indexes are not table-owned records +and therefore use their own typed scopes. In a table scope the payload is +exactly the three IDs followed by a typed suffix. Name indexes include the +storage namespace so a moved table can be found without a catalog-wide scan. + +`file/` is the only persistent file record. It holds +the complete selected descriptor and provides logical GET resolution, ordered +S3 listing, inspection and GC. Publish it with one create-only CAS after bytes +are durable. There are no separate path/public/id indexes. Lifecycle tasks +carry the canonical filename; file ID remains a value field and locator binding. +Multipart/operation/GC state is temporary workflow state, not another file index. + +The native filename is `f/v1/..parquet`; use the last dot +before the extension to delimit the URL-safe, dot-free locator. Authenticate +the exact filename as well as the native descriptor to prevent alternate-name +aliases. The locator contains a unique file ID, so identical readable stems +remain distinct. Retry the same injection operation with the original file ID +and return the selected URL; a conflicting exact key fails rather than +overwriting. Ordinary SDK filenames retain their exact chosen key and the same +create-only collision rule. Include the readable stem in the total key limit. + +Rename/move changes name mappings and the head's live name/epoch fields using +the existing lifecycle state machine. The head's key, storage identity, +metadata generation, file URLs and chunk ownership remain unchanged. Dropping +the old namespace cannot cascade into files of a moved table; namespace ID +reuse is forbidden and cleanup follows table ownership rather than names. + +## Standard metadata, large objects and prefix listing + +The supported format profile remains the repository's pinned Iceberg v1/v2/v3 +specification. The live upstream spec may describe newer versions; this layout +change does not advertise those versions. References: +[table metadata](https://iceberg.apache.org/spec/#table-metadata), +[snapshots](https://iceberg.apache.org/spec/#snapshots), and +[manifests](https://iceberg.apache.org/spec/#manifests). + +```text +table/head -> current metadata JSON file + snapshots[] -> manifest-list -> manifests -> data/delete files + refs, snapshot-log, metadata-log, schemas, partition specs +``` + +- Snapshots and refs retain their standard JSON representation. They do not + require individual authoritative KV keys. Preserve the v1 direct `manifests` + alternative when reading old snapshots; v2/v3 use `manifest-list`. +- Metadata JSON, manifest lists, manifests, data/delete and statistics files + all use the one `file/` record model. File kind + does not introduce another index. Standard metadata/Avro fields, sequence + inheritance, schemas and exact path-based delete references remain intact. +- A large metadata JSON is a native object whose bytes are written into + IcebergTable chunks. Its KV value contains a bounded native descriptor/root, + canonical length and integrity metadata, not the complete JSON or snapshots + array. Preserve the existing small-file inline threshold; crossing it selects + native chunk storage without changing FileIO behavior. Large extent vectors + likewise use a bounded chunk directory rather than growing the KV value. +- `head` stays bounded: selected metadata pointer, generation and existing + identity/commit state. Publish immutable files durably before the head CAS. + Concurrent commits still validate their base; no head can select partial + bytes. A commit retry may create a new manifest list/metadata generation; + it does not overwrite already published files to retain the old filename. +- CROWDB-generated metadata names preserve useful public prefixes: + `metadata/.metadata.json`, `metadata/.avro` and + `statistics/.puffin`. Existing clients may choose other safe paths. + Native parquet injection retains the direct `f/v1/...` filename. Metadata + objects may resolve their single file record; direct parquet reads remain + independent of metadata/list queries. Large metadata storage does not force + all objects into the direct parquet naming convention. +- Table listing maps the public prefix `/metadata/` to the bounded + KV prefix `/file/metadata/`. A more specific filename prefix maps + identically, without scanning `data/`, `f/`, statistics, operations or GC. + Full table list scans only `/file/`. Prefix upper bounds and + continuation cursors are constructed from the canonical encoded key. +- Apply the existing list page record/byte/output limits to metadata listing + as well. Build each page incrementally; never collect or sort the full file + set. Delimiter rollups and tombstone filtering advance a bounded cursor, and + return a continuation when the scan budget ends even if the page is sparse. + List reads file descriptors only, never opens metadata JSON or manifests. +- Object listing and snapshot enumeration are different operations. A prefix + list shows retained metadata files, not the snapshots inside one JSON file; + no `snapshot/` objects or private SDK pagination fields are synthesized. + Use the existing generation-bound validated projections for REFS loads; + preserve ALL response semantics. Projections remain optional, paged and + discardable, and cannot participate in commit authority or GC reachability + proof. Missing/corrupt projections use the bounded canonical parser. +- Chunk-backed storage alone does not make parsing unbounded metadata safe. + Keep explicit metadata-byte, collection, parser-memory and response budgets; + reject oversized unsupported requests clearly without truncating valid + snapshots or refs. Reuse the existing streaming/bounded read paths where + available; do not claim that the existing commit parser is fully streaming. +- Physical GC follows retained snapshot/branch/tag and metadata retention + rules, not filename order, list age or absence from the current snapshot. + Snapshot expiration is a metadata update; files shared by another retained + snapshot remain readable. Historical JSON files follow their own retention + policy rather than being immediately deleted on head replacement. + +## File URL and native read boundary + +The Iceberg `file_path` remains a stable URL. Published CROWDB chunk +locators (`ChunkId` plus offset and logical span) are stable while the file is +retained; they become invalid only after the file is deleted and its physical +storage is reclaimed. This makes a native read locator a viable optimization, +but it still cannot be assumed to be one scalar location: a streamed file can +have multiple locations and a tree-backed file can require its directory root. + +The preferred read fast path is therefore: + +```text +file_path URL -> immutable file authority or native read locator + -> ChunkRoot/Locations -> ChunkIoClient range stream + -> native chunk storage +``` + +There is only one public URL: the URL stored as Iceberg `file_path`. If +ordinary SDK GET and range GET are to bypass the file-location lookup, this URL +must be a native read locator, for example a compact signed file-root/location +token under the table prefix. The server validates the catalog/table grant, +file length, integrity metadata and locator binding before +reading. Large location vectors use a bounded chunk-resident directory root; +a token requiring a KV mapping would reintroduce the location lookup. + +An SDK chooses its path before PUT. Reserving a target when PUT arrives cannot +put the newly allocated location into the already chosen URL. Such files retain +their logical URL and use `record_resolution`. Native injection owns the upload +and manifest-building sequence: finalize bytes and descriptor, publish the +file, return one direct URL, then build manifests with that URL. An internal +operation ID supports retry without publishing another file. This boundary is +the agreed native injection boundary; ordinary SDK writers keep logical URLs. + +The direct-read locator is versioned and authenticated. It binds catalog, +storage namespace, table, file ID, length, integrity metadata, and a compact +chunk root or complete location descriptor. It has a bounded URI +size and carries no arbitrary client-controlled chunk ID. A valid direct read +calls the chunk API with the complete validated locator and skips the Iceberg +file repository; a fallback read is explicit in the file record and is never +presented as direct. Persistent locators have no credential expiry or credential +nonce: refreshed HTTP grants must continue reading retained snapshots. Include +a signing-key version and retain verification keys while their files survive; +the locator signature grants no access by itself. Inline/compressed metadata +may use the logical path; direct mode requires random-readable native bytes. + +The current chunk reader calls `query_layout`/`query_chunk`; removing +`FileRepository::resolve` does not by itself remove chunk metadata RPCs. +Measure file-location resolution, file-state checks, chunk-layout resolution +and data reads separately, including cold cache. The first three must cause +zero KV RPCs on the direct path; adding a hidden lookup fails acceptance. +Stable chunk IDs/offsets do +not imply immutable replica addresses. A direct chunk API must preserve normal +repair/failover behavior and never embed transient host placement in file URLs. + +Logical deletion affects metadata/list and schedules physical reclamation; it +does not promise immediate revocation of an already known direct URL. A valid +grant and locator can read retained bytes until GC. GET/HEAD/Range GET perform +no file tombstone lookup, and need no per-file revocation cache. List, deletion, +GC and recovery may use KV normally. Physical storage must reject reclaimed +chunk identities/extents and prevent old locators reading newly allocated data; +check this at the storage boundary without a per-read chunk-KV query. + +The upload state machine is `Staging -> Published -> Deleted -> Reclaimed`, +or `Staging -> Aborted -> Reclaimed`. Incomplete or aborted records never +appear in ListObjectsV2. Complete PUT and multipart retries are idempotent; +conflicting content or overwrite attempts fail without changing a published +record. Equal length alone never establishes an identical retry. Logical +deletion removes list visibility; direct reads remain possible during retention. +GC/read concurrency follows native storage lifetime rules: return retained bytes +or a storage error, never bytes from a replacement allocation. + +The native metadata record uses the public relative filename as its map key +and stores file ID, length, integrity metadata, format, root and native +locations as its value. The public URL is derived from table identity and +filename. This preserves inspection, listing, GC, and +recovery clarity. A file's canonical URL and ETag never change during its +lifetime, including rename, credential rotation and snapshot rollback. + +Arrow/PyArrow derives the bucket from the URI host and the object key from the +URI path. For a `FileSelector`, Arrow removes a trailing slash from the base +directory and sends `ListObjectsV2` with `Prefix = key + "/"`; non-recursive +selectors also send `Delimiter = "/"`. An exact file operation normally +becomes HEAD/GET (and Range for reads); if HEAD misses, Arrow may test an +implicit directory with `Prefix = object-key + "/"`. The native FileIO surface +must therefore keep table-scoped listing, even though Iceberg manifests +normally provide explicit file paths and do not require a full table scan for +reads. + +The server enforces this scope instead of trusting the client: every list +request must name the complete `///` +prefix authorized for the request. Empty, catalog-only, namespace-only, +foreign-table, and bucket-wide prefixes are rejected; bucket-wide bucket +listing is not exposed. The missing-file fallback is allowed only for the +exact `/` child prefix and cannot widen the table grant. This is a +security and isolation rule, not an Arrow workaround. + +The list compatibility contract includes `prefix`, `delimiter`, `max-keys`, +`continuation-token`, `start-after`, `encoding-type=url`, common prefixes, +`IsTruncated`, and `NextContinuationToken`. Exact-object existence fallbacks +may list `/` to distinguish a missing object from an implicit +directory. Listing remains table-scoped and never grants catalog-wide +discovery. + +The fixed virtual bucket has narrow control-plane behavior. HeadBucket succeeds +only when the Iceberg endpoint and table grant are valid; the configured +endpoint supplies region information, so Arrow does not need AWS bucket +discovery. ListBuckets returns `403 AccessDenied`; authenticated GetBucketLocation +returns the configured region. CreateBucket and DeleteBucket are rejected. No operation creates a +bucket per table, and every object operation still requires the immutable +three-ID prefix and a matching grant. + +## Request and lifecycle constraints + +- Canonical object keys are at most 1,024 UTF-8 bytes, including the three IDs + and locator. Preserve relative-key case, literal percent/plus and internal + repeated slashes as in the existing FileIO contract. Reject leading slash, + dot traversal, backslash, controls, query/fragment delimiters and malformed + ID encodings. Decode HTTP percent escapes once at the transport boundary; + neither stored URLs nor KV keys are repeatedly decoded or normalized. + Namespace/name index components use an unambiguous length/escape codec. +- The complete table prefix ends in `/`. Authorization compares parsed IDs and + component boundaries, not textual `starts_with(table-id)`. Descendant list + prefixes may end inside a filename; S3 prefix matching need not end in `/`. + Arrow's exact-missing-file `/` probe needs no remembered prior HEAD. +- List selected published objects, including files awaiting a snapshot commit; + exclude incomplete uploads, losing candidates and tombstones. List is not a + snapshot reachability query. Return public object keys (not full URIs), size, + ETag and the existing fixed epoch LastModified. No internal logical alias is + additionally advertised for a direct file. Listing `data/` need not return + direct objects whose actual public keys are under `f/`. +- Order and paginate `file/` by public filename. Support delimiter `/`, URL + encoding, start-after and common prefixes, with the existing 256-record/ + 4-MiB scan and 2-MiB XML limits, at most 1,000 returned entries. Max-keys zero + returns an empty result. Continuations bind principal, grant nonce/expiry, + catalog epoch, full storage identity, prefix, delimiter and encoding; + foreign/expired tokens fail before scanning. Advance through zero-result + pages and common-prefix subtrees. Pagination remains a forward live traversal. +- GET/HEAD retain S3 content length, stable ETag and conditional request + semantics. Support closed, open-ended and suffix single byte ranges; clamp + an excessive end, return 206/Content-Range for a valid range and 416 for an + unsatisfiable range. Reject multiple ranges explicitly. Never interpret a + requested offset relative to the underlying chunk rather than the file, or + read neighbouring shared-chunk data. ETag is not necessarily a content MD5. +- Ordinary PUT and multipart keys cannot enter reserved `f/`. Injection issues + those URLs only after a selected durable publication; unknown or forged + tokens fail closed. Unknown length and multipart SDK uploads stay on their + original logical key. Completing multipart validates the ordered part set, + freezes it once, and replays the same outcome after response loss. Abort or + TTL expiry reclaims only unselected allocations, never a published winner. +- Writer/reader grants include table-scoped list; cleanup-only grants do not. + DELETE/DeleteObjects require explicit cleanup authority and retained-reference + checks. Failed per-object batch deletions return individual errors. Physical + GC waits for admitted-read retention and durable proof; shared chunks cannot + be reclaimed while another live extent references them. Snapshot expiration + alone is not file deletion. Keep locator file IDs non-reusable to prevent + old URLs reading later allocations at reused offsets. +- Rename/move leaves existing table-bound grants valid until their normal + expiry/revocation; issuance via the old name fails. Drop prevents new + writes/credentials; direct URLs with still-valid grants can read retained + bytes until physical reclamation. Purge + reclaims only the old table's identity; recreate cannot acquire its grants. +- Unsupported CopyObject, UploadPartCopy and bucket administration return an + explicit S3 error without mutation. A filesystem directory has no separate + native record; list synthesizes common prefixes. Do not claim broad SDK + compatibility for directory creation or copy/rename helpers until tested. +- Branch/tag references and historical snapshots keep their file URLs through + rollback, compaction and snapshot expiration. Compaction creates new immutable + files; it never overwrites old locator targets. Commit retains existing CAS, + requirement validation and response-loss behavior. Table `set-location` may + accept the existing canonical base with a trailing slash but cannot relocate + the immutable identity; unsupported location-provider overrides fail clearly. + +## Decisions + +- [x] **Freeze the physical table path at table creation**: use an immutable + ID path containing the catalog ID, the namespace ID at creation, and the + table ID. A rename changes the name index and table metadata name, while a + cross-namespace move changes the live namespace mapping without moving file + bytes or rewriting every manifest reference. The path's namespace component + is therefore a storage namespace ID, not necessarily the table's current + namespace after a move. Files use a canonical prefix such as + `s3://iceberg////`. +- [x] **Represent the default namespace explicitly**: create or recover one + durable default `NamespaceId` for the catalog and use it in the same path and + KV key rules as every named namespace. No special implicit path is needed. +- [x] **Keep S3-shaped URIs as the compatibility surface**: clients use the + S3 SDK and SigV4 over the native Iceberg listener, but the URI does not imply + a general S3 metadata bucket or an external S3 data store. The existing + `ChunkType::IcebergTable` and `FileBlockStore` remain the byte owner. +- [x] **Use a fixed compatibility bucket**: use `iceberg` as the virtual S3 + bucket for every Iceberg file URL. Keep catalog, storage namespace, and table + IDs in the object key. Do not create one bucket per table; enforce table + scope in request validation. +- [x] **Reject unscoped listing**: require a complete table prefix for every + ListObjectsV2 request, reject empty or broader prefixes, and preserve the + Arrow trailing-slash directory fallback only within that table scope. +- [x] **Expose one file URL**: the Iceberg `file_path` is the only public file + location. Keep clear parquet keys and native locator metadata inside the KV + value; do not add a second SDK-visible location field. +- **Proposed rollout**: new catalogs use v2. Existing v1 catalogs retain v1 + reads, writes, grants and URLs; reject in-place version relabeling. Migration + is outside this requirement. Dispatch by catalog layout version and never + merge v1/v2 table scans or reinterpret a v1 suffix as a v2 path. +- [x] **Choose the relative file layout**: the SDK may choose any safe logical + relative name below the table prefix. Native injection uses the reserved + `f/v1/..parquet` form; ordinary SDK writes retain the supplied key. + Both use one `file/` record. The TPC loader passes only its file name to the provider and + does not add a second table directory. + +## Phase 1: identity and canonical path model + +- [x] **Reconcile the permanent design contract**: update + `doc/design/access-server/iceberge/design-crowdb-iceberg.md` and the upload + flow to the fixed `iceberg` bucket, three-ID key, injection-before-manifest + behavior, no manifest rewrite, and direct-vs-record read modes. This must + land before implementation is considered complete. + +- [x] **Define the immutable storage-path codec**: `StorageTablePath` contains + `CatalogId`, storage `NamespaceId` and `TableId`; its lower-hex path is + independent of live names. `NativeFileLocation` uses the fixed virtual bucket + and preserves the exact filename. Boundary, UTF-8, alias and legacy isolation + tests pass in `native_storage_path_test.rs`, `file_location_test.rs`, + and `key_test.rs`. + Files: `lib/crowdb-access-iceberg/src/key/storage_path.rs`, + `lib/crowdb-access-iceberg/src/file/native_location.rs`. +- [~] **Carry immutable storage identity through runtime records**: use + `StorageTablePath` in table heads/mappings, chunk-directory owners, + credentials and creation metadata; add layout dispatch without changing the + v1 catalog contract. The standalone codec does not activate new catalogs. + Files: `lib/crowdb-access-iceberg/src/file/location.rs`, + `lib/crowdb-access-iceberg/src/file/directory.rs`, + `lib/crowdb-access-iceberg/src/table/record.rs`. +- [ ] **Define canonical URI encoding and validation**: encode IDs in fixed + lower-case forms, reserve `f/` for server-issued locators, preserve relative + UTF-8 keys, and reject aliases, traversal, authority changes, and malformed + prefixes. Round trips must recover all three IDs and the relative file key. + Files: `lib/crowdb-access-iceberg/src/file/location.rs`, + `lib/crowdb-access-iceberg/tests/file_location_test.rs`. +- [ ] **Assign the storage path once** during table creation, including staged + creation, and bind it into the table head and initial metadata. Rename and + cross-namespace move must retain it; a newly created table gets a new table + identity and path. + Files: `lib/crowdb-access-iceberg/src/commit/create/publisher/preparation.rs`, + `lib/crowdb-access-iceberg/src/table/lifecycle/`, + `lib/crowdb-access-iceberg/src/commit/create.rs`. + +## Phase 2: KV key directory redesign + +- [ ] **Remove redundant planned indexes**: name lookup returns the complete + storage identity; carry it through callers, multipart, GC and recovery so + no table-ID reverse index is introduced. Store metadata JSON, Avro manifests + and parquet through the same single file record. Keep existing bounded + metadata projection pages under the owning table/generation as derived data. + Files: `lib/crowdb-access-iceberg/src/table/`, + `lib/crowdb-access-iceberg/src/metadata_projection/`, + `lib/crowdb-access-iceberg/src/key/codec.rs`. + +- [ ] **Introduce one table-path prefix encoder** for catalog, storage + namespace, and table IDs. All table-owned file keys must use it rather than + assembling ad-hoc suffixes. + Files: `lib/crowdb-access-iceberg/src/key.rs`, + `lib/crowdb-access-iceberg/src/key/codec.rs`, + `lib/crowdb-access-iceberg/src/file/key.rs`. +- [ ] **Move the single file authority under the prefix**: + `table-path/file/` is the complete selected record. + Do not introduce separate file name, public-key or file-ID indexes. Preserve bounded + ordered scans and reject a range + whose catalog, namespace, or table prefix differs from the request. + Files: `lib/crowdb-access-iceberg/src/file/key.rs`, + `lib/crowdb-access-iceberg/src/file/listing.rs`, + `lib/crowdb-access-iceberg/src/file/listing/repository.rs`, + `lib/crowdb-access-iceberg/src/catalog/storage.rs`. +- [ ] **Audit every table-owned auxiliary key**: multipart records, write + intents/fences, GC candidates/nodes/pending state, and recovery scans must + either carry the same table-path prefix or explicitly document why an + operation-scoped key is independent. Update range bounds and foreign-record + checks together. + Files: `lib/crowdb-access-iceberg/src/file/`, + `lib/crowdb-access-iceberg/src/gc/`, + `lib/crowdb-access-iceberg/src/record/`, + `lib/crowdb-access-iceberg/src/key/codec.rs`. +- [x] **Version the table key envelope and routed-store dispatch**: + `TableStorageKey` uses a v2 envelope followed by the + slash-separated immutable table path and scope. File keys contain only the + public filename; `file_range` bounds metadata/filename scans directly. + `StorageKey` selects the decoder by version without fallback. Routed point + operations accept either validated envelope, while scan bounds reject mixed + versions and out-of-range returned keys. Legacy decoding and cross-table/ + namespace range isolation are tested. + Files: `lib/crowdb-access-iceberg/src/key/table_storage.rs`, + `lib/crowdb-access-iceberg/src/key/storage.rs`, + `lib/crowdb-access-iceberg/src/catalog/storage.rs`, + `lib/crowdb-access-iceberg/tests/native_storage_path_test.rs`. +- [ ] **Dispatch catalogs and recovery by persisted layout**: bind the selected + layout in catalog authority before creating new native paths. Retain v1 + reads/writes/grants for v1 catalogs; reject relabeling and never merge layouts + in recovery scans. Repository and lifecycle wiring must use the selected + layout together; key codec support alone is not activation. + Files: `lib/crowdb-access-iceberg/src/catalog/`, + `lib/crowdb-access-iceberg/src/record/authority.rs`, + `lib/crowdb-protocol/src/fbs/iceberg.fbs`, + `lib/crowdb-access-iceberg/tests/*recovery*test.rs`. + +## Phase 3: FileIO and S3-compatible access + +- [ ] **Implement native injection finalization**: the internal ingestion entry + accepts the full storage identity, readable stem, upload identity and bytes; + finish durable native writes, bind the exact filename to a signed descriptor, + CAS one file record, then return its one canonical URL. Persist/reuse the + existing operation identity for uncertain outcomes; no separate permanent + filename/public/file-ID indexes. SDK PUT continues to return its normal S3 + response and does not pretend to rename the client's path. + Files: `lib/crowdb-access-iceberg/src/file/`, + `app/crowdb-access-server/src/iceberg/`, `tools/load-tpch.py`. +- [ ] **Enforce the direct chunk boundary**: introduce/extend the native range + read interface accepting the authenticated full descriptor/root; instrument + file-location, file-state and chunk-KV queries separately. Implement the + missing path rather than falling back to `query_chunk` on a cache miss. + Exercise cold routing, multi-extent reads, repair, GC races and offset reuse. + Files: `lib/crowdb-access-iceberg/src/file/blocks.rs`, + `lib/crowdb-chunk-client/src/chunk/chunk_reader.rs`, + `lib/crowdb-chunk-client/tests/chunk_reader_test.rs`. +- [ ] **Reuse bounded permission validation**: define permission-cache entries + by principal, catalog epoch, full storage identity and operations; bound + validity by grants and applicable authority leases. Misses validate authority, + expired entries fail closed, and direct reads never load per-file state. + Keep permission cache accounting separate from native location resolution. + Use existing lock-free cache mechanisms; do not introduce a lock as a shortcut. + Files: `lib/crowdb-access-iceberg/src/file/credentials.rs`, + `app/crowdb-access-server/src/iceberg/file_admission.rs`. + +- [ ] **Bind grants and credential prefixes to the full storage path** so a + credential issued for one table cannot address another namespace/table path. + Update refresh responses and staged-table credentials consistently. + Files: `lib/crowdb-access-iceberg/src/file/credentials.rs`, + `lib/crowdb-access-iceberg/src/wire/credentials.rs`, + `app/crowdb-access-server/src/iceberg/table_credentials.rs`. +- [ ] **Parse and route the new URI prefix** in exact object, list, multipart, + cleanup, and continuation-token flows. A table rename must continue to use + the same URI and file records after the new name is loaded from the catalog. + Files: `app/crowdb-access-server/src/iceberg/file_request.rs`, + `app/crowdb-access-server/src/iceberg/file_http.rs`, + `app/crowdb-access-server/src/iceberg/file_list_request.rs`, + `app/crowdb-access-server/src/iceberg/file_admission.rs`. +- [ ] **Keep physical bytes native**: no filesystem or general S3 object + authority is introduced. Uploads continue through native chunk writes and + publication remains a KV authority CAS. A direct read receives a complete + validated chunk locator without `FileRepository::resolve`; a + `record_resolution` file uses the existing file record path. + Files: `app/crowdb-access-server/src/iceberg/file_http/stream.rs`, + `lib/crowdb-access-iceberg/src/file/blocks.rs`, + `lib/crowdb-access-iceberg/src/file/repository.rs`, + `lib/crowdb-chunk-client/src/chunk/chunk_reader.rs`. +- [ ] **Constrain catalog location-provider output**: reject a provider result + outside the canonical table prefix before upload. This prevents the loader + from recording a URI that the native grant cannot serve. + Files: `/cjdata/cpp/crowdb-tpc-loader/src/crowdb_tpc_loader/backend.py`, + `/cjdata/cpp/crowdb-tpc-loader/src/crowdb_tpc_loader/loader.py`. +- [ ] **Use one SDK-visible file URL**: use its public filename as the single + native KV file key, retain the native chunk descriptor and read + mode in its value, and make the single URL written to Iceberg manifests the + bound location. Implement native injection finalization before manifest + creation. Ordinary PUT retains its logical URL and `record_resolution`; + neither path rewrites existing manifests or requires a new SDK field. + Files: `lib/crowdb-access-iceberg/src/file/record.rs`, + `lib/crowdb-access-iceberg/src/file/location.rs`, + `app/crowdb-access-server/src/iceberg/file_http/`, + `lib/crowdb-access-iceberg/src/wire/credentials.rs`. +- [ ] **Add direct locator validation**: bind locator version, table identity, + file identity, length, integrity metadata and chunk root; authenticate the + current HTTP grant independently of the durable locator signature; + reject tampering before chunk access and enforce one contiguous range. + Files: `lib/crowdb-access-iceberg/src/file/location.rs`, + `lib/crowdb-access-iceberg/src/file/record.rs`, + `app/crowdb-access-server/src/iceberg/file_http.rs`. + +## Phase 4: metadata, rename, and lifecycle invariants + +- [ ] **Keep large metadata as native objects**: preserve the inline threshold, + route larger metadata JSON/Avro into native chunk storage, bound descriptors + with directory roots and keep snapshot arrays out of TableHead. Exercise + canonical bytes through FileIO and ALL/REFS, including missing/corrupt + generation projections and explicit parser/response limit errors. + Files: `lib/crowdb-access-iceberg/src/file/record.rs`, + `lib/crowdb-access-iceberg/src/table/metadata/`, + `lib/crowdb-access-iceberg/src/metadata_projection/`, + `lib/crowdb-access-iceberg/src/commit/files.rs`. +- [ ] **Implement bounded metadata prefix list**: map the public metadata or + filename prefix to the exact `file/` range; stream ordered records into a + bounded page and persist an authenticated continuation at scan-budget + exhaustion. Handle tombstones, sparse pages, delimiter common prefixes and + zero max-keys without payload reads, table-wide sorting or unrelated scans. + Files: `lib/crowdb-access-iceberg/src/file/listing/`, + `app/crowdb-access-server/src/iceberg/file_list_request.rs`. +- [ ] **Preserve standard metadata semantics**: cover snapshots/refs/logs and + v1 direct manifests, v2/v3 sequence inheritance and existing v3 lineage; + preserve exact delete-file references and immutable manifest bytes. Commit + retries may write new manifest lists; atomic head selection and retained + reference GC remain authoritative. Add no private snapshot KV authority. + Files: `lib/crowdb-access-iceberg/src/commit/`, + `lib/crowdb-access-iceberg/src/table/metadata/`, + `lib/crowdb-access-iceberg/src/gc/`, + `app/crowdb-access-server/tests/iceberg_table_lifecycle_test.rs`. + +- [ ] **Validate table metadata location against the immutable storage path**; + `set-location` may normalize a trailing slash but cannot change the path + identity. Metadata generations and manifest references remain below that + path. + Files: `lib/crowdb-access-iceberg/src/table/metadata/root.rs`, + `lib/crowdb-access-iceberg/src/commit/evaluator/initial.rs`, + `lib/crowdb-access-iceberg/src/commit/evaluator/scalar.rs`, + `lib/crowdb-access-iceberg/src/commit/files.rs`. +- [ ] **Preserve rename behavior**: same-namespace rename and cross-namespace + move update only name/mapping authority; old names disappear, the storage + path remains readable through the new table identity, and a recreated name + receives a new path. + Files: `lib/crowdb-access-iceberg/src/table/lifecycle/`, + `app/crowdb-access-server/tests/iceberg_table_lifecycle_test.rs`, + `e2e-test/clients/iceberg-java/src/main/java/TestIcebergCatalogWrites.java`. +- [ ] **Define grant and URL behavior across moves**: existing immutable URLs + continue to bind to the storage namespace ID while the table is live; new + credentials resolve the current name mapping and old-name resolution fails. + Drop/recreate receives a new table ID, and direct and fallback locators reject + the old table after purge. + Files: `lib/crowdb-access-iceberg/src/file/credentials.rs`, + `app/crowdb-access-server/src/iceberg/file_admission.rs`. +- [ ] **Recheck GC and recovery ownership** against the immutable storage path + before reclaiming files or chunk roots. Logical drop and physical purge must + not confuse a recreated table with a retained old path. + Files: `lib/crowdb-access-iceberg/src/gc/`, + `lib/crowdb-access-iceberg/src/file/multipart_recovery.rs`, + `lib/crowdb-access-iceberg/src/record/`. + +## Phase 5: loader and container injection + +- [ ] **Align the TPC loader with the canonical table path**: use stable data + names below the provider's `data/` directory, verify the returned URI is + table-scoped, and keep the run ID in recovery/report metadata rather than in + the object path. + Files: `/cjdata/cpp/crowdb-tpc-loader/src/crowdb_tpc_loader/backend.py`, + `/cjdata/cpp/crowdb-tpc-loader/src/crowdb_tpc_loader/loader.py`, + `/cjdata/cpp/crowdb-tpc-loader/tests/unit/test_commits.py`. +- [ ] **Separate or remove the optional S3 mirror** in + `tools/load-tpch.py`. If retained for the container demo, document its + bucket/key as an independent S3 object copy and do not treat it as the + Iceberg table file layout. + Files: `tools/load-tpch.py`, + `container/single-node-container/README.md`. + +## Acceptance tests + +Unit: + +- [ ] Canonical storage path and URI round trips include catalog, storage + namespace, table, and relative file key. +- [ ] New and old key layouts cannot collide; table-path scans exclude foreign + namespace/table records. +- [ ] File grants, list prefixes, continuation tokens, multipart sessions, GC + keys, and chunk-directory owners reject mismatched identities. + +Integration: + +- [ ] Store metadata above the inline threshold with many snapshots; assert + bounded head/record size, chunk-backed bytes, correct ALL/REFS and projection + fallback. Exceed each configured limit and assert an explicit error with no + truncated success or partially published head. +- [ ] Generate many metadata and manifest files with interleaved other file + prefixes. Page full-table, `metadata/` and filename-prefix lists; record scan + bounds, peak page memory and payload-read counts. Assert no unrelated scans, + no payload reads and complete results on an unchanged eligible set. +- [ ] Race metadata commits and retry their lost responses; reload every + selected version with official clients and verify snapshot/manifest references, + retained branches/tags and protected shared-file GC behavior. + +- [ ] Publish native injection, load its direct URL through unmodified readers, + and publish a separate SDK-written logical URL. Assert one URL per file and + correct position deletes, deletion-vector references and historical reads. +- [ ] Crash before and after the single file-record CAS, restart and + replay; assert no ghost list entries, no lost selected file and no GC of a + losing operation's shared winner. Retry equal-length different bytes. +- [ ] Renew credentials and rotate locator signing keys; retained snapshot URLs + stay readable. Delete concurrently with direct read and restart: list omits + the deleted file, but a valid direct URL reads bytes until reclamation. After + reclamation/reuse, the old URL must never return another file's bytes. +- [ ] Count location, deletion-state and chunk-layout RPCs independently for + cold/warm HEAD and footer ranges. Include multi-location files, directory-root + boundary reads, chunk repair/failover and URI size limits. No zero-KV claim + may be inferred from merely skipping `FileRepository::resolve`; assert zero + file-location, file-state and chunk-KV resolution RPCs for direct reads. +- [ ] Test public-key order and delimiter grouping when internal logical names + sort differently; test max-keys zero, empty pages and credential refresh. +- [ ] Test suffix/open-ended/out-of-bounds ranges, ETag conditions, unknown + length upload, multipart abort-vs-complete races and response-loss retries. +- [ ] Move a table, drop/recreate its old namespace, then resolve and read via + the new name; assert no cascade deletion or storage namespace ID reuse. + +- [ ] Create a table in the default namespace and an explicit namespace; + inspect metadata, KV keys, and native file locations. +- [ ] PUT/GET/HEAD/list/multipart a parquet file through an S3 SDK and verify + the object is below the canonical table prefix, bytes remain in native + Iceberg chunk storage, the URL is unchanged by commit, and direct versus + fallback reads are observable. +- [ ] Exercise Arrow HeadBucket, exact HEAD miss directory fallback, recursive + and non-recursive ListObjectsV2, continuation expiry, and rejection of empty, + catalog-only, namespace-only, foreign-table, and bucket-wide prefixes. +- [ ] Rename within a namespace and move across namespaces; reload metadata and + read the original file through the new name without moving or duplicating it. +- [ ] Drop and recreate the same name; verify the new table cannot see the old + path and GC/recovery retain the correct ownership. + +E2E: + +- [ ] Run PyIceberg, Java Iceberg, and Rust Iceberg lifecycle flows with native + FileIO credentials and the new URI prefix. +- [ ] Run the container injection flow and verify the Iceberg report's + `remote_uri` values separately from any optional general-S3 mirror report. +- [ ] Exercise response loss, restart, multipart recovery, and retired-table + grace paths with the new key ranges. +- [ ] Exercise v1 legacy reads/writes, rejected in-place activation, v1/v2 collision checks, + direct locator tampering, foreign-table use, checksum/length conflicts, and + the no-manifest-rewrite guarantee. + +## Consolidated file list + +- Identity and URI: `lib/crowdb-access-iceberg/src/file/`, + `lib/crowdb-access-iceberg/src/table/record.rs`. +- KV keys and scans: `lib/crowdb-access-iceberg/src/key/`, + `lib/crowdb-access-iceberg/src/file/key.rs`, + `lib/crowdb-access-iceberg/src/catalog/storage.rs`. +- Metadata scale: `lib/crowdb-access-iceberg/src/metadata_projection/`, + `lib/crowdb-access-iceberg/src/table/metadata/`. +- Native direct reads: `lib/crowdb-chunk-client/src/chunk/chunk_reader.rs`, + `lib/crowdb-chunk-client/tests/chunk_reader_test.rs`. +- HTTP/FileIO: `app/crowdb-access-server/src/iceberg/`, + `lib/crowdb-access-iceberg/src/wire/credentials.rs`. +- Lifecycle and GC: `lib/crowdb-access-iceberg/src/commit/`, + `lib/crowdb-access-iceberg/src/table/lifecycle/`, + `lib/crowdb-access-iceberg/src/gc/`. +- Loader/container: `/cjdata/cpp/crowdb-tpc-loader/src/crowdb_tpc_loader/`, + `/cjdata/cpp/crowdb/tools/load-tpch.py`, + `container/single-node-container/README.md`. diff --git a/lib/crowdb-access-iceberg/src/catalog/storage.rs b/lib/crowdb-access-iceberg/src/catalog/storage.rs index 83711846c..96ac6a54a 100644 --- a/lib/crowdb-access-iceberg/src/catalog/storage.rs +++ b/lib/crowdb-access-iceberg/src/catalog/storage.rs @@ -11,7 +11,7 @@ use crowdb_protocol::chunk_kv::{ }; use crate::error::ValidationError; -use crate::key::{CatalogScope, IcebergKey, MAX_KEY_BYTES}; +use crate::key::{CatalogScope, IcebergKey, StorageKey}; use crate::record::MAX_RECORD_BYTES; #[derive(Debug, thiserror::Error)] @@ -155,11 +155,11 @@ impl RoutedCatalogStore { identity: ClientRequestId, ) -> Result { if matches!( - IcebergKey::decode(key)?, - IcebergKey::System { + StorageKey::decode(key)?, + StorageKey::Legacy(IcebergKey::System { scope: crate::key::SystemScope::ActiveRoot, .. - } + }) ) { return Err(ValidationError::Key.into()); } @@ -200,14 +200,8 @@ impl RoutedCatalogStore { let (Some(start), Some(end)) = (&request.start, &request.end) else { return Err(ValidationError::Key.into()); }; - if start.as_slice() < b"ICE\0\x01".as_slice() - || end.as_slice() > b"ICE\0\x02".as_slice() - || start >= end - || start.len() > MAX_KEY_BYTES - || end.len() > MAX_KEY_BYTES - { - return Err(ValidationError::Key.into()); - } + let range = start.clone()..end.clone(); + StorageKey::validate_range(&range)?; if request.max_items == 0 || request.max_items > 256 || request.max_bytes == 0 @@ -226,7 +220,10 @@ impl RoutedCatalogStore { if value.revision == 0 { return Err(StoreError::Response); } - IcebergKey::decode(&value.key)?; + StorageKey::decode(&value.key)?; + if !range.contains(&value.key) { + return Err(StoreError::Response); + } validate_value(&value.value)?; } Ok(page) @@ -246,7 +243,7 @@ impl CatalogStore for RoutedCatalogStore { } async fn get(&self, key: &[u8]) -> Result, StoreError> { - IcebergKey::decode(key)?; + StorageKey::decode(key)?; self.counters.get.fetch_add(1, Ordering::Relaxed); count_request(|counters| &counters.get); let response = self.client.get(key.to_vec(), None).await?; @@ -263,7 +260,7 @@ impl CatalogStore for RoutedCatalogStore { value: &[u8], identity: ClientRequestId, ) -> Result { - IcebergKey::decode(key)?; + StorageKey::decode(key)?; validate_value(value)?; let operation = match expected { Some(expected) => { diff --git a/lib/crowdb-access-iceberg/src/file.rs b/lib/crowdb-access-iceberg/src/file.rs index 4c56eb0e3..d56f9a776 100644 --- a/lib/crowdb-access-iceberg/src/file.rs +++ b/lib/crowdb-access-iceberg/src/file.rs @@ -21,6 +21,9 @@ mod multipart_list; mod multipart_recovery; mod multipart_repository; mod multipart_selection; +mod native_location; +pub(crate) use native_location::validate_native_relative_key; +pub use native_location::{NativeFileLocation, NATIVE_ICEBERG_BUCKET}; mod parquet; pub use parquet::ParquetPageLimits; pub(crate) use parquet::{ParquetColumnReader, ParquetColumnValue}; diff --git a/lib/crowdb-access-iceberg/src/file/native_location.rs b/lib/crowdb-access-iceberg/src/file/native_location.rs new file mode 100644 index 000000000..e827ec8a8 --- /dev/null +++ b/lib/crowdb-access-iceberg/src/file/native_location.rs @@ -0,0 +1,99 @@ +use std::{fmt, str::FromStr}; + +use crate::{error::ValidationError, key::StorageTablePath}; + +use super::{location::validate_relative_key, MAX_OBJECT_KEY_BYTES}; + +pub const NATIVE_ICEBERG_BUCKET: &str = "iceberg"; + +/// A canonical SDK-visible object name in native Iceberg storage. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct NativeFileLocation { + table: StorageTablePath, + relative_key: String, +} + +impl StorageTablePath { + #[must_use] + pub fn uri(self) -> String { + format!("s3://{NATIVE_ICEBERG_BUCKET}/{self}") + } + + /// # Errors + /// Rejects unsafe relative keys and object keys exceeding the SDK byte limit. + pub fn file(self, relative_key: &str) -> Result { + validate_native_relative_key(relative_key)?; + Ok(NativeFileLocation { + table: self, + relative_key: relative_key.to_owned(), + }) + } + + /// Admission for names chosen by ordinary SDK writers. + /// + /// # Errors + /// Rejects invalid keys and the server-owned direct locator directory. + pub fn logical_file(self, relative_key: &str) -> Result { + if relative_key == "f" || relative_key.starts_with("f/") { + return Err(ValidationError::Key); + } + self.file(relative_key) + } +} + +impl NativeFileLocation { + /// # Errors + /// Rejects foreign buckets, malformed table prefixes and unsafe filenames. + pub fn from_object_key(bucket: &str, key: &str) -> Result { + if bucket != NATIVE_ICEBERG_BUCKET { + return Err(ValidationError::Key); + } + if key.len() > MAX_OBJECT_KEY_BYTES { + return Err(ValidationError::KeyTooLarge); + } + let (table, relative) = StorageTablePath::split_object_key(key)?; + table.file(relative) + } + + #[must_use] + pub const fn table(&self) -> StorageTablePath { + self.table + } + + #[must_use] + pub fn relative_key(&self) -> &str { + &self.relative_key + } + + #[must_use] + pub fn object_key(&self) -> String { + format!("{}{}", self.table, self.relative_key) + } +} + +impl fmt::Display for NativeFileLocation { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "s3://{NATIVE_ICEBERG_BUCKET}/{}{}", + self.table, self.relative_key + ) + } +} + +impl FromStr for NativeFileLocation { + type Err = ValidationError; + + fn from_str(value: &str) -> Result { + let body = value.strip_prefix("s3://").ok_or(ValidationError::Key)?; + let (bucket, key) = body.split_once('/').ok_or(ValidationError::Key)?; + Self::from_object_key(bucket, key) + } +} + +pub(crate) fn validate_native_relative_key(key: &str) -> Result<(), ValidationError> { + if key.len() > MAX_OBJECT_KEY_BYTES - StorageTablePath::PREFIX_BYTES { + return Err(ValidationError::KeyTooLarge); + } + validate_relative_key(key) +} diff --git a/lib/crowdb-access-iceberg/src/key.rs b/lib/crowdb-access-iceberg/src/key.rs index ada9471ee..9af6443b2 100644 --- a/lib/crowdb-access-iceberg/src/key.rs +++ b/lib/crowdb-access-iceberg/src/key.rs @@ -3,7 +3,13 @@ mod codec; mod identity; mod name; +mod storage; +mod storage_path; +mod table_storage; pub use codec::{CatalogScope, IcebergKey, SystemScope, MAX_KEY_BYTES}; pub use identity::{CatalogId, FileId, NamespaceId, OperationId, TableId}; pub use name::NameSuffix; +pub use storage::StorageKey; +pub use storage_path::StorageTablePath; +pub use table_storage::{TableStorageKey, TableStorageScope}; diff --git a/lib/crowdb-access-iceberg/src/key/storage.rs b/lib/crowdb-access-iceberg/src/key/storage.rs new file mode 100644 index 000000000..3994a6437 --- /dev/null +++ b/lib/crowdb-access-iceberg/src/key/storage.rs @@ -0,0 +1,47 @@ +use std::ops::Range; + +use crate::error::ValidationError; + +use super::{IcebergKey, TableStorageKey, MAX_KEY_BYTES}; + +/// Dispatches storage envelopes without trying another layout on decode failure. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum StorageKey { + Legacy(IcebergKey), + NativeTable(TableStorageKey), +} + +impl StorageKey { + /// # Errors + /// Rejects unsupported versions and malformed keys in their selected layout. + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_KEY_BYTES { + return Err(ValidationError::KeyTooLarge); + } + if bytes.get(..4) != Some(b"ICE\0") { + return Err(ValidationError::Key); + } + match bytes.get(4).copied().ok_or(ValidationError::Key)? { + 1 => IcebergKey::decode(bytes).map(Self::Legacy), + 2 => TableStorageKey::decode(bytes).map(Self::NativeTable), + version => Err(ValidationError::KeyVersion(version)), + } + } + + /// # Errors + /// Rejects unbounded, reversed or oversized scans and scans crossing layouts. + pub fn validate_range(range: &Range>) -> Result<(), ValidationError> { + if range.start.len() > MAX_KEY_BYTES || range.end.len() > MAX_KEY_BYTES { + return Err(ValidationError::KeyTooLarge); + } + let (lower, upper): (&[u8], &[u8]) = match range.start.get(..5) { + Some(b"ICE\0\x01") => (b"ICE\0\x01", b"ICE\0\x02"), + Some(b"ICE\0\x02") => (b"ICE\0\x02", b"ICE\0\x03"), + _ => return Err(ValidationError::Key), + }; + if range.start.as_slice() < lower || range.end.as_slice() > upper || range.start >= range.end { + return Err(ValidationError::Key); + } + Ok(()) + } +} diff --git a/lib/crowdb-access-iceberg/src/key/storage_path.rs b/lib/crowdb-access-iceberg/src/key/storage_path.rs new file mode 100644 index 000000000..06190b784 --- /dev/null +++ b/lib/crowdb-access-iceberg/src/key/storage_path.rs @@ -0,0 +1,74 @@ +use std::{fmt, str::FromStr}; + +use crate::error::ValidationError; + +use super::{CatalogId, NamespaceId, TableId}; + +/// Immutable storage ownership, independent of a table's current name or namespace. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct StorageTablePath { + pub catalog: CatalogId, + pub storage_namespace: NamespaceId, + pub table: TableId, +} + +impl StorageTablePath { + pub const PREFIX_BYTES: usize = 99; + + /// # Errors + /// Rejects aliases, noncanonical IDs and incomplete table prefixes. + pub fn from_prefix(prefix: &str) -> Result { + let (path, relative) = Self::split_object_key(prefix)?; + if !relative.is_empty() { + return Err(ValidationError::Key); + } + Ok(path) + } + + /// Separates a complete immutable table prefix from its undecoded suffix. + /// + /// # Errors + /// Rejects missing components, zero IDs and noncanonical identity encodings. + pub fn split_object_key(key: &str) -> Result<(Self, &str), ValidationError> { + let mut fields = key.splitn(4, '/'); + let catalog = fields.next().ok_or(ValidationError::Key)?; + let namespace = fields.next().ok_or(ValidationError::Key)?; + let table = fields.next().ok_or(ValidationError::Key)?; + let relative = fields.next().ok_or(ValidationError::Key)?; + let path = Self { + catalog: canonical_id(catalog)?, + storage_namespace: canonical_id(namespace)?, + table: canonical_id(table)?, + }; + Ok((path, relative)) + } +} + +impl fmt::Display for StorageTablePath { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "{}/{}/{}/", + self.catalog, self.storage_namespace, self.table + ) + } +} + +impl FromStr for StorageTablePath { + type Err = ValidationError; + + fn from_str(value: &str) -> Result { + Self::from_prefix(value) + } +} + +pub(super) fn canonical_id>(value: &str) -> Result { + if value.len() != 32 + || !value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(ValidationError::Key); + } + value.parse() +} diff --git a/lib/crowdb-access-iceberg/src/key/table_storage.rs b/lib/crowdb-access-iceberg/src/key/table_storage.rs new file mode 100644 index 000000000..335fcc29b --- /dev/null +++ b/lib/crowdb-access-iceberg/src/key/table_storage.rs @@ -0,0 +1,153 @@ +use std::ops::Range; + +use crate::{error::ValidationError, file::validate_native_relative_key}; + +use super::{storage_path::canonical_id, OperationId, StorageTablePath, MAX_KEY_BYTES}; + +const HEADER: &[u8] = b"ICE\0\x02"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TableStorageScope { + Head, + File, + Projection, + Operation, + Multipart, + Gc, +} + +impl TableStorageScope { + const fn component(self) -> &'static str { + match self { + Self::Head => "head", + Self::File => "file", + Self::Projection => "projection", + Self::Operation => "operation", + Self::Multipart => "multipart", + Self::Gc => "gc", + } + } + + fn parse(value: &str) -> Result { + match value { + "head" => Ok(Self::Head), + "file" => Ok(Self::File), + "projection" => Ok(Self::Projection), + "operation" => Ok(Self::Operation), + "multipart" => Ok(Self::Multipart), + "gc" => Ok(Self::Gc), + _ => Err(ValidationError::Key), + } + } +} + +/// Version-isolated table keys; filenames are their only file authority suffix. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct TableStorageKey { + pub table: StorageTablePath, + pub scope: TableStorageScope, + pub suffix: String, +} + +impl TableStorageKey { + /// # Errors + /// Rejects unsafe suffixes, invalid head keys and oversized keys. + pub fn encode(&self) -> Result, ValidationError> { + self.validate()?; + let mut bytes = scope_prefix(self.table, self.scope); + bytes.extend_from_slice(self.suffix.as_bytes()); + if bytes.len() > MAX_KEY_BYTES { + return Err(ValidationError::KeyTooLarge); + } + Ok(bytes) + } + + /// # Errors + /// Rejects legacy keys, unknown scopes, aliases and invalid suffixes. + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_KEY_BYTES { + return Err(ValidationError::KeyTooLarge); + } + let payload = bytes.strip_prefix(HEADER).ok_or(ValidationError::Key)?; + let payload = std::str::from_utf8(payload).map_err(|_| ValidationError::Key)?; + let (table, relative) = StorageTablePath::split_object_key(payload)?; + let (component, suffix) = relative.split_once('/').unwrap_or((relative, "")); + let scope = TableStorageScope::parse(component)?; + if (scope == TableStorageScope::Head) == relative.contains('/') { + return Err(ValidationError::Key); + } + let key = Self { + table, + scope, + suffix: suffix.to_owned(), + }; + key.validate()?; + Ok(key) + } + + #[must_use] + pub fn table_range(table: StorageTablePath) -> Range> { + let mut start = HEADER.to_vec(); + start.extend_from_slice(table.to_string().as_bytes()); + prefix_range(start) + } + + /// A bounded ordered file scan, including arbitrary public filename prefixes. + /// + /// # Errors + /// Rejects escaping prefixes and prefixes exceeding the object-key byte limit. + pub fn file_range( + table: StorageTablePath, + filename_prefix: &str, + ) -> Result>, ValidationError> { + if !filename_prefix.is_empty() { + validate_native_relative_key(filename_prefix)?; + } + let mut start = scope_prefix(table, TableStorageScope::File); + start.extend_from_slice(filename_prefix.as_bytes()); + Ok(prefix_range(start)) + } + + fn validate(&self) -> Result<(), ValidationError> { + match self.scope { + TableStorageScope::Head if self.suffix.is_empty() => Ok(()), + TableStorageScope::Head => Err(ValidationError::Key), + TableStorageScope::File => validate_native_relative_key(&self.suffix), + TableStorageScope::Projection => { + crate::file::validate_relative_key(&self.suffix)?; + let (generation, child) = self.suffix.split_once('/').ok_or(ValidationError::Key)?; + let number = generation.parse::().map_err(|_| ValidationError::Key)?; + if number == 0 || number.to_string() != generation || child.is_empty() { + return Err(ValidationError::Key); + } + Ok(()) + } + TableStorageScope::Operation | TableStorageScope::Multipart | TableStorageScope::Gc => { + crate::file::validate_relative_key(&self.suffix)?; + let identity = self.suffix.split('/').next().ok_or(ValidationError::Key)?; + canonical_id::(identity).map(|_| ()) + } + } + } +} + +fn scope_prefix(table: StorageTablePath, scope: TableStorageScope) -> Vec { + let mut bytes = HEADER.to_vec(); + bytes.extend_from_slice(table.to_string().as_bytes()); + bytes.extend_from_slice(scope.component().as_bytes()); + if scope != TableStorageScope::Head { + bytes.push(b'/'); + } + bytes +} + +fn prefix_range(start: Vec) -> Range> { + let mut end = start.clone(); + while let Some(byte) = end.pop() { + if byte != u8::MAX { + end.push(byte + 1); + break; + } + } + start..end +} diff --git a/lib/crowdb-access-iceberg/tests/native_storage_path_test.rs b/lib/crowdb-access-iceberg/tests/native_storage_path_test.rs new file mode 100644 index 000000000..e6e2edb6c --- /dev/null +++ b/lib/crowdb-access-iceberg/tests/native_storage_path_test.rs @@ -0,0 +1,225 @@ +use crowdb_access_iceberg::{ + file::{NativeFileLocation, MAX_OBJECT_KEY_BYTES}, + key::{ + CatalogId, CatalogScope, IcebergKey, NamespaceId, StorageKey, StorageTablePath, TableId, + TableStorageKey, TableStorageScope, + }, +}; + +fn table() -> StorageTablePath { + StorageTablePath { + catalog: CatalogId::from_bytes(&[0xab; 16]).unwrap(), + storage_namespace: NamespaceId::from_bytes(&[0xcd; 16]).unwrap(), + table: TableId::from_bytes(&[0xef; 16]).unwrap(), + } +} + +fn key(table: StorageTablePath, scope: TableStorageScope, suffix: &str) -> TableStorageKey { + TableStorageKey { + table, + scope, + suffix: suffix.to_owned(), + } +} + +#[test] +fn native_uri_preserves_immutable_identity_and_exact_filename() { + let table = table(); + assert_eq!(table.to_string().len(), StorageTablePath::PREFIX_BYTES); + assert_eq!(table.to_string().parse::().unwrap(), table); + assert_eq!(table.uri(), format!("s3://iceberg/{table}")); + for filename in ["metadata/v1.json", "数据/a+b", "a%2Fb", "a//b", "a/", "a b"] { + let location = table.logical_file(filename).unwrap(); + assert_eq!(location.table(), table); + assert_eq!(location.relative_key(), filename); + assert_eq!( + location.to_string().parse::().unwrap(), + location + ); + assert_eq!( + NativeFileLocation::from_object_key("iceberg", &location.object_key()).unwrap(), + location + ); + } + assert_ne!(table.file("a%2Fb").unwrap(), table.file("a/b").unwrap()); + assert!(table.logical_file("f/v1/name.token.parquet").is_err()); + assert!(table.file("f/v1/name.token.parquet").is_ok()); +} + +#[test] +fn native_uri_rejects_aliases_incomplete_scopes_and_escaping_names() { + let table = table(); + let uri = table.file("metadata/v1.json").unwrap().to_string(); + for invalid in [ + uri.replacen("abab", "ABAB", 1), + uri.replacen("abababab", "abab-abab", 1), + uri.replacen("iceberg/", "iceberg:9000/", 1), + uri.replacen("iceberg/", "user@iceberg/", 1), + uri.replacen("iceberg/", "iceberg-old/", 1), + uri.replacen("s3:", "s3a:", 1), + uri.replacen(&table.catalog.to_string(), &"0".repeat(32), 1), + ] { + assert!(invalid.parse::().is_err(), "{invalid}"); + } + for filename in ["", "/a", "../a", "a/../b", "a/./b", "a\\b", "a?b", "a#b", "a\0b"] { + assert!(table.file(filename).is_err(), "{filename:?}"); + } + for prefix in [ + format!("{}/", table.catalog), + format!("{}/{}/", table.catalog, table.storage_namespace), + table.to_string().trim_end_matches('/').to_owned(), + format!("{table}metadata/"), + ] { + assert!(prefix.parse::().is_err(), "{prefix}"); + } + assert!(table.uri().parse::().is_err()); +} + +#[test] +fn filename_limit_counts_three_ids_and_utf8_bytes() { + let table = table(); + let available = MAX_OBJECT_KEY_BYTES - StorageTablePath::PREFIX_BYTES; + let filename = "x".repeat(available); + let location = table.file(&filename).unwrap(); + assert_eq!(location.object_key().len(), MAX_OBJECT_KEY_BYTES); + assert_eq!( + location.to_string().parse::().unwrap(), + location + ); + assert!(table.file(&(filename + "x")).is_err()); + assert!(table.file(&("x".repeat(available - 1) + "冰")).is_err()); +} + +#[test] +fn table_keys_are_version_isolated_and_round_trip() { + let table = table(); + let legacy = IcebergKey::Catalog { + catalog: table.catalog, + scope: CatalogScope::File, + suffix: vec![1; 16], + } + .encode() + .unwrap(); + assert!(TableStorageKey::decode(&legacy).is_err()); + assert!(matches!( + StorageKey::decode(&legacy).unwrap(), + StorageKey::Legacy(_) + )); + for (scope, suffix, component) in [ + (TableStorageScope::Head, "", "head"), + (TableStorageScope::File, "metadata/v1.json", "file/"), + (TableStorageScope::Projection, "1/receipt", "projection/"), + ( + TableStorageScope::Operation, + "01010101010101010101010101010101/state", + "operation/", + ), + ( + TableStorageScope::Multipart, + "01010101010101010101010101010101/part/1", + "multipart/", + ), + ( + TableStorageScope::Gc, + "01010101010101010101010101010101/state", + "gc/", + ), + ] { + let key = key(table, scope, suffix); + let bytes = key.encode().unwrap(); + assert_eq!(TableStorageKey::decode(&bytes).unwrap(), key); + assert_eq!( + StorageKey::decode(&bytes).unwrap(), + StorageKey::NativeTable(key.clone()) + ); + assert!(IcebergKey::decode(&bytes).is_err()); + assert!(TableStorageKey::table_range(table).contains(&bytes)); + assert_eq!(&bytes[5..], format!("{table}{component}{suffix}").as_bytes()); + } +} + +#[test] +fn scan_validation_never_combines_storage_versions() { + let table = table(); + let legacy = IcebergKey::catalog_range(table.catalog); + let native = TableStorageKey::table_range(table); + assert!(StorageKey::validate_range(&legacy).is_ok()); + assert!(StorageKey::validate_range(&native).is_ok()); + assert!(StorageKey::validate_range(&TableStorageKey::file_range(table, "metadata/").unwrap()).is_ok()); + assert!(StorageKey::validate_range(&(legacy.start..native.end.clone())).is_err()); + assert!(StorageKey::validate_range(&(native.end.clone()..native.start.clone())).is_err()); + assert!(StorageKey::validate_range(&(native.start.clone()..native.start)).is_err()); + assert!(StorageKey::decode(b"ICE\0\x03").is_err()); + assert!(StorageKey::decode(b"ICE\0\x02not-a-path").is_err()); +} + +#[test] +fn prefix_scans_exclude_other_kinds_namespaces_and_tables() { + let table = table(); + let prefix = TableStorageKey::file_range(table, "metadata/v1").unwrap(); + for filename in [ + "metadata/v1.json", + "metadata/v10.json", + "metadata/v1/manifest.avro", + ] { + assert!(prefix.contains(&key(table, TableStorageScope::File, filename).encode().unwrap())); + } + for filename in ["data/v1.parquet", "metadata/v2.json", "metadata/v0.json"] { + assert!(!prefix.contains(&key(table, TableStorageScope::File, filename).encode().unwrap())); + } + for foreign in [ + StorageTablePath { + table: TableId::random(), + ..table + }, + StorageTablePath { + storage_namespace: NamespaceId::random(), + ..table + }, + StorageTablePath { + catalog: CatalogId::random(), + ..table + }, + ] { + let bytes = key(foreign, TableStorageScope::File, "metadata/v1.json") + .encode() + .unwrap(); + assert!(!prefix.contains(&bytes)); + assert!(!TableStorageKey::table_range(table).contains(&bytes)); + } + let all_files = TableStorageKey::file_range(table, "").unwrap(); + assert!(!all_files.contains(&key(table, TableStorageScope::Head, "").encode().unwrap())); + assert!(!all_files.contains( + &key(table, TableStorageScope::Projection, "1/receipt") + .encode() + .unwrap() + )); + assert!(TableStorageKey::file_range(table, "../").is_err()); +} + +#[test] +fn malformed_keys_cannot_alias_head_or_filename() { + let table = table(); + for (scope, suffix) in [ + (TableStorageScope::Head, "file"), + (TableStorageScope::File, ""), + (TableStorageScope::File, "../file"), + (TableStorageScope::Operation, "../operation"), + (TableStorageScope::Operation, "upload/state"), + (TableStorageScope::Gc, "00000000000000000000000000000000/state"), + ( + TableStorageScope::Multipart, + "ABABABABABABABABABABABABABABABAB/part", + ), + (TableStorageScope::Projection, "0/receipt"), + (TableStorageScope::Projection, "01/receipt"), + (TableStorageScope::Projection, "1/"), + ] { + assert!(key(table, scope, suffix).encode().is_err()); + } + for suffix in ["head/", "head/file", "file/", "unknown/x", "file/../x"] { + let mut bytes = b"ICE\0\x02".to_vec(); + bytes.extend_from_slice(format!("{table}{suffix}").as_bytes()); + assert!(TableStorageKey::decode(&bytes).is_err(), "{suffix}"); + } +} From a5e5c439e22015d417703d8e628406e50c0bb14f Mon Sep 17 00:00:00 2001 From: Gian Date: Wed, 14 Oct 2026 18:55:08 +0800 Subject: [PATCH 02/15] Add native Iceberg file records and direct read locators --- app/crowdb-access-server/src/iceberg.rs | 2 + .../src/iceberg/file_bucket_request.rs | 43 ++++ .../src/iceberg/file_http.rs | 106 +++++---- .../src/iceberg/file_http/bucket.rs | 67 ++++++ .../src/iceberg/file_http/delete.rs | 6 +- .../src/iceberg/file_list_request.rs | 8 +- .../src/iceberg/file_request.rs | 8 +- .../src/iceberg/table_credentials.rs | 19 +- .../src/iceberg/table_read.rs | 1 + .../src/iceberg/table_write.rs | 1 + .../tests/common/iceberg_file_blocks.rs | 1 + .../tests/common/iceberg_table_http.rs | 2 + .../tests/common/iceberg_upload.rs | 1 + .../tests/iceberg_file_admission_test.rs | 3 + .../tests/iceberg_file_auth_test.rs | 1 + .../tests/iceberg_file_recovery_test.rs | 1 + .../tests/iceberg_file_request_test.rs | 56 +++++ .../tests/iceberg_file_response_test.rs | 1 + .../tests/iceberg_table_acceptance_test.rs | 1 + ...34-access-iceberg-native-storage-layout.md | 10 +- doc/backlog/backlog.md | 6 +- .../iceberge/design-crowdb-iceberg.md | 8 +- doc/working/plan-iceberg-storage-path.md | 57 ++++- .../access_iceberg/file_http/capacity.rs | 1 + .../access_iceberg/file_http/capacity/gc.rs | 1 + .../tests/access_iceberg/file_http/listing.rs | 1 + .../tests/access_iceberg/file_http/setup.rs | 2 + e2e-test/tests/access_iceberg/gc/seed.rs | 2 + e2e-test/tests/access_iceberg/native_files.rs | 1 + e2e-test/tests/access_s3/protocol_http.rs | 1 + lib/crowdb-access-iceberg/Cargo.toml | 1 + .../src/commit/create/operation.rs | 1 + .../src/commit/create/publisher/completion.rs | 6 +- .../commit/create/publisher/preparation.rs | 1 + .../commit/create/publisher/reservation.rs | 4 +- .../src/commit/direct.rs | 7 +- .../src/commit/files/fence.rs | 4 +- .../src/commit/files/snapshots.rs | 4 +- .../src/commit/journal.rs | 4 +- .../src/commit/publication/completion.rs | 8 +- lib/crowdb-access-iceberg/src/file.rs | 4 +- lib/crowdb-access-iceberg/src/file/blocks.rs | 3 + .../src/file/credentials.rs | 5 +- .../src/file/credentials/token.rs | 72 ++++-- lib/crowdb-access-iceberg/src/file/digest.rs | 26 ++- .../src/file/direct_location.rs | 191 ++++++++++++++++ .../src/file/directory.rs | 23 +- lib/crowdb-access-iceberg/src/file/key.rs | 16 +- lib/crowdb-access-iceberg/src/file/listing.rs | 9 +- .../src/file/listing/repository.rs | 18 +- .../src/file/location.rs | 47 +++- .../src/file/repository.rs | 51 +++-- .../src/file/write_intent.rs | 2 +- lib/crowdb-access-iceberg/src/gc/admission.rs | 4 +- lib/crowdb-access-iceberg/src/gc/fence.rs | 16 +- .../src/gc/proof/traversal.rs | 4 +- .../src/gc/repository.rs | 19 +- .../src/gc/worker/live.rs | 6 +- lib/crowdb-access-iceberg/src/key/storage.rs | 8 + .../src/namespace/drop_probe.rs | 2 +- lib/crowdb-access-iceberg/src/record.rs | 2 + .../src/record/envelope.rs | 38 +++- .../src/record/multipart/fields.rs | 11 +- .../src/record/storage_key.rs | 31 +++ lib/crowdb-access-iceberg/src/record/table.rs | 8 + .../src/record/write_intent.rs | 12 +- lib/crowdb-access-iceberg/src/table.rs | 2 +- lib/crowdb-access-iceberg/src/table/key.rs | 12 + .../src/table/lifecycle/completion.rs | 8 +- .../src/table/lifecycle/operation.rs | 1 + .../src/table/lifecycle/reservation.rs | 4 +- lib/crowdb-access-iceberg/src/table/list.rs | 2 +- lib/crowdb-access-iceberg/src/table/record.rs | 14 ++ .../src/table/repository.rs | 4 +- .../src/wire/credentials.rs | 3 + .../tests/commit_manifest_provenance_test.rs | 1 + .../tests/common/deletion_vector.rs | 1 + .../tests/common/file.rs | 1 + .../tests/common/manifest_entry.rs | 1 + .../tests/common/manifest_list.rs | 1 + .../tests/common/multipart.rs | 1 + .../tests/common/parquet_integers.rs | 1 + .../tests/common/parquet_metadata.rs | 1 + .../tests/common/parquet_nullable.rs | 1 + .../tests/common/projection.rs | 1 + .../tests/common/puffin.rs | 1 + .../tests/common/selected_parquet.rs | 1 + .../tests/common/store.rs | 12 +- .../tests/common/table_metadata.rs | 1 + .../tests/common/table_read.rs | 2 + .../tests/direct_file_location_test.rs | 143 ++++++++++++ .../tests/file_assembly_pipeline_test.rs | 1 + .../tests/file_assembly_test.rs | 1 + .../tests/file_avro_test.rs | 1 + .../tests/file_checkpoint_test.rs | 1 + .../tests/file_credentials_test.rs | 32 +++ .../tests/file_digest_test.rs | 1 + .../tests/file_directory_test.rs | 18 ++ .../tests/file_format_test.rs | 1 + .../tests/file_json_test.rs | 1 + .../tests/file_listing_test.rs | 2 + .../tests/file_location_test.rs | 1 + .../tests/file_record_test.rs | 1 + .../tests/file_scale_test.rs | 1 + .../tests/file_seal_test.rs | 1 + .../tests/file_staging_test.rs | 1 + .../tests/file_stream_test.rs | 1 + .../tests/gc_reachability_test.rs | 1 + .../tests/gc_tree_test.rs | 2 + .../tests/manifest_metadata_test.rs | 1 + .../tests/multipart_record_test.rs | 14 ++ .../tests/native_file_runtime_test.rs | 211 ++++++++++++++++++ .../tests/parquet_delete_encoding_test.rs | 1 + .../tests/parquet_nullable_test.rs | 1 + .../tests/parquet_position_delete_test.rs | 2 + .../tests/parquet_scalar_test.rs | 1 + .../tests/table_commit_publication_test.rs | 1 + .../tests/table_repository_test.rs | 96 ++++++++ .../tests/wire_credentials_test.rs | 6 + lib/crowdb-protocol/src/fbs/iceberg.fbs | 3 + 120 files changed, 1536 insertions(+), 175 deletions(-) create mode 100644 app/crowdb-access-server/src/iceberg/file_bucket_request.rs create mode 100644 app/crowdb-access-server/src/iceberg/file_http/bucket.rs create mode 100644 lib/crowdb-access-iceberg/src/file/direct_location.rs create mode 100644 lib/crowdb-access-iceberg/src/record/storage_key.rs create mode 100644 lib/crowdb-access-iceberg/tests/direct_file_location_test.rs create mode 100644 lib/crowdb-access-iceberg/tests/native_file_runtime_test.rs diff --git a/app/crowdb-access-server/src/iceberg.rs b/app/crowdb-access-server/src/iceberg.rs index cac8ad8d8..67d4866e6 100644 --- a/app/crowdb-access-server/src/iceberg.rs +++ b/app/crowdb-access-server/src/iceberg.rs @@ -5,9 +5,11 @@ mod connection; mod file_admission; mod file_auth; mod file_body; +mod file_bucket_request; mod file_complete; mod file_http; mod file_list_request; +pub use file_bucket_request::{parse_file_bucket, FileBucketRequest}; mod file_list_response; mod file_recovery; mod file_request; diff --git a/app/crowdb-access-server/src/iceberg/file_bucket_request.rs b/app/crowdb-access-server/src/iceberg/file_bucket_request.rs new file mode 100644 index 000000000..d36a61abf --- /dev/null +++ b/app/crowdb-access-server/src/iceberg/file_bucket_request.rs @@ -0,0 +1,43 @@ +use hyper::{Method, Uri}; + +use super::file_request::{decode, query, FileRequestError}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum FileBucketRequest { + Head(String), + Location(String), +} + +/// # Errors +/// Rejects unsupported bucket selectors and malformed transport encodings. +pub fn parse_file_bucket(method: &Method, uri: &Uri) -> Result, FileRequestError> { + if uri + .path_and_query() + .map_or(true, |value| value.as_str().len() > 8192) + { + return Err(FileRequestError::Invalid); + } + if uri.path().trim_end_matches('/').split('/').count() != 2 { + return Ok(None); + } + let path = decode(uri.path())?; + let bucket = path.strip_prefix('/').ok_or(FileRequestError::Invalid)?; + let bucket = bucket.strip_suffix('/').unwrap_or(bucket); + if bucket.is_empty() || bucket.contains('/') { + return Ok(None); + } + let mut fields = query(uri.query())?; + if method == Method::HEAD { + if !fields.is_empty() { + return Err(FileRequestError::Unsupported); + } + return Ok(Some(FileBucketRequest::Head(bucket.to_owned()))); + } + if method == Method::GET && fields.remove("location").as_deref() == Some("") { + if !fields.is_empty() { + return Err(FileRequestError::Unsupported); + } + return Ok(Some(FileBucketRequest::Location(bucket.to_owned()))); + } + Ok(None) +} diff --git a/app/crowdb-access-server/src/iceberg/file_http.rs b/app/crowdb-access-server/src/iceberg/file_http.rs index 8b0384118..e0e021de5 100644 --- a/app/crowdb-access-server/src/iceberg/file_http.rs +++ b/app/crowdb-access-server/src/iceberg/file_http.rs @@ -6,8 +6,9 @@ use crowdb_access_iceberg::catalog::{ CatalogError, CatalogLifecycle, CatalogRepository, CatalogStore, RootState, }; use crowdb_access_iceberg::file::{ - resolve_range, FileBlockStore, FileGrantError, FileGrantIssuer, FileOperation, FileRecord, - FileRepository, FileSealer, MultipartLister, MultipartPartStore, MultipartRepository, RangeError, + resolve_range, DirectFileLocations, FileBlockStore, FileGrantError, FileGrantIssuer, FileOperation, + FileRecord, FileRepository, FileSealer, MultipartLister, MultipartPartStore, MultipartRepository, + RangeError, }; use crowdb_access_iceberg::gc::{GcRepository, GcStore}; use crowdb_access_iceberg::key::OperationId; @@ -28,6 +29,7 @@ use super::file_response::{FileS3ErrorCode, MultipartResponses}; use super::file_upload::FileUploadBudget; use crate::upload_flow::body_encoding::UploadBody; +mod bucket; mod delete; mod listing; mod metrics; @@ -45,6 +47,7 @@ pub(super) struct FileHttp { lister: MultipartLister, blocks: Arc, issuer: FileGrantIssuer, + direct_locations: DirectFileLocations, responses: FileResponseBudget, uploads: FileUploadBudget, small_threshold_exclusive: usize, @@ -94,6 +97,8 @@ impl FileHttp { return Err(FileGrantError::Invalid); } Ok(Self { + direct_locations: DirectFileLocations::new(1, std::collections::BTreeMap::from([(1, secret)])) + .map_err(|_| FileGrantError::Invalid)?, list_tokens: crowdb_access_iceberg::file::FileListTokens::new(secret) .map_err(|_| FileGrantError::Invalid)?, repository: FileRepository::new(store.clone()), @@ -139,6 +144,43 @@ impl FileHttp { Ok(()) } + fn receive_provider( + &self, + request: &mut Request, + operation: FileOperation, + streaming: bool, + ) -> Option> { + let declared_receive = request + .headers() + .get("x-amz-decoded-content-length") + .or_else(|| request.headers().get(CONTENT_LENGTH)) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()); + if matches!(operation, FileOperation::Put | FileOperation::UploadPart) + && !streaming + && !request + .headers() + .get("content-encoding") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.split(',').any(|part| part.trim() == "aws-chunked")) + { + self.native_allocator.as_ref().and_then(|allocator| { + if let Some(length) = declared_receive + .and_then(|length| usize::try_from(length).ok()) + .filter(|length| *length > 0 && *length < self.small_threshold_exclusive) + { + crate::http_receive::install_small_body_receive_provider(request, allocator, length) + } else { + Some(crate::http_receive::install_native_body_receive_provider( + request, allocator, + )) + } + }) + } else { + None + } + } + pub(super) async fn dispatch( self: &Arc, catalog: &CatalogRepository, @@ -161,6 +203,13 @@ impl FileHttp { mut request: Request, request_timeout: Duration, ) -> Result, FileS3ErrorCode> { + if let Some(probe) = super::file_bucket_request::parse_file_bucket(request.method(), request.uri()) + .map_err(request_error)? + { + return self + .bucket_probe(catalog, &request, &probe, request_timeout) + .await; + } if request.method() == Method::POST && matches!(request.uri().query(), Some("delete" | "delete=")) { return self.delete_objects(catalog, request, request_timeout).await; } @@ -194,38 +243,8 @@ impl FileHttp { let admission = FileTransferAdmission::authorize(&grant, &file_request, self.limits, session.as_ref(), now_ms) .map_err(admission_error)?; - let declared_receive = request - .headers() - .get("x-amz-decoded-content-length") - .or_else(|| request.headers().get(CONTENT_LENGTH)) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.parse::().ok()); - let native_receiver = if matches!( - file_request.operation, - FileOperation::Put | FileOperation::UploadPart - ) && streaming.is_none() - && !request - .headers() - .get("content-encoding") - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value.split(',').any(|part| part.trim() == "aws-chunked")) - { - self.native_allocator.as_ref().and_then(|allocator| { - if let Some(length) = declared_receive - .and_then(|length| usize::try_from(length).ok()) - .filter(|length| *length > 0 && *length < self.small_threshold_exclusive) - { - crate::http_receive::install_small_body_receive_provider(&mut request, allocator, length) - } else { - Some(crate::http_receive::install_native_body_receive_provider( - &mut request, - allocator, - )) - } - }) - } else { - None - }; + let native_receiver = + self.receive_provider(&mut request, file_request.operation, streaming.is_some()); match file_request.operation { FileOperation::Head | FileOperation::Get => { self.read(&file_request, &request, root.context, &admission).await @@ -349,12 +368,19 @@ impl FileHttp { context: crowdb_access_iceberg::catalog::CatalogContext, admission: &FileTransferAdmission, ) -> Result, FileS3ErrorCode> { - let record = self - .repository - .load(context, &file_request.location) - .await - .map_err(catalog_error)? - .ok_or(FileS3ErrorCode::NoSuchKey)?; + let record = match self + .direct_locations + .resolve(&file_request.location) + .map_err(|_| FileS3ErrorCode::AccessDenied)? + { + Some(record) => record, + None => self + .repository + .load(context, &file_request.location) + .await + .map_err(catalog_error)? + .ok_or(FileS3ErrorCode::NoSuchKey)?, + }; let range = match request .headers() .get_all(RANGE) diff --git a/app/crowdb-access-server/src/iceberg/file_http/bucket.rs b/app/crowdb-access-server/src/iceberg/file_http/bucket.rs new file mode 100644 index 000000000..c6a6ec627 --- /dev/null +++ b/app/crowdb-access-server/src/iceberg/file_http/bucket.rs @@ -0,0 +1,67 @@ +use std::time::Duration; + +use crowdb_access_iceberg::catalog::{CatalogLifecycle, CatalogRepository, RootState}; +use crowdb_access_iceberg::file::{FileOperation, TableLocation}; +use crowdb_access_s3::auth::RawAuthRequest; +use hyper::header::CONTENT_LENGTH; +use hyper::{body::Incoming, Request, Response}; + +use super::{ + authenticate_file_transfer, catalog_error, now_ms, set_header, FileHttp, FileS3ErrorCode, IcebergBody, +}; +use crate::iceberg::FileBucketRequest; + +impl FileHttp { + pub(super) async fn bucket_probe( + &self, + catalog: &CatalogRepository, + request: &Request, + probe: &FileBucketRequest, + request_timeout: Duration, + ) -> Result, FileS3ErrorCode> { + let (root, authority) = catalog.status().await.map_err(catalog_error)?; + if root.state != RootState::Ready + || authority.lifecycle != CatalogLifecycle::Ready + || request_timeout.is_zero() + || request_timeout > Duration::from_millis(authority.admission_bounds.request_ms) + { + return Err(FileS3ErrorCode::SlowDown); + } + let (grant, streaming) = authenticate_file_transfer( + &self.issuer, + root.context, + RawAuthRequest::from_parts(request.method(), request.uri(), request.headers()), + &self.region, + now_ms()?, + ) + .map_err(|_| FileS3ErrorCode::AccessDenied)?; + if streaming.is_some() || !grant.operations.allows(FileOperation::Head) { + return Err(FileS3ErrorCode::AccessDenied); + } + let table = TableLocation { + catalog: grant.context.catalog, + table: grant.table, + storage_namespace: grant.storage_namespace, + }; + let (bucket, location) = match probe { + FileBucketRequest::Head(bucket) => (bucket, false), + FileBucketRequest::Location(bucket) => (bucket, true), + }; + if *bucket != table.bucket() { + return Err(FileS3ErrorCode::AccessDenied); + } + let bytes = if location { + format!("{}", self.region).into_bytes() + } else { + Vec::new() + }; + let length = bytes.len(); + let mut response = Response::new(IcebergBody::new(bytes)); + set_header(&mut response, CONTENT_LENGTH, &length.to_string())?; + response.headers_mut().insert( + "x-amz-bucket-region", + hyper::header::HeaderValue::from_str(&self.region).map_err(|_| FileS3ErrorCode::InternalError)?, + ); + Ok(response) + } +} diff --git a/app/crowdb-access-server/src/iceberg/file_http/delete.rs b/app/crowdb-access-server/src/iceberg/file_http/delete.rs index 425f072b0..26bca9b71 100644 --- a/app/crowdb-access-server/src/iceberg/file_http/delete.rs +++ b/app/crowdb-access-server/src/iceberg/file_http/delete.rs @@ -9,7 +9,7 @@ use crowdb_access_iceberg::gc::{ avro_links, metadata_links, AvroMarkCursor, AvroMarkLimits, GcLimits, ReachableFile, ReachableKind, }; use crowdb_access_iceberg::record::StorageRecord; -use crowdb_access_iceberg::table::head_key; +use crowdb_access_iceberg::table::storage_head_key; use crowdb_access_s3::auth::RawAuthRequest; use http_body_util::BodyExt; use hyper::body::Incoming; @@ -69,6 +69,7 @@ impl FileHttp { return Err(FileS3ErrorCode::AccessDenied); } let bucket = TableLocation { + storage_namespace: grant.storage_namespace, catalog: root.context.catalog, table: grant.table, } @@ -111,6 +112,7 @@ impl FileHttp { return Err(FileS3ErrorCode::InvalidRequest); } let table = TableLocation { + storage_namespace: grant.storage_namespace, catalog: root.context.catalog, table: grant.table, }; @@ -223,7 +225,7 @@ impl FileHttp { context: CatalogContext, table: TableLocation, ) -> Result, FileS3ErrorCode> { - let key = head_key(context.catalog, table.table); + let key = storage_head_key(table); let Some(value) = self .store .get(&key.encode().map_err(|_| FileS3ErrorCode::InternalError)?) diff --git a/app/crowdb-access-server/src/iceberg/file_list_request.rs b/app/crowdb-access-server/src/iceberg/file_list_request.rs index d7c59c7d7..cfec9a149 100644 --- a/app/crowdb-access-server/src/iceberg/file_list_request.rs +++ b/app/crowdb-access-server/src/iceberg/file_list_request.rs @@ -2,6 +2,7 @@ use super::file_request::{decode, query, FileRequestError}; use crowdb_access_iceberg::file::{FileListRequest, TableLocation}; +use crowdb_access_iceberg::key::StorageTablePath; use hyper::{Method, Uri}; /// Parses `ListObjectsV2` independently of exact-object `FileIO`. @@ -28,7 +29,12 @@ pub fn parse_file_list(method: &Method, uri: &Uri) -> Result FileOperation::DeleteObject, None => return Err(FileRequestError::Unsupported), }; + if location.table().storage_namespace.is_some() + && matches!(operation, FileOperation::Put | FileOperation::CreateMultipart) + && (location.relative_key() == "f" || location.relative_key().starts_with("f/")) + { + return Err(FileRequestError::Invalid); + } Ok(Self { location, operation, diff --git a/app/crowdb-access-server/src/iceberg/table_credentials.rs b/app/crowdb-access-server/src/iceberg/table_credentials.rs index c566420ea..cbad30eee 100644 --- a/app/crowdb-access-server/src/iceberg/table_credentials.rs +++ b/app/crowdb-access-server/src/iceberg/table_credentials.rs @@ -38,6 +38,7 @@ pub(super) struct TableFileConfig { #[derive(Clone, Copy)] pub(super) struct TableFileTarget<'a> { + pub(super) storage_namespace: Option, pub(super) namespace: &'a NamespaceIdentifier, pub(super) name: &'a str, pub(super) table: TableId, @@ -74,6 +75,7 @@ impl TableFileConfig { access.context, access.authority, FileDelegationTarget { + storage_namespace: target.storage_namespace, table: target.table, format_version: target.version, staged: target.staged, @@ -205,14 +207,19 @@ impl TableCredentials { .await .map_err(|_| service_unavailable())?; let mut ttl_ms = 900_000; - let (table, version, staged) = if let Some(selected) = + let (table, version, staged, storage_namespace) = if let Some(selected) = selected.filter(|selected| selector.map_or(true, |table| table == selected.head.table)) { self.tables .ensure_current(context, &selected) .await .map_err(|_| service_unavailable())?; - (selected.head.table, selected.head.format_version, false) + ( + selected.head.table, + selected.head.format_version, + false, + selected.head.metadata_location.table().storage_namespace, + ) } else { let table = selector.ok_or_else(missing_table)?; let identity = OperationId::from_bytes(table.as_bytes()).map_err(|_| bad_request())?; @@ -249,13 +256,19 @@ impl TableCredentials { { return Err(service_unavailable()); } - (table, operation.candidate.format_version, true) + ( + table, + operation.candidate.format_version, + true, + operation.candidate.metadata_location.table().storage_namespace, + ) }; self.issue_response( repository, context, principal, FileDelegationTarget { + storage_namespace, table, format_version: version, staged, diff --git a/app/crowdb-access-server/src/iceberg/table_read.rs b/app/crowdb-access-server/src/iceberg/table_read.rs index 5a55ecaf2..1300f38ec 100644 --- a/app/crowdb-access-server/src/iceberg/table_read.rs +++ b/app/crowdb-access-server/src/iceberg/table_read.rs @@ -207,6 +207,7 @@ impl TableHttp { config.append( &mut bytes, super::table_credentials::TableFileTarget { + storage_namespace: head.metadata_location.table().storage_namespace, namespace, name, table: head.table, diff --git a/app/crowdb-access-server/src/iceberg/table_write.rs b/app/crowdb-access-server/src/iceberg/table_write.rs index 6c607fe09..717c50cc4 100644 --- a/app/crowdb-access-server/src/iceberg/table_write.rs +++ b/app/crowdb-access-server/src/iceberg/table_write.rs @@ -258,6 +258,7 @@ impl TableWrites { config.append( &mut body, super::table_credentials::TableFileTarget { + storage_namespace: table.storage_namespace, namespace: &namespace, name: &name, table: table.table, diff --git a/app/crowdb-access-server/tests/common/iceberg_file_blocks.rs b/app/crowdb-access-server/tests/common/iceberg_file_blocks.rs index c9f893c16..b59d26388 100644 --- a/app/crowdb-access-server/tests/common/iceberg_file_blocks.rs +++ b/app/crowdb-access-server/tests/common/iceberg_file_blocks.rs @@ -25,6 +25,7 @@ impl TestFileBlocks { FileRecord { file: FileId::random(), location: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), } diff --git a/app/crowdb-access-server/tests/common/iceberg_table_http.rs b/app/crowdb-access-server/tests/common/iceberg_table_http.rs index 1d112e560..65c4205c0 100644 --- a/app/crowdb-access-server/tests/common/iceberg_table_http.rs +++ b/app/crowdb-access-server/tests/common/iceberg_table_http.rs @@ -158,6 +158,7 @@ impl TestTableHttp { pub async fn install_with_location(&self, name: &str, trailing_slash: bool) -> (TableHead, Vec) { let location = TableLocation { + storage_namespace: None, catalog: self.context.catalog, table: TableId::random(), }; @@ -218,6 +219,7 @@ impl TestTableHttp { self.put( &name_key(head.catalog, head.namespace, name).unwrap(), &StorageRecord::TableMapping(TableMapping { + storage_namespace: None, catalog: head.catalog, namespace: head.namespace, name: name.into(), diff --git a/app/crowdb-access-server/tests/common/iceberg_upload.rs b/app/crowdb-access-server/tests/common/iceberg_upload.rs index f61d6f917..3debf5043 100644 --- a/app/crowdb-access-server/tests/common/iceberg_upload.rs +++ b/app/crowdb-access-server/tests/common/iceberg_upload.rs @@ -17,6 +17,7 @@ use sha2::{Digest, Sha256}; pub fn owner() -> FileIdentity { FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/app/crowdb-access-server/tests/iceberg_file_admission_test.rs b/app/crowdb-access-server/tests/iceberg_file_admission_test.rs index 2195ea810..1f8bad7c3 100644 --- a/app/crowdb-access-server/tests/iceberg_file_admission_test.rs +++ b/app/crowdb-access-server/tests/iceberg_file_admission_test.rs @@ -20,6 +20,7 @@ use hyper::Method; fn grant(operations: &[FileOperation]) -> FileGrant { FileGrant { + storage_namespace: None, context: CatalogContext { catalog: CatalogId::random(), activation_epoch: 1, @@ -37,6 +38,7 @@ fn grant(operations: &[FileOperation]) -> FileGrant { fn location(grant: &FileGrant) -> crowdb_access_iceberg::file::FileLocation { TableLocation { + storage_namespace: None, catalog: grant.context.catalog, table: grant.table, } @@ -112,6 +114,7 @@ fn authorization_intersects_grant_operation_scope_and_service_limits() { )); let mut wrong = get_request.clone(); wrong.location = TableLocation { + storage_namespace: None, catalog: grant.context.catalog, table: TableId::random(), } diff --git a/app/crowdb-access-server/tests/iceberg_file_auth_test.rs b/app/crowdb-access-server/tests/iceberg_file_auth_test.rs index 9e7138530..b56f94a90 100644 --- a/app/crowdb-access-server/tests/iceberg_file_auth_test.rs +++ b/app/crowdb-access-server/tests/iceberg_file_auth_test.rs @@ -18,6 +18,7 @@ fn credentials() -> (FileGrantIssuer, FileCredentials) { let issuer = FileGrantIssuer::new([42; 32], 60_000).unwrap(); let credentials = issuer .issue(FileGrant { + storage_namespace: None, context: CatalogContext { catalog: CatalogId::random(), activation_epoch: 1, diff --git a/app/crowdb-access-server/tests/iceberg_file_recovery_test.rs b/app/crowdb-access-server/tests/iceberg_file_recovery_test.rs index f05b9620a..eed11c7aa 100644 --- a/app/crowdb-access-server/tests/iceberg_file_recovery_test.rs +++ b/app/crowdb-access-server/tests/iceberg_file_recovery_test.rs @@ -112,6 +112,7 @@ async fn setup_with_bounds_and_file_limit( let started = now_ms(); let credentials = issuer .issue(FileGrant { + storage_namespace: None, context, table: table.table, principal: [7; 32], diff --git a/app/crowdb-access-server/tests/iceberg_file_request_test.rs b/app/crowdb-access-server/tests/iceberg_file_request_test.rs index f333dd98b..3fad6d857 100644 --- a/app/crowdb-access-server/tests/iceberg_file_request_test.rs +++ b/app/crowdb-access-server/tests/iceberg_file_request_test.rs @@ -6,11 +6,67 @@ use hyper::Method; fn table() -> TableLocation { TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), } } +#[test] +fn bucket_probes_are_distinct_from_unscoped_object_lists() { + use crowdb_access_server::iceberg::{parse_file_bucket, FileBucketRequest}; + assert_eq!( + parse_file_bucket(&Method::HEAD, &"/iceberg".parse().unwrap()).unwrap(), + Some(FileBucketRequest::Head("iceberg".into())) + ); + assert_eq!( + parse_file_bucket(&Method::GET, &"/iceberg?location".parse().unwrap()).unwrap(), + Some(FileBucketRequest::Location("iceberg".into())) + ); + assert!( + parse_file_bucket(&Method::GET, &"/iceberg?list-type=2".parse().unwrap()) + .unwrap() + .is_none() + ); + assert!(parse_file_bucket(&Method::GET, &"/".parse().unwrap()) + .unwrap() + .is_none()); + assert!(parse_file_bucket(&Method::GET, &"/iceberg?location&prefix=".parse().unwrap()).is_err()); +} + +#[test] +fn three_id_http_paths_and_list_prefixes_use_the_fixed_bucket() { + let table = TableLocation { + storage_namespace: Some(crowdb_access_iceberg::key::NamespaceId::random()), + ..table() + }; + let object = format!("/iceberg/{}metadata/v1.json", table.object_prefix()); + let request = FileRequest::parse(&Method::GET, &object.parse().unwrap()).unwrap(); + assert_eq!(request.location.table(), table); + let list = format!("/iceberg?list-type=2&prefix={}metadata%2F", table.object_prefix()); + assert_eq!( + parse_file_list(&Method::GET, &list.parse().unwrap()) + .unwrap() + .unwrap() + .table, + table + ); + for prefix in [ + String::new(), + format!("{}/", table.catalog), + format!("{}/{}/", table.catalog, table.storage_namespace.unwrap()), + ] { + assert!(parse_file_list( + &Method::GET, + &format!("/iceberg?list-type=2&prefix={prefix}").parse().unwrap() + ) + .is_err()); + } + let reserved = format!("/iceberg/{}f/v1/name.token.parquet", table.object_prefix()); + assert!(FileRequest::parse(&Method::PUT, &reserved.parse().unwrap()).is_err()); + assert!(FileRequest::parse(&Method::GET, &reserved.parse().unwrap()).is_ok()); +} + #[test] fn native_list_requests_preserve_table_prefix_and_validate_all_selectors() { let table = table(); diff --git a/app/crowdb-access-server/tests/iceberg_file_response_test.rs b/app/crowdb-access-server/tests/iceberg_file_response_test.rs index 59d36f71e..5902af450 100644 --- a/app/crowdb-access-server/tests/iceberg_file_response_test.rs +++ b/app/crowdb-access-server/tests/iceberg_file_response_test.rs @@ -15,6 +15,7 @@ use sha2::{Digest, Sha256}; fn session() -> MultipartSession { let table = TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }; diff --git a/app/crowdb-access-server/tests/iceberg_table_acceptance_test.rs b/app/crowdb-access-server/tests/iceberg_table_acceptance_test.rs index 919215108..b64492d88 100644 --- a/app/crowdb-access-server/tests/iceberg_table_acceptance_test.rs +++ b/app/crowdb-access-server/tests/iceberg_table_acceptance_test.rs @@ -95,6 +95,7 @@ fn mapping(test: &TestTableHttp, head: &TableHead, name: &str, state: TableMappi test.put( &name_key(head.catalog, head.namespace, name).unwrap(), &StorageRecord::TableMapping(TableMapping { + storage_namespace: None, catalog: head.catalog, namespace: head.namespace, name: name.into(), diff --git a/doc/backlog/R234-access-iceberg-native-storage-layout.md b/doc/backlog/R234-access-iceberg-native-storage-layout.md index eac7fff5a..636e3b1a8 100644 --- a/doc/backlog/R234-access-iceberg-native-storage-layout.md +++ b/doc/backlog/R234-access-iceberg-native-storage-layout.md @@ -197,8 +197,9 @@ Defined edge behavior: Existing logical-URL resolution may still reject its tombstone. Direct GET, HEAD and Range GET must not use that resolver or add per-file state queries. Grant signature/scope/expiry validation remains independent of file lookup. - The native read API must avoid chunk-KV location resolution on the direct - path; GC and storage reads must prevent reclaimed extents from aliasing new + The direct path skips Iceberg file-location/lifecycle KV queries and retains + native chunk-layer layout resolution for cold reads, repair and migration. + GC and storage reads must prevent reclaimed extents from aliasing new data, including after restart and physical space reuse. - Uses the standard Iceberg URI requirement and Arrow S3 behavior as compatibility references, not as a requirement to introduce an external S3 @@ -286,8 +287,9 @@ Defined edge behavior: and recreated, assert live lookup still returns the frozen storage path and the files survive. Invariant: ICEBERG-PATH-1. E2E test. - Given files with multiple extents, when reading footer and boundary ranges, - assert correct bytes, bounded URLs and zero file-location, deletion-state - and chunk-KV resolution RPCs on the direct path. Invariants: ICEBERG-PATH-3 and + assert correct bytes, bounded URLs and zero Iceberg file-location and + file-deletion-state KV RPCs. Chunk layout resolution remains supported. + Invariants: ICEBERG-PATH-3 and ICEBERG-PATH-8. Integration test. - Given concurrent multipart completion/abort, same-length conflicting PUTs and conditional/range reads, when requests retry after response loss, assert diff --git a/doc/backlog/backlog.md b/doc/backlog/backlog.md index 6e487f84e..3f6e95ad5 100644 --- a/doc/backlog/backlog.md +++ b/doc/backlog/backlog.md @@ -45,8 +45,10 @@ live in the linked document; implementation follows ID directory, single file record, injection direct URL, large metadata objects and bounded table/metadata prefix listing. **Area:** access server / Iceberg / Chunk-KV. **Complexity:** High. - **Status:** Implementation started; native identity/URI/key codecs and storage - envelope dispatch verified. Runtime publication and direct reads pending. + **Status:** Implementation in progress; codecs, native logical file publication + and bounded prefix listing verified. Catalog activation, lifecycle keys and + direct reads pending. Direct reads skip Iceberg file metadata KV and retain + native chunk layout resolution. ## Chunk, Chunk-KV, and recovery diff --git a/doc/design/access-server/iceberge/design-crowdb-iceberg.md b/doc/design/access-server/iceberge/design-crowdb-iceberg.md index 352a2ae2f..a297af6c9 100644 --- a/doc/design/access-server/iceberge/design-crowdb-iceberg.md +++ b/doc/design/access-server/iceberge/design-crowdb-iceberg.md @@ -433,8 +433,9 @@ logical prefixes regardless of their native object size. Position-delete and deletion-vector path references always use the original canonical URL. Direct HEAD/GET/Range GET validate request permissions and locator authenticity -and read the native descriptor without file-location, file-tombstone or -chunk-KV resolution queries. A KV-resolved opaque token is not a direct +and read the native descriptor without Iceberg file-location or file-tombstone +KV queries. Chunk-layer layout resolution remains available for replica/EC +routing, cold reads, repair and migration. A KV-resolved opaque file token is not a direct locator. Large files use bounded chunk-resident directory traversal, not an unbounded URL. Native chunk routing preserves repair/failover without embedding transient replica addresses in URLs. Reclaimed identities and reused extents @@ -970,7 +971,8 @@ must inspect every item. - **ICE-I11 — Single file authority:** one filename-keyed record selects each published file; list and lifecycle use that record without secondary file indexes. - **ICE-I12 — Direct native read:** a direct URL supplies its immutable native - descriptor without file-location, file-state or chunk-KV resolution queries. + descriptor without Iceberg file-location or file-state KV queries; native + chunk layout resolution and repair retain their normal behavior. Logical deletion removes list visibility; physical GC ends byte availability. - **ICE-I13 — Bounded metadata scale:** large metadata bytes live in native objects, descriptors/head remain bounded, prefix listing reads bounded KV diff --git a/doc/working/plan-iceberg-storage-path.md b/doc/working/plan-iceberg-storage-path.md index f1b0c08f0..3257c9d06 100644 --- a/doc/working/plan-iceberg-storage-path.md +++ b/doc/working/plan-iceberg-storage-path.md @@ -15,9 +15,14 @@ Goal: make catalog, namespace, and table identities explicit in the Iceberg KV and FileIO path model, keep file bytes in the native CROWDB Iceberg storage, and preserve file locations across table rename and cross-namespace move. -Status: implementation started. Immutable storage identity, native URI codec and -version-isolated table-key codec and routed-store dispatch are implemented; runtime publication and direct -reads are not switched to the new layout yet. Verification is tracked below. +Status: implementation in progress. Native identity/URI/key codecs, storage +dispatch, logical file publication/resolution and bounded prefix listing are +implemented. File/directory/grant/name-mapping records preserve storage namespace. +Native head lookup/publication/rename/drop/GC fences and direct URL reads are +implemented. Catalog v2 activation, remaining workflow/GC/projection keys and +injection HTTP/loader/recovery integration remain pending. Direct reads skip +Iceberg file-location/state KV and preserve +the chunk layer's normal layout resolution, as confirmed in the chat. Verified foundation gates: @@ -26,6 +31,36 @@ Verified foundation gates: - `pixi run -- cargo fmt --all -- --check` passed. - `pixi run rs-lint` passed, including workspace all-target Clippy. +Verified runtime work: + +- `TableLocation`/`FileLocation` preserve native and legacy URLs; one native + `canonical_file_key` binds a record to the full path and filename. +- `FileRepository` publishes/resolves one native record and recovers publication + response loss. Prefix listing uses scanned records without per-file GETs; + a 1,200-object mixed-prefix fixture verifies ordered bounded pages. +- Grant v2, directory v2, digest checkpoint v2, multipart/write-intent owner + codecs and name mappings preserve storage namespace. Mapping/head bindings + retain it when the live namespace changes. +- `storage_head_key` is used by table selection, commits, creation publication, + rename/drop, namespace drop probes and GC fences. A native lookup/move test + verifies the head key and metadata URL remain unchanged across a live move. +- `DirectFileLocations` supplies authenticated, bounded versioned parquet + locators with a verification key ring. HTTP reads reconstruct their file + descriptor directly; tests cover multi-extent ranges, tampering, old-key + retention, publication response loss, logical deletion and physical removal + with no Iceberg file KV reads. `finalize_native_parquet` requires the caller's + durably selected owner/tree; an upload recovery journal/HTTP entry is pending. +- `FileGrantIssuer` retains at most 64 validated credentials in an ArcSwap + cache, rechecking context and time on hits. Expired grants and changed catalog + epochs fail closed. Requests still verify SigV4 and file scope individually. +- HTTP exact/list parsing supports three IDs and rejects broad list prefixes; + ordinary SDK PUT/multipart creation cannot occupy the reserved `f/` directory. +- Iceberg all-target tests, focused snapshot/multipart/mapping/direct tests, + HTTP request/admission tests, workspace Clippy and fmt passed. Credential + issuance now derives namespace from selected table/draft metadata. Fixed + bucket HeadBucket/region probes are wired; bucket parser tests pass. Native + catalog activation and full native lifecycle/GC/SDK flows remain unverified. + ## Target structure Iceberg storage owns the logical directory. Native chunk placement remains @@ -248,9 +283,10 @@ may use the logical path; direct mode requires random-readable native bytes. The current chunk reader calls `query_layout`/`query_chunk`; removing `FileRepository::resolve` does not by itself remove chunk metadata RPCs. -Measure file-location resolution, file-state checks, chunk-layout resolution -and data reads separately, including cold cache. The first three must cause -zero KV RPCs on the direct path; adding a hidden lookup fails acceptance. +Measure Iceberg file-location resolution, file-state checks, chunk-layout +resolution and data reads separately, including cold cache. The first two +must cause zero KV RPCs on the direct path. Native chunk layout resolution is +retained for cold routing, repair and migration. Stable chunk IDs/offsets do not imply immutable replica addresses. A direct chunk API must preserve normal repair/failover behavior and never embed transient host placement in file URLs. @@ -261,7 +297,7 @@ grant and locator can read retained bytes until GC. GET/HEAD/Range GET perform no file tombstone lookup, and need no per-file revocation cache. List, deletion, GC and recovery may use KV normally. Physical storage must reject reclaimed chunk identities/extents and prevent old locators reading newly allocated data; -check this at the storage boundary without a per-read chunk-KV query. +check this at the storage boundary without an Iceberg file-record lookup. The upload state machine is `Staging -> Published -> Deleted -> Reclaimed`, or `Staging -> Aborted -> Reclaimed`. Incomplete or aborted records never @@ -517,8 +553,8 @@ three-ID prefix and a matching grant. `app/crowdb-access-server/src/iceberg/`, `tools/load-tpch.py`. - [ ] **Enforce the direct chunk boundary**: introduce/extend the native range read interface accepting the authenticated full descriptor/root; instrument - file-location, file-state and chunk-KV queries separately. Implement the - missing path rather than falling back to `query_chunk` on a cache miss. + Iceberg file-location, file-state and chunk layout queries separately. + Retain `query_chunk` for native layout resolution, including cache misses. Exercise cold routing, multi-extent reads, repair, GC races and offset reuse. Files: `lib/crowdb-access-iceberg/src/file/blocks.rs`, `lib/crowdb-chunk-client/src/chunk/chunk_reader.rs`, @@ -689,7 +725,8 @@ Integration: cold/warm HEAD and footer ranges. Include multi-location files, directory-root boundary reads, chunk repair/failover and URI size limits. No zero-KV claim may be inferred from merely skipping `FileRepository::resolve`; assert zero - file-location, file-state and chunk-KV resolution RPCs for direct reads. + Iceberg file-location and file-state KV RPCs for direct reads. Normal chunk + layout resolution remains supported and is measured independently. - [ ] Test public-key order and delimiter grouping when internal logical names sort differently; test max-keys zero, empty pages and credential refresh. - [ ] Test suffix/open-ended/out-of-bounds ranges, ETag conditions, unknown diff --git a/e2e-test/tests/access_iceberg/file_http/capacity.rs b/e2e-test/tests/access_iceberg/file_http/capacity.rs index 6a9a55012..f284aa9a9 100644 --- a/e2e-test/tests/access_iceberg/file_http/capacity.rs +++ b/e2e-test/tests/access_iceberg/file_http/capacity.rs @@ -40,6 +40,7 @@ async fn full_simulated_disk_preserves_file_authority_then_recovers_after_compac let context = seed_catalog(&stack).await; let owner = FileIdentity { table: TableLocation { + storage_namespace: None, catalog: context.catalog, table: TableId::random(), }, diff --git a/e2e-test/tests/access_iceberg/file_http/capacity/gc.rs b/e2e-test/tests/access_iceberg/file_http/capacity/gc.rs index a43eb0a59..0cb60069e 100644 --- a/e2e-test/tests/access_iceberg/file_http/capacity/gc.rs +++ b/e2e-test/tests/access_iceberg/file_http/capacity/gc.rs @@ -14,6 +14,7 @@ pub(super) async fn seed_gc_workspace_task( ) { let store = stack.store().await; let table = TableLocation { + storage_namespace: None, catalog: context.catalog, table: TableId::random(), }; diff --git a/e2e-test/tests/access_iceberg/file_http/listing.rs b/e2e-test/tests/access_iceberg/file_http/listing.rs index 8c5400ef1..1f536437b 100644 --- a/e2e-test/tests/access_iceberg/file_http/listing.rs +++ b/e2e-test/tests/access_iceberg/file_http/listing.rs @@ -51,6 +51,7 @@ async fn native_list_objects_v2_serves_selected_files_and_rejects_foreign_scope( assert!(xml.contains(""), "{xml}"); assert!(xml.contains("data%2Fnested%2F"), "{xml}"); let foreign = crowdb_access_iceberg::file::TableLocation { + storage_namespace: None, catalog: table.catalog, table: crowdb_access_iceberg::key::TableId::random(), }; diff --git a/e2e-test/tests/access_iceberg/file_http/setup.rs b/e2e-test/tests/access_iceberg/file_http/setup.rs index d19dcdcf2..32af523f3 100644 --- a/e2e-test/tests/access_iceberg/file_http/setup.rs +++ b/e2e-test/tests/access_iceberg/file_http/setup.rs @@ -89,6 +89,7 @@ pub(super) async fn setup_with_bounds_and_file_limit( let started = now_ms(); let credentials = issuer .issue(FileGrant { + storage_namespace: None, context, table: table.table, principal: [7; 32], @@ -149,6 +150,7 @@ pub(super) async fn cleanup_client( client: client.client.clone(), credentials: issuer .issue(FileGrant { + storage_namespace: None, context, table: table.table, principal: [9; 32], diff --git a/e2e-test/tests/access_iceberg/gc/seed.rs b/e2e-test/tests/access_iceberg/gc/seed.rs index 983f95aa5..09a4273c6 100644 --- a/e2e-test/tests/access_iceberg/gc/seed.rs +++ b/e2e-test/tests/access_iceberg/gc/seed.rs @@ -60,6 +60,7 @@ pub(super) async fn seed_table( pub(super) async fn seed_head(store: &RoutedCatalogStore, context: CatalogContext) -> TableId { let table = TableId::random(); let location = TableLocation { + storage_namespace: None, catalog: context.catalog, table, }; @@ -122,6 +123,7 @@ pub(super) async fn seed_files( let file = FileRecord { file: FileId::random(), location: TableLocation { + storage_namespace: None, catalog: context.catalog, table, } diff --git a/e2e-test/tests/access_iceberg/native_files.rs b/e2e-test/tests/access_iceberg/native_files.rs index 9449e03cd..d10ef0345 100644 --- a/e2e-test/tests/access_iceberg/native_files.rs +++ b/e2e-test/tests/access_iceberg/native_files.rs @@ -106,6 +106,7 @@ async fn native_file_tree_publication_and_ranges_survive_catalog_storage_restart let context = seed_root(&stack).await; let owner = FileIdentity { table: TableLocation { + storage_namespace: None, catalog: context.catalog, table: TableId::random(), }, diff --git a/e2e-test/tests/access_s3/protocol_http.rs b/e2e-test/tests/access_s3/protocol_http.rs index 41d94ff81..518b731c9 100644 --- a/e2e-test/tests/access_s3/protocol_http.rs +++ b/e2e-test/tests/access_s3/protocol_http.rs @@ -99,6 +99,7 @@ async fn write_iceberg(iceberg_addr: SocketAddr, seeds: &[String]) { let context = repository.status().await.unwrap().0.context; let credentials = issuer .issue(FileGrant { + storage_namespace: None, context, table: table.table, principal: [7; 32], diff --git a/lib/crowdb-access-iceberg/Cargo.toml b/lib/crowdb-access-iceberg/Cargo.toml index a0aa10ef5..0807448f2 100644 --- a/lib/crowdb-access-iceberg/Cargo.toml +++ b/lib/crowdb-access-iceberg/Cargo.toml @@ -14,6 +14,7 @@ workspace = true test-util = [] [dependencies] +arc-swap = "1.9" async-trait = "0.1" base64 = "0.22" bincode = "1.3" diff --git a/lib/crowdb-access-iceberg/src/commit/create/operation.rs b/lib/crowdb-access-iceberg/src/commit/create/operation.rs index db9ad40e9..1a564a280 100644 --- a/lib/crowdb-access-iceberg/src/commit/create/operation.rs +++ b/lib/crowdb-access-iceberg/src/commit/create/operation.rs @@ -80,6 +80,7 @@ impl TableCreateOperation { #[must_use] pub fn mapping(&self, state: TableMappingState) -> TableMapping { TableMapping { + storage_namespace: self.candidate.metadata_location.table().storage_namespace, catalog: self.context.catalog, namespace: self.candidate.namespace, name: self.candidate.name.clone(), diff --git a/lib/crowdb-access-iceberg/src/commit/create/publisher/completion.rs b/lib/crowdb-access-iceberg/src/commit/create/publisher/completion.rs index 2e5caa74e..463242f7a 100644 --- a/lib/crowdb-access-iceberg/src/commit/create/publisher/completion.rs +++ b/lib/crowdb-access-iceberg/src/commit/create/publisher/completion.rs @@ -6,7 +6,7 @@ use crate::{ namespace::authority_key, operation::mutation_identity, record::StorageRecord, - table::{head_key, name_key, TableMappingState}, + table::{name_key, storage_head_key, TableMappingState}, }; impl TableCreator { @@ -16,7 +16,7 @@ impl TableCreator { self.check_admission(operation).await?; } let head = &operation.candidate; - let key = head_key(head.catalog, head.table).encode()?; + let key = storage_head_key(head.storage_location()).encode()?; let bytes = StorageRecord::TableHead(Box::new(head.clone())).encode()?; let result = self .store @@ -98,7 +98,7 @@ impl TableCreator { .map_err(CatalogError::from)?; } else if operation.phase == Phase::Complete { let head = &operation.candidate; - let key = head_key(head.catalog, head.table).encode()?; + let key = storage_head_key(head.storage_location()).encode()?; let before = StorageRecord::TableHead(Box::new(head.clone())).encode()?; let mut settled = head.clone(); settled.pending_operation = None; diff --git a/lib/crowdb-access-iceberg/src/commit/create/publisher/preparation.rs b/lib/crowdb-access-iceberg/src/commit/create/publisher/preparation.rs index ec182c9d8..ed439f677 100644 --- a/lib/crowdb-access-iceberg/src/commit/create/publisher/preparation.rs +++ b/lib/crowdb-access-iceberg/src/commit/create/publisher/preparation.rs @@ -98,6 +98,7 @@ fn target( staged: bool, ) -> Result { let table = TableLocation { + storage_namespace: None, catalog: request.context.catalog, table: if staged { TableId::from_bytes(request.identity.operation.as_bytes())? diff --git a/lib/crowdb-access-iceberg/src/commit/create/publisher/reservation.rs b/lib/crowdb-access-iceberg/src/commit/create/publisher/reservation.rs index b3508b1aa..f7df7c271 100644 --- a/lib/crowdb-access-iceberg/src/commit/create/publisher/reservation.rs +++ b/lib/crowdb-access-iceberg/src/commit/create/publisher/reservation.rs @@ -5,7 +5,7 @@ use crate::{ error::ValidationError, operation::mutation_identity, record::StorageRecord, - table::{head_key, name_key, TableMappingState}, + table::{name_key, storage_head_key, TableMappingState}, }; impl TableCreator { @@ -59,7 +59,7 @@ impl TableCreator { Box::pin(self.resume_with_budget(operation.context, existing.operation, budget)).await?; return Ok(()); } - let head_key = head_key(existing.catalog, existing.table); + let head_key = storage_head_key(existing.storage_location()); let value_head = self .store .get(&head_key.encode()?) diff --git a/lib/crowdb-access-iceberg/src/commit/direct.rs b/lib/crowdb-access-iceberg/src/commit/direct.rs index aa2a13cf7..3d58f9c4b 100644 --- a/lib/crowdb-access-iceberg/src/commit/direct.rs +++ b/lib/crowdb-access-iceberg/src/commit/direct.rs @@ -8,7 +8,8 @@ use crate::{ operation::{mutation_identity, RequestIdentity}, record::StorageRecord, table::{ - head_key, read_table_metadata_document, SelectedTable, TableHead, TableLifecycle, TableMetadataError, + read_table_metadata_document, storage_head_key, SelectedTable, TableHead, TableLifecycle, + TableMetadataError, }, }; @@ -115,7 +116,7 @@ pub async fn publish_direct_commit( } Err(error) => return Err(error.into()), } - let key = head_key(before.catalog, before.table).encode()?; + let key = storage_head_key(before.storage_location()).encode()?; let expected = StorageRecord::TableHead(Box::new(before.clone())).encode()?; let value = StorageRecord::TableHead(Box::new(candidate.clone())).encode()?; let outcome = store @@ -132,7 +133,7 @@ pub async fn publish_direct_commit( CasOutcome::Conflict(Some(observed)) if observed.bytes == value => Ok(body), CasOutcome::Conflict(Some(observed)) => { let StorageRecord::TableHead(current) = - StorageRecord::decode(&head_key(before.catalog, before.table), &observed.bytes)? + StorageRecord::decode(&storage_head_key(before.storage_location()), &observed.bytes)? else { return Err(ValidationError::Record.into()); }; diff --git a/lib/crowdb-access-iceberg/src/commit/files/fence.rs b/lib/crowdb-access-iceberg/src/commit/files/fence.rs index 4d800e678..b555a8421 100644 --- a/lib/crowdb-access-iceberg/src/commit/files/fence.rs +++ b/lib/crowdb-access-iceberg/src/commit/files/fence.rs @@ -4,7 +4,7 @@ use crate::{ catalog::{CatalogContext, CatalogError, CatalogStore}, commit::{PriorManifestSource, TableCreateJournal, TableCreateOperation}, record::StorageRecord, - table::{head_key, name_key, TableMappingState, TableRepository}, + table::{name_key, storage_head_key, TableMappingState, TableRepository}, }; pub(super) enum CandidateFence { @@ -47,7 +47,7 @@ impl CandidateFence { if StorageRecord::decode(&key, &value.bytes)? != StorageRecord::TableMapping(mapping) { return Err(CatalogError::Conflict); } - let head = head_key(context.catalog, operation.candidate.table).encode()?; + let head = storage_head_key(operation.candidate.storage_location()).encode()?; if store.get(&head).await?.is_some() || journal .load(context, operation.identity.operation) diff --git a/lib/crowdb-access-iceberg/src/commit/files/snapshots.rs b/lib/crowdb-access-iceberg/src/commit/files/snapshots.rs index 9024509e0..7504ab16b 100644 --- a/lib/crowdb-access-iceberg/src/commit/files/snapshots.rs +++ b/lib/crowdb-access-iceberg/src/commit/files/snapshots.rs @@ -48,7 +48,7 @@ impl CandidateFileSource { { return Err(Error::Binding); } - self.validate_snapshot_set(Some(prior), &limits).await + Box::pin(self.validate_snapshot_set(Some(prior), &limits)).await } /// Validates all initial snapshots, including parent-child DV checks within a staged transaction. @@ -61,7 +61,7 @@ impl CandidateFileSource { if self.fence.prior().is_some() { return Err(Error::Binding); } - self.validate_snapshot_set(None, &limits).await + Box::pin(self.validate_snapshot_set(None, &limits)).await } async fn validate_snapshot_set( diff --git a/lib/crowdb-access-iceberg/src/commit/journal.rs b/lib/crowdb-access-iceberg/src/commit/journal.rs index 8fa0450c5..0a737073e 100644 --- a/lib/crowdb-access-iceberg/src/commit/journal.rs +++ b/lib/crowdb-access-iceberg/src/commit/journal.rs @@ -7,7 +7,7 @@ use crate::{ key::{CatalogScope, IcebergKey, OperationId}, operation::{mutation_identity, PayloadStore}, record::StorageRecord, - table::head_key, + table::storage_head_key, }; /// CAS journal for update intents; it neither validates files nor mutates table heads. @@ -147,7 +147,7 @@ impl TableCommitJournal { operation: &TableCommitOperation, next: TableCommitPhase, ) -> Result<(), CatalogError> { - let key = head_key(operation.before.catalog, operation.before.table); + let key = storage_head_key(operation.before.storage_location()); let value = self.store.get(&key.encode()?).await?.ok_or(CatalogError::Busy)?; let StorageRecord::TableHead(head) = StorageRecord::decode(&key, &value.bytes)? else { return Err(ValidationError::Record.into()); diff --git a/lib/crowdb-access-iceberg/src/commit/publication/completion.rs b/lib/crowdb-access-iceberg/src/commit/publication/completion.rs index b0f4c890c..d342a3c60 100644 --- a/lib/crowdb-access-iceberg/src/commit/publication/completion.rs +++ b/lib/crowdb-access-iceberg/src/commit/publication/completion.rs @@ -6,7 +6,7 @@ use crate::{ file::FileRepository, operation::{mutation_identity, PayloadStore, MAX_PAYLOAD_BYTES}, record::StorageRecord, - table::{head_key, read_table_metadata_document, SelectedTable, TableMetadataLimits}, + table::{read_table_metadata_document, storage_head_key, SelectedTable, TableMetadataLimits}, }; impl Publisher { @@ -14,7 +14,7 @@ impl Publisher { &self, operation: &TableCommitOperation, ) -> Result, Error> { - let key = head_key(operation.before.catalog, operation.before.table); + let key = storage_head_key(operation.before.storage_location()); let value = self .store .get(&key.encode()?) @@ -85,7 +85,7 @@ impl Publisher { let candidate = operation.candidate.as_ref().ok_or(ValidationError::Record)?; let prepared_body = self.success_body(operation).await?; self.current(operation).await?; - let key = head_key(candidate.catalog, candidate.table).encode()?; + let key = storage_head_key(candidate.storage_location()).encode()?; let before = StorageRecord::TableHead(Box::new(operation.before.clone())).encode()?; let after = StorageRecord::TableHead(Box::new(candidate.clone())).encode()?; let result = self @@ -155,7 +155,7 @@ impl Publisher { let candidate = operation.candidate.as_ref().ok_or(ValidationError::Record)?; let mut settled = candidate.clone(); settled.pending_operation = None; - let key = head_key(candidate.catalog, candidate.table).encode()?; + let key = storage_head_key(candidate.storage_location()).encode()?; let before = StorageRecord::TableHead(Box::new(candidate.clone())).encode()?; let after = StorageRecord::TableHead(Box::new(settled)).encode()?; self.store diff --git a/lib/crowdb-access-iceberg/src/file.rs b/lib/crowdb-access-iceberg/src/file.rs index d56f9a776..a5e8e8313 100644 --- a/lib/crowdb-access-iceberg/src/file.rs +++ b/lib/crowdb-access-iceberg/src/file.rs @@ -7,6 +7,7 @@ mod content; mod credentials; mod deletion_vector; mod digest; +mod direct_location; mod directory; mod format; mod json; @@ -56,12 +57,13 @@ pub use deletion_vector::{ DeletionVectorPositions, DeletionVectorReference, DeletionVectorStats, }; pub use digest::FileDigest; +pub use direct_location::DirectFileLocations; pub use directory::{ChunkDirectory, ChunkEntry, FileIdentity, MAX_DIRECTORY_ENTRIES}; pub use format::{ probe_orc_footer, probe_parquet_footer, probe_puffin_footer, FormatProbeError, OrcFooter, PuffinFooter, }; pub use json::{JsonSealError, JsonSealer}; -pub use key::{file_key, location_key}; +pub use key::{canonical_file_key, file_key, location_key}; pub(crate) use location::validate_relative_key; pub use location::{FileLocation, TableLocation, MAX_OBJECT_KEY_BYTES}; pub use multipart::{ diff --git a/lib/crowdb-access-iceberg/src/file/blocks.rs b/lib/crowdb-access-iceberg/src/file/blocks.rs index 880878bb8..fbf9e420d 100644 --- a/lib/crowdb-access-iceberg/src/file/blocks.rs +++ b/lib/crowdb-access-iceberg/src/file/blocks.rs @@ -187,6 +187,9 @@ impl FileBlockStore for NativeFileBlocks { } let mut key = b"iceberg-file-block-v1".to_vec(); key.extend_from_slice(owner.table.catalog.as_bytes()); + if let Some(namespace) = owner.table.storage_namespace { + key.extend_from_slice(namespace.as_bytes()); + } key.extend_from_slice(owner.table.table.as_bytes()); key.extend_from_slice(owner.file.as_bytes()); let mut writer = self.client.prepare_small_write_for_key(bytes.len(), &key).await?; diff --git a/lib/crowdb-access-iceberg/src/file/credentials.rs b/lib/crowdb-access-iceberg/src/file/credentials.rs index 75ffe08ea..42bcbc040 100644 --- a/lib/crowdb-access-iceberg/src/file/credentials.rs +++ b/lib/crowdb-access-iceberg/src/file/credentials.rs @@ -1,5 +1,5 @@ use crate::catalog::CatalogContext; -use crate::key::{OperationId, TableId}; +use crate::key::{NamespaceId, OperationId, TableId}; use super::FileLocation; @@ -53,6 +53,7 @@ impl FileOperations { #[derive(Clone, Debug, Eq, PartialEq)] pub struct FileGrant { + pub storage_namespace: Option, pub context: CatalogContext, pub table: TableId, pub principal: [u8; 32], @@ -76,6 +77,7 @@ impl FileGrant { ) -> Result<(), FileGrantError> { if location.table().catalog != self.context.catalog || location.table().table != self.table + || location.table().storage_namespace != self.storage_namespace || !self.operations.allows(operation) { return Err(FileGrantError::Forbidden); @@ -117,6 +119,7 @@ pub enum FileGrantError { Bounds, } +#[derive(Clone)] pub struct FileCredentials { grant: FileGrant, access_key_id: String, diff --git a/lib/crowdb-access-iceberg/src/file/credentials/token.rs b/lib/crowdb-access-iceberg/src/file/credentials/token.rs index de71c69e5..1932900d1 100644 --- a/lib/crowdb-access-iceberg/src/file/credentials/token.rs +++ b/lib/crowdb-access-iceberg/src/file/credentials/token.rs @@ -1,11 +1,13 @@ +use arc_swap::ArcSwap; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; use data_encoding::BASE32_NOPAD; use hmac::{Hmac, Mac}; use sha2::Sha256; +use std::{collections::BTreeMap, sync::Arc}; use subtle::ConstantTimeEq; use crate::catalog::CatalogContext; -use crate::key::{CatalogId, OperationId, TableId}; +use crate::key::{CatalogId, NamespaceId, OperationId, TableId}; use super::{FileCredentials, FileGrant, FileGrantError, FileOperations}; @@ -17,6 +19,7 @@ type Signer = Hmac; pub struct FileGrantIssuer { key: [u8; 32], max_ttl_ms: u64, + validated: ArcSwap>, } impl FileGrantIssuer { @@ -26,7 +29,11 @@ impl FileGrantIssuer { if key == [0; 32] || max_ttl_ms == 0 { return Err(FileGrantError::Invalid); } - Ok(Self { key, max_ttl_ms }) + Ok(Self { + key, + max_ttl_ms, + validated: ArcSwap::from_pointee(BTreeMap::new()), + }) } /// # Errors @@ -77,27 +84,41 @@ impl FileGrantIssuer { context: CatalogContext, now_ms: u64, ) -> Result { - if session_token.len() != TOKEN_CHARACTERS { + if ![TOKEN_CHARACTERS, 228].contains(&session_token.len()) { return Err(FileGrantError::Invalid); } + if let Some(cached) = self.validated.load().get(session_token) { + validate_request(&cached.grant, context, now_ms)?; + return Ok(cached.clone()); + } let bytes = URL_SAFE_NO_PAD .decode(session_token) .map_err(|_| FileGrantError::Invalid)?; - if bytes.len() != TOKEN_BYTES { + if ![TOKEN_BYTES, TOKEN_BYTES + 16].contains(&bytes.len()) { return Err(FileGrantError::Invalid); } - self.signer(b"iceberg-file-grant-v1", &bytes[..CLAIM_BYTES]) - .verify_slice(&bytes[CLAIM_BYTES..]) + let claim_bytes = bytes.len() - 32; + self.signer(b"iceberg-file-grant-v1", &bytes[..claim_bytes]) + .verify_slice(&bytes[claim_bytes..]) .map_err(|_| FileGrantError::Invalid)?; - let grant = decode(&bytes[..CLAIM_BYTES])?; + let grant = decode(&bytes[..claim_bytes])?; grant.validate(self.max_ttl_ms)?; - if grant.context != context { - return Err(FileGrantError::Forbidden); - } - if now_ms < grant.issued_ms || now_ms >= grant.expires_ms { - return Err(FileGrantError::Expired); - } - Ok(self.credentials(grant, &bytes)) + validate_request(&grant, context, now_ms)?; + let credentials = self.credentials(grant, &bytes); + self.remember(&credentials, now_ms); + Ok(credentials) + } + + fn remember(&self, credentials: &FileCredentials, now_ms: u64) { + self.validated.rcu(|current| { + let mut next = (**current).clone(); + next.retain(|_, value| value.grant.expires_ms > now_ms); + while next.len() >= 64 { + next.pop_first(); + } + next.insert(credentials.session_token.clone(), credentials.clone()); + Arc::new(next) + }); } fn signer(&self, domain: &[u8], bytes: &[u8]) -> Signer { @@ -125,9 +146,19 @@ impl FileGrantIssuer { } } +fn validate_request(grant: &FileGrant, context: CatalogContext, now_ms: u64) -> Result<(), FileGrantError> { + if grant.context != context { + return Err(FileGrantError::Forbidden); + } + if now_ms < grant.issued_ms || now_ms >= grant.expires_ms { + return Err(FileGrantError::Expired); + } + Ok(()) +} + fn encode(grant: &FileGrant) -> Vec { let mut bytes = Vec::with_capacity(TOKEN_BYTES); - bytes.push(1); + bytes.push(if grant.storage_namespace.is_some() { 2 } else { 1 }); bytes.extend_from_slice(grant.context.catalog.as_bytes()); bytes.extend_from_slice(&grant.context.activation_epoch.to_be_bytes()); bytes.extend_from_slice(grant.table.as_bytes()); @@ -138,11 +169,15 @@ fn encode(grant: &FileGrant) -> Vec { bytes.extend_from_slice(&grant.operations.0.to_be_bytes()); bytes.extend_from_slice(&grant.max_request_bytes.to_be_bytes()); bytes.extend_from_slice(&grant.max_file_bytes.to_be_bytes()); + if let Some(namespace) = grant.storage_namespace { + bytes.extend_from_slice(namespace.as_bytes()); + } bytes } fn decode(bytes: &[u8]) -> Result { - if bytes.len() != CLAIM_BYTES || bytes[0] != 1 { + if !((bytes.len() == CLAIM_BYTES && bytes[0] == 1) || (bytes.len() == CLAIM_BYTES + 16 && bytes[0] == 2)) + { return Err(FileGrantError::Invalid); } let array = |start, end| bytes.get(start..end).ok_or(FileGrantError::Invalid); @@ -154,6 +189,11 @@ fn decode(bytes: &[u8]) -> Result { )) }; Ok(FileGrant { + storage_namespace: if bytes[0] == 2 { + Some(NamespaceId::from_bytes(&bytes[CLAIM_BYTES..]).map_err(|_| FileGrantError::Invalid)?) + } else { + None + }, context: CatalogContext { catalog: CatalogId::from_bytes(array(1, 17)?).map_err(|_| FileGrantError::Invalid)?, activation_epoch: number(17)?, diff --git a/lib/crowdb-access-iceberg/src/file/digest.rs b/lib/crowdb-access-iceberg/src/file/digest.rs index 15e4a5817..2ce897c88 100644 --- a/lib/crowdb-access-iceberg/src/file/digest.rs +++ b/lib/crowdb-access-iceberg/src/file/digest.rs @@ -89,7 +89,11 @@ impl FileDigest { #[must_use] pub fn checkpoint(&self) -> Vec { let mut bytes = Vec::with_capacity(CHECKPOINT_BYTES); - bytes.extend_from_slice(b"ICHS\x01"); + bytes.extend_from_slice(if self.owner.table.storage_namespace.is_some() { + b"ICHS\x02" + } else { + b"ICHS\x01" + }); bytes.extend_from_slice(self.owner.table.catalog.as_bytes()); bytes.extend_from_slice(self.owner.table.table.as_bytes()); bytes.extend_from_slice(self.owner.file.as_bytes()); @@ -98,6 +102,9 @@ impl FileDigest { bytes.extend_from_slice(&word.to_be_bytes()); } bytes.extend_from_slice(&self.tail); + if let Some(namespace) = self.owner.table.storage_namespace { + bytes.extend_from_slice(namespace.as_bytes()); + } let checksum = Sha256::digest(&bytes); bytes.extend_from_slice(&checksum); bytes @@ -108,12 +115,23 @@ impl FileDigest { /// # Errors /// Rejects wrong versions, identities, lengths, checksums and noncanonical tails. pub fn restore(owner: FileIdentity, bytes: &[u8]) -> Result { - if bytes.len() != CHECKPOINT_BYTES - || &bytes[..5] != b"ICHS\x01" + let extra = if owner.table.storage_namespace.is_some() { + 16 + } else { + 0 + }; + let version = if extra == 0 { b"ICHS\x01" } else { b"ICHS\x02" }; + let checksum_offset = CHECKSUM_OFFSET + extra; + if bytes.len() != CHECKPOINT_BYTES + extra + || &bytes[..5] != version || &bytes[5..21] != owner.table.catalog.as_bytes() || &bytes[21..37] != owner.table.table.as_bytes() || &bytes[37..53] != owner.file.as_bytes() - || Sha256::digest(&bytes[..CHECKSUM_OFFSET]).as_slice() != &bytes[CHECKSUM_OFFSET..] + || owner + .table + .storage_namespace + .is_some_and(|id| &bytes[CHECKSUM_OFFSET..checksum_offset] != id.as_bytes()) + || Sha256::digest(&bytes[..checksum_offset]).as_slice() != &bytes[checksum_offset..] { return Err(FileIoError::Bounds); } diff --git a/lib/crowdb-access-iceberg/src/file/direct_location.rs b/lib/crowdb-access-iceberg/src/file/direct_location.rs new file mode 100644 index 000000000..494c482cc --- /dev/null +++ b/lib/crowdb-access-iceberg/src/file/direct_location.rs @@ -0,0 +1,191 @@ +use std::collections::BTreeMap; + +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use hmac::{Hmac, Mac}; +use sha2::Sha256; + +use crate::{error::ValidationError, key::FileId}; + +use super::{ChunkRoot, ContentFormat, FileContent, FileKind, FileLocation, FileRecord, TableLocation}; + +const PAYLOAD_BYTES: usize = 142; +const TOKEN_BYTES: usize = PAYLOAD_BYTES + 32; + +/// Persistent file locators use a key ring independent of expiring HTTP grants. +pub struct DirectFileLocations { + active_version: u32, + keys: BTreeMap, +} + +impl DirectFileLocations { + /// # Errors + /// Rejects missing active keys, reserved versions and uninitialized keys. + pub fn new(active_version: u32, keys: BTreeMap) -> Result { + if !keys.contains_key(&active_version) + || keys.iter().any(|(version, key)| *version == 0 || *key == [0; 32]) + { + return Err(ValidationError::Record); + } + Ok(Self { active_version, keys }) + } + + /// Called only after the supplied native tree is durable. + /// + /// # Errors + /// Rejects legacy tables, invalid stems and inconsistent native file descriptors. + pub fn parquet( + &self, + table: TableLocation, + stem: &str, + file: FileId, + length: u64, + digest: [u8; 32], + root: ChunkRoot, + ) -> Result { + if table.storage_namespace.is_none() || stem.contains('/') { + return Err(ValidationError::Key); + } + super::validate_relative_key(stem)?; + let mut payload = Vec::with_capacity(TOKEN_BYTES); + payload.push(1); + payload.extend_from_slice(&self.active_version.to_be_bytes()); + payload.extend_from_slice(file.as_bytes()); + payload.extend_from_slice(&length.to_be_bytes()); + payload.extend_from_slice(&digest); + encode_root(&mut payload, &root); + let signature = self + .signer(self.active_version, table, stem, &payload)? + .finalize() + .into_bytes(); + payload.extend_from_slice(&signature); + let location = table.file(&format!( + "f/v1/{stem}.{}.parquet", + URL_SAFE_NO_PAD.encode(payload) + ))?; + let record = FileRecord { + file, + location, + kind: FileKind::Data, + format: ContentFormat::Parquet, + length, + digest, + content: FileContent::Chunks { root: Some(root) }, + hint: None, + }; + record.validate()?; + Ok(record) + } + + /// Reconstructs a read descriptor without consulting any file record or state. + /// HTTP permission validation remains the caller's separate responsibility. + /// + /// # Errors + /// Rejects reserved-path corruption, altered identities and unknown signing keys. + pub fn resolve(&self, location: &FileLocation) -> Result, ValidationError> { + if location.table().storage_namespace.is_none() || !location.relative_key().starts_with("f/") { + return Ok(None); + } + let name = location + .relative_key() + .strip_prefix("f/v1/") + .ok_or(ValidationError::Key)?; + let name = name.strip_suffix(".parquet").ok_or(ValidationError::Key)?; + let (stem, token) = name.rsplit_once('.').ok_or(ValidationError::Key)?; + if stem.contains('/') || token.len() != TOKEN_BYTES.div_ceil(3) * 4 { + return Err(ValidationError::Key); + } + super::validate_relative_key(stem)?; + let payload = URL_SAFE_NO_PAD.decode(token).map_err(|_| ValidationError::Key)?; + if payload.len() != TOKEN_BYTES || payload[0] != 1 { + return Err(ValidationError::Key); + } + let version = u32::from_be_bytes(payload[1..5].try_into().map_err(|_| ValidationError::Key)?); + self.signer(version, location.table(), stem, &payload[..PAYLOAD_BYTES])? + .verify_slice(&payload[PAYLOAD_BYTES..]) + .map_err(|_| ValidationError::IdentityMismatch)?; + let mut fields = Cursor(&payload[5..PAYLOAD_BYTES]); + let file = FileId::from_bytes(&fields.take::<16>()?)?; + let length = fields.number()?; + let digest = fields.take()?; + let root = decode_root(&mut fields)?; + let record = FileRecord { + file, + location: location.clone(), + kind: FileKind::Data, + format: ContentFormat::Parquet, + length, + digest, + content: FileContent::Chunks { root: Some(root) }, + hint: None, + }; + record.validate()?; + Ok(Some(record)) + } + + fn signer( + &self, + version: u32, + table: TableLocation, + stem: &str, + payload: &[u8], + ) -> Result, ValidationError> { + let key = self.keys.get(&version).ok_or(ValidationError::IdentityMismatch)?; + let path = table.storage_path().ok_or(ValidationError::Key)?; + let mut signer = Hmac::::new_from_slice(key).map_err(|_| ValidationError::Record)?; + signer.update(b"iceberg-native-parquet-locator-v1"); + signer.update(path.to_string().as_bytes()); + signer.update(&(stem.len() as u64).to_be_bytes()); + signer.update(stem.as_bytes()); + signer.update(payload); + Ok(signer) + } +} + +fn encode_root(bytes: &mut Vec, root: &ChunkRoot) { + for field in [ + root.chunk.high, + root.chunk.low, + root.offset, + root.physical_length, + root.logical_offset, + root.logical_length, + ] { + bytes.extend_from_slice(&field.to_be_bytes()); + } + bytes.push(root.height); + bytes.extend_from_slice(&root.digest); +} + +fn decode_root(fields: &mut Cursor<'_>) -> Result { + Ok(ChunkRoot { + chunk: crowdb_protocol::common::ChunkId { + high: fields.number()?, + low: fields.number()?, + }, + offset: fields.number()?, + physical_length: fields.number()?, + logical_offset: fields.number()?, + logical_length: fields.number()?, + height: fields.take::<1>()?[0], + digest: fields.take()?, + }) +} + +struct Cursor<'a>(&'a [u8]); + +impl Cursor<'_> { + fn take(&mut self) -> Result<[u8; N], ValidationError> { + let value = self + .0 + .get(..N) + .ok_or(ValidationError::Record)? + .try_into() + .map_err(|_| ValidationError::Record)?; + self.0 = &self.0[N..]; + Ok(value) + } + + fn number(&mut self) -> Result { + Ok(u64::from_be_bytes(self.take()?)) + } +} diff --git a/lib/crowdb-access-iceberg/src/file/directory.rs b/lib/crowdb-access-iceberg/src/file/directory.rs index 98a1298f4..61ba0d41c 100644 --- a/lib/crowdb-access-iceberg/src/file/directory.rs +++ b/lib/crowdb-access-iceberg/src/file/directory.rs @@ -1,7 +1,7 @@ use crowdb_protocol::common::ChunkId; use crate::error::ValidationError; -use crate::key::{CatalogId, FileId, TableId}; +use crate::key::{CatalogId, FileId, NamespaceId, TableId}; use super::content::{MAX_CHUNK_DIRECTORY_BYTES, MAX_CHUNK_TREE_HEIGHT}; use super::{ChunkRoot, TableLocation}; @@ -72,8 +72,15 @@ impl ChunkDirectory { pub(crate) fn encode_frontier(&self) -> Result, ValidationError> { self.frontier_length()?; let mut bytes = Vec::with_capacity(HEADER_BYTES + ENTRY_BYTES * self.entries.len()); - bytes.extend_from_slice(b"ICEN\x01"); + bytes.extend_from_slice(if self.owner.table.storage_namespace.is_some() { + b"ICEN\x02" + } else { + b"ICEN\x01" + }); bytes.extend_from_slice(self.owner.table.catalog.as_bytes()); + if let Some(namespace) = self.owner.table.storage_namespace { + bytes.extend_from_slice(namespace.as_bytes()); + } bytes.extend_from_slice(self.owner.table.table.as_bytes()); bytes.extend_from_slice(self.owner.file.as_bytes()); bytes.push(self.height); @@ -121,12 +128,20 @@ impl ChunkDirectory { return Err(ValidationError::RecordTooLarge); } let mut cursor = Cursor(bytes); - if cursor.take::<5>()? != *b"ICEN\x01" { + let version = cursor.take::<5>()?; + if version != *b"ICEN\x01" && version != *b"ICEN\x02" { return Err(ValidationError::Record); } + let catalog = CatalogId::from_bytes(&cursor.take::<16>()?)?; + let storage_namespace = if version == *b"ICEN\x02" { + Some(NamespaceId::from_bytes(&cursor.take::<16>()?)?) + } else { + None + }; let actual_owner = FileIdentity { table: TableLocation { - catalog: CatalogId::from_bytes(&cursor.take::<16>()?)?, + storage_namespace, + catalog, table: TableId::from_bytes(&cursor.take::<16>()?)?, }, file: FileId::from_bytes(&cursor.take::<16>()?)?, diff --git a/lib/crowdb-access-iceberg/src/file/key.rs b/lib/crowdb-access-iceberg/src/file/key.rs index 633f30f17..9e0321192 100644 --- a/lib/crowdb-access-iceberg/src/file/key.rs +++ b/lib/crowdb-access-iceberg/src/file/key.rs @@ -1,7 +1,21 @@ -use crate::key::{CatalogId, CatalogScope, FileId, IcebergKey}; +use crate::key::{ + CatalogId, CatalogScope, FileId, IcebergKey, StorageKey, TableStorageKey, TableStorageScope, +}; use super::FileLocation; +#[must_use] +pub fn canonical_file_key(location: &FileLocation) -> StorageKey { + match location.table().storage_path() { + Some(table) => StorageKey::NativeTable(TableStorageKey { + table, + scope: TableStorageScope::File, + suffix: location.relative_key().to_owned(), + }), + None => StorageKey::Legacy(location_key(location)), + } +} + #[must_use] pub fn file_key(catalog: CatalogId, file: FileId) -> IcebergKey { IcebergKey::Catalog { diff --git a/lib/crowdb-access-iceberg/src/file/listing.rs b/lib/crowdb-access-iceberg/src/file/listing.rs index 1726bce69..fdc0445df 100644 --- a/lib/crowdb-access-iceberg/src/file/listing.rs +++ b/lib/crowdb-access-iceberg/src/file/listing.rs @@ -4,7 +4,7 @@ use crowdb_chunk_kv_client::MultiScanRequest; use crowdb_protocol::chunk_kv::ScanDirection; use crate::error::ValidationError; -use crate::key::{CatalogScope, IcebergKey}; +use crate::key::{CatalogScope, IcebergKey, TableStorageKey}; use super::{TableLocation, MAX_OBJECT_KEY_BYTES}; @@ -66,6 +66,10 @@ impl FileListRequest { .prefix .strip_prefix(&self.table.object_prefix()) .ok_or(ValidationError::Key)?; + if let Some(table) = self.table.storage_path() { + let range = TableStorageKey::file_range(table, relative)?; + return Ok((range.start, range.end)); + } let mut start = IcebergKey::catalog_range(self.table.catalog).start; start.push(CatalogScope::FileLocation as u8); start.extend_from_slice(self.table.table.as_bytes()); @@ -78,6 +82,9 @@ impl FileListRequest { let relative = object_key .strip_prefix(&self.table.object_prefix()) .ok_or(ValidationError::Key)?; + if let Some(table) = self.table.storage_path() { + return Ok(TableStorageKey::file_range(table, relative)?.start); + } let mut key = IcebergKey::catalog_range(self.table.catalog).start; key.push(CatalogScope::FileLocation as u8); key.extend_from_slice(self.table.table.as_bytes()); diff --git a/lib/crowdb-access-iceberg/src/file/listing/repository.rs b/lib/crowdb-access-iceberg/src/file/listing/repository.rs index cf2cd47f4..b2aa374e2 100644 --- a/lib/crowdb-access-iceberg/src/file/listing/repository.rs +++ b/lib/crowdb-access-iceberg/src/file/listing/repository.rs @@ -1,6 +1,6 @@ use crate::catalog::{check_context, CatalogError, StoreError}; use crate::error::ValidationError; -use crate::key::IcebergKey; +use crate::key::{IcebergKey, StorageKey}; use crate::record::StorageRecord; use super::{successor, FileListPage, FileListRequest, FileListTokens, FileLocationScan, ListedFile}; @@ -23,6 +23,7 @@ impl FileRepository { request.validate()?; if grant.context.catalog != request.table.catalog || grant.table != request.table.table + || grant.storage_namespace != request.table.storage_namespace || !grant.operations.allows(FileOperation::ListObjects) || now_ms < grant.issued_ms || now_ms >= grant.expires_ms @@ -87,18 +88,21 @@ impl FileRepository { consumed += 1; start.clone_from(&item.key); start.push(0); - let key = IcebergKey::decode(&item.key)?; + let key = StorageKey::decode(&item.key)?; if matches!( - StorageRecord::decode(&key, &item.value)?, + StorageRecord::decode_storage(&key, &item.value)?, StorageRecord::DeletedFile(_) ) { continue; } - let IcebergKey::Catalog { suffix, .. } = key else { - return Err(ValidationError::Key.into()); + let relative = match &key { + StorageKey::Legacy(IcebergKey::Catalog { suffix, .. }) => { + std::str::from_utf8(suffix.get(16..).ok_or(ValidationError::Key)?) + .map_err(|_| ValidationError::Key)? + } + StorageKey::NativeTable(key) => key.suffix.as_str(), + StorageKey::Legacy(_) => return Err(ValidationError::Key.into()), }; - let relative = std::str::from_utf8(suffix.get(16..).ok_or(ValidationError::Key)?) - .map_err(|_| ValidationError::Key)?; let location = request.table.file(relative)?; let record = self.resolve_value(&location, &item.value).await?; let object_key = location.object_key(); diff --git a/lib/crowdb-access-iceberg/src/file/location.rs b/lib/crowdb-access-iceberg/src/file/location.rs index 7667ed6a2..cbb79be35 100644 --- a/lib/crowdb-access-iceberg/src/file/location.rs +++ b/lib/crowdb-access-iceberg/src/file/location.rs @@ -4,7 +4,7 @@ use std::str::FromStr; use data_encoding::BASE32_NOPAD; use crate::error::ValidationError; -use crate::key::{CatalogId, TableId}; +use crate::key::{CatalogId, NamespaceId, StorageTablePath, TableId}; pub const MAX_OBJECT_KEY_BYTES: usize = 1024; const TABLE_PREFIX_BYTES: usize = 35; @@ -12,6 +12,7 @@ const BUCKET_BYTES: usize = 34; #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub struct TableLocation { + pub storage_namespace: Option, pub catalog: CatalogId, pub table: TableId, } @@ -19,6 +20,9 @@ pub struct TableLocation { impl TableLocation { #[must_use] pub fn bucket(self) -> String { + if self.storage_namespace.is_some() { + return super::NATIVE_ICEBERG_BUCKET.to_owned(); + } format!( "iceberg-{}", BASE32_NOPAD.encode(self.catalog.as_bytes()).to_ascii_lowercase() @@ -27,13 +31,31 @@ impl TableLocation { #[must_use] pub fn object_prefix(self) -> String { + if let Some(path) = self.storage_path() { + return path.to_string(); + } format!("t/{}/", self.table) } + #[must_use] + pub const fn storage_path(self) -> Option { + match self.storage_namespace { + Some(storage_namespace) => Some(StorageTablePath { + catalog: self.catalog, + storage_namespace, + table: self.table, + }), + None => None, + } + } + /// # Errors /// Rejects keys outside this table or keys that cannot be represented exactly. pub fn file(self, relative_key: &str) -> Result { validate_relative_key(relative_key)?; + if self.storage_namespace.is_some() { + super::validate_native_relative_key(relative_key)?; + } Ok(FileLocation { table: self, relative_key: relative_key.to_owned(), @@ -53,6 +75,9 @@ impl FromStr for TableLocation { fn from_str(value: &str) -> Result { let body = value.strip_prefix("s3://").ok_or(ValidationError::Key)?; let (bucket, prefix) = body.split_once('/').ok_or(ValidationError::Key)?; + if bucket == super::NATIVE_ICEBERG_BUCKET { + return StorageTablePath::from_prefix(prefix).map(Self::from); + } if prefix.len() != TABLE_PREFIX_BYTES || !prefix.ends_with('/') { return Err(ValidationError::Key); } @@ -73,6 +98,10 @@ impl FileLocation { if key.len() > MAX_OBJECT_KEY_BYTES { return Err(ValidationError::KeyTooLarge); } + if bucket == super::NATIVE_ICEBERG_BUCKET { + let (table, relative) = StorageTablePath::split_object_key(key)?; + return TableLocation::from(table).file(relative); + } let rest = key.strip_prefix("t/").ok_or(ValidationError::Key)?; let (table, relative) = rest.split_once('/').ok_or(ValidationError::Key)?; let table = parse_table(bucket, &format!("t/{table}"))?; @@ -95,6 +124,16 @@ impl FileLocation { } } +impl From for TableLocation { + fn from(path: StorageTablePath) -> Self { + Self { + catalog: path.catalog, + storage_namespace: Some(path.storage_namespace), + table: path.table, + } + } +} + impl fmt::Display for FileLocation { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { write!(formatter, "{}{}", self.table, self.relative_key) @@ -124,7 +163,11 @@ fn parse_table(bucket: &str, prefix: &str) -> Result Result { + if context.catalog != owner.table.catalog { + return Err(ValidationError::IdentityMismatch.into()); + } + let root = tree.root.ok_or(ValidationError::Record)?; + let candidate = locations.parquet(owner.table, stem, owner.file, tree.length, tree.digest, root)?; + self.publish(context, &candidate).await + } + /// # Errors /// Rejects retired contexts, corrupt bindings and missing published authorities. pub async fn load( @@ -53,12 +74,12 @@ impl FileRepository { ) -> Result { let location = &expected.location; self.check_context(context, location).await?; - let key = location_key(location); + let key = canonical_file_key(location); let encoded = key.encode()?; let Some(current) = self.store.get(&encoded).await? else { return Err(CatalogError::Conflict); }; - if let StorageRecord::DeletedFile(deleted) = StorageRecord::decode(&key, ¤t.bytes)? { + if let StorageRecord::DeletedFile(deleted) = StorageRecord::decode_storage(&key, ¤t.bytes)? { return if deleted.file == *expected { Ok(*deleted) } else { @@ -87,11 +108,12 @@ impl FileRepository { CasOutcome::Applied(_) => Ok(deleted), CasOutcome::Conflict(Some(value)) if matches!( - StorageRecord::decode(&key, &value.bytes)?, + StorageRecord::decode_storage(&key, &value.bytes)?, StorageRecord::DeletedFile(_) ) => { - let StorageRecord::DeletedFile(current) = StorageRecord::decode(&key, &value.bytes)? else { + let StorageRecord::DeletedFile(current) = StorageRecord::decode_storage(&key, &value.bytes)? + else { unreachable!() }; if current.file == *expected { @@ -113,11 +135,11 @@ impl FileRepository { location: &FileLocation, ) -> Result, CatalogError> { self.check_context(context, location).await?; - let key = location_key(location); + let key = canonical_file_key(location); let Some(value) = self.store.get(&key.encode()?).await? else { return Ok(None); }; - match StorageRecord::decode(&key, &value.bytes)? { + match StorageRecord::decode_storage(&key, &value.bytes)? { StorageRecord::DeletedFile(deleted) => Ok(Some(*deleted)), _ => Ok(None), } @@ -130,7 +152,7 @@ impl FileRepository { ) -> Result { candidate.validate()?; self.check_context(context, &candidate.location).await?; - let key = location_key(&candidate.location).encode()?; + let key = canonical_file_key(&candidate.location).encode()?; let bytes = StorageRecord::File(Box::new(candidate.clone())).encode()?; let result = self .store @@ -139,8 +161,9 @@ impl FileRepository { let published = match result { CasOutcome::Applied(_) => candidate.clone(), CasOutcome::Conflict(Some(value)) => { - let record_key = location_key(&candidate.location); - if let StorageRecord::DeletedFile(deleted) = StorageRecord::decode(&record_key, &value.bytes)? + let record_key = canonical_file_key(&candidate.location); + if let StorageRecord::DeletedFile(deleted) = + StorageRecord::decode_storage(&record_key, &value.bytes)? { if candidate.file == deleted.file.file { return Err(CatalogError::Conflict); @@ -179,12 +202,12 @@ impl FileRepository { } async fn resolve(&self, location: &FileLocation) -> Result, CatalogError> { - let key = location_key(location); + let key = canonical_file_key(location); let Some(value) = self.store.get(&key.encode()?).await? else { return Ok(None); }; if matches!( - StorageRecord::decode(&key, &value.bytes)?, + StorageRecord::decode_storage(&key, &value.bytes)?, StorageRecord::DeletedFile(_) ) { return Ok(None); @@ -197,8 +220,8 @@ impl FileRepository { location: &FileLocation, bytes: &[u8], ) -> Result { - let key = location_key(location); - let record = match StorageRecord::decode(&key, bytes)? { + let key = canonical_file_key(location); + let record = match StorageRecord::decode_storage(&key, bytes)? { StorageRecord::File(record) => record, StorageRecord::FileMapping(mapping) => { let key = file_key(location.table().catalog, mapping.file); diff --git a/lib/crowdb-access-iceberg/src/file/write_intent.rs b/lib/crowdb-access-iceberg/src/file/write_intent.rs index 914d900ed..bd3ae76e8 100644 --- a/lib/crowdb-access-iceberg/src/file/write_intent.rs +++ b/lib/crowdb-access-iceberg/src/file/write_intent.rs @@ -120,7 +120,7 @@ async fn check_writer(store: &dyn CatalogStore, owner: FileIdentity) -> Result<( if root.state != RootState::Ready || root.context.catalog != owner.table.catalog { return Err(CatalogError::Busy); } - let key = crate::table::head_key(owner.table.catalog, owner.table.table); + let key = crate::table::storage_head_key(owner.table); if let Some(value) = store.get(&key.encode()?).await? { let StorageRecord::TableHead(head) = StorageRecord::decode(&key, &value.bytes)? else { return Err(ValidationError::Record.into()); diff --git a/lib/crowdb-access-iceberg/src/gc/admission.rs b/lib/crowdb-access-iceberg/src/gc/admission.rs index bb0c50751..329d00059 100644 --- a/lib/crowdb-access-iceberg/src/gc/admission.rs +++ b/lib/crowdb-access-iceberg/src/gc/admission.rs @@ -5,7 +5,7 @@ use crate::{ key::{CatalogScope, IcebergKey, OperationId, SystemScope}, operation::{ledger_locate, LedgerLocation, ManagementAction, ManagementOperation, ManagementPhase}, record::StorageRecord, - table::{head_key, TablePurgeTask}, + table::{storage_head_key, TablePurgeTask}, }; use super::{CandidatePhase, GcCandidate, GcLimits, GcRepository, GcTask, GcTaskKind, TreeReclaimCursor}; @@ -198,7 +198,7 @@ impl GcRepository { } return Ok(existing); } - let head_key = head_key(context.catalog, marker.head.table); + let head_key = storage_head_key(marker.head.storage_location()); let head = self .store .get(&head_key.encode()?) diff --git a/lib/crowdb-access-iceberg/src/gc/fence.rs b/lib/crowdb-access-iceberg/src/gc/fence.rs index 5dce34d93..2da92ed51 100644 --- a/lib/crowdb-access-iceberg/src/gc/fence.rs +++ b/lib/crowdb-access-iceberg/src/gc/fence.rs @@ -2,7 +2,7 @@ use crate::{ catalog::CatalogError, error::ValidationError, record::StorageRecord, - table::{head_key, TableHead, TableLifecycle}, + table::{storage_head_key, TableHead, TableLifecycle}, }; use super::{GcPhase, GcRepository, GcStalledReason, GcTask, GcTaskKind}; @@ -23,7 +23,7 @@ impl GcRepository { return Ok(task.clone()); } let expected = fenced_head(task)?; - let key = head_key(expected.catalog, expected.table); + let key = storage_head_key(expected.storage_location()); if let Some(value) = self.store.get(&key.encode()?).await? { let StorageRecord::TableHead(current) = StorageRecord::decode(&key, &value.bytes)? else { return Err(ValidationError::Record.into()); @@ -55,7 +55,7 @@ impl GcRepository { .operation_fence .checked_add(2) .ok_or(ValidationError::GenerationExhausted)?; - let key = head_key(released.catalog, released.table); + let key = storage_head_key(released.storage_location()); let Some(value) = self.store.get(&key.encode()?).await? else { return Ok(false); }; @@ -76,8 +76,8 @@ impl GcRepository { return Err(CatalogError::Busy); } let fenced = fenced_head(task)?; - self.change( - &head_key(before.catalog, before.table), + self.change_storage( + &storage_head_key(before.storage_location()), Some(&StorageRecord::TableHead(Box::new(before.clone()))), &StorageRecord::TableHead(Box::new(fenced.clone())), ) @@ -89,7 +89,7 @@ impl GcRepository { /// Rejects lost ownership; no deletion may follow a failed verification. pub async fn verify_table_fence(&self, task: &GcTask) -> Result<(), CatalogError> { let expected = fenced_head(task)?; - let key = head_key(expected.catalog, expected.table); + let key = storage_head_key(expected.storage_location()); let value = self .store .get(&key.encode()?) @@ -110,8 +110,8 @@ impl GcRepository { .operation_fence .checked_add(1) .ok_or(ValidationError::GenerationExhausted)?; - self.change( - &head_key(before.catalog, before.table), + self.change_storage( + &storage_head_key(before.storage_location()), Some(&StorageRecord::TableHead(Box::new(before))), &StorageRecord::TableHead(Box::new(released)), ) diff --git a/lib/crowdb-access-iceberg/src/gc/proof/traversal.rs b/lib/crowdb-access-iceberg/src/gc/proof/traversal.rs index 2b4216c1f..dcba37be5 100644 --- a/lib/crowdb-access-iceberg/src/gc/proof/traversal.rs +++ b/lib/crowdb-access-iceberg/src/gc/proof/traversal.rs @@ -6,7 +6,7 @@ use crate::{ file::{AvroDatumLimits, AvroLimits, FileBlockStore, FileReader}, key::{CatalogScope, IcebergKey}, record::StorageRecord, - table::{head_key, TableHead, TableMetadataDocument, TableMetadataLimits}, + table::{storage_head_key, TableHead, TableMetadataDocument, TableMetadataLimits}, }; use super::super::{ @@ -28,7 +28,7 @@ impl GcRepository { } check_context(self.store.as_ref(), task.context).await?; let head = task.head.as_ref().ok_or(ValidationError::Record)?; - let key = head_key(head.catalog, head.table); + let key = storage_head_key(head.storage_location()); let value = self .store .get(&key.encode()?) diff --git a/lib/crowdb-access-iceberg/src/gc/repository.rs b/lib/crowdb-access-iceberg/src/gc/repository.rs index 348587ac4..27c3fa98b 100644 --- a/lib/crowdb-access-iceberg/src/gc/repository.rs +++ b/lib/crowdb-access-iceberg/src/gc/repository.rs @@ -173,7 +173,24 @@ impl GcRepository { before: Option<&StorageRecord>, after: &StorageRecord, ) -> Result<(), CatalogError> { - self.check_retirement_mutation(key, after).await?; + self.change_storage(&crate::key::StorageKey::Legacy(key.clone()), before, after) + .await + } + + pub(super) async fn change_storage( + &self, + key: &crate::key::StorageKey, + before: Option<&StorageRecord>, + after: &StorageRecord, + ) -> Result<(), CatalogError> { + match key { + crate::key::StorageKey::Legacy(key) => self.check_retirement_mutation(key, after).await?, + crate::key::StorageKey::NativeTable(key) => { + if self.retirement(key.table.catalog).await?.is_some() { + return Err(CatalogError::Busy); + } + } + } let key = key.encode()?; let after = after.encode()?; let before = before.map(StorageRecord::encode).transpose()?; diff --git a/lib/crowdb-access-iceberg/src/gc/worker/live.rs b/lib/crowdb-access-iceberg/src/gc/worker/live.rs index 33372a6f1..03c4b16d5 100644 --- a/lib/crowdb-access-iceberg/src/gc/worker/live.rs +++ b/lib/crowdb-access-iceberg/src/gc/worker/live.rs @@ -1,7 +1,7 @@ use crate::{ catalog::check_context, file::{AvroDatumLimits, AvroLimits}, - table::{head_key, TableMetadataLimits}, + table::{storage_head_key, TableMetadataLimits}, }; use super::{ @@ -21,7 +21,7 @@ impl GcWorker { }; if !changed_context { let head = task.head.as_ref().ok_or(ValidationError::Record)?; - let key = head_key(head.catalog, head.table); + let key = storage_head_key(head.storage_location()); if let Some(value) = self.repository.store.get(&key.encode()?).await? { let StorageRecord::TableHead(current) = StorageRecord::decode(&key, &value.bytes)? else { return Err(ValidationError::Record.into()); @@ -168,7 +168,7 @@ impl GcWorker { return Ok(self.repository.verify_table_fence(task).await?); } let head = task.head.as_ref().ok_or(ValidationError::Record)?; - let key = head_key(head.catalog, head.table); + let key = storage_head_key(head.storage_location()); let value = self .repository .store diff --git a/lib/crowdb-access-iceberg/src/key/storage.rs b/lib/crowdb-access-iceberg/src/key/storage.rs index 3994a6437..4871ca648 100644 --- a/lib/crowdb-access-iceberg/src/key/storage.rs +++ b/lib/crowdb-access-iceberg/src/key/storage.rs @@ -12,6 +12,14 @@ pub enum StorageKey { } impl StorageKey { + /// # Errors + /// Rejects invalid keys in the explicitly selected layout. + pub fn encode(&self) -> Result, ValidationError> { + match self { + Self::Legacy(key) => key.encode(), + Self::NativeTable(key) => key.encode(), + } + } /// # Errors /// Rejects unsupported versions and malformed keys in their selected layout. pub fn decode(bytes: &[u8]) -> Result { diff --git a/lib/crowdb-access-iceberg/src/namespace/drop_probe.rs b/lib/crowdb-access-iceberg/src/namespace/drop_probe.rs index 83c6af4cb..51d2d7214 100644 --- a/lib/crowdb-access-iceberg/src/namespace/drop_probe.rs +++ b/lib/crowdb-access-iceberg/src/namespace/drop_probe.rs @@ -165,7 +165,7 @@ impl NamespaceDropper { .await?; return Ok(true); } - let key = crate::table::head_key(mapping.catalog, mapping.table); + let key = crate::table::storage_head_key(mapping.storage_location()); let value = self .creator .names diff --git a/lib/crowdb-access-iceberg/src/record.rs b/lib/crowdb-access-iceberg/src/record.rs index d62d952ca..6748ee252 100644 --- a/lib/crowdb-access-iceberg/src/record.rs +++ b/lib/crowdb-access-iceberg/src/record.rs @@ -14,6 +14,7 @@ mod namespace_operation; mod payload; mod retry; mod root; +mod storage_key; mod table; mod table_commit; mod table_create; @@ -21,3 +22,4 @@ mod table_lifecycle; mod write_intent; pub use envelope::{StorageRecord, MAX_RECORD_BYTES}; +pub use storage_key::StorageRecordKey; diff --git a/lib/crowdb-access-iceberg/src/record/envelope.rs b/lib/crowdb-access-iceberg/src/record/envelope.rs index 1b1150a59..ab83f9806 100644 --- a/lib/crowdb-access-iceberg/src/record/envelope.rs +++ b/lib/crowdb-access-iceberg/src/record/envelope.rs @@ -7,7 +7,7 @@ use crate::file::{ file_key, location_key, DeletedFile, FileMapping, FileRecord, MultipartAdmissionRecord, MultipartPart, MultipartSession, }; -use crate::key::{CatalogScope, IcebergKey, SystemScope}; +use crate::key::{CatalogScope, IcebergKey, StorageKey, SystemScope}; use crate::namespace::{authority_key, name_key, NamespaceAuthority, NamespaceMapping, NamespaceOperation}; use crate::operation::{ledger_key_matches, ManagementOperation, PayloadPage, RetryRecord, RetryResult}; @@ -196,7 +196,35 @@ impl StorageRecord { /// # Errors /// Rejects malformed envelopes, unknown schemas, invalid fields or mismatched keys. - pub fn decode(key: &IcebergKey, bytes: &[u8]) -> Result { + pub fn decode(key: &Key, bytes: &[u8]) -> Result { + let record = Self::decode_bytes(bytes)?; + key.validate_record(&record)?; + Ok(record) + } + + /// # Errors + /// Rejects foreign storage identities, record kinds and key layouts. + pub fn decode_storage(key: &StorageKey, bytes: &[u8]) -> Result { + Self::decode(key, bytes) + } + + pub(super) fn validate_storage_key(&self, key: &StorageKey) -> Result<(), ValidationError> { + if let StorageKey::Legacy(key) = key { + return self.validate_key(key); + } + let expected = match self { + Self::File(file) => crate::file::canonical_file_key(&file.location), + Self::DeletedFile(file) => crate::file::canonical_file_key(&file.file.location), + Self::TableHead(head) => crate::table::storage_head_key(head.storage_location()), + _ => return Err(ValidationError::Record), + }; + if expected != *key { + return Err(ValidationError::IdentityMismatch); + } + Ok(()) + } + + fn decode_bytes(bytes: &[u8]) -> Result { if bytes.len() > MAX_RECORD_BYTES { return Err(ValidationError::RecordTooLarge); } @@ -207,9 +235,7 @@ impl StorageRecord { if envelope.schema_version() != SCHEMA_VERSION { return Err(ValidationError::RecordVersion(envelope.schema_version())); } - let record = Self::decode_value(envelope)?; - record.validate_key(key)?; - Ok(record) + Self::decode_value(envelope) } fn decode_value(envelope: FBIcebergRecord<'_>) -> Result { @@ -390,7 +416,7 @@ impl StorageRecord { Ok(record) } - fn validate_key(&self, key: &IcebergKey) -> Result<(), ValidationError> { + pub(super) fn validate_key(&self, key: &IcebergKey) -> Result<(), ValidationError> { if let Some(result) = self.validate_gc_key(key) { return result; } diff --git a/lib/crowdb-access-iceberg/src/record/multipart/fields.rs b/lib/crowdb-access-iceberg/src/record/multipart/fields.rs index 8b2852ea6..ab0fdcc1d 100644 --- a/lib/crowdb-access-iceberg/src/record/multipart/fields.rs +++ b/lib/crowdb-access-iceberg/src/record/multipart/fields.rs @@ -3,7 +3,7 @@ use flatbuffers::{FlatBufferBuilder, WIPOffset}; use crate::error::ValidationError; use crate::file::{FileIdentity, FileTree, TableLocation}; -use crate::key::{CatalogId, FileId, TableId}; +use crate::key::{CatalogId, FileId, NamespaceId, TableId}; use crate::record::file::{decode_root, encode_root}; pub(super) fn encode_owner<'buffer>( @@ -13,12 +13,17 @@ pub(super) fn encode_owner<'buffer>( let catalog = builder.create_vector(owner.table.catalog.as_bytes()); let table_id = builder.create_vector(owner.table.table.as_bytes()); let file_id = builder.create_vector(owner.file.as_bytes()); + let storage_namespace = owner + .table + .storage_namespace + .map(|id| builder.create_vector(id.as_bytes())); FBFileIdentity::create( builder, &FBFileIdentityArgs { catalog: Some(catalog), table_id: Some(table_id), file_id: Some(file_id), + storage_namespace, }, ) } @@ -26,6 +31,10 @@ pub(super) fn encode_owner<'buffer>( pub(super) fn decode_owner(value: FBFileIdentity<'_>) -> Result { Ok(FileIdentity { table: TableLocation { + storage_namespace: value + .storage_namespace() + .map(|id| NamespaceId::from_bytes(id.bytes())) + .transpose()?, catalog: CatalogId::from_bytes(value.catalog().bytes())?, table: TableId::from_bytes(value.table_id().bytes())?, }, diff --git a/lib/crowdb-access-iceberg/src/record/storage_key.rs b/lib/crowdb-access-iceberg/src/record/storage_key.rs new file mode 100644 index 000000000..d098fa0d0 --- /dev/null +++ b/lib/crowdb-access-iceberg/src/record/storage_key.rs @@ -0,0 +1,31 @@ +use crate::{ + error::ValidationError, + key::{IcebergKey, StorageKey}, +}; + +use super::StorageRecord; + +/// Only versioned Iceberg storage keys can bind a decoded authority record. +pub trait StorageRecordKey: sealed::Sealed { + /// # Errors + /// Rejects a record belonging to another key, storage owner or layout. + fn validate_record(&self, record: &StorageRecord) -> Result<(), ValidationError>; +} + +impl StorageRecordKey for IcebergKey { + fn validate_record(&self, record: &StorageRecord) -> Result<(), ValidationError> { + record.validate_key(self) + } +} + +impl StorageRecordKey for StorageKey { + fn validate_record(&self, record: &StorageRecord) -> Result<(), ValidationError> { + record.validate_storage_key(self) + } +} + +mod sealed { + pub trait Sealed {} + impl Sealed for crate::key::IcebergKey {} + impl Sealed for crate::key::StorageKey {} +} diff --git a/lib/crowdb-access-iceberg/src/record/table.rs b/lib/crowdb-access-iceberg/src/record/table.rs index b4220889b..0b3ac3c2e 100644 --- a/lib/crowdb-access-iceberg/src/record/table.rs +++ b/lib/crowdb-access-iceberg/src/record/table.rs @@ -102,6 +102,9 @@ pub(super) fn encode_mapping<'buffer>( let namespace_id = builder.create_vector(mapping.namespace.as_bytes()); let name = builder.create_string(&mapping.name); let operation = builder.create_vector(mapping.operation.as_bytes()); + let storage_namespace = mapping + .storage_namespace + .map(|id| builder.create_vector(id.as_bytes())); Ok(FBTableMapping::create( builder, &FBTableMappingArgs { @@ -115,12 +118,17 @@ pub(super) fn encode_mapping<'buffer>( TableMappingState::Reserved => 0, TableMappingState::Published => 1, }, + storage_namespace, }, )) } pub(super) fn decode_mapping(value: FBTableMapping<'_>) -> Result { let mapping = TableMapping { + storage_namespace: value + .storage_namespace() + .map(|id| NamespaceId::from_bytes(id.bytes())) + .transpose()?, catalog: CatalogId::from_bytes(value.catalog().bytes())?, table: TableId::from_bytes(value.table_id().bytes())?, namespace: NamespaceId::from_bytes(value.namespace_id().bytes())?, diff --git a/lib/crowdb-access-iceberg/src/record/write_intent.rs b/lib/crowdb-access-iceberg/src/record/write_intent.rs index f9cea357b..db10170a5 100644 --- a/lib/crowdb-access-iceberg/src/record/write_intent.rs +++ b/lib/crowdb-access-iceberg/src/record/write_intent.rs @@ -4,7 +4,7 @@ use flatbuffers::{FlatBufferBuilder, WIPOffset}; use crate::{ error::ValidationError, file::{FileIdentity, FileWriteIntent, TableLocation}, - key::{CatalogId, FileId, OperationId, TableId}, + key::{CatalogId, FileId, NamespaceId, OperationId, TableId}, }; pub(super) fn encode<'buffer>( @@ -16,6 +16,11 @@ pub(super) fn encode<'buffer>( let table_id = builder.create_vector(intent.owner.table.table.as_bytes()); let file_id = builder.create_vector(intent.owner.file.as_bytes()); let identity = builder.create_vector(intent.identity.as_bytes()); + let storage_namespace = intent + .owner + .table + .storage_namespace + .map(|id| builder.create_vector(id.as_bytes())); let root = super::file::encode_root(builder, &intent.root); Ok(FBFileWriteIntent::create( builder, @@ -28,6 +33,7 @@ pub(super) fn encode<'buffer>( created_ms: intent.created_ms, not_before_ms: intent.not_before_ms, deleting: intent.deleting, + storage_namespace, }, )) } @@ -37,6 +43,10 @@ pub(super) fn decode(value: FBFileWriteIntent<'_>) -> Result crate::key::StorageKey { + match location.storage_path() { + Some(table) => crate::key::StorageKey::NativeTable(crate::key::TableStorageKey { + table, + scope: crate::key::TableStorageScope::Head, + suffix: String::new(), + }), + None => crate::key::StorageKey::Legacy(head_key(location.catalog, location.table)), + } +} + #[must_use] pub fn head_key(catalog: CatalogId, table: TableId) -> IcebergKey { IcebergKey::Catalog { diff --git a/lib/crowdb-access-iceberg/src/table/lifecycle/completion.rs b/lib/crowdb-access-iceberg/src/table/lifecycle/completion.rs index 37302166f..4f5cfc1db 100644 --- a/lib/crowdb-access-iceberg/src/table/lifecycle/completion.rs +++ b/lib/crowdb-access-iceberg/src/table/lifecycle/completion.rs @@ -5,7 +5,7 @@ use crate::{ error::ValidationError, operation::mutation_identity, record::StorageRecord, - table::{head_key, name_key, TableMapping, TableMappingState}, + table::{name_key, storage_head_key, TableMapping, TableMappingState}, }; use std::time::{SystemTime, UNIX_EPOCH}; @@ -15,7 +15,7 @@ impl TableLifecycles { if operation.is_rename() { self.check_admission(operation).await?; } - let key = head_key(operation.context.catalog, operation.before.table); + let key = storage_head_key(operation.before.storage_location()); let before = StorageRecord::TableHead(Box::new(operation.before.clone())).encode()?; let after = StorageRecord::TableHead(Box::new(operation.candidate.clone())).encode()?; let encoded = key.encode()?; @@ -49,7 +49,7 @@ impl TableLifecycles { pub(super) async fn complete(&self, operation: &TableLifecycleOperation) -> Result<(), CatalogError> { self.current(operation).await?; - let head_key = head_key(operation.context.catalog, operation.before.table); + let head_key = storage_head_key(operation.before.storage_location()); let value = self .store .get(&head_key.encode()?) @@ -139,7 +139,7 @@ impl TableLifecycles { } else if operation.phase == Phase::Complete { self.remove_mapping(&operation.source).await?; if operation.is_rename() { - let key = head_key(operation.context.catalog, operation.before.table).encode()?; + let key = storage_head_key(operation.before.storage_location()).encode()?; let before = StorageRecord::TableHead(Box::new(operation.candidate.clone())).encode()?; let mut settled = operation.candidate.clone(); settled.pending_operation = None; diff --git a/lib/crowdb-access-iceberg/src/table/lifecycle/operation.rs b/lib/crowdb-access-iceberg/src/table/lifecycle/operation.rs index 361466ee0..0748d5c69 100644 --- a/lib/crowdb-access-iceberg/src/table/lifecycle/operation.rs +++ b/lib/crowdb-access-iceberg/src/table/lifecycle/operation.rs @@ -126,6 +126,7 @@ impl TableLifecycleOperation { pub(super) fn destination(&self, state: TableMappingState) -> TableMapping { TableMapping { + storage_namespace: self.candidate.metadata_location.table().storage_namespace, catalog: self.context.catalog, namespace: self.candidate.namespace, name: self.candidate.name.clone(), diff --git a/lib/crowdb-access-iceberg/src/table/lifecycle/reservation.rs b/lib/crowdb-access-iceberg/src/table/lifecycle/reservation.rs index 8a583c1e0..8baca14e5 100644 --- a/lib/crowdb-access-iceberg/src/table/lifecycle/reservation.rs +++ b/lib/crowdb-access-iceberg/src/table/lifecycle/reservation.rs @@ -5,7 +5,7 @@ use crate::{ key::{NamespaceId, OperationId}, operation::mutation_identity, record::StorageRecord, - table::{head_key, name_key, TableMapping, TableMappingState}, + table::{name_key, storage_head_key, TableMapping, TableMappingState}, }; impl TableLifecycles { @@ -50,7 +50,7 @@ impl TableLifecycles { .await?; return Ok(()); } - let head_key = head_key(existing.catalog, existing.table); + let head_key = storage_head_key(existing.storage_location()); let head = self .store .get(&head_key.encode()?) diff --git a/lib/crowdb-access-iceberg/src/table/list.rs b/lib/crowdb-access-iceberg/src/table/list.rs index 3f7dc57dc..aea847221 100644 --- a/lib/crowdb-access-iceberg/src/table/list.rs +++ b/lib/crowdb-access-iceberg/src/table/list.rs @@ -178,7 +178,7 @@ impl TableLister { if mapping.state != super::TableMappingState::Published { continue; } - let key = super::head_key(context.catalog, mapping.table); + let key = super::storage_head_key(mapping.storage_location()); let Some(stored) = self.store.get(&key.encode()?).await? else { continue; }; diff --git a/lib/crowdb-access-iceberg/src/table/record.rs b/lib/crowdb-access-iceberg/src/table/record.rs index f1abce04c..9b68f134c 100644 --- a/lib/crowdb-access-iceberg/src/table/record.rs +++ b/lib/crowdb-access-iceberg/src/table/record.rs @@ -31,6 +31,10 @@ pub struct TableHead { } impl TableHead { + #[must_use] + pub fn storage_location(&self) -> crate::file::TableLocation { + self.metadata_location.table() + } /// # Errors /// Rejects invalid revisions, identity, version and unfenced tombstones. pub fn validate(&self) -> Result<(), ValidationError> { @@ -58,6 +62,7 @@ pub enum TableMappingState { #[derive(Clone, Debug, Eq, PartialEq)] pub struct TableMapping { + pub storage_namespace: Option, pub catalog: CatalogId, pub namespace: NamespaceId, pub name: String, @@ -68,6 +73,14 @@ pub struct TableMapping { } impl TableMapping { + #[must_use] + pub const fn storage_location(&self) -> crate::file::TableLocation { + crate::file::TableLocation { + catalog: self.catalog, + table: self.table, + storage_namespace: self.storage_namespace, + } + } /// # Errors /// Rejects invalid names and zero name epochs. pub fn validate(&self) -> Result<(), ValidationError> { @@ -85,6 +98,7 @@ impl TableMapping { && self.catalog == head.catalog && self.namespace == head.namespace && self.table == head.table + && self.storage_namespace == head.metadata_location.table().storage_namespace && self.name == head.name && self.name_epoch == head.name_epoch } diff --git a/lib/crowdb-access-iceberg/src/table/repository.rs b/lib/crowdb-access-iceberg/src/table/repository.rs index a1f7b83c2..16264d966 100644 --- a/lib/crowdb-access-iceberg/src/table/repository.rs +++ b/lib/crowdb-access-iceberg/src/table/repository.rs @@ -59,7 +59,7 @@ impl TableRepository { if selected.head.catalog != context.catalog { return Err(ValidationError::IdentityMismatch.into()); } - let key = super::head_key(context.catalog, selected.head.table); + let key = super::storage_head_key(selected.head.storage_location()); let value = self .store .get(&key.encode()?) @@ -90,7 +90,7 @@ impl TableRepository { if mapping.state != TableMappingState::Published { return Ok(None); } - let key = super::head_key(context.catalog, mapping.table); + let key = super::storage_head_key(mapping.storage_location()); let Some(value) = self.store.get(&key.encode()?).await? else { return Ok(None); }; diff --git a/lib/crowdb-access-iceberg/src/wire/credentials.rs b/lib/crowdb-access-iceberg/src/wire/credentials.rs index fa520e68d..121e3986b 100644 --- a/lib/crowdb-access-iceberg/src/wire/credentials.rs +++ b/lib/crowdb-access-iceberg/src/wire/credentials.rs @@ -22,6 +22,7 @@ impl From for StorageCredential { let grant = credentials.grant(); Self { prefix: TableLocation { + storage_namespace: grant.storage_namespace, catalog: grant.context.catalog, table: grant.table, } @@ -59,6 +60,7 @@ pub struct FileDelegationLimits { #[derive(Clone, Copy, Debug)] pub struct FileDelegationTarget { + pub storage_namespace: Option, pub table: TableId, pub format_version: u8, pub staged: bool, @@ -159,6 +161,7 @@ impl FileDelegationLimits { digest.update(b"crowdb-iceberg-file-principal-v1"); digest.update(principal.name.as_bytes()); issuer.issue(FileGrant { + storage_namespace: target.storage_namespace, context, table: target.table, principal: digest.finalize().into(), diff --git a/lib/crowdb-access-iceberg/tests/commit_manifest_provenance_test.rs b/lib/crowdb-access-iceberg/tests/commit_manifest_provenance_test.rs index bb4e1ff5a..737c675b5 100644 --- a/lib/crowdb-access-iceberg/tests/commit_manifest_provenance_test.rs +++ b/lib/crowdb-access-iceberg/tests/commit_manifest_provenance_test.rs @@ -133,6 +133,7 @@ async fn candidate_sources_separate_reachable_history_from_new_definition_bound_ upload ); let foreign = crowdb_access_iceberg::file::TableLocation { + storage_namespace: None, catalog: fixture.namespace.context.catalog, table: crowdb_access_iceberg::key::TableId::random(), } diff --git a/lib/crowdb-access-iceberg/tests/common/deletion_vector.rs b/lib/crowdb-access-iceberg/tests/common/deletion_vector.rs index 8c7900d98..51b5c441a 100644 --- a/lib/crowdb-access-iceberg/tests/common/deletion_vector.rs +++ b/lib/crowdb-access-iceberg/tests/common/deletion_vector.rs @@ -69,6 +69,7 @@ pub async fn record( cardinality: u64, ) -> (FileRecord, DeletionVectorReference) { let table = TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }; diff --git a/lib/crowdb-access-iceberg/tests/common/file.rs b/lib/crowdb-access-iceberg/tests/common/file.rs index b35b610a5..1444a7cea 100644 --- a/lib/crowdb-access-iceberg/tests/common/file.rs +++ b/lib/crowdb-access-iceberg/tests/common/file.rs @@ -25,6 +25,7 @@ impl TestFile { activation_epoch: 1, }, table: TableLocation { + storage_namespace: None, catalog, table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/common/manifest_entry.rs b/lib/crowdb-access-iceberg/tests/common/manifest_entry.rs index 7042ed129..69519949e 100644 --- a/lib/crowdb-access-iceberg/tests/common/manifest_entry.rs +++ b/lib/crowdb-access-iceberg/tests/common/manifest_entry.rs @@ -13,6 +13,7 @@ pub struct TestManifestEntry { pub fn table() -> TableLocation { TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[1; 16]).unwrap(), table: TableId::from_bytes(&[2; 16]).unwrap(), } diff --git a/lib/crowdb-access-iceberg/tests/common/manifest_list.rs b/lib/crowdb-access-iceberg/tests/common/manifest_list.rs index a9778153c..d623395d3 100644 --- a/lib/crowdb-access-iceberg/tests/common/manifest_list.rs +++ b/lib/crowdb-access-iceberg/tests/common/manifest_list.rs @@ -10,6 +10,7 @@ pub struct TestManifestList { pub fn table() -> TableLocation { TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[1; 16]).unwrap(), table: TableId::from_bytes(&[2; 16]).unwrap(), } diff --git a/lib/crowdb-access-iceberg/tests/common/multipart.rs b/lib/crowdb-access-iceberg/tests/common/multipart.rs index acd515cac..92f76e857 100644 --- a/lib/crowdb-access-iceberg/tests/common/multipart.rs +++ b/lib/crowdb-access-iceberg/tests/common/multipart.rs @@ -8,6 +8,7 @@ use crowdb_access_iceberg::operation::PayloadReference; pub fn session() -> MultipartSession { let table = TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }; diff --git a/lib/crowdb-access-iceberg/tests/common/parquet_integers.rs b/lib/crowdb-access-iceberg/tests/common/parquet_integers.rs index 444afc9e5..4a0bf9cc6 100644 --- a/lib/crowdb-access-iceberg/tests/common/parquet_integers.rs +++ b/lib/crowdb-access-iceberg/tests/common/parquet_integers.rs @@ -70,6 +70,7 @@ pub async fn file( stored_content( &bytes, TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[1; 16]).unwrap(), table: TableId::from_bytes(&[2; 16]).unwrap(), }, diff --git a/lib/crowdb-access-iceberg/tests/common/parquet_metadata.rs b/lib/crowdb-access-iceberg/tests/common/parquet_metadata.rs index e56d6cf7c..b43dacefc 100644 --- a/lib/crowdb-access-iceberg/tests/common/parquet_metadata.rs +++ b/lib/crowdb-access-iceberg/tests/common/parquet_metadata.rs @@ -121,6 +121,7 @@ pub async fn stored(footer: &[u8], body_bytes: usize) -> (Arc, FileR stored_content( &bytes, TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/common/parquet_nullable.rs b/lib/crowdb-access-iceberg/tests/common/parquet_nullable.rs index 717f7c2bd..1bffe5466 100644 --- a/lib/crowdb-access-iceberg/tests/common/parquet_nullable.rs +++ b/lib/crowdb-access-iceberg/tests/common/parquet_nullable.rs @@ -105,6 +105,7 @@ pub async fn file_with_type_length( stored_content( &bytes, TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[1; 16]).unwrap(), table: TableId::from_bytes(&[2; 16]).unwrap(), }, diff --git a/lib/crowdb-access-iceberg/tests/common/projection.rs b/lib/crowdb-access-iceberg/tests/common/projection.rs index 33273610d..698aca829 100644 --- a/lib/crowdb-access-iceberg/tests/common/projection.rs +++ b/lib/crowdb-access-iceberg/tests/common/projection.rs @@ -44,6 +44,7 @@ impl TestProjection { let blocks = Arc::new(TestBlocks::default()); let owner = FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/common/puffin.rs b/lib/crowdb-access-iceberg/tests/common/puffin.rs index eff1d2e50..9dd672a84 100644 --- a/lib/crowdb-access-iceberg/tests/common/puffin.rs +++ b/lib/crowdb-access-iceberg/tests/common/puffin.rs @@ -9,6 +9,7 @@ use std::sync::Arc; pub fn referenced() -> crowdb_access_iceberg::file::FileLocation { TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), } diff --git a/lib/crowdb-access-iceberg/tests/common/selected_parquet.rs b/lib/crowdb-access-iceberg/tests/common/selected_parquet.rs index 4c3fc9879..47b7b3ec1 100644 --- a/lib/crowdb-access-iceberg/tests/common/selected_parquet.rs +++ b/lib/crowdb-access-iceberg/tests/common/selected_parquet.rs @@ -68,6 +68,7 @@ pub fn entry(content: FileContentKind) -> ManifestScalarEntry { }, file: ManifestFileFields { location: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), } diff --git a/lib/crowdb-access-iceberg/tests/common/store.rs b/lib/crowdb-access-iceberg/tests/common/store.rs index 35269da41..4a4a8c86a 100644 --- a/lib/crowdb-access-iceberg/tests/common/store.rs +++ b/lib/crowdb-access-iceberg/tests/common/store.rs @@ -7,6 +7,7 @@ use std::sync::{ use arc_swap::ArcSwap; use async_trait::async_trait; use crowdb_access_iceberg::catalog::{CasOutcome, CatalogStore, StoreError, StoredValue}; +use crowdb_access_iceberg::key::{CatalogScope, IcebergKey, StorageKey, TableStorageKey, TableStorageScope}; use crowdb_protocol::chunk_kv::ClientRequestId; #[derive(Default)] @@ -65,11 +66,14 @@ impl TestStore { fn lose_file_reply(&self, key: &[u8]) -> bool { matches!( - crowdb_access_iceberg::key::IcebergKey::decode(key), - Ok(crowdb_access_iceberg::key::IcebergKey::Catalog { - scope: crowdb_access_iceberg::key::CatalogScope::FileLocation, + StorageKey::decode(key), + Ok(StorageKey::Legacy(IcebergKey::Catalog { + scope: CatalogScope::FileLocation, .. - }) + }) | StorageKey::NativeTable(TableStorageKey { + scope: TableStorageScope::File, + .. + })) ) && self.file_record_reply_loss.swap(false, Ordering::SeqCst) } diff --git a/lib/crowdb-access-iceberg/tests/common/table_metadata.rs b/lib/crowdb-access-iceberg/tests/common/table_metadata.rs index 55679204c..ec25120e6 100644 --- a/lib/crowdb-access-iceberg/tests/common/table_metadata.rs +++ b/lib/crowdb-access-iceberg/tests/common/table_metadata.rs @@ -8,6 +8,7 @@ use sha2::{Digest, Sha256}; pub fn table() -> TableLocation { TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[1; 16]).unwrap(), table: TableId::from_bytes(&[2; 16]).unwrap(), } diff --git a/lib/crowdb-access-iceberg/tests/common/table_read.rs b/lib/crowdb-access-iceberg/tests/common/table_read.rs index b41961b35..6de91dd34 100644 --- a/lib/crowdb-access-iceberg/tests/common/table_read.rs +++ b/lib/crowdb-access-iceberg/tests/common/table_read.rs @@ -37,6 +37,7 @@ impl TestTable { head.catalog = fixture.context.catalog; head.namespace = parent.namespace; let location = TableLocation { + storage_namespace: None, catalog: head.catalog, table: head.table, }; @@ -85,6 +86,7 @@ impl TestTable { .await .unwrap(); let mapping = TableMapping { + storage_namespace: None, catalog: head.catalog, namespace: head.namespace, name: head.name.clone(), diff --git a/lib/crowdb-access-iceberg/tests/direct_file_location_test.rs b/lib/crowdb-access-iceberg/tests/direct_file_location_test.rs new file mode 100644 index 000000000..464b58e56 --- /dev/null +++ b/lib/crowdb-access-iceberg/tests/direct_file_location_test.rs @@ -0,0 +1,143 @@ +#[path = "common/file_blocks.rs"] +mod blocks; +#[path = "common/store.rs"] +mod common; +#[path = "common/file.rs"] +mod fixture; + +use std::{collections::BTreeMap, sync::Arc}; + +use crowdb_access_iceberg::{ + file::{ + ByteRange, DirectFileLocations, FileIdentity, FileReader, FileRepository, FileTreeWriter, + TableLocation, + }, + key::{CatalogId, FileId, NamespaceId, TableId}, +}; + +fn table() -> TableLocation { + TableLocation { + storage_namespace: Some(NamespaceId::random()), + catalog: CatalogId::random(), + table: TableId::random(), + } +} + +#[tokio::test] +async fn injection_replay_and_logical_delete_do_not_add_direct_read_metadata_queries() { + use std::sync::atomic::Ordering; + let mut fixture = fixture::TestFile::new(common::TestStore::default()).await; + fixture.table.storage_namespace = Some(NamespaceId::random()); + let owner = FileIdentity { + table: fixture.table, + file: FileId::random(), + }; + let blocks = Arc::new(blocks::TestBlocks::default()); + let mut writer = FileTreeWriter::new(blocks.clone(), owner, 1024).unwrap(); + writer.push(b"retained parquet bytes").await.unwrap(); + let tree = writer.finish().await.unwrap(); + let signer = DirectFileLocations::new(1, BTreeMap::from([(1, [1; 32])])).unwrap(); + let repository = FileRepository::new(fixture.store.clone()); + repository + .publish(fixture.context, &fixture.record("metadata/v1.json", b"{}")) + .await + .unwrap(); + fixture.store.file_record_reply_loss.store(true, Ordering::SeqCst); + assert!(repository + .finalize_native_parquet(fixture.context, owner, "part", tree.clone(), &signer) + .await + .is_err()); + let published = repository + .finalize_native_parquet(fixture.context, owner, "part", tree, &signer) + .await + .unwrap(); + repository + .mark_deleted(fixture.context, &published, 100) + .await + .unwrap(); + assert!(repository + .load(fixture.context, &published.location) + .await + .unwrap() + .is_none()); + let before = fixture.store.reads.load(Ordering::SeqCst); + let record = signer.resolve(&published.location).unwrap().unwrap(); + let mut reader = FileReader::new(blocks.clone(), record, None, 1024).unwrap(); + assert_eq!(reader.next().await.unwrap().unwrap(), b"retained parquet bytes"); + assert_eq!(fixture.store.reads.load(Ordering::SeqCst), before); + blocks.values.store(Arc::new(BTreeMap::new())); + let mut reclaimed = FileReader::new( + blocks, + signer.resolve(&published.location).unwrap().unwrap(), + None, + 1024, + ) + .unwrap(); + assert!(reclaimed.next().await.is_err()); +} + +#[tokio::test] +async fn direct_urls_read_multi_extent_ranges_without_any_file_repository() { + let blocks = Arc::new(blocks::TestBlocks::default()); + let table = table(); + let file = FileId::random(); + let owner = FileIdentity { table, file }; + let mut writer = FileTreeWriter::new(blocks.clone(), owner, 3).unwrap(); + writer.push(b"abcdefghij").await.unwrap(); + let tree = writer.finish().await.unwrap(); + let signer = DirectFileLocations::new(1, BTreeMap::from([(1, [1; 32])])).unwrap(); + let record = signer + .parquet( + table, + "part.0001", + file, + tree.length, + tree.digest, + tree.root.unwrap(), + ) + .unwrap(); + let location = record.location.to_string().parse().unwrap(); + let resolved = signer.resolve(&location).unwrap().unwrap(); + assert_eq!(resolved, record); + assert!(location.object_key().len() <= 1024); + let mut reader = FileReader::new(blocks, resolved, Some(ByteRange { start: 2, end: 9 }), 3).unwrap(); + let mut bytes = Vec::new(); + while let Some(chunk) = reader.next().await.unwrap() { + bytes.extend_from_slice(&chunk); + } + assert_eq!(bytes, b"cdefghi"); +} + +#[tokio::test] +async fn locator_rotation_preserves_old_urls_and_tampering_never_resolves() { + let blocks = Arc::new(blocks::TestBlocks::default()); + let table = table(); + let file = FileId::random(); + let mut writer = FileTreeWriter::new(blocks, FileIdentity { table, file }, 1024).unwrap(); + writer.push(b"retained bytes").await.unwrap(); + let tree = writer.finish().await.unwrap(); + let signer = DirectFileLocations::new(1, BTreeMap::from([(1, [1; 32])])).unwrap(); + let record = signer + .parquet(table, "part", file, tree.length, tree.digest, tree.root.unwrap()) + .unwrap(); + let rotated = DirectFileLocations::new(2, BTreeMap::from([(1, [1; 32]), (2, [2; 32])])).unwrap(); + assert_eq!(rotated.resolve(&record.location).unwrap(), Some(record.clone())); + let mut foreign = table; + foreign.storage_namespace = Some(NamespaceId::random()); + assert!(rotated + .resolve(&foreign.file(record.location.relative_key()).unwrap()) + .is_err()); + let changed_name = record.location.relative_key().replacen("part.", "other.", 1); + assert!(rotated.resolve(&table.file(&changed_name).unwrap()).is_err()); + let uri = record.location.to_string(); + let marker = uri.find("part.").unwrap() + 5; + let mut changed = uri.into_bytes(); + changed[marker + 20] = if changed[marker + 20] == b'A' { b'B' } else { b'A' }; + assert!(rotated + .resolve(&String::from_utf8(changed).unwrap().parse().unwrap()) + .is_err()); + assert!(rotated + .resolve(&table.file("metadata/v1.json").unwrap()) + .unwrap() + .is_none()); +} diff --git a/lib/crowdb-access-iceberg/tests/file_assembly_pipeline_test.rs b/lib/crowdb-access-iceberg/tests/file_assembly_pipeline_test.rs index cefb85cf0..8be491210 100644 --- a/lib/crowdb-access-iceberg/tests/file_assembly_pipeline_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_assembly_pipeline_test.rs @@ -57,6 +57,7 @@ async fn setup() -> (Arc, FileAssembly, AssemblyPart, FileId let store = Arc::new(TestPipelineBlocks::default()); let owner = FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_assembly_test.rs b/lib/crowdb-access-iceberg/tests/file_assembly_test.rs index f281088d2..df9129395 100644 --- a/lib/crowdb-access-iceberg/tests/file_assembly_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_assembly_test.rs @@ -12,6 +12,7 @@ use std::sync::{atomic::Ordering, Arc}; fn owner() -> FileIdentity { FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_avro_test.rs b/lib/crowdb-access-iceberg/tests/file_avro_test.rs index c33b4c7bc..842e86cff 100644 --- a/lib/crowdb-access-iceberg/tests/file_avro_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_avro_test.rs @@ -124,6 +124,7 @@ fn block(bytes: &mut Vec, count: i64, encoded: &[u8]) { async fn record(store: Arc, bytes: &[u8]) -> FileRecord { let owner = FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_checkpoint_test.rs b/lib/crowdb-access-iceberg/tests/file_checkpoint_test.rs index df1ed95cc..7e3384634 100644 --- a/lib/crowdb-access-iceberg/tests/file_checkpoint_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_checkpoint_test.rs @@ -13,6 +13,7 @@ use std::sync::{atomic::Ordering, Arc}; fn owner() -> FileIdentity { FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_credentials_test.rs b/lib/crowdb-access-iceberg/tests/file_credentials_test.rs index 6857043c4..b3c251801 100644 --- a/lib/crowdb-access-iceberg/tests/file_credentials_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_credentials_test.rs @@ -6,6 +6,7 @@ use crowdb_access_iceberg::key::{CatalogId, OperationId, TableId}; fn grant() -> FileGrant { FileGrant { + storage_namespace: None, context: CatalogContext { catalog: CatalogId::random(), activation_epoch: 1, @@ -48,10 +49,39 @@ fn delegated_credentials_verify_across_servers_without_storing_secret_material() assert_ne!(issued.access_key_id(), another.access_key_id()); } +#[test] +fn cached_validation_still_checks_expiry_and_catalog_epoch() { + let issuer = FileGrantIssuer::new([2; 32], 10_000).unwrap(); + let grant = grant(); + let credentials = issuer.issue(grant.clone()).unwrap(); + for _ in 0..2 { + assert_eq!( + issuer + .verify_token(credentials.session_token(), grant.context, grant.issued_ms) + .unwrap() + .grant(), + &grant + ); + } + assert!(matches!( + issuer.verify_token(credentials.session_token(), grant.context, grant.expires_ms), + Err(FileGrantError::Expired) + )); + let changed = CatalogContext { + activation_epoch: grant.context.activation_epoch + 1, + ..grant.context + }; + assert!(matches!( + issuer.verify_token(credentials.session_token(), changed, grant.issued_ms), + Err(FileGrantError::Forbidden) + )); +} + #[test] fn grants_bind_exact_table_operations_and_independent_byte_limits() { let grant = grant(); let location = TableLocation { + storage_namespace: None, catalog: grant.context.catalog, table: grant.table, } @@ -72,10 +102,12 @@ fn grants_bind_exact_table_operations_and_independent_byte_limits() { ); for table in [ TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: grant.table, }, TableLocation { + storage_namespace: None, catalog: grant.context.catalog, table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_digest_test.rs b/lib/crowdb-access-iceberg/tests/file_digest_test.rs index 7b963a05b..a87d08857 100644 --- a/lib/crowdb-access-iceberg/tests/file_digest_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_digest_test.rs @@ -5,6 +5,7 @@ use sha2::{Digest, Sha256}; fn owner() -> FileIdentity { FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_directory_test.rs b/lib/crowdb-access-iceberg/tests/file_directory_test.rs index c45a57346..a11d6b2b0 100644 --- a/lib/crowdb-access-iceberg/tests/file_directory_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_directory_test.rs @@ -8,6 +8,7 @@ fn directory(count: usize) -> ChunkDirectory { ChunkDirectory { owner: FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, @@ -34,6 +35,23 @@ fn directory(count: usize) -> ChunkDirectory { } } +#[test] +fn native_directories_preserve_storage_namespace_and_reject_foreign_owners() { + let mut node = directory(2); + node.owner.table.storage_namespace = Some(crowdb_access_iceberg::key::NamespaceId::random()); + let bytes = node.encode().unwrap(); + assert_eq!(&bytes[..5], b"ICEN\x02"); + assert_eq!( + ChunkDirectory::decode(&bytes, node.owner, node.height, node.length().unwrap()).unwrap(), + node + ); + let mut foreign = node.owner; + foreign.table.storage_namespace = Some(crowdb_access_iceberg::key::NamespaceId::random()); + assert!(ChunkDirectory::decode(&bytes, foreign, node.height, node.length().unwrap()).is_err()); + foreign.table.storage_namespace = None; + assert!(ChunkDirectory::decode(&bytes, foreign, node.height, node.length().unwrap()).is_err()); +} + #[test] fn directories_bind_every_child_to_a_bounded_height_owner_and_span() { for count in [1, 2, MAX_DIRECTORY_ENTRIES] { diff --git a/lib/crowdb-access-iceberg/tests/file_format_test.rs b/lib/crowdb-access-iceberg/tests/file_format_test.rs index cf0b3f71b..cbab4348d 100644 --- a/lib/crowdb-access-iceberg/tests/file_format_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_format_test.rs @@ -13,6 +13,7 @@ use crowdb_access_iceberg::key::{CatalogId, FileId, TableId}; async fn record(store: Arc, bytes: &[u8], block_size: usize) -> FileRecord { let owner = FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_json_test.rs b/lib/crowdb-access-iceberg/tests/file_json_test.rs index 12b453aa8..6e1cbbbf8 100644 --- a/lib/crowdb-access-iceberg/tests/file_json_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_json_test.rs @@ -14,6 +14,7 @@ use crowdb_access_iceberg::key::{CatalogId, FileId, TableId}; async fn record(store: Arc, bytes: &[u8], block_bytes: usize) -> FileRecord { let owner = FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_listing_test.rs b/lib/crowdb-access-iceberg/tests/file_listing_test.rs index 3539c78d4..3e587c034 100644 --- a/lib/crowdb-access-iceberg/tests/file_listing_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_listing_test.rs @@ -14,6 +14,7 @@ use fixture::TestFile; fn grant(fixture: &TestFile) -> FileGrant { FileGrant { + storage_namespace: None, context: fixture.context, table: fixture.table.table, principal: [7; 32], @@ -80,6 +81,7 @@ async fn two_table_principals_cannot_discover_each_others_published_names() { let selected = fixture.record("data/own.json", b"{}"); repository.publish(fixture.context, &selected).await.unwrap(); let other_table = crowdb_access_iceberg::file::TableLocation { + storage_namespace: None, catalog: fixture.context.catalog, table: TableId::random(), }; diff --git a/lib/crowdb-access-iceberg/tests/file_location_test.rs b/lib/crowdb-access-iceberg/tests/file_location_test.rs index d7f780e10..a0a702588 100644 --- a/lib/crowdb-access-iceberg/tests/file_location_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_location_test.rs @@ -3,6 +3,7 @@ use crowdb_access_iceberg::key::{CatalogId, TableId}; fn table() -> TableLocation { TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[0xff; 16]).unwrap(), table: TableId::from_bytes(&[0xab; 16]).unwrap(), } diff --git a/lib/crowdb-access-iceberg/tests/file_record_test.rs b/lib/crowdb-access-iceberg/tests/file_record_test.rs index ed251e8a8..ac83159e2 100644 --- a/lib/crowdb-access-iceberg/tests/file_record_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_record_test.rs @@ -12,6 +12,7 @@ fn record(input: &[u8]) -> FileRecord { FileRecord { file: FileId::random(), location: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), } diff --git a/lib/crowdb-access-iceberg/tests/file_scale_test.rs b/lib/crowdb-access-iceberg/tests/file_scale_test.rs index dacadbb6d..8af3b5521 100644 --- a/lib/crowdb-access-iceberg/tests/file_scale_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_scale_test.rs @@ -16,6 +16,7 @@ use crowdb_access_iceberg::{ async fn logical_tib(store: &blocks::TestBlocks) -> FileRecord { let owner = FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_seal_test.rs b/lib/crowdb-access-iceberg/tests/file_seal_test.rs index 426dd3208..c056a4a39 100644 --- a/lib/crowdb-access-iceberg/tests/file_seal_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_seal_test.rs @@ -14,6 +14,7 @@ use crowdb_access_iceberg::record::StorageRecord; fn owner() -> FileIdentity { FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_staging_test.rs b/lib/crowdb-access-iceberg/tests/file_staging_test.rs index e237da1dd..253d8de0e 100644 --- a/lib/crowdb-access-iceberg/tests/file_staging_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_staging_test.rs @@ -12,6 +12,7 @@ use std::sync::Arc; fn owner() -> FileIdentity { FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_stream_test.rs b/lib/crowdb-access-iceberg/tests/file_stream_test.rs index 5157ef6ac..8db08a7c3 100644 --- a/lib/crowdb-access-iceberg/tests/file_stream_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_stream_test.rs @@ -13,6 +13,7 @@ use crowdb_access_iceberg::key::{CatalogId, FileId, TableId}; fn owner() -> FileIdentity { FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/gc_reachability_test.rs b/lib/crowdb-access-iceberg/tests/gc_reachability_test.rs index 73f078328..2969b5490 100644 --- a/lib/crowdb-access-iceberg/tests/gc_reachability_test.rs +++ b/lib/crowdb-access-iceberg/tests/gc_reachability_test.rs @@ -16,6 +16,7 @@ mod blocks; fn table() -> TableLocation { TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), } diff --git a/lib/crowdb-access-iceberg/tests/gc_tree_test.rs b/lib/crowdb-access-iceberg/tests/gc_tree_test.rs index 5e2def6aa..a26237edc 100644 --- a/lib/crowdb-access-iceberg/tests/gc_tree_test.rs +++ b/lib/crowdb-access-iceberg/tests/gc_tree_test.rs @@ -18,6 +18,7 @@ async fn directory_deletion_resumes_without_rereading_deleted_children() { let blocks = Arc::new(blocks::TestBlocks::default()); let owner = FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, @@ -97,6 +98,7 @@ async fn corrupt_directory_never_authorizes_a_child_deletion() { let blocks = blocks::TestBlocks::default(); let owner = FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/manifest_metadata_test.rs b/lib/crowdb-access-iceberg/tests/manifest_metadata_test.rs index ef17f0281..7170e051b 100644 --- a/lib/crowdb-access-iceberg/tests/manifest_metadata_test.rs +++ b/lib/crowdb-access-iceberg/tests/manifest_metadata_test.rs @@ -8,6 +8,7 @@ use crowdb_access_iceberg::manifest::{ fn table() -> TableLocation { TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[1; 16]).unwrap(), table: TableId::from_bytes(&[2; 16]).unwrap(), } diff --git a/lib/crowdb-access-iceberg/tests/multipart_record_test.rs b/lib/crowdb-access-iceberg/tests/multipart_record_test.rs index 22a1507ba..d5ac466a8 100644 --- a/lib/crowdb-access-iceberg/tests/multipart_record_test.rs +++ b/lib/crowdb-access-iceberg/tests/multipart_record_test.rs @@ -11,6 +11,20 @@ use crowdb_access_iceberg::key::{CatalogId, CatalogScope, FileId, IcebergKey}; use crowdb_access_iceberg::record::StorageRecord; use std::sync::Arc; +#[tokio::test] +async fn native_multipart_identity_round_trips_the_full_storage_path() { + let mut session = fixtures::session(); + session.owner.table.storage_namespace = Some(crowdb_access_iceberg::key::NamespaceId::random()); + session.location = session.owner.table.file(session.location.relative_key()).unwrap(); + let record = StorageRecord::MultipartSession(Box::new(session.clone())); + assert_eq!( + StorageRecord::decode(&session.key(), &record.encode().unwrap()).unwrap(), + record + ); + session.owner.table.storage_namespace = Some(crowdb_access_iceberg::key::NamespaceId::random()); + assert!(session.validate().is_err()); +} + #[tokio::test] async fn multipart_records_round_trip_open_partial_publishing_published_and_aborted_states() { let store = Arc::new(TestBlocks::default()); diff --git a/lib/crowdb-access-iceberg/tests/native_file_runtime_test.rs b/lib/crowdb-access-iceberg/tests/native_file_runtime_test.rs new file mode 100644 index 000000000..e8aef610b --- /dev/null +++ b/lib/crowdb-access-iceberg/tests/native_file_runtime_test.rs @@ -0,0 +1,211 @@ +#[path = "common/store.rs"] +mod common; +#[path = "common/file.rs"] +mod fixture; + +use std::sync::atomic::Ordering; + +use crowdb_access_iceberg::{ + file::{ + canonical_file_key, FileDigest, FileGrant, FileGrantIssuer, FileIdentity, FileListRequest, + FileListTokens, FileOperation, FileOperations, FileRepository, TableLocation, + }, + key::{NamespaceId, OperationId, StorageKey, StorageTablePath, TableStorageScope}, + record::StorageRecord, +}; + +async fn native_fixture() -> fixture::TestFile { + let mut fixture = fixture::TestFile::new(common::TestStore::default()).await; + fixture.table = StorageTablePath { + catalog: fixture.context.catalog, + storage_namespace: NamespaceId::random(), + table: fixture.table.table, + } + .into(); + fixture +} + +fn grant(fixture: &fixture::TestFile) -> FileGrant { + FileGrant { + storage_namespace: fixture.table.storage_namespace, + context: fixture.context, + table: fixture.table.table, + principal: [1; 32], + nonce: OperationId::random(), + issued_ms: 100, + expires_ms: 10_000, + operations: FileOperations::new(&[FileOperation::Get, FileOperation::ListObjects]).unwrap(), + max_request_bytes: 1024, + max_file_bytes: 1024, + } +} + +#[tokio::test] +async fn native_publication_has_one_filename_record_and_conflicts_preserve_bytes() { + let fixture = native_fixture().await; + let repository = FileRepository::new(fixture.store.clone()); + let candidate = fixture.record("metadata/v1.json", b"{}"); + assert_eq!( + repository.publish(fixture.context, &candidate).await.unwrap(), + candidate + ); + let equal = fixture.record("metadata/v1.json", b"{}"); + assert_eq!( + repository.publish(fixture.context, &equal).await.unwrap(), + candidate + ); + assert!(repository + .publish(fixture.context, &fixture.record("metadata/v1.json", b"[]")) + .await + .is_err()); + assert_eq!( + repository + .load(fixture.context, &candidate.location) + .await + .unwrap(), + Some(candidate.clone()) + ); + let key = canonical_file_key(&candidate.location); + let snapshot = fixture.store.values.load_full(); + let files: Vec<_> = snapshot.keys().filter(|key| matches!(StorageKey::decode(key), Ok(StorageKey::NativeTable(key)) if key.scope == TableStorageScope::File)).collect(); + assert_eq!(files, [&key.encode().unwrap()]); + assert_eq!( + StorageRecord::decode_storage(&key, &snapshot[&key.encode().unwrap()].bytes).unwrap(), + StorageRecord::File(Box::new(candidate.clone())) + ); + let mut foreign = candidate.location.table(); + foreign.storage_namespace = Some(NamespaceId::random()); + let foreign = canonical_file_key(&foreign.file(candidate.location.relative_key()).unwrap()); + assert!(StorageRecord::decode_storage(&foreign, &snapshot[&key.encode().unwrap()].bytes).is_err()); +} + +#[tokio::test] +async fn native_publication_lost_reply_replays_the_same_single_record() { + let fixture = native_fixture().await; + let repository = FileRepository::new(fixture.store.clone()); + let candidate = fixture.record("metadata/v1.json", b"{}"); + fixture.store.file_record_reply_loss.store(true, Ordering::SeqCst); + assert!(repository.publish(fixture.context, &candidate).await.is_err()); + let restarted = FileRepository::new(fixture.store.clone()); + assert_eq!( + restarted.publish(fixture.context, &candidate).await.unwrap(), + candidate + ); + assert_eq!( + restarted + .load(fixture.context, &candidate.location) + .await + .unwrap(), + Some(candidate) + ); +} + +#[tokio::test] +async fn many_metadata_files_page_by_public_prefix_without_file_resolution_gets() { + let fixture = native_fixture().await; + let repository = FileRepository::new(fixture.store.clone()); + for index in 0..600 { + for prefix in ["metadata", "other"] { + let record = fixture.record(&format!("{prefix}/v{index:04}.json"), b"{}"); + repository.publish(fixture.context, &record).await.unwrap(); + } + } + let tokens = FileListTokens::new([1; 32]).unwrap(); + let grant = grant(&fixture); + let mut request = FileListRequest { + table: fixture.table, + prefix: format!("{}metadata/", fixture.table.object_prefix()), + delimiter: None, + encoding_url: false, + max_keys: 73, + continuation_token: None, + start_after: None, + }; + let mut keys = Vec::new(); + loop { + let before = fixture.store.reads.load(Ordering::SeqCst); + let page = repository.list(&grant, &request, &tokens, 200).await.unwrap(); + // The two Ready root checks are independent of file-record resolution. + assert_eq!(fixture.store.reads.load(Ordering::SeqCst) - before, 2); + keys.extend(page.files.into_iter().map(|file| file.key)); + request.continuation_token = page.next_continuation_token; + if request.continuation_token.is_none() { + break; + } + } + let expected: Vec<_> = (0..600) + .map(|index| format!("{}metadata/v{index:04}.json", fixture.table.object_prefix())) + .collect(); + assert_eq!(keys, expected); + request.prefix = format!("{}metadata/v000", fixture.table.object_prefix()); + let page = repository.list(&grant, &request, &tokens, 200).await.unwrap(); + assert_eq!(page.files.len(), 10); + let mut foreign_grant = grant.clone(); + foreign_grant.storage_namespace = Some(NamespaceId::random()); + assert!(repository + .list(&foreign_grant, &request, &tokens, 200) + .await + .is_err()); +} + +#[tokio::test] +async fn native_grants_and_digest_checkpoints_bind_the_storage_namespace() { + let fixture = native_fixture().await; + let issuer = FileGrantIssuer::new([2; 32], 10_000).unwrap(); + let grant = grant(&fixture); + let credentials = issuer.issue(grant.clone()).unwrap(); + let verified = issuer + .verify( + credentials.access_key_id(), + credentials.session_token(), + fixture.context, + 200, + ) + .unwrap(); + assert_eq!(verified.grant(), &grant); + let location = fixture.table.file("metadata/v1.json").unwrap(); + verified + .grant() + .authorize(FileOperation::Get, &location, 0, 2) + .unwrap(); + let mut foreign = fixture.table; + foreign.storage_namespace = Some(NamespaceId::random()); + assert!(verified + .grant() + .authorize( + FileOperation::Get, + &foreign.file("metadata/v1.json").unwrap(), + 0, + 2 + ) + .is_err()); + let owner = FileIdentity { + table: fixture.table, + file: crowdb_access_iceberg::key::FileId::random(), + }; + let mut digest = FileDigest::new(owner); + digest.update(b"hello").unwrap(); + let checkpoint = digest.checkpoint(); + let restored = FileDigest::restore(owner, &checkpoint).unwrap(); + assert_eq!(restored.finish(), digest.finish()); + assert!(FileDigest::restore( + FileIdentity { + table: foreign, + ..owner + }, + &checkpoint + ) + .is_err()); + let legacy: TableLocation = TableLocation { + storage_namespace: None, + ..fixture.table + }; + assert!(FileDigest::restore( + FileIdentity { + table: legacy, + ..owner + }, + &checkpoint + ) + .is_err()); +} diff --git a/lib/crowdb-access-iceberg/tests/parquet_delete_encoding_test.rs b/lib/crowdb-access-iceberg/tests/parquet_delete_encoding_test.rs index 7bd262f9a..b157ab622 100644 --- a/lib/crowdb-access-iceberg/tests/parquet_delete_encoding_test.rs +++ b/lib/crowdb-access-iceberg/tests/parquet_delete_encoding_test.rs @@ -30,6 +30,7 @@ impl PositionDeleteTargets for TestTargets { fn table() -> TableLocation { TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[1; 16]).unwrap(), table: TableId::from_bytes(&[2; 16]).unwrap(), } diff --git a/lib/crowdb-access-iceberg/tests/parquet_nullable_test.rs b/lib/crowdb-access-iceberg/tests/parquet_nullable_test.rs index 49703497f..43e66fc00 100644 --- a/lib/crowdb-access-iceberg/tests/parquet_nullable_test.rs +++ b/lib/crowdb-access-iceberg/tests/parquet_nullable_test.rs @@ -29,6 +29,7 @@ async fn official_parquet_java_v1_v2_nested_and_all_null_files_decode() { let (store, record) = fixture::stored_content( &bytes, TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[1; 16]).unwrap(), table: TableId::from_bytes(&[2; 16]).unwrap(), }, diff --git a/lib/crowdb-access-iceberg/tests/parquet_position_delete_test.rs b/lib/crowdb-access-iceberg/tests/parquet_position_delete_test.rs index b1eac8cf0..68d47e501 100644 --- a/lib/crowdb-access-iceberg/tests/parquet_position_delete_test.rs +++ b/lib/crowdb-access-iceberg/tests/parquet_position_delete_test.rs @@ -34,6 +34,7 @@ async fn official_iceberg_java_zstd_v1_and_v2_delete_files_decode() { use crowdb_access_iceberg::file::TableLocation; use crowdb_access_iceberg::key::{CatalogId, TableId}; let table = TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[1; 16]).unwrap(), table: TableId::from_bytes(&[2; 16]).unwrap(), }; @@ -56,6 +57,7 @@ async fn canonical_storage_does_not_make_corrupted_sdk_pages_valid() { use crowdb_access_iceberg::file::TableLocation; use crowdb_access_iceberg::key::{CatalogId, TableId}; let table = TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[1; 16]).unwrap(), table: TableId::from_bytes(&[2; 16]).unwrap(), }; diff --git a/lib/crowdb-access-iceberg/tests/parquet_scalar_test.rs b/lib/crowdb-access-iceberg/tests/parquet_scalar_test.rs index 5b824336f..61901df9a 100644 --- a/lib/crowdb-access-iceberg/tests/parquet_scalar_test.rs +++ b/lib/crowdb-access-iceberg/tests/parquet_scalar_test.rs @@ -34,6 +34,7 @@ async fn official_java_v1_v2_nullable_scalar_files_decode_with_and_without_dicti let (store, record) = fixture::stored_content( &bytes, crowdb_access_iceberg::file::TableLocation { + storage_namespace: None, catalog: crowdb_access_iceberg::key::CatalogId::from_bytes(&[1; 16]).unwrap(), table: crowdb_access_iceberg::key::TableId::from_bytes(&[2; 16]).unwrap(), }, diff --git a/lib/crowdb-access-iceberg/tests/table_commit_publication_test.rs b/lib/crowdb-access-iceberg/tests/table_commit_publication_test.rs index d289c9561..dda2fd5b2 100644 --- a/lib/crowdb-access-iceberg/tests/table_commit_publication_test.rs +++ b/lib/crowdb-access-iceberg/tests/table_commit_publication_test.rs @@ -168,6 +168,7 @@ async fn lifecycle_request( .put( name_key(head.catalog, head.namespace, &head.name).unwrap(), StorageRecord::TableMapping(TableMapping { + storage_namespace: None, catalog: head.catalog, namespace: head.namespace, name: head.name.clone(), diff --git a/lib/crowdb-access-iceberg/tests/table_repository_test.rs b/lib/crowdb-access-iceberg/tests/table_repository_test.rs index 53cc66910..521e1f8da 100644 --- a/lib/crowdb-access-iceberg/tests/table_repository_test.rs +++ b/lib/crowdb-access-iceberg/tests/table_repository_test.rs @@ -36,6 +36,7 @@ fn head(record: &FileRecord) -> TableHead { fn mapping(head: &TableHead) -> TableMapping { TableMapping { + storage_namespace: None, catalog: head.catalog, namespace: head.namespace, name: head.name.clone(), @@ -75,6 +76,101 @@ async fn new_fixture() -> fixture::TestFile { fixture } +#[tokio::test] +async fn name_mappings_preserve_storage_namespace_across_live_namespace_moves() { + let fixture = new_fixture().await; + let mut file = fixture.record("metadata/one.json", b"{}"); + let mut location = file.location.table(); + let storage_namespace = NamespaceId::random(); + location.storage_namespace = Some(storage_namespace); + file.location = location.file("metadata/one.json").unwrap(); + let mut head = head(&file); + let mut mapping = mapping(&head); + mapping.storage_namespace = Some(storage_namespace); + assert!(mapping.resolves(&head)); + FileRepository::new(fixture.store.clone()) + .publish(fixture.context, &file) + .await + .unwrap(); + let canonical = crowdb_access_iceberg::table::storage_head_key(head.storage_location()); + let encoded = canonical.encode().unwrap(); + let initial = StorageRecord::TableHead(Box::new(head.clone())).encode().unwrap(); + fixture + .store + .compare_exchange( + &encoded, + None, + &initial, + mutation_identity(&encoded, None, &initial), + ) + .await + .unwrap(); + put( + &fixture.store, + name_key(head.catalog, head.namespace, &head.name).unwrap(), + StorageRecord::TableMapping(mapping.clone()), + ) + .await; + let repository = TableRepository::new(fixture.store.clone()); + assert_eq!( + repository + .select(fixture.context, head.namespace, &head.name) + .await + .unwrap() + .unwrap() + .head, + head + ); + let original = head.metadata_location.clone(); + head.namespace = NamespaceId::random(); + head.name = "moved".into(); + head.name_epoch += 1; + mapping.namespace = head.namespace; + mapping.name.clone_from(&head.name); + mapping.name_epoch = head.name_epoch; + assert!(mapping.resolves(&head)); + assert_eq!(head.metadata_location, original); + assert_eq!( + crowdb_access_iceberg::table::storage_head_key(head.storage_location()), + canonical + ); + let moved = StorageRecord::TableHead(Box::new(head.clone())).encode().unwrap(); + fixture + .store + .compare_exchange( + &encoded, + Some(&initial), + &moved, + mutation_identity(&encoded, Some(&initial), &moved), + ) + .await + .unwrap(); + put( + &fixture.store, + name_key(head.catalog, head.namespace, &head.name).unwrap(), + StorageRecord::TableMapping(mapping.clone()), + ) + .await; + let selected = repository + .select(fixture.context, head.namespace, &head.name) + .await + .unwrap() + .unwrap(); + repository + .ensure_current(fixture.context, &selected) + .await + .unwrap(); + assert_eq!(selected.metadata.location, original); + let key = name_key(head.catalog, head.namespace, &head.name).unwrap(); + let record = StorageRecord::TableMapping(mapping.clone()); + assert_eq!( + StorageRecord::decode(&key, &record.encode().unwrap()).unwrap(), + record + ); + mapping.storage_namespace = Some(NamespaceId::random()); + assert!(!mapping.resolves(&head)); +} + #[tokio::test] async fn selection_pins_one_head_and_immutable_metadata_generation() { let fixture = new_fixture().await; diff --git a/lib/crowdb-access-iceberg/tests/wire_credentials_test.rs b/lib/crowdb-access-iceberg/tests/wire_credentials_test.rs index 7a1f04bb6..e01346403 100644 --- a/lib/crowdb-access-iceberg/tests/wire_credentials_test.rs +++ b/lib/crowdb-access-iceberg/tests/wire_credentials_test.rs @@ -33,6 +33,7 @@ fn authority(context: CatalogContext) -> CatalogAuthority { fn target(table: TableId) -> FileDelegationTarget { FileDelegationTarget { + storage_namespace: None, table, format_version: 2, staged: false, @@ -46,6 +47,7 @@ fn only_independent_writer_receives_file_mutations() { let context = context(); let table = TableId::random(); let location = TableLocation { + storage_namespace: None, catalog: context.catalog, table, } @@ -99,6 +101,7 @@ fn only_independent_writer_receives_file_mutations() { Err(FileGrantError::Bounds) ); let other = TableLocation { + storage_namespace: None, catalog: context.catalog, table: TableId::random(), } @@ -136,6 +139,7 @@ fn delegated_file_operations_follow_selected_version_and_draft_stage() { context, &authority, FileDelegationTarget { + storage_namespace: None, table, format_version: 2, staged: true, @@ -151,6 +155,7 @@ fn delegated_file_operations_follow_selected_version_and_draft_stage() { context, &authority, FileDelegationTarget { + storage_namespace: None, table, format_version: 3, staged: false, @@ -201,6 +206,7 @@ fn refresh_rotates_credentials_and_serializes_official_sdk_properties() { assert_eq!( credentials[0]["prefix"], TableLocation { + storage_namespace: None, catalog: context.catalog, table } diff --git a/lib/crowdb-protocol/src/fbs/iceberg.fbs b/lib/crowdb-protocol/src/fbs/iceberg.fbs index 9fbe50fab..411220abf 100644 --- a/lib/crowdb-protocol/src/fbs/iceberg.fbs +++ b/lib/crowdb-protocol/src/fbs/iceberg.fbs @@ -189,6 +189,7 @@ table FBFileIdentity { catalog:[ubyte] (required); table_id:[ubyte] (required); file_id:[ubyte] (required); + storage_namespace:[ubyte]; } table FBFileTree { @@ -312,6 +313,7 @@ table FBTableMapping { name_epoch:ulong; operation:[ubyte] (required); state:ubyte = 255; + storage_namespace:[ubyte]; } table FBTableCommitOperation { @@ -450,6 +452,7 @@ table FBFileWriteIntent { created_ms:ulong; not_before_ms:ulong; deleting:bool; + storage_namespace:[ubyte]; } table FBGcEntry { From 05035984545a2c7652cd542811e9ab5499bfedbf Mon Sep 17 00:00:00 2001 From: Gian Date: Wed, 14 Oct 2026 23:43:18 +0800 Subject: [PATCH 03/15] Enable native Iceberg catalogs and durable injection URLs --- app/crowdb-access-server/Cargo.toml | 2 +- app/crowdb-access-server/src/config.rs | 17 + app/crowdb-access-server/src/iceberg.rs | 2 + .../src/iceberg/file_body.rs | 7 + .../src/iceberg/file_http.rs | 133 +++--- .../src/iceberg/file_http/injection.rs | 94 +++++ .../src/iceberg/file_http/read.rs | 110 +++++ .../src/iceberg/file_read_conditions.rs | 71 ++++ .../src/iceberg/file_request.rs | 18 + .../src/iceberg/file_response.rs | 4 + .../src/iceberg/gc_control.rs | 30 +- .../src/iceberg/gc_runtime.rs | 2 + app/crowdb-access-server/src/iceberg/http.rs | 22 +- .../src/iceberg/runtime.rs | 7 + .../src/iceberg/table_credentials.rs | 114 ++++- .../tests/common/iceberg_commit_fault.rs | 11 +- .../common/iceberg_commit_recovery/blocks.rs | 9 + .../tests/common/iceberg_store.rs | 56 ++- .../tests/iceberg_commit_recovery_test.rs | 3 +- .../tests/iceberg_commit_sdk_test.rs | 8 +- .../tests/iceberg_file_recovery_test.rs | 2 +- .../tests/iceberg_file_request_test.rs | 38 ++ .../tests/iceberg_file_response_test.rs | 34 ++ .../iceberg_native_injection_http_test.rs | 397 ++++++++++++++++++ .../tests/iceberg_table_acceptance_test.rs | 26 +- .../tests/iceberg_table_lifecycle_test.rs | 12 +- container/single-node-container/README.md | 7 +- .../design-crowdb-iceberg-upload-flow.md | 26 +- .../iceberge/design-crowdb-iceberg.md | 26 +- doc/working/plan-iceberg-storage-path.md | 68 ++- e2e-test/tests/access_iceberg/native_files.rs | 1 + lib/crowdb-access-iceberg/src/catalog.rs | 5 +- .../src/catalog/repository.rs | 2 +- .../src/catalog/state.rs | 73 ++++ .../commit/create/publisher/preparation.rs | 55 ++- lib/crowdb-access-iceberg/src/file.rs | 2 + .../src/file/direct_location.rs | 40 +- .../src/file/injection.rs | 202 +++++++++ lib/crowdb-access-iceberg/src/file/reader.rs | 37 ++ lib/crowdb-access-iceberg/src/file/writer.rs | 5 + lib/crowdb-access-iceberg/src/gc/discovery.rs | 75 ++-- lib/crowdb-access-iceberg/src/gc/mark.rs | 4 +- lib/crowdb-access-iceberg/src/gc/storage.rs | 41 +- lib/crowdb-access-iceberg/src/gc/task.rs | 33 +- lib/crowdb-access-iceberg/src/gc/worker.rs | 6 +- .../src/gc/worker/cleanup.rs | 59 +++ .../src/gc/worker/inactive.rs | 3 +- .../src/gc/worker/live.rs | 4 +- .../src/gc/worker/multipart.rs | 2 + .../src/gc/worker/native.rs | 82 ++++ .../src/gc/worker/sweep.rs | 21 +- .../src/gc/worker/terminal.rs | 1 + .../src/gc/worker/writes.rs | 1 + .../src/metadata_projection/model.rs | 16 +- .../src/namespace/create.rs | 9 + .../src/namespace/repository.rs | 28 ++ lib/crowdb-access-iceberg/src/record.rs | 1 + .../src/record/authority.rs | 14 +- .../src/record/envelope.rs | 33 +- lib/crowdb-access-iceberg/src/record/gc.rs | 2 + .../src/record/injection.rs | 45 ++ .../src/record/storage_key.rs | 9 + .../tests/common/store.rs | 11 + .../tests/direct_file_location_test.rs | 33 ++ .../tests/gc_proof_test.rs | 1 + .../tests/gc_record_test.rs | 2 + .../tests/gc_worker_test.rs | 71 +++- .../tests/metadata_projection_test.rs | 34 ++ .../tests/native_injection_test.rs | 90 ++++ .../tests/record_test.rs | 42 ++ .../tests/table_create_publication_test.rs | 82 ++++ lib/crowdb-protocol/src/fbs/iceberg.fbs | 13 +- tools/load-tpch.py | 4 +- 73 files changed, 2417 insertions(+), 233 deletions(-) create mode 100644 app/crowdb-access-server/src/iceberg/file_http/injection.rs create mode 100644 app/crowdb-access-server/src/iceberg/file_http/read.rs create mode 100644 app/crowdb-access-server/src/iceberg/file_read_conditions.rs create mode 100644 app/crowdb-access-server/tests/iceberg_native_injection_http_test.rs create mode 100644 lib/crowdb-access-iceberg/src/file/injection.rs create mode 100644 lib/crowdb-access-iceberg/src/gc/worker/native.rs create mode 100644 lib/crowdb-access-iceberg/src/record/injection.rs create mode 100644 lib/crowdb-access-iceberg/tests/native_injection_test.rs diff --git a/app/crowdb-access-server/Cargo.toml b/app/crowdb-access-server/Cargo.toml index 54273ec8e..46618583e 100644 --- a/app/crowdb-access-server/Cargo.toml +++ b/app/crowdb-access-server/Cargo.toml @@ -20,6 +20,7 @@ s3 = [ ] [dependencies] +arc-swap = "1.9" hex = "0.4" hmac = "0.12" async-trait = "0.1" @@ -59,7 +60,6 @@ bincode = "1" crowdb-access-server = { path = ".", default-features = false, features = ["test-util"] } futures = "0.3" crowdb-console-shared = { path = "../../lib/crowdb-console-shared" } -arc-swap = "1.9" async-trait = "0.1" crowdb-chunkdb-client = { path = "../../lib/crowdb-chunkdb-client" } crowdb-diskdb-client = { path = "../../lib/crowdb-diskdb-client" } diff --git a/app/crowdb-access-server/src/config.rs b/app/crowdb-access-server/src/config.rs index 779cacbbe..b87eae877 100644 --- a/app/crowdb-access-server/src/config.rs +++ b/app/crowdb-access-server/src/config.rs @@ -206,6 +206,7 @@ pub struct S3Config { #[derive(Clone, Debug, Default, Deserialize, Serialize)] #[serde(default)] pub struct IcebergConfig { + pub locator_signing: Option, /// Overrides the legacy shared small-write policy for Iceberg only. pub small_write: Option, pub listen: Option, @@ -223,6 +224,22 @@ pub struct IcebergConfig { pub gc: IcebergGcConfig, } +#[derive(Clone, Deserialize, Serialize)] +pub struct IcebergLocatorSigningConfig { + pub active_version: u32, + pub keys: std::collections::BTreeMap, +} + +impl std::fmt::Debug for IcebergLocatorSigningConfig { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("IcebergLocatorSigningConfig") + .field("active_version", &self.active_version) + .field("key_versions", &self.keys.keys().collect::>()) + .finish() + } +} + /// Dataset's listener and read-admission limits. Dataset is optional so an /// existing S3/Iceberg deployment keeps its current startup behavior. #[derive(Clone, Debug, Deserialize, Serialize)] diff --git a/app/crowdb-access-server/src/iceberg.rs b/app/crowdb-access-server/src/iceberg.rs index 67d4866e6..1a33f11cd 100644 --- a/app/crowdb-access-server/src/iceberg.rs +++ b/app/crowdb-access-server/src/iceberg.rs @@ -11,9 +11,11 @@ mod file_http; mod file_list_request; pub use file_bucket_request::{parse_file_bucket, FileBucketRequest}; mod file_list_response; +mod file_read_conditions; mod file_recovery; mod file_request; pub use file_list_request::parse_file_list; +pub use file_read_conditions::{file_read_condition, FileReadCondition}; mod file_response; mod file_selection; mod file_upload; diff --git a/app/crowdb-access-server/src/iceberg/file_body.rs b/app/crowdb-access-server/src/iceberg/file_body.rs index 913cdcaa7..c94897627 100644 --- a/app/crowdb-access-server/src/iceberg/file_body.rs +++ b/app/crowdb-access-server/src/iceberg/file_body.rs @@ -108,6 +108,13 @@ pub struct FileReadBody { } impl FileReadBody { + /// # Errors + /// Rejects preflight bytes for a stream or a different file owner/root. + pub fn with_root_directory(mut self, bytes: Vec) -> Result { + let reader = self.reader.take().ok_or(FileIoError::Bounds)?; + self.reader = Some(reader.with_root_directory(bytes)?); + Ok(self) + } fn finish(&mut self) { self.reader = None; self.pending = None; diff --git a/app/crowdb-access-server/src/iceberg/file_http.rs b/app/crowdb-access-server/src/iceberg/file_http.rs index e0e021de5..d43c524e1 100644 --- a/app/crowdb-access-server/src/iceberg/file_http.rs +++ b/app/crowdb-access-server/src/iceberg/file_http.rs @@ -6,9 +6,8 @@ use crowdb_access_iceberg::catalog::{ CatalogError, CatalogLifecycle, CatalogRepository, CatalogStore, RootState, }; use crowdb_access_iceberg::file::{ - resolve_range, DirectFileLocations, FileBlockStore, FileGrantError, FileGrantIssuer, FileOperation, - FileRecord, FileRepository, FileSealer, MultipartLister, MultipartPartStore, MultipartRepository, - RangeError, + DirectFileLocations, FileBlockStore, FileGrantError, FileGrantIssuer, FileOperation, FileRecord, + FileRepository, FileSealer, MultipartLister, MultipartPartStore, MultipartRepository, RangeError, }; use crowdb_access_iceberg::gc::{GcRepository, GcStore}; use crowdb_access_iceberg::key::OperationId; @@ -17,8 +16,8 @@ use crowdb_access_s3::auth::{RawAuthRequest, StreamingPayloadVerifier}; use crowdb_access_s3::native_buffer::{NativeBodyAllocator, NativeBodyReceiver}; use crowdb_chunk_client::LargeWritePolicy; use hyper::body::Incoming; -use hyper::http::header::{ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, ETAG, RANGE}; -use hyper::{Method, Request, Response, StatusCode}; +use hyper::http::header::{CONTENT_LENGTH, ETAG}; +use hyper::{Method, Request, Response}; use super::body::IcebergBody; use super::file_admission::{FileAdmissionError, FileServiceLimits, FileTransferAdmission}; @@ -31,14 +30,22 @@ use crate::upload_flow::body_encoding::UploadBody; mod bucket; mod delete; +mod injection; mod listing; mod metrics; mod multipart; +mod read; mod stream; use metrics::UploadFlowMetrics; pub use metrics::UploadFlowSnapshot; +struct CatalogLocators { + catalog: crowdb_access_iceberg::key::CatalogId, + key: Option, + locations: DirectFileLocations, +} + pub(super) struct FileHttp { repository: FileRepository, store: Arc, @@ -48,6 +55,8 @@ pub(super) struct FileHttp { blocks: Arc, issuer: FileGrantIssuer, direct_locations: DirectFileLocations, + locator_cache: arc_swap::ArcSwapOption, + locator_keys_configured: bool, responses: FileResponseBudget, uploads: FileUploadBudget, small_threshold_exclusive: usize, @@ -60,6 +69,36 @@ pub(super) struct FileHttp { } impl FileHttp { + pub(super) fn set_direct_locations(&mut self, locations: DirectFileLocations) { + self.direct_locations = locations; + self.locator_keys_configured = true; + self.locator_cache.store(None); + } + + fn catalog_locations( + &self, + authority: &crowdb_access_iceberg::catalog::CatalogAuthority, + ) -> Result, FileS3ErrorCode> { + if let Some(cached) = self.locator_cache.load_full() { + if cached.catalog == authority.catalog && cached.key == authority.locator_key { + return Ok(cached); + } + } + let locations = if let Some(key) = authority.locator_key { + self.direct_locations + .for_catalog(key, self.locator_keys_configured) + .map_err(|_| FileS3ErrorCode::InternalError)? + } else { + self.direct_locations.clone() + }; + let locators = Arc::new(CatalogLocators { + catalog: authority.catalog, + key: authority.locator_key, + locations, + }); + self.locator_cache.store(Some(locators.clone())); + Ok(locators) + } pub(super) fn native_receive_metrics( &self, ) -> Option { @@ -97,6 +136,8 @@ impl FileHttp { return Err(FileGrantError::Invalid); } Ok(Self { + locator_cache: arc_swap::ArcSwapOption::empty(), + locator_keys_configured: false, direct_locations: DirectFileLocations::new(1, std::collections::BTreeMap::from([(1, secret)])) .map_err(|_| FileGrantError::Invalid)?, list_tokens: crowdb_access_iceberg::file::FileListTokens::new(secret) @@ -245,11 +286,27 @@ impl FileHttp { .map_err(admission_error)?; let native_receiver = self.receive_provider(&mut request, file_request.operation, streaming.is_some()); + let locators = self.catalog_locations(&authority)?; match file_request.operation { FileOperation::Head | FileOperation::Get => { - self.read(&file_request, &request, root.context, &admission).await + self.read( + &file_request, + &request, + root.context, + &admission, + &locators.locations, + ) + .await } FileOperation::Put => { + if file_request.injection.is_some() { + if streaming.is_some() { + return Err(FileS3ErrorCode::InvalidRequest); + } + return self + .inject(&file_request, request, &admission, now_ms, &locators.locations) + .await; + } self.put( &file_request, request, @@ -360,70 +417,6 @@ impl FileHttp { set_header(&mut response, ETAG, &etag(&published))?; Ok(response) } - - async fn read( - &self, - file_request: &FileRequest, - request: &Request, - context: crowdb_access_iceberg::catalog::CatalogContext, - admission: &FileTransferAdmission, - ) -> Result, FileS3ErrorCode> { - let record = match self - .direct_locations - .resolve(&file_request.location) - .map_err(|_| FileS3ErrorCode::AccessDenied)? - { - Some(record) => record, - None => self - .repository - .load(context, &file_request.location) - .await - .map_err(catalog_error)? - .ok_or(FileS3ErrorCode::NoSuchKey)?, - }; - let range = match request - .headers() - .get_all(RANGE) - .iter() - .collect::>() - .as_slice() - { - [] => None, - [value] => Some(value.to_str().map_err(|_| FileS3ErrorCode::InvalidRange)?), - _ => return Err(FileS3ErrorCode::InvalidRange), - }; - let range = resolve_range(range, record.length).map_err(range_error)?; - let bytes = range.map_or(record.length, |range| range.end - range.start); - let body = if request.method() == Method::HEAD { - admission.check_bytes(0, record.length).map_err(admission_error)?; - IcebergBody::new(Vec::new()) - } else { - IcebergBody::file( - admission - .read_body(&self.responses, self.blocks.clone(), record.clone(), range) - .map_err(admission_error)?, - ) - }; - let mut response = Response::new(body); - *response.status_mut() = if range.is_some() { - StatusCode::PARTIAL_CONTENT - } else { - StatusCode::OK - }; - set_header(&mut response, CONTENT_LENGTH, &bytes.to_string())?; - set_header(&mut response, ETAG, &etag(&record))?; - response - .headers_mut() - .insert(ACCEPT_RANGES, hyper::header::HeaderValue::from_static("bytes")); - if let Some(range) = range { - set_header( - &mut response, - CONTENT_RANGE, - &format!("bytes {}-{}/{}", range.start, range.end - 1, record.length), - )?; - } - Ok(response) - } } fn set_header( diff --git a/app/crowdb-access-server/src/iceberg/file_http/injection.rs b/app/crowdb-access-server/src/iceberg/file_http/injection.rs new file mode 100644 index 000000000..18b81f2d7 --- /dev/null +++ b/app/crowdb-access-server/src/iceberg/file_http/injection.rs @@ -0,0 +1,94 @@ +use crowdb_access_iceberg::{ + file::{FileIdentity, NativeInjectionRepository, NativeInjectionRequest}, + key::FileId, +}; +use hyper::{ + body::Incoming, + header::{CONTENT_LENGTH, CONTENT_TYPE, ETAG}, + Request, Response, +}; + +use super::{ + admission_error, catalog_error, etag, multipart, now_ms, set_header, FileHttp, FileRequest, + FileS3ErrorCode, FileTransferAdmission, IcebergBody, UploadBody, +}; + +impl FileHttp { + pub(super) async fn inject( + &self, + file_request: &FileRequest, + request: Request, + admission: &FileTransferAdmission, + current_ms: u64, + locations: &crowdb_access_iceberg::file::DirectFileLocations, + ) -> Result, FileS3ErrorCode> { + let digest = multipart::signed_digest(request.headers().get("x-amz-content-sha256"))? + .ok_or(FileS3ErrorCode::InvalidRequest)?; + let stem = file_request + .location + .relative_key() + .rsplit('/') + .next() + .and_then(|filename| filename.strip_suffix(".parquet")) + .ok_or(FileS3ErrorCode::InvalidRequest)? + .to_owned(); + let injection = NativeInjectionRequest { + context: admission.context(), + table: file_request.location.table(), + identity: file_request.injection.ok_or(FileS3ErrorCode::InvalidRequest)?, + principal: admission.principal(), + stem, + digest, + }; + let repository = NativeInjectionRepository::new(self.store.clone()); + crowdb_access_iceberg::file::DirectFileLocations::validate_stem(&injection.stem) + .map_err(|_| FileS3ErrorCode::InvalidRequest)?; + let file = if let Some(file) = repository + .replay(&injection, current_ms) + .await + .map_err(catalog_error)? + { + file + } else { + let (parts, incoming) = request.into_parts(); + let mut body = UploadBody::new(incoming, &parts.headers, None, admission.request_byte_limit()) + .map_err(multipart::encoding_error)?; + let owner = FileIdentity { + table: injection.table, + file: FileId::random(), + }; + let length = body.decoded_length(); + let tree = admission + .receive( + &self.uploads, + &mut body, + self.blocks.clone(), + owner, + length, + Some(digest), + ) + .await + .map_err(|error| { + body.failure() + .map_or_else(|| admission_error(error), multipart::encoding_error) + })?; + crowdb_access_iceberg::file::FileSealer::new(self.blocks.clone(), self.limits.max_file_bytes) + .map_err(|_| FileS3ErrorCode::InternalError)? + .seal_uploaded(owner, file_request.location.clone(), tree.clone()) + .await + .map_err(multipart::seal_error)?; + repository + .finish(&injection, owner, tree, locations, now_ms()?) + .await + .map_err(catalog_error)? + }; + let bytes = serde_json::to_vec(&serde_json::json!({"location": file.location.to_string()})) + .map_err(|_| FileS3ErrorCode::InternalError)?; + let length = bytes.len(); + let mut response = Response::new(IcebergBody::new(bytes)); + set_header(&mut response, CONTENT_LENGTH, &length.to_string())?; + set_header(&mut response, CONTENT_TYPE, "application/json")?; + set_header(&mut response, ETAG, &etag(&file))?; + Ok(response) + } +} diff --git a/app/crowdb-access-server/src/iceberg/file_http/read.rs b/app/crowdb-access-server/src/iceberg/file_http/read.rs new file mode 100644 index 000000000..71cce0e6f --- /dev/null +++ b/app/crowdb-access-server/src/iceberg/file_http/read.rs @@ -0,0 +1,110 @@ +use crowdb_access_iceberg::file::{resolve_range, ByteRange}; +use hyper::{ + body::Incoming, + header::{ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, ETAG, RANGE}, + Method, Request, Response, StatusCode, +}; + +use super::{ + admission_error, catalog_error, etag, range_error, set_header, FileHttp, FileRequest, FileS3ErrorCode, + FileTransferAdmission, IcebergBody, +}; +use crate::iceberg::file_read_conditions::{file_read_condition, FileReadCondition}; + +impl FileHttp { + pub(super) async fn read( + &self, + file_request: &FileRequest, + request: &Request, + context: crowdb_access_iceberg::catalog::CatalogContext, + admission: &FileTransferAdmission, + locations: &crowdb_access_iceberg::file::DirectFileLocations, + ) -> Result, FileS3ErrorCode> { + let (record, direct) = match locations + .resolve(&file_request.location) + .map_err(|_| FileS3ErrorCode::AccessDenied)? + { + Some(record) => (record, true), + None => ( + self.repository + .load(context, &file_request.location) + .await + .map_err(catalog_error)? + .ok_or(FileS3ErrorCode::NoSuchKey)?, + false, + ), + }; + admission.check_bytes(0, record.length).map_err(admission_error)?; + let root_directory = if direct { + Some( + crowdb_access_iceberg::file::FileReader::probe_root(self.blocks.as_ref(), &record) + .await + .map_err(|_| FileS3ErrorCode::NoSuchKey)?, + ) + } else { + None + }; + let condition = file_read_condition(request.headers(), &etag(&record))?; + let FileReadCondition::Read { use_range } = condition else { + let mut response = Response::new(IcebergBody::new(Vec::new())); + *response.status_mut() = StatusCode::NOT_MODIFIED; + set_header(&mut response, ETAG, &etag(&record))?; + return Ok(response); + }; + let range = requested_range(request, record.length, use_range)?; + let bytes = range.map_or(record.length, |range| range.end - range.start); + let body = if request.method() == Method::HEAD { + admission.check_bytes(0, record.length).map_err(admission_error)?; + IcebergBody::new(Vec::new()) + } else { + let body = admission + .read_body(&self.responses, self.blocks.clone(), record.clone(), range) + .map_err(admission_error)?; + let body = if let Some(bytes) = root_directory { + body.with_root_directory(bytes) + .map_err(|_| FileS3ErrorCode::NoSuchKey)? + } else { + body + }; + IcebergBody::file(body) + }; + let mut response = Response::new(body); + *response.status_mut() = if range.is_some() { + StatusCode::PARTIAL_CONTENT + } else { + StatusCode::OK + }; + set_header(&mut response, CONTENT_LENGTH, &bytes.to_string())?; + set_header(&mut response, ETAG, &etag(&record))?; + response + .headers_mut() + .insert(ACCEPT_RANGES, hyper::header::HeaderValue::from_static("bytes")); + if let Some(range) = range { + set_header( + &mut response, + CONTENT_RANGE, + &format!("bytes {}-{}/{}", range.start, range.end - 1, record.length), + )?; + } + Ok(response) + } +} + +fn requested_range( + request: &Request, + length: u64, + use_range: bool, +) -> Result, FileS3ErrorCode> { + let range = match request + .headers() + .get_all(RANGE) + .iter() + .collect::>() + .as_slice() + { + [] => None, + [value] => Some(value.to_str().map_err(|_| FileS3ErrorCode::InvalidRange)?), + _ => return Err(FileS3ErrorCode::InvalidRange), + }; + resolve_range(if use_range { range } else { None }, length).map_err(range_error) +} diff --git a/app/crowdb-access-server/src/iceberg/file_read_conditions.rs b/app/crowdb-access-server/src/iceberg/file_read_conditions.rs new file mode 100644 index 000000000..1bca42d7b --- /dev/null +++ b/app/crowdb-access-server/src/iceberg/file_read_conditions.rs @@ -0,0 +1,71 @@ +use hyper::header::{HeaderMap, IF_MATCH, IF_NONE_MATCH, IF_RANGE}; + +use super::file_response::FileS3ErrorCode; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FileReadCondition { + NotModified, + Read { use_range: bool }, +} + +/// # Errors +/// Rejects malformed condition lists and mismatched strong object preconditions. +pub fn file_read_condition(headers: &HeaderMap, etag: &str) -> Result { + if let Some(matches) = condition(headers, IF_MATCH, etag, false)? { + if !matches { + return Err(FileS3ErrorCode::PreconditionFailed); + } + } + if condition(headers, IF_NONE_MATCH, etag, true)? == Some(true) { + return Ok(FileReadCondition::NotModified); + } + let use_range = match headers.get_all(IF_RANGE).iter().collect::>().as_slice() { + [] => true, + [value] => value.to_str().map_err(|_| FileS3ErrorCode::InvalidRequest)? == etag, + _ => return Err(FileS3ErrorCode::InvalidRequest), + }; + Ok(FileReadCondition::Read { use_range }) +} + +fn condition( + headers: &HeaderMap, + name: hyper::header::HeaderName, + etag: &str, + weak: bool, +) -> Result, FileS3ErrorCode> { + let mut result = None; + let mut count = 0; + let mut bytes = 0; + for value in headers.get_all(name) { + let value = value.to_str().map_err(|_| FileS3ErrorCode::InvalidRequest)?; + bytes += value.len(); + if bytes > 4096 { + return Err(FileS3ErrorCode::InvalidRequest); + } + for tag in value.split(',') { + count += 1; + let tag = tag.trim(); + if count > 64 { + return Err(FileS3ErrorCode::InvalidRequest); + } + let comparison = if weak { + tag.strip_prefix("W/").unwrap_or(tag) + } else { + tag + }; + let syntax = tag.strip_prefix("W/").unwrap_or(tag); + if syntax != "*" + && (syntax.len() < 2 + || !syntax.starts_with('"') + || !syntax.ends_with('"') + || syntax[1..syntax.len() - 1] + .bytes() + .any(|byte| byte == b'"' || byte <= 0x20 || byte == 0x7f)) + { + return Err(FileS3ErrorCode::InvalidRequest); + } + result = Some(result.unwrap_or(false) || comparison == "*" || comparison == etag); + } + } + Ok(result) +} diff --git a/app/crowdb-access-server/src/iceberg/file_request.rs b/app/crowdb-access-server/src/iceberg/file_request.rs index 76ff428df..283432ceb 100644 --- a/app/crowdb-access-server/src/iceberg/file_request.rs +++ b/app/crowdb-access-server/src/iceberg/file_request.rs @@ -25,6 +25,7 @@ pub enum MultipartRequest { #[derive(Clone, Debug, Eq, PartialEq)] pub struct FileRequest { + pub injection: Option, pub location: FileLocation, pub operation: FileOperation, pub multipart: Option, @@ -54,7 +55,23 @@ impl FileRequest { .parse() .map_err(|_| FileRequestError::Invalid)?; let mut query = query(uri.query())?; + let injection = query + .remove("crowdb-native-upload") + .map(|identity| { + if method != Method::PUT + || location.table().storage_namespace.is_none() + || !location.relative_key().ends_with(".parquet") + { + return Err(FileRequestError::Invalid); + } + crowdb_access_iceberg::operation::RequestIdentity::parse(&identity, u64::MAX) + .map_err(|_| FileRequestError::Invalid) + }) + .transpose()?; let multipart = multipart(method, &mut query)?; + if injection.is_some() && multipart.is_some() { + return Err(FileRequestError::Invalid); + } if !query.is_empty() { return Err(FileRequestError::Unsupported); } @@ -77,6 +94,7 @@ impl FileRequest { return Err(FileRequestError::Invalid); } Ok(Self { + injection, location, operation, multipart, diff --git a/app/crowdb-access-server/src/iceberg/file_response.rs b/app/crowdb-access-server/src/iceberg/file_response.rs index b4d858990..77d7d344f 100644 --- a/app/crowdb-access-server/src/iceberg/file_response.rs +++ b/app/crowdb-access-server/src/iceberg/file_response.rs @@ -33,6 +33,7 @@ pub enum FileS3ErrorCode { Conflict, BadDigest, EntityTooSmall, + PreconditionFailed, } impl FileS3ErrorCode { @@ -48,6 +49,7 @@ impl FileS3ErrorCode { Self::InvalidRange => StatusCode::RANGE_NOT_SATISFIABLE, Self::SlowDown => StatusCode::SERVICE_UNAVAILABLE, Self::Conflict => StatusCode::CONFLICT, + Self::PreconditionFailed => StatusCode::PRECONDITION_FAILED, } } @@ -65,6 +67,7 @@ impl FileS3ErrorCode { Self::Conflict => "The immutable object already exists with different content", Self::BadDigest => "The supplied digest does not match the uploaded content", Self::EntityTooSmall => "A nonfinal upload part is smaller than 5 MiB", + Self::PreconditionFailed => "An object precondition was not satisfied", } } @@ -82,6 +85,7 @@ impl FileS3ErrorCode { Self::Conflict => "OperationAborted", Self::BadDigest => "BadDigest", Self::EntityTooSmall => "EntityTooSmall", + Self::PreconditionFailed => "PreconditionFailed", } } } diff --git a/app/crowdb-access-server/src/iceberg/gc_control.rs b/app/crowdb-access-server/src/iceberg/gc_control.rs index bc22a3e82..5f0cc2582 100644 --- a/app/crowdb-access-server/src/iceberg/gc_control.rs +++ b/app/crowdb-access-server/src/iceberg/gc_control.rs @@ -5,10 +5,10 @@ use crowdb_access_iceberg::{ CatalogContext, CatalogRepository, CatalogStore, ManagementPrivilege, RootState, RoutedCatalogStore, }, gc::{GcLimits, GcRepository, GcTask}, - key::{CatalogId, OperationId, TableId}, + key::{CatalogId, OperationId, StorageTablePath, TableId}, operation::{ManagementAction, ManagementPhase}, record::StorageRecord, - table::{head_key, TableHead, TableLifecycle}, + table::{storage_head_key, TableHead, TableLifecycle}, wire::BearerAuthenticator, }; @@ -88,13 +88,28 @@ async fn start_table( table: &str, limits: GcLimits, ) -> Result<(), BoxError> { - let (root, _) = catalog.status().await?; + let (root, authority) = catalog.status().await?; if root.state != RootState::Ready { return Err("catalog is not ready".into()); } - let table: TableId = table.parse()?; + let table = match authority.storage_layout { + crowdb_access_iceberg::catalog::CatalogStorageLayout::Native => { + let path = StorageTablePath::from_prefix(&format!("{}/", table.trim_end_matches('/')))?; + if path.catalog != root.context.catalog { + return Err("table storage path belongs to another catalog".into()); + } + path.into() + } + crowdb_access_iceberg::catalog::CatalogStorageLayout::Legacy => { + crowdb_access_iceberg::file::TableLocation { + catalog: root.context.catalog, + table: table.parse::()?, + storage_namespace: None, + } + } + }; let identity: OperationId = identity.parse()?; - let head = load_head(store, root.context.catalog, table).await?; + let head = load_head(store, table).await?; if head.lifecycle != TableLifecycle::Tombstone { return Err("live-table GC is disabled; only tombstoned tables can be reclaimed".into()); } @@ -151,10 +166,9 @@ async fn start_retired( async fn load_head( store: &RoutedCatalogStore, - catalog: CatalogId, - table: TableId, + table: crowdb_access_iceberg::file::TableLocation, ) -> Result { - let key = head_key(catalog, table); + let key = storage_head_key(table); let value = store.get(&key.encode()?).await?.ok_or("table head is missing")?; let StorageRecord::TableHead(head) = StorageRecord::decode(&key, &value.bytes)? else { return Err("table head has an invalid record".into()); diff --git a/app/crowdb-access-server/src/iceberg/gc_runtime.rs b/app/crowdb-access-server/src/iceberg/gc_runtime.rs index dc00ee18c..1c32bb617 100644 --- a/app/crowdb-access-server/src/iceberg/gc_runtime.rs +++ b/app/crowdb-access-server/src/iceberg/gc_runtime.rs @@ -286,6 +286,7 @@ async fn scan_and_advance( }); } let scan = GcScan { + native: false, catalog, scope: Some(CatalogScope::GcTask), prefix: Vec::new(), @@ -388,6 +389,7 @@ async fn scan_purge( limits: GcLimits, ) -> Result, Box> { let scan = GcScan { + native: false, catalog: context.catalog, scope: Some(CatalogScope::Reclamation), prefix: Vec::new(), diff --git a/app/crowdb-access-server/src/iceberg/http.rs b/app/crowdb-access-server/src/iceberg/http.rs index c2a869048..d855dcd58 100644 --- a/app/crowdb-access-server/src/iceberg/http.rs +++ b/app/crowdb-access-server/src/iceberg/http.rs @@ -107,6 +107,23 @@ impl IcebergHttpService { Ok(self) } + /// Installs persistent locator keys, including retained older key versions. + /// # Errors + /// Rejects configuration after the file service has been shared. + pub fn with_direct_file_locations( + mut self, + locations: crowdb_access_iceberg::file::DirectFileLocations, + ) -> Result { + Arc::get_mut( + self.files + .as_mut() + .ok_or(crowdb_access_iceberg::file::FileGrantError::Invalid)?, + ) + .ok_or(crowdb_access_iceberg::file::FileGrantError::Invalid)? + .set_direct_locations(locations); + Ok(self) + } + /// Applies the configured shared-chunk routing threshold to native file uploads. /// # Errors /// Rejects an unavailable file service or an invalid threshold. @@ -250,7 +267,10 @@ impl IcebergHttpService { &self, request: Request, ) -> Result, IcebergErrorResponse> { - if request.uri().path().starts_with("/iceberg-") { + if request.uri().path().starts_with("/iceberg-") + || request.uri().path() == "/iceberg" + || request.uri().path().starts_with("/iceberg/") + { return Ok(match &self.files { Some(files) => { Box::pin(files.dispatch(&self.repository, request, self.request_timeout)).await diff --git a/app/crowdb-access-server/src/iceberg/runtime.rs b/app/crowdb-access-server/src/iceberg/runtime.rs index 38a285dad..60195ebdb 100644 --- a/app/crowdb-access-server/src/iceberg/runtime.rs +++ b/app/crowdb-access-server/src/iceberg/runtime.rs @@ -179,6 +179,13 @@ async fn start_listener( )? .with_small_object_threshold(access_config.iceberg_small_write().threshold_exclusive())? .with_large_write_policy(large_write)?; + if let Some(signing) = &access_config.iceberg.locator_signing { + service = + service.with_direct_file_locations(crowdb_access_iceberg::file::DirectFileLocations::new( + signing.active_version, + signing.keys.clone(), + )?)?; + } if authority.admission_bounds.delegated_access_ms >= 900_000 { let endpoint = std::env::var("CROWDB_ICEBERG_PUBLIC_URI").unwrap_or_else(|_| format!("http://{address}")); diff --git a/app/crowdb-access-server/src/iceberg/table_credentials.rs b/app/crowdb-access-server/src/iceberg/table_credentials.rs index cbad30eee..11b3c434b 100644 --- a/app/crowdb-access-server/src/iceberg/table_credentials.rs +++ b/app/crowdb-access-server/src/iceberg/table_credentials.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, sync::Arc}; +use std::{collections::BTreeMap, fmt::Write, sync::Arc}; use crowdb_access_iceberg::{ catalog::{ @@ -7,9 +7,9 @@ use crowdb_access_iceberg::{ }, commit::{TableCreateJournal, TableCreatePhase}, file::FileGrantIssuer, - key::{OperationId, TableId}, + key::{NamespaceId, OperationId, TableId}, namespace::{NamespaceIdentifier, NamespaceRepository, NamespaceStore}, - table::TableRepository, + table::{storage_head_key, TableLifecycle, TableRepository}, wire::{ FileDelegationLimits, FileDelegationTarget, IcebergErrorResponse, LoadCredentialsResponse, Principal, }, @@ -64,6 +64,19 @@ impl TableFileConfig { let endpoint = request_host.and_then(|host| self.endpoint_for_host(host)); let mut properties = self.properties(target.namespace, target.name, target.table, endpoint.as_deref()); + if target.storage_namespace.is_some() { + properties.insert("crowdb.native-injection", "true".into()); + properties.insert( + "crowdb.native-injection-max-bytes", + (1024_u64 * 1024 * 1024).to_string(), + ); + if let Some(namespace) = target.storage_namespace { + let endpoint = properties + .get_mut("client.refresh-credentials-endpoint") + .ok_or_else(service_unavailable)?; + write!(endpoint, "&storage-namespace-id={namespace}").expect("string writes do not fail"); + } + } let credentials = FileDelegationLimits { ttl_ms: 900_000, max_request_bytes: 1024 * 1024 * 1024, @@ -183,21 +196,19 @@ impl TableCredentials { principal: Principal, request: &Request, ) -> Result, IcebergErrorResponse> { - if request.method() != hyper::Method::GET { - return Err(super::table_read::unsupported()); - } - let path = request - .uri() - .path() - .strip_suffix("/credentials") - .ok_or_else(bad_request)?; - let target = super::table_write::request::parse(&path.parse().map_err(|_| bad_request())?)?; - let name = target.name.ok_or_else(bad_request)?; - let (selector, cleanup) = selector(request.uri().query())?; + let (namespace, name) = credential_target(request)?; + let (selector, storage_namespace, cleanup) = selector(request.uri().query())?; let now = now_ms()?; + if let (Some(table), Some(namespace)) = (selector, storage_namespace) { + if let Some(target) = self.native_target(context, table, namespace, cleanup).await? { + return self + .issue_response(repository, context, principal, target, 900_000, now, cleanup) + .await; + } + } let parent = self .namespaces - .load(context, &target.namespace) + .load(context, &namespace) .await .map_err(|_| service_unavailable())? .ok_or_else(missing_table)?; @@ -232,11 +243,14 @@ impl TableCredentials { let stage = operation.stage.as_ref().ok_or_else(missing_table)?; if !principal.namespace_write || operation.principal != principal.name - || operation.namespace != target.namespace + || operation.namespace != namespace || operation.candidate.namespace != parent.namespace || operation.candidate.name != name || operation.candidate.table != table || operation.phase != TableCreatePhase::Staged + || storage_namespace.is_some_and(|namespace| { + operation.candidate.storage_location().storage_namespace != Some(namespace) + }) { return Err(missing_table()); } @@ -280,6 +294,45 @@ impl TableCredentials { .await } + async fn native_target( + &self, + context: CatalogContext, + table: TableId, + namespace: NamespaceId, + cleanup: bool, + ) -> Result, IcebergErrorResponse> { + let key = storage_head_key(crowdb_access_iceberg::file::TableLocation { + catalog: context.catalog, + storage_namespace: Some(namespace), + table, + }); + let Some(value) = self + .store + .get(&key.encode().map_err(|_| bad_request())?) + .await + .map_err(|_| service_unavailable())? + else { + return Ok(None); + }; + let crowdb_access_iceberg::record::StorageRecord::TableHead(head) = + crowdb_access_iceberg::record::StorageRecord::decode(&key, &value.bytes) + .map_err(|_| service_unavailable())? + else { + return Err(service_unavailable()); + }; + if head.lifecycle != TableLifecycle::Ready + && !(cleanup && head.lifecycle == TableLifecycle::Tombstone) + { + return Err(missing_table()); + } + Ok(Some(FileDelegationTarget { + table, + storage_namespace: Some(namespace), + format_version: head.format_version, + staged: false, + })) + } + #[allow(clippy::too_many_arguments)] async fn issue_response( &self, @@ -323,8 +376,11 @@ impl TableCredentials { } } -fn selector(query: Option<&str>) -> Result<(Option, bool), IcebergErrorResponse> { +fn selector( + query: Option<&str>, +) -> Result<(Option, Option, bool), IcebergErrorResponse> { let mut selector = None; + let mut namespace = None; let mut cleanup = false; for pair in query .unwrap_or_default() @@ -336,9 +392,31 @@ fn selector(query: Option<&str>) -> Result<(Option, bool), IcebergError "table-id" if selector.is_none() => { selector = Some(decode_query(value)?.parse().map_err(|_| bad_request())?); } + "storage-namespace-id" if namespace.is_none() => { + namespace = Some(decode_query(value)?.parse().map_err(|_| bad_request())?); + } "cleanup" if !cleanup && decode_query(value)? == "true" => cleanup = true, _ => return Err(bad_request()), } } - Ok((selector, cleanup)) + if namespace.is_some() && selector.is_none() { + return Err(bad_request()); + } + Ok((selector, namespace, cleanup)) +} + +fn credential_target( + request: &Request, +) -> Result<(NamespaceIdentifier, String), IcebergErrorResponse> { + if request.method() != hyper::Method::GET { + return Err(super::table_read::unsupported()); + } + let path = request + .uri() + .path() + .strip_suffix("/credentials") + .ok_or_else(bad_request)?; + let target = super::table_write::request::parse(&path.parse().map_err(|_| bad_request())?)?; + let name = target.name.ok_or_else(bad_request)?; + Ok((target.namespace, name)) } diff --git a/app/crowdb-access-server/tests/common/iceberg_commit_fault.rs b/app/crowdb-access-server/tests/common/iceberg_commit_fault.rs index 229e812b6..603c8b6db 100644 --- a/app/crowdb-access-server/tests/common/iceberg_commit_fault.rs +++ b/app/crowdb-access-server/tests/common/iceberg_commit_fault.rs @@ -9,7 +9,7 @@ use crowdb_access_iceberg::{ catalog::{CasOutcome, CatalogStore, StoreError, StoredValue}, file::{ChunkRoot, FileBlockStore, FileIdentity, FileIoError, MultipartPartScan, MultipartPartStore}, gc::{GcScan, GcStore, GcSystemScan}, - key::{CatalogScope, IcebergKey}, + key::{CatalogScope, IcebergKey, StorageKey, TableStorageKey, TableStorageScope}, namespace::{ChildScan, NamespaceStore}, record::StorageRecord, }; @@ -114,7 +114,7 @@ impl CatalogStore for Test value: &[u8], identity: ClientRequestId, ) -> Result { - let record_key = IcebergKey::decode(key)?; + let record_key = StorageKey::decode(key)?; let label = match StorageRecord::decode(&record_key, value)? { StorageRecord::TableCreateOperation(operation) => format!("create-{:?}", operation.phase), StorageRecord::TableCommitOperation(operation) => format!("commit-{:?}", operation.phase), @@ -122,10 +122,13 @@ impl CatalogStore for Test StorageRecord::File(_) if matches!( record_key, - IcebergKey::Catalog { + StorageKey::Legacy(IcebergKey::Catalog { scope: CatalogScope::FileLocation, .. - } + }) | StorageKey::NativeTable(TableStorageKey { + scope: TableStorageScope::File, + .. + }) ) => { "file-location-authority".into() diff --git a/app/crowdb-access-server/tests/common/iceberg_commit_recovery/blocks.rs b/app/crowdb-access-server/tests/common/iceberg_commit_recovery/blocks.rs index 1ec2a5b03..db8182e92 100644 --- a/app/crowdb-access-server/tests/common/iceberg_commit_recovery/blocks.rs +++ b/app/crowdb-access-server/tests/common/iceberg_commit_recovery/blocks.rs @@ -34,6 +34,15 @@ impl TestBlocks { values: self.values.clone(), } } + + #[allow(dead_code)] + pub fn remove_root(&self, root: &ChunkRoot) { + self.values.rcu(|values| { + let mut next = (**values).clone(); + next.remove(&root.chunk.low); + next + }); + } } #[async_trait] diff --git a/app/crowdb-access-server/tests/common/iceberg_store.rs b/app/crowdb-access-server/tests/common/iceberg_store.rs index 00e5d4043..1d69a0691 100644 --- a/app/crowdb-access-server/tests/common/iceberg_store.rs +++ b/app/crowdb-access-server/tests/common/iceberg_store.rs @@ -53,6 +53,8 @@ pub struct TestStore { pub read_delay_ms: AtomicU64, pub scan_delay_ms: AtomicU64, pub scans: AtomicU64, + pub native_file_reads: AtomicU64, + pub deny_native_file_reads: AtomicBool, pub lose_reply_kind: AtomicU8, pub pause_file_read: AtomicBool, pub file_read_entered: tokio::sync::Notify, @@ -174,14 +176,30 @@ impl CatalogStore for TestStore { }) } async fn get(&self, key: &[u8]) -> Result, StoreError> { + if matches!(crowdb_access_iceberg::key::StorageKey::decode(key), + Ok(crowdb_access_iceberg::key::StorageKey::NativeTable(key)) + if key.scope == crowdb_access_iceberg::key::TableStorageScope::File) + { + self.native_file_reads.fetch_add(1, Ordering::SeqCst); + if self.deny_native_file_reads.load(Ordering::SeqCst) { + return Err(StoreError::Response); + } + } let value = self.values.load().get(key).cloned(); if matches!( - crowdb_access_iceberg::key::IcebergKey::decode(key), - Ok(crowdb_access_iceberg::key::IcebergKey::Catalog { - scope: crowdb_access_iceberg::key::CatalogScope::File - | crowdb_access_iceberg::key::CatalogScope::FileLocation, - .. - }) + crowdb_access_iceberg::key::StorageKey::decode(key), + Ok(crowdb_access_iceberg::key::StorageKey::Legacy( + crowdb_access_iceberg::key::IcebergKey::Catalog { + scope: crowdb_access_iceberg::key::CatalogScope::File + | crowdb_access_iceberg::key::CatalogScope::FileLocation, + .. + } + ) | crowdb_access_iceberg::key::StorageKey::NativeTable( + crowdb_access_iceberg::key::TableStorageKey { + scope: crowdb_access_iceberg::key::TableStorageScope::File, + .. + } + )) ) && self.pause_file_read.swap(false, Ordering::SeqCst) { self.file_read_entered.notify_one(); @@ -202,11 +220,18 @@ impl CatalogStore for TestStore { ) -> Result { identity.validate().unwrap(); if matches!( - crowdb_access_iceberg::key::IcebergKey::decode(key), - Ok(crowdb_access_iceberg::key::IcebergKey::Catalog { - scope: crowdb_access_iceberg::key::CatalogScope::TableHead, - .. - }) + crowdb_access_iceberg::key::StorageKey::decode(key), + Ok(crowdb_access_iceberg::key::StorageKey::Legacy( + crowdb_access_iceberg::key::IcebergKey::Catalog { + scope: crowdb_access_iceberg::key::CatalogScope::TableHead, + .. + } + ) | crowdb_access_iceberg::key::StorageKey::NativeTable( + crowdb_access_iceberg::key::TableStorageKey { + scope: crowdb_access_iceberg::key::TableStorageScope::Head, + .. + } + )) ) && self.pause_head_cas.swap(false, Ordering::SeqCst) { self.head_cas_entered.notify_one(); @@ -230,8 +255,10 @@ impl CatalogStore for TestStore { if Arc::ptr_eq(¤t, &self.values.compare_and_swap(¤t, Arc::new(next))) { let mode = self.lose_reply_kind.load(Ordering::SeqCst); let lose = mode == 1 - || match crowdb_access_iceberg::key::IcebergKey::decode(key) { - Ok(crowdb_access_iceberg::key::IcebergKey::Catalog { scope, .. }) => { + || match crowdb_access_iceberg::key::StorageKey::decode(key) { + Ok(crowdb_access_iceberg::key::StorageKey::Legacy( + crowdb_access_iceberg::key::IcebergKey::Catalog { scope, .. }, + )) => { (mode == 2 && scope == crowdb_access_iceberg::key::CatalogScope::NamespaceAuthority) || (mode == 3 && scope == crowdb_access_iceberg::key::CatalogScope::Operation) @@ -244,6 +271,9 @@ impl CatalogStore for TestStore { Ok(crowdb_access_iceberg::record::StorageRecord::TableCommitOperation(operation)) if operation.phase == crowdb_access_iceberg::commit::TableCommitPhase::Rejected)) } + Ok(crowdb_access_iceberg::key::StorageKey::NativeTable(key)) => { + mode == 5 && key.scope == crowdb_access_iceberg::key::TableStorageScope::Head + } _ => false, }; if lose && self.lose_reply_kind.swap(0, Ordering::SeqCst) != 0 { diff --git a/app/crowdb-access-server/tests/iceberg_commit_recovery_test.rs b/app/crowdb-access-server/tests/iceberg_commit_recovery_test.rs index b22d631e3..393319e89 100644 --- a/app/crowdb-access-server/tests/iceberg_commit_recovery_test.rs +++ b/app/crowdb-access-server/tests/iceberg_commit_recovery_test.rs @@ -102,7 +102,8 @@ async fn table_publication_recovers_before_and_after_every_durable_write() { ]), "direct updates must publish immutable metadata through one head CAS" ); - assert_eq!(count, 4); + // The native tree persists data and its owner-bound directory before publication. + assert_eq!(count, 6); } else { assert!(labels.iter().any(|label| label.starts_with("create-"))); } diff --git a/app/crowdb-access-server/tests/iceberg_commit_sdk_test.rs b/app/crowdb-access-server/tests/iceberg_commit_sdk_test.rs index a38e188f3..7a61285f4 100644 --- a/app/crowdb-access-server/tests/iceberg_commit_sdk_test.rs +++ b/app/crowdb-access-server/tests/iceberg_commit_sdk_test.rs @@ -9,7 +9,7 @@ mod common; #[allow(dead_code)] mod fixture; -use crowdb_access_iceberg::key::IcebergKey; +use crowdb_access_iceberg::key::{IcebergKey, StorageKey}; use serde_json::{json, Value}; use std::{sync::atomic::Ordering, time::Duration}; @@ -66,11 +66,11 @@ async fn official_commit_loses_real_head_cas_without_rebase_and_keeps_hidden_ret assert!( fixture.store.values.load().keys().all(|key| { !matches!( - IcebergKey::decode(key).unwrap(), - IcebergKey::Catalog { + StorageKey::decode(key).unwrap(), + StorageKey::Legacy(IcebergKey::Catalog { scope: crowdb_access_iceberg::key::CatalogScope::TableCommitOperation, .. - } + }) ) }), "ordinary commits must not write a phase journal" diff --git a/app/crowdb-access-server/tests/iceberg_file_recovery_test.rs b/app/crowdb-access-server/tests/iceberg_file_recovery_test.rs index eed11c7aa..d9d978873 100644 --- a/app/crowdb-access-server/tests/iceberg_file_recovery_test.rs +++ b/app/crowdb-access-server/tests/iceberg_file_recovery_test.rs @@ -112,7 +112,7 @@ async fn setup_with_bounds_and_file_limit( let started = now_ms(); let credentials = issuer .issue(FileGrant { - storage_namespace: None, + storage_namespace: table.storage_namespace, context, table: table.table, principal: [7; 32], diff --git a/app/crowdb-access-server/tests/iceberg_file_request_test.rs b/app/crowdb-access-server/tests/iceberg_file_request_test.rs index 3fad6d857..739d7940d 100644 --- a/app/crowdb-access-server/tests/iceberg_file_request_test.rs +++ b/app/crowdb-access-server/tests/iceberg_file_request_test.rs @@ -12,6 +12,44 @@ fn table() -> TableLocation { } } +#[test] +fn native_injection_accepts_only_new_layout_parquet_put_and_uuidv7_identity() { + let mut table = table(); + table.storage_namespace = Some(crowdb_access_iceberg::key::NamespaceId::random()); + let identity = "0199aabc-0000-7000-8000-000000000001"; + let uri = path( + table, + &format!("data/part.parquet?crowdb-native-upload={identity}"), + ); + assert!(FileRequest::parse(&Method::PUT, &uri) + .unwrap() + .injection + .is_some()); + assert!(FileRequest::parse(&Method::GET, &uri).is_err()); + assert!(FileRequest::parse( + &Method::PUT, + &path(table, "data/part.parquet?crowdb-native-upload=bad") + ) + .is_err()); + assert!(FileRequest::parse( + &Method::PUT, + &path( + table, + &format!("metadata/v1.json?crowdb-native-upload={identity}") + ) + ) + .is_err()); + table.storage_namespace = None; + assert!(FileRequest::parse( + &Method::PUT, + &path( + table, + &format!("data/part.parquet?crowdb-native-upload={identity}") + ) + ) + .is_err()); +} + #[test] fn bucket_probes_are_distinct_from_unscoped_object_lists() { use crowdb_access_server::iceberg::{parse_file_bucket, FileBucketRequest}; diff --git a/app/crowdb-access-server/tests/iceberg_file_response_test.rs b/app/crowdb-access-server/tests/iceberg_file_response_test.rs index 5902af450..b8338b147 100644 --- a/app/crowdb-access-server/tests/iceberg_file_response_test.rs +++ b/app/crowdb-access-server/tests/iceberg_file_response_test.rs @@ -208,3 +208,37 @@ fn complete_uses_only_a_published_matching_record() { session.published = None; assert!(MultipartResponses::complete(&session, &record, "https://storage.example/a").is_err()); } +#[test] +fn read_conditions_preserve_strong_preconditions_and_conditional_range_semantics() { + use crowdb_access_server::iceberg::{file_read_condition, FileReadCondition}; + use hyper::header::{HeaderMap, HeaderValue, IF_MATCH, IF_NONE_MATCH, IF_RANGE}; + let tag = "\"current\""; + let mut headers = HeaderMap::new(); + headers.insert(IF_MATCH, HeaderValue::from_static("\"old\", \"current\"")); + headers.insert(IF_RANGE, HeaderValue::from_static("\"old\"")); + assert_eq!( + file_read_condition(&headers, tag).unwrap(), + FileReadCondition::Read { use_range: false } + ); + headers.insert(IF_RANGE, HeaderValue::from_static("\"current\"")); + assert_eq!( + file_read_condition(&headers, tag).unwrap(), + FileReadCondition::Read { use_range: true } + ); + headers.insert(IF_MATCH, HeaderValue::from_static("W/\"current\"")); + assert_eq!( + file_read_condition(&headers, tag), + Err(FileS3ErrorCode::PreconditionFailed) + ); + headers.remove(IF_MATCH); + headers.insert(IF_NONE_MATCH, HeaderValue::from_static("W/\"current\"")); + assert_eq!( + file_read_condition(&headers, tag).unwrap(), + FileReadCondition::NotModified + ); + headers.insert(IF_NONE_MATCH, HeaderValue::from_static("\"old\", invalid")); + assert_eq!( + file_read_condition(&headers, tag), + Err(FileS3ErrorCode::InvalidRequest) + ); +} diff --git a/app/crowdb-access-server/tests/iceberg_native_injection_http_test.rs b/app/crowdb-access-server/tests/iceberg_native_injection_http_test.rs new file mode 100644 index 000000000..966dcf8cc --- /dev/null +++ b/app/crowdb-access-server/tests/iceberg_native_injection_http_test.rs @@ -0,0 +1,397 @@ +#[path = "common/iceberg_commit_recovery/blocks.rs"] +#[allow(dead_code)] +mod blocks; +#[path = "common/iceberg_commit_recovery/fixture.rs"] +#[allow(dead_code, unused_imports)] +mod common; +#[path = "common/iceberg_store.rs"] +mod memory; +#[path = "common/iceberg_signed_file.rs"] +#[allow(dead_code)] +mod signed; +#[path = "common/iceberg_commit_recovery/store.rs"] +mod store; + +use crowdb_access_iceberg::{ + catalog::{CatalogRepository, ClearBounds, ManagementPrivilege}, + file::{ + DirectFileLocations, FileGrant, FileGrantIssuer, FileOperation, FileOperations, FileRepository, + TableLocation, + }, + key::{NamespaceId, OperationId, TableId}, + operation::{ManagementAction, ManagementRequest, RequestIdentity}, + wire::BearerAuthenticator, +}; +use crowdb_access_server::iceberg::{serve, IcebergHttpService}; +use reqwest::Method; +use std::{ + collections::BTreeMap, + sync::{atomic::Ordering, Arc}, + time::Duration, +}; + +fn authentication() -> BearerAuthenticator { + BearerAuthenticator::new(&"r".repeat(32), &"w".repeat(32), &"m".repeat(32), &"c".repeat(32)).unwrap() +} + +fn rotated_authentication() -> BearerAuthenticator { + BearerAuthenticator::new(&"a".repeat(32), &"b".repeat(32), &"d".repeat(32), &"e".repeat(32)).unwrap() +} + +async fn rest( + address: std::net::SocketAddr, + method: Method, + path: &str, + body: serde_json::Value, + serial: u64, +) -> serde_json::Value { + let now = common::now_ms(); + let identity = format!("{:08x}-{:04x}-7000-8000-{serial:012x}", now >> 16, now & 0xffff); + let request = reqwest::Client::new() + .request(method, format!("http://{address}{path}")) + .bearer_auth("w".repeat(32)) + .header("idempotency-key", identity); + let request = if body.is_null() { + request + } else { + request.json(&body) + }; + let response = request.send().await.unwrap(); + let status = response.status(); + let text = response.text().await.unwrap(); + assert!(status.is_success(), "{path}: {status}: {text}"); + if text.is_empty() { + serde_json::Value::Null + } else { + serde_json::from_str(&text).unwrap() + } +} + +#[tokio::test] +async fn full_storage_identity_refresh_survives_move_and_default_namespace_recreation() { + use serde_json::json; + let (_stack, _repository, _table, client, server) = setup().await; + let create = json!({"name":"events","schema":{"type":"struct","schema-id":0, + "fields":[{"id":1,"name":"id","type":"long","required":true}]}}); + let initial = rest( + client.address, + Method::POST, + "/v1/namespaces/default/tables", + create.clone(), + 2, + ) + .await; + let original = initial["metadata"]["location"].as_str().unwrap(); + let refresh = initial["config"]["client.refresh-credentials-endpoint"] + .as_str() + .unwrap(); + assert!(refresh.contains("storage-namespace-id=")); + rest( + client.address, + Method::POST, + "/v1/namespaces", + json!({"namespace":["dest"]}), + 3, + ) + .await; + rest( + client.address, + Method::POST, + "/v1/tables/rename", + json!({ + "source":{"namespace":["default"],"name":"events"}, + "destination":{"namespace":["dest"],"name":"renamed"}}), + 4, + ) + .await; + rest( + client.address, + Method::DELETE, + "/v1/namespaces/default", + json!(null), + 5, + ) + .await; + let recreated = rest( + client.address, + Method::POST, + "/v1/namespaces/default/tables", + create, + 6, + ) + .await; + assert_ne!(recreated["metadata"]["location"].as_str().unwrap(), original); + let response = reqwest::Client::new() + .get(format!("http://{}{refresh}", client.address)) + .bearer_auth("r".repeat(32)) + .send() + .await + .unwrap(); + let status = response.status(); + let text = response.text().await.unwrap(); + assert_eq!(status, 200, "{text}"); + let refreshed: serde_json::Value = serde_json::from_str(&text).unwrap(); + assert_eq!( + refreshed["storage-credentials"][0]["prefix"] + .as_str() + .unwrap() + .trim_end_matches('/'), + original.trim_end_matches('/') + ); + let moved = reqwest::Client::new() + .get(format!( + "http://{}/v1/namespaces/dest/tables/renamed", + client.address + )) + .bearer_auth("r".repeat(32)) + .send() + .await + .unwrap() + .json::() + .await + .unwrap(); + assert_eq!(moved["metadata"]["location"].as_str().unwrap(), original); + server.abort(); + let _ = server.await; +} + +async fn listen( + stack: &common::TestCommitFixture, + repository: Arc, + rotate: bool, +) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) { + let auth = if rotate { + rotated_authentication() + } else { + authentication() + }; + let mut service = IcebergHttpService::new(repository, auth, Duration::from_secs(300)) + .with_namespaces(stack.store()) + .unwrap() + .with_fileio(stack.store(), stack.blocks.clone(), "us-east-1".into()) + .unwrap() + .with_tables(stack.store(), stack.blocks.clone()) + .unwrap(); + if rotate { + service = service + .with_direct_file_locations(DirectFileLocations::new(2, BTreeMap::from([(2, [4; 32])])).unwrap()) + .unwrap(); + } + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + service = service + .with_table_credentials(stack.store(), format!("http://{address}")) + .unwrap(); + let task = tokio::spawn(async move { + serve(listener, Arc::new(service), std::future::pending()) + .await + .unwrap(); + }); + (address, task) +} + +async fn setup() -> ( + common::TestCommitFixture, + Arc, + TableLocation, + signed::TestFileClient, + tokio::task::JoinHandle<()>, +) { + let stack = common::TestCommitFixture::new(); + let repository = Arc::new( + CatalogRepository::new( + stack.store(), + ClearBounds { + request_ms: 300_000, + delegated_access_ms: 900_000, + ..ClearBounds::default() + }, + ) + .unwrap(), + ); + let now = common::now_ms(); + repository + .execute( + ManagementRequest { + identity: RequestIdentity { + operation: OperationId::random(), + issued_ms: now, + }, + principal: "manager".into(), + action: ManagementAction::Initialize, + expected_epoch: 0, + display_name: "native".into(), + confirmation: None, + capabilities: None, + }, + ManagementPrivilege::Manage, + now, + ) + .await + .unwrap(); + common::activate(&repository).await; + assert_eq!( + repository.status().await.unwrap().1.storage_layout, + crowdb_access_iceberg::catalog::CatalogStorageLayout::Native + ); + let context = repository.status().await.unwrap().0.context; + let table = TableLocation { + catalog: context.catalog, + table: TableId::random(), + storage_namespace: Some(NamespaceId::random()), + }; + let credentials = FileGrantIssuer::new(authentication().namespace_token_key(), 900_000) + .unwrap() + .issue(FileGrant { + context, + storage_namespace: table.storage_namespace, + table: table.table, + principal: [3; 32], + nonce: OperationId::random(), + issued_ms: now, + expires_ms: now + 900_000, + max_request_bytes: 1024 * 1024, + max_file_bytes: 1024 * 1024, + operations: FileOperations::new(&[ + FileOperation::Put, + FileOperation::Get, + FileOperation::Head, + FileOperation::ListObjects, + ]) + .unwrap(), + }) + .unwrap(); + let (address, server) = listen(&stack, repository.clone(), false).await; + let client = signed::TestFileClient { + client: reqwest::Client::new(), + credentials, + address, + }; + (stack, repository, table, client, server) +} + +#[tokio::test] +async fn direct_http_reads_skip_file_kv_after_upload_restart_rotation_and_logical_delete() { + let (stack, repository, table, mut client, server) = setup().await; + let context = repository.status().await.unwrap().0.context; + let now = common::now_ms(); + let identity = format!("{:08x}-{:04x}-7000-8000-000000000001", now >> 16, now & 0xffff); + let path = format!("/iceberg/{}data/part.parquet", table.storage_path().unwrap()); + let query = format!("crowdb-native-upload={identity}"); + let bytes = b"PAR1datafoot\x04\0\0\0PAR1"; + let response = client.send(Method::PUT, &path, &query, bytes, false).await; + let status = response.status(); + let body = response.text().await.unwrap(); + assert_eq!(status, 200, "{body}"); + let uri = serde_json::from_str::(&body).unwrap()["location"] + .as_str() + .unwrap() + .to_owned(); + let location = uri.parse::().unwrap(); + let direct = format!("/iceberg/{}", location.object_key()); + assert!(location.relative_key().starts_with("f/v1/part.")); + let replay = client.send(Method::PUT, &path, &query, bytes, false).await; + assert_eq!(replay.status(), 200); + assert_eq!(replay.text().await.unwrap(), body); + let file = FileRepository::new(stack.store()) + .load(context, &location) + .await + .unwrap() + .unwrap(); + server.abort(); + let _ = server.await; + let (address, server) = listen(&stack, repository, true).await; + client.address = address; + let mut grant = client.credentials.grant().clone(); + grant.nonce = OperationId::random(); + client.credentials = FileGrantIssuer::new(rotated_authentication().namespace_token_key(), 900_000) + .unwrap() + .issue(grant) + .unwrap(); + let kv = stack.store(); + kv.native_file_reads.store(0, Ordering::SeqCst); + kv.deny_native_file_reads.store(true, Ordering::SeqCst); + assert_eq!( + client + .send(Method::HEAD, "/iceberg", "", b"", false) + .await + .status(), + 200 + ); + let head = client.send(Method::HEAD, &direct, "", b"", false).await; + assert_eq!(head.status(), 200); + assert_eq!(head.headers()["content-length"], bytes.len().to_string()); + let tag = head.headers()["etag"].to_str().unwrap().to_owned(); + assert_conditional_ranges(&client, &direct, &tag).await; + let response = client + .send( + Method::GET, + "/iceberg", + &format!("list-type=2&prefix={}", table.storage_path().unwrap()), + b"", + false, + ) + .await; + let list = response.text().await.unwrap(); + assert!(list.contains("f/v1/part."), "{list}"); + assert_eq!(kv.native_file_reads.load(Ordering::SeqCst), 0); + kv.deny_native_file_reads.store(false, Ordering::SeqCst); + FileRepository::new(kv.clone()) + .mark_deleted(context, &file, common::now_ms()) + .await + .unwrap(); + kv.native_file_reads.store(0, Ordering::SeqCst); + kv.deny_native_file_reads.store(true, Ordering::SeqCst); + let response = client.send(Method::GET, &direct, "", b"", false).await; + assert_eq!(response.status(), 200); + assert_eq!(response.bytes().await.unwrap(), bytes.as_slice()); + let response = client + .send( + Method::GET, + "/iceberg", + &format!("list-type=2&prefix={}", table.storage_path().unwrap()), + b"", + false, + ) + .await; + assert!(!response.text().await.unwrap().contains("f/v1/part.")); + assert_eq!(kv.native_file_reads.load(Ordering::SeqCst), 0); + let crowdb_access_iceberg::file::FileContent::Chunks { root: Some(root) } = &file.content else { + panic!() + }; + stack.blocks.remove_root(root); + for method in [Method::HEAD, Method::GET] { + let response = client + .request(method, &direct, "", b"", false, None) + .header("If-None-Match", &tag) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 404); + } + assert_eq!(kv.native_file_reads.load(Ordering::SeqCst), 0); + server.abort(); + let _ = server.await; +} + +async fn assert_conditional_ranges(client: &signed::TestFileClient, direct: &str, tag: &str) { + let response = client + .send_range(Method::GET, direct, "", b"", false, Some("bytes=-4")) + .await; + assert_eq!(response.status(), 206); + assert_eq!(response.bytes().await.unwrap(), b"PAR1".as_slice()); + let response = client + .request(Method::GET, direct, "", b"", false, None) + .header("If-None-Match", tag) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 304); + let response = client + .request(Method::GET, direct, "", b"", false, None) + .header("If-Match", "\"wrong\"") + .send() + .await + .unwrap(); + assert_eq!(response.status(), 412); +} diff --git a/app/crowdb-access-server/tests/iceberg_table_acceptance_test.rs b/app/crowdb-access-server/tests/iceberg_table_acceptance_test.rs index b64492d88..17aeb501b 100644 --- a/app/crowdb-access-server/tests/iceberg_table_acceptance_test.rs +++ b/app/crowdb-access-server/tests/iceberg_table_acceptance_test.rs @@ -11,7 +11,7 @@ use std::{collections::BTreeMap, sync::atomic::Ordering, time::Duration}; use crowdb_access_iceberg::{ catalog::StoredValue, - key::{CatalogScope, IcebergKey, OperationId, TableId}, + key::{CatalogScope, IcebergKey, OperationId, StorageKey, TableId, TableStorageKey, TableStorageScope}, record::StorageRecord, table::{head_key, name_key, TableHead, TableLifecycle, TableMapping, TableMappingState}, }; @@ -164,15 +164,18 @@ fn authority(test: &TestTableHttp) -> BTreeMap, StoredValue> { .iter() .filter(|(key, _)| { matches!( - IcebergKey::decode(key), - Ok(IcebergKey::Catalog { + StorageKey::decode(key), + Ok(StorageKey::Legacy(IcebergKey::Catalog { scope: CatalogScope::TableHead | CatalogScope::TableName | CatalogScope::File | CatalogScope::FileLocation | CatalogScope::Reclamation, .. - }) + }) | StorageKey::NativeTable(TableStorageKey { + scope: TableStorageScope::Head | TableStorageScope::File, + .. + })) ) }) .map(|(key, value)| (key.clone(), value.clone())) @@ -232,11 +235,14 @@ async fn lost_drop_publication_reply_replays_once_and_preserves_files_for_both_p let after = authority(&test); for (key, value) in before { if matches!( - IcebergKey::decode(&key).unwrap(), - IcebergKey::Catalog { + StorageKey::decode(&key).unwrap(), + StorageKey::Legacy(IcebergKey::Catalog { scope: CatalogScope::File | CatalogScope::FileLocation, .. - } + }) | StorageKey::NativeTable(TableStorageKey { + scope: TableStorageScope::File, + .. + }) ) { assert_eq!(after.get(&key), Some(&value)); } @@ -245,11 +251,11 @@ async fn lost_drop_publication_reply_replays_once_and_preserves_files_for_both_p .keys() .filter(|key| { matches!( - IcebergKey::decode(key).unwrap(), - IcebergKey::Catalog { + StorageKey::decode(key).unwrap(), + StorageKey::Legacy(IcebergKey::Catalog { scope: CatalogScope::Reclamation, .. - } + }) ) }) .count(); diff --git a/app/crowdb-access-server/tests/iceberg_table_lifecycle_test.rs b/app/crowdb-access-server/tests/iceberg_table_lifecycle_test.rs index 06da2ac96..1fd30c9d6 100644 --- a/app/crowdb-access-server/tests/iceberg_table_lifecycle_test.rs +++ b/app/crowdb-access-server/tests/iceberg_table_lifecycle_test.rs @@ -232,7 +232,17 @@ async fn credential_refresh_follows_exact_renamed_identity_and_stops_after_drop( .await, ) .await; - value(test.request(Method::GET, old, "w", None).await, 404).await; + let refreshed = value(test.request(Method::GET, old, "w", None).await, 200).await; + assert_eq!( + refreshed["storage-credentials"][0]["prefix"] + .as_str() + .unwrap() + .trim_end_matches('/'), + created["metadata"]["location"] + .as_str() + .unwrap() + .trim_end_matches('/') + ); let renamed = "/v1/namespaces/analytics/tables/renamed"; let moved = value(test.request(Method::GET, renamed, "w", None).await, 200).await; let current = moved["config"]["client.refresh-credentials-endpoint"] diff --git a/container/single-node-container/README.md b/container/single-node-container/README.md index c5fff2339..963514844 100644 --- a/container/single-node-container/README.md +++ b/container/single-node-container/README.md @@ -88,8 +88,11 @@ pixi run clean-container `start-container` builds the image, starts `crowdb-single-node`, exposes ports 9090–9094 on the container network, waits for readiness, and prints client credentials. Docker creates an anonymous data volume unless configured -otherwise. `inject-container` loads TPC-H SF1 into Iceberg and copies its -Parquet files to the container's S3 endpoint. The loader retains its data and +otherwise. `inject-container` loads TPC-H SF1 into native Iceberg and copies its +Parquet files to the container's S3 endpoint as an independent benchmark copy. +The S3 report's bucket and keys are not Iceberg locations or backing storage; +Iceberg manifests use the native listener's returned table-scoped URLs. +The loader retains its data and reports under `.crowdb-runtime/artifacts/tpc-loader/`. `clean-container` removes the named container, other exited containers created from the selected image, all untagged images that are not referenced by any container, and all diff --git a/doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md b/doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md index db0c3bc05..fc8c5c7bd 100644 --- a/doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md +++ b/doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md @@ -24,12 +24,13 @@ Depends on: [Native Iceberg Storage](design-crowdb-iceberg.md) and Native injection owns both byte ingestion and the subsequent construction of Iceberg file references: -1. Finish durable IcebergTable chunk writes and seal a complete native - descriptor or bounded chunk-resident directory root. +1. Finish durable IcebergTable chunk writes and seal an owner-bound directory + root. Even a one-leaf native tree includes its file identity in that root. 2. Form `f/v1/..parquet` under the immutable `///` prefix. Bind the locator's unique file identity, length, descriptor and exact filename by signature. -3. Publish one `file/` record with create-only CAS. Return the one +3. Durably select the descriptor in the UUIDv7 injection operation before + publishing one `file/` record with create-only CAS. Return the one `s3://iceberg//` URL after publication. A retry uses the same operation/file identity and returns the selected URL; conflicts never overwrite published content. @@ -39,11 +40,28 @@ Iceberg file references: safe cleanup flow; they do not change a previously returned URL. Direct HEAD/GET/range requests validate cached permissions and the locator, -then read native storage without file-record or chunk-KV location lookup. +then read native storage without Iceberg file-location or file-state KV lookup. +The chunk layer still performs its normal replica/EC layout resolution. +HEAD and conditional reads probe the physical directory root; GET reuses the +verified directory in its pull reader. Reclaimed or reused roots fail closed. List and cleanup still use the single file record. Logical deletion removes the file from list; its known direct URL may read retained bytes until physical GC. Reclaimed identities cannot alias newly allocated bytes. +The native injection entry is a signed PUT with +`crowdb-native-upload=` on a table-scoped parquet upload URI. It requires +an explicit SHA-256 payload hash and returns one JSON `location` after durable +selection and publication. The operation binds its principal, full table path, +readable stem, payload digest and retry deadline. A missing publication is +completed from the selected receipt after response loss; a changed or deleted +publication cannot be replayed. The listener advertises the single-request byte +bound. Native loader parts stay within it; normal SDK multipart retains its +chosen logical URL. + +The loader journals the stable upload URI before transfer and the returned +location before adding standard manifest references. Its report preserves the +upload URI for recovery; it does not add a file-location field to Iceberg metadata. + ### Standard SDK uploads SDK location providers choose logical filenames before PUT. These uploads diff --git a/doc/design/access-server/iceberge/design-crowdb-iceberg.md b/doc/design/access-server/iceberge/design-crowdb-iceberg.md index a297af6c9..05327798f 100644 --- a/doc/design/access-server/iceberge/design-crowdb-iceberg.md +++ b/doc/design/access-server/iceberge/design-crowdb-iceberg.md @@ -332,8 +332,9 @@ contract, not delegation to the general S3 authority. File publication, immutability, authorization, and deletion remain under Iceberg control. Public URIs use the fixed virtual bucket `iceberg`: `s3://iceberg////`. -Typed locations use lower-case unpadded base32 catalog IDs and lower-case hex -namespace and table IDs. Relative UTF-8 object keys preserve case, literal percent signs, plus +Native typed locations use lower-case fixed-width hexadecimal for all three +128-bit IDs. Legacy compatibility buckets retain their base32 catalog component. +Relative UTF-8 object keys preserve case, literal percent signs, plus signs and repeated internal slashes; the whole object key is bounded to 1,024 bytes. Dot traversal, leading slash, backslash, controls, query and fragment delimiters are rejected rather than normalized. HTTP percent decoding belongs @@ -441,6 +442,11 @@ unbounded URL. Native chunk routing preserves repair/failover without embedding transient replica addresses in URLs. Reclaimed identities and reused extents must never make an old locator return a different file's bytes. +Native trees retain an owner-bound directory root even when they contain one +leaf. Its digest and encoded full file identity distinguish an identical-content +replacement at a reused extent. Direct request preflight checks that root before +HEAD or conditional success, and GET reuses its verified bytes in the pull reader. + Authorization uses bounded validated grant/context caches and signature, scope and expiry checks. An authorization cache miss may load authority; it must not become a per-file location/state lookup. Expired entries are not @@ -452,6 +458,22 @@ verification material for their files' lifetime. A locator is not a bearer permission. Cold/warm authorization costs and storage resolution counts are measured separately. +Native catalog authority persists the default locator verification/signing key, +independent of access tokens. Key version 1 is reserved for that catalog key; +renaming and capability activation preserve it. Additional signing versions are +configured through `iceberg.locator_signing`, with an active version and retained +verification keys. Configured version 1 must match the persisted catalog key. +Every serving node uses the same configured versions and retains them until the +corresponding files are physically reclaimed. A lock-free cache selects the +current catalog's key ring; clearing to a new catalog selects its new default key. +Native authority uses record envelope 2 so a legacy server cannot silently ignore +the storage layout. Legacy authorities retain envelope 1 and omitted layout +fields decode as the legacy layout. + +Credential refresh URLs carry the table ID and frozen storage namespace ID. +The listener resolves that immutable head directly, so refreshing an old endpoint +after a move or namespace recreation does not require a table-ID reverse index. + Legacy chunk-backed files use bounded leaf blocks and immutable chunk-resident directory pages, with at most 256 children per page and eight directory levels. Each page binds its catalog, table and file identity, child heights and covered byte count. diff --git a/doc/working/plan-iceberg-storage-path.md b/doc/working/plan-iceberg-storage-path.md index 3257c9d06..7911e668e 100644 --- a/doc/working/plan-iceberg-storage-path.md +++ b/doc/working/plan-iceberg-storage-path.md @@ -15,14 +15,47 @@ Goal: make catalog, namespace, and table identities explicit in the Iceberg KV and FileIO path model, keep file bytes in the native CROWDB Iceberg storage, and preserve file locations across table rename and cross-namespace move. -Status: implementation in progress. Native identity/URI/key codecs, storage -dispatch, logical file publication/resolution and bounded prefix listing are -implemented. File/directory/grant/name-mapping records preserve storage namespace. -Native head lookup/publication/rename/drop/GC fences and direct URL reads are -implemented. Catalog v2 activation, remaining workflow/GC/projection keys and -injection HTTP/loader/recovery integration remain pending. Direct reads skip -Iceberg file-location/state KV and preserve -the chunk layer's normal layout resolution, as confirmed in the chat. +Status: implementation in progress. Native catalog layout selection, default +namespace creation, table/file/projection paths, direct URL reads, upload +selection/replay and the loader's returned-location handoff are implemented. +Remaining work is the auxiliary-key/recovery audit, complete native SDK/container +acceptance, and final design/backlog cleanup. Direct reads skip Iceberg +file-location/state KV and retain the chunk layer's normal layout resolution, +as confirmed in the chat. + +Latest verified behavior: + +- Native catalog authorities persist layout 2 in a record envelope that older + servers reject; omitted legacy layout fields remain version 1. Rename and + capability activation preserve layout. New initialization/clear chooses 2. + The authority also persists the reserved version-1 locator key; access-token + rotation cannot invalidate file URLs. Configured later signing versions retain + that catalog key automatically. Both credential and locator key rotation pass + the native HTTP read test. +- Native default namespace creation uses normal durable namespace publication + before table metadata writes. Native creation and replay retain one frozen + namespace, one head and one filename authority. +- Native GC discovers v2 file records through bounded v2 scans, resolves standard + graph references by canonical filename, deletes that authority and removes + native projection pages at catalog retirement. Restart-at-every-step tests pass. +- Injection PUT selects an operation receipt before file publication. Both lost + receipt and lost publication replies replay the same FileId, root and URL; + changed digest/principal/identity and deleted-file replays fail closed. +- Direct locators require an owner-bound directory root even for one leaf. + An identical replacement at a reused root extent cannot revive the old URL. + HTTP probes root ownership for HEAD/GET/conditions and reuses the preflight + directory in its pull reader, without file-location/state KV. +- Native HTTP integration forbids all file KV GETs and verifies cold restart, + locator-key rotation, range reads, ETag conditions, scoped listing, logical + delete retention, and HEAD/GET failure after physical root removal. +- Credential refresh includes the frozen namespace in the existing endpoint URL. + Moving a table, dropping/recreating its default namespace and refreshing through + the old endpoint retains its old prefix without any table-ID reverse index. +- The TPC loader journals its UUIDv7 upload URI before transfer and the selected + returned URL before manifest registration. Python verification: 222 passed, + 7 explicitly skipped integrations; Ruff check/format passed. Existing unrelated + loader edits are preserved. A pre-existing region-only assertion was corrected + to validate the actual table name for the nation case as well. Verified foundation gates: @@ -134,8 +167,9 @@ table-ID reverse index: callers and recovery tasks carry the full identity. acceleration, not another file record or authoritative snapshot store. The first three path segments are parsed by fixed identity encodings and -positions. The canonical encodings remain lower-case unpadded base32 for the -catalog ID and lower-case fixed-width hexadecimal for namespace and table IDs; +positions. The native canonical encodings are lower-case fixed-width hexadecimal for all +three 128-bit IDs (32 bytes of ASCII each). Legacy buckets retain their base32 +catalog component; labels such as `namespace` and `table` are rejected rather than accepted as aliases. @@ -475,7 +509,7 @@ three-ID prefix and a matching grant. prefixes. Round trips must recover all three IDs and the relative file key. Files: `lib/crowdb-access-iceberg/src/file/location.rs`, `lib/crowdb-access-iceberg/tests/file_location_test.rs`. -- [ ] **Assign the storage path once** during table creation, including staged +- [x] **Assign the storage path once** during table creation, including staged creation, and bind it into the table head and initial metadata. Rename and cross-namespace move must retain it; a newly created table gets a new table identity and path. @@ -494,13 +528,13 @@ three-ID prefix and a matching grant. `lib/crowdb-access-iceberg/src/metadata_projection/`, `lib/crowdb-access-iceberg/src/key/codec.rs`. -- [ ] **Introduce one table-path prefix encoder** for catalog, storage +- [x] **Introduce one table-path prefix encoder** for catalog, storage namespace, and table IDs. All table-owned file keys must use it rather than assembling ad-hoc suffixes. Files: `lib/crowdb-access-iceberg/src/key.rs`, `lib/crowdb-access-iceberg/src/key/codec.rs`, `lib/crowdb-access-iceberg/src/file/key.rs`. -- [ ] **Move the single file authority under the prefix**: +- [x] **Move the single file authority under the prefix**: `table-path/file/` is the complete selected record. Do not introduce separate file name, public-key or file-ID indexes. Preserve bounded ordered scans and reject a range @@ -509,7 +543,7 @@ three-ID prefix and a matching grant. `lib/crowdb-access-iceberg/src/file/listing.rs`, `lib/crowdb-access-iceberg/src/file/listing/repository.rs`, `lib/crowdb-access-iceberg/src/catalog/storage.rs`. -- [ ] **Audit every table-owned auxiliary key**: multipart records, write +- [~] **Audit every table-owned auxiliary key**: multipart records, write intents/fences, GC candidates/nodes/pending state, and recovery scans must either carry the same table-path prefix or explicitly document why an operation-scoped key is independent. Update range bounds and foreign-record @@ -559,7 +593,7 @@ three-ID prefix and a matching grant. Files: `lib/crowdb-access-iceberg/src/file/blocks.rs`, `lib/crowdb-chunk-client/src/chunk/chunk_reader.rs`, `lib/crowdb-chunk-client/tests/chunk_reader_test.rs`. -- [ ] **Reuse bounded permission validation**: define permission-cache entries +- [x] **Reuse bounded permission validation**: define permission-cache entries by principal, catalog epoch, full storage identity and operations; bound validity by grants and applicable authority leases. Misses validate authority, expired entries fail closed, and direct reads never load per-file state. @@ -605,7 +639,7 @@ three-ID prefix and a matching grant. `lib/crowdb-access-iceberg/src/file/location.rs`, `app/crowdb-access-server/src/iceberg/file_http/`, `lib/crowdb-access-iceberg/src/wire/credentials.rs`. -- [ ] **Add direct locator validation**: bind locator version, table identity, +- [x] **Add direct locator validation**: bind locator version, table identity, file identity, length, integrity metadata and chunk root; authenticate the current HTTP grant independently of the durable locator signature; reject tampering before chunk access and enforce one contiguous range. @@ -624,7 +658,7 @@ three-ID prefix and a matching grant. `lib/crowdb-access-iceberg/src/table/metadata/`, `lib/crowdb-access-iceberg/src/metadata_projection/`, `lib/crowdb-access-iceberg/src/commit/files.rs`. -- [ ] **Implement bounded metadata prefix list**: map the public metadata or +- [x] **Implement bounded metadata prefix list**: map the public metadata or filename prefix to the exact `file/` range; stream ordered records into a bounded page and persist an authenticated continuation at scan-budget exhaustion. Handle tombstones, sparse pages, delimiter common prefixes and diff --git a/e2e-test/tests/access_iceberg/native_files.rs b/e2e-test/tests/access_iceberg/native_files.rs index d10ef0345..2c9209326 100644 --- a/e2e-test/tests/access_iceberg/native_files.rs +++ b/e2e-test/tests/access_iceberg/native_files.rs @@ -42,6 +42,7 @@ async fn assert_checkpoint_intent( let intents = crowdb_access_iceberg::gc::GcStore::scan_gc( stack.store().await.as_ref(), crowdb_access_iceberg::gc::GcScan { + native: false, catalog: owner.table.catalog, scope: Some(crowdb_access_iceberg::key::CatalogScope::FileWriteIntent), prefix: owner.table.table.as_bytes().to_vec(), diff --git a/lib/crowdb-access-iceberg/src/catalog.rs b/lib/crowdb-access-iceberg/src/catalog.rs index 0fcaacdb5..030e9556d 100644 --- a/lib/crowdb-access-iceberg/src/catalog.rs +++ b/lib/crowdb-access-iceberg/src/catalog.rs @@ -14,7 +14,10 @@ pub(crate) use context::check_context; pub use deadline::ClearBounds; pub use repository::{CatalogError, CatalogRepository, ManagementPrivilege}; pub use root::{ActiveCatalogRecord, RootState}; -pub use state::{CatalogAuthority, CatalogContext, CatalogLifecycle, ClearTransition}; +pub use state::{ + CatalogAuthority, CatalogContext, CatalogLifecycle, CatalogLocatorKey, CatalogStorageLayout, + ClearTransition, +}; pub use storage::{ CasOutcome, CatalogStore, CatalogStoreOperationCounts, CatalogStoreOperationMeter, RoutedCatalogStore, StoreError, StoredValue, diff --git a/lib/crowdb-access-iceberg/src/catalog/repository.rs b/lib/crowdb-access-iceberg/src/catalog/repository.rs index 58f177c80..04b9371f8 100644 --- a/lib/crowdb-access-iceberg/src/catalog/repository.rs +++ b/lib/crowdb-access-iceberg/src/catalog/repository.rs @@ -272,7 +272,7 @@ impl CatalogRepository { .as_ref() .ok_or(ValidationError::Record)? .activated(request.capabilities.ok_or(ValidationError::Record)?)?, - _ => CatalogAuthority::new(candidate, request.display_name.clone())?, + _ => CatalogAuthority::new_native(candidate, request.display_name.clone())?, }; result.admission_bounds = bounds; Ok(ManagementOperation { diff --git a/lib/crowdb-access-iceberg/src/catalog/state.rs b/lib/crowdb-access-iceberg/src/catalog/state.rs index 27bb186d9..84f530fb1 100644 --- a/lib/crowdb-access-iceberg/src/catalog/state.rs +++ b/lib/crowdb-access-iceberg/src/catalog/state.rs @@ -42,8 +42,63 @@ pub enum CatalogLifecycle { Retired, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u8)] +pub enum CatalogStorageLayout { + Legacy = 1, + Native = 2, +} + +#[derive(Clone, Copy, Eq, PartialEq)] +pub struct CatalogLocatorKey([u8; 32]); + +impl CatalogLocatorKey { + /// # Errors + /// Rejects an uninitialized persistent locator secret. + pub fn new(bytes: [u8; 32]) -> Result { + if bytes == [0; 32] { + return Err(ValidationError::Record); + } + Ok(Self(bytes)) + } + + #[must_use] + pub const fn bytes(self) -> [u8; 32] { + self.0 + } + + fn generate() -> Self { + use sha2::{Digest, Sha256}; + let mut digest = Sha256::new(); + digest.update(b"crowdb-native-locator-key-v1"); + digest.update(uuid::Uuid::new_v4().as_bytes()); + digest.update(uuid::Uuid::new_v4().as_bytes()); + Self(digest.finalize().into()) + } +} + +impl std::fmt::Debug for CatalogLocatorKey { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("CatalogLocatorKey([redacted])") + } +} + +impl CatalogStorageLayout { + /// # Errors + /// Rejects unrecognized persisted layouts without attempting a fallback. + pub fn from_version(version: u8) -> Result { + match version { + 1 => Ok(Self::Legacy), + 2 => Ok(Self::Native), + _ => Err(ValidationError::KeyVersion(version)), + } + } +} + #[derive(Clone, Debug, Eq, PartialEq)] pub struct CatalogAuthority { + pub storage_layout: CatalogStorageLayout, + pub locator_key: Option, pub catalog: CatalogId, pub display_name: String, pub name_generation: u64, @@ -58,6 +113,8 @@ impl CatalogAuthority { /// Rejects invalid display names; no table format support is enabled initially. pub fn new(catalog: CatalogId, display_name: String) -> Result { let authority = Self { + storage_layout: CatalogStorageLayout::Legacy, + locator_key: None, catalog, display_name, name_generation: 1, @@ -70,6 +127,17 @@ impl CatalogAuthority { Ok(authority) } + /// Creates an authority for a new catalog without relabeling existing data. + /// # Errors + /// Rejects invalid display names. + pub fn new_native(catalog: CatalogId, display_name: String) -> Result { + Ok(Self { + storage_layout: CatalogStorageLayout::Native, + locator_key: Some(CatalogLocatorKey::generate()), + ..Self::new(catalog, display_name)? + }) + } + /// # Errors /// Rejects malformed names, zero generations and inconsistent capabilities. pub fn validate(&self) -> Result<(), ValidationError> { @@ -77,6 +145,9 @@ impl CatalogAuthority { if self.name_generation == 0 || self.config_generation == 0 { return Err(ValidationError::Record); } + if (self.storage_layout == CatalogStorageLayout::Native) != self.locator_key.is_some() { + return Err(ValidationError::Record); + } self.admission_bounds.completion_deadline(0)?; self.capabilities.validate() } @@ -94,6 +165,8 @@ impl CatalogAuthority { .checked_add(1) .ok_or(ValidationError::GenerationExhausted)?; Ok(Self { + storage_layout: self.storage_layout, + locator_key: self.locator_key, catalog: self.catalog, display_name, name_generation, diff --git a/lib/crowdb-access-iceberg/src/commit/create/publisher/preparation.rs b/lib/crowdb-access-iceberg/src/commit/create/publisher/preparation.rs index ed439f677..7d29d67ea 100644 --- a/lib/crowdb-access-iceberg/src/commit/create/publisher/preparation.rs +++ b/lib/crowdb-access-iceberg/src/commit/create/publisher/preparation.rs @@ -5,6 +5,29 @@ use super::{ }; impl TableCreator { + async fn storage_layout( + &self, + context: crate::catalog::CatalogContext, + ) -> Result { + let authority_key = crate::key::IcebergKey::Catalog { + catalog: context.catalog, + scope: crate::key::CatalogScope::Authority, + suffix: Vec::new(), + }; + let value = self + .store + .get(&authority_key.encode()?) + .await + .map_err(CatalogError::from)? + .ok_or(CatalogError::Uninitialized)?; + let crate::record::StorageRecord::Authority(authority) = + crate::record::StorageRecord::decode(&authority_key, &value.bytes)? + else { + return Err(ValidationError::Record.into()); + }; + Ok(authority.storage_layout) + } + pub(super) async fn prepare( &self, request: &TableCreationRequest, @@ -14,10 +37,20 @@ impl TableCreator { if decoded.stage_create() != expires_ms.is_some() { return Err(Error::Unsupported("staged table creation")); } - let namespace = NamespaceRepository::from_parts(self.store.clone(), self.names.clone()) - .load(request.context, &request.namespace) - .await? - .ok_or(Error::NamespaceMissing)?; + let layout = self.storage_layout(request.context).await?; + let repository = NamespaceRepository::from_parts(self.store.clone(), self.names.clone()); + let namespace = if layout == crate::catalog::CatalogStorageLayout::Native + && request.namespace.components() == ["default"] + { + repository + .ensure_default(request.context, request.identity.issued_ms) + .await? + } else { + repository + .load(request.context, &request.namespace) + .await? + .ok_or(Error::NamespaceMissing)? + }; if let Some(expiry) = expires_ms { if request.timestamp_ms < 0 || expiry <= request.timestamp_ms { return Err(ValidationError::Record.into()); @@ -41,7 +74,13 @@ impl TableCreator { .entry("created-at") .or_insert(serde_json::Value::String(created)); } - let target = target(request, namespace.namespace, decoded.name(), expires_ms.is_some())?; + let target = target( + request, + namespace.namespace, + decoded.name(), + expires_ms.is_some(), + layout, + )?; let initial = evaluate_table_creation(&decoded, target, request.timestamp_ms, limits())?; let response = if expires_ms.is_some() { super::staged::stage_response(initial.document.canonical())? @@ -96,9 +135,13 @@ fn target( namespace: crate::key::NamespaceId, name: &str, staged: bool, + layout: crate::catalog::CatalogStorageLayout, ) -> Result { let table = TableLocation { - storage_namespace: None, + storage_namespace: match layout { + crate::catalog::CatalogStorageLayout::Legacy => None, + crate::catalog::CatalogStorageLayout::Native => Some(namespace), + }, catalog: request.context.catalog, table: if staged { TableId::from_bytes(request.identity.operation.as_bytes())? diff --git a/lib/crowdb-access-iceberg/src/file.rs b/lib/crowdb-access-iceberg/src/file.rs index a5e8e8313..d2b45ad19 100644 --- a/lib/crowdb-access-iceberg/src/file.rs +++ b/lib/crowdb-access-iceberg/src/file.rs @@ -8,6 +8,8 @@ mod credentials; mod deletion_vector; mod digest; mod direct_location; +mod injection; +pub use injection::{NativeInjection, NativeInjectionRepository, NativeInjectionRequest}; mod directory; mod format; mod json; diff --git a/lib/crowdb-access-iceberg/src/file/direct_location.rs b/lib/crowdb-access-iceberg/src/file/direct_location.rs index 494c482cc..cc45d6e4d 100644 --- a/lib/crowdb-access-iceberg/src/file/direct_location.rs +++ b/lib/crowdb-access-iceberg/src/file/direct_location.rs @@ -12,12 +12,38 @@ const PAYLOAD_BYTES: usize = 142; const TOKEN_BYTES: usize = PAYLOAD_BYTES + 32; /// Persistent file locators use a key ring independent of expiring HTTP grants. +#[derive(Clone)] pub struct DirectFileLocations { active_version: u32, keys: BTreeMap, } impl DirectFileLocations { + /// # Errors + /// Rejects a configured replacement of the catalog's reserved key version. + pub fn for_catalog( + &self, + key: crate::catalog::CatalogLocatorKey, + strict: bool, + ) -> Result { + let key = key.bytes(); + if strict && self.keys.get(&1).is_some_and(|configured| *configured != key) { + return Err(ValidationError::IdentityMismatch); + } + let mut keys = self.keys.clone(); + keys.insert(1, key); + Self::new(self.active_version, keys) + } + /// # Errors + /// Rejects stems which cannot fit the final authenticated object key. + pub fn validate_stem(stem: &str) -> Result<(), ValidationError> { + const LOCATOR_KEY_BYTES: usize = 99 + 5 + 1 + 232 + 8; + super::validate_relative_key(stem)?; + if stem.contains('/') || stem.len() > super::MAX_OBJECT_KEY_BYTES - LOCATOR_KEY_BYTES { + return Err(ValidationError::Key); + } + Ok(()) + } /// # Errors /// Rejects missing active keys, reserved versions and uninitialized keys. pub fn new(active_version: u32, keys: BTreeMap) -> Result { @@ -42,10 +68,10 @@ impl DirectFileLocations { digest: [u8; 32], root: ChunkRoot, ) -> Result { - if table.storage_namespace.is_none() || stem.contains('/') { + if table.storage_namespace.is_none() || root.height == 0 { return Err(ValidationError::Key); } - super::validate_relative_key(stem)?; + Self::validate_stem(stem)?; let mut payload = Vec::with_capacity(TOKEN_BYTES); payload.push(1); payload.extend_from_slice(&self.active_version.to_be_bytes()); @@ -65,7 +91,7 @@ impl DirectFileLocations { let record = FileRecord { file, location, - kind: FileKind::Data, + kind: FileKind::Unbound, format: ContentFormat::Parquet, length, digest, @@ -82,6 +108,9 @@ impl DirectFileLocations { /// # Errors /// Rejects reserved-path corruption, altered identities and unknown signing keys. pub fn resolve(&self, location: &FileLocation) -> Result, ValidationError> { + if location.table().storage_namespace.is_some() && location.relative_key() == "f" { + return Err(ValidationError::Key); + } if location.table().storage_namespace.is_none() || !location.relative_key().starts_with("f/") { return Ok(None); } @@ -108,10 +137,13 @@ impl DirectFileLocations { let length = fields.number()?; let digest = fields.take()?; let root = decode_root(&mut fields)?; + if root.height == 0 { + return Err(ValidationError::Record); + } let record = FileRecord { file, location: location.clone(), - kind: FileKind::Data, + kind: FileKind::Unbound, format: ContentFormat::Parquet, length, digest, diff --git a/lib/crowdb-access-iceberg/src/file/injection.rs b/lib/crowdb-access-iceberg/src/file/injection.rs new file mode 100644 index 000000000..8d5b63b48 --- /dev/null +++ b/lib/crowdb-access-iceberg/src/file/injection.rs @@ -0,0 +1,202 @@ +use std::sync::Arc; + +use crate::{ + catalog::{check_context, CasOutcome, CatalogContext, CatalogError, CatalogStore}, + error::ValidationError, + key::{StorageKey, TableStorageKey, TableStorageScope}, + operation::{mutation_identity, RequestIdentity, RETRY_WINDOW_MS}, + record::StorageRecord, +}; + +use super::{DirectFileLocations, FileIdentity, FileRecord, FileRepository, FileTree, TableLocation}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct NativeInjection { + pub identity: RequestIdentity, + pub activation_epoch: u64, + pub principal: [u8; 32], + pub stem: String, + pub file: FileRecord, +} + +impl NativeInjection { + /// # Errors + /// Rejects legacy ownership, unsupported content and unbounded retention. + pub fn validate(&self) -> Result<(), ValidationError> { + self.file.validate()?; + DirectFileLocations::validate_stem(&self.stem)?; + if self.activation_epoch == 0 + || self.file.location.table().storage_namespace.is_none() + || self.stem.contains('/') + || !self + .file + .location + .relative_key() + .starts_with(&format!("f/v1/{}.", self.stem)) + { + return Err(ValidationError::Record); + } + self.identity + .issued_ms + .checked_add(RETRY_WINDOW_MS) + .ok_or(ValidationError::Deadline)?; + Ok(()) + } + + /// # Errors + /// Rejects legacy storage identities. + pub fn key(&self) -> Result { + injection_key(self.file.location.table(), self.identity) + } + + #[must_use] + pub fn retained(&self, now_ms: u64) -> bool { + now_ms <= self.identity.issued_ms.saturating_add(RETRY_WINDOW_MS) + } +} + +#[derive(Clone, Debug)] +pub struct NativeInjectionRequest { + pub context: CatalogContext, + pub table: TableLocation, + pub identity: RequestIdentity, + pub principal: [u8; 32], + pub stem: String, + pub digest: [u8; 32], +} + +impl NativeInjectionRequest { + fn validate(&self, now_ms: u64) -> Result<(), ValidationError> { + self.context.validate()?; + self.identity.validate(now_ms)?; + DirectFileLocations::validate_stem(&self.stem)?; + if self.table.catalog != self.context.catalog + || self.table.storage_namespace.is_none() + || self.stem.contains('/') + { + return Err(ValidationError::IdentityMismatch); + } + Ok(()) + } + + fn matches(&self, receipt: &NativeInjection) -> bool { + receipt.identity == self.identity + && receipt.activation_epoch == self.context.activation_epoch + && receipt.file.location.table() == self.table + && receipt.principal == self.principal + && receipt.stem == self.stem + && receipt.file.digest == self.digest + } +} + +pub struct NativeInjectionRepository { + store: Arc, +} + +impl NativeInjectionRepository { + #[must_use] + pub fn new(store: Arc) -> Self { + Self { store } + } + + /// Replays a selected upload without allocating a new tree or changing its URL. + /// # Errors + /// Rejects expired, foreign, changed or logically deleted upload retries. + pub async fn replay( + &self, + request: &NativeInjectionRequest, + now_ms: u64, + ) -> Result, CatalogError> { + request.validate(now_ms)?; + check_context(self.store.as_ref(), request.context).await?; + let key = injection_key(request.table, request.identity)?; + let Some(value) = self.store.get(&key.encode()?).await? else { + return Ok(None); + }; + let StorageRecord::NativeInjection(receipt) = StorageRecord::decode(&key, &value.bytes)? else { + return Err(ValidationError::Record.into()); + }; + if !request.matches(&receipt) { + return Err(CatalogError::Conflict); + } + self.publish(request.context, &receipt.file).await.map(Some) + } + + /// Selects one durable descriptor before publishing its sole filename authority. + /// # Errors + /// Rejects changed payloads and preserves the durable winner after uncertain replies. + pub async fn finish( + &self, + request: &NativeInjectionRequest, + owner: FileIdentity, + tree: FileTree, + locations: &DirectFileLocations, + now_ms: u64, + ) -> Result { + request.validate(now_ms)?; + if owner.table != request.table || tree.digest != request.digest { + return Err(CatalogError::Conflict); + } + check_context(self.store.as_ref(), request.context).await?; + let file = locations.parquet( + owner.table, + &request.stem, + owner.file, + tree.length, + tree.digest, + tree.root.ok_or(ValidationError::Record)?, + )?; + let receipt = NativeInjection { + identity: request.identity, + activation_epoch: request.context.activation_epoch, + principal: request.principal, + stem: request.stem.clone(), + file, + }; + let key = receipt.key()?; + let bytes = StorageRecord::NativeInjection(Box::new(receipt.clone())).encode()?; + let encoded = key.encode()?; + let selected = match self + .store + .compare_exchange(&encoded, None, &bytes, mutation_identity(&encoded, None, &bytes)) + .await + { + Ok(CasOutcome::Applied(_)) => receipt, + outcome => { + let value = match outcome { + Ok(CasOutcome::Conflict(Some(value))) => value, + Ok(CasOutcome::Conflict(None)) => return Err(CatalogError::Conflict), + Err(error) => self + .store + .get(&encoded) + .await? + .ok_or(CatalogError::Store(error))?, + Ok(CasOutcome::Applied(_)) => return Err(ValidationError::Record.into()), + }; + let StorageRecord::NativeInjection(receipt) = StorageRecord::decode(&key, &value.bytes)? + else { + return Err(ValidationError::Record.into()); + }; + if !request.matches(&receipt) { + return Err(CatalogError::Conflict); + } + *receipt + } + }; + self.publish(request.context, &selected.file).await + } + + async fn publish(&self, context: CatalogContext, file: &FileRecord) -> Result { + FileRepository::new(self.store.clone()) + .publish(context, file) + .await + } +} + +fn injection_key(table: TableLocation, identity: RequestIdentity) -> Result { + Ok(StorageKey::NativeTable(TableStorageKey { + table: table.storage_path().ok_or(ValidationError::Key)?, + scope: TableStorageScope::Operation, + suffix: format!("{}/injection", identity.operation), + })) +} diff --git a/lib/crowdb-access-iceberg/src/file/reader.rs b/lib/crowdb-access-iceberg/src/file/reader.rs index 1c17e717f..8aa53a507 100644 --- a/lib/crowdb-access-iceberg/src/file/reader.rs +++ b/lib/crowdb-access-iceberg/src/file/reader.rs @@ -33,6 +33,43 @@ struct ReadSource { } impl FileReader { + /// Checks physical directory ownership without any file authority lookup. + /// # Errors + /// Rejects absent, reclaimed, reused or corrupt roots and non-directory descriptors. + pub async fn probe_root(store: &dyn FileBlockStore, record: &FileRecord) -> Result, FileIoError> { + record.validate()?; + let FileContent::Chunks { root: Some(root) } = &record.content else { + return Err(FileIoError::Bounds); + }; + if root.height == 0 { + return Err(FileIoError::Bounds); + } + let bytes = store.read(root).await?; + verify_block(root, &bytes)?; + ChunkDirectory::decode( + &bytes, + FileIdentity { + table: record.location.table(), + file: record.file, + }, + root.height, + record.length, + )?; + Ok(bytes) + } + + /// Reuses a preflight directory without keeping more than one cached directory. + /// # Errors + /// Rejects bytes which do not belong to this reader's root or owner. + pub fn with_root_directory(mut self, bytes: Vec) -> Result { + let FileContent::Chunks { root: Some(root) } = &self.record.content else { + return Err(FileIoError::Bounds); + }; + verify_block(root, &bytes)?; + ChunkDirectory::decode(&bytes, self.record.owner, root.height, self.record.length)?; + self.leaf_directory = Some((root.clone(), bytes)); + Ok(self) + } /// # Errors /// Rejects invalid records, out-of-file intervals and unbounded frame sizes. pub fn new( diff --git a/lib/crowdb-access-iceberg/src/file/writer.rs b/lib/crowdb-access-iceberg/src/file/writer.rs index 7afd0f61f..b3b2b5ce9 100644 --- a/lib/crowdb-access-iceberg/src/file/writer.rs +++ b/lib/crowdb-access-iceberg/src/file/writer.rs @@ -104,6 +104,11 @@ impl FileTreeWriter { continue; } if self.levels[level].len() == 1 && self.levels[level + 1..].iter().all(Vec::is_empty) { + if level == 0 && self.owner.table.storage_namespace.is_some() { + let entry = self.flush_directory(level).await?; + self.append(entry).await?; + continue; + } let entry = self.levels[level].pop().ok_or(FileIoError::Bounds)?; if entry.length != self.length { return Err(FileIoError::Bounds); diff --git a/lib/crowdb-access-iceberg/src/gc/discovery.rs b/lib/crowdb-access-iceberg/src/gc/discovery.rs index 082ea454a..82165a7ab 100644 --- a/lib/crowdb-access-iceberg/src/gc/discovery.rs +++ b/lib/crowdb-access-iceberg/src/gc/discovery.rs @@ -2,7 +2,7 @@ use crate::{ catalog::{CatalogContext, CatalogError}, error::ValidationError, file::MultipartPhase, - key::{CatalogScope, IcebergKey, OperationId}, + key::{CatalogScope, IcebergKey, OperationId, StorageKey, TableStorageScope}, record::StorageRecord, table::{TableHead, TableLifecycle}, }; @@ -83,23 +83,36 @@ impl GcRepository { if !matches!(task.phase, GcPhase::Discover | GcPhase::Rescan) || task.paused { return Err(CatalogError::Busy); } + let native = task.discovery_scope == task.last_discovery_scope(); let scan = GcScan { + native, catalog: task.context.catalog, - scope: Some(if task.kind == GcTaskKind::MultipartCleanup { - match task.discovery_scope { - 0 => CatalogScope::FileLocation, - 1 => CatalogScope::MultipartPart, - _ => CatalogScope::MultipartSession, - } + scope: if native { + None } else { - match task.discovery_scope { - 0 => CatalogScope::File, - 1 => CatalogScope::FileLocation, - 2 => CatalogScope::MultipartPart, - _ => CatalogScope::MultipartSession, - } - }), - prefix: Vec::new(), + Some(if task.kind == GcTaskKind::MultipartCleanup { + match task.discovery_scope { + 0 => CatalogScope::FileLocation, + 1 => CatalogScope::MultipartPart, + _ => CatalogScope::MultipartSession, + } + } else { + match task.discovery_scope { + 0 => CatalogScope::File, + 1 => CatalogScope::FileLocation, + 2 => CatalogScope::MultipartPart, + _ => CatalogScope::MultipartSession, + } + }) + }, + prefix: if native { + task.head + .as_ref() + .and_then(|head| head.storage_location().storage_path()) + .map_or_else(Vec::new, |table| table.to_string().into_bytes()) + } else { + Vec::new() + }, after: task.scan_after.clone(), items: usize::from(limits.page_items), bytes: (crate::record::MAX_RECORD_BYTES + crate::key::MAX_KEY_BYTES) @@ -108,7 +121,7 @@ impl GcRepository { let page = self.store.scan_gc(scan.clone()).await?; scan.validate_page(&page)?; for item in &page.items { - let key = IcebergKey::decode(&item.key)?; + let key = StorageKey::decode(&item.key)?; if let Some(candidate) = self .discovery_candidate(task, &key, &item.value, limits, now_ms) .await? @@ -121,13 +134,7 @@ impl GcRepository { next.scan_after.clone_from(&last.key); } else { next.scan_after.clear(); - if task.discovery_scope - < if task.kind == GcTaskKind::MultipartCleanup { - 2 - } else { - 3 - } - { + if task.discovery_scope < task.last_discovery_scope() { next.discovery_scope += 1; } else if task.phase == GcPhase::Rescan { next.discovery_scope = 0; @@ -157,26 +164,28 @@ impl GcRepository { async fn discovery_candidate( &self, task: &GcTask, - key: &IcebergKey, + key: &StorageKey, bytes: &[u8], limits: GcLimits, now_ms: u64, ) -> Result, CatalogError> { let mut deleted_ms = None; let (file, part, assembly) = match key { - IcebergKey::Catalog { + StorageKey::NativeTable(key) if key.scope != TableStorageScope::File => return Ok(None), + StorageKey::Legacy(IcebergKey::Catalog { scope: CatalogScope::File, .. - } => { + }) => { let StorageRecord::File(file) = StorageRecord::decode(key, bytes)? else { return Err(ValidationError::Record.into()); }; (*file, None, None) } - IcebergKey::Catalog { + StorageKey::NativeTable(_) + | StorageKey::Legacy(IcebergKey::Catalog { scope: CatalogScope::FileLocation, .. - } => match StorageRecord::decode(key, bytes)? { + }) => match StorageRecord::decode(key, bytes)? { StorageRecord::File(file) if task.kind != GcTaskKind::MultipartCleanup => (*file, None, None), StorageRecord::DeletedFile(record) => { deleted_ms = Some(record.deleted_ms); @@ -185,10 +194,10 @@ impl GcRepository { StorageRecord::File(_) | StorageRecord::FileMapping(_) => return Ok(None), _ => return Err(ValidationError::Record.into()), }, - IcebergKey::Catalog { + StorageKey::Legacy(IcebergKey::Catalog { scope: CatalogScope::MultipartPart, .. - } => { + }) => { let StorageRecord::MultipartPart(part) = StorageRecord::decode(key, bytes)? else { return Err(ValidationError::Record.into()); }; @@ -197,10 +206,10 @@ impl GcRepository { } (GcCandidate::part_file(&part)?, Some(*part), None) } - IcebergKey::Catalog { + StorageKey::Legacy(IcebergKey::Catalog { scope: CatalogScope::MultipartSession, .. - } => { + }) => { let StorageRecord::MultipartSession(session) = StorageRecord::decode(key, bytes)? else { return Err(ValidationError::Record.into()); }; @@ -215,7 +224,7 @@ impl GcRepository { } (GcCandidate::assembly_file(&session)?, None, Some(session)) } - _ => return Err(ValidationError::Record.into()), + StorageKey::Legacy(_) => return Err(ValidationError::Record.into()), }; if task .head diff --git a/lib/crowdb-access-iceberg/src/gc/mark.rs b/lib/crowdb-access-iceberg/src/gc/mark.rs index a7355a93c..e926289b0 100644 --- a/lib/crowdb-access-iceberg/src/gc/mark.rs +++ b/lib/crowdb-access-iceberg/src/gc/mark.rs @@ -1,7 +1,7 @@ use crate::{ catalog::CatalogError, error::ValidationError, - file::{file_key, location_key, FileIoError, FileRecord}, + file::{canonical_file_key, file_key, FileIoError, FileRecord}, record::StorageRecord, table::TableMetadataError, }; @@ -27,7 +27,7 @@ impl GcRepository { &self, location: &crate::file::FileLocation, ) -> Result { - let key = location_key(location); + let key = canonical_file_key(location); let value = self .store .get(&key.encode()?) diff --git a/lib/crowdb-access-iceberg/src/gc/storage.rs b/lib/crowdb-access-iceberg/src/gc/storage.rs index 4fe3f603e..20f63c6ab 100644 --- a/lib/crowdb-access-iceberg/src/gc/storage.rs +++ b/lib/crowdb-access-iceberg/src/gc/storage.rs @@ -5,11 +5,12 @@ use crowdb_protocol::chunk_kv::{ClientRequestId, ScanDirection}; use crate::{ catalog::{CasOutcome, CatalogStore, RoutedCatalogStore, StoreError}, error::ValidationError, - key::{CatalogId, CatalogScope, IcebergKey, SystemScope}, + key::{CatalogId, CatalogScope, IcebergKey, StorageKey, StorageTablePath, SystemScope}, }; #[derive(Clone, Debug)] pub struct GcScan { + pub native: bool, pub catalog: CatalogId, pub scope: Option, pub prefix: Vec, @@ -107,6 +108,9 @@ impl GcScan { if self.items == 0 || self.items > 256 || self.bytes == 0 || self.bytes > 16 * 1024 * 1024 { return Err(ValidationError::RecordTooLarge); } + if self.native { + return self.native_request(); + } if self.prefix.len() > 40 || (!self.prefix.is_empty() && self.scope.is_none()) { return Err(ValidationError::Key); } @@ -151,6 +155,41 @@ impl GcScan { }) } + fn native_request(&self) -> Result { + if self.scope.is_some() { + return Err(ValidationError::Key); + } + let mut start = b"ICE\0\x02".to_vec(); + if self.prefix.is_empty() { + start.extend_from_slice(format!("{}/", self.catalog).as_bytes()); + } else { + let prefix = std::str::from_utf8(&self.prefix).map_err(|_| ValidationError::Key)?; + let table = StorageTablePath::from_prefix(prefix)?; + if table.catalog != self.catalog { + return Err(ValidationError::IdentityMismatch); + } + start.extend_from_slice(&self.prefix); + } + let mut end = start.clone(); + *end.last_mut().ok_or(ValidationError::Key)? += 1; + if !self.after.is_empty() { + if self.after < start || self.after >= end { + return Err(ValidationError::Key); + } + StorageKey::decode(&self.after)?; + start.clone_from(&self.after); + start.push(0); + } + Ok(MultiScanRequest { + start: Some(start), + end: Some(end), + direction: ScanDirection::Forward, + max_items: self.items, + max_bytes: self.bytes, + continuation: None, + }) + } + /// # Errors /// Rejects unordered, foreign, oversized or failed scan pages. pub fn validate_page(&self, page: &MultiScanPage) -> Result<(), StoreError> { diff --git a/lib/crowdb-access-iceberg/src/gc/task.rs b/lib/crowdb-access-iceberg/src/gc/task.rs index 75f85e4ae..f7fe28f48 100644 --- a/lib/crowdb-access-iceberg/src/gc/task.rs +++ b/lib/crowdb-access-iceberg/src/gc/task.rs @@ -1,7 +1,7 @@ use crate::{ catalog::CatalogContext, error::ValidationError, - key::{CatalogScope, IcebergKey, OperationId}, + key::{CatalogScope, IcebergKey, OperationId, StorageKey}, table::TableHead, }; @@ -33,6 +33,8 @@ pub enum GcPhase { SweepWrites, VerifyCleanup, CleanupGc, + CleanupNative, + RootsNative, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -78,6 +80,14 @@ pub struct GcTask { } impl GcTask { + pub(super) const fn last_discovery_scope(&self) -> u8 { + if matches!(self.kind, GcTaskKind::MultipartCleanup) { + 3 + } else { + 4 + } + } + pub(crate) fn retirement_key(catalog: crate::key::CatalogId) -> IcebergKey { IcebergKey::Catalog { catalog, @@ -101,12 +111,7 @@ impl GcTask { self.context.validate()?; self.proof.validate(self)?; if self.revision == 0 - || self.discovery_scope - > if self.kind == GcTaskKind::MultipartCleanup { - 2 - } else { - 3 - } + || self.discovery_scope > self.last_discovery_scope() || (self.discovery_scope != 0 && !matches!(self.phase, GcPhase::Discover | GcPhase::Rescan)) || self.created_ms == 0 || self.not_before_ms < self.created_ms @@ -121,6 +126,7 @@ impl GcTask { | GcPhase::PreSweepSystem | GcPhase::VerifyCleanup | GcPhase::CleanupGc + | GcPhase::CleanupNative ) && self.kind != GcTaskKind::RetiredCatalog) || (matches!( self.kind, @@ -139,8 +145,17 @@ impl GcTask { GcPhase::CleanupSystem | GcPhase::RootsSystem | GcPhase::PreSweepSystem ) { super::GcSystemScan::validate_cursor(&self.scan_after)?; - } else if !IcebergKey::catalog_range(self.context.catalog).contains(&self.scan_after) { - return Err(ValidationError::Key); + } else { + match StorageKey::decode(&self.scan_after)? { + StorageKey::Legacy(IcebergKey::Catalog { catalog, .. }) + if catalog == self.context.catalog => {} + StorageKey::NativeTable(key) + if key.table.catalog == self.context.catalog + && self.head.as_ref().map_or(true, |head| { + head.storage_location().storage_path() == Some(key.table) + }) => {} + _ => return Err(ValidationError::Key), + } } } if let Some(head) = &self.head { diff --git a/lib/crowdb-access-iceberg/src/gc/worker.rs b/lib/crowdb-access-iceberg/src/gc/worker.rs index 9c95c47f6..cdf64a0b4 100644 --- a/lib/crowdb-access-iceberg/src/gc/worker.rs +++ b/lib/crowdb-access-iceberg/src/gc/worker.rs @@ -5,7 +5,7 @@ use crowdb_chunk_client::ReclaimOutcome; use crate::{ catalog::{CatalogError, CatalogLifecycle, RootState}, error::ValidationError, - file::{file_key, location_key, FileBlockStore, FileIoError}, + file::{file_key, FileBlockStore, FileIoError}, key::{CatalogScope, IcebergKey, SystemScope}, record::StorageRecord, }; @@ -21,6 +21,7 @@ mod cleanup; mod inactive; mod live; mod multipart; +mod native; mod sweep; mod system; mod terminal; @@ -107,6 +108,7 @@ impl GcWorker { GcPhase::Roots if task.kind == GcTaskKind::LiveTable => self.live_roots(task, now_ms).await, GcPhase::Mark if task.kind == GcTaskKind::LiveTable => self.live_mark(task).await, GcPhase::Roots => self.inactive_roots(task, now_ms).await, + GcPhase::RootsNative => self.native_roots(task, now_ms).await, GcPhase::Fence if task.kind == GcTaskKind::LiveTable => self.live_fence(task, now_ms).await, GcPhase::Fence => self.fence(task, now_ms).await, GcPhase::Sweep => self.sweep(task, now_ms).await, @@ -116,6 +118,7 @@ impl GcWorker { GcPhase::SweepWrites => self.sweep_writes(task, now_ms).await, GcPhase::CleanupSystem => self.cleanup_system(task, now_ms).await, GcPhase::CleanupCatalog => self.cleanup_catalog(task, now_ms).await, + GcPhase::CleanupNative => self.cleanup_native(task, now_ms).await, GcPhase::VerifyCleanup => self.verify_cleanup(task).await, GcPhase::CleanupGc => self.cleanup_gc(task).await, GcPhase::RootsSystem | GcPhase::PreSweepSystem => self.scan_system_protection(task, now_ms).await, @@ -223,6 +226,7 @@ impl GcWorker { fn candidates(&self, task: &GcTask) -> GcScan { GcScan { + native: false, catalog: task.context.catalog, scope: Some(CatalogScope::GcCandidate), prefix: task diff --git a/lib/crowdb-access-iceberg/src/gc/worker/cleanup.rs b/lib/crowdb-access-iceberg/src/gc/worker/cleanup.rs index e47f3b009..5aab0c8c0 100644 --- a/lib/crowdb-access-iceberg/src/gc/worker/cleanup.rs +++ b/lib/crowdb-access-iceberg/src/gc/worker/cleanup.rs @@ -90,6 +90,7 @@ impl GcWorker { return Err(CatalogError::Busy.into()); } let scan = GcScan { + native: false, catalog: task.context.catalog, scope: None, prefix: Vec::new(), @@ -115,6 +116,63 @@ impl GcWorker { } } let mut next = task.progress()?; + if let Some(last) = page.items.last() { + next.scan_after.clone_from(&last.key); + } else { + next.scan_after.clear(); + next.phase = GcPhase::CleanupNative; + } + self.repository.update(task, &next).await?; + Ok(next) + } + + pub(super) async fn cleanup_native(&self, task: &GcTask, now_ms: u64) -> Result { + self.verify_inactive(task).await?; + if task.kind != GcTaskKind::RetiredCatalog || now_ms < task.not_before_ms { + return Err(CatalogError::Busy.into()); + } + let scan = GcScan { + native: true, + catalog: task.context.catalog, + scope: None, + prefix: Vec::new(), + after: task.scan_after.clone(), + items: usize::from(self.limits.page_items), + bytes: self.limits.step_bytes.min(16 * 1024 * 1024) as usize, + }; + let page = self + .repository + .store + .scan_gc(scan.clone()) + .await + .map_err(CatalogError::from)?; + scan.validate_page(&page).map_err(CatalogError::from)?; + for item in &page.items { + let crate::key::StorageKey::NativeTable(key) = crate::key::StorageKey::decode(&item.key)? else { + return Err(ValidationError::Key.into()); + }; + match key.scope { + crate::key::TableStorageScope::Projection => {} + crate::key::TableStorageScope::Head => { + StorageRecord::decode_storage(&crate::key::StorageKey::NativeTable(key), &item.value)?; + } + crate::key::TableStorageScope::Operation => { + let StorageRecord::NativeInjection(receipt) = StorageRecord::decode_storage( + &crate::key::StorageKey::NativeTable(key), + &item.value, + )? + else { + return Err(ValidationError::Record.into()); + }; + if receipt.retained(now_ms) { + return Err(CatalogError::Busy.into()); + } + } + _ => return Err(CatalogError::Busy.into()), + } + self.delete_exact(&item.key, &item.value).await?; + } + let mut next = task.progress()?; if let Some(last) = page.items.last() { next.scan_after.clone_from(&last.key); } else { @@ -182,6 +240,7 @@ impl GcWorker { return Err(CatalogError::Busy.into()); } let parts = GcScan { + native: false, catalog: task.context.catalog, scope: Some(CatalogScope::MultipartPart), prefix: session.upload.as_bytes().to_vec(), diff --git a/lib/crowdb-access-iceberg/src/gc/worker/inactive.rs b/lib/crowdb-access-iceberg/src/gc/worker/inactive.rs index 7d2b98e60..265779a58 100644 --- a/lib/crowdb-access-iceberg/src/gc/worker/inactive.rs +++ b/lib/crowdb-access-iceberg/src/gc/worker/inactive.rs @@ -47,6 +47,7 @@ impl GcWorker { self.repository.verify_table_fence(task).await?; } let scan = GcScan { + native: false, catalog: task.context.catalog, scope: None, prefix: Vec::new(), @@ -97,7 +98,7 @@ impl GcWorker { next.scan_after.clone_from(&last.key); } else { next.scan_after.clear(); - next.phase = GcPhase::Rescan; + next.phase = GcPhase::RootsNative; } self.repository.update(task, &next).await?; Ok(next) diff --git a/lib/crowdb-access-iceberg/src/gc/worker/live.rs b/lib/crowdb-access-iceberg/src/gc/worker/live.rs index 03c4b16d5..44b8de8e2 100644 --- a/lib/crowdb-access-iceberg/src/gc/worker/live.rs +++ b/lib/crowdb-access-iceberg/src/gc/worker/live.rs @@ -77,6 +77,7 @@ impl GcWorker { return Ok(self.repository.start_proof(task).await?); } let scan = GcScan { + native: false, catalog: task.context.catalog, scope: None, prefix: Vec::new(), @@ -126,8 +127,7 @@ impl GcWorker { next.scan_after.clone_from(&last.key); } else { next.scan_after.clear(); - next.proof.complete = false; - next.phase = GcPhase::Mark; + next.phase = GcPhase::RootsNative; } self.repository.update(task, &next).await?; Ok(next) diff --git a/lib/crowdb-access-iceberg/src/gc/worker/multipart.rs b/lib/crowdb-access-iceberg/src/gc/worker/multipart.rs index 8dd618223..d3f49419f 100644 --- a/lib/crowdb-access-iceberg/src/gc/worker/multipart.rs +++ b/lib/crowdb-access-iceberg/src/gc/worker/multipart.rs @@ -11,6 +11,7 @@ impl GcWorker { ) -> Result { self.verify_inactive(task).await?; let scan = GcScan { + native: false, catalog: task.context.catalog, scope: Some(CatalogScope::MultipartSession), prefix: Vec::new(), @@ -40,6 +41,7 @@ impl GcWorker { .await? { let parts = GcScan { + native: false, catalog: task.context.catalog, scope: Some(CatalogScope::MultipartPart), prefix: session.upload.as_bytes().to_vec(), diff --git a/lib/crowdb-access-iceberg/src/gc/worker/native.rs b/lib/crowdb-access-iceberg/src/gc/worker/native.rs new file mode 100644 index 000000000..3a75baa3a --- /dev/null +++ b/lib/crowdb-access-iceberg/src/gc/worker/native.rs @@ -0,0 +1,82 @@ +use crate::key::{StorageKey, TableStorageScope}; + +use super::{ + CatalogError, GcPhase, GcScan, GcStalledReason, GcTask, GcTaskKind, GcWorkError, GcWorker, StorageRecord, + ValidationError, +}; + +impl GcWorker { + pub(super) async fn native_roots(&self, task: &GcTask, now_ms: u64) -> Result { + if task.kind == GcTaskKind::LiveTable { + self.verify_live_head(task).await?; + } else { + self.verify_inactive(task).await?; + } + let scan = GcScan { + native: true, + catalog: task.context.catalog, + scope: None, + prefix: task + .head + .as_ref() + .and_then(|head| head.storage_location().storage_path()) + .map_or_else(Vec::new, |table| table.to_string().into_bytes()), + after: task.scan_after.clone(), + items: usize::from(self.limits.page_items), + bytes: self.limits.step_bytes.min(16 * 1024 * 1024) as usize, + }; + let page = self + .repository + .store + .scan_gc(scan.clone()) + .await + .map_err(CatalogError::from)?; + scan.validate_page(&page).map_err(CatalogError::from)?; + let mut next = task.progress()?; + for item in &page.items { + let StorageKey::NativeTable(key) = StorageKey::decode(&item.key)? else { + return Err(ValidationError::Key.into()); + }; + if key.scope != TableStorageScope::Operation { + continue; + } + let StorageRecord::NativeInjection(receipt) = + StorageRecord::decode_storage(&StorageKey::NativeTable(key), &item.value)? + else { + return Err(ValidationError::Record.into()); + }; + if receipt.retained(now_ms) { + next.stalled = GcStalledReason::Protected; + next.scan_after.clear(); + next.retry_at_ms = receipt + .identity + .issued_ms + .saturating_add(crate::operation::RETRY_WINDOW_MS) + .saturating_add(1); + if task.kind == GcTaskKind::LiveTable && task.fenced { + self.repository.release_table_fence(task).await?; + next.fenced = false; + next.phase = GcPhase::Complete; + } else { + next.phase = GcPhase::Waiting; + } + self.repository.update(task, &next).await?; + return Ok(next); + } + self.delete_exact(&item.key, &item.value).await?; + } + if let Some(last) = page.items.last() { + next.scan_after.clone_from(&last.key); + } else { + next.scan_after.clear(); + next.phase = if task.kind == GcTaskKind::LiveTable { + next.proof.complete = false; + GcPhase::Mark + } else { + GcPhase::Rescan + }; + } + self.repository.update(task, &next).await?; + Ok(next) + } +} diff --git a/lib/crowdb-access-iceberg/src/gc/worker/sweep.rs b/lib/crowdb-access-iceberg/src/gc/worker/sweep.rs index 6ae23e2d0..0175db71b 100644 --- a/lib/crowdb-access-iceberg/src/gc/worker/sweep.rs +++ b/lib/crowdb-access-iceberg/src/gc/worker/sweep.rs @@ -1,10 +1,10 @@ use crate::{catalog::CasOutcome, operation::mutation_identity}; +use crate::{file::canonical_file_key, record::StorageRecordKey}; use crowdb_protocol::chunkdb::rpc::Location; use super::{ - file_key, location_key, CandidatePhase, CatalogError, GcCandidate, GcPhase, GcStalledReason, GcTask, - GcTaskKind, GcWorkError, GcWorker, IcebergKey, ReclaimOutcome, ReclaimStep, StorageRecord, - ValidationError, + file_key, CandidatePhase, CatalogError, GcCandidate, GcPhase, GcStalledReason, GcTask, GcTaskKind, + GcWorkError, GcWorker, IcebergKey, ReclaimOutcome, ReclaimStep, StorageRecord, ValidationError, }; impl GcWorker { @@ -249,7 +249,7 @@ impl GcWorker { { return Ok(()); } - let key = location_key(&candidate.file.location); + let key = canonical_file_key(&candidate.file.location); let value = self .repository .store @@ -311,7 +311,7 @@ impl GcWorker { file: candidate.file.file, location: candidate.file.location.clone(), }; - let location = location_key(&mapping.location); + let location = canonical_file_key(&mapping.location); if let Some(value) = self .repository .store @@ -337,6 +337,9 @@ impl GcWorker { _ => return Err(CatalogError::Conflict.into()), } } + if mapping.location.table().storage_namespace.is_some() { + return Ok(()); + } self.remove_record( &file_key(mapping.location.table().catalog, mapping.file), &StorageRecord::File(Box::new(candidate.file.clone())), @@ -344,8 +347,12 @@ impl GcWorker { .await } - async fn remove_record(&self, key: &IcebergKey, record: &StorageRecord) -> Result<(), GcWorkError> { - let key = key.encode()?; + async fn remove_record( + &self, + key: &Key, + record: &StorageRecord, + ) -> Result<(), GcWorkError> { + let key = key.storage_bytes()?; let bytes = record.encode()?; match self .repository diff --git a/lib/crowdb-access-iceberg/src/gc/worker/terminal.rs b/lib/crowdb-access-iceberg/src/gc/worker/terminal.rs index a14829573..f85534754 100644 --- a/lib/crowdb-access-iceberg/src/gc/worker/terminal.rs +++ b/lib/crowdb-access-iceberg/src/gc/worker/terminal.rs @@ -86,6 +86,7 @@ impl GcWorker { fn terminal_scan(&self, task: &GcTask) -> GcScan { GcScan { + native: false, catalog: task.context.catalog, scope: None, prefix: Vec::new(), diff --git a/lib/crowdb-access-iceberg/src/gc/worker/writes.rs b/lib/crowdb-access-iceberg/src/gc/worker/writes.rs index 019248a5d..cd411de68 100644 --- a/lib/crowdb-access-iceberg/src/gc/worker/writes.rs +++ b/lib/crowdb-access-iceberg/src/gc/worker/writes.rs @@ -18,6 +18,7 @@ impl GcWorker { return Err(CatalogError::Busy.into()); } let scan = GcScan { + native: false, catalog: task.context.catalog, scope: Some(CatalogScope::FileWriteIntent), prefix: task diff --git a/lib/crowdb-access-iceberg/src/metadata_projection/model.rs b/lib/crowdb-access-iceberg/src/metadata_projection/model.rs index 16b75841e..6535a05d7 100644 --- a/lib/crowdb-access-iceberg/src/metadata_projection/model.rs +++ b/lib/crowdb-access-iceberg/src/metadata_projection/model.rs @@ -1,11 +1,12 @@ use std::collections::BTreeMap; +use std::fmt::Write; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use sha2::{Digest, Sha256}; use crate::file::{FileRecord, TableLocation}; -use crate::key::{CatalogScope, IcebergKey}; +use crate::key::{CatalogScope, IcebergKey, TableStorageKey, TableStorageScope}; pub const PROJECTION_VERSION: u16 = 2; pub const PROJECTION_PAGE_BYTES: usize = 32 * 1024; @@ -32,6 +33,19 @@ impl ProjectionIdentity { } pub(super) fn key(self, child: u16, page: u16) -> Option> { + if let Some(table) = self.table.storage_path() { + let mut digest = String::with_capacity(64); + for byte in self.digest { + write!(digest, "{byte:02x}").expect("string writes do not fail"); + } + return TableStorageKey { + table, + scope: TableStorageScope::Projection, + suffix: format!("{}/{digest}/{}/{child}/{page}", self.generation, self.version), + } + .encode() + .ok(); + } let mut suffix = self.table.table.as_bytes().to_vec(); suffix.extend_from_slice(&self.generation.to_be_bytes()); suffix.extend_from_slice(&self.digest); diff --git a/lib/crowdb-access-iceberg/src/namespace/create.rs b/lib/crowdb-access-iceberg/src/namespace/create.rs index 8465b522f..6d1af80e2 100644 --- a/lib/crowdb-access-iceberg/src/namespace/create.rs +++ b/lib/crowdb-access-iceberg/src/namespace/create.rs @@ -29,6 +29,15 @@ pub struct NamespaceCreator { } impl NamespaceCreator { + pub(crate) fn from_parts( + store: Arc, + names: Arc, + ) -> Self { + Self { + repository: NamespaceRepository::from_parts(store, names.clone()), + names, + } + } #[must_use] pub fn new(store: Arc) -> Self { Self { diff --git a/lib/crowdb-access-iceberg/src/namespace/repository.rs b/lib/crowdb-access-iceberg/src/namespace/repository.rs index 8014953a7..b1a1ab5c7 100644 --- a/lib/crowdb-access-iceberg/src/namespace/repository.rs +++ b/lib/crowdb-access-iceberg/src/namespace/repository.rs @@ -13,6 +13,34 @@ pub struct NamespaceRepository { } impl NamespaceRepository { + pub(crate) async fn ensure_default( + &self, + context: CatalogContext, + issued_ms: u64, + ) -> Result { + let identifier = NamespaceIdentifier::new(vec!["default".into()])?; + if let Some(authority) = self.load(context, &identifier).await? { + return Ok(authority); + } + let outcome = super::NamespaceCreator::from_parts(self.store.clone(), self.names.clone()) + .create(&super::NamespaceCreateRequest { + context, + identity: crate::operation::RequestIdentity { + operation: crate::key::OperationId::random(), + issued_ms, + }, + principal: "native-default-namespace".into(), + identifier: identifier.clone(), + properties: super::NamespaceProperties::default(), + }) + .await; + if let Some(authority) = self.load(context, &identifier).await? { + Ok(authority) + } else { + outcome?; + Err(CatalogError::Busy) + } + } pub(crate) fn from_parts(store: Arc, names: Arc) -> Self { Self { store, names } } diff --git a/lib/crowdb-access-iceberg/src/record.rs b/lib/crowdb-access-iceberg/src/record.rs index 6748ee252..205f4925c 100644 --- a/lib/crowdb-access-iceberg/src/record.rs +++ b/lib/crowdb-access-iceberg/src/record.rs @@ -6,6 +6,7 @@ mod file; mod gc; mod gc_key; mod gc_node; +mod injection; mod management; mod multipart; mod multipart_admission; diff --git a/lib/crowdb-access-iceberg/src/record/authority.rs b/lib/crowdb-access-iceberg/src/record/authority.rs index ebe5989f0..337cf2fdb 100644 --- a/lib/crowdb-access-iceberg/src/record/authority.rs +++ b/lib/crowdb-access-iceberg/src/record/authority.rs @@ -1,7 +1,9 @@ use crowdb_protocol::iceberg_fb::{FBCatalogAuthority, FBCatalogAuthorityArgs, FBCatalogLifecycle}; use flatbuffers::{FlatBufferBuilder, WIPOffset}; -use crate::catalog::{Capabilities, CatalogAuthority, CatalogLifecycle, ClearBounds}; +use crate::catalog::{ + Capabilities, CatalogAuthority, CatalogLifecycle, CatalogLocatorKey, CatalogStorageLayout, ClearBounds, +}; use crate::error::ValidationError; use crate::key::CatalogId; @@ -12,9 +14,14 @@ pub(super) fn encode<'buffer>( authority.validate()?; let catalog = builder.create_vector(authority.catalog.as_bytes()); let display_name = builder.create_string(&authority.display_name); + let locator_key = authority + .locator_key + .map(|key| builder.create_vector(&key.bytes())); Ok(FBCatalogAuthority::create( builder, &FBCatalogAuthorityArgs { + locator_key, + storage_layout: authority.storage_layout as u8, catalog: Some(catalog), display_name: Some(display_name), name_generation: authority.name_generation, @@ -42,6 +49,11 @@ pub(super) fn decode(value: FBCatalogAuthority<'_>) -> Result), FileWriteIntent(Box), GcNode(Box), GcTask(Box), @@ -53,7 +54,13 @@ impl StorageRecord { let envelope = FBIcebergRecord::create( &mut builder, &FBIcebergRecordArgs { - schema_version: SCHEMA_VERSION, + schema_version: if matches!(self, Self::Authority(authority) + if authority.storage_layout == crate::catalog::CatalogStorageLayout::Native) + { + 2 + } else { + SCHEMA_VERSION + }, value_type, value: Some(value), }, @@ -74,6 +81,10 @@ impl StorageRecord { return Ok(value); } Ok(match self { + Self::NativeInjection(receipt) => ( + FBRecordValue::FBNativeInjection, + super::injection::encode(builder, receipt)?.as_union_value(), + ), Self::FileWriteIntent(intent) => ( FBRecordValue::FBFileWriteIntent, super::write_intent::encode(builder, intent)?.as_union_value(), @@ -213,6 +224,7 @@ impl StorageRecord { return self.validate_key(key); } let expected = match self { + Self::NativeInjection(receipt) => receipt.key()?, Self::File(file) => crate::file::canonical_file_key(&file.location), Self::DeletedFile(file) => crate::file::canonical_file_key(&file.file.location), Self::TableHead(head) => crate::table::storage_head_key(head.storage_location()), @@ -232,14 +244,29 @@ impl StorageRecord { return Err(ValidationError::Record); } let envelope = fb::root_as_fbiceberg_record(bytes).map_err(|_| ValidationError::Record)?; - if envelope.schema_version() != SCHEMA_VERSION { + if envelope.schema_version() != SCHEMA_VERSION + && !(envelope.schema_version() == 2 && envelope.value_type() == FBRecordValue::FBCatalogAuthority) + { return Err(ValidationError::RecordVersion(envelope.schema_version())); } - Self::decode_value(envelope) + let record = Self::decode_value(envelope)?; + let native_authority = matches!(&record, Self::Authority(authority) + if authority.storage_layout == crate::catalog::CatalogStorageLayout::Native); + if (envelope.schema_version() == 2) != native_authority { + return Err(ValidationError::RecordVersion(envelope.schema_version())); + } + Ok(record) } fn decode_value(envelope: FBIcebergRecord<'_>) -> Result { match envelope.value_type() { + FBRecordValue::FBNativeInjection => { + return Ok(Self::NativeInjection(Box::new(super::injection::decode( + envelope + .value_as_fbnative_injection() + .ok_or(ValidationError::Record)?, + )?))); + } FBRecordValue::FBFileWriteIntent => { return Ok(Self::FileWriteIntent(Box::new(super::write_intent::decode( envelope diff --git a/lib/crowdb-access-iceberg/src/record/gc.rs b/lib/crowdb-access-iceberg/src/record/gc.rs index 425ca38ad..4ec4a5871 100644 --- a/lib/crowdb-access-iceberg/src/record/gc.rs +++ b/lib/crowdb-access-iceberg/src/record/gc.rs @@ -154,6 +154,8 @@ fn decode_phase(value: u8) -> Result { 13 => GcPhase::SweepWrites, 14 => GcPhase::VerifyCleanup, 15 => GcPhase::CleanupGc, + 16 => GcPhase::CleanupNative, + 17 => GcPhase::RootsNative, _ => return Err(ValidationError::Record), }) } diff --git a/lib/crowdb-access-iceberg/src/record/injection.rs b/lib/crowdb-access-iceberg/src/record/injection.rs new file mode 100644 index 000000000..fc19d8cd0 --- /dev/null +++ b/lib/crowdb-access-iceberg/src/record/injection.rs @@ -0,0 +1,45 @@ +use crowdb_protocol::iceberg_fb::{FBNativeInjection, FBNativeInjectionArgs}; +use flatbuffers::{FlatBufferBuilder, WIPOffset}; + +use crate::{error::ValidationError, file::NativeInjection, key::OperationId, operation::RequestIdentity}; + +pub(super) fn encode<'buffer>( + builder: &mut FlatBufferBuilder<'buffer>, + value: &NativeInjection, +) -> Result>, ValidationError> { + value.validate()?; + let operation = builder.create_vector(value.identity.operation.as_bytes()); + let principal = builder.create_vector(&value.principal); + let stem = builder.create_string(&value.stem); + let file = super::file::encode(builder, &value.file)?; + Ok(FBNativeInjection::create( + builder, + &FBNativeInjectionArgs { + operation: Some(operation), + issued_ms: value.identity.issued_ms, + activation_epoch: value.activation_epoch, + principal: Some(principal), + stem: Some(stem), + file: Some(file), + }, + )) +} + +pub(super) fn decode(value: FBNativeInjection<'_>) -> Result { + let receipt = NativeInjection { + identity: RequestIdentity { + operation: OperationId::from_bytes(value.operation().bytes())?, + issued_ms: value.issued_ms(), + }, + activation_epoch: value.activation_epoch(), + principal: value + .principal() + .bytes() + .try_into() + .map_err(|_| ValidationError::Record)?, + stem: value.stem().into(), + file: super::file::decode(value.file())?, + }; + receipt.validate()?; + Ok(receipt) +} diff --git a/lib/crowdb-access-iceberg/src/record/storage_key.rs b/lib/crowdb-access-iceberg/src/record/storage_key.rs index d098fa0d0..5b9891b42 100644 --- a/lib/crowdb-access-iceberg/src/record/storage_key.rs +++ b/lib/crowdb-access-iceberg/src/record/storage_key.rs @@ -7,18 +7,27 @@ use super::StorageRecord; /// Only versioned Iceberg storage keys can bind a decoded authority record. pub trait StorageRecordKey: sealed::Sealed { + /// # Errors + /// Rejects malformed or oversized keys in their selected layout. + fn storage_bytes(&self) -> Result, ValidationError>; /// # Errors /// Rejects a record belonging to another key, storage owner or layout. fn validate_record(&self, record: &StorageRecord) -> Result<(), ValidationError>; } impl StorageRecordKey for IcebergKey { + fn storage_bytes(&self) -> Result, ValidationError> { + self.encode() + } fn validate_record(&self, record: &StorageRecord) -> Result<(), ValidationError> { record.validate_key(self) } } impl StorageRecordKey for StorageKey { + fn storage_bytes(&self) -> Result, ValidationError> { + self.encode() + } fn validate_record(&self, record: &StorageRecord) -> Result<(), ValidationError> { record.validate_storage_key(self) } diff --git a/lib/crowdb-access-iceberg/tests/common/store.rs b/lib/crowdb-access-iceberg/tests/common/store.rs index 4a4a8c86a..bbed28412 100644 --- a/lib/crowdb-access-iceberg/tests/common/store.rs +++ b/lib/crowdb-access-iceberg/tests/common/store.rs @@ -15,6 +15,7 @@ pub struct TestStore { pub values: ArcSwap, StoredValue>>, pub fail_after: AtomicUsize, pub file_record_reply_loss: AtomicBool, + pub native_injection_reply_loss: AtomicBool, pub gc_workspace_denied: AtomicBool, #[allow(dead_code)] pub gc_delete_reply_loss: AtomicBool, @@ -65,6 +66,16 @@ impl TestStore { } fn lose_file_reply(&self, key: &[u8]) -> bool { + if matches!( + StorageKey::decode(key), + Ok(StorageKey::NativeTable(TableStorageKey { + scope: TableStorageScope::Operation, + .. + })) + ) && self.native_injection_reply_loss.swap(false, Ordering::SeqCst) + { + return true; + } matches!( StorageKey::decode(key), Ok(StorageKey::Legacy(IcebergKey::Catalog { diff --git a/lib/crowdb-access-iceberg/tests/direct_file_location_test.rs b/lib/crowdb-access-iceberg/tests/direct_file_location_test.rs index 464b58e56..8febc80ae 100644 --- a/lib/crowdb-access-iceberg/tests/direct_file_location_test.rs +++ b/lib/crowdb-access-iceberg/tests/direct_file_location_test.rs @@ -23,6 +23,39 @@ fn table() -> TableLocation { } } +#[tokio::test] +async fn identical_replacement_at_the_same_root_extent_cannot_revive_a_reclaimed_locator() { + let blocks = Arc::new(blocks::TestBlocks::default()); + let table = table(); + let signer = DirectFileLocations::new(1, BTreeMap::from([(1, [1; 32])])).unwrap(); + let bytes = b"identical parquet bytes"; + let mut files = Vec::new(); + for _ in 0..2 { + let file = FileId::random(); + let mut writer = FileTreeWriter::new(blocks.clone(), FileIdentity { table, file }, 1024).unwrap(); + writer.push(bytes).await.unwrap(); + let tree = writer.finish().await.unwrap(); + let root = tree.root.unwrap(); + assert!(root.height > 0); + files.push(( + signer + .parquet(table, "part", file, tree.length, tree.digest, root.clone()) + .unwrap(), + root, + )); + } + assert_ne!(files[0].0.location, files[1].0.location); + let mut reused = (*blocks.values.load_full()).clone(); + reused.insert(files[0].1.chunk.low, reused[&files[1].1.chunk.low].clone()); + blocks.values.store(Arc::new(reused)); + let old = signer.resolve(&files[0].0.location).unwrap().unwrap(); + let mut reader = + FileReader::new(blocks.clone(), old, Some(ByteRange { start: 0, end: 3 }), 1024).unwrap(); + assert!(reader.next().await.is_err()); + let mut reader = FileReader::new(blocks, files[1].0.clone(), None, 1024).unwrap(); + assert_eq!(reader.next().await.unwrap().unwrap(), bytes); +} + #[tokio::test] async fn injection_replay_and_logical_delete_do_not_add_direct_read_metadata_queries() { use std::sync::atomic::Ordering; diff --git a/lib/crowdb-access-iceberg/tests/gc_proof_test.rs b/lib/crowdb-access-iceberg/tests/gc_proof_test.rs index 715c2e842..9a1c21b37 100644 --- a/lib/crowdb-access-iceberg/tests/gc_proof_test.rs +++ b/lib/crowdb-access-iceberg/tests/gc_proof_test.rs @@ -473,6 +473,7 @@ async fn live_sweep_recovers_a_lost_final_fence_release_response() { if task.phase == GcPhase::SweepWrites { let page = store .scan_gc(GcScan { + native: false, catalog: task.context.catalog, scope: Some(CatalogScope::FileWriteIntent), prefix: task.head.as_ref().unwrap().table.as_bytes().to_vec(), diff --git a/lib/crowdb-access-iceberg/tests/gc_record_test.rs b/lib/crowdb-access-iceberg/tests/gc_record_test.rs index 177f9bd3f..89534d65a 100644 --- a/lib/crowdb-access-iceberg/tests/gc_record_test.rs +++ b/lib/crowdb-access-iceberg/tests/gc_record_test.rs @@ -53,6 +53,8 @@ fn task_codec_rejects_foreign_keys_and_invalid_progress() { assert!(task.validate().is_err()); task.scan_after.clear(); task.discovery_scope = 4; + task.validate().unwrap(); + task.discovery_scope = 5; assert!(task.validate().is_err()); task.discovery_scope = 1; task.validate().unwrap(); diff --git a/lib/crowdb-access-iceberg/tests/gc_worker_test.rs b/lib/crowdb-access-iceberg/tests/gc_worker_test.rs index 0da02d896..0af668b38 100644 --- a/lib/crowdb-access-iceberg/tests/gc_worker_test.rs +++ b/lib/crowdb-access-iceberg/tests/gc_worker_test.rs @@ -39,7 +39,23 @@ async fn fixture( GcLimits, FileId, ) { - let fixture = common::file::TestFile::new(common::TestStore::default()).await; + fixture_at(retire, false).await +} + +async fn fixture_at( + retire: bool, + native: bool, +) -> ( + common::file::TestFile, + Arc, + GcTask, + GcLimits, + FileId, +) { + let mut fixture = common::file::TestFile::new(common::TestStore::default()).await; + if native { + fixture.table.storage_namespace = Some(crowdb_access_iceberg::key::NamespaceId::random()); + } let blocks = Arc::new(gc_blocks::TestReclaimBlocks::default()); let owner = FileIdentity { table: fixture.table, @@ -231,6 +247,58 @@ async fn retired_file_reclamation_survives_worker_restart_at_every_step() { assert!(!published_file_exists(&fixture, file).await); } +#[tokio::test] +async fn native_retirement_discovers_and_reclaims_single_filename_records_after_restart() { + use crowdb_access_iceberg::file::canonical_file_key; + use crowdb_access_iceberg::key::{StorageKey, TableStorageKey, TableStorageScope}; + let (fixture, blocks, mut task, limits, _) = fixture_at(true, true).await; + let file = canonical_file_key(&fixture.table.file("data/object.parquet").unwrap()) + .encode() + .unwrap(); + let projection = TableStorageKey { + table: fixture.table.storage_path().unwrap(), + scope: TableStorageScope::Projection, + suffix: "1/derived/1/0".into(), + } + .encode() + .unwrap(); + fixture + .store + .compare_exchange( + &projection, + None, + b"derived", + mutation_identity(&projection, None, b"derived"), + ) + .await + .unwrap(); + for _ in 0..350 { + let repository = GcRepository::new(fixture.store.clone()); + let worker = GcWorker::new(repository.clone(), blocks.clone(), limits).unwrap(); + task = worker.step(&task, 2000_u64.max(task.retry_at_ms)).await.unwrap(); + task = repository + .task(task.context.catalog, task.identity) + .await + .unwrap() + .unwrap(); + if task.phase == GcPhase::Complete { + break; + } + } + assert_eq!(task.phase, GcPhase::Complete, "{task:?}"); + assert_eq!(task.deleted, 1); + assert_eq!(task.reclaimed_bytes, 4096); + assert!(blocks.blocks.values.load().is_empty()); + assert!(fixture.store.get(&file).await.unwrap().is_none()); + assert!(fixture.store.get(&projection).await.unwrap().is_none()); + assert!(!fixture + .store + .values + .load() + .keys() + .any(|key| matches!(StorageKey::decode(key), Ok(StorageKey::NativeTable(_))))); +} + #[tokio::test] async fn retired_streamed_file_reclaims_location_after_worker_restart() { let (fixture, blocks, mut task, limits, _) = fixture(true).await; @@ -896,6 +964,7 @@ async fn missing_file_claim_stops_sweep_before_any_physical_deletion() { let page = fixture .store .scan_gc(GcScan { + native: false, catalog: task.context.catalog, scope: Some(CatalogScope::GcClaim), prefix: Vec::new(), diff --git a/lib/crowdb-access-iceberg/tests/metadata_projection_test.rs b/lib/crowdb-access-iceberg/tests/metadata_projection_test.rs index 5f8111316..04d3a578b 100644 --- a/lib/crowdb-access-iceberg/tests/metadata_projection_test.rs +++ b/lib/crowdb-access-iceberg/tests/metadata_projection_test.rs @@ -14,6 +14,40 @@ use sha2::{Digest, Sha256}; const JSON: &[u8] = b"{ \n \"refs\" : {\"main\": { \"snapshot-id\": 123 }}, \"unknown\" : [1, 2] }\n"; +#[tokio::test] +async fn native_projection_pages_are_bound_to_storage_namespace_and_generation() { + use crowdb_access_iceberg::key::{NamespaceId, StorageKey, TableStorageScope}; + let mut fixture = TestProjection::new(JSON).await; + let mut table = fixture.record.location.table(); + table.storage_namespace = Some(NamespaceId::random()); + fixture.record.location = table.file("metadata/v7.json").unwrap(); + assert!(fixture.projection.put(&fixture.record, 7, JSON).await); + for key in fixture.store.values.load().keys() { + let StorageKey::NativeTable(key) = StorageKey::decode(key).unwrap() else { + panic!() + }; + assert_eq!(key.table, table.storage_path().unwrap()); + assert_eq!(key.scope, TableStorageScope::Projection); + assert!(key.suffix.starts_with("7/")); + } + assert!(matches!( + fixture + .projection + .select(fixture.record.clone(), 7, &["refs"]) + .await + .unwrap(), + MetadataRead::Selected(_) + )); + assert_eq!(fixture.blocks.reads.load(Ordering::SeqCst), 0); + table.storage_namespace = Some(NamespaceId::random()); + let mut foreign = fixture.record.clone(); + foreign.location = table.file("metadata/v7.json").unwrap(); + assert!(!matches!( + fixture.projection.select(foreign, 7, &["refs"]).await, + Ok(MetadataRead::Selected(_)) + )); +} + #[tokio::test] async fn selected_children_preserve_bytes_without_reading_canonical_blocks() { let fixture = TestProjection::new(JSON).await; diff --git a/lib/crowdb-access-iceberg/tests/native_injection_test.rs b/lib/crowdb-access-iceberg/tests/native_injection_test.rs new file mode 100644 index 000000000..1007a88d1 --- /dev/null +++ b/lib/crowdb-access-iceberg/tests/native_injection_test.rs @@ -0,0 +1,90 @@ +#[path = "common/file_blocks.rs"] +mod blocks; +#[path = "common/store.rs"] +mod common; +#[path = "common/file.rs"] +#[allow(dead_code)] +mod fixture; + +use crowdb_access_iceberg::{ + file::{ + DirectFileLocations, FileIdentity, FileRepository, FileTreeWriter, NativeInjectionRepository, + NativeInjectionRequest, + }, + key::{FileId, NamespaceId, OperationId, StorageKey, TableStorageScope}, + operation::{RequestIdentity, RETRY_WINDOW_MS}, +}; +use std::{ + collections::BTreeMap, + sync::{atomic::Ordering, Arc}, +}; + +#[tokio::test] +async fn durable_injection_selection_survives_both_lost_replies_and_key_rotation() { + for selection_loss in [true, false] { + let mut fixture = fixture::TestFile::new(common::TestStore::default()).await; + fixture.table.storage_namespace = Some(NamespaceId::random()); + let blocks = Arc::new(blocks::TestBlocks::default()); + let owner = FileIdentity { + table: fixture.table, + file: FileId::random(), + }; + let mut writer = FileTreeWriter::new(blocks, owner, 128).unwrap(); + writer.push(&[7; 1024]).await.unwrap(); + let tree = writer.finish().await.unwrap(); + let request = NativeInjectionRequest { + context: fixture.context, + table: fixture.table, + identity: RequestIdentity { + operation: OperationId::random(), + issued_ms: 100, + }, + principal: [3; 32], + stem: "part-0001".into(), + digest: tree.digest, + }; + let signer = DirectFileLocations::new(1, BTreeMap::from([(1, [1; 32])])).unwrap(); + let repository = NativeInjectionRepository::new(fixture.store.clone()); + assert!(repository.replay(&request, 101).await.unwrap().is_none()); + if selection_loss { + fixture + .store + .native_injection_reply_loss + .store(true, Ordering::SeqCst); + repository + .finish(&request, owner, tree, &signer, 101) + .await + .unwrap(); + } else { + fixture.store.file_record_reply_loss.store(true, Ordering::SeqCst); + assert!(repository + .finish(&request, owner, tree, &signer, 101) + .await + .is_err()); + } + let restarted = NativeInjectionRepository::new(fixture.store.clone()); + let file = restarted.replay(&request, 102).await.unwrap().unwrap(); + assert_eq!(file.file, owner.file); + let rotated = DirectFileLocations::new(2, BTreeMap::from([(1, [1; 32]), (2, [2; 32])])).unwrap(); + assert_eq!(rotated.resolve(&file.location).unwrap(), Some(file.clone())); + assert_eq!(restarted.replay(&request, 103).await.unwrap(), Some(file.clone())); + let records = fixture.store.values.load_full(); + assert_eq!(records.keys().filter(|key| matches!(StorageKey::decode(key), Ok(StorageKey::NativeTable(key)) if key.scope == TableStorageScope::File)).count(), 1); + assert_eq!(records.keys().filter(|key| matches!(StorageKey::decode(key), Ok(StorageKey::NativeTable(key)) if key.scope == TableStorageScope::Operation)).count(), 1); + let mut changed = request.clone(); + changed.digest[0] ^= 1; + assert!(restarted.replay(&changed, 104).await.is_err()); + changed = request.clone(); + changed.principal[0] ^= 1; + assert!(restarted.replay(&changed, 104).await.is_err()); + changed = request.clone(); + changed.table.storage_namespace = Some(NamespaceId::random()); + assert!(restarted.replay(&changed, 104).await.unwrap().is_none()); + assert!(restarted.replay(&request, 100 + RETRY_WINDOW_MS).await.is_err()); + FileRepository::new(fixture.store.clone()) + .mark_deleted(fixture.context, &file, 105) + .await + .unwrap(); + assert!(restarted.replay(&request, 106).await.is_err()); + } +} diff --git a/lib/crowdb-access-iceberg/tests/record_test.rs b/lib/crowdb-access-iceberg/tests/record_test.rs index afc19f107..8760fd981 100644 --- a/lib/crowdb-access-iceberg/tests/record_test.rs +++ b/lib/crowdb-access-iceberg/tests/record_test.rs @@ -17,6 +17,46 @@ fn root_key() -> IcebergKey { } } +#[test] +fn native_catalog_authority_uses_a_version_legacy_servers_cannot_silently_relabel() { + use crowdb_access_iceberg::catalog::{Capabilities, CatalogStorageLayout}; + let catalog = CatalogId::random(); + let key = IcebergKey::Catalog { + catalog, + scope: CatalogScope::Authority, + suffix: Vec::new(), + }; + for (authority, version) in [ + (CatalogAuthority::new(catalog, "legacy".into()).unwrap(), 1), + (CatalogAuthority::new_native(catalog, "native".into()).unwrap(), 2), + ] { + let record = StorageRecord::Authority(authority.clone()); + let bytes = record.encode().unwrap(); + assert_eq!( + fb::root_as_fbiceberg_record(&bytes).unwrap().schema_version(), + version + ); + assert_eq!(StorageRecord::decode(&key, &bytes).unwrap(), record); + let activated = authority + .activated(Capabilities::from_bits(0x3fff).unwrap()) + .unwrap(); + assert_eq!(activated.storage_layout, authority.storage_layout); + assert_eq!(activated.locator_key, authority.locator_key); + assert_eq!( + activated.renamed("renamed".into()).unwrap().storage_layout, + authority.storage_layout + ); + assert_eq!( + activated.renamed("renamed".into()).unwrap().locator_key, + authority.locator_key + ); + assert_eq!( + authority.storage_layout == CatalogStorageLayout::Native, + version == 2 + ); + } +} + #[test] fn every_root_phase_round_trips_with_its_exact_catalog_and_deadline() { let previous = CatalogContext { @@ -130,6 +170,8 @@ fn authority_codec_rejects_unknown_lifecycle_capabilities_and_invalid_generation let authority = FBCatalogAuthority::create( &mut builder, &FBCatalogAuthorityArgs { + locator_key: None, + storage_layout: 1, catalog: Some(identity), display_name: Some(name), lifecycle, diff --git a/lib/crowdb-access-iceberg/tests/table_create_publication_test.rs b/lib/crowdb-access-iceberg/tests/table_create_publication_test.rs index 7cac625d3..80b1cac15 100644 --- a/lib/crowdb-access-iceberg/tests/table_create_publication_test.rs +++ b/lib/crowdb-access-iceberg/tests/table_create_publication_test.rs @@ -22,6 +22,88 @@ use crowdb_access_iceberg::{ table::{read_table_metadata_document, TableMetadataLimits, TableRepository}, }; +#[tokio::test] +async fn native_catalog_creation_freezes_namespace_in_storage_identity_and_replays() { + use crowdb_access_iceberg::catalog::{CatalogAuthority, CatalogStorageLayout}; + use crowdb_access_iceberg::key::{CatalogScope, IcebergKey, StorageKey, TableStorageScope}; + let test = TestCreation::new().await; + let authority = CatalogAuthority::new_native(test.fixture.context.catalog, "native".into()).unwrap(); + test.fixture + .put( + IcebergKey::Catalog { + catalog: authority.catalog, + scope: CatalogScope::Authority, + suffix: Vec::new(), + }, + StorageRecord::Authority(authority.clone()), + ) + .await; + let creator = test.creator(); + let result = creator.create(&test.request).await.unwrap(); + assert_eq!(creator.create(&test.request).await.unwrap(), result); + let selected = TableRepository::new(test.fixture.store.clone()) + .select(test.fixture.context, test.parent.namespace, "events") + .await + .unwrap() + .unwrap(); + let path = selected.head.storage_location().storage_path().unwrap(); + assert_eq!(path.storage_namespace, test.parent.namespace); + assert_eq!(path.catalog, test.fixture.context.catalog); + assert!(selected + .head + .metadata_location + .to_string() + .starts_with(&format!("s3://iceberg/{path}"))); + let keys = test.fixture.store.values.load_full(); + let native: Vec<_> = keys + .keys() + .filter_map(|key| match StorageKey::decode(key) { + Ok(StorageKey::NativeTable(key)) => Some(key), + _ => None, + }) + .collect(); + assert_eq!( + native + .iter() + .filter(|key| key.scope == TableStorageScope::Head) + .count(), + 1 + ); + assert_eq!( + native + .iter() + .filter(|key| key.scope == TableStorageScope::File) + .count(), + 1 + ); + assert!(native.iter().all(|key| key.table == path)); + assert_eq!( + authority.renamed("renamed".into()).unwrap().storage_layout, + CatalogStorageLayout::Native + ); + let mut default = test.request.clone(); + default.namespace = + crowdb_access_iceberg::namespace::NamespaceIdentifier::new(vec!["default".into()]).unwrap(); + default.identity.operation = OperationId::random(); + let created = creator.create(&default).await.unwrap(); + assert_eq!(creator.create(&default).await.unwrap(), created); + let namespace = NamespaceRepository::new(test.fixture.store.clone()) + .load(test.fixture.context, &default.namespace) + .await + .unwrap() + .unwrap(); + let selected = TableRepository::new(test.fixture.store.clone()) + .select(test.fixture.context, namespace.namespace, "events") + .await + .unwrap() + .unwrap(); + assert_eq!( + selected.head.storage_location().storage_namespace, + Some(namespace.namespace) + ); + assert_ne!(namespace.namespace, test.parent.namespace); +} + #[tokio::test] async fn response_reserve_rejects_before_durable_creation() { let test = TestCreation::new().await; diff --git a/lib/crowdb-protocol/src/fbs/iceberg.fbs b/lib/crowdb-protocol/src/fbs/iceberg.fbs index 411220abf..f5a4b2dc9 100644 --- a/lib/crowdb-protocol/src/fbs/iceberg.fbs +++ b/lib/crowdb-protocol/src/fbs/iceberg.fbs @@ -35,6 +35,8 @@ table FBCatalogAuthority { root_lease_ms:ulong; delegated_access_ms:ulong; clock_skew_ms:ulong; + storage_layout:ubyte = 1; + locator_key:[ubyte]; } table FBManagementOperation { @@ -480,7 +482,16 @@ table FBGcNode { complete:bool; } -union FBRecordValue { FBActiveCatalog, FBCatalogAuthority, FBManagementOperation, FBRetryRecord, FBNamespaceAuthority, FBNamespaceMapping, FBPayloadPage, FBRetryResult, FBNamespaceOperation, FBFileRecord, FBFileMapping, FBMultipartSession, FBMultipartPart, FBMultipartAdmission, FBTableHead, FBTableMapping, FBTableCommitOperation, FBTableCreateOperation, FBTableLifecycleOperation, FBTablePurgeTask, FBGcTask, FBGcCandidate, FBGcPage, FBGcNode, FBFileWriteIntent, FBDeletedFile } +table FBNativeInjection { + operation:[ubyte] (required); + issued_ms:ulong; + activation_epoch:ulong; + principal:[ubyte] (required); + stem:string (required); + file:FBFileRecord (required); +} + +union FBRecordValue { FBActiveCatalog, FBCatalogAuthority, FBManagementOperation, FBRetryRecord, FBNamespaceAuthority, FBNamespaceMapping, FBPayloadPage, FBRetryResult, FBNamespaceOperation, FBFileRecord, FBFileMapping, FBMultipartSession, FBMultipartPart, FBMultipartAdmission, FBTableHead, FBTableMapping, FBTableCommitOperation, FBTableCreateOperation, FBTableLifecycleOperation, FBTablePurgeTask, FBGcTask, FBGcCandidate, FBGcPage, FBGcNode, FBFileWriteIntent, FBDeletedFile, FBNativeInjection } table FBIcebergRecord { schema_version:ushort; diff --git a/tools/load-tpch.py b/tools/load-tpch.py index 90e7f251e..90d49d213 100644 --- a/tools/load-tpch.py +++ b/tools/load-tpch.py @@ -5,6 +5,8 @@ Run: pixi run -e iceberg-e2e python tools/load-tpch.py Local files and both upload reports remain under .crowdb-runtime/artifacts/. +The S3 upload is an independent benchmark copy. Its bucket and keys do not +describe or back the native Iceberg table's file locations. """ import argparse @@ -144,7 +146,7 @@ def copy_objects(args, report_path, work, properties, client): except ClientError as error: if error.response["Error"]["Code"] != "BucketAlreadyOwnedByYou": raise - copied = {"iceberg_report": str(report_path), "bucket": bucket, "status": "running", "objects": []} + copied = {"iceberg_report": str(report_path), "purpose": "independent-s3-benchmark-copy", "bucket": bucket, "status": "running", "objects": []} output = work / "s3-report.json" save_report(output, copied) for table in tables.values(): From fa23493fee8433f7e045020aae00502525ba250b Mon Sep 17 00:00:00 2001 From: Gian Date: Wed, 14 Oct 2026 23:52:40 +0800 Subject: [PATCH 04/15] Dispatch garbage collection through the persisted catalog layout --- lib/crowdb-access-iceberg/src/gc/claim.rs | 2 +- lib/crowdb-access-iceberg/src/gc/discovery.rs | 27 ++++++++++++++++--- .../src/gc/repository.rs | 19 +++++++++++++ .../tests/gc_worker_test.rs | 6 ++++- 4 files changed, 49 insertions(+), 5 deletions(-) diff --git a/lib/crowdb-access-iceberg/src/gc/claim.rs b/lib/crowdb-access-iceberg/src/gc/claim.rs index 54dbd7b6a..4fa52d2b6 100644 --- a/lib/crowdb-access-iceberg/src/gc/claim.rs +++ b/lib/crowdb-access-iceberg/src/gc/claim.rs @@ -95,7 +95,7 @@ impl GcRepository { || previous .head .as_ref() - .is_some_and(|head| head.table != candidate.file.location.table().table) + .is_some_and(|head| head.storage_location() != candidate.file.location.table()) || !matches!( (task.kind, previous.kind), ( diff --git a/lib/crowdb-access-iceberg/src/gc/discovery.rs b/lib/crowdb-access-iceberg/src/gc/discovery.rs index 82165a7ab..99ccb9de4 100644 --- a/lib/crowdb-access-iceberg/src/gc/discovery.rs +++ b/lib/crowdb-access-iceberg/src/gc/discovery.rs @@ -1,5 +1,5 @@ use crate::{ - catalog::{CatalogContext, CatalogError}, + catalog::{CatalogContext, CatalogError, CatalogStorageLayout}, error::ValidationError, file::MultipartPhase, key::{CatalogScope, IcebergKey, OperationId, StorageKey, TableStorageScope}, @@ -118,7 +118,28 @@ impl GcRepository { bytes: (crate::record::MAX_RECORD_BYTES + crate::key::MAX_KEY_BYTES) .min(limits.step_bytes as usize), }; - let page = self.store.scan_gc(scan.clone()).await?; + let layout = self.catalog_layout(task.context.catalog).await?; + if task.head.as_ref().is_some_and(|head| { + head.storage_location().storage_namespace.is_some() != (layout == CatalogStorageLayout::Native) + }) { + return Err(ValidationError::IdentityMismatch.into()); + } + let data_scope = task.discovery_scope == 0 + || (task.kind != GcTaskKind::MultipartCleanup && task.discovery_scope == 1); + let allowed = if native { + layout == CatalogStorageLayout::Native + } else { + !data_scope || layout == CatalogStorageLayout::Legacy + }; + let page = if allowed { + self.store.scan_gc(scan.clone()).await? + } else { + crowdb_chunk_kv_client::MultiScanPage { + items: Vec::new(), + continuation: None, + terminal_failure: None, + } + }; scan.validate_page(&page)?; for item in &page.items { let key = StorageKey::decode(&item.key)?; @@ -229,7 +250,7 @@ impl GcRepository { if task .head .as_ref() - .is_some_and(|head| head.table != file.location.table().table) + .is_some_and(|head| head.storage_location() != file.location.table()) { return Ok(None); } diff --git a/lib/crowdb-access-iceberg/src/gc/repository.rs b/lib/crowdb-access-iceberg/src/gc/repository.rs index 27c3fa98b..67380d2b4 100644 --- a/lib/crowdb-access-iceberg/src/gc/repository.rs +++ b/lib/crowdb-access-iceberg/src/gc/repository.rs @@ -16,6 +16,25 @@ pub struct GcRepository { } impl GcRepository { + pub(super) async fn catalog_layout( + &self, + catalog: CatalogId, + ) -> Result { + let key = IcebergKey::Catalog { + catalog, + scope: CatalogScope::Authority, + suffix: Vec::new(), + }; + let value = self + .store + .get(&key.encode()?) + .await? + .ok_or(ValidationError::Record)?; + let StorageRecord::Authority(authority) = StorageRecord::decode(&key, &value.bytes)? else { + return Err(ValidationError::Record.into()); + }; + Ok(authority.storage_layout) + } #[must_use] pub fn new(store: Arc) -> Self { Self { store } diff --git a/lib/crowdb-access-iceberg/tests/gc_worker_test.rs b/lib/crowdb-access-iceberg/tests/gc_worker_test.rs index 0af668b38..1f8e869ed 100644 --- a/lib/crowdb-access-iceberg/tests/gc_worker_test.rs +++ b/lib/crowdb-access-iceberg/tests/gc_worker_test.rs @@ -78,7 +78,11 @@ async fn fixture_at( .publish(fixture.context, &file) .await .unwrap(); - let mut retired = CatalogAuthority::new(fixture.context.catalog, "retired".into()).unwrap(); + let mut retired = if native { + CatalogAuthority::new_native(fixture.context.catalog, "retired".into()).unwrap() + } else { + CatalogAuthority::new(fixture.context.catalog, "retired".into()).unwrap() + }; if retire { retired.lifecycle = CatalogLifecycle::Retired; } From 9e8217692ba22f73d163c07206486bfe46f0a71c Mon Sep 17 00:00:00 2001 From: Gian Date: Wed, 14 Oct 2026 23:58:18 +0800 Subject: [PATCH 05/15] Bind recovery ownership and native SDK fixtures to complete storage identities --- .../iceberge/design-crowdb-iceberg.md | 31 ++++++++- doc/working/plan-iceberg-storage-path.md | 14 ++-- e2e-test/clients/iceberg-python/injection.py | 69 +++++++++++++++++++ e2e-test/clients/iceberg-python/listing.py | 5 ++ .../access_iceberg/file_http/capacity.rs | 6 +- .../access_iceberg/file_http/capacity/gc.rs | 8 +-- .../tests/access_iceberg/file_http/listing.rs | 19 +++-- .../tests/access_iceberg/file_http/setup.rs | 4 +- e2e-test/tests/access_iceberg/gc/seed.rs | 30 +++++--- e2e-test/tests/access_iceberg/native_files.rs | 2 +- lib/crowdb-access-iceberg/src/gc/discovery.rs | 31 +++++---- .../src/gc/proof/traversal.rs | 5 +- .../src/gc/worker/inactive.rs | 14 ++-- 13 files changed, 189 insertions(+), 49 deletions(-) create mode 100644 e2e-test/clients/iceberg-python/injection.py diff --git a/doc/design/access-server/iceberge/design-crowdb-iceberg.md b/doc/design/access-server/iceberge/design-crowdb-iceberg.md index 05327798f..e39bc5a66 100644 --- a/doc/design/access-server/iceberge/design-crowdb-iceberg.md +++ b/doc/design/access-server/iceberge/design-crowdb-iceberg.md @@ -82,9 +82,7 @@ has a durable ID under the same rules as named namespaces. ├── head ├── file/ ├── projection// - ├── operation//... - ├── multipart//... - └── gc//... + └── operation//injection ``` Catalog and namespace control records have their own typed scopes. Namespace @@ -103,6 +101,33 @@ selected content. Retries preserve the selected file identity and URL. Workflow records exist only where recovery or multipart/GC progress requires them, not as three mandatory records for every file. +Service recovery journals are separate from that directory. Their existing +catalog control scopes remain explicitly versioned binary keys, even in a +native catalog. They are not file lookup indexes and cannot produce an object +listing. The service must discover abandoned work before a table head exists +and after a table has disappeared, without a table-ID reverse index: + +- Create, commit and lifecycle journals and payload pages are keyed by operation + identity; creation can precede name/head publication and rename spans two live + namespaces. Their selected heads carry the frozen storage identity. +- Multipart sessions and parts are keyed by upload identity/part number. Catalog + admission accounts for all sessions, including unpublished and expired uploads. + Every session and part carries and validates the full storage identity. +- Write intents/fences track allocated but potentially unpublished native + extents, including failed creation. Catalog maintenance scans their bounded + control scopes; the value binds the full table/file owner and exact chunk root. +- GC tasks, candidates, claims, proof nodes and pages belong to the recovery + worker. A retirement task can cover several tables and adopts interrupted + purge work. Claims coordinate physical deletion across tasks/generations; + candidate values retain the canonical filename and full storage identity. + +These records are retained only for retry, fencing and reclamation. They do not +add another published file authority or participate in direct GET/HEAD. GC data +discovery chooses v1 or v2 from the persisted catalog layout; independent +service-control scans never merge the two file namespaces. Head, file and +projection records use the native table directory. The native injection receipt +already knows its selected table and is stored under that directory. + ### Standard metadata and bounded scale TableHead selects an immutable standard metadata JSON file. Its `snapshots` diff --git a/doc/working/plan-iceberg-storage-path.md b/doc/working/plan-iceberg-storage-path.md index 7911e668e..82f0eb31c 100644 --- a/doc/working/plan-iceberg-storage-path.md +++ b/doc/working/plan-iceberg-storage-path.md @@ -155,9 +155,7 @@ the serialized key must still version and validate each fixed ID segment: ├── head ├── file/ ├── projection// - ├── operation/ - ├── multipart/ - └── gc/ + └── operation//injection ``` The namespace ID in a table's directory is always its storage namespace ID. @@ -543,7 +541,7 @@ three-ID prefix and a matching grant. `lib/crowdb-access-iceberg/src/file/listing.rs`, `lib/crowdb-access-iceberg/src/file/listing/repository.rs`, `lib/crowdb-access-iceberg/src/catalog/storage.rs`. -- [~] **Audit every table-owned auxiliary key**: multipart records, write +- [x] **Audit every table-owned auxiliary key**: multipart records, write intents/fences, GC candidates/nodes/pending state, and recovery scans must either carry the same table-path prefix or explicitly document why an operation-scoped key is independent. Update range bounds and foreign-record @@ -552,6 +550,14 @@ three-ID prefix and a matching grant. `lib/crowdb-access-iceberg/src/gc/`, `lib/crowdb-access-iceberg/src/record/`, `lib/crowdb-access-iceberg/src/key/codec.rs`. + Head/file/projection/injection records use the frozen native prefix. The + permanent design explicitly lists independent catalog service journals: + unpublished write intents/fences, upload sessions/parts, catalog admission, + operation/payload journals and GC task/claim/proof workspace. They recover + work without a published table and coordinate catalog-wide reclamation; + they never resolve or list a published file. Their values bind full storage + identity. GC layout dispatch and claim adoption, inactive protection and + proof traversal now compare full identity rather than only table ID. - [x] **Version the table key envelope and routed-store dispatch**: `TableStorageKey` uses a v2 envelope followed by the slash-separated immutable table path and scope. File keys contain only the diff --git a/e2e-test/clients/iceberg-python/injection.py b/e2e-test/clients/iceberg-python/injection.py new file mode 100644 index 000000000..e78d0a61f --- /dev/null +++ b/e2e-test/clients/iceberg-python/injection.py @@ -0,0 +1,69 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Native injection followed by unmodified PyIceberg and Arrow reads.""" + +import hashlib +import secrets +import time +import uuid + +from botocore.auth import S3SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.credentials import Credentials +import pyarrow as pa +import pyarrow.parquet as pq +from pyiceberg.catalog import load_catalog +from pyiceberg.schema import Schema +from pyiceberg.types import LongType, NestedField +import requests + + +def upload(table, data): + properties = table.io.properties + random = secrets.randbits(74) + identity = uuid.UUID(int=(int(time.time() * 1000) << 80) | (7 << 76) + | ((random >> 62) << 64) | (2 << 62) | (random & ((1 << 62) - 1))) + uri = table.location() + f"/data/selected-{identity}.parquet" + url = properties["s3.endpoint"].rstrip("/") + "/" + uri.removeprefix("s3://") + url += f"?crowdb-native-upload={identity}" + credentials = Credentials(properties["s3.access-key-id"], properties["s3.secret-access-key"], + properties["s3.session-token"]) + selected = None + for _ in range(2): + request = AWSRequest(method="PUT", url=url, data=data, + headers={"x-amz-content-sha256": hashlib.sha256(data).hexdigest()}) + S3SigV4Auth(credentials, "s3", "us-east-1").add_auth(request) + response = requests.put(url, data=data, headers=dict(request.headers), timeout=60) + response.raise_for_status() + location = response.json()["location"] + assert location.startswith(table.location() + "/f/v1/"), location + assert selected is None or selected == location + selected = location + return selected + + +def verify(config): + catalog = load_catalog("native", type="rest", uri=config["endpoint"], token="w" * 32) + identifier = ("direct_moved", "selected") + if config.get("verify_restart"): + table = catalog.load_table(identifier) + else: + table = catalog.create_table(("default", "selected"), Schema(NestedField(1, "id", LongType()))) + sink = pa.BufferOutputStream() + expected = pa.table({"id": pa.array([1, 2, 3], type=pa.int64())}) + pq.write_table(expected, sink) + location = upload(table, sink.getvalue().to_pybytes()) + table.add_files([location]) + original = table.scan().plan_files()[0].file.file_path + assert original == location + catalog.create_namespace("direct_moved") + catalog.rename_table(("default", "selected"), identifier) + catalog.drop_namespace("default") + catalog.create_namespace("default") + table = catalog.load_table(identifier) + assert table.scan().plan_files()[0].file.file_path == original + assert table.scan().to_arrow().column("id").to_pylist() == [1, 2, 3] + assert "/f/v1/" in table.scan().plan_files()[0].file.file_path + # PyIceberg's Arrow FileIO opens the returned URL and performs Parquet footer + # range reads. No additional location field is supplied to either SDK. + print("Native injection: retry, manifest registration, Arrow query, move and retained URL passed") diff --git a/e2e-test/clients/iceberg-python/listing.py b/e2e-test/clients/iceberg-python/listing.py index 9318ca6e4..fcef72086 100644 --- a/e2e-test/clients/iceberg-python/listing.py +++ b/e2e-test/clients/iceberg-python/listing.py @@ -9,10 +9,14 @@ import pyarrow.fs as fs import pyiceberg from pyiceberg.io.pyarrow import PyArrowFile +from injection import verify def main(): config = json.load(sys.stdin) + if config.get("verify_restart"): + verify(config) + return endpoint = urlsplit(config["endpoint"]) filesystem = fs.S3FileSystem( access_key=config["access_key"], secret_key=config["secret_key"], @@ -33,6 +37,7 @@ def main(): names = sorted(file.path for file in selected if file.type == fs.FileType.File) assert names == sorted(prefix + suffix for suffix in ["a.json", "nested/b.json", "雪&<>+%.json"]), names print(f"PyIceberg {pyiceberg.__version__} / PyArrow {pyarrow.__version__}: exact creation, HEAD/GET and prefix discovery passed") + verify(config) if __name__ == "__main__": diff --git a/e2e-test/tests/access_iceberg/file_http/capacity.rs b/e2e-test/tests/access_iceberg/file_http/capacity.rs index f284aa9a9..7c269bf58 100644 --- a/e2e-test/tests/access_iceberg/file_http/capacity.rs +++ b/e2e-test/tests/access_iceberg/file_http/capacity.rs @@ -9,14 +9,14 @@ mod workspace; use crowdb_access_iceberg::{ catalog::{CatalogContext, CatalogRepository, CatalogStore, ClearBounds, ManagementPrivilege}, file::{ - location_key, ContentFormat, FileContent, FileIdentity, FileKind, FileLocation, FileReader, + canonical_file_key, ContentFormat, FileContent, FileIdentity, FileKind, FileLocation, FileReader, FileRecord, FileRepository, FileTreeWriter, NativeFileBlocks, TableLocation, }, gc::{GcLimits, GcPhase, GcRepository, GcStalledReason, GcTask, GcWorker}, key::{FileId, OperationId, TableId}, operation::{mutation_identity, ManagementAction, ManagementRequest, RequestIdentity}, record::StorageRecord, - table::{head_key, TableHead, TableLifecycle, TablePurgeTask}, + table::{storage_head_key, TableHead, TableLifecycle, TablePurgeTask}, }; use crowdb_chunk_client::{ChunkIoClient, ChunkIoClientConfig, SmallWritePolicy}; use crowdb_diskdb_client::{DiskdbClient, DiskdbClientError, DiskdbRpcTransport}; @@ -40,7 +40,7 @@ async fn full_simulated_disk_preserves_file_authority_then_recovers_after_compac let context = seed_catalog(&stack).await; let owner = FileIdentity { table: TableLocation { - storage_namespace: None, + storage_namespace: Some(crowdb_access_iceberg::key::NamespaceId::random()), catalog: context.catalog, table: TableId::random(), }, diff --git a/e2e-test/tests/access_iceberg/file_http/capacity/gc.rs b/e2e-test/tests/access_iceberg/file_http/capacity/gc.rs index 0cb60069e..2642dffeb 100644 --- a/e2e-test/tests/access_iceberg/file_http/capacity/gc.rs +++ b/e2e-test/tests/access_iceberg/file_http/capacity/gc.rs @@ -14,7 +14,7 @@ pub(super) async fn seed_gc_workspace_task( ) { let store = stack.store().await; let table = TableLocation { - storage_namespace: None, + storage_namespace: Some(crowdb_access_iceberg::key::NamespaceId::random()), catalog: context.catalog, table: TableId::random(), }; @@ -50,7 +50,7 @@ pub(super) async fn seed_gc_workspace_task( operation_fence: 2, pending_operation: Some(OperationId::random()), }; - let key = head_key(context.catalog, table.table).encode().unwrap(); + let key = storage_head_key(table).encode().unwrap(); let bytes = StorageRecord::TableHead(Box::new(head.clone())).encode().unwrap(); store .compare_exchange(&key, None, &bytes, mutation_identity(&key, None, &bytes)) @@ -128,7 +128,7 @@ pub(super) async fn assert_gc_workspace_stall( assert!(stack .store() .await - .get(&location_key(file).encode().unwrap()) + .get(&canonical_file_key(file).encode().unwrap()) .await .unwrap() .is_some()); @@ -159,7 +159,7 @@ pub(super) async fn assert_gc_workspace_recovered( assert!(stack .store() .await - .get(&location_key(file).encode().unwrap()) + .get(&canonical_file_key(file).encode().unwrap()) .await .unwrap() .is_none()); diff --git a/e2e-test/tests/access_iceberg/file_http/listing.rs b/e2e-test/tests/access_iceberg/file_http/listing.rs index 1f536437b..b3eb11a53 100644 --- a/e2e-test/tests/access_iceberg/file_http/listing.rs +++ b/e2e-test/tests/access_iceberg/file_http/listing.rs @@ -51,7 +51,7 @@ async fn native_list_objects_v2_serves_selected_files_and_rejects_foreign_scope( assert!(xml.contains(""), "{xml}"); assert!(xml.contains("data%2Fnested%2F"), "{xml}"); let foreign = crowdb_access_iceberg::file::TableLocation { - storage_namespace: None, + storage_namespace: table.storage_namespace, catalog: table.catalog, table: crowdb_access_iceberg::key::TableId::random(), }; @@ -86,13 +86,24 @@ async fn native_list_objects_v2_serves_selected_files_and_rejects_foreign_scope( #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore = "requires the pinned PyIceberg/PyArrow environment and native storage"] async fn official_pyarrow_native_fileio_creates_exact_files_and_lists_table_prefixes() { - use std::io::Write as _; - use std::process::{Command, Stdio}; - let (_stack, _process, client, table) = setup().await; + let (stack, process, client, table) = setup().await; let config = serde_json::json!({ "endpoint": format!("http://{}", client.address), "bucket": table.bucket(), "prefix": table.object_prefix(), "access_key": client.credentials.access_key_id(), "secret_key": client.credentials.secret_access_key(), "session_token": client.credentials.session_token() }); + run_python(config.clone()).await; + process.stop().await; + stack.restart_catalog().await; + let restarted = super::super::frontend::TestFrontend::start(13).await; + let mut config = config; + config["endpoint"] = format!("http://{}", restarted.address()).into(); + config["verify_restart"] = true.into(); + run_python(config).await; +} + +async fn run_python(config: serde_json::Value) { + use std::io::Write as _; + use std::process::{Command, Stdio}; let status = tokio::task::spawn_blocking(move || { let mut child = Command::new("timeout") .arg("60") diff --git a/e2e-test/tests/access_iceberg/file_http/setup.rs b/e2e-test/tests/access_iceberg/file_http/setup.rs index 32af523f3..a358402e3 100644 --- a/e2e-test/tests/access_iceberg/file_http/setup.rs +++ b/e2e-test/tests/access_iceberg/file_http/setup.rs @@ -89,7 +89,7 @@ pub(super) async fn setup_with_bounds_and_file_limit( let started = now_ms(); let credentials = issuer .issue(FileGrant { - storage_namespace: None, + storage_namespace: table.storage_namespace, context, table: table.table, principal: [7; 32], @@ -150,7 +150,7 @@ pub(super) async fn cleanup_client( client: client.client.clone(), credentials: issuer .issue(FileGrant { - storage_namespace: None, + storage_namespace: table.storage_namespace, context, table: table.table, principal: [9; 32], diff --git a/e2e-test/tests/access_iceberg/gc/seed.rs b/e2e-test/tests/access_iceberg/gc/seed.rs index 09a4273c6..a5a863862 100644 --- a/e2e-test/tests/access_iceberg/gc/seed.rs +++ b/e2e-test/tests/access_iceberg/gc/seed.rs @@ -9,11 +9,11 @@ use crowdb_access_iceberg::{ catalog::{ CatalogContext, CatalogRepository, CatalogStore, ClearBounds, ManagementPrivilege, RoutedCatalogStore, }, - file::{file_key, ContentFormat, FileContent, FileKind, FileRecord, TableLocation}, + file::{canonical_file_key, ContentFormat, FileContent, FileKind, FileRecord, TableLocation}, key::{FileId, NamespaceId, OperationId, TableId}, operation::{mutation_identity, ManagementAction, ManagementRequest, RequestIdentity}, record::StorageRecord, - table::{head_key, TableHead, TableLifecycle, TablePurgeTask}, + table::{storage_head_key, TableHead, TableLifecycle, TablePurgeTask}, }; use sha2::{Digest, Sha256}; use std::sync::Arc; @@ -60,7 +60,7 @@ pub(super) async fn seed_table( pub(super) async fn seed_head(store: &RoutedCatalogStore, context: CatalogContext) -> TableId { let table = TableId::random(); let location = TableLocation { - storage_namespace: None, + storage_namespace: Some(NamespaceId::from_bytes(table.as_bytes()).unwrap()), catalog: context.catalog, table, }; @@ -81,7 +81,13 @@ pub(super) async fn seed_head(store: &RoutedCatalogStore, context: CatalogContex operation_fence: 1, pending_operation: None, }; - let key = head_key(context.catalog, table).encode().unwrap(); + let key = storage_head_key(TableLocation { + catalog: context.catalog, + storage_namespace: Some(NamespaceId::from_bytes(table.as_bytes()).unwrap()), + table, + }) + .encode() + .unwrap(); let bytes = StorageRecord::TableHead(Box::new(head)).encode().unwrap(); store .compare_exchange(&key, None, &bytes, mutation_identity(&key, None, &bytes)) @@ -91,7 +97,11 @@ pub(super) async fn seed_head(store: &RoutedCatalogStore, context: CatalogContex } pub(super) async fn tombstone_head(store: &RoutedCatalogStore, context: CatalogContext, table: TableId) { - let key = head_key(context.catalog, table); + let key = storage_head_key(TableLocation { + catalog: context.catalog, + storage_namespace: Some(NamespaceId::from_bytes(table.as_bytes()).unwrap()), + table, + }); let previous = store.get(&key.encode().unwrap()).await.unwrap().unwrap(); let StorageRecord::TableHead(mut head) = StorageRecord::decode(&key, &previous.bytes).unwrap() else { panic!("table head"); @@ -123,7 +133,7 @@ pub(super) async fn seed_files( let file = FileRecord { file: FileId::random(), location: TableLocation { - storage_namespace: None, + storage_namespace: Some(NamespaceId::from_bytes(table.as_bytes()).unwrap()), catalog: context.catalog, table, } @@ -136,7 +146,7 @@ pub(super) async fn seed_files( content: FileContent::select_inline(FileKind::Metadata, payload_json.as_bytes()).unwrap(), hint: None, }; - let key = file_key(context.catalog, file.file).encode().unwrap(); + let key = canonical_file_key(&file.location).encode().unwrap(); let bytes = StorageRecord::File(Box::new(file)).encode().unwrap(); store .compare_exchange(&key, None, &bytes, mutation_identity(&key, None, &bytes)) @@ -147,7 +157,11 @@ pub(super) async fn seed_files( pub(super) async fn seed_purge_marker(store: &RoutedCatalogStore, context: CatalogContext, table: TableId) { tombstone_head(store, context, table).await; - let key = head_key(context.catalog, table); + let key = storage_head_key(TableLocation { + catalog: context.catalog, + storage_namespace: Some(NamespaceId::from_bytes(table.as_bytes()).unwrap()), + table, + }); let head = store.get(&key.encode().unwrap()).await.unwrap().unwrap(); let StorageRecord::TableHead(head) = StorageRecord::decode(&key, &head.bytes).unwrap() else { panic!("table head"); diff --git a/e2e-test/tests/access_iceberg/native_files.rs b/e2e-test/tests/access_iceberg/native_files.rs index 2c9209326..5b2051cad 100644 --- a/e2e-test/tests/access_iceberg/native_files.rs +++ b/e2e-test/tests/access_iceberg/native_files.rs @@ -107,7 +107,7 @@ async fn native_file_tree_publication_and_ranges_survive_catalog_storage_restart let context = seed_root(&stack).await; let owner = FileIdentity { table: TableLocation { - storage_namespace: None, + storage_namespace: Some(crowdb_access_iceberg::key::NamespaceId::random()), catalog: context.catalog, table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/src/gc/discovery.rs b/lib/crowdb-access-iceberg/src/gc/discovery.rs index 99ccb9de4..4298042e3 100644 --- a/lib/crowdb-access-iceberg/src/gc/discovery.rs +++ b/lib/crowdb-access-iceberg/src/gc/discovery.rs @@ -118,20 +118,7 @@ impl GcRepository { bytes: (crate::record::MAX_RECORD_BYTES + crate::key::MAX_KEY_BYTES) .min(limits.step_bytes as usize), }; - let layout = self.catalog_layout(task.context.catalog).await?; - if task.head.as_ref().is_some_and(|head| { - head.storage_location().storage_namespace.is_some() != (layout == CatalogStorageLayout::Native) - }) { - return Err(ValidationError::IdentityMismatch.into()); - } - let data_scope = task.discovery_scope == 0 - || (task.kind != GcTaskKind::MultipartCleanup && task.discovery_scope == 1); - let allowed = if native { - layout == CatalogStorageLayout::Native - } else { - !data_scope || layout == CatalogStorageLayout::Legacy - }; - let page = if allowed { + let page = if self.discovery_layout_allows(task, native).await? { self.store.scan_gc(scan.clone()).await? } else { crowdb_chunk_kv_client::MultiScanPage { @@ -182,6 +169,22 @@ impl GcRepository { Ok(next) } + async fn discovery_layout_allows(&self, task: &GcTask, native: bool) -> Result { + let layout = self.catalog_layout(task.context.catalog).await?; + if task.head.as_ref().is_some_and(|head| { + head.storage_location().storage_namespace.is_some() != (layout == CatalogStorageLayout::Native) + }) { + return Err(ValidationError::IdentityMismatch.into()); + } + let data_scope = task.discovery_scope == 0 + || (task.kind != GcTaskKind::MultipartCleanup && task.discovery_scope == 1); + Ok(if native { + layout == CatalogStorageLayout::Native + } else { + !data_scope || layout == CatalogStorageLayout::Legacy + }) + } + async fn discovery_candidate( &self, task: &GcTask, diff --git a/lib/crowdb-access-iceberg/src/gc/proof/traversal.rs b/lib/crowdb-access-iceberg/src/gc/proof/traversal.rs index dcba37be5..581546086 100644 --- a/lib/crowdb-access-iceberg/src/gc/proof/traversal.rs +++ b/lib/crowdb-access-iceberg/src/gc/proof/traversal.rs @@ -49,7 +49,10 @@ impl GcRepository { head: &TableHead, ) -> Result<(), CatalogError> { if head.catalog != task.context.catalog - || task.head.as_ref().map_or(true, |root| root.table != head.table) + || task + .head + .as_ref() + .map_or(true, |root| root.storage_location() != head.storage_location()) { return Err(ValidationError::IdentityMismatch.into()); } diff --git a/lib/crowdb-access-iceberg/src/gc/worker/inactive.rs b/lib/crowdb-access-iceberg/src/gc/worker/inactive.rs index 265779a58..01cd93257 100644 --- a/lib/crowdb-access-iceberg/src/gc/worker/inactive.rs +++ b/lib/crowdb-access-iceberg/src/gc/worker/inactive.rs @@ -122,7 +122,11 @@ impl GcWorker { } pub(super) fn protects(task: &GcTask, record: &StorageRecord, now_ms: u64, grace_ms: u64) -> bool { - let owns = |table| task.head.as_ref().map_or(true, |head| head.table == table); + let owns = |table| { + task.head + .as_ref() + .map_or(true, |head| head.storage_location() == table) + }; let retained = |issued: u64| { now_ms < issued @@ -131,7 +135,7 @@ pub(super) fn protects(task: &GcTask, record: &StorageRecord, now_ms: u64, grace }; match record { StorageRecord::TableCommitOperation(operation) => { - owns(operation.before.table) + owns(operation.before.storage_location()) && ((task.kind == GcTaskKind::RetiredCatalog && retained(operation.identity.issued_ms)) || !matches!( operation.phase, @@ -139,7 +143,7 @@ pub(super) fn protects(task: &GcTask, record: &StorageRecord, now_ms: u64, grace )) } StorageRecord::TableCreateOperation(operation) => { - owns(operation.candidate.table) + owns(operation.candidate.storage_location()) && ((task.kind == GcTaskKind::RetiredCatalog && retained(operation.identity.issued_ms)) || !matches!( operation.phase, @@ -147,7 +151,7 @@ pub(super) fn protects(task: &GcTask, record: &StorageRecord, now_ms: u64, grace )) } StorageRecord::TableLifecycleOperation(operation) => { - owns(operation.before.table) + owns(operation.before.storage_location()) && ((task.kind == GcTaskKind::RetiredCatalog && retained(operation.identity.issued_ms)) || !matches!( operation.phase, @@ -162,7 +166,7 @@ pub(super) fn protects(task: &GcTask, record: &StorageRecord, now_ms: u64, grace | crate::file::MultipartPhase::Aborted | crate::file::MultipartPhase::Conflicted ); - owns(session.owner.table.table) + owns(session.owner.table) && (!terminal || (task.kind == GcTaskKind::RetiredCatalog && now_ms < session.expires_ms.saturating_add(grace_ms))) From caadd43012f99b9598860a98f1d8bbf62d56b853 Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 15 Oct 2026 00:14:03 +0800 Subject: [PATCH 06/15] Restore native multipart digest checkpoints and verify loader URL handoff --- .../design-crowdb-iceberg-upload-flow.md | 2 +- doc/working/plan-iceberg-storage-path.md | 49 ++++++++++--------- e2e-test/clients/iceberg-python/injection.py | 23 ++++++++- e2e-test/tests/access_iceberg/gc.rs | 7 ++- .../src/record/multipart/completion.rs | 2 +- .../tests/multipart_record_test.rs | 42 ++++++++++++++++ 6 files changed, 97 insertions(+), 28 deletions(-) diff --git a/doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md b/doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md index fc8c5c7bd..4dfbf222f 100644 --- a/doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md +++ b/doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md @@ -69,7 +69,7 @@ retain that filename and resolve the single file record for reads. They do not require a pre-upload reservation API, a second SDK location field, or post-upload rewriting of manifests and path-based delete references. -The TPC loader hashes each local Parquet file through bounded reads, checks +For a catalog without native injection, the TPC loader hashes each local Parquet file through bounded reads, checks that its unique target is absent, then sends a known-size S3 request using the vended native credentials. Below 256 MiB it uses a single PUT; larger files use bounded multipart parts aligned to complete frame payloads. The upload diff --git a/doc/working/plan-iceberg-storage-path.md b/doc/working/plan-iceberg-storage-path.md index 82f0eb31c..fc15606ea 100644 --- a/doc/working/plan-iceberg-storage-path.md +++ b/doc/working/plan-iceberg-storage-path.md @@ -18,8 +18,8 @@ and preserve file locations across table rename and cross-namespace move. Status: implementation in progress. Native catalog layout selection, default namespace creation, table/file/projection paths, direct URL reads, upload selection/replay and the loader's returned-location handoff are implemented. -Remaining work is the auxiliary-key/recovery audit, complete native SDK/container -acceptance, and final design/backlog cleanup. Direct reads skip Iceberg +The auxiliary-key/recovery audit is complete. Remaining work is native +SDK/container acceptance and final design/backlog cleanup. Direct reads skip Iceberg file-location/state KV and retain the chunk layer's normal layout resolution, as confirmed in the chat. @@ -82,7 +82,8 @@ Verified runtime work: descriptor directly; tests cover multi-extent ranges, tampering, old-key retention, publication response loss, logical deletion and physical removal with no Iceberg file KV reads. `finalize_native_parquet` requires the caller's - durably selected owner/tree; an upload recovery journal/HTTP entry is pending. + durably selected owner/tree; injection now selects a durable receipt before + publishing the filename authority. - `FileGrantIssuer` retains at most 64 validated credentials in an ArcSwap cache, rechecking context and time on hits. Expired grants and changed catalog epochs fail closed. Requests still verify SigV4 and file scope individually. @@ -91,8 +92,8 @@ Verified runtime work: - Iceberg all-target tests, focused snapshot/multipart/mapping/direct tests, HTTP request/admission tests, workspace Clippy and fmt passed. Credential issuance now derives namespace from selected table/draft metadata. Fixed - bucket HeadBucket/region probes are wired; bucket parser tests pass. Native - catalog activation and full native lifecycle/GC/SDK flows remain unverified. + bucket HeadBucket/region probes are wired; bucket parser tests pass. Native catalog activation, lifecycle and GC unit/integration tests pass; + native container SDK flows are being verified. ## Target structure @@ -494,14 +495,14 @@ three-ID prefix and a matching grant. and `key_test.rs`. Files: `lib/crowdb-access-iceberg/src/key/storage_path.rs`, `lib/crowdb-access-iceberg/src/file/native_location.rs`. -- [~] **Carry immutable storage identity through runtime records**: use +- [x] **Carry immutable storage identity through runtime records**: use `StorageTablePath` in table heads/mappings, chunk-directory owners, credentials and creation metadata; add layout dispatch without changing the v1 catalog contract. The standalone codec does not activate new catalogs. Files: `lib/crowdb-access-iceberg/src/file/location.rs`, `lib/crowdb-access-iceberg/src/file/directory.rs`, `lib/crowdb-access-iceberg/src/table/record.rs`. -- [ ] **Define canonical URI encoding and validation**: encode IDs in fixed +- [x] **Define canonical URI encoding and validation**: encode IDs in fixed lower-case forms, reserve `f/` for server-issued locators, preserve relative UTF-8 keys, and reject aliases, traversal, authority changes, and malformed prefixes. Round trips must recover all three IDs and the relative file key. @@ -517,7 +518,7 @@ three-ID prefix and a matching grant. ## Phase 2: KV key directory redesign -- [ ] **Remove redundant planned indexes**: name lookup returns the complete +- [x] **Remove redundant planned indexes**: name lookup returns the complete storage identity; carry it through callers, multipart, GC and recovery so no table-ID reverse index is introduced. Store metadata JSON, Avro manifests and parquet through the same single file record. Keep existing bounded @@ -570,7 +571,7 @@ three-ID prefix and a matching grant. `lib/crowdb-access-iceberg/src/key/storage.rs`, `lib/crowdb-access-iceberg/src/catalog/storage.rs`, `lib/crowdb-access-iceberg/tests/native_storage_path_test.rs`. -- [ ] **Dispatch catalogs and recovery by persisted layout**: bind the selected +- [x] **Dispatch catalogs and recovery by persisted layout**: bind the selected layout in catalog authority before creating new native paths. Retain v1 reads/writes/grants for v1 catalogs; reject relabeling and never merge layouts in recovery scans. Repository and lifecycle wiring must use the selected @@ -582,7 +583,7 @@ three-ID prefix and a matching grant. ## Phase 3: FileIO and S3-compatible access -- [ ] **Implement native injection finalization**: the internal ingestion entry +- [x] **Implement native injection finalization**: the internal ingestion entry accepts the full storage identity, readable stem, upload identity and bytes; finish durable native writes, bind the exact filename to a signed descriptor, CAS one file record, then return its one canonical URL. Persist/reuse the @@ -591,7 +592,7 @@ three-ID prefix and a matching grant. response and does not pretend to rename the client's path. Files: `lib/crowdb-access-iceberg/src/file/`, `app/crowdb-access-server/src/iceberg/`, `tools/load-tpch.py`. -- [ ] **Enforce the direct chunk boundary**: introduce/extend the native range +- [x] **Enforce the direct chunk boundary**: introduce/extend the native range read interface accepting the authenticated full descriptor/root; instrument Iceberg file-location, file-state and chunk layout queries separately. Retain `query_chunk` for native layout resolution, including cache misses. @@ -608,20 +609,20 @@ three-ID prefix and a matching grant. Files: `lib/crowdb-access-iceberg/src/file/credentials.rs`, `app/crowdb-access-server/src/iceberg/file_admission.rs`. -- [ ] **Bind grants and credential prefixes to the full storage path** so a +- [x] **Bind grants and credential prefixes to the full storage path** so a credential issued for one table cannot address another namespace/table path. Update refresh responses and staged-table credentials consistently. Files: `lib/crowdb-access-iceberg/src/file/credentials.rs`, `lib/crowdb-access-iceberg/src/wire/credentials.rs`, `app/crowdb-access-server/src/iceberg/table_credentials.rs`. -- [ ] **Parse and route the new URI prefix** in exact object, list, multipart, +- [x] **Parse and route the new URI prefix** in exact object, list, multipart, cleanup, and continuation-token flows. A table rename must continue to use the same URI and file records after the new name is loaded from the catalog. Files: `app/crowdb-access-server/src/iceberg/file_request.rs`, `app/crowdb-access-server/src/iceberg/file_http.rs`, `app/crowdb-access-server/src/iceberg/file_list_request.rs`, `app/crowdb-access-server/src/iceberg/file_admission.rs`. -- [ ] **Keep physical bytes native**: no filesystem or general S3 object +- [x] **Keep physical bytes native**: no filesystem or general S3 object authority is introduced. Uploads continue through native chunk writes and publication remains a KV authority CAS. A direct read receives a complete validated chunk locator without `FileRepository::resolve`; a @@ -630,12 +631,12 @@ three-ID prefix and a matching grant. `lib/crowdb-access-iceberg/src/file/blocks.rs`, `lib/crowdb-access-iceberg/src/file/repository.rs`, `lib/crowdb-chunk-client/src/chunk/chunk_reader.rs`. -- [ ] **Constrain catalog location-provider output**: reject a provider result +- [x] **Constrain catalog location-provider output**: reject a provider result outside the canonical table prefix before upload. This prevents the loader from recording a URI that the native grant cannot serve. Files: `/cjdata/cpp/crowdb-tpc-loader/src/crowdb_tpc_loader/backend.py`, `/cjdata/cpp/crowdb-tpc-loader/src/crowdb_tpc_loader/loader.py`. -- [ ] **Use one SDK-visible file URL**: use its public filename as the single +- [x] **Use one SDK-visible file URL**: use its public filename as the single native KV file key, retain the native chunk descriptor and read mode in its value, and make the single URL written to Iceberg manifests the bound location. Implement native injection finalization before manifest @@ -655,7 +656,7 @@ three-ID prefix and a matching grant. ## Phase 4: metadata, rename, and lifecycle invariants -- [ ] **Keep large metadata as native objects**: preserve the inline threshold, +- [x] **Keep large metadata as native objects**: preserve the inline threshold, route larger metadata JSON/Avro into native chunk storage, bound descriptors with directory roots and keep snapshot arrays out of TableHead. Exercise canonical bytes through FileIO and ALL/REFS, including missing/corrupt @@ -671,7 +672,7 @@ three-ID prefix and a matching grant. zero max-keys without payload reads, table-wide sorting or unrelated scans. Files: `lib/crowdb-access-iceberg/src/file/listing/`, `app/crowdb-access-server/src/iceberg/file_list_request.rs`. -- [ ] **Preserve standard metadata semantics**: cover snapshots/refs/logs and +- [x] **Preserve standard metadata semantics**: cover snapshots/refs/logs and v1 direct manifests, v2/v3 sequence inheritance and existing v3 lineage; preserve exact delete-file references and immutable manifest bytes. Commit retries may write new manifest lists; atomic head selection and retained @@ -681,7 +682,7 @@ three-ID prefix and a matching grant. `lib/crowdb-access-iceberg/src/gc/`, `app/crowdb-access-server/tests/iceberg_table_lifecycle_test.rs`. -- [ ] **Validate table metadata location against the immutable storage path**; +- [x] **Validate table metadata location against the immutable storage path**; `set-location` may normalize a trailing slash but cannot change the path identity. Metadata generations and manifest references remain below that path. @@ -689,21 +690,21 @@ three-ID prefix and a matching grant. `lib/crowdb-access-iceberg/src/commit/evaluator/initial.rs`, `lib/crowdb-access-iceberg/src/commit/evaluator/scalar.rs`, `lib/crowdb-access-iceberg/src/commit/files.rs`. -- [ ] **Preserve rename behavior**: same-namespace rename and cross-namespace +- [x] **Preserve rename behavior**: same-namespace rename and cross-namespace move update only name/mapping authority; old names disappear, the storage path remains readable through the new table identity, and a recreated name receives a new path. Files: `lib/crowdb-access-iceberg/src/table/lifecycle/`, `app/crowdb-access-server/tests/iceberg_table_lifecycle_test.rs`, `e2e-test/clients/iceberg-java/src/main/java/TestIcebergCatalogWrites.java`. -- [ ] **Define grant and URL behavior across moves**: existing immutable URLs +- [x] **Define grant and URL behavior across moves**: existing immutable URLs continue to bind to the storage namespace ID while the table is live; new credentials resolve the current name mapping and old-name resolution fails. Drop/recreate receives a new table ID, and direct and fallback locators reject the old table after purge. Files: `lib/crowdb-access-iceberg/src/file/credentials.rs`, `app/crowdb-access-server/src/iceberg/file_admission.rs`. -- [ ] **Recheck GC and recovery ownership** against the immutable storage path +- [x] **Recheck GC and recovery ownership** against the immutable storage path before reclaiming files or chunk roots. Logical drop and physical purge must not confuse a recreated table with a retained old path. Files: `lib/crowdb-access-iceberg/src/gc/`, @@ -712,14 +713,14 @@ three-ID prefix and a matching grant. ## Phase 5: loader and container injection -- [ ] **Align the TPC loader with the canonical table path**: use stable data +- [x] **Align the TPC loader with the canonical table path**: use stable data names below the provider's `data/` directory, verify the returned URI is table-scoped, and keep the run ID in recovery/report metadata rather than in the object path. Files: `/cjdata/cpp/crowdb-tpc-loader/src/crowdb_tpc_loader/backend.py`, `/cjdata/cpp/crowdb-tpc-loader/src/crowdb_tpc_loader/loader.py`, `/cjdata/cpp/crowdb-tpc-loader/tests/unit/test_commits.py`. -- [ ] **Separate or remove the optional S3 mirror** in +- [x] **Separate or remove the optional S3 mirror** in `tools/load-tpch.py`. If retained for the container demo, document its bucket/key as an independent S3 object copy and do not treat it as the Iceberg table file layout. diff --git a/e2e-test/clients/iceberg-python/injection.py b/e2e-test/clients/iceberg-python/injection.py index e78d0a61f..90957da0c 100644 --- a/e2e-test/clients/iceberg-python/injection.py +++ b/e2e-test/clients/iceberg-python/injection.py @@ -3,7 +3,10 @@ """Native injection followed by unmodified PyIceberg and Arrow reads.""" import hashlib +import os +from pathlib import Path import secrets +import tempfile import time import uuid @@ -19,6 +22,17 @@ def upload(table, data): + if os.environ.get("CROWDB_E2E_NATIVE_LOADER") == "1": + from crowdb_tpc_loader.native_upload import new_upload_id, upload as native_upload + + uri = table.location() + f"/data/selected-{new_upload_id()}.parquet" + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "selected.parquet" + path.write_bytes(data) + first = native_upload(table.io.properties, uri, path, len(data), 4096, lambda _: None) + second = native_upload(table.io.properties, uri, path, len(data), 4096, lambda _: None) + assert first[0] == second[0] + return first[0] properties = table.io.properties random = secrets.randbits(74) identity = uuid.UUID(int=(int(time.time() * 1000) << 80) | (7 << 76) @@ -62,7 +76,14 @@ def verify(config): catalog.create_namespace("default") table = catalog.load_table(identifier) assert table.scan().plan_files()[0].file.file_path == original - assert table.scan().to_arrow().column("id").to_pylist() == [1, 2, 3] + selected = table.scan().to_arrow() + assert selected.column("id").to_pylist() == [1, 2, 3] + if os.environ.get("CROWDB_E2E_NATIVE_LOADER") == "1": + import duckdb + + with duckdb.connect() as connection: + connection.register("selected", selected) + assert connection.execute("SELECT count(*), sum(id) FROM selected").fetchone() == (3, 6) assert "/f/v1/" in table.scan().plan_files()[0].file.file_path # PyIceberg's Arrow FileIO opens the returned URL and performs Parquet footer # range reads. No additional location field is supplied to either SDK. diff --git a/e2e-test/tests/access_iceberg/gc.rs b/e2e-test/tests/access_iceberg/gc.rs index 56bd3ef83..8ccf0e4bd 100644 --- a/e2e-test/tests/access_iceberg/gc.rs +++ b/e2e-test/tests/access_iceberg/gc.rs @@ -41,7 +41,12 @@ async fn authenticated_gc_controls_survive_separate_processes() { let stack = TestNativeStorage::new(); let (store, context, table) = seed_table(&stack).await; let identity = OperationId::random().to_string(); - let table_id = table.to_string(); + let table_id = crowdb_access_iceberg::key::StorageTablePath { + catalog: context.catalog, + storage_namespace: crowdb_access_iceberg::key::NamespaceId::from_bytes(table.as_bytes()).unwrap(), + table, + } + .to_string(); let catalog_id = context.catalog.to_string(); let denied = command(&stack, 'w', &["start-table", &identity, &table_id]); assert!(!denied.status.success()); diff --git a/lib/crowdb-access-iceberg/src/record/multipart/completion.rs b/lib/crowdb-access-iceberg/src/record/multipart/completion.rs index d4a63b3ee..c70d608af 100644 --- a/lib/crowdb-access-iceberg/src/record/multipart/completion.rs +++ b/lib/crowdb-access-iceberg/src/record/multipart/completion.rs @@ -65,7 +65,7 @@ pub(super) fn decode(value: FBMultipartCompletion<'_>) -> Result Date: Thu, 15 Oct 2026 00:32:09 +0800 Subject: [PATCH 07/15] Verify native file access and durable loader reports with official SDKs --- doc/working/plan-iceberg-storage-path.md | 37 ++++++++++++- .../main/java/TestIcebergFileOperations.java | 4 +- e2e-test/clients/iceberg-python/listing.py | 7 +++ .../clients/iceberg-python/native_loader.py | 55 +++++++++++++++++++ .../tests/access_iceberg/file_http/listing.rs | 3 +- 5 files changed, 101 insertions(+), 5 deletions(-) create mode 100644 e2e-test/clients/iceberg-python/native_loader.py diff --git a/doc/working/plan-iceberg-storage-path.md b/doc/working/plan-iceberg-storage-path.md index fc15606ea..6babdc631 100644 --- a/doc/working/plan-iceberg-storage-path.md +++ b/doc/working/plan-iceberg-storage-path.md @@ -19,7 +19,8 @@ Status: implementation in progress. Native catalog layout selection, default namespace creation, table/file/projection paths, direct URL reads, upload selection/replay and the loader's returned-location handoff are implemented. The auxiliary-key/recovery audit is complete. Remaining work is native -SDK/container acceptance and final design/backlog cleanup. Direct reads skip Iceberg +SDK/container acceptance and final design/backlog cleanup. Direct reads skip +Iceberg file-location/state KV and retain the chunk layer's normal layout resolution, as confirmed in the chat. @@ -52,7 +53,7 @@ Latest verified behavior: Moving a table, dropping/recreating its default namespace and refreshing through the old endpoint retains its old prefix without any table-ID reverse index. - The TPC loader journals its UUIDv7 upload URI before transfer and the selected - returned URL before manifest registration. Python verification: 222 passed, + returned URL before manifest registration. Python verification: 226 passed, 7 explicitly skipped integrations; Ruff check/format passed. Existing unrelated loader edits are preserved. A pre-existing region-only assertion was corrected to validate the actual table name for the nation case as well. @@ -92,7 +93,8 @@ Verified runtime work: - Iceberg all-target tests, focused snapshot/multipart/mapping/direct tests, HTTP request/admission tests, workspace Clippy and fmt passed. Credential issuance now derives namespace from selected table/draft metadata. Fixed - bucket HeadBucket/region probes are wired; bucket parser tests pass. Native catalog activation, lifecycle and GC unit/integration tests pass; + bucket HeadBucket/region probes are wired; bucket parser tests pass. Native + catalog activation, lifecycle and GC unit/integration tests pass; native container SDK flows are being verified. ## Target structure @@ -727,6 +729,35 @@ three-ID prefix and a matching grant. Files: `tools/load-tpch.py`, `container/single-node-container/README.md`. +## Native container verification + +- Fresh image source verification and the two scoped listing cases pass with + PyIceberg 0.11.1 and Arrow 25.0.1. The production loader upload helper selects + one URL on retry; standard manifest registration, Arrow read and SQL aggregate + pass before/after a cross-namespace move and catalog storage restart. +- First complete core partition: 23 passed, two failures. The GC CLI fixture + still supplied a table ID; it now supplies the complete frozen path. Tree + multipart found a real decoder gap: native digest checkpoints are 205 bytes, + but completion decoding only accepted legacy 189-byte checkpoints. Both bounded + shapes now decode, with full owner/checksum validation unchanged; the new + partial native checkpoint round-trip and existing recovery/completion tests pass. +- Core partition rerun: all 25 cases pass, including native tree multipart + reload, real chunk-KV restart, GC controls and large streamed/multipart IO. +- Apache Iceberg 1.11.0 Java catalog snapshot/staged creation and opaque + data/delete reads pass. S3FileIO also passes after its foreign-table fixture + was corrected to mutate a table ID within the valid three-ID prefix, retaining + the 403 AccessDenied assertion. +- Official Rust lifecycle and lost-response/native restart cases pass. The + retirement test is waiting through its full delegated-access grace period. +- PyIceberg 0.10.0 matches the loader adapter's declared range. Its complete + production dataflow creates a fixture table, uploads through native injection, + persists the selected URL in a real load report, commits the standard manifest, + then reloads the report/table and executes SQL over native reads after restart. + This focused fixture verifies ingestion; it does not claim an SF1 benchmark. +- The loader feature is committed separately with its 226-test/Ruff gate and + fresh native container read/SQL verification. Pre-existing version, docs and + ordinary filename edits remain unstaged in that repository. + ## Acceptance tests Unit: diff --git a/e2e-test/clients/iceberg-java/src/main/java/TestIcebergFileOperations.java b/e2e-test/clients/iceberg-java/src/main/java/TestIcebergFileOperations.java index c0fed1ba6..7b82de346 100644 --- a/e2e-test/clients/iceberg-java/src/main/java/TestIcebergFileOperations.java +++ b/e2e-test/clients/iceberg-java/src/main/java/TestIcebergFileOperations.java @@ -31,7 +31,9 @@ public static void run(S3Client client, String prefix) { reject(403, "AccessDenied", () -> client.deleteObject(request -> request.bucket(bucket).key(key))); reject(400, "InvalidRequest", () -> client.putObject(request -> request.bucket(bucket) .key(root + "../escape.json"), RequestBody.fromBytes(bytes))); - String foreign = "t/" + (root.charAt(2) == '0' ? '1' : '0') + root.substring(3) + "metadata/foreign.json"; + int tableOffset = root.startsWith("t/") ? 2 : root.lastIndexOf('/', root.length() - 2) + 1; + String foreign = root.substring(0, tableOffset) + (root.charAt(tableOffset) == '0' ? '1' : '0') + + root.substring(tableOffset + 1) + "metadata/foreign.json"; reject(403, "AccessDenied", () -> client.putObject(request -> request.bucket(bucket).key(foreign), RequestBody.fromBytes(bytes))); if (!etag.equals(client.headObject(request -> request.bucket(bucket).key(key)).eTag()) diff --git a/e2e-test/clients/iceberg-python/listing.py b/e2e-test/clients/iceberg-python/listing.py index fcef72086..0ca7624c0 100644 --- a/e2e-test/clients/iceberg-python/listing.py +++ b/e2e-test/clients/iceberg-python/listing.py @@ -2,6 +2,7 @@ # Licensed under the Apache License, Version 2.0. import json +import os import sys from urllib.parse import urlsplit @@ -16,6 +17,9 @@ def main(): config = json.load(sys.stdin) if config.get("verify_restart"): verify(config) + if os.environ.get("CROWDB_E2E_NATIVE_LOADER_FLOW") == "1": + from native_loader import verify as verify_loader + verify_loader(config) return endpoint = urlsplit(config["endpoint"]) filesystem = fs.S3FileSystem( @@ -38,6 +42,9 @@ def main(): assert names == sorted(prefix + suffix for suffix in ["a.json", "nested/b.json", "雪&<>+%.json"]), names print(f"PyIceberg {pyiceberg.__version__} / PyArrow {pyarrow.__version__}: exact creation, HEAD/GET and prefix discovery passed") verify(config) + if os.environ.get("CROWDB_E2E_NATIVE_LOADER_FLOW") == "1": + from native_loader import verify as verify_loader + verify_loader(config) if __name__ == "__main__": diff --git a/e2e-test/clients/iceberg-python/native_loader.py b/e2e-test/clients/iceberg-python/native_loader.py new file mode 100644 index 000000000..6c0aadad3 --- /dev/null +++ b/e2e-test/clients/iceberg-python/native_loader.py @@ -0,0 +1,55 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Exercise the production loader's upload, report and commit dataflow.""" + +from decimal import Decimal +import json +from pathlib import Path +import tempfile + +import duckdb +import pyarrow as pa +import pyarrow.parquet as pq + +from crowdb_tpc_loader.backend import IcebergBackend +from crowdb_tpc_loader.loader import load_tables +from crowdb_tpc_loader.models import Options, ParquetPart, TableData +from crowdb_tpc_loader.report import RunReport +from crowdb_tpc_loader.security import Redactor + + +def verify(config): + report_path = Path(config["report_dir"]) / "native-load-report.json" + options = Options("load", "tpch", Decimal("0.01"), catalog_uri=config["endpoint"], + token="w" * 32, namespace=("native_loader",), report_file=report_path, + catalog_properties={"py-io-impl": "pyiceberg.io.pyarrow.PyArrowFileIO"}) + backend = IcebergBackend(options, Redactor([options.token])) + backend.connect() + if not config.get("verify_restart"): + backend.ensure_namespace() + report = RunReport(options, "native-container-fixture", backend.redactor) + report.paths.append(report_path) + report.data["generator"] = {"implementation": "acceptance-fixture", "version": "1"} + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + backend.set_staging(root) + schema = pa.schema([pa.field("id", pa.int64())]) + path = root / "part.parquet" + pq.write_table(pa.table({"id": [1, 2, 3]}, schema=schema), path) + part = ParquetPart(path, 3, path.stat().st_size, schema) + data = TableData("region", (part,), schema) + report.generated_table(data, root) + load_tables(backend, {data.name: data}, report, 4096, print, upload_workers=1) + assert report.table("region")["status"] == "succeeded", report.data + saved = json.loads(report_path.read_text()) + item = saved["tables"]["region"]["files"][0] + assert item["state"] == "registered", item + assert "/f/v1/" in item["remote_uri"] and "/data/" in item["upload_uri"] + assert item["remote_uri"] != item["upload_uri"] + table = backend.catalog.load_table((*options.namespace, "region")) + assert [task.file.file_path for task in table.scan().plan_files()] == [item["remote_uri"]] + selected = table.scan().to_arrow() + with duckdb.connect() as connection: + connection.register("selected", selected) + assert connection.execute("SELECT count(*), sum(id) FROM selected").fetchone() == (3, 6) + print("Production loader: selected URL report, standard commit and native SQL read passed") diff --git a/e2e-test/tests/access_iceberg/file_http/listing.rs b/e2e-test/tests/access_iceberg/file_http/listing.rs index b3eb11a53..6455331c8 100644 --- a/e2e-test/tests/access_iceberg/file_http/listing.rs +++ b/e2e-test/tests/access_iceberg/file_http/listing.rs @@ -90,7 +90,8 @@ async fn official_pyarrow_native_fileio_creates_exact_files_and_lists_table_pref let config = serde_json::json!({ "endpoint": format!("http://{}", client.address), "bucket": table.bucket(), "prefix": table.object_prefix(), "access_key": client.credentials.access_key_id(), "secret_key": client.credentials.secret_access_key(), - "session_token": client.credentials.session_token() }); + "session_token": client.credentials.session_token(), + "report_dir": std::env::var("CROWDB_E2E_ARTIFACTS").unwrap() }); run_python(config.clone()).await; process.stop().await; stack.restart_catalog().await; From bc1d9c1d1b5b9c269d5a0b783083c433bba864cf Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 15 Oct 2026 00:41:29 +0800 Subject: [PATCH 08/15] Close the completed native Iceberg storage requirement --- ...34-access-iceberg-native-storage-layout.md | 326 ------- doc/backlog/backlog.md | 8 - doc/working/plan-iceberg-storage-path.md | 853 ------------------ 3 files changed, 1187 deletions(-) delete mode 100644 doc/backlog/R234-access-iceberg-native-storage-layout.md delete mode 100644 doc/working/plan-iceberg-storage-path.md diff --git a/doc/backlog/R234-access-iceberg-native-storage-layout.md b/doc/backlog/R234-access-iceberg-native-storage-layout.md deleted file mode 100644 index 636e3b1a8..000000000 --- a/doc/backlog/R234-access-iceberg-native-storage-layout.md +++ /dev/null @@ -1,326 +0,0 @@ - - - -### R234: access server / Iceberg — Native storage path and SDK-compatible file access - -## Problem - -Iceberg table files currently use a mixed location model: table identity is -encoded in one location form while file authority and parquet data can be -resolved through unrelated keys or directories. This makes the KV layout hard -to inspect, leaves rename and cross-namespace move invariants implicit, and -adds avoidable metadata work before a native file read. The root design is -[`design-crowdb-iceberg.md`](../design/access-server/iceberge/design-crowdb-iceberg.md) -and the execution details are in -[`plan-iceberg-storage-path.md`](../working/plan-iceberg-storage-path.md). - -The public contract cannot add a second chunk-location field: Iceberg clients -consume one `data_file.file_path` URL and third-party SDKs cannot be changed. -The URL must remain S3-shaped for compatibility while the bytes remain owned -by native `ChunkType::IcebergTable` storage. A client or server must also not -be able to list the whole Iceberg bucket by omitting a prefix. - -## Solution - -The requirement establishes these invariants: - -- **ICEBERG-PATH-1 — Immutable identity path:** every table-owned path starts - with `///`. The namespace ID is - captured at table creation. Rename and cross-namespace move update the live - name mapping only; they do not move file bytes or rewrite file URLs. -- **ICEBERG-PATH-2 — Fixed compatibility bucket:** every public URI uses the - fixed virtual bucket `iceberg`; the three IDs are object-key segments. A - table ID is never used as a bucket. -- **ICEBERG-PATH-3 — One public file URL:** Iceberg `file_path` is the only - SDK-visible file location. It uses the fixed bucket and immutable table - prefix. Native injection can issue a direct locator after durable upload, - before constructing manifests. Unmodified SDK writers keep their chosen - logical URL and resolve its native file record. Native injection determines - the final URL before manifest construction; readers need no additional field. -- **ICEBERG-PATH-4 — One file record:** each published file has exactly one - `file/` record under the immutable table prefix. - The filename is also the public URL suffix and list key suffix. Native - injection can use `f/v1/..parquet`; the authenticated - locator includes a unique file identity, so repeated readable stems do not - collide. The value contains file ID, length, integrity metadata, format, - native descriptor and lifecycle state. No path/public/id secondary file - indexes are created. List, inspection, recovery and GC use this same record. -- **ICEBERG-PATH-5 — Table-scoped list:** ListObjectsV2 accepts only a complete - table prefix `///` belonging to - the authorized request. Empty, catalog-only, namespace-only, foreign-table, - and bucket-wide prefixes are rejected. `prefix`, `delimiter`, `max-keys`, - continuation, start-after, URL encoding, common prefixes, and truncation - fields retain S3 behavior. -- **ICEBERG-PATH-6 — Native lifecycle:** file publication binds one immutable - file identity to one canonical public URL. Logical deletion removes the file - from the selected index/list and schedules delayed GC. A holder of its direct - URL with a valid grant may still read retained bytes until physical reclamation. - Direct reads do not query a file tombstone or location mapping. - Retained chunk IDs and offsets remain stable until deletion and reclamation. -- **ICEBERG-PATH-7 — SDK write compatibility:** ordinary SDK location providers - choose names locally before PUT; PUT admission cannot change that URL to - contain locations allocated later. Preserve the original URL for these - files. An internal injection operation may return one final direct URL after - publication; consumers put that URL into manifests using standard fields. -- **ICEBERG-PATH-8 — Locator classes:** logical URLs use the public-key file - index; direct URLs carry a versioned authenticated native descriptor or a - chunk-resident directory root. A token resolved through another KV mapping - is not a direct locator. Both classes bind the full table identity. Locator - authenticity is separate from access authorization: the persistent URL has - no credential expiry, while each HTTP request requires a valid grant. -- **ICEBERG-PATH-9 — No hidden rewrite:** metadata and manifest references are - kept byte-for-byte at the SDK-supplied URL. Commit publication does not scan - or rewrite manifests. Direct URLs are chosen before the first manifest is - constructed. Existing data-file names, position-delete paths, deletion-vector - references and historical snapshots are never silently converted to aliases. -- **ICEBERG-PATH-10 — Versioned migration:** the new slash-component layout is - encoded as a new key-layout version. Existing v1 binary keys remain readable - by the legacy decoder and are never reinterpreted as v2. New catalogs use v2; - existing catalogs retain v1 reads, writes and URLs. In-place conversion is - outside this requirement; activation must reject attempts to relabel a v1 - catalog as v2. Never rewrite historical metadata just to change its keys. -- **ICEBERG-PATH-11 — Control-plane isolation:** `iceberg` is one virtual - bucket, not a general bucket authority. HeadBucket and region/endpoint - discovery are satisfied by the Iceberg listener and table grant; bucket - creation/deletion and unscoped ListBuckets/ListObjects are rejected. Object - operations require a complete table prefix and a grant for that immutable - storage path. - -The model is defined in the permanent Iceberg design; the working plan tracks -implementation and verification. Documentation agreement is not evidence that -the current runtime already implements the new layout or direct-read path. - -The request path is: - -```text -Iceberg file_path URL - -> fixed bucket and three-ID table scope - -> validated native file locator - -> chunk read or table-scoped KV index operation - -> IcebergTable chunk storage -``` - -1. Add one canonical identity/path codec for catalog ID, storage namespace ID, - table ID, and safe relative file keys. Use it for table metadata, file - authorities, file-by-path indexes, multipart records, GC records, grants, - and continuation tokens. Reject traversal, malformed encodings, aliases, - and identity mismatches. -2. Assign the immutable storage path during normal and staged table creation. - Persist the fixed path in the table head and initial metadata. Keep the - live `(namespace, name)` mapping separate so rename, move, drop, and - recreate cannot alias retained files. -3. Move all table-owned KV records below the canonical identity prefix. Publish - one `file/` record with a create-only CAS after - native bytes are durable. Preserve the selected record on collision; - replay the same upload identity/result, and reject conflicting uploads. - Carry the canonical filename in lifecycle tasks rather than adding an ID - index. Bound every scan to the table prefix. -4. Keep the native Iceberg chunk type and `FileBlockStore` as the byte owner. - In native injection, finalize bytes and a bounded read descriptor, durably - publish the file, then return its one direct URL. Ordinary PUT/multipart - publishes a descriptor under its original URL. Implement exact GET, HEAD, - Range GET and cleanup for both classes without silently switching names. -5. Validate the fixed bucket and complete table prefix at the HTTP/FileIO - boundary. Do not expose bucket-wide `ListBuckets` or unscoped object list - results. For a missing exact object, support the SDK fallback list of - `/` without widening the authorization scope. -6. Return list results from the native file index with public `file_path` keys, - not by scanning chunk placement. Preserve Arrow/PyArrow directory behavior: - a `FileSelector` base directory becomes a key prefix with one trailing `/`, - while an exact file first uses HEAD and may list `/` only as a directory - fallback. Arrow's S3 implementation derives the bucket from the URI host - and the key from the URI path; it does not infer catalog or namespace from - the bucket. -7. Align the loader and container injection flow with the returned canonical - URI. Any optional general-S3 mirror remains a separate demo artifact and is - not an Iceberg file authority. -8. Reconcile the permanent Iceberg design and upload-flow documents with this - contract. In particular, replace the older bucket/table-prefix examples, - state that commit does not rewrite manifest paths, and define the direct - chunk-read API as separate from Iceberg file-record lookup. Do not mark R234 - complete while those documents disagree with the implemented contract. - -Defined edge behavior: - -- Catalog/name mappings and table head are sufficient to find the complete - storage identity; no table-ID reverse index is required. One file-record - type covers metadata JSON, manifest-list, manifest, data/delete and statistics - files. Snapshots and refs remain standard JSON fields, including the v1 - direct-manifest compatibility branch. No extra snapshot authority is added. -- Large metadata is a native chunk-backed object. Head and file values stay - bounded; the full snapshot array belongs in the canonical metadata JSON. - Existing validated generation projections may accelerate REFS reads but are - disposable. ALL responses and format-version requirements stay intact. -- Public `metadata/` and `statistics/` prefixes map directly to the equivalent - bounded `file/` KV ranges. List pages never decode those object payloads, - accumulate the complete table or scan unrelated file kinds. Snapshot - enumeration inside JSON is separate from file listing. Metadata/parser/ - response budgets reject oversize input explicitly, never truncate history. -- Head publication follows durable standard files and validates the prior - generation. Retained snapshots, branches and tags protect shared files from - GC. Removing an entry from the current snapshot alone proves no deletion right. - -- The three-ID path is the payload inside Iceberg storage; omit a redundant - `iceberg/` component. Catalog/namespace/name control records use separate - scopes. Live name lookup returns both table ID and storage namespace ID. - Rename may update live identity/epoch fields, but never the head key or file - locations. Removing an old namespace cannot delete a moved table's files. -- Keys retain the existing 1,024-byte bound and relative UTF-8 rules; HTTP - percent decoding occurs only at the transport boundary. Reserved `f/` paths - are issued only by native publication. Direct directory roots handle files - with many extents without an unbounded URL. Locator signatures retain their - verification keys for the file lifetime and never substitute for HTTP grants. -- List enumerates selected immutable public objects, including files awaiting - commit, not snapshot reachability. It excludes unfinished/losing/deleted - records and never emits an extra alias for a direct file. Return object keys, - not full URLs. Retain existing bounded pages, live forward traversal and - grant-bound continuation tokens; a refreshed credential starts a new scan. -- Preserve conditional HEAD/GET, single-range reads and create-only PUT. - Identical retry requires content identity, not merely equal length. Multipart - completion selects one ordered part set and replays after response loss; - abort cannot remove a published file. Explicit cleanup grants and retained - reference checks govern single/batch deletion. Snapshot expiration alone - does not delete physical bytes. Copy and bucket mutations remain unsupported. -- Existing table-bound grants survive rename/move until normal expiry or - revocation. New grants resolve the live name. Branches, tags, time travel, - rollback and compaction preserve existing file URLs and retained native - extents. Recreated names and reused physical offsets cannot revive old URLs. - -## Dependencies - -- Depends on the existing native Iceberg table, FileIO, chunk, and KV - authority designs listed in the working plan, plus a chunk read operation - that accepts a validated complete native locator without resolving a file - record through the Iceberg KV server. -- Changes direct-file deletion semantics explicitly: metadata/list visibility - ends at logical deletion, while retained bytes remain addressable until GC. - Existing logical-URL resolution may still reject its tombstone. Direct GET, - HEAD and Range GET must not use that resolver or add per-file state queries. - Grant signature/scope/expiry validation remains independent of file lookup. - The direct path skips Iceberg file-location/lifecycle KV queries and retains - native chunk-layer layout resolution for cold reads, repair and migration. - GC and storage reads must prevent reclaimed extents from aliasing new - data, including after restart and physical space reuse. -- Uses the standard Iceberg URI requirement and Arrow S3 behavior as - compatibility references, not as a requirement to introduce an external S3 - store: [Iceberg specification](https://iceberg.apache.org/spec/), - [Arrow S3 filesystem](https://raw.githubusercontent.com/apache/arrow/apache-arrow-25.0.0/cpp/src/arrow/filesystem/s3fs.cc). - -## Acceptance - -- Given a table created in the default namespace and a named namespace, when - their table heads and file indexes are inspected, assert every table-owned - key starts with the catalog, storage namespace, and table IDs in that order. - Invariant: ICEBERG-PATH-1. Unit test. -- Given a table URL such as - `s3://iceberg/cat/ns/table/f/file.parquet`, when Arrow parses it, assert the - bucket is `iceberg` and the object key starts with `cat/ns/table/`. Assert a - table ID is never parsed as the bucket. Invariants: ICEBERG-PATH-2 and - ICEBERG-PATH-3. Integration test. -- Given a file published before a same-namespace rename or cross-namespace - move, when the file is loaded through the new table name, assert its single - `file_path` and native chunk ownership are unchanged. Invariants: - ICEBERG-PATH-1 and ICEBERG-PATH-3. E2E test. -- Given a published parquet record, when a client performs HEAD, full GET, and - footer Range GET using only `file_path`, assert the server can serve the - request without requiring a second public location field and routes bytes to - native chunk storage. Invariants: ICEBERG-PATH-3 and ICEBERG-PATH-6. - Integration test. -- Given a third-party SDK PUT to its locally chosen URL and a native injection, - when both files are published, assert the SDK file keeps its original URL - while injection returns one direct URL before manifest construction. Neither - requires a second field or subsequent manifest rewrite. Invariants: - ICEBERG-PATH-7 and ICEBERG-PATH-9. Integration test. -- Given a direct locator token, when its version, table ID, file ID, length, - or chunk-root binding is changed, or its HTTP grant expires, assert the - request is rejected before chunk access. Given a valid token, assert direct - reads perform neither file-location nor file-lifecycle KV queries. - Invariants: ICEBERG-PATH-8 and ICEBERG-PATH-6. Integration test. -- Given a file key and its native file index record, when metadata and recovery - inspect the record, assert its key suffix is the public filename and its - value contains length, integrity and native locator metadata, with no - additional path/public/id file records. Invariant: - ICEBERG-PATH-4. Unit test. -- Given an authorized table list with recursive and non-recursive selectors, - when the server receives ListObjectsV2 requests, assert the prefix is the - complete table prefix, delimiter and pagination are honored, and public - object keys are returned. Invariant: ICEBERG-PATH-5. Integration test. -- Given an empty, catalog-only, namespace-only, foreign-table, or bucket-wide - list prefix, when a caller requests ListObjectsV2, assert the request is - rejected without revealing keys from another table. Invariant: - ICEBERG-PATH-5. E2E test. -- Given an exact missing file request, when Arrow performs HEAD followed by its - directory fallback list, assert only `/` is considered and no - broader prefix is queried. Invariant: ICEBERG-PATH-5. Integration test. -- Given a logical file delete followed by delayed physical GC, when the old URL - is read with a valid grant, assert list excludes the file while its direct URL - still reads retained bytes. After GC, the URL fails and cannot read replacement - bytes at a reused extent. Retained locators never change. Invariant: - ICEBERG-PATH-6. E2E test. -- Given a dropped table and a recreated table with the same name, when both - paths are inspected, assert the recreated table receives a new table ID and - cannot read the old table's records or chunks. Invariants: ICEBERG-PATH-1 and - ICEBERG-PATH-6. E2E test. -- Given a v1 catalog, when v2 layout activation is requested, assert activation - is rejected while legacy reads and writes remain available. Given a new v2 - catalog, assert v1 keys cannot decode as v2 and scans never merge layouts. - Invariant: ICEBERG-PATH-10. Integration test. -- Given HeadBucket, ListBuckets, CreateBucket, DeleteBucket, and a list without - a complete table prefix, assert the fixed virtual bucket returns only the - documented compatibility response and never reveals another table's keys. - Invariant: ICEBERG-PATH-11. Integration test. -- Given logical names and public keys with different ordering, when paginating - a table list with delimiters, assert ordered public keys, no aliases and - correct continuation through empty pages. Invariants: ICEBERG-PATH-4 and - ICEBERG-PATH-5. Integration test. -- Given a crash before or after the single file-record CAS, when recovery retries, - assert only selected records become visible and GC preserves the selected - bytes. Invariants: ICEBERG-PATH-4 and ICEBERG-PATH-6. Integration test. -- Given two injections with the same readable name and a retry of one upload, - when publication finishes, assert distinct files have distinct locator-bearing - filenames, the retry returns its original filename, and a conflicting exact - key never overwrites the winner. Invariant: ICEBERG-PATH-4. Integration test. -- Given a retained snapshot, new HTTP credentials and rotated signing keys, - when reading its direct URL, assert the URL is unchanged and still readable; - an expired grant remains rejected. Invariant: ICEBERG-PATH-8. Integration test. -- Given a moved table and its old namespace, when that namespace is dropped - and recreated, assert live lookup still returns the frozen storage path and - the files survive. Invariant: ICEBERG-PATH-1. E2E test. -- Given files with multiple extents, when reading footer and boundary ranges, - assert correct bytes, bounded URLs and zero Iceberg file-location and - file-deletion-state KV RPCs. Chunk layout resolution remains supported. - Invariants: ICEBERG-PATH-3 and - ICEBERG-PATH-8. Integration test. -- Given concurrent multipart completion/abort, same-length conflicting PUTs - and conditional/range reads, when requests retry after response loss, assert - immutable selected bytes, one completion result and documented HTTP errors. - Invariants: ICEBERG-PATH-6 and ICEBERG-PATH-7. Integration test. -- Given branches, tags, position deletes and historical snapshots, when moving, - compacting and rolling back the table, assert original paths and delete - applicability remain intact. Invariants: ICEBERG-PATH-1 and ICEBERG-PATH-9. - E2E test. - - -- Given metadata exceeding the inline threshold with many snapshots, when - publishing and reloading it through ALL/REFS, assert chunk-backed canonical - bytes, bounded head/file values and identical required table semantics with - projections present, missing or corrupt. Invariants: ICEBERG-PATH-4 and - ICEBERG-PATH-9; permanent ICE-I2 and ICE-I13. Integration test. -- Given many metadata/manifest files mixed with data files, when listing the - table, metadata directory and a filename prefix over multiple pages, assert - correct public keys, bounded per-page work, no payload reads and no scans - outside the chosen prefix. Invariant: ICEBERG-PATH-5. Integration test. -- Given v1/v2/v3 fixtures and two concurrent commits, when clients reload the - selected head and historical snapshots, assert the standard reference graph, - version-specific fields and a single complete winning generation; losing - candidates remain invisible to table load. Invariants: ICEBERG-PATH-9 and - permanent ICE-I3/ICE-I7. E2E test. - -Required implementation gates (documentation editing does not run these): - -- `pixi run -- cargo test -p crowdb-access-iceberg --all-targets` -- `pixi run -- cargo test -p crowdb-access-server --all-targets` -- `pixi run -- cargo test -p crowdb-chunk-client --all-targets` -- `pixi run -- cargo clippy -p crowdb-access-iceberg -p crowdb-access-server -p crowdb-chunk-client --all-targets -- -D warnings` -- `pixi run -- cargo fmt --all -- --check` -- `pixi run rs-lint` diff --git a/doc/backlog/backlog.md b/doc/backlog/backlog.md index 3f6e95ad5..1a607f253 100644 --- a/doc/backlog/backlog.md +++ b/doc/backlog/backlog.md @@ -41,14 +41,6 @@ live in the linked document; implementation follows - **[R196](R196-access-upload-benchmark-regression.md)** — S3 and Iceberg HTTP upload benchmark regression. **Area:** access server / CLI / benchmarks. **Complexity:** Medium. **Dependencies:** completed upload correctness paths. -- **[R234](R234-access-iceberg-native-storage-layout.md)** — native Iceberg - ID directory, single file record, injection direct URL, large metadata objects - and bounded table/metadata prefix listing. - **Area:** access server / Iceberg / Chunk-KV. **Complexity:** High. - **Status:** Implementation in progress; codecs, native logical file publication - and bounded prefix listing verified. Catalog activation, lifecycle keys and - direct reads pending. Direct reads skip Iceberg file metadata KV and retain - native chunk layout resolution. ## Chunk, Chunk-KV, and recovery diff --git a/doc/working/plan-iceberg-storage-path.md b/doc/working/plan-iceberg-storage-path.md deleted file mode 100644 index 6babdc631..000000000 --- a/doc/working/plan-iceberg-storage-path.md +++ /dev/null @@ -1,853 +0,0 @@ - - - -# Iceberg Storage Path Plan - -Requirement: [R234](../backlog/R234-access-iceberg-native-storage-layout.md) - -Upstream references: - -- [Iceberg table specification](../design/access-server/iceberge/iceberg-table-spec-1.11.0.md) -- [CROWDB Iceberg design](../design/access-server/iceberge/design-crowdb-iceberg.md) -- [CROWDB Iceberg upload flow](../design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md) - -Goal: make catalog, namespace, and table identities explicit in the Iceberg -KV and FileIO path model, keep file bytes in the native CROWDB Iceberg storage, -and preserve file locations across table rename and cross-namespace move. - -Status: implementation in progress. Native catalog layout selection, default -namespace creation, table/file/projection paths, direct URL reads, upload -selection/replay and the loader's returned-location handoff are implemented. -The auxiliary-key/recovery audit is complete. Remaining work is native -SDK/container acceptance and final design/backlog cleanup. Direct reads skip -Iceberg -file-location/state KV and retain the chunk layer's normal layout resolution, -as confirmed in the chat. - -Latest verified behavior: - -- Native catalog authorities persist layout 2 in a record envelope that older - servers reject; omitted legacy layout fields remain version 1. Rename and - capability activation preserve layout. New initialization/clear chooses 2. - The authority also persists the reserved version-1 locator key; access-token - rotation cannot invalidate file URLs. Configured later signing versions retain - that catalog key automatically. Both credential and locator key rotation pass - the native HTTP read test. -- Native default namespace creation uses normal durable namespace publication - before table metadata writes. Native creation and replay retain one frozen - namespace, one head and one filename authority. -- Native GC discovers v2 file records through bounded v2 scans, resolves standard - graph references by canonical filename, deletes that authority and removes - native projection pages at catalog retirement. Restart-at-every-step tests pass. -- Injection PUT selects an operation receipt before file publication. Both lost - receipt and lost publication replies replay the same FileId, root and URL; - changed digest/principal/identity and deleted-file replays fail closed. -- Direct locators require an owner-bound directory root even for one leaf. - An identical replacement at a reused root extent cannot revive the old URL. - HTTP probes root ownership for HEAD/GET/conditions and reuses the preflight - directory in its pull reader, without file-location/state KV. -- Native HTTP integration forbids all file KV GETs and verifies cold restart, - locator-key rotation, range reads, ETag conditions, scoped listing, logical - delete retention, and HEAD/GET failure after physical root removal. -- Credential refresh includes the frozen namespace in the existing endpoint URL. - Moving a table, dropping/recreating its default namespace and refreshing through - the old endpoint retains its old prefix without any table-ID reverse index. -- The TPC loader journals its UUIDv7 upload URI before transfer and the selected - returned URL before manifest registration. Python verification: 226 passed, - 7 explicitly skipped integrations; Ruff check/format passed. Existing unrelated - loader edits are preserved. A pre-existing region-only assertion was corrected - to validate the actual table name for the nation case as well. - -Verified foundation gates: - -- `pixi run -- cargo test -p crowdb-access-iceberg --all-targets` passed, - including seven native storage-path cases and existing legacy/lifecycle tests. -- `pixi run -- cargo fmt --all -- --check` passed. -- `pixi run rs-lint` passed, including workspace all-target Clippy. - -Verified runtime work: - -- `TableLocation`/`FileLocation` preserve native and legacy URLs; one native - `canonical_file_key` binds a record to the full path and filename. -- `FileRepository` publishes/resolves one native record and recovers publication - response loss. Prefix listing uses scanned records without per-file GETs; - a 1,200-object mixed-prefix fixture verifies ordered bounded pages. -- Grant v2, directory v2, digest checkpoint v2, multipart/write-intent owner - codecs and name mappings preserve storage namespace. Mapping/head bindings - retain it when the live namespace changes. -- `storage_head_key` is used by table selection, commits, creation publication, - rename/drop, namespace drop probes and GC fences. A native lookup/move test - verifies the head key and metadata URL remain unchanged across a live move. -- `DirectFileLocations` supplies authenticated, bounded versioned parquet - locators with a verification key ring. HTTP reads reconstruct their file - descriptor directly; tests cover multi-extent ranges, tampering, old-key - retention, publication response loss, logical deletion and physical removal - with no Iceberg file KV reads. `finalize_native_parquet` requires the caller's - durably selected owner/tree; injection now selects a durable receipt before - publishing the filename authority. -- `FileGrantIssuer` retains at most 64 validated credentials in an ArcSwap - cache, rechecking context and time on hits. Expired grants and changed catalog - epochs fail closed. Requests still verify SigV4 and file scope individually. -- HTTP exact/list parsing supports three IDs and rejects broad list prefixes; - ordinary SDK PUT/multipart creation cannot occupy the reserved `f/` directory. -- Iceberg all-target tests, focused snapshot/multipart/mapping/direct tests, - HTTP request/admission tests, workspace Clippy and fmt passed. Credential - issuance now derives namespace from selected table/draft metadata. Fixed - bucket HeadBucket/region probes are wired; bucket parser tests pass. Native - catalog activation, lifecycle and GC unit/integration tests pass; - native container SDK flows are being verified. - -## Target structure - -Iceberg storage owns the logical directory. Native chunk placement remains -opaque and is still selected by the Iceberg chunk type, allocator, and -`FileBlockStore`; the directory below is the authority path and the stable -name used to resolve a file. The namespace component is the storage namespace -ID captured when the table is created: - -```text -/// -├── metadata/ -│ ├── .metadata.json -│ ├── .avro -│ └── .avro -├── data/ -│ ├── /data-.parquet -│ ├── /delete-.parquet -│ └── /delete-vector-.puffin -└── statistics/ - └── .puffin -``` - -`metadata/`, `data/`, and `statistics/` are conventional defaults, not a new -Iceberg semantic requirement. The server must accept any safe relative key -below the immutable table prefix so ordinary Iceberg clients can choose their -own file names and subdirectories. Every metadata, manifest, data, delete, -Puffin, and statistics reference generated by CROWDB must remain under this -prefix. Opaque client manifests are not recursively validated by normal commit; -FileIO enforces scope whenever an actual reference is accessed. Importing -external files requires explicit native ingestion, not an external authority. - -This is the logical Iceberg file namespace. The single public `file_path` may -render a native locator such as `f/.parquet` under the same -table prefix. The readable stem may be included in the public filename itself; -there is no second internal filename index or second public location field. - -Native injection returns a versioned locator file component after bytes and -their descriptor are durable, before creating any referencing manifest: -`s3://iceberg////f/v1/..parquet`. -Ordinary SDK writes retain their chosen `data/...` or other safe relative key; -their location provider runs in the client and cannot be assumed to call a -CROWDB reservation API. Both forms have exactly one public URL per file. The -fixed `iceberg` compatibility bucket is always the URI bucket; the -object key carries the three ID segments. A table ID is never promoted to a -bucket. The `s3://` wrapper exists for SDK compatibility; the object is still -read from native Iceberg authority and chunk storage. - -The KV layout uses the same identity tuple as a slash-separated logical path; -the serialized key must still version and validate each fixed ID segment: - -```text -/ -├── head -├── namespace/name/ -> namespace-id -└── / - ├── head - ├── name/ -> (storage-namespace-id, table-id) - └── / - ├── head - ├── file/ - ├── projection// - └── operation//injection -``` - -The namespace ID in a table's directory is always its storage namespace ID. -The name mapping may live in a different namespace after a move. There is no -table-ID reverse index: callers and recovery tasks carry the full identity. -`projection/` represents the existing optional, rebuildable metadata read -acceleration, not another file record or authoritative snapshot store. - -The first three path segments are parsed by fixed identity encodings and -positions. The native canonical encodings are lower-case fixed-width hexadecimal for all -three 128-bit IDs (32 bytes of ASCII each). Legacy buckets retain their base32 -catalog component; -labels such as `namespace` and `table` are rejected rather than accepted as -aliases. - -The displayed paths are inside Iceberg storage. Neither the bucket name nor -`iceberg/` is repeated in the KV payload; a binary storage-family/version header -may precede it. Catalog/namespace control indexes are not table-owned records -and therefore use their own typed scopes. In a table scope the payload is -exactly the three IDs followed by a typed suffix. Name indexes include the -storage namespace so a moved table can be found without a catalog-wide scan. - -`file/` is the only persistent file record. It holds -the complete selected descriptor and provides logical GET resolution, ordered -S3 listing, inspection and GC. Publish it with one create-only CAS after bytes -are durable. There are no separate path/public/id indexes. Lifecycle tasks -carry the canonical filename; file ID remains a value field and locator binding. -Multipart/operation/GC state is temporary workflow state, not another file index. - -The native filename is `f/v1/..parquet`; use the last dot -before the extension to delimit the URL-safe, dot-free locator. Authenticate -the exact filename as well as the native descriptor to prevent alternate-name -aliases. The locator contains a unique file ID, so identical readable stems -remain distinct. Retry the same injection operation with the original file ID -and return the selected URL; a conflicting exact key fails rather than -overwriting. Ordinary SDK filenames retain their exact chosen key and the same -create-only collision rule. Include the readable stem in the total key limit. - -Rename/move changes name mappings and the head's live name/epoch fields using -the existing lifecycle state machine. The head's key, storage identity, -metadata generation, file URLs and chunk ownership remain unchanged. Dropping -the old namespace cannot cascade into files of a moved table; namespace ID -reuse is forbidden and cleanup follows table ownership rather than names. - -## Standard metadata, large objects and prefix listing - -The supported format profile remains the repository's pinned Iceberg v1/v2/v3 -specification. The live upstream spec may describe newer versions; this layout -change does not advertise those versions. References: -[table metadata](https://iceberg.apache.org/spec/#table-metadata), -[snapshots](https://iceberg.apache.org/spec/#snapshots), and -[manifests](https://iceberg.apache.org/spec/#manifests). - -```text -table/head -> current metadata JSON file - snapshots[] -> manifest-list -> manifests -> data/delete files - refs, snapshot-log, metadata-log, schemas, partition specs -``` - -- Snapshots and refs retain their standard JSON representation. They do not - require individual authoritative KV keys. Preserve the v1 direct `manifests` - alternative when reading old snapshots; v2/v3 use `manifest-list`. -- Metadata JSON, manifest lists, manifests, data/delete and statistics files - all use the one `file/` record model. File kind - does not introduce another index. Standard metadata/Avro fields, sequence - inheritance, schemas and exact path-based delete references remain intact. -- A large metadata JSON is a native object whose bytes are written into - IcebergTable chunks. Its KV value contains a bounded native descriptor/root, - canonical length and integrity metadata, not the complete JSON or snapshots - array. Preserve the existing small-file inline threshold; crossing it selects - native chunk storage without changing FileIO behavior. Large extent vectors - likewise use a bounded chunk directory rather than growing the KV value. -- `head` stays bounded: selected metadata pointer, generation and existing - identity/commit state. Publish immutable files durably before the head CAS. - Concurrent commits still validate their base; no head can select partial - bytes. A commit retry may create a new manifest list/metadata generation; - it does not overwrite already published files to retain the old filename. -- CROWDB-generated metadata names preserve useful public prefixes: - `metadata/.metadata.json`, `metadata/.avro` and - `statistics/.puffin`. Existing clients may choose other safe paths. - Native parquet injection retains the direct `f/v1/...` filename. Metadata - objects may resolve their single file record; direct parquet reads remain - independent of metadata/list queries. Large metadata storage does not force - all objects into the direct parquet naming convention. -- Table listing maps the public prefix `/metadata/` to the bounded - KV prefix `/file/metadata/`. A more specific filename prefix maps - identically, without scanning `data/`, `f/`, statistics, operations or GC. - Full table list scans only `/file/`. Prefix upper bounds and - continuation cursors are constructed from the canonical encoded key. -- Apply the existing list page record/byte/output limits to metadata listing - as well. Build each page incrementally; never collect or sort the full file - set. Delimiter rollups and tombstone filtering advance a bounded cursor, and - return a continuation when the scan budget ends even if the page is sparse. - List reads file descriptors only, never opens metadata JSON or manifests. -- Object listing and snapshot enumeration are different operations. A prefix - list shows retained metadata files, not the snapshots inside one JSON file; - no `snapshot/` objects or private SDK pagination fields are synthesized. - Use the existing generation-bound validated projections for REFS loads; - preserve ALL response semantics. Projections remain optional, paged and - discardable, and cannot participate in commit authority or GC reachability - proof. Missing/corrupt projections use the bounded canonical parser. -- Chunk-backed storage alone does not make parsing unbounded metadata safe. - Keep explicit metadata-byte, collection, parser-memory and response budgets; - reject oversized unsupported requests clearly without truncating valid - snapshots or refs. Reuse the existing streaming/bounded read paths where - available; do not claim that the existing commit parser is fully streaming. -- Physical GC follows retained snapshot/branch/tag and metadata retention - rules, not filename order, list age or absence from the current snapshot. - Snapshot expiration is a metadata update; files shared by another retained - snapshot remain readable. Historical JSON files follow their own retention - policy rather than being immediately deleted on head replacement. - -## File URL and native read boundary - -The Iceberg `file_path` remains a stable URL. Published CROWDB chunk -locators (`ChunkId` plus offset and logical span) are stable while the file is -retained; they become invalid only after the file is deleted and its physical -storage is reclaimed. This makes a native read locator a viable optimization, -but it still cannot be assumed to be one scalar location: a streamed file can -have multiple locations and a tree-backed file can require its directory root. - -The preferred read fast path is therefore: - -```text -file_path URL -> immutable file authority or native read locator - -> ChunkRoot/Locations -> ChunkIoClient range stream - -> native chunk storage -``` - -There is only one public URL: the URL stored as Iceberg `file_path`. If -ordinary SDK GET and range GET are to bypass the file-location lookup, this URL -must be a native read locator, for example a compact signed file-root/location -token under the table prefix. The server validates the catalog/table grant, -file length, integrity metadata and locator binding before -reading. Large location vectors use a bounded chunk-resident directory root; -a token requiring a KV mapping would reintroduce the location lookup. - -An SDK chooses its path before PUT. Reserving a target when PUT arrives cannot -put the newly allocated location into the already chosen URL. Such files retain -their logical URL and use `record_resolution`. Native injection owns the upload -and manifest-building sequence: finalize bytes and descriptor, publish the -file, return one direct URL, then build manifests with that URL. An internal -operation ID supports retry without publishing another file. This boundary is -the agreed native injection boundary; ordinary SDK writers keep logical URLs. - -The direct-read locator is versioned and authenticated. It binds catalog, -storage namespace, table, file ID, length, integrity metadata, and a compact -chunk root or complete location descriptor. It has a bounded URI -size and carries no arbitrary client-controlled chunk ID. A valid direct read -calls the chunk API with the complete validated locator and skips the Iceberg -file repository; a fallback read is explicit in the file record and is never -presented as direct. Persistent locators have no credential expiry or credential -nonce: refreshed HTTP grants must continue reading retained snapshots. Include -a signing-key version and retain verification keys while their files survive; -the locator signature grants no access by itself. Inline/compressed metadata -may use the logical path; direct mode requires random-readable native bytes. - -The current chunk reader calls `query_layout`/`query_chunk`; removing -`FileRepository::resolve` does not by itself remove chunk metadata RPCs. -Measure Iceberg file-location resolution, file-state checks, chunk-layout -resolution and data reads separately, including cold cache. The first two -must cause zero KV RPCs on the direct path. Native chunk layout resolution is -retained for cold routing, repair and migration. -Stable chunk IDs/offsets do -not imply immutable replica addresses. A direct chunk API must preserve normal -repair/failover behavior and never embed transient host placement in file URLs. - -Logical deletion affects metadata/list and schedules physical reclamation; it -does not promise immediate revocation of an already known direct URL. A valid -grant and locator can read retained bytes until GC. GET/HEAD/Range GET perform -no file tombstone lookup, and need no per-file revocation cache. List, deletion, -GC and recovery may use KV normally. Physical storage must reject reclaimed -chunk identities/extents and prevent old locators reading newly allocated data; -check this at the storage boundary without an Iceberg file-record lookup. - -The upload state machine is `Staging -> Published -> Deleted -> Reclaimed`, -or `Staging -> Aborted -> Reclaimed`. Incomplete or aborted records never -appear in ListObjectsV2. Complete PUT and multipart retries are idempotent; -conflicting content or overwrite attempts fail without changing a published -record. Equal length alone never establishes an identical retry. Logical -deletion removes list visibility; direct reads remain possible during retention. -GC/read concurrency follows native storage lifetime rules: return retained bytes -or a storage error, never bytes from a replacement allocation. - -The native metadata record uses the public relative filename as its map key -and stores file ID, length, integrity metadata, format, root and native -locations as its value. The public URL is derived from table identity and -filename. This preserves inspection, listing, GC, and -recovery clarity. A file's canonical URL and ETag never change during its -lifetime, including rename, credential rotation and snapshot rollback. - -Arrow/PyArrow derives the bucket from the URI host and the object key from the -URI path. For a `FileSelector`, Arrow removes a trailing slash from the base -directory and sends `ListObjectsV2` with `Prefix = key + "/"`; non-recursive -selectors also send `Delimiter = "/"`. An exact file operation normally -becomes HEAD/GET (and Range for reads); if HEAD misses, Arrow may test an -implicit directory with `Prefix = object-key + "/"`. The native FileIO surface -must therefore keep table-scoped listing, even though Iceberg manifests -normally provide explicit file paths and do not require a full table scan for -reads. - -The server enforces this scope instead of trusting the client: every list -request must name the complete `///` -prefix authorized for the request. Empty, catalog-only, namespace-only, -foreign-table, and bucket-wide prefixes are rejected; bucket-wide bucket -listing is not exposed. The missing-file fallback is allowed only for the -exact `/` child prefix and cannot widen the table grant. This is a -security and isolation rule, not an Arrow workaround. - -The list compatibility contract includes `prefix`, `delimiter`, `max-keys`, -`continuation-token`, `start-after`, `encoding-type=url`, common prefixes, -`IsTruncated`, and `NextContinuationToken`. Exact-object existence fallbacks -may list `/` to distinguish a missing object from an implicit -directory. Listing remains table-scoped and never grants catalog-wide -discovery. - -The fixed virtual bucket has narrow control-plane behavior. HeadBucket succeeds -only when the Iceberg endpoint and table grant are valid; the configured -endpoint supplies region information, so Arrow does not need AWS bucket -discovery. ListBuckets returns `403 AccessDenied`; authenticated GetBucketLocation -returns the configured region. CreateBucket and DeleteBucket are rejected. No operation creates a -bucket per table, and every object operation still requires the immutable -three-ID prefix and a matching grant. - -## Request and lifecycle constraints - -- Canonical object keys are at most 1,024 UTF-8 bytes, including the three IDs - and locator. Preserve relative-key case, literal percent/plus and internal - repeated slashes as in the existing FileIO contract. Reject leading slash, - dot traversal, backslash, controls, query/fragment delimiters and malformed - ID encodings. Decode HTTP percent escapes once at the transport boundary; - neither stored URLs nor KV keys are repeatedly decoded or normalized. - Namespace/name index components use an unambiguous length/escape codec. -- The complete table prefix ends in `/`. Authorization compares parsed IDs and - component boundaries, not textual `starts_with(table-id)`. Descendant list - prefixes may end inside a filename; S3 prefix matching need not end in `/`. - Arrow's exact-missing-file `/` probe needs no remembered prior HEAD. -- List selected published objects, including files awaiting a snapshot commit; - exclude incomplete uploads, losing candidates and tombstones. List is not a - snapshot reachability query. Return public object keys (not full URIs), size, - ETag and the existing fixed epoch LastModified. No internal logical alias is - additionally advertised for a direct file. Listing `data/` need not return - direct objects whose actual public keys are under `f/`. -- Order and paginate `file/` by public filename. Support delimiter `/`, URL - encoding, start-after and common prefixes, with the existing 256-record/ - 4-MiB scan and 2-MiB XML limits, at most 1,000 returned entries. Max-keys zero - returns an empty result. Continuations bind principal, grant nonce/expiry, - catalog epoch, full storage identity, prefix, delimiter and encoding; - foreign/expired tokens fail before scanning. Advance through zero-result - pages and common-prefix subtrees. Pagination remains a forward live traversal. -- GET/HEAD retain S3 content length, stable ETag and conditional request - semantics. Support closed, open-ended and suffix single byte ranges; clamp - an excessive end, return 206/Content-Range for a valid range and 416 for an - unsatisfiable range. Reject multiple ranges explicitly. Never interpret a - requested offset relative to the underlying chunk rather than the file, or - read neighbouring shared-chunk data. ETag is not necessarily a content MD5. -- Ordinary PUT and multipart keys cannot enter reserved `f/`. Injection issues - those URLs only after a selected durable publication; unknown or forged - tokens fail closed. Unknown length and multipart SDK uploads stay on their - original logical key. Completing multipart validates the ordered part set, - freezes it once, and replays the same outcome after response loss. Abort or - TTL expiry reclaims only unselected allocations, never a published winner. -- Writer/reader grants include table-scoped list; cleanup-only grants do not. - DELETE/DeleteObjects require explicit cleanup authority and retained-reference - checks. Failed per-object batch deletions return individual errors. Physical - GC waits for admitted-read retention and durable proof; shared chunks cannot - be reclaimed while another live extent references them. Snapshot expiration - alone is not file deletion. Keep locator file IDs non-reusable to prevent - old URLs reading later allocations at reused offsets. -- Rename/move leaves existing table-bound grants valid until their normal - expiry/revocation; issuance via the old name fails. Drop prevents new - writes/credentials; direct URLs with still-valid grants can read retained - bytes until physical reclamation. Purge - reclaims only the old table's identity; recreate cannot acquire its grants. -- Unsupported CopyObject, UploadPartCopy and bucket administration return an - explicit S3 error without mutation. A filesystem directory has no separate - native record; list synthesizes common prefixes. Do not claim broad SDK - compatibility for directory creation or copy/rename helpers until tested. -- Branch/tag references and historical snapshots keep their file URLs through - rollback, compaction and snapshot expiration. Compaction creates new immutable - files; it never overwrites old locator targets. Commit retains existing CAS, - requirement validation and response-loss behavior. Table `set-location` may - accept the existing canonical base with a trailing slash but cannot relocate - the immutable identity; unsupported location-provider overrides fail clearly. - -## Decisions - -- [x] **Freeze the physical table path at table creation**: use an immutable - ID path containing the catalog ID, the namespace ID at creation, and the - table ID. A rename changes the name index and table metadata name, while a - cross-namespace move changes the live namespace mapping without moving file - bytes or rewriting every manifest reference. The path's namespace component - is therefore a storage namespace ID, not necessarily the table's current - namespace after a move. Files use a canonical prefix such as - `s3://iceberg////`. -- [x] **Represent the default namespace explicitly**: create or recover one - durable default `NamespaceId` for the catalog and use it in the same path and - KV key rules as every named namespace. No special implicit path is needed. -- [x] **Keep S3-shaped URIs as the compatibility surface**: clients use the - S3 SDK and SigV4 over the native Iceberg listener, but the URI does not imply - a general S3 metadata bucket or an external S3 data store. The existing - `ChunkType::IcebergTable` and `FileBlockStore` remain the byte owner. -- [x] **Use a fixed compatibility bucket**: use `iceberg` as the virtual S3 - bucket for every Iceberg file URL. Keep catalog, storage namespace, and table - IDs in the object key. Do not create one bucket per table; enforce table - scope in request validation. -- [x] **Reject unscoped listing**: require a complete table prefix for every - ListObjectsV2 request, reject empty or broader prefixes, and preserve the - Arrow trailing-slash directory fallback only within that table scope. -- [x] **Expose one file URL**: the Iceberg `file_path` is the only public file - location. Keep clear parquet keys and native locator metadata inside the KV - value; do not add a second SDK-visible location field. -- **Proposed rollout**: new catalogs use v2. Existing v1 catalogs retain v1 - reads, writes, grants and URLs; reject in-place version relabeling. Migration - is outside this requirement. Dispatch by catalog layout version and never - merge v1/v2 table scans or reinterpret a v1 suffix as a v2 path. -- [x] **Choose the relative file layout**: the SDK may choose any safe logical - relative name below the table prefix. Native injection uses the reserved - `f/v1/..parquet` form; ordinary SDK writes retain the supplied key. - Both use one `file/` record. The TPC loader passes only its file name to the provider and - does not add a second table directory. - -## Phase 1: identity and canonical path model - -- [x] **Reconcile the permanent design contract**: update - `doc/design/access-server/iceberge/design-crowdb-iceberg.md` and the upload - flow to the fixed `iceberg` bucket, three-ID key, injection-before-manifest - behavior, no manifest rewrite, and direct-vs-record read modes. This must - land before implementation is considered complete. - -- [x] **Define the immutable storage-path codec**: `StorageTablePath` contains - `CatalogId`, storage `NamespaceId` and `TableId`; its lower-hex path is - independent of live names. `NativeFileLocation` uses the fixed virtual bucket - and preserves the exact filename. Boundary, UTF-8, alias and legacy isolation - tests pass in `native_storage_path_test.rs`, `file_location_test.rs`, - and `key_test.rs`. - Files: `lib/crowdb-access-iceberg/src/key/storage_path.rs`, - `lib/crowdb-access-iceberg/src/file/native_location.rs`. -- [x] **Carry immutable storage identity through runtime records**: use - `StorageTablePath` in table heads/mappings, chunk-directory owners, - credentials and creation metadata; add layout dispatch without changing the - v1 catalog contract. The standalone codec does not activate new catalogs. - Files: `lib/crowdb-access-iceberg/src/file/location.rs`, - `lib/crowdb-access-iceberg/src/file/directory.rs`, - `lib/crowdb-access-iceberg/src/table/record.rs`. -- [x] **Define canonical URI encoding and validation**: encode IDs in fixed - lower-case forms, reserve `f/` for server-issued locators, preserve relative - UTF-8 keys, and reject aliases, traversal, authority changes, and malformed - prefixes. Round trips must recover all three IDs and the relative file key. - Files: `lib/crowdb-access-iceberg/src/file/location.rs`, - `lib/crowdb-access-iceberg/tests/file_location_test.rs`. -- [x] **Assign the storage path once** during table creation, including staged - creation, and bind it into the table head and initial metadata. Rename and - cross-namespace move must retain it; a newly created table gets a new table - identity and path. - Files: `lib/crowdb-access-iceberg/src/commit/create/publisher/preparation.rs`, - `lib/crowdb-access-iceberg/src/table/lifecycle/`, - `lib/crowdb-access-iceberg/src/commit/create.rs`. - -## Phase 2: KV key directory redesign - -- [x] **Remove redundant planned indexes**: name lookup returns the complete - storage identity; carry it through callers, multipart, GC and recovery so - no table-ID reverse index is introduced. Store metadata JSON, Avro manifests - and parquet through the same single file record. Keep existing bounded - metadata projection pages under the owning table/generation as derived data. - Files: `lib/crowdb-access-iceberg/src/table/`, - `lib/crowdb-access-iceberg/src/metadata_projection/`, - `lib/crowdb-access-iceberg/src/key/codec.rs`. - -- [x] **Introduce one table-path prefix encoder** for catalog, storage - namespace, and table IDs. All table-owned file keys must use it rather than - assembling ad-hoc suffixes. - Files: `lib/crowdb-access-iceberg/src/key.rs`, - `lib/crowdb-access-iceberg/src/key/codec.rs`, - `lib/crowdb-access-iceberg/src/file/key.rs`. -- [x] **Move the single file authority under the prefix**: - `table-path/file/` is the complete selected record. - Do not introduce separate file name, public-key or file-ID indexes. Preserve bounded - ordered scans and reject a range - whose catalog, namespace, or table prefix differs from the request. - Files: `lib/crowdb-access-iceberg/src/file/key.rs`, - `lib/crowdb-access-iceberg/src/file/listing.rs`, - `lib/crowdb-access-iceberg/src/file/listing/repository.rs`, - `lib/crowdb-access-iceberg/src/catalog/storage.rs`. -- [x] **Audit every table-owned auxiliary key**: multipart records, write - intents/fences, GC candidates/nodes/pending state, and recovery scans must - either carry the same table-path prefix or explicitly document why an - operation-scoped key is independent. Update range bounds and foreign-record - checks together. - Files: `lib/crowdb-access-iceberg/src/file/`, - `lib/crowdb-access-iceberg/src/gc/`, - `lib/crowdb-access-iceberg/src/record/`, - `lib/crowdb-access-iceberg/src/key/codec.rs`. - Head/file/projection/injection records use the frozen native prefix. The - permanent design explicitly lists independent catalog service journals: - unpublished write intents/fences, upload sessions/parts, catalog admission, - operation/payload journals and GC task/claim/proof workspace. They recover - work without a published table and coordinate catalog-wide reclamation; - they never resolve or list a published file. Their values bind full storage - identity. GC layout dispatch and claim adoption, inactive protection and - proof traversal now compare full identity rather than only table ID. -- [x] **Version the table key envelope and routed-store dispatch**: - `TableStorageKey` uses a v2 envelope followed by the - slash-separated immutable table path and scope. File keys contain only the - public filename; `file_range` bounds metadata/filename scans directly. - `StorageKey` selects the decoder by version without fallback. Routed point - operations accept either validated envelope, while scan bounds reject mixed - versions and out-of-range returned keys. Legacy decoding and cross-table/ - namespace range isolation are tested. - Files: `lib/crowdb-access-iceberg/src/key/table_storage.rs`, - `lib/crowdb-access-iceberg/src/key/storage.rs`, - `lib/crowdb-access-iceberg/src/catalog/storage.rs`, - `lib/crowdb-access-iceberg/tests/native_storage_path_test.rs`. -- [x] **Dispatch catalogs and recovery by persisted layout**: bind the selected - layout in catalog authority before creating new native paths. Retain v1 - reads/writes/grants for v1 catalogs; reject relabeling and never merge layouts - in recovery scans. Repository and lifecycle wiring must use the selected - layout together; key codec support alone is not activation. - Files: `lib/crowdb-access-iceberg/src/catalog/`, - `lib/crowdb-access-iceberg/src/record/authority.rs`, - `lib/crowdb-protocol/src/fbs/iceberg.fbs`, - `lib/crowdb-access-iceberg/tests/*recovery*test.rs`. - -## Phase 3: FileIO and S3-compatible access - -- [x] **Implement native injection finalization**: the internal ingestion entry - accepts the full storage identity, readable stem, upload identity and bytes; - finish durable native writes, bind the exact filename to a signed descriptor, - CAS one file record, then return its one canonical URL. Persist/reuse the - existing operation identity for uncertain outcomes; no separate permanent - filename/public/file-ID indexes. SDK PUT continues to return its normal S3 - response and does not pretend to rename the client's path. - Files: `lib/crowdb-access-iceberg/src/file/`, - `app/crowdb-access-server/src/iceberg/`, `tools/load-tpch.py`. -- [x] **Enforce the direct chunk boundary**: introduce/extend the native range - read interface accepting the authenticated full descriptor/root; instrument - Iceberg file-location, file-state and chunk layout queries separately. - Retain `query_chunk` for native layout resolution, including cache misses. - Exercise cold routing, multi-extent reads, repair, GC races and offset reuse. - Files: `lib/crowdb-access-iceberg/src/file/blocks.rs`, - `lib/crowdb-chunk-client/src/chunk/chunk_reader.rs`, - `lib/crowdb-chunk-client/tests/chunk_reader_test.rs`. -- [x] **Reuse bounded permission validation**: define permission-cache entries - by principal, catalog epoch, full storage identity and operations; bound - validity by grants and applicable authority leases. Misses validate authority, - expired entries fail closed, and direct reads never load per-file state. - Keep permission cache accounting separate from native location resolution. - Use existing lock-free cache mechanisms; do not introduce a lock as a shortcut. - Files: `lib/crowdb-access-iceberg/src/file/credentials.rs`, - `app/crowdb-access-server/src/iceberg/file_admission.rs`. - -- [x] **Bind grants and credential prefixes to the full storage path** so a - credential issued for one table cannot address another namespace/table path. - Update refresh responses and staged-table credentials consistently. - Files: `lib/crowdb-access-iceberg/src/file/credentials.rs`, - `lib/crowdb-access-iceberg/src/wire/credentials.rs`, - `app/crowdb-access-server/src/iceberg/table_credentials.rs`. -- [x] **Parse and route the new URI prefix** in exact object, list, multipart, - cleanup, and continuation-token flows. A table rename must continue to use - the same URI and file records after the new name is loaded from the catalog. - Files: `app/crowdb-access-server/src/iceberg/file_request.rs`, - `app/crowdb-access-server/src/iceberg/file_http.rs`, - `app/crowdb-access-server/src/iceberg/file_list_request.rs`, - `app/crowdb-access-server/src/iceberg/file_admission.rs`. -- [x] **Keep physical bytes native**: no filesystem or general S3 object - authority is introduced. Uploads continue through native chunk writes and - publication remains a KV authority CAS. A direct read receives a complete - validated chunk locator without `FileRepository::resolve`; a - `record_resolution` file uses the existing file record path. - Files: `app/crowdb-access-server/src/iceberg/file_http/stream.rs`, - `lib/crowdb-access-iceberg/src/file/blocks.rs`, - `lib/crowdb-access-iceberg/src/file/repository.rs`, - `lib/crowdb-chunk-client/src/chunk/chunk_reader.rs`. -- [x] **Constrain catalog location-provider output**: reject a provider result - outside the canonical table prefix before upload. This prevents the loader - from recording a URI that the native grant cannot serve. - Files: `/cjdata/cpp/crowdb-tpc-loader/src/crowdb_tpc_loader/backend.py`, - `/cjdata/cpp/crowdb-tpc-loader/src/crowdb_tpc_loader/loader.py`. -- [x] **Use one SDK-visible file URL**: use its public filename as the single - native KV file key, retain the native chunk descriptor and read - mode in its value, and make the single URL written to Iceberg manifests the - bound location. Implement native injection finalization before manifest - creation. Ordinary PUT retains its logical URL and `record_resolution`; - neither path rewrites existing manifests or requires a new SDK field. - Files: `lib/crowdb-access-iceberg/src/file/record.rs`, - `lib/crowdb-access-iceberg/src/file/location.rs`, - `app/crowdb-access-server/src/iceberg/file_http/`, - `lib/crowdb-access-iceberg/src/wire/credentials.rs`. -- [x] **Add direct locator validation**: bind locator version, table identity, - file identity, length, integrity metadata and chunk root; authenticate the - current HTTP grant independently of the durable locator signature; - reject tampering before chunk access and enforce one contiguous range. - Files: `lib/crowdb-access-iceberg/src/file/location.rs`, - `lib/crowdb-access-iceberg/src/file/record.rs`, - `app/crowdb-access-server/src/iceberg/file_http.rs`. - -## Phase 4: metadata, rename, and lifecycle invariants - -- [x] **Keep large metadata as native objects**: preserve the inline threshold, - route larger metadata JSON/Avro into native chunk storage, bound descriptors - with directory roots and keep snapshot arrays out of TableHead. Exercise - canonical bytes through FileIO and ALL/REFS, including missing/corrupt - generation projections and explicit parser/response limit errors. - Files: `lib/crowdb-access-iceberg/src/file/record.rs`, - `lib/crowdb-access-iceberg/src/table/metadata/`, - `lib/crowdb-access-iceberg/src/metadata_projection/`, - `lib/crowdb-access-iceberg/src/commit/files.rs`. -- [x] **Implement bounded metadata prefix list**: map the public metadata or - filename prefix to the exact `file/` range; stream ordered records into a - bounded page and persist an authenticated continuation at scan-budget - exhaustion. Handle tombstones, sparse pages, delimiter common prefixes and - zero max-keys without payload reads, table-wide sorting or unrelated scans. - Files: `lib/crowdb-access-iceberg/src/file/listing/`, - `app/crowdb-access-server/src/iceberg/file_list_request.rs`. -- [x] **Preserve standard metadata semantics**: cover snapshots/refs/logs and - v1 direct manifests, v2/v3 sequence inheritance and existing v3 lineage; - preserve exact delete-file references and immutable manifest bytes. Commit - retries may write new manifest lists; atomic head selection and retained - reference GC remain authoritative. Add no private snapshot KV authority. - Files: `lib/crowdb-access-iceberg/src/commit/`, - `lib/crowdb-access-iceberg/src/table/metadata/`, - `lib/crowdb-access-iceberg/src/gc/`, - `app/crowdb-access-server/tests/iceberg_table_lifecycle_test.rs`. - -- [x] **Validate table metadata location against the immutable storage path**; - `set-location` may normalize a trailing slash but cannot change the path - identity. Metadata generations and manifest references remain below that - path. - Files: `lib/crowdb-access-iceberg/src/table/metadata/root.rs`, - `lib/crowdb-access-iceberg/src/commit/evaluator/initial.rs`, - `lib/crowdb-access-iceberg/src/commit/evaluator/scalar.rs`, - `lib/crowdb-access-iceberg/src/commit/files.rs`. -- [x] **Preserve rename behavior**: same-namespace rename and cross-namespace - move update only name/mapping authority; old names disappear, the storage - path remains readable through the new table identity, and a recreated name - receives a new path. - Files: `lib/crowdb-access-iceberg/src/table/lifecycle/`, - `app/crowdb-access-server/tests/iceberg_table_lifecycle_test.rs`, - `e2e-test/clients/iceberg-java/src/main/java/TestIcebergCatalogWrites.java`. -- [x] **Define grant and URL behavior across moves**: existing immutable URLs - continue to bind to the storage namespace ID while the table is live; new - credentials resolve the current name mapping and old-name resolution fails. - Drop/recreate receives a new table ID, and direct and fallback locators reject - the old table after purge. - Files: `lib/crowdb-access-iceberg/src/file/credentials.rs`, - `app/crowdb-access-server/src/iceberg/file_admission.rs`. -- [x] **Recheck GC and recovery ownership** against the immutable storage path - before reclaiming files or chunk roots. Logical drop and physical purge must - not confuse a recreated table with a retained old path. - Files: `lib/crowdb-access-iceberg/src/gc/`, - `lib/crowdb-access-iceberg/src/file/multipart_recovery.rs`, - `lib/crowdb-access-iceberg/src/record/`. - -## Phase 5: loader and container injection - -- [x] **Align the TPC loader with the canonical table path**: use stable data - names below the provider's `data/` directory, verify the returned URI is - table-scoped, and keep the run ID in recovery/report metadata rather than in - the object path. - Files: `/cjdata/cpp/crowdb-tpc-loader/src/crowdb_tpc_loader/backend.py`, - `/cjdata/cpp/crowdb-tpc-loader/src/crowdb_tpc_loader/loader.py`, - `/cjdata/cpp/crowdb-tpc-loader/tests/unit/test_commits.py`. -- [x] **Separate or remove the optional S3 mirror** in - `tools/load-tpch.py`. If retained for the container demo, document its - bucket/key as an independent S3 object copy and do not treat it as the - Iceberg table file layout. - Files: `tools/load-tpch.py`, - `container/single-node-container/README.md`. - -## Native container verification - -- Fresh image source verification and the two scoped listing cases pass with - PyIceberg 0.11.1 and Arrow 25.0.1. The production loader upload helper selects - one URL on retry; standard manifest registration, Arrow read and SQL aggregate - pass before/after a cross-namespace move and catalog storage restart. -- First complete core partition: 23 passed, two failures. The GC CLI fixture - still supplied a table ID; it now supplies the complete frozen path. Tree - multipart found a real decoder gap: native digest checkpoints are 205 bytes, - but completion decoding only accepted legacy 189-byte checkpoints. Both bounded - shapes now decode, with full owner/checksum validation unchanged; the new - partial native checkpoint round-trip and existing recovery/completion tests pass. -- Core partition rerun: all 25 cases pass, including native tree multipart - reload, real chunk-KV restart, GC controls and large streamed/multipart IO. -- Apache Iceberg 1.11.0 Java catalog snapshot/staged creation and opaque - data/delete reads pass. S3FileIO also passes after its foreign-table fixture - was corrected to mutate a table ID within the valid three-ID prefix, retaining - the 403 AccessDenied assertion. -- Official Rust lifecycle and lost-response/native restart cases pass. The - retirement test is waiting through its full delegated-access grace period. -- PyIceberg 0.10.0 matches the loader adapter's declared range. Its complete - production dataflow creates a fixture table, uploads through native injection, - persists the selected URL in a real load report, commits the standard manifest, - then reloads the report/table and executes SQL over native reads after restart. - This focused fixture verifies ingestion; it does not claim an SF1 benchmark. -- The loader feature is committed separately with its 226-test/Ruff gate and - fresh native container read/SQL verification. Pre-existing version, docs and - ordinary filename edits remain unstaged in that repository. - -## Acceptance tests - -Unit: - -- [ ] Canonical storage path and URI round trips include catalog, storage - namespace, table, and relative file key. -- [ ] New and old key layouts cannot collide; table-path scans exclude foreign - namespace/table records. -- [ ] File grants, list prefixes, continuation tokens, multipart sessions, GC - keys, and chunk-directory owners reject mismatched identities. - -Integration: - -- [ ] Store metadata above the inline threshold with many snapshots; assert - bounded head/record size, chunk-backed bytes, correct ALL/REFS and projection - fallback. Exceed each configured limit and assert an explicit error with no - truncated success or partially published head. -- [ ] Generate many metadata and manifest files with interleaved other file - prefixes. Page full-table, `metadata/` and filename-prefix lists; record scan - bounds, peak page memory and payload-read counts. Assert no unrelated scans, - no payload reads and complete results on an unchanged eligible set. -- [ ] Race metadata commits and retry their lost responses; reload every - selected version with official clients and verify snapshot/manifest references, - retained branches/tags and protected shared-file GC behavior. - -- [ ] Publish native injection, load its direct URL through unmodified readers, - and publish a separate SDK-written logical URL. Assert one URL per file and - correct position deletes, deletion-vector references and historical reads. -- [ ] Crash before and after the single file-record CAS, restart and - replay; assert no ghost list entries, no lost selected file and no GC of a - losing operation's shared winner. Retry equal-length different bytes. -- [ ] Renew credentials and rotate locator signing keys; retained snapshot URLs - stay readable. Delete concurrently with direct read and restart: list omits - the deleted file, but a valid direct URL reads bytes until reclamation. After - reclamation/reuse, the old URL must never return another file's bytes. -- [ ] Count location, deletion-state and chunk-layout RPCs independently for - cold/warm HEAD and footer ranges. Include multi-location files, directory-root - boundary reads, chunk repair/failover and URI size limits. No zero-KV claim - may be inferred from merely skipping `FileRepository::resolve`; assert zero - Iceberg file-location and file-state KV RPCs for direct reads. Normal chunk - layout resolution remains supported and is measured independently. -- [ ] Test public-key order and delimiter grouping when internal logical names - sort differently; test max-keys zero, empty pages and credential refresh. -- [ ] Test suffix/open-ended/out-of-bounds ranges, ETag conditions, unknown - length upload, multipart abort-vs-complete races and response-loss retries. -- [ ] Move a table, drop/recreate its old namespace, then resolve and read via - the new name; assert no cascade deletion or storage namespace ID reuse. - -- [ ] Create a table in the default namespace and an explicit namespace; - inspect metadata, KV keys, and native file locations. -- [ ] PUT/GET/HEAD/list/multipart a parquet file through an S3 SDK and verify - the object is below the canonical table prefix, bytes remain in native - Iceberg chunk storage, the URL is unchanged by commit, and direct versus - fallback reads are observable. -- [ ] Exercise Arrow HeadBucket, exact HEAD miss directory fallback, recursive - and non-recursive ListObjectsV2, continuation expiry, and rejection of empty, - catalog-only, namespace-only, foreign-table, and bucket-wide prefixes. -- [ ] Rename within a namespace and move across namespaces; reload metadata and - read the original file through the new name without moving or duplicating it. -- [ ] Drop and recreate the same name; verify the new table cannot see the old - path and GC/recovery retain the correct ownership. - -E2E: - -- [ ] Run PyIceberg, Java Iceberg, and Rust Iceberg lifecycle flows with native - FileIO credentials and the new URI prefix. -- [ ] Run the container injection flow and verify the Iceberg report's - `remote_uri` values separately from any optional general-S3 mirror report. -- [ ] Exercise response loss, restart, multipart recovery, and retired-table - grace paths with the new key ranges. -- [ ] Exercise v1 legacy reads/writes, rejected in-place activation, v1/v2 collision checks, - direct locator tampering, foreign-table use, checksum/length conflicts, and - the no-manifest-rewrite guarantee. - -## Consolidated file list - -- Identity and URI: `lib/crowdb-access-iceberg/src/file/`, - `lib/crowdb-access-iceberg/src/table/record.rs`. -- KV keys and scans: `lib/crowdb-access-iceberg/src/key/`, - `lib/crowdb-access-iceberg/src/file/key.rs`, - `lib/crowdb-access-iceberg/src/catalog/storage.rs`. -- Metadata scale: `lib/crowdb-access-iceberg/src/metadata_projection/`, - `lib/crowdb-access-iceberg/src/table/metadata/`. -- Native direct reads: `lib/crowdb-chunk-client/src/chunk/chunk_reader.rs`, - `lib/crowdb-chunk-client/tests/chunk_reader_test.rs`. -- HTTP/FileIO: `app/crowdb-access-server/src/iceberg/`, - `lib/crowdb-access-iceberg/src/wire/credentials.rs`. -- Lifecycle and GC: `lib/crowdb-access-iceberg/src/commit/`, - `lib/crowdb-access-iceberg/src/table/lifecycle/`, - `lib/crowdb-access-iceberg/src/gc/`. -- Loader/container: `/cjdata/cpp/crowdb-tpc-loader/src/crowdb_tpc_loader/`, - `/cjdata/cpp/crowdb/tools/load-tpch.py`, - `container/single-node-container/README.md`. From 980e0c93b793c20a959796aff6014151fe21f710 Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 15 Oct 2026 10:49:45 +0800 Subject: [PATCH 09/15] Fix Console credential and metadata rendering --- .../ui/src/access/Workbench.test.tsx | 23 ++++++++ app/crowdb-web/ui/src/access/Workbench.tsx | 37 ++++++++++-- app/crowdb-web/ui/src/iceberg/Fields.test.tsx | 31 ++++++++++ app/crowdb-web/ui/src/iceberg/Fields.tsx | 58 ++++++++++++++++++- .../ui/src/iceberg/ReferenceExplorer.tsx | 4 +- app/crowdb-web/ui/src/views/S3View.tsx | 2 +- .../e2e/fixtures/defaultDeployments.ts | 4 +- 7 files changed, 149 insertions(+), 10 deletions(-) create mode 100644 app/crowdb-web/ui/src/access/Workbench.test.tsx create mode 100644 app/crowdb-web/ui/src/iceberg/Fields.test.tsx diff --git a/app/crowdb-web/ui/src/access/Workbench.test.tsx b/app/crowdb-web/ui/src/access/Workbench.test.tsx new file mode 100644 index 000000000..f911c2ab9 --- /dev/null +++ b/app/crowdb-web/ui/src/access/Workbench.test.tsx @@ -0,0 +1,23 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { beforeEach, expect, it, vi } from 'vitest'; +import { DocsHelp } from './Workbench'; + +beforeEach(() => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(JSON.stringify({ + s3: { endpoint: 'http://localhost:9091', region: 'us-east-1', access_key_id: 'access', secret_access_key: 'secret' }, + iceberg: { uri: 'http://localhost:9092', token: 'token' }, + })))); + Object.defineProperty(navigator, 'clipboard', { configurable: true, value: { writeText: vi.fn().mockResolvedValue(undefined) } }); +}); + +it('reveals credentials and reports a successful copy', async () => { + render(); + fireEvent.click(screen.getByText('Connection credentials')); + fireEvent.click(screen.getByRole('button', { name: 'Show local credentials' })); + await waitFor(() => expect(screen.getByText(/ICEBERG_URI=http:\/\/localhost:9092/)).toBeVisible()); + fireEvent.click(screen.getByRole('button', { name: 'Copy env' })); + await waitFor(() => expect(screen.getByRole('status')).toHaveTextContent('Copied environment variables')); + expect(navigator.clipboard.writeText).toHaveBeenCalledWith(expect.stringContaining('AWS_ACCESS_KEY_ID=access')); +}); diff --git a/app/crowdb-web/ui/src/access/Workbench.tsx b/app/crowdb-web/ui/src/access/Workbench.tsx index 4bd7caeeb..0d5d70139 100644 --- a/app/crowdb-web/ui/src/access/Workbench.tsx +++ b/app/crowdb-web/ui/src/access/Workbench.tsx @@ -13,16 +13,29 @@ export const buttonClass = 'tw-rounded tw-border tw-border-border tw-px-3 tw-py- export function DocsHelp({ href, title, auth }: { href: string; title: string; auth?: string }) { const [credentials, setCredentials] = useState(null); const [error, setError] = useState(''); + const [copyStatus, setCopyStatus] = useState(''); + const endpointForBrowser = (value: string | null) => { + if (!value) return ''; + try { + const url = new URL(value); + const browserHost = window.location.hostname; + if (browserHost && ['localhost', '127.0.0.1', '[::1]', '::1'].includes(url.hostname)) url.hostname = browserHost; + return url.toString().replace(/\/$/, ''); + } catch { + return value; + } + }; const reveal = async () => { setError(''); + setCopyStatus(''); try { const response = await fetch(`${getApiBase()}/access/credentials`, { cache: 'no-store' }); if (!response.ok) throw new Error(`HTTP ${response.status}`); const value = await response.json() as { s3: Record; iceberg: Record }; const lines = [ - `ICEBERG_URI=${value.iceberg.uri ?? ''}`, + `ICEBERG_URI=${endpointForBrowser(value.iceberg.uri)}`, `ICEBERG_TOKEN=${value.iceberg.token ?? ''}`, - `AWS_ENDPOINT_URL=${value.s3.endpoint ?? ''}`, + `AWS_ENDPOINT_URL=${endpointForBrowser(value.s3.endpoint)}`, `AWS_DEFAULT_REGION=${value.s3.region ?? ''}`, `AWS_ACCESS_KEY_ID=${value.s3.access_key_id ?? ''}`, `AWS_SECRET_ACCESS_KEY=${value.s3.secret_access_key ?? ''}`, @@ -32,13 +45,29 @@ export function DocsHelp({ href, title, auth }: { href: string; title: string; a setError(`Credentials are unavailable: ${String(reason)}`); } }; - const copy = () => { if (credentials) void navigator.clipboard?.writeText(credentials); }; + const copy = async () => { + if (!credentials) return; + setCopyStatus(''); + try { + if (navigator.clipboard?.writeText) await navigator.clipboard.writeText(credentials); + else { + const input = document.createElement('textarea'); + input.value = credentials; input.setAttribute('readonly', ''); input.style.position = 'fixed'; input.style.opacity = '0'; + document.body.appendChild(input); input.select(); + if (!document.execCommand('copy')) throw new Error('Clipboard is unavailable'); + input.remove(); + } + setCopyStatus('Copied environment variables'); + } catch (reason) { + setCopyStatus(`Copy failed: ${String(reason)}`); + } + }; return
{title}
Help
- {auth &&
Connection credentials

{auth}

{credentials && }
{error &&

{error}

}{credentials && 
{credentials}
}
} + {auth &&
Connection credentials

{auth}

{credentials && }
{error &&

{error}

}{copyStatus &&

{copyStatus}

}{credentials && 
{credentials}
}
}
; } export function Workbench({ sidebar, children, detail, showActivity = true, resizableSidebar = true, help }: { resizableSidebar?: boolean; showActivity?: boolean; sidebar: ReactNode; children: ReactNode; detail?: ReactNode; help?: ReactNode }) { diff --git a/app/crowdb-web/ui/src/iceberg/Fields.test.tsx b/app/crowdb-web/ui/src/iceberg/Fields.test.tsx new file mode 100644 index 000000000..748fdfcea --- /dev/null +++ b/app/crowdb-web/ui/src/iceberg/Fields.test.tsx @@ -0,0 +1,31 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { expect, it } from 'vitest'; +import { Fields, SchemaTable } from './Fields'; + +it('renders file metrics records as a table', () => { + render(); + const table = screen.getByRole('table', { name: 'File metrics' }); + expect(table).toHaveTextContent('null count'); + expect(table).toHaveTextContent('id'); + expect(table).toHaveTextContent('name'); + expect(table.querySelectorAll('tbody tr')).toHaveLength(2); +}); + +it('renders nested writer fields as an expandable table', () => { + render(); + const table = screen.getByRole('table', { name: 'Writer schema' }); + expect(table).toHaveTextContent('status'); + expect(table).toHaveTextContent('data_file'); + expect(table).not.toHaveTextContent('file_path'); + const button = screen.getByRole('button', { name: 'Expand field details' }); + fireEvent.click(button); + return waitFor(() => expect(table).toHaveTextContent('file_path')); +}); diff --git a/app/crowdb-web/ui/src/iceberg/Fields.tsx b/app/crowdb-web/ui/src/iceberg/Fields.tsx index 5e7eac3a2..d9700f643 100644 --- a/app/crowdb-web/ui/src/iceberg/Fields.tsx +++ b/app/crowdb-web/ui/src/iceberg/Fields.tsx @@ -18,15 +18,71 @@ export function byteSize(value: string | number | undefined): string { } export function Fields({ values, stacked = false }: { values: Record; stacked?: boolean }) { if (Object.keys(values).length > 50) return ; - return
{Object.entries(values).slice(0, 50).map(([key, value]) =>
{key.replaceAll('_', ' ').replaceAll('-', ' ')}
{typeof value === 'object' && value !== null ? stacked ? : : scalar(value)}
)}
; + return
{Object.entries(values).slice(0, 50).map(([key, value]) =>
{key.replaceAll('_', ' ').replaceAll('-', ' ')}
{key === 'metrics' && Array.isArray(value) && value.every(item => asObject(item)) ? asObject(item)!)} /> : typeof value === 'object' && value !== null ? stacked ? : : scalar(value)}
)}
; +} +function ObjectArrayTable({ label, values }: { label: string; values: SchemaNode[] }) { + const headings = [...new Set(values.flatMap(value => Object.keys(value)))]; + return
{headings.map(heading => )}{values.map((value, index) => {headings.map(heading => )})}
#{heading.replaceAll('_', ' ')}
{index}{typeof value[heading] === 'object' && value[heading] !== null ? JSON.stringify(value[heading]) : scalar(value[heading])}
; } export function Structured({ value }: { value: unknown }) { const [page, setPage] = useState(0); if (value == null || typeof value !== 'object') return <>{scalar(value)}; + if (Array.isArray(value) && value.length > 0 && value.every(item => asObject(item))) return asObject(item)!)} />; const entries = Object.entries(value); const start = Math.min(page * 30, Math.max(0, Math.floor((entries.length - 1) / 30) * 30)); return <>
{entries.slice(start, start + 30).map(([key, item]) =>
{key}
{item && typeof item === 'object' ? : scalar(item)}
)}
{entries.length > 30 &&
{start + 1}–{Math.min(start + 30, entries.length)} of {entries.length}
}; } +type SchemaNode = Record; +const asObject = (value: unknown): SchemaNode | null => value && typeof value === 'object' && !Array.isArray(value) ? value as SchemaNode : null; +const schemaFields = (value: unknown): SchemaNode[] => { + if (Array.isArray(value)) return value.flatMap(item => schemaFields(item)); + const object = asObject(value); + if (!object || !Array.isArray(object.fields)) return []; + return object.fields.filter((field): field is SchemaNode => !!asObject(field)).map(field => asObject(field)!); +}; +const schemaType = (value: unknown): string => { + if (Array.isArray(value)) return value.map(schemaType).join(' | '); + if (typeof value === 'string') return value; + const object = asObject(value); + if (!object) return scalar(value); + if (typeof object.type === 'string') { + if (object.logicalType) return `${object.type} (${String(object.logicalType)})`; + if (object.type === 'array' && object.items) return `array<${schemaType(object.items)}>`; + if (object.type === 'map' && object.values) return `map`; + return object.name ? `${object.type} ${String(object.name)}` : object.type; + } + if (object.fields) return object.name ? `record ${String(object.name)}` : 'record'; + return JSON.stringify(value); +}; +const schemaChildren = (field: SchemaNode): SchemaNode[] => { + const direct = schemaFields(field); + if (direct.length) return direct; + return schemaFields(field.type); +}; +function SchemaFieldRows({ fields, depth = 0 }: { fields: SchemaNode[]; depth?: number }) { + return <>{fields.map((field, index) => )}; +} +function SchemaFieldRow({ field, depth }: { field: SchemaNode; depth: number }) { + const [open, setOpen] = useState(false); + const children = schemaChildren(field); + const fieldType = schemaType(field.type ?? field); + const required = field.required === true || (Array.isArray(field.type) && !field.type.some(item => item === 'null')); + return <> + + {children.length > 0 ? : String(field.name ?? 'field')} + {fieldType} + {required ? 'Yes' : 'No'} + {scalar(field.id ?? field['field-id'])} + {field.doc ? String(field.doc) : children.length ? `${children.length} nested fields` : '—'} + + {open && } + ; +} +export function SchemaTable({ value }: { value: unknown }) { + const fields = schemaFields(value); + if (!fields.length) return ; + return
{['Field', 'Type', 'Required', 'ID', 'Details'].map(heading => )}
{heading}
; +} function Expandable({ value }: { value: object }) { const [open, setOpen] = useState(false); return
setOpen(event.currentTarget.open)}>{Object.keys(value).length} {Array.isArray(value) ? 'items' : 'fields'}{open && }
; diff --git a/app/crowdb-web/ui/src/iceberg/ReferenceExplorer.tsx b/app/crowdb-web/ui/src/iceberg/ReferenceExplorer.tsx index a41299e6c..ae876d9cd 100644 --- a/app/crowdb-web/ui/src/iceberg/ReferenceExplorer.tsx +++ b/app/crowdb-web/ui/src/iceberg/ReferenceExplorer.tsx @@ -1,7 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. import { buttonClass } from '../access/Workbench'; -import { Fields, Records, Structured, byteSize } from './Fields'; +import { Fields, Records, SchemaTable, byteSize } from './Fields'; import { ManifestTable } from './ManifestTable'; import { ParquetInspector } from './ParquetInspector'; import type { Inspector } from './useInspection'; @@ -18,7 +18,7 @@ export function InspectionView({ inspector, propertyHost }: { inspector: Inspect {data?.kind === 'unsupported' && <>} />

{data.reason}

} {data?.kind === 'parquet' && } {data?.kind === 'manifest-list' && } - {data?.kind === 'manifest' && <> [, f.status, f.content, f.format, f.records, byteSize(f.size)])} />
Writer schema
} + {data?.kind === 'manifest' && <> [, f.status, f.content, f.format, f.records, byteSize(f.size)])} />
Writer schema
} {data && data.kind !== 'parquet' && Number(data.offset) > 0 && } {data?.next != null && } {data?.kind === 'parquet' && data.groups?.[0]?.index !== '0' && } diff --git a/app/crowdb-web/ui/src/views/S3View.tsx b/app/crowdb-web/ui/src/views/S3View.tsx index eb6c38056..aaf3d31d2 100644 --- a/app/crowdb-web/ui/src/views/S3View.tsx +++ b/app/crowdb-web/ui/src/views/S3View.tsx @@ -197,7 +197,7 @@ export function S3View({ active, readonly, onChunk }: { active: boolean; readonl const bytes = await previewBytes(response); setPreview(new TextDecoder('utf-8', { fatal: false }).decode(bytes)); }, 'Loaded first 4 KiB')}>Preview first 4 KiB - {!readonly && } + {!readonly && } } {error &&

{error}

} diff --git a/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts b/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts index 9c3859da1..930e6cc4c 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts +++ b/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts @@ -28,7 +28,7 @@ export function observeDefaultDeployments(page: Page) { if (rejected) expect(rejected.status(), await rejected.text()).toBe(201); const completed = nodes.find(node => replies.get(`${node}:${kind}`)?.some(response => response.status() === 201)); if (completed !== undefined) return completed; - const timeout = process.platform === 'darwin' ? 30_000 : 3_000; + const timeout = process.platform === 'darwin' ? 30_000 : 10_000; const response = await page.waitForResponse(response => pending.has(key(response) ?? '') && (response.status() === 201 || !expectedWait(response)), { timeout }); expect(response.status(), await response.text()).toBe(201); @@ -36,7 +36,7 @@ export function observeDefaultDeployments(page: Page) { }, async verify(node: number, kind: string) { const identity = `${node}:${kind}`; - const timeout = process.platform === 'darwin' ? 30_000 : 3_000; + const timeout = process.platform === 'darwin' ? 30_000 : 10_000; const response = replies.get(identity)?.find(response => response.status() === 201) ?? await page.waitForResponse(response => key(response) === identity && response.status() === 201, { timeout }); for (const rejected of replies.get(identity) ?? []) { From a395fa5aff46258a8cee1a7369f6ef33f52fee8e Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 15 Oct 2026 12:03:02 +0800 Subject: [PATCH 10/15] Allow longer deployment observation in CI --- .../clients/console-ui/e2e/fixtures/defaultDeployments.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts b/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts index 930e6cc4c..fccc4ce63 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts +++ b/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts @@ -28,7 +28,7 @@ export function observeDefaultDeployments(page: Page) { if (rejected) expect(rejected.status(), await rejected.text()).toBe(201); const completed = nodes.find(node => replies.get(`${node}:${kind}`)?.some(response => response.status() === 201)); if (completed !== undefined) return completed; - const timeout = process.platform === 'darwin' ? 30_000 : 10_000; + const timeout = 30_000; const response = await page.waitForResponse(response => pending.has(key(response) ?? '') && (response.status() === 201 || !expectedWait(response)), { timeout }); expect(response.status(), await response.text()).toBe(201); @@ -36,7 +36,7 @@ export function observeDefaultDeployments(page: Page) { }, async verify(node: number, kind: string) { const identity = `${node}:${kind}`; - const timeout = process.platform === 'darwin' ? 30_000 : 10_000; + const timeout = 30_000; const response = replies.get(identity)?.find(response => response.status() === 201) ?? await page.waitForResponse(response => key(response) === identity && response.status() === 201, { timeout }); for (const rejected of replies.get(identity) ?? []) { From 97f17c339d07889d59b81fb200cf5232f7eedcd7 Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 15 Oct 2026 12:34:15 +0800 Subject: [PATCH 11/15] Wait for durable service plans in UI E2E --- .../e2e/fixtures/defaultDeployments.ts | 23 +++++++++++-------- .../console-ui/e2e/flows/71-s3-native.spec.ts | 4 +++- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts b/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts index fccc4ce63..787cc6e2a 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts +++ b/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts @@ -21,24 +21,29 @@ export function observeDefaultDeployments(page: Page) { }); return { async nextCompleted(nodes: number[], kind: string) { - const pending = new Set(nodes.map(node => `${node}:${kind}`)); const expectedWait = (response: Response) => response.status() === 409 && ['chunkdb', 'chunk-kv'].includes(kind); const rejected = nodes.flatMap(node => replies.get(`${node}:${kind}`) ?? []) .find(response => response.status() !== 201 && !expectedWait(response)); if (rejected) expect(rejected.status(), await rejected.text()).toBe(201); const completed = nodes.find(node => replies.get(`${node}:${kind}`)?.some(response => response.status() === 201)); if (completed !== undefined) return completed; - const timeout = 30_000; - const response = await page.waitForResponse(response => pending.has(key(response) ?? '') - && (response.status() === 201 || !expectedWait(response)), { timeout }); - expect(response.status(), await response.text()).toBe(201); - return Number(key(response)!.split(':')[0]); + // The deploy request can be queued behind disk recovery, so its response + // is not a useful readiness signal. Follow the durable service plan and + // let the enclosing Playwright test budget report a genuine hang. + for (;;) { + const plans = await page.evaluate(async () => { + const response = await fetch('/api/service-plans'); + return response.json(); + }); + const ready = nodes.find(node => plans[node]?.steps?.[kind]?.state === 'deployed'); + if (ready !== undefined) return ready; + await new Promise(resolve => setTimeout(resolve, 100)); + } }, async verify(node: number, kind: string) { const identity = `${node}:${kind}`; - const timeout = 30_000; - const response = replies.get(identity)?.find(response => response.status() === 201) - ?? await page.waitForResponse(response => key(response) === identity && response.status() === 201, { timeout }); + const response = replies.get(identity)?.find(response => response.status() === 201); + if (!response) return; for (const rejected of replies.get(identity) ?? []) { if (rejected === response) break; const body = await rejected.json(); diff --git a/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts b/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts index 0a093cab8..c9486fb68 100644 --- a/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts @@ -24,7 +24,9 @@ test('S3 native multipart upload, HEAD, bounded preview and full round trip', as await step('native S3 DOM setup', async () => { await page.goto('/?domain=S3'); await expect(page.getByLabel('Access key', { exact: true })).toHaveCount(0); - await page.getByRole('navigation', { name: 'S3 buckets' }).getByRole('button', { name: bucket, exact: true }).click(); + const bucketButton = page.getByRole('navigation', { name: 'S3 buckets' }).getByRole('button', { name: bucket, exact: true }); + await expect(bucketButton).toBeVisible({ timeout: 10_000 }); + await bucketButton.click(); await page.getByText('Bucket actions', { exact: true }).click(); await page.getByLabel('Object key', { exact: true }).fill(key); await page.getByLabel('Object file').setInputFiles({ name: 'multipart.txt', mimeType: 'text/plain', buffer: bytes }); From 840b4e801f7381d23df50dfd60f78a5a663fb7b3 Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 15 Oct 2026 18:56:46 +0800 Subject: [PATCH 12/15] Avoid decoding successful S3 PUT responses --- e2e-test/tests/access_s3/protocol_http.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/e2e-test/tests/access_s3/protocol_http.rs b/e2e-test/tests/access_s3/protocol_http.rs index 518b731c9..f3c4178bf 100644 --- a/e2e-test/tests/access_s3/protocol_http.rs +++ b/e2e-test/tests/access_s3/protocol_http.rs @@ -128,7 +128,14 @@ async fn write_iceberg(iceberg_addr: SocketAddr, seeds: &[String]) { table.file(&format!("data/{name}.bin")).unwrap().object_key() ); let put = file_client.send(Method::PUT, &path, "", &bytes, true).await; - assert_eq!(put.status(), 200, "{}", put.text().await.unwrap()); + if put.status() != 200 { + let status = put.status(); + let body = put + .text() + .await + .unwrap_or_else(|error| format!("")); + panic!("PUT {path} returned {status}: {body}"); + } let get = file_client.send(Method::GET, &path, "", b"", false).await; assert_eq!(get.status(), 200); assert_eq!(get.bytes().await.unwrap().as_ref(), bytes); From 9d0ba0a00e4714ab800a8cfea5e76a33b50958d8 Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 15 Oct 2026 21:19:39 +0800 Subject: [PATCH 13/15] Fix native lifecycle and Iceberg grant scope --- app/crowdb-web/ui/src/shell/Sidebar.test.tsx | 13 +++++++++++++ app/crowdb-web/ui/src/shell/Sidebar.tsx | 8 +++++++- e2e-test/tests/access_s3/protocol_http.rs | 2 +- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/app/crowdb-web/ui/src/shell/Sidebar.test.tsx b/app/crowdb-web/ui/src/shell/Sidebar.test.tsx index afdc59632..fe1987463 100644 --- a/app/crowdb-web/ui/src/shell/Sidebar.test.tsx +++ b/app/crowdb-web/ui/src/shell/Sidebar.test.tsx @@ -96,6 +96,19 @@ describe('Sidebar · Cluster tree projection', () => { expect(queryByText('PKV-11')).toBeNull(); }); + it('keeps a stopped registered PKV service visible for restart actions', () => { + const stoppedRack = { + ...rack, + nodes: rack.nodes.map(node => node.id === 10 ? { ...node, kv_server: undefined } : node), + }; + const { getByText } = renderSidebar(Domain.Cluster, { + racks: [stoppedRack], + servers: [], + allServers: [{ id: 'kv-10', node_id: 10, service_type: 'paxos-kv', health: 'down' }], + }); + expect(getByText('PKV-10', { exact: true })).toBeTruthy(); + }); + it('renders assigned disk groups and disks under the owning DiskDB service', () => { const { getByTestId, getByText } = renderSidebar(Domain.Cluster); const diskdbSubtree = getByTestId('tree-node-DDB-10'); diff --git a/app/crowdb-web/ui/src/shell/Sidebar.tsx b/app/crowdb-web/ui/src/shell/Sidebar.tsx index db23c312c..9ba14f0bf 100644 --- a/app/crowdb-web/ui/src/shell/Sidebar.tsx +++ b/app/crowdb-web/ui/src/shell/Sidebar.tsx @@ -140,7 +140,13 @@ export function Sidebar({ const kinds: ServiceKind[] = plan ? [...serviceOrder] : []; if (!plan) { for (const service of nodeServices) if (isAuxiliaryKind(service.service_type)) kinds.push(service.service_type); - if (serverByNodeId.has(nodeId)) kinds.push('paxos-kv'); + // Keep a registered KV service visible after it is stopped. The + // physical node projection may omit `kv_server` while the process + // is down, but `/api/servers` still owns the deployment record and + // the restart action must remain available for that record. + if (serverByNodeId.has(nodeId) || nodeServices.some(service => service.service_type === 'paxos-kv')) { + kinds.push('paxos-kv'); + } if (diskdbNodeIds?.has(nodeId)) kinds.push('diskdb'); } for (const kind of [...new Set(kinds)]) { diff --git a/e2e-test/tests/access_s3/protocol_http.rs b/e2e-test/tests/access_s3/protocol_http.rs index f3c4178bf..0a93fd258 100644 --- a/e2e-test/tests/access_s3/protocol_http.rs +++ b/e2e-test/tests/access_s3/protocol_http.rs @@ -99,7 +99,7 @@ async fn write_iceberg(iceberg_addr: SocketAddr, seeds: &[String]) { let context = repository.status().await.unwrap().0.context; let credentials = issuer .issue(FileGrant { - storage_namespace: None, + storage_namespace: table.storage_namespace, context, table: table.table, principal: [7; 32], From 5a66ab2b4b62d30e739412dfd4c55a98e1bd47c2 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 16 Oct 2026 00:32:51 +0800 Subject: [PATCH 14/15] Require complete storage ownership before chunk startup --- app/crowdb-web/src/services/dependencies.rs | 15 ++++++ .../tests/storage_readiness_test.rs | 5 ++ app/crowdb-web/ui/src/views/S3View.tsx | 2 +- .../e2e/fixtures/defaultDeployments.ts | 23 ++++---- .../console-ui/e2e/flows/71-s3-native.spec.ts | 2 +- .../tests/access_s3/node_failure/recovery.rs | 52 +++++++++++++++---- 6 files changed, 74 insertions(+), 25 deletions(-) diff --git a/app/crowdb-web/src/services/dependencies.rs b/app/crowdb-web/src/services/dependencies.rs index ee7894402..4c626f57d 100644 --- a/app/crowdb-web/src/services/dependencies.rs +++ b/app/crowdb-web/src/services/dependencies.rs @@ -56,6 +56,21 @@ pub(super) async fn storage_wait_reason(state: &AppState) -> Option { if diskio.is_empty() || groups.is_empty() { return Some("Waiting: configure disk groups and publish live DiskIO ownership".into()); } + let Ok(diskdb_instances) = registry.read_all_diskdb_instances().await else { + return Some("Waiting: live DiskDB ownership registry is unavailable".into()); + }; + let mut diskdb_owned = HashSet::new(); + for (_, instance) in diskdb_instances { + if instance.rpc_endpoint.is_empty() { + continue; + } + if let Some(ownership) = instance.extra.and_then(|extra| extra.diskdb) { + diskdb_owned.extend(ownership.owned_dg_ids); + } + } + if groups.iter().any(|group| !diskdb_owned.contains(&group.dg_id)) { + return Some("Waiting: every configured disk group needs live DiskDB ownership".into()); + } let mut owned = HashSet::new(); for server in diskio { let instance = instances.iter().find(|(_, value)| { diff --git a/app/crowdb-web/tests/storage_readiness_test.rs b/app/crowdb-web/tests/storage_readiness_test.rs index 3e061d16d..daa64ec3d 100644 --- a/app/crowdb-web/tests/storage_readiness_test.rs +++ b/app/crowdb-web/tests/storage_readiness_test.rs @@ -73,6 +73,11 @@ async fn chunk_storage_requires_live_complete_ownership_even_when_diskio_pid_is_ .heartbeat_diskio(1, endpoint, &[11, 12], &[]) .await .unwrap(); + assert_eq!(readiness(&app).await["ready"], false); + registry + .heartbeat_diskdb(1, "127.0.0.1:14010", &[11, 12], &[]) + .await + .unwrap(); assert_eq!(readiness(&app).await["ready"], true); registry.unregister("diskio", 1).await.unwrap(); assert_eq!(readiness(&app).await["ready"], false); diff --git a/app/crowdb-web/ui/src/views/S3View.tsx b/app/crowdb-web/ui/src/views/S3View.tsx index aaf3d31d2..cf0c0250a 100644 --- a/app/crowdb-web/ui/src/views/S3View.tsx +++ b/app/crowdb-web/ui/src/views/S3View.tsx @@ -122,7 +122,7 @@ export function S3View({ active, readonly, onChunk }: { active: boolean; readonl const response = await request('HEAD', objectPath(bucket, row.key)); setDetail(Object.fromEntries(response.headers.entries())); }, `Inspected ${row.key}`); }; - useEffect(() => { if (origin) void run(listBuckets, 'Buckets refreshed'); }, [origin]); + useEffect(() => { if (active && origin) void run(listBuckets, 'Buckets refreshed'); }, [active, origin]); useNavigationSnapshot(Domain.S3, 'object-query', () => { const state = { bucket, bucketStart, bucketFilter, prefix, listedPrefix, cursor, previous: [...previous], selected: selected ? { ...selected } : null, diff --git a/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts b/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts index 787cc6e2a..930e6cc4c 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts +++ b/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts @@ -21,29 +21,24 @@ export function observeDefaultDeployments(page: Page) { }); return { async nextCompleted(nodes: number[], kind: string) { + const pending = new Set(nodes.map(node => `${node}:${kind}`)); const expectedWait = (response: Response) => response.status() === 409 && ['chunkdb', 'chunk-kv'].includes(kind); const rejected = nodes.flatMap(node => replies.get(`${node}:${kind}`) ?? []) .find(response => response.status() !== 201 && !expectedWait(response)); if (rejected) expect(rejected.status(), await rejected.text()).toBe(201); const completed = nodes.find(node => replies.get(`${node}:${kind}`)?.some(response => response.status() === 201)); if (completed !== undefined) return completed; - // The deploy request can be queued behind disk recovery, so its response - // is not a useful readiness signal. Follow the durable service plan and - // let the enclosing Playwright test budget report a genuine hang. - for (;;) { - const plans = await page.evaluate(async () => { - const response = await fetch('/api/service-plans'); - return response.json(); - }); - const ready = nodes.find(node => plans[node]?.steps?.[kind]?.state === 'deployed'); - if (ready !== undefined) return ready; - await new Promise(resolve => setTimeout(resolve, 100)); - } + const timeout = process.platform === 'darwin' ? 30_000 : 10_000; + const response = await page.waitForResponse(response => pending.has(key(response) ?? '') + && (response.status() === 201 || !expectedWait(response)), { timeout }); + expect(response.status(), await response.text()).toBe(201); + return Number(key(response)!.split(':')[0]); }, async verify(node: number, kind: string) { const identity = `${node}:${kind}`; - const response = replies.get(identity)?.find(response => response.status() === 201); - if (!response) return; + const timeout = process.platform === 'darwin' ? 30_000 : 10_000; + const response = replies.get(identity)?.find(response => response.status() === 201) + ?? await page.waitForResponse(response => key(response) === identity && response.status() === 201, { timeout }); for (const rejected of replies.get(identity) ?? []) { if (rejected === response) break; const body = await rejected.json(); diff --git a/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts b/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts index c9486fb68..945e21898 100644 --- a/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts @@ -25,7 +25,7 @@ test('S3 native multipart upload, HEAD, bounded preview and full round trip', as await page.goto('/?domain=S3'); await expect(page.getByLabel('Access key', { exact: true })).toHaveCount(0); const bucketButton = page.getByRole('navigation', { name: 'S3 buckets' }).getByRole('button', { name: bucket, exact: true }); - await expect(bucketButton).toBeVisible({ timeout: 10_000 }); + await expect(bucketButton).toBeVisible(); await bucketButton.click(); await page.getByText('Bucket actions', { exact: true }).click(); await page.getByLabel('Object key', { exact: true }).fill(key); diff --git a/e2e-test/tests/access_s3/node_failure/recovery.rs b/e2e-test/tests/access_s3/node_failure/recovery.rs index 81b4eed16..d623ebd48 100644 --- a/e2e-test/tests/access_s3/node_failure/recovery.rs +++ b/e2e-test/tests/access_s3/node_failure/recovery.rs @@ -5,6 +5,7 @@ use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport}; use crowdb_kv_client::{HardwareClient, RangeBindingClient, ServiceRegistryClient}; use crowdb_protocol::chunkdb::rpc::{ChunkType, ListChunksRequest, QueryChunkRequest, Strip, StripType}; use crowdb_protocol::common::{ChunkId, HwStatus}; +use std::collections::HashSet; use std::sync::Arc; use std::time::Duration; @@ -106,26 +107,59 @@ async fn ready(http: &reqwest::Client, endpoint: &str) { } pub(super) async fn storage_ready() { + let registry = ServiceRegistryClient::new(super::kv_client()); + let hardware = HardwareClient::new(super::kv_client()); let deadline = tokio::time::Instant::now() + Duration::from_secs(15); loop { let result = crowdb_diskio_client::DiskioClient::connect_with_clients( - ServiceRegistryClient::new(super::kv_client()), - HardwareClient::new(super::kv_client()), + registry.clone(), + hardware.clone(), crowdb_diskio_client::DiskioClientConfig::default(), ) .await; - match result { - Ok(client) if client.status().disks == 3 => return, - result => assert!( - tokio::time::Instant::now() < deadline, - "restarted DiskIO owners must publish all three disks: {:?}", - result.as_ref().map(|client| client.status().disks) - ), + let diskdb = diskdb_missing(®istry, &hardware).await; + if matches!(&result, Ok(client) if client.status().disks == 3) + && matches!(&diskdb, Ok(missing) if missing.is_empty()) + { + return; } + assert!( + tokio::time::Instant::now() < deadline, + "restarted storage topology is not ready: DiskIO={:?}, DiskDB={diskdb:?}", + result.as_ref().map(|client| client.status().disks) + ); tokio::time::sleep(Duration::from_millis(100)).await; } } +async fn diskdb_missing( + registry: &ServiceRegistryClient, + hardware: &HardwareClient, +) -> Result, String> { + let instances = registry + .read_all_diskdb_instances() + .await + .map_err(|error| format!("registry: {error}"))?; + let groups = hardware + .list_disk_groups() + .await + .map_err(|error| format!("hardware: {error}"))?; + let mut owned = HashSet::new(); + for (_, instance) in instances { + if instance.rpc_endpoint.is_empty() { + continue; + } + if let Some(diskdb) = instance.extra.and_then(|extra| extra.diskdb) { + owned.extend(diskdb.owned_dg_ids); + } + } + Ok(groups + .iter() + .map(|group| group.dg_id) + .filter(|group| !owned.contains(group)) + .collect()) +} + pub(super) async fn repaired(chunks: &[ChunkId]) { let kv = Arc::new(super::kv_client()); let client = ChunkdbClient::new( From 68a4c5508bd8fc9a64270d524f26556c5fff2ff6 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 16 Oct 2026 07:10:23 +0800 Subject: [PATCH 15/15] Keep service lifecycle menus available during authority recovery --- app/crowdb-web/src/services/dependencies.rs | 15 --------------- app/crowdb-web/tests/storage_readiness_test.rs | 5 ----- app/crowdb-web/ui/src/menus/useClusterMenus.tsx | 7 ++++++- 3 files changed, 6 insertions(+), 21 deletions(-) diff --git a/app/crowdb-web/src/services/dependencies.rs b/app/crowdb-web/src/services/dependencies.rs index 4c626f57d..ee7894402 100644 --- a/app/crowdb-web/src/services/dependencies.rs +++ b/app/crowdb-web/src/services/dependencies.rs @@ -56,21 +56,6 @@ pub(super) async fn storage_wait_reason(state: &AppState) -> Option { if diskio.is_empty() || groups.is_empty() { return Some("Waiting: configure disk groups and publish live DiskIO ownership".into()); } - let Ok(diskdb_instances) = registry.read_all_diskdb_instances().await else { - return Some("Waiting: live DiskDB ownership registry is unavailable".into()); - }; - let mut diskdb_owned = HashSet::new(); - for (_, instance) in diskdb_instances { - if instance.rpc_endpoint.is_empty() { - continue; - } - if let Some(ownership) = instance.extra.and_then(|extra| extra.diskdb) { - diskdb_owned.extend(ownership.owned_dg_ids); - } - } - if groups.iter().any(|group| !diskdb_owned.contains(&group.dg_id)) { - return Some("Waiting: every configured disk group needs live DiskDB ownership".into()); - } let mut owned = HashSet::new(); for server in diskio { let instance = instances.iter().find(|(_, value)| { diff --git a/app/crowdb-web/tests/storage_readiness_test.rs b/app/crowdb-web/tests/storage_readiness_test.rs index daa64ec3d..3e061d16d 100644 --- a/app/crowdb-web/tests/storage_readiness_test.rs +++ b/app/crowdb-web/tests/storage_readiness_test.rs @@ -73,11 +73,6 @@ async fn chunk_storage_requires_live_complete_ownership_even_when_diskio_pid_is_ .heartbeat_diskio(1, endpoint, &[11, 12], &[]) .await .unwrap(); - assert_eq!(readiness(&app).await["ready"], false); - registry - .heartbeat_diskdb(1, "127.0.0.1:14010", &[11, 12], &[]) - .await - .unwrap(); assert_eq!(readiness(&app).await["ready"], true); registry.unregister("diskio", 1).await.unwrap(); assert_eq!(readiness(&app).await["ready"], false); diff --git a/app/crowdb-web/ui/src/menus/useClusterMenus.tsx b/app/crowdb-web/ui/src/menus/useClusterMenus.tsx index aa217811a..e0bd3e815 100644 --- a/app/crowdb-web/ui/src/menus/useClusterMenus.tsx +++ b/app/crowdb-web/ui/src/menus/useClusterMenus.tsx @@ -14,7 +14,12 @@ export function useClusterMenus({ readonly, managed, managementAuthorized, domai /** Build per-layer context menu items for a normalized target. */ return useCallback( (t: MenuTarget): MenuItemOrSeparator[] => { - if (readonly || managed || !managementAuthorized) return []; + // Existing service lifecycle actions must remain available while the + // authority probe is down. Stopping a KV service makes that probe + // unavailable, but restart/stop/delete are exactly the recovery actions + // needed to bring the authority back. + const isServiceTarget = t.type === 'Server' && t.serviceType != null; + if (readonly || managed || (!managementAuthorized && !isServiceTarget)) return []; const items: MenuItemOrSeparator[] = []; const p = t.parentIds || {}; if (physicalActive && t.type === 'Server' && isAuxiliaryKind(t.serviceType)) {