diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..97a08c9 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,24 @@ +# Dependabot keeps dependencies current: one pull request a week per ecosystem with minor and patch updates, and one +# per major update. A new version is proposed once it's 7 days old (security updates at once). +version: 2 + +updates: + - package-ecosystem: github-actions + directory: / + schedule: { interval: weekly } + cooldown: { default-days: 7 } + groups: + actions: { update-types: [minor, patch] } + + - package-ecosystem: cargo + directory: / + schedule: { interval: weekly } + cooldown: { default-days: 7 } + groups: + cargo: { update-types: [minor, patch] } + ignore: + # iceberg (pinned with =, a test dependency) fixes the Arrow and Parquet majors: move them with it, by hand. + - dependency-name: "arrow-*" + update-types: [version-update:semver-major] + - dependency-name: parquet + update-types: [version-update:semver-major] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..99d007e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,54 @@ +# Checks every push to main and every pull request. Needs no secrets. +name: CI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.event_name == 'pull_request' && github.ref || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 + +jobs: + native: + name: Native (format, lint, tests) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: rustup toolchain install # from rust-toolchain.toml + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + - run: cargo fmt --all --check + - run: cargo clippy --workspace --all-features --all-targets -- -D warnings + - run: cargo test --workspace --all-features + # Each feature builds on its own, and with none, without warnings. + - name: Features one at a time + env: + RUSTFLAGS: -D warnings + run: | + cargo check -p lakebed-iceberg --no-default-features + cargo check -p lakebed-iceberg --features write + cargo check -p lakebed-iceberg --features arrow + cargo check -p lakebed-iceberg --features parquet-stats + + wasm: + name: WebAssembly (browser and WASI targets) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: rustup toolchain install # from rust-toolchain.toml + - run: rustup target add wasm32-unknown-unknown wasm32-wasip2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + - run: cargo clippy --workspace --all-features --target wasm32-unknown-unknown -- -D warnings + - run: cargo build --workspace --all-features --target wasm32-wasip2 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2c393a2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/target +.env* +.DS_Store diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..409e128 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,38 @@ +# Git hooks, run by prek (https://prek.j178.dev): `uv tool install prek && prek install --install-hooks`. +# On commit: formatters and fast checks on the staged files. On push: clippy, natively and for wasm32. +# Everything: `prek run --all-files --hook-stage pre-push`. CI runs the same checks plus the tests. +default_install_hook_types: [pre-commit, pre-push] + +repos: + - repo: builtin + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-merge-conflict + - id: check-yaml + - id: check-toml + - id: check-added-large-files + args: [--maxkb=1024] + - id: detect-private-key + + - repo: local + hooks: + - id: rustfmt + name: rustfmt + entry: rustfmt --edition 2024 + language: system + types: [rust] + - id: clippy + name: clippy + entry: cargo clippy --workspace --all-features --all-targets --quiet -- -D warnings + language: system + types: [rust] + pass_filenames: false + stages: [pre-push] + - id: clippy-wasm + name: clippy (wasm32) + entry: cargo clippy --workspace --all-features --target wasm32-unknown-unknown --quiet -- -D warnings + language: system + types: [rust] + pass_filenames: false + stages: [pre-push] diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..5a7647d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,30 @@ +# Agent instructions + +lakebed: table formats (Apache Iceberg first) read and written in Rust without I/O of their own, for WebAssembly and +native targets. One crate per format (`crates/lakebed-`) on a shared I/O boundary (`crates/lakebed-core`). +Timelake and lakeglass use it. + +## Before changing code + +- Follow [CONTRIBUTING.md](CONTRIBUTING.md). It is the code guide, and it applies to you. +- For anything beyond a small fix, check the specs ([spec/README.md](spec/README.md)). Agree on the design with the user + before building something new, and record the decision in the spec's Decisions table. + +## Rules + +- **Done means checked.** Before you report work as finished, run `prek run --all-files --hook-stage pre-push` and + `cargo test --workspace --all-features`. Report failures as failures. +- **Fix the code, not the checks.** Don't loosen lint or format configs. Don't add `#[allow]`. Use + `#[expect(..., reason = "...")]` only when the tool is genuinely wrong. +- **Keep things in sync:** a change in behaviour updates its spec (status, design, history), the tests, and + `CHANGELOG.md` (under `Unreleased`) in the same commit. +- **Everything builds for WASM.** Every crate builds for `wasm32-unknown-unknown`: no threads, no blocking I/O, no + `std::time`, no C dependencies. All I/O goes through the `Host` trait. CI checks it. +- **Readers don't trust files.** Bytes from a store can be anything: check lengths, return errors, never panic. +- **Interop is tested.** What a format crate writes must read in that format's reference implementation and the other + way round; the tests show it. +- **Every behaviour a caller might want to change is an option**, named for its mechanism. +- **Ideas, not code.** Never copy or translate another project's code. Credit ideas in `PRIOR_ART.md` + (CONTRIBUTING.md, "Credit"). +- **Tests run offline.** No network, cloud accounts, or real buckets in tests. +- **Never commit secrets** or `.env*` files. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..6905974 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,21 @@ +# Changelog + +All notable changes to lakebed. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and +versions follow [Semantic Versioning](https://semver.org/). + +## Unreleased + +### Added + +- `lakebed-core`: the `Host` trait for object stores, clocks, and randomness; `MemoryHost`; one `Error` type. +- `lakebed-iceberg`: Iceberg table metadata, schemas, partition specs, manifest lists, and manifests read for format + versions 1 to 3, including delete files and deletion vector references; values in Iceberg's single-value encoding. +- `lakebed-iceberg` feature `write`: format v2 manifests, manifest lists, and metadata updates, and commits to + file-system tables with options (`CommitOptions`) and hooks (`CommitHooks`). A writer whose clock is behind the + table's last update dates its commit at that update, so the snapshot log stays in order. +- `lakebed-iceberg` features `arrow` (schemas as Arrow schemas with field ids) and `parquet-stats` (a data file's + Iceberg statistics from its Parquet footer; a bound or null count only when every row group gives it exactly, and + decimals as Iceberg's shortest big-endian bytes). +- `lakebed`: re-exports the format crates behind a feature each. + +The Iceberg code started as Timelake's shared writer (`timelake-format`), made general. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..43c994c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..c3fbea5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,85 @@ +# Contributing to lakebed + +This is the code guide for everyone who changes this repository, people and AI agents alike. Formatters and linters +enforce the mechanical rules (see [Checks](#checks)). This guide covers what they can't: how code should read. +[spec/](spec/README.md) holds the design. + +## Principles + +1. **Readable first.** Someone new should understand a function from its name, its signature, and one pass through + the body. +2. **One crate per format.** A format's crate depends on `lakebed-core` and on nothing of another format's, so adding + one never touches the others. Formats are shown as they are: no shared model flattens their differences. +3. **No I/O of our own.** Crates read and write through `lakebed_core::Host`. They never open sockets, files, or + threads, so they build for `wasm32-unknown-unknown`. +4. **One definition of each thing.** Before writing a helper, constant, or type, look for an existing one. +5. **Behaviour, spec, and tests change together.** A change users can see updates its spec, adds or updates tests, and + adds a line to [CHANGELOG.md](CHANGELOG.md). +6. **Behaviour is configurable.** Where callers could reasonably want something else (retries, limits, what to write), + it is an option with a default, named for its mechanism (`merge_manifests_at`, not `fast`). + +## Comments and docs + +- **Public items get a short doc comment**: what it does, and anything a caller must know (errors, units, + invariants). One to three lines is usually enough. +- **Comments explain why, not what.** Explain a non-obvious choice, a workaround, or a rule from a format's spec, with + a link to the spec section. +- **No long comments.** If an explanation needs a page, it belongs in a spec. Link to it from the code. + +## Credit + +lakebed learns from other projects' ideas, never from copying their code. [PRIOR_ART.md](PRIOR_ART.md) lists the +ideas we took. + +- **Never copy code**, whatever its license: not by pasting, and not by translating it into another language. +- **Credit ideas from open-source projects** with a row in PRIOR_ART.md and a short comment where the idea is + implemented. +- **Don't name open-core, dual-licensed, or source-available projects, or commercial products, as a source.** + +## Rust + +- **Errors:** `lakebed_core::Error`, with the variant that says what a caller can do about it (`NotFound`, `Conflict`, + `CommitUnknown`, `Unsupported`, `Corrupt`). No `unwrap()` outside tests. +- **Types over strings:** enums and newtypes instead of strings and magic numbers. Parse once at the boundary. +- **Don't trust files.** Check lengths and offsets before using them, cap allocations, and return an error instead of + panicking on a malformed file. Tests feed readers truncated and garbage input. +- **Async:** host futures aren't `Send` (JavaScript promises aren't). Never block inside async code. +- **Suppressing a lint:** `#[expect(lint, reason = "...")]` with a real reason. Never `#[allow]`. + +## Tests + +- **A bug fix starts with a test that fails**, then the fix. +- **Tests run offline**, against `MemoryHost` or a host over the reference implementation's in-memory storage. +- **Interop:** writers are tested against the format's reference reader, and readers against the reference writer. + +## Checks + +Install once: `uv tool install prek && prek install --install-hooks`. Hooks then run on commit and on push. Before you +push, or before calling work done: + +```bash +prek run --all-files --hook-stage pre-push # rustfmt, clippy (native and wasm32) +cargo test --workspace --all-features # unit, doc, and interop tests +``` + +## Dependencies + +Keep dependencies on their newest versions; check the latest release when adding one. Arrow and Parquet follow the +major version iceberg-rust's newest release uses (the interop tests build both). Actions are pinned to a commit with +the version in a comment (`uses: actions/checkout@ # v7.0.1`). + +## Commits + +- **Subject:** a short imperative sentence with the area first, for example `Core: …`, `Iceberg: …`, or `Spec: …`. + Add a body when the why isn't obvious. +- **Scope:** one logical change per commit. + +## Changelog + +[CHANGELOG.md](CHANGELOG.md) is for users, not a list of commits. Anything a user can notice gets one line under +`## Unreleased`, in `Added`, `Changed`, `Deprecated`, `Removed`, `Fixed`, or `Security`, in the same commit as the +change. + +## License + +Contributions are licensed under the Apache License 2.0, like the rest of the project (see [LICENSE](LICENSE)). diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..963b662 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,3309 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "apache-avro" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" +dependencies = [ + "bigdecimal", + "bon", + "crc32fast", + "digest 0.10.7", + "log", + "miniz_oxide 0.8.9", + "num-bigint", + "quad-rand", + "rand 0.9.5", + "regex-lite", + "serde", + "serde_bytes", + "serde_json", + "snap", + "strum 0.27.2", + "strum_macros 0.27.2", + "thiserror", + "uuid", + "zstd", +] + +[[package]] +name = "apache-avro" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c" +dependencies = [ + "bigdecimal", + "bon", + "digest 0.11.3", + "log", + "miniz_oxide 0.9.1", + "num-bigint", + "ouroboros", + "quad-rand", + "rand 0.10.3", + "regex-lite", + "serde", + "serde_bytes", + "serde_json", + "strum 0.28.0", + "thiserror", + "uuid", +] + +[[package]] +name = "array-init" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" + +[[package]] +name = "arrow-arith" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64 0.22.1", + "chrono", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-data" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ipc" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", +] + +[[package]] +name = "arrow-ord" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-schema" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" + +[[package]] +name = "arrow-select" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "as-any" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063" + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bigdecimal" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fa3f3d8cbf4dffcfe4991de61d012bef509a409ecbe9dd41049bfe32b4d4653" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "bimap" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bnum" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f781dba93de3a5ef6dc5b17c9958b208f6f3f021623b360fb605ea51ce443f10" +dependencies = [ + "serde", + "serde-big-array", +] + +[[package]] +name = "bon" +version = "3.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "214f5df094ce551a10a30ffb9c70243d61f121a3d985a6495933e181dee6a7d2" +dependencies = [ + "bon-macros", +] + +[[package]] +name = "bon-macros" +version = "3.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2706da6c749998cc555d04184909608956a69ad3f8ab1a076fbc867b4a3c3ce5" +dependencies = [ + "darling 0.24.1", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877bf67ac8e52449eb2f9e98a344ea163beb70dd1f9eebe3cc5357dcfeaee4df" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core 0.24.1", + "darling_macro 0.24.1", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.9", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core 0.24.1", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt-parser" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "dissimilar" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeda16ab4059c5fd2a83f2b9c9e9c981327b18aa8e3b313f7e6563799d4f093e" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "expect-test" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63af43ff4431e848fb47472a920f14fa71c24de13255a5692e93d4e90302acb0" +dependencies = [ + "dissimilar", + "once_cell", +] + +[[package]] +name = "fastnum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020d1b59a944bc239d79903fbac2eda2365138b44890c27979562f6592059dcd" +dependencies = [ + "bnum", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags 2.13.2", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide 0.9.1", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", +] + +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "iceberg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e50fb53c7480f414911ab76b6e14c0042d54ad5c30f6ac2bf2f52a487a25716" +dependencies = [ + "aes-gcm", + "anyhow", + "apache-avro 0.21.0", + "array-init", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "arrow-string", + "as-any", + "async-trait", + "backon", + "base64 0.22.1", + "bimap", + "bytes", + "chrono", + "derive_builder", + "expect-test", + "fastnum", + "flate2", + "fnv", + "futures", + "itertools", + "moka", + "murmur3", + "once_cell", + "ordered-float 4.6.0", + "parquet", + "rand 0.9.5", + "reqwest", + "roaring", + "serde", + "serde_bytes", + "serde_derive", + "serde_json", + "serde_repr", + "serde_with", + "strum 0.27.2", + "tokio", + "tracing", + "typed-builder", + "typetag", + "url", + "uuid", + "zeroize", + "zstd", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + +[[package]] +name = "inventory" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c" +dependencies = [ + "rustversion", +] + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lakebed" +version = "0.1.0" +dependencies = [ + "lakebed-core", + "lakebed-iceberg", +] + +[[package]] +name = "lakebed-core" +version = "0.1.0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "lakebed-iceberg" +version = "0.1.0" +dependencies = [ + "apache-avro 0.22.0", + "arrow-array", + "arrow-schema", + "bytes", + "futures", + "futures-util", + "iceberg", + "lakebed-core", + "parquet", + "serde_json", + "tokio", +] + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "murmur3" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ouroboros" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "parquet" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64 0.22.1", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.17.1", + "lz4_flex", + "num-bigint", + "num-integer", + "num-traits", + "paste", + "ring", + "seq-macro", + "simdutf8", + "snap", + "thrift", + "tokio", + "twox-hash", + "zstd", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" +dependencies = [ + "proc-macro2", + "syn 3.0.9", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "version_check", + "yansi", +] + +[[package]] +name = "quad-rand" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "roaring" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1" +dependencies = [ + "bytemuck", + "byteorder", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9adc193c780ef8f159aee8b61e2d5801aaa555e6eb0947fe45530ec506296f" +dependencies = [ + "base64 0.23.1", + "bs58", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.2", + "jiff", + "schemars 0.9.0", + "schemars 1.2.3", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49" +dependencies = [ + "darling 0.24.1", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "snap" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros 0.27.2", +] + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros 0.28.0", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "ordered-float 2.10.1", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokio" +version = "1.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c168a474b95bfecfe908a65d4bdffc523144b649a74c09c286afb35d53e0bc" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.13.2", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "twox-hash" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2" + +[[package]] +name = "typed-builder" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd9d30e3a08026c78f246b173243cf07b3696d274debd26680773b6773c2afc7" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "typetag" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" +dependencies = [ + "js-sys", + "tokio", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.2+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..c32eb9e --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,43 @@ +[workspace] +resolver = "2" +members = ["crates/lakebed-core", "crates/lakebed-iceberg", "crates/lakebed"] + +[workspace.package] +version = "0.1.0" +edition = "2024" +rust-version = "1.99" +license = "Apache-2.0" +repository = "https://github.com/polythings/lakebed" +keywords = ["lakehouse", "iceberg", "wasm", "table-format"] +categories = ["database-implementations", "wasm"] + +[workspace.dependencies] +lakebed-core = { path = "crates/lakebed-core", version = "0.1.0" } +lakebed-iceberg = { path = "crates/lakebed-iceberg", version = "0.1.0", default-features = false } +thiserror = "2" +serde_json = "1" +apache-avro = { version = "0.22", default-features = false } +futures-util = { version = "0.3", default-features = false, features = ["alloc"] } +# Arrow and Parquet stay on the major that iceberg-rust's newest release (0.10.1) uses, so applications using both +# (Timelake's engine) and the interop tests build one copy of each. +arrow-schema = "58" +parquet = { version = "58", default-features = false } +iceberg = "=0.10.1" +tokio = { version = "1", features = ["macros", "rt"] } + +[workspace.lints.rust] +unsafe_code = "forbid" +unreachable_pub = "warn" +missing_debug_implementations = "warn" +missing_docs = "warn" +rust_2018_idioms = { level = "warn", priority = -1 } + +[workspace.lints.clippy] +pedantic = { level = "warn", priority = -1 } +missing_errors_doc = "allow" +missing_panics_doc = "allow" +must_use_candidate = "allow" +allow_attributes = "warn" +dbg_macro = "warn" +todo = "warn" +unwrap_used = "warn" diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..0ce932b --- /dev/null +++ b/NOTICE @@ -0,0 +1,6 @@ +lakebed +Copyright 2026 Oleksandr Litus and the lakebed contributors + +This product is licensed under the Apache License, Version 2.0 (see LICENSE). + +Ideas lakebed borrowed from other projects are credited in PRIOR_ART.md. diff --git a/PRIOR_ART.md b/PRIOR_ART.md new file mode 100644 index 0000000..7622547 --- /dev/null +++ b/PRIOR_ART.md @@ -0,0 +1,22 @@ +# Prior art + +lakebed stands on other open-source projects and learns from more. This file credits the ideas we took. We take ideas, +never code: every line of lakebed is our own. How we credit is in [CONTRIBUTING.md](CONTRIBUTING.md#credit). + +## Foundations + +| Project | License | What lakebed uses | +|---|---|---| +| [Apache Iceberg](https://iceberg.apache.org) | Apache-2.0 | The table format and its specification | +| [iceberg-rust](https://github.com/apache/iceberg-rust) | Apache-2.0 | The reference the interop tests check against (a test dependency only) | +| [Apache Avro](https://avro.apache.org) (`apache-avro`) | Apache-2.0 | Reading and writing manifests and manifest lists | +| [arrow-rs](https://github.com/apache/arrow-rs) (`arrow-schema`, `parquet`) | Apache-2.0 | Arrow schemas; Parquet footers for statistics | + +## Ideas + +| Idea | From | License of the source | Where in lakebed | +|---|---|---|---| +| A format library with no I/O of its own: the application supplies storage through a trait the library calls | [delta-kernel-rs](https://github.com/delta-io/delta-kernel-rs) (its `Engine` trait) | Apache-2.0 | `lakebed-core`'s `Host` | +| One crate per backend under a small shared core, re-exported by an umbrella crate with a feature each | [arrow-rs](https://github.com/apache/arrow-rs), [Apache OpenDAL](https://github.com/apache/opendal) | Apache-2.0 | The workspace layout | +| Commits to tables without a catalog: `v.metadata.json` written only if absent, `version-hint.text` for readers | Iceberg's file-system ("Hadoop") tables | Apache-2.0 | `lakebed_iceberg::commit` | +| Avro-safe names for partition fields (`_x` and the character's code in hex) | Iceberg's Java implementation | Apache-2.0 | Manifest writing | diff --git a/README.md b/README.md new file mode 100644 index 0000000..c4a52da --- /dev/null +++ b/README.md @@ -0,0 +1,56 @@ +# lakebed + +Table formats read and written without I/O of their own, in Rust, for WebAssembly and native targets. Each format's +metadata (Apache Iceberg first) is parsed and written by plain code that asks a `Host` for bytes: the application +implements the host over its object store, so the same crate runs in a browser, a Cloudflare Worker, a CLI, or a +server. + +lakebed is what [Timelake](https://github.com/polythings/timelake) and [lakeglass](https://github.com/polythings/lakeglass) +share. The official Rust crates for these formats build on tokio and HTTP clients, which don't compile to +`wasm32-unknown-unknown`. + +## Crates + +| Crate | What it does | +|---|---| +| [`lakebed-core`](crates/lakebed-core) | The `Host` trait (ranged reads, listing, conditional writes, clock, randomness), errors, an in-memory host | +| [`lakebed-iceberg`](crates/lakebed-iceberg) | Apache Iceberg: metadata, manifests, and commits | +| [`lakebed`](crates/lakebed) | Re-exports the format crates, one feature each | + +More formats get their own crate as they are added (spec 2). + +## Iceberg + +| Part | Status | +|---|---| +| Table metadata, format v1 to v3: schemas, partition specs, snapshots, properties | Read | +| Manifest lists and manifests: data files, delete files, deletion vector references, column statistics | Read | +| Format v2 manifests, manifest lists, and metadata updates | Write (feature `write`) | +| Commits to file-system tables (`v.metadata.json` with a conditional put), with hooks | Write (feature `write`) | +| Schema as an Arrow schema with field ids | Feature `arrow` | +| A data file's Iceberg statistics from its Parquet footer | Feature `parquet-stats` | + +What lakebed writes, iceberg-rust reads, and the other way round: the tests commit to one table with both in turn. + +```rust +use lakebed_iceberg::Table; + +async fn files(host: &H) -> lakebed_core::Result { + let mut table = Table::load(host, "s3://bucket/warehouse/db/events").await?; + Ok(table.live_files(host).await?.len()) +} +``` + +## Development + +```bash +cargo test --workspace --all-features # tests, offline +cargo build --workspace --all-features --target wasm32-unknown-unknown # the WebAssembly build +prek run --all-files --hook-stage pre-push # formatters and linters +``` + +[CONTRIBUTING.md](CONTRIBUTING.md) is the code guide, and [spec/](spec/README.md) holds the design. + +## License + +Apache License 2.0 ([LICENSE](LICENSE)). diff --git a/clippy.toml b/clippy.toml new file mode 100644 index 0000000..731d87f --- /dev/null +++ b/clippy.toml @@ -0,0 +1,2 @@ +allow-unwrap-in-tests = true +doc-valid-idents = ["WebAssembly", ".."] diff --git a/crates/lakebed-core/Cargo.toml b/crates/lakebed-core/Cargo.toml new file mode 100644 index 0000000..b414a77 --- /dev/null +++ b/crates/lakebed-core/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "lakebed-core" +description = "The I/O boundary shared by lakebed's table format crates: a host trait for object stores, clocks, and randomness" +readme = "README.md" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true + +[dependencies] +thiserror.workspace = true + +[lints] +workspace = true diff --git a/crates/lakebed-core/README.md b/crates/lakebed-core/README.md new file mode 100644 index 0000000..9d74b43 --- /dev/null +++ b/crates/lakebed-core/README.md @@ -0,0 +1,12 @@ +# lakebed-core + +The I/O boundary of [lakebed](https://github.com/polythings/lakebed)'s table format crates. Format crates do no I/O of +their own: they call a `Host` that the application implements over its object store, clock, and randomness, so they +build for `wasm32-unknown-unknown` and native targets alike. + +- `Host`: get, ranged get, exists, put, put-if-absent (the commit primitive), multipart upload, list, delete, the time, + random numbers, and waiting. Its futures aren't `Send`, so a host can wrap JavaScript promises. +- `MemoryHost`: a host in memory with a fixed clock and seeded randomness, for tests. +- `Error`: one error type, whose variant says what a caller can do (`NotFound`, `Conflict`, `CommitUnknown`, …). + +Apache License 2.0. diff --git a/crates/lakebed-core/src/error.rs b/crates/lakebed-core/src/error.rs new file mode 100644 index 0000000..662fd96 --- /dev/null +++ b/crates/lakebed-core/src/error.rs @@ -0,0 +1,30 @@ +/// An error reading or writing a table. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + /// The request doesn't fit the table, or a value is out of range. + #[error("invalid: {0}")] + Invalid(String), + /// The table uses something this version can read but not write, or not read at all. + #[error("unsupported: {0}")] + Unsupported(String), + /// Another writer changed the table in a way this commit can't build on. + #[error("conflict: {0}")] + Conflict(String), + /// A commit whose outcome can't be told: its metadata may be in the table. Nothing it wrote + /// may be deleted; load the table again to see whether it committed. + #[error("commit outcome unknown: {0}")] + CommitUnknown(String), + /// An object the table needs is missing. + #[error("not found: {0}")] + NotFound(String), + /// The store failed. + #[error("store: {0}")] + Store(String), + /// A file isn't what its format says it should be. + #[error("unexpected file contents: {0}")] + Corrupt(String), +} + +/// `Result` with [`Error`]. +pub type Result = std::result::Result; diff --git a/crates/lakebed-core/src/host.rs b/crates/lakebed-core/src/host.rs new file mode 100644 index 0000000..4151c36 --- /dev/null +++ b/crates/lakebed-core/src/host.rs @@ -0,0 +1,55 @@ +use std::future::Future; + +use crate::Result; + +/// An object found by [`Host::list`]. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Listed { + /// Absolute URI. + pub uri: String, + /// Size in bytes. + pub size: u64, +} + +/// What a table format needs from where it runs: an object store, a clock, randomness, and a way +/// to wait. Paths are absolute URIs, as table metadata records them. +/// +/// The futures aren't required to be `Send`, so hosts can wrap JavaScript promises. +pub trait Host { + /// A multipart upload in progress. + type Upload: Upload; + + /// The object at `uri`, or `None` if there is none. + fn get(&self, uri: &str) -> impl Future>>>; + /// `len` bytes of the object at `uri` from `start`. + fn get_range(&self, uri: &str, start: u64, len: u64) -> impl Future>>; + /// Whether an object exists at `uri`. + fn exists(&self, uri: &str) -> impl Future>; + /// Writes (or replaces) the object at `uri`. + fn put(&self, uri: &str, bytes: Vec) -> impl Future>; + /// Writes the object at `uri` unless one exists; `false` if one did. The commit primitive. + fn put_if_absent(&self, uri: &str, bytes: Vec) -> impl Future>; + /// Starts a multipart upload to `uri`. + fn upload(&self, uri: &str) -> impl Future>; + /// Objects whose URI starts with `prefix`. + fn list(&self, prefix: &str) -> impl Future>>; + /// Deletes objects; missing ones are not an error. + fn delete(&self, uris: &[String]) -> impl Future>; + /// Milliseconds since the Unix epoch. + fn now_ms(&self) -> i64; + /// A random number. + fn random(&self) -> u64; + /// Waits `ms` milliseconds. + fn sleep(&self, ms: u64) -> impl Future; +} + +/// A multipart upload: parts in order, then [`Upload::complete`]. +pub trait Upload { + /// Sends the next part. Every part but the last must be at least the store's minimum (5 MiB on + /// S3 and R2). + fn put_part(&mut self, bytes: Vec) -> impl Future>; + /// Completes the upload: the object appears. + fn complete(self) -> impl Future>; + /// Drops the parts sent. + fn abort(self) -> impl Future; +} diff --git a/crates/lakebed-core/src/lib.rs b/crates/lakebed-core/src/lib.rs new file mode 100644 index 0000000..5975f62 --- /dev/null +++ b/crates/lakebed-core/src/lib.rs @@ -0,0 +1,31 @@ +//! The I/O boundary shared by lakebed's table format crates. Format crates do no I/O of their own: +//! they read and write through a [`Host`] the application implements over its object store (an +//! `object_store` client natively, `fetch` or an R2 binding in WebAssembly), so they build for +//! `wasm32-unknown-unknown` and native targets alike. +//! +//! ``` +//! use lakebed_core::{Host, MemoryHost}; +//! +//! # block_on(async { +//! let host = MemoryHost::new(0); +//! host.put("memory://t/a", b"x".to_vec()).await?; +//! assert!(host.exists("memory://t/a").await?); +//! # Ok::<(), lakebed_core::Error>(()) +//! # }).unwrap(); +//! # // MemoryHost's futures are ready at once, so one poll runs them. +//! # fn block_on(f: F) -> F::Output { +//! # let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); +//! # let mut f = std::pin::pin!(f); +//! # loop { if let std::task::Poll::Ready(v) = f.as_mut().poll(&mut cx) { return v; } } +//! # } +//! ``` + +mod error; +mod host; +mod memory; +mod uuid; + +pub use error::{Error, Result}; +pub use host::{Host, Listed, Upload}; +pub use memory::{MemoryHost, MemoryUpload}; +pub use uuid::{uuid_v4, uuid_v7}; diff --git a/crates/lakebed-core/src/memory.rs b/crates/lakebed-core/src/memory.rs new file mode 100644 index 0000000..919ebce --- /dev/null +++ b/crates/lakebed-core/src/memory.rs @@ -0,0 +1,138 @@ +use std::collections::BTreeMap; +use std::future::{Future, ready}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +use crate::{Error, Host, Listed, Result, Upload}; + +/// A [`Host`] in memory, with a fixed clock and a seeded random sequence: for tests and examples. +/// Clones share the same objects. +#[derive(Clone, Debug, Default)] +pub struct MemoryHost { + objects: Arc>>>, + clock: Arc>, + seed: Arc>, +} + +impl MemoryHost { + /// An empty store at time `now_ms`. + pub fn new(now_ms: i64) -> Self { + let host = Self::default(); + *lock(&host.clock) = now_ms; + host + } + + /// Moves the clock forward. + pub fn advance(&self, ms: i64) { + *lock(&self.clock) += ms; + } + + /// The URIs stored, sorted. + pub fn uris(&self) -> Vec { + self.objects().keys().cloned().collect() + } + + fn objects(&self) -> MutexGuard<'_, BTreeMap>> { + lock(&self.objects) + } +} + +fn lock(m: &Mutex) -> MutexGuard<'_, T> { + m.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// An upload to a [`MemoryHost`]. +#[derive(Debug)] +pub struct MemoryUpload { + host: MemoryHost, + uri: String, + bytes: Vec, +} + +impl Upload for MemoryUpload { + fn put_part(&mut self, bytes: Vec) -> impl Future> { + self.bytes.extend(bytes); + ready(Ok(())) + } + + fn complete(self) -> impl Future> { + self.host.objects().insert(self.uri, self.bytes); + ready(Ok(())) + } + + fn abort(self) -> impl Future { + ready(()) + } +} + +impl Host for MemoryHost { + type Upload = MemoryUpload; + + fn get(&self, uri: &str) -> impl Future>>> { + ready(Ok(self.objects().get(uri).cloned())) + } + + fn get_range(&self, uri: &str, start: u64, len: u64) -> impl Future>> { + let objects = self.objects(); + ready(objects.get(uri).ok_or_else(|| Error::NotFound(uri.to_string())).map(|bytes| { + let start = usize::try_from(start).unwrap_or(usize::MAX).min(bytes.len()); + let end = start.saturating_add(usize::try_from(len).unwrap_or(usize::MAX)).min(bytes.len()); + bytes[start..end].to_vec() + })) + } + + fn exists(&self, uri: &str) -> impl Future> { + ready(Ok(self.objects().contains_key(uri))) + } + + fn put(&self, uri: &str, bytes: Vec) -> impl Future> { + self.objects().insert(uri.to_string(), bytes); + ready(Ok(())) + } + + fn put_if_absent(&self, uri: &str, bytes: Vec) -> impl Future> { + let mut objects = self.objects(); + let absent = !objects.contains_key(uri); + if absent { + objects.insert(uri.to_string(), bytes); + } + ready(Ok(absent)) + } + + fn upload(&self, uri: &str) -> impl Future> { + ready(Ok(MemoryUpload { host: self.clone(), uri: uri.to_string(), bytes: vec![] })) + } + + fn list(&self, prefix: &str) -> impl Future>> { + let listed = self + .objects() + .range(prefix.to_string()..) + .take_while(|(k, _)| k.starts_with(prefix)) + .map(|(k, v)| Listed { uri: k.clone(), size: v.len() as u64 }) + .collect(); + ready(Ok(listed)) + } + + fn delete(&self, uris: &[String]) -> impl Future> { + let mut objects = self.objects(); + for uri in uris { + objects.remove(uri); + } + ready(Ok(())) + } + + fn now_ms(&self) -> i64 { + *lock(&self.clock) + } + + fn random(&self) -> u64 { + // A linear congruential generator (Knuth's MMIX constants), with the high bits folded in. + let mut seed = lock(&self.seed); + *seed = seed.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407); + *seed ^ (*seed >> 29) + } + + fn sleep(&self, ms: u64) -> impl Future { + self.advance(i64::try_from(ms).unwrap_or(i64::MAX)); + ready(()) + } +} diff --git a/crates/lakebed-core/src/uuid.rs b/crates/lakebed-core/src/uuid.rs new file mode 100644 index 0000000..320b951 --- /dev/null +++ b/crates/lakebed-core/src/uuid.rs @@ -0,0 +1,55 @@ +use std::fmt::Write as _; + +use crate::Host; + +/// A random UUID (version 4) from `host`. +pub fn uuid_v4(host: &impl Host) -> String { + let mut b = [0u8; 16]; + b[..8].copy_from_slice(&host.random().to_be_bytes()); + b[8..].copy_from_slice(&host.random().to_be_bytes()); + b[6] = (b[6] & 0x0f) | 0x40; + b[8] = (b[8] & 0x3f) | 0x80; + format_uuid(&b) +} + +/// A time-ordered UUID (version 7) from `host`'s clock and randomness: names made with it sort by +/// creation time. +pub fn uuid_v7(host: &impl Host) -> String { + let mut b = [0u8; 16]; + let ms = u64::try_from(host.now_ms()).unwrap_or(0); + b[..6].copy_from_slice(&ms.to_be_bytes()[2..]); + b[6..14].copy_from_slice(&host.random().to_be_bytes()); + b[14..].copy_from_slice(&host.random().to_be_bytes()[..2]); + b[6] = (b[6] & 0x0f) | 0x70; + b[8] = (b[8] & 0x3f) | 0x80; + format_uuid(&b) +} + +fn format_uuid(b: &[u8; 16]) -> String { + let hex = b.iter().fold(String::with_capacity(32), |mut s, x| { + let _ = write!(s, "{x:02x}"); + s + }); + format!("{}-{}-{}-{}-{}", &hex[..8], &hex[8..12], &hex[12..16], &hex[16..20], &hex[20..]) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::MemoryHost; + + #[test] + fn uuids_are_well_formed() { + let host = MemoryHost::new(1_767_225_600_000); + let v4 = uuid_v4(&host); + let v7 = uuid_v7(&host); + for (u, version) in [(&v4, '4'), (&v7, '7')] { + assert_eq!(u.len(), 36); + assert_eq!(u.chars().nth(14), Some(version)); + assert!(matches!(u.chars().nth(19), Some('8' | '9' | 'a' | 'b'))); + } + // Version 7 starts with the time, so later ones sort later. + host.advance(1); + assert!(uuid_v7(&host) > v7); + } +} diff --git a/crates/lakebed-iceberg/Cargo.toml b/crates/lakebed-iceberg/Cargo.toml new file mode 100644 index 0000000..946ec88 --- /dev/null +++ b/crates/lakebed-iceberg/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "lakebed-iceberg" +description = "Apache Iceberg table metadata, manifests and commits without I/O of its own, for WebAssembly and native" +readme = "README.md" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true + +[features] +default = [] +# Writing: manifests, manifest lists and table metadata updates, and the commit protocol over a host. +write = [] +# The table schema as an Arrow schema, with Iceberg field ids in the field metadata. +arrow = ["dep:arrow-schema"] +# Iceberg's statistics of a data file (counts and bounds per column) from its Parquet footer. +parquet-stats = ["dep:parquet"] + +[dependencies] +lakebed-core.workspace = true +apache-avro.workspace = true +serde_json.workspace = true +futures-util.workspace = true +arrow-schema = { workspace = true, optional = true } +parquet = { workspace = true, optional = true } + +# The interop tests check that iceberg-rust reads what this crate writes and the other way round. +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +iceberg.workspace = true +tokio.workspace = true +parquet = { workspace = true, features = ["arrow"] } +arrow-array = "58" +arrow-schema.workspace = true +bytes = "1" +futures = "0.3" + +[package.metadata.docs.rs] +all-features = true + +[lints] +workspace = true diff --git a/crates/lakebed-iceberg/README.md b/crates/lakebed-iceberg/README.md new file mode 100644 index 0000000..cb493a8 --- /dev/null +++ b/crates/lakebed-iceberg/README.md @@ -0,0 +1,15 @@ +# lakebed-iceberg + +[Apache Iceberg](https://iceberg.apache.org) tables read and written without I/O of their own, through a +[`lakebed_core::Host`](../lakebed-core), so they work in WebAssembly (browsers, Cloudflare Workers) as well as natively. + +| Feature | What it adds | +|---|---| +| (always) | Table metadata of format v1 to v3, schemas, partition specs, snapshots; manifest lists and manifests with data and delete files; values in Iceberg's single-value encoding; loading a table from its location or a metadata file | +| `write` | Format v2 manifests, manifest lists, and metadata updates; commits to file-system tables with `CommitOptions` and `CommitHooks` | +| `arrow` | The schema as an Arrow schema, with field ids | +| `parquet-stats` | A data file's Iceberg statistics (counts, bounds, split offsets) from its Parquet footer | + +The tests commit to one table with lakebed and iceberg-rust in turn, each reading what the other wrote. + +Apache License 2.0. diff --git a/crates/lakebed-iceberg/src/arrow.rs b/crates/lakebed-iceberg/src/arrow.rs new file mode 100644 index 0000000..1dd9ba2 --- /dev/null +++ b/crates/lakebed-iceberg/src/arrow.rs @@ -0,0 +1,103 @@ +//! Iceberg schemas as Arrow schemas (feature `arrow`), with each field's Iceberg id in its +//! metadata under [`FIELD_ID_KEY`], as Parquet writers and iceberg-rust's `schema_to_arrow_schema` +//! carry it. + +use std::collections::HashMap; +use std::sync::Arc; + +use arrow_schema::{DataType, Field, Fields, TimeUnit}; + +use crate::types::{NestedField, PrimitiveType, Schema, Type}; +use crate::{Error, Result}; + +/// The Arrow field metadata key carrying an Iceberg field id. +pub const FIELD_ID_KEY: &str = "PARQUET:field_id"; + +/// The time zone of `timestamptz` columns in Arrow. +pub const UTC: &str = "+00:00"; + +/// `schema` as an Arrow schema. +pub fn schema_to_arrow(schema: &Schema) -> Result { + Ok(arrow_schema::Schema::new(fields(&schema.fields)?)) +} + +fn fields(fields: &[NestedField]) -> Result> { + fields.iter().map(|f| field(&f.name, f.id, !f.required, &f.ty)).collect() +} + +fn field(name: &str, id: i32, nullable: bool, ty: &Type) -> Result { + let metadata = HashMap::from([(FIELD_ID_KEY.to_string(), id.to_string())]); + Ok(Field::new(name, data_type(ty)?, nullable).with_metadata(metadata)) +} + +/// The Arrow type of an Iceberg type. +pub fn data_type(ty: &Type) -> Result { + Ok(match ty { + Type::Primitive(p) => primitive(p), + Type::Struct(f) => DataType::Struct(Fields::from(fields(f)?)), + Type::List { element_id, element_required, element } => { + DataType::List(Arc::new(field("element", *element_id, !element_required, element)?)) + } + Type::Map { key_id, key, value_id, value_required, value } => { + let entries = Field::new( + "key_value", + DataType::Struct(Fields::from(vec![ + field("key", *key_id, false, key)?, + field("value", *value_id, !value_required, value)?, + ])), + false, + ); + DataType::Map(Arc::new(entries), false) + } + Type::Other(name) => return Err(Error::Unsupported(format!("{name} columns in Arrow"))), + }) +} + +fn primitive(p: &PrimitiveType) -> DataType { + match p { + PrimitiveType::Boolean => DataType::Boolean, + PrimitiveType::Int => DataType::Int32, + PrimitiveType::Long => DataType::Int64, + PrimitiveType::Float => DataType::Float32, + PrimitiveType::Double => DataType::Float64, + PrimitiveType::Decimal { precision, scale } => { + DataType::Decimal128(*precision, i8::try_from(*scale).unwrap_or(i8::MAX)) + } + PrimitiveType::Date => DataType::Date32, + PrimitiveType::Time => DataType::Time64(TimeUnit::Microsecond), + PrimitiveType::Timestamp => DataType::Timestamp(TimeUnit::Microsecond, None), + PrimitiveType::Timestamptz => DataType::Timestamp(TimeUnit::Microsecond, Some(UTC.into())), + PrimitiveType::TimestampNs => DataType::Timestamp(TimeUnit::Nanosecond, None), + PrimitiveType::TimestamptzNs => DataType::Timestamp(TimeUnit::Nanosecond, Some(UTC.into())), + PrimitiveType::String => DataType::Utf8, + PrimitiveType::Uuid => DataType::FixedSizeBinary(16), + PrimitiveType::Fixed(n) => DataType::FixedSizeBinary(i32::try_from(*n).unwrap_or(i32::MAX)), + PrimitiveType::Binary => DataType::LargeBinary, + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn fields_carry_ids() { + let schema = Schema::from_json(&json!({"fields": [ + {"id": 1, "name": "time", "required": true, "type": "timestamptz"}, + {"id": 2, "name": "tags", "required": false, "type": + {"type": "map", "key-id": 3, "key": "string", "value-id": 4, "value-required": true, "value": "decimal(10,2)"}} + ]})) + .unwrap(); + let arrow = schema_to_arrow(&schema).unwrap(); + let time = arrow.field(0); + assert_eq!(time.data_type(), &DataType::Timestamp(TimeUnit::Microsecond, Some(UTC.into()))); + assert!(!time.is_nullable()); + assert_eq!(time.metadata()[FIELD_ID_KEY], "1"); + let DataType::Map(entries, _) = arrow.field(1).data_type() else { panic!("not a map") }; + let DataType::Struct(kv) = entries.data_type() else { panic!("not a struct") }; + assert_eq!(kv[1].metadata()[FIELD_ID_KEY], "4"); + assert_eq!(kv[1].data_type(), &DataType::Decimal128(10, 2)); + } +} diff --git a/crates/lakebed-iceberg/src/avro.rs b/crates/lakebed-iceberg/src/avro.rs new file mode 100644 index 0000000..08544c5 --- /dev/null +++ b/crates/lakebed-iceberg/src/avro.rs @@ -0,0 +1,86 @@ +//! Reading Avro records by field name, as manifests and manifest lists are read: optional fields +//! are unions with null, and fields a format version lacks are absent. + +use std::collections::HashMap; + +use apache_avro::Reader; +use apache_avro::types::Value; + +use crate::{Error, Result, corrupt}; + +pub(crate) fn error(e: impl std::fmt::Display) -> Error { + corrupt(format!("avro: {e}")) +} + +/// An Avro container file's header metadata. +pub(crate) type Header = HashMap>; + +/// The records of an Avro container file, and its header metadata. +pub(crate) fn records(bytes: &[u8]) -> Result<(Vec, Header)> { + let reader = Reader::new(bytes).map_err(error)?; + let metadata = reader.user_metadata().clone(); + let records = reader.map(|v| v.map_err(error)).collect::>()?; + Ok((records, metadata)) +} + +/// The value inside an optional field's union. +pub(crate) fn strip_union(v: &Value) -> &Value { + match v { + Value::Union(_, inner) => strip_union(inner), + other => other, + } +} + +/// Field `name` of `record`, or `None` if it is absent or null. +pub(crate) fn field<'a>(record: &'a Value, name: &str) -> Option<&'a Value> { + match strip_union(record) { + Value::Record(fields) => { + fields.iter().find(|(n, _)| n == name).map(|(_, v)| strip_union(v)).filter(|v| **v != Value::Null) + } + _ => None, + } +} + +/// The first of `names` that `record` has (fields renamed between format versions). +pub(crate) fn field_of<'a>(record: &'a Value, names: &[&str]) -> Option<&'a Value> { + names.iter().find_map(|n| field(record, n)) +} + +pub(crate) fn long(v: Option<&Value>) -> Option { + match v? { + Value::Long(n) => Some(*n), + Value::Int(n) | Value::Date(n) => Some(i64::from(*n)), + _ => None, + } +} + +pub(crate) fn int(v: Option<&Value>) -> Option { + match v? { + Value::Int(n) | Value::Date(n) => Some(*n), + Value::Long(n) => i32::try_from(*n).ok(), + _ => None, + } +} + +pub(crate) fn string(v: Option<&Value>) -> Option<&str> { + match v? { + Value::String(s) => Some(s), + _ => None, + } +} + +pub(crate) fn required(v: Option, what: &str) -> Result { + v.ok_or_else(|| corrupt(format!("manifest without {what}"))) +} + +/// An optional Avro value. +#[cfg(feature = "write")] +pub(crate) fn some(v: Value) -> Value { + Value::Union(1, Box::new(v)) +} + +/// A null optional Avro value. +#[cfg(feature = "write")] +pub(crate) fn null() -> Value { + Value::Union(0, Box::new(Value::Null)) +} diff --git a/crates/lakebed-iceberg/src/commit.rs b/crates/lakebed-iceberg/src/commit.rs new file mode 100644 index 0000000..eb3d396 --- /dev/null +++ b/crates/lakebed-iceberg/src/commit.rs @@ -0,0 +1,673 @@ +//! Commits to file-system tables (feature `write`): every file of a snapshot (manifests, manifest +//! list, and whatever the caller adds) is uploaded, then `v.metadata.json` is written only if +//! absent. When another writer got there first, the commit reloads the table and tries again on top +//! of it. `version-hint.text` is written after the commit; readers don't rely on it. +//! +//! Applications extend a commit through [`CommitHooks`]: files of their own written with the +//! snapshot, summary properties, a check that skips a commit already made, and how to reload. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::fmt; +use std::future::Future; +use std::hash::BuildHasher; +use std::sync::Arc; + +use lakebed_core::{Host, uuid_v4}; + +use crate::manifest::{ + DataFile, Entry, ManifestContext, ManifestFile, ManifestWrite, Status, manifest_file, write_manifest, + write_manifest_list, +}; +use crate::metadata::NewSnapshot; +use crate::table::{Table, hint_uri, metadata_uri}; +use crate::{Error, Result, summary}; + +/// What a commit changes. +#[derive(Clone, Debug)] +pub enum Change { + /// Add data files. + Append { + /// The files. + files: Vec, + }, + /// Remove files and add files with the same rows (compaction). Each added file keeps the data + /// sequence number in `sequence_numbers`, so its rows keep their order against other files'. + Replace { + /// Files removed. + removed: Vec, + /// Files added. + added: Vec, + /// Data sequence number of each added file. + sequence_numbers: Vec, + }, + /// Remove files (retention). + Delete { + /// Files removed. + removed: Vec, + }, +} + +impl Change { + /// Files the change adds. + pub fn added(&self) -> &[DataFile] { + match self { + Self::Append { files } => files, + Self::Replace { added, .. } => added, + Self::Delete { .. } => &[], + } + } + + /// Files the change removes. + pub fn removed(&self) -> &[Entry] { + match self { + Self::Append { .. } => &[], + Self::Replace { removed, .. } | Self::Delete { removed } => removed, + } + } + + fn data_sequence(&self, i: usize) -> Option { + match self { + Self::Replace { sequence_numbers, .. } => sequence_numbers.get(i).copied(), + _ => None, + } + } + + /// The snapshot operation: `append`, `replace` or `delete`. + pub fn operation(&self) -> &'static str { + match self { + Self::Append { .. } => "append", + Self::Replace { .. } => "replace", + Self::Delete { .. } => "delete", + } + } +} + +/// How a commit behaves. The defaults are the ones Timelake's engine uses. +#[derive(Clone, Debug)] +pub struct CommitOptions { + /// Attempts before a commit that keeps losing the race gives up with [`Error::Conflict`]. + pub max_attempts: u64, + /// The wait before attempt `n` is a random number of milliseconds in this range, times `n`. + pub backoff_ms: (u64, u64), + /// An append merges the table's manifests into one once there would be more than this many. + pub merge_manifests_at: usize, + /// Write `version-hint.text` after each commit, for readers of stores that can't list. + pub write_version_hint: bool, +} + +impl Default for CommitOptions { + fn default() -> Self { + Self { max_attempts: 10, backoff_ms: (5, 50), merge_manifests_at: 8, write_version_hint: true } + } +} + +/// How a commit went. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CommitInfo { + /// The new snapshot (when skipped, the current one). + pub snapshot_id: i64, + /// The table version after the commit. + pub version: u64, + /// Attempts it took. + pub attempts: u64, + /// [`CommitHooks::skip`] found the change already committed; nothing was written. + pub skipped: bool, +} + +/// What an application adds to a snapshot. +#[derive(Clone, Debug, Default)] +pub struct Extra { + /// Files to upload with the snapshot's manifests: `(uri, bytes)`. A commit that loses the race + /// deletes them. + pub files: Vec<(String, Vec)>, + /// Summary properties. They override the caller's; Iceberg's counters override them. + pub summary: HashMap, +} + +/// An application's part in a commit. Every method has a default that does nothing extra. +pub trait CommitHooks { + /// The error the hooks (and so the commit) return. + type Error: From + fmt::Debug; + + /// The newest version of the table, after another writer won the race. + fn reload(&mut self, host: &H, location: &str) -> impl Future> { + async move { Ok(Table::load(host, location).await?) } + } + + /// Whether `base` already holds this change (an idempotent append retried), so nothing is + /// committed. Checked before every attempt. + fn skip(&mut self, _base: &Table) -> bool { + false + } + + /// Files and summary properties to add to snapshot `snapshot_id` on top of `base`. Called once + /// per attempt. + fn extra( + &mut self, + _host: &H, + _base: &Table, + _snapshot_id: i64, + ) -> impl Future> { + async { Ok(Extra::default()) } + } +} + +/// Hooks that add nothing. +#[derive(Clone, Copy, Debug, Default)] +pub struct NoHooks; + +impl CommitHooks for NoHooks { + type Error = Error; +} + +/// Commits `change` on top of `base` (or the newest version, after a lost race), with `props` in +/// the snapshot summary. The returned table knows its files when `base` did. +pub async fn commit, S: BuildHasher>( + host: &H, + base: Table, + change: &Change, + props: &HashMap, + options: &CommitOptions, + hooks: &mut K, +) -> Result<(Table, CommitInfo), K::Error> { + base.metadata.check_writable()?; + if let Change::Replace { added, sequence_numbers, .. } = change + && added.len() != sequence_numbers.len() + { + let (files, numbers) = (added.len(), sequence_numbers.len()); + return Err(Error::Invalid(format!("replace: {files} added files but {numbers} sequence numbers")).into()); + } + let mut base = base; + for attempt in 1..=options.max_attempts { + if attempt > 1 { + let (lo, hi) = options.backoff_ms; + host.sleep((lo + host.random() % hi.saturating_sub(lo).max(1)) * attempt).await; + base = hooks.reload(host, &base.location).await?; + } + if hooks.skip(&base) { + let info = CommitInfo { + snapshot_id: base.metadata.current_snapshot_id().unwrap_or_default(), + version: base.version, + attempts: attempt, + skipped: true, + }; + return Ok((base, info)); + } + if let Some((table, snapshot_id)) = attempt_commit(host, &base, change, props, options, hooks).await? { + let info = CommitInfo { snapshot_id, version: table.version, attempts: attempt, skipped: false }; + return Ok((table, info)); + } + } + Err(Error::Conflict(format!("{}: gave up after {} attempts", base.location, options.max_attempts)).into()) +} + +/// Writes table metadata `bytes` at `uri` unless a version is there; `false` if another writer's +/// is. +/// +/// A conditional PUT that reports failure may have landed all the same: a retry after a first try +/// that landed gets "already exists", and a timeout says nothing. The version is then read back: +/// our own metadata there is the commit, another writer's means it lost. If that can't be told, +/// the error is [`Error::CommitUnknown`], and callers must not delete what the metadata references. +pub async fn put_metadata(host: &H, uri: &str, bytes: Vec) -> Result { + let unknown = |e: &dyn fmt::Display| Error::CommitUnknown(format!("{uri}: {e}")); + match host.put_if_absent(uri, bytes.clone()).await { + Ok(true) => Ok(true), + // Ours (an earlier try landed) or another writer's. Absent now, it was never ours: our + // version would still be there. + Ok(false) => match host.get(uri).await { + Ok(found) => Ok(found.is_some_and(|found| found == bytes)), + Err(e) => Err(unknown(&e)), + }, + // Absent now, the PUT may still land. + Err(e) => match host.get(uri).await { + Ok(Some(found)) => Ok(found == bytes), + Ok(None) | Err(_) => Err(unknown(&e)), + }, + } +} + +/// A snapshot id not in the table yet. +fn new_snapshot_id(host: &impl Host, base: &Table) -> i64 { + loop { + let id = i64::try_from(host.random() >> 1).unwrap_or(1).max(1); + if !base.metadata.has_snapshot(id) { + return id; + } + } +} + +/// One attempt: writes the snapshot's files and publishes it. `None` if another writer committed +/// this version first (this attempt's files are deleted). With an unknown outcome, nothing is +/// deleted. +async fn attempt_commit, S: BuildHasher>( + host: &H, + base: &Table, + change: &Change, + props: &HashMap, + options: &CommitOptions, + hooks: &mut K, +) -> Result, K::Error> { + let snapshot_id = new_snapshot_id(host, base); + let sequence_number = base.metadata.next_sequence_number(); + let mut uploads: Vec<(String, Vec)> = vec![]; + let mut unknown = false; + let result = async { + let manifests = write_manifests(host, base, change, snapshot_id, options, &mut uploads).await?; + let extra = hooks.extra(host, base, snapshot_id).await?; + uploads.extend(extra.files); + let list_uri = format!("{}/metadata/snap-{snapshot_id}-{}.avro", base.location, uuid_v4(host)); + let parent = base.metadata.current_snapshot_id(); + let list = write_manifest_list(&manifests, snapshot_id, parent, sequence_number)?; + uploads.push((list_uri.clone(), list)); + let puts = uploads.iter().map(|(uri, bytes)| host.put(uri, bytes.clone())); + futures_util::future::try_join_all(puts).await?; + + let mut summary: HashMap = props.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); + summary.extend(extra.summary); + summary.extend(counters(change, &base.metadata.summary())); + // Snapshot and metadata logs must stay in time order, so a writer whose clock is behind the + // last one's dates its commit at the table's last update. + let now = host.now_ms().max(base.metadata.last_updated_ms()); + let snapshot = NewSnapshot { + snapshot_id, + sequence_number, + timestamp_ms: now, + manifest_list: list_uri, + operation: change.operation().into(), + summary, + }; + let metadata = base.metadata.with_snapshot(&snapshot, &base.metadata_file, now)?; + let version = base.version + 1; + let ours = put_metadata(host, &metadata_uri(&base.location, version), metadata.to_json().into_bytes()) + .await + .inspect_err(|e| unknown = matches!(e, Error::CommitUnknown(_)))?; + Ok::<_, K::Error>(ours.then_some((metadata, version))) + } + .await; + let (metadata, version) = match result { + Ok(Some(committed)) => committed, + Ok(None) => { + delete_uploads(host, uploads).await; + return Ok(None); + } + Err(e) => { + if !unknown { + delete_uploads(host, uploads).await; + } + return Err(e); + } + }; + + let files = base.files.as_ref().map(|files| { + let removed: HashSet<&str> = change.removed().iter().map(|e| e.file.path.as_str()).collect(); + let mut next: Vec = change + .added() + .iter() + .enumerate() + .map(|(i, f)| Entry { + status: Status::Added, + snapshot_id, + sequence_number: change.data_sequence(i).unwrap_or(sequence_number), + file_sequence_number: sequence_number, + file: Arc::new(f.clone()), + }) + .collect(); + next.extend(files.iter().filter(|e| !removed.contains(e.file.path.as_str())).cloned()); + next + }); + let table = Table { files, ..Table::new(base.location.clone(), version, metadata) }; + if options.write_version_hint { + let _ = host.put(&hint_uri(&table.location), version.to_string().into_bytes()).await; + } + Ok(Some((table, snapshot_id))) +} + +async fn delete_uploads(host: &impl Host, uploads: Vec<(String, Vec)>) { + let written: Vec = uploads.into_iter().map(|(uri, _)| uri).collect(); + let _ = host.delete(&written).await; +} + +/// Writes a manifest of the added files, carries the base snapshot's manifests over, and rewrites +/// the ones holding removed files (or all of them once there are too many). +async fn write_manifests( + host: &H, + base: &Table, + change: &Change, + snapshot_id: i64, + options: &CommitOptions, + uploads: &mut Vec<(String, Vec)>, +) -> Result> { + let metadata = &base.metadata; + let schema_json = metadata.schema_json()?.to_string(); + let (spec_id, spec_fields) = metadata.partition_spec_json()?; + let spec_fields_json = spec_fields.to_string(); + let partition_type = metadata.partition_type()?; + let ctx = ManifestContext { + schema_json: &schema_json, + schema_id: metadata.schema_id(), + spec_fields_json: &spec_fields_json, + spec_id, + partition_type: &partition_type, + }; + let mut write = |kind: &str, entries: &[ManifestWrite]| -> Result { + let uri = format!("{}/metadata/{}-{kind}-m.avro", base.location, uuid_v4(host)); + let bytes = write_manifest(&ctx, entries)?; + let mf = manifest_file(&uri, &bytes, spec_id, snapshot_id, entries, &partition_type); + uploads.push((uri, bytes)); + Ok(mf) + }; + let mut new = vec![]; + let added = change.added(); + if !added.is_empty() { + let entries: Vec = added + .iter() + .enumerate() + .map(|(i, f)| ManifestWrite { + status: Status::Added, + snapshot_id: Some(snapshot_id), + sequence_number: change.data_sequence(i), + file_sequence_number: None, + file: Arc::new(f.clone()), + }) + .collect(); + new.push(write("added", &entries)?); + } + + let current = base.manifests(host).await?; + let removed: HashSet<&str> = change.removed().iter().map(|e| e.file.path.as_str()).collect(); + let merge = removed.is_empty() && current.len() + new.len() > options.merge_manifests_at; + if removed.is_empty() && !merge { + new.extend(current); + return Ok(new); + } + let loaded = futures_util::future::try_join_all(current.iter().map(|m| Table::manifest(host, m))).await?; + let (mut kept, mut deleted, mut carried) = (vec![], vec![], vec![]); + for (manifest, entries) in current.into_iter().zip(loaded) { + let live: Vec<&Entry> = entries.iter().filter(|e| e.status != Status::Deleted).collect(); + let touched = live.iter().any(|e| removed.contains(e.file.path.as_str())); + // Delete manifests and other specs' manifests are carried as they are. + let mergeable = manifest.content == crate::ManifestContent::Data && manifest.partition_spec_id == spec_id; + if !mergeable || (!merge && !touched) { + if touched { + return Err(Error::Unsupported(format!( + "{}: removing files of an older partition spec", + base.location + ))); + } + carried.push(manifest); + continue; + } + for e in live { + let (status, snapshot) = if removed.contains(e.file.path.as_str()) { + (Status::Deleted, snapshot_id) + } else { + (Status::Existing, e.snapshot_id) + }; + let w = ManifestWrite { + status, + snapshot_id: Some(snapshot), + sequence_number: Some(e.sequence_number), + file_sequence_number: Some(e.file_sequence_number), + file: Arc::clone(&e.file), + }; + if status == Status::Deleted { deleted.push(w) } else { kept.push(w) } + } + } + if deleted.len() != removed.len() { + return Err(Error::Conflict(format!( + "{}: {} of {} files to remove are no longer in the table (concurrent maintenance?)", + base.location, + removed.len() - deleted.len(), + removed.len() + ))); + } + if !kept.is_empty() { + new.push(write("merged", &kept)?); + } + if !deleted.is_empty() { + new.push(write("deleted", &deleted)?); + } + new.extend(carried); + Ok(new) +} + +/// Iceberg's standard snapshot summary counters after `change` on a table whose current summary is +/// `base`. Delete-file totals carry over: commits here don't add or remove delete files. +pub fn counters(change: &Change, base: &HashMap) -> BTreeMap { + let total = |k: &str| base.get(k).and_then(|v| v.parse::().ok()).unwrap_or(0); + let (added, removed) = (change.added(), change.removed()); + let added_rows: u64 = added.iter().map(|f| f.record_count).sum(); + let added_bytes: u64 = added.iter().map(|f| f.file_size).sum(); + let removed_rows: u64 = removed.iter().map(|e| e.file.record_count).sum(); + let removed_bytes: u64 = removed.iter().map(|e| e.file.file_size).sum(); + let (added_files, removed_files) = (added.len() as u64, removed.len() as u64); + let mut out = BTreeMap::new(); + for (k, v) in [ + (summary::ADDED_DATA_FILES, added_files), + (summary::ADDED_RECORDS, added_rows), + (summary::ADDED_FILES_SIZE, added_bytes), + (summary::DELETED_DATA_FILES, removed_files), + (summary::DELETED_RECORDS, removed_rows), + (summary::REMOVED_FILES_SIZE, removed_bytes), + ] { + if v != 0 { + out.insert(k.to_string(), v.to_string()); + } + } + for (k, v) in [ + (summary::TOTAL_DATA_FILES, (total(summary::TOTAL_DATA_FILES) + added_files).saturating_sub(removed_files)), + (summary::TOTAL_RECORDS, (total(summary::TOTAL_RECORDS) + added_rows).saturating_sub(removed_rows)), + (summary::TOTAL_FILES_SIZE, (total(summary::TOTAL_FILES_SIZE) + added_bytes).saturating_sub(removed_bytes)), + (summary::TOTAL_DELETE_FILES, total(summary::TOTAL_DELETE_FILES)), + (summary::TOTAL_POSITION_DELETES, total(summary::TOTAL_POSITION_DELETES)), + (summary::TOTAL_EQUALITY_DELETES, total(summary::TOTAL_EQUALITY_DELETES)), + ] { + out.insert(k.to_string(), v.to_string()); + } + out +} + +#[cfg(test)] +mod tests { + use lakebed_core::MemoryHost; + use serde_json::json; + + use super::*; + use crate::{Datum, Metadata, block_on}; + + const LOCATION: &str = "memory://wh/t"; + + fn create(host: &MemoryHost, format_version: u8) -> Table { + let metadata = json!({ + "format-version": format_version, "table-uuid": "u", "location": LOCATION, "last-sequence-number": 0, + "last-updated-ms": 0, "last-column-id": 2, "current-schema-id": 0, + "schemas": [{"schema-id": 0, "type": "struct", "fields": [ + {"id": 1, "name": "time", "required": true, "type": "timestamptz"}, + {"id": 2, "name": "device", "required": true, "type": "string"}]}], + "default-spec-id": 0, "last-partition-id": 1000, + "partition-specs": [{"spec-id": 0, "fields": [ + {"source-id": 1, "field-id": 1000, "name": "time_day", "transform": "day"}]}], + "default-sort-order-id": 0, "sort-orders": [{"order-id": 0, "fields": []}], + "properties": {}, "current-snapshot-id": -1, "snapshots": [], "snapshot-log": [], "metadata-log": [] + }); + let table = Table::new(LOCATION, 1, Metadata::from_json(metadata).unwrap()); + block_on(host.put(&table.metadata_file, table.metadata.to_json().into_bytes())).unwrap(); + Table { files: Some(vec![]), ..table } + } + + fn file(name: &str, day: i32, rows: u64) -> DataFile { + DataFile { + path: format!("{LOCATION}/data/{name}.parquet"), + partition: vec![Some(Datum::Date(day))], + record_count: rows, + file_size: rows * 10, + ..DataFile::default() + } + } + + fn append(files: Vec) -> Change { + Change::Append { files } + } + + fn try_run>( + host: &MemoryHost, + base: Table, + change: &Change, + hooks: &mut K, + ) -> Result<(Table, CommitInfo), K::Error> { + block_on(commit(host, base, change, &HashMap::new(), &CommitOptions::default(), hooks)) + } + + fn run>( + host: &MemoryHost, + base: Table, + change: &Change, + hooks: &mut K, + ) -> (Table, CommitInfo) { + try_run(host, base, change, hooks).unwrap() + } + + fn files_named(t: &Table, name: &str) -> Vec { + t.files.iter().flatten().filter(|e| e.file.path.ends_with(name)).cloned().collect() + } + + /// The live files of the newest version, read back from the store: (name, data sequence + /// number, file sequence number). + fn live(host: &MemoryHost) -> Vec<(String, i64, i64)> { + let mut t = block_on(Table::load(host, LOCATION)).unwrap(); + let mut files: Vec<_> = block_on(t.live_files(host)) + .unwrap() + .iter() + .map(|e| { + ( + e.file.path.rsplit('/').next().unwrap_or_default().to_string(), + e.sequence_number, + e.file_sequence_number, + ) + }) + .collect(); + files.sort(); + files + } + + #[test] + fn append_replace_and_delete() { + let host = MemoryHost::new(1_000); + let t = create(&host, 2); + let (t, info) = run(&host, t, &append(vec![file("a", 1, 10), file("b", 2, 20)]), &mut NoHooks); + assert_eq!((info.version, info.attempts, info.skipped), (2, 1, false)); + assert_eq!(live(&host), [("a.parquet".into(), 1, 1), ("b.parquet".into(), 1, 1)]); + + // Compaction keeps the data sequence number of what it replaces. + let change = Change::Replace { + removed: files_named(&t, "a.parquet"), + added: vec![file("c", 1, 10)], + sequence_numbers: vec![1], + }; + let (t, _) = run(&host, t, &change, &mut NoHooks); + assert_eq!(live(&host), [("b.parquet".into(), 1, 1), ("c.parquet".into(), 1, 2)]); + assert_eq!(t.files.as_ref().map(Vec::len), Some(2), "the commit derives the files"); + + let (t, _) = run(&host, t.clone(), &Change::Delete { removed: files_named(&t, "b.parquet") }, &mut NoHooks); + assert_eq!(live(&host), [("c.parquet".into(), 1, 2)]); + let summary = t.metadata.summary(); + assert_eq!((summary["operation"].as_str(), summary[crate::summary::TOTAL_RECORDS].as_str()), ("delete", "10")); + assert_eq!(summary[crate::summary::DELETED_DATA_FILES], "1"); + assert_eq!(t.metadata.metadata_log_len(), 3); + assert_eq!(block_on(host.get(&hint_uri(LOCATION))).unwrap().as_deref(), Some(&b"4"[..])); + } + + #[test] + fn a_lost_race_retries_on_the_newer_version() { + let host = MemoryHost::new(1_000); + let stale = create(&host, 2); + let (_, first) = run(&host, stale.clone(), &append(vec![file("a", 1, 10)]), &mut NoHooks); + let (_, second) = run(&host, stale, &append(vec![file("b", 1, 10)]), &mut NoHooks); + assert_eq!((first.version, second.version, second.attempts), (2, 3, 2)); + assert_eq!(live(&host).len(), 2); + // The lost attempt's manifests and list were deleted: every list is a snapshot's. + let lists = host.uris().iter().filter(|u| u.contains("/snap-")).count(); + assert_eq!(lists, 2); + } + + #[test] + fn removing_files_that_are_gone_is_a_conflict() { + let host = MemoryHost::new(1_000); + let (t, _) = run(&host, create(&host, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); + let removed = t.files.clone().unwrap_or_default(); + let (t, _) = run(&host, t, &Change::Delete { removed: removed.clone() }, &mut NoHooks); + let err = try_run(&host, t, &Change::Delete { removed }, &mut NoHooks); + assert!(matches!(err, Err(Error::Conflict(_))), "{err:?}"); + } + + /// Hooks that skip a batch already committed, and add a file and summary properties. + struct Batch(&'static str); + + impl CommitHooks for Batch { + type Error = Error; + + fn skip(&mut self, base: &Table) -> bool { + base.metadata.ancestors().iter().any(|s| s.summary.get("batch").map(String::as_str) == Some(self.0)) + } + + fn extra(&mut self, _: &MemoryHost, base: &Table, snapshot_id: i64) -> impl Future> { + let uri = format!("{}/metadata/extra-{snapshot_id}", base.location); + let summary = + HashMap::from([("batch".to_string(), self.0.to_string()), ("extra".to_string(), uri.clone())]); + std::future::ready(Ok(Extra { files: vec![(uri, b"x".to_vec())], summary })) + } + } + + #[test] + fn hooks_skip_and_extend_commits() { + let host = MemoryHost::new(1_000); + let (t, info) = run(&host, create(&host, 2), &append(vec![file("a", 1, 10)]), &mut Batch("b1")); + let extra = t.metadata.summary()["extra"].clone(); + assert!(block_on(host.exists(&extra)).unwrap()); + let (t, again) = run(&host, t, &append(vec![file("a", 1, 10)]), &mut Batch("b1")); + assert!(again.skipped); + assert_eq!((again.version, again.snapshot_id), (info.version, info.snapshot_id)); + assert_eq!(t.version, 2); + } + + #[test] + fn many_manifests_merge() { + let host = MemoryHost::new(1_000); + let mut t = create(&host, 2); + for i in 0..9 { + t = run(&host, t, &append(vec![file(&format!("f{i}"), i, 1)]), &mut NoHooks).0; + } + assert_eq!(block_on(t.manifests(&host)).unwrap().len(), 2, "eight merged into one, plus the new one"); + assert_eq!(live(&host).len(), 9); + } + + #[test] + fn a_clock_behind_the_table_keeps_the_logs_in_order() { + let host = MemoryHost::new(5_000); + let (t, _) = run(&host, create(&host, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); + let late = MemoryHost::new(1_000); + for uri in host.uris() { + block_on(late.put(&uri, block_on(host.get(&uri)).unwrap().unwrap())).unwrap(); + } + let (t, _) = run(&late, t, &append(vec![file("b", 1, 10)]), &mut NoHooks); + assert_eq!((t.metadata.current_snapshot().unwrap().timestamp_ms, t.metadata.last_updated_ms()), (5_000, 5_000)); + } + + #[test] + fn a_replace_needs_a_sequence_number_per_added_file() { + let host = MemoryHost::new(1_000); + let (t, _) = run(&host, create(&host, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); + let removed = files_named(&t, "a.parquet"); + let change = + Change::Replace { removed, added: vec![file("b", 1, 5), file("c", 1, 5)], sequence_numbers: vec![1] }; + let err = try_run(&host, t, &change, &mut NoHooks); + assert!(matches!(err, Err(Error::Invalid(_))), "{err:?}"); + } + + #[test] + fn only_v2_tables_are_written() { + let host = MemoryHost::new(1_000); + let err = try_run(&host, create(&host, 1), &append(vec![]), &mut NoHooks); + assert!(matches!(err, Err(Error::Unsupported(_)))); + } +} diff --git a/crates/lakebed-iceberg/src/datum.rs b/crates/lakebed-iceberg/src/datum.rs new file mode 100644 index 0000000..975bf55 --- /dev/null +++ b/crates/lakebed-iceberg/src/datum.rs @@ -0,0 +1,287 @@ +//! Typed values, and Iceberg's single-value binary serialization, which manifests use for column +//! bounds and manifest lists for partition bounds. +//! + +use std::cmp::Ordering; + +use crate::types::PrimitiveType; +use crate::{Result, corrupt}; + +/// A value of a primitive type. Dates, times and timestamps keep Iceberg's integer encoding. +#[derive(Clone, Debug, PartialEq)] +pub enum Datum { + /// `boolean`. + Boolean(bool), + /// `int`. + Int(i32), + /// `long`. + Long(i64), + /// `float`. + Float(f32), + /// `double`. + Double(f64), + /// `decimal(P,S)`: the unscaled value. + Decimal { + /// The value times 10^`scale`. + unscaled: i128, + /// Digits in all. + precision: u8, + /// Digits after the point. + scale: u8, + }, + /// `date`: days since 1970-01-01. + Date(i32), + /// `time`: microseconds since midnight. + Time(i64), + /// `timestamp`: microseconds since the epoch. + Timestamp(i64), + /// `timestamptz`: microseconds since the epoch, UTC. + Timestamptz(i64), + /// `timestamp_ns`: nanoseconds since the epoch. + TimestampNs(i64), + /// `timestamptz_ns`: nanoseconds since the epoch, UTC. + TimestamptzNs(i64), + /// `string`. + String(String), + /// `uuid`, big-endian. + Uuid([u8; 16]), + /// `fixed[L]`. + Fixed(Vec), + /// `binary`. + Binary(Vec), +} + +fn sized(bytes: &[u8], ty: &PrimitiveType) -> Result<[u8; N]> { + bytes.try_into().map_err(|_| corrupt(format!("a {ty} value of {} bytes", bytes.len()))) +} + +impl Datum { + /// Decodes Iceberg's single-value serialization of a `ty` value. A `long` or `double` stored in + /// 4 bytes was written before the column was promoted from `int` or `float`, and is widened. + pub fn from_bytes(ty: &PrimitiveType, bytes: &[u8]) -> Result { + Ok(match ty { + PrimitiveType::Boolean => match bytes { + [b] => Self::Boolean(*b != 0), + _ => return Err(corrupt(format!("a boolean value of {} bytes", bytes.len()))), + }, + PrimitiveType::Int => Self::Int(i32::from_le_bytes(sized(bytes, ty)?)), + PrimitiveType::Long if bytes.len() == 4 => Self::Long(i32::from_le_bytes(sized(bytes, ty)?).into()), + PrimitiveType::Long => Self::Long(i64::from_le_bytes(sized(bytes, ty)?)), + PrimitiveType::Float => Self::Float(f32::from_le_bytes(sized(bytes, ty)?)), + PrimitiveType::Double if bytes.len() == 4 => Self::Double(f32::from_le_bytes(sized(bytes, ty)?).into()), + PrimitiveType::Double => Self::Double(f64::from_le_bytes(sized(bytes, ty)?)), + PrimitiveType::Decimal { precision, scale } => { + if bytes.is_empty() || bytes.len() > 16 { + return Err(corrupt(format!("a decimal value of {} bytes", bytes.len()))); + } + Self::Decimal { unscaled: from_be(bytes), precision: *precision, scale: *scale } + } + PrimitiveType::Date => Self::Date(i32::from_le_bytes(sized(bytes, ty)?)), + PrimitiveType::Time => Self::Time(i64::from_le_bytes(sized(bytes, ty)?)), + PrimitiveType::Timestamp => Self::Timestamp(i64::from_le_bytes(sized(bytes, ty)?)), + PrimitiveType::Timestamptz => Self::Timestamptz(i64::from_le_bytes(sized(bytes, ty)?)), + PrimitiveType::TimestampNs => Self::TimestampNs(i64::from_le_bytes(sized(bytes, ty)?)), + PrimitiveType::TimestamptzNs => Self::TimestamptzNs(i64::from_le_bytes(sized(bytes, ty)?)), + PrimitiveType::String => { + Self::String(String::from_utf8(bytes.to_vec()).map_err(|_| corrupt("a string value that isn't UTF-8"))?) + } + PrimitiveType::Uuid => Self::Uuid(sized(bytes, ty)?), + PrimitiveType::Fixed(_) => Self::Fixed(bytes.to_vec()), + PrimitiveType::Binary => Self::Binary(bytes.to_vec()), + }) + } + + /// Iceberg's single-value serialization of this value. + pub fn to_bytes(&self) -> Vec { + match self { + Self::Boolean(b) => vec![u8::from(*b)], + Self::Int(v) | Self::Date(v) => v.to_le_bytes().to_vec(), + Self::Long(v) + | Self::Time(v) + | Self::Timestamp(v) + | Self::Timestamptz(v) + | Self::TimestampNs(v) + | Self::TimestamptzNs(v) => v.to_le_bytes().to_vec(), + Self::Float(v) => v.to_le_bytes().to_vec(), + Self::Double(v) => v.to_le_bytes().to_vec(), + Self::Decimal { unscaled, .. } => minimal_be(*unscaled), + Self::String(s) => s.as_bytes().to_vec(), + Self::Uuid(u) => u.to_vec(), + Self::Fixed(b) | Self::Binary(b) => b.clone(), + } + } + + /// The value's type. + pub fn primitive_type(&self) -> PrimitiveType { + match self { + Self::Boolean(_) => PrimitiveType::Boolean, + Self::Int(_) => PrimitiveType::Int, + Self::Long(_) => PrimitiveType::Long, + Self::Float(_) => PrimitiveType::Float, + Self::Double(_) => PrimitiveType::Double, + Self::Decimal { precision, scale, .. } => PrimitiveType::Decimal { precision: *precision, scale: *scale }, + Self::Date(_) => PrimitiveType::Date, + Self::Time(_) => PrimitiveType::Time, + Self::Timestamp(_) => PrimitiveType::Timestamp, + Self::Timestamptz(_) => PrimitiveType::Timestamptz, + Self::TimestampNs(_) => PrimitiveType::TimestampNs, + Self::TimestamptzNs(_) => PrimitiveType::TimestamptzNs, + Self::String(_) => PrimitiveType::String, + Self::Uuid(_) => PrimitiveType::Uuid, + Self::Fixed(b) => PrimitiveType::Fixed(b.len() as u64), + Self::Binary(_) => PrimitiveType::Binary, + } + } + + /// The value as an integer, for the integer-encoded types (`int`, `long`, dates, times, and + /// timestamps). + pub fn as_i64(&self) -> Option { + match self { + Self::Int(v) | Self::Date(v) => Some((*v).into()), + Self::Long(v) + | Self::Time(v) + | Self::Timestamp(v) + | Self::Timestamptz(v) + | Self::TimestampNs(v) + | Self::TimestamptzNs(v) => Some(*v), + _ => None, + } + } + + /// Whether this is a float or double NaN. + pub fn is_nan(&self) -> bool { + match self { + Self::Float(v) => v.is_nan(), + Self::Double(v) => v.is_nan(), + _ => false, + } + } +} + +/// Values of the same type compare by value (strings and bytes lexicographically, as Iceberg +/// orders them); values of different types don't compare. +impl PartialOrd for Datum { + fn partial_cmp(&self, other: &Self) -> Option { + match (self, other) { + (Self::Boolean(a), Self::Boolean(b)) => a.partial_cmp(b), + (Self::Int(a), Self::Int(b)) | (Self::Date(a), Self::Date(b)) => a.partial_cmp(b), + (Self::Long(a), Self::Long(b)) + | (Self::Time(a), Self::Time(b)) + | (Self::Timestamp(a), Self::Timestamp(b)) + | (Self::Timestamptz(a), Self::Timestamptz(b)) + | (Self::TimestampNs(a), Self::TimestampNs(b)) + | (Self::TimestamptzNs(a), Self::TimestamptzNs(b)) => a.partial_cmp(b), + (Self::Float(a), Self::Float(b)) => a.partial_cmp(b), + (Self::Double(a), Self::Double(b)) => a.partial_cmp(b), + (Self::Decimal { unscaled: a, scale: sa, .. }, Self::Decimal { unscaled: b, scale: sb, .. }) + if sa == sb => + { + a.partial_cmp(b) + } + (Self::String(a), Self::String(b)) => a.partial_cmp(b), + (Self::Uuid(a), Self::Uuid(b)) => a.partial_cmp(b), + (Self::Fixed(a), Self::Fixed(b)) | (Self::Binary(a), Self::Binary(b)) => a.partial_cmp(b), + _ => None, + } + } +} + +/// The shortest big-endian two's complement bytes of `v` (Iceberg's decimal encoding). +pub fn minimal_be(v: i128) -> Vec { + let bytes = v.to_be_bytes(); + let mut start = 0; + while start < bytes.len() - 1 { + let (b, next) = (bytes[start], bytes[start + 1]); + if (b == 0 && next & 0x80 == 0) || (b == 0xff && next & 0x80 != 0) { + start += 1; + } else { + break; + } + } + bytes[start..].to_vec() +} + +/// A big-endian two's complement value of at most 16 bytes, as an integer. +pub fn from_be(bytes: &[u8]) -> i128 { + let negative = bytes.first().is_some_and(|b| b & 0x80 != 0); + let mut v: i128 = if negative { -1 } else { 0 }; + for &b in &bytes[bytes.len().saturating_sub(16)..] { + v = (v << 8) | i128::from(b); + } + v +} + +#[cfg(test)] +mod tests { + use super::*; + + fn round_trip(d: &Datum) { + assert_eq!(&Datum::from_bytes(&d.primitive_type(), &d.to_bytes()).unwrap(), d); + } + + #[test] + fn values_round_trip() { + for d in [ + Datum::Boolean(true), + Datum::Int(-7), + Datum::Long(1 << 40), + Datum::Float(0.5), + Datum::Double(-1.25), + Datum::Decimal { unscaled: -12_345, precision: 10, scale: 2 }, + Datum::Date(-1), + Datum::Time(3_600_000_000), + Datum::Timestamptz(1_700_000_000_000_000), + Datum::TimestamptzNs(1_700_000_000_000_000_001), + Datum::String("truck_1".into()), + Datum::Uuid([7; 16]), + Datum::Fixed(vec![1, 2, 3]), + Datum::Binary(vec![]), + ] { + round_trip(&d); + } + } + + #[test] + fn encodings_follow_the_spec() { + assert_eq!(Datum::Int(7).to_bytes(), 7i32.to_le_bytes()); + assert_eq!(Datum::Boolean(false).to_bytes(), [0]); + assert_eq!(Datum::Decimal { unscaled: 128, precision: 5, scale: 0 }.to_bytes(), [0x00, 0x80]); + assert_eq!(Datum::String("ab".into()).to_bytes(), b"ab"); + } + + #[test] + fn promoted_columns_widen() { + assert_eq!(Datum::from_bytes(&PrimitiveType::Long, &(-3i32).to_le_bytes()).unwrap(), Datum::Long(-3)); + assert_eq!(Datum::from_bytes(&PrimitiveType::Double, &0.5f32.to_le_bytes()).unwrap(), Datum::Double(0.5)); + } + + #[test] + fn malformed_values_are_errors() { + assert!(Datum::from_bytes(&PrimitiveType::Long, &[1, 2, 3]).is_err()); + assert!(Datum::from_bytes(&PrimitiveType::Int, &[]).is_err()); + assert!(Datum::from_bytes(&PrimitiveType::Boolean, &[]).is_err()); + assert!(Datum::from_bytes(&PrimitiveType::String, &[0xff]).is_err()); + assert!(Datum::from_bytes(&PrimitiveType::Uuid, &[0; 15]).is_err()); + let decimal = PrimitiveType::Decimal { precision: 38, scale: 0 }; + assert!(Datum::from_bytes(&decimal, &[0; 17]).is_err()); + } + + #[test] + fn only_same_types_compare() { + assert!(Datum::Int(1) < Datum::Int(2)); + assert!(Datum::String("a".into()) < Datum::String("b".into())); + assert_eq!(Datum::Int(1).partial_cmp(&Datum::Long(2)), None); + assert_eq!(Datum::Double(f64::NAN).partial_cmp(&Datum::Double(1.0)), None); + } + + #[test] + fn decimal_bytes_are_minimal() { + assert_eq!(minimal_be(0), [0]); + assert_eq!(minimal_be(127), [0x7f]); + assert_eq!(minimal_be(128), [0x00, 0x80]); + assert_eq!(minimal_be(-1), [0xff]); + assert_eq!(minimal_be(-129), [0xff, 0x7f]); + assert_eq!(from_be(&minimal_be(-129)), -129); + assert_eq!(from_be(&minimal_be(1 << 40)), 1 << 40); + } +} diff --git a/crates/lakebed-iceberg/src/lib.rs b/crates/lakebed-iceberg/src/lib.rs new file mode 100644 index 0000000..446b6fb --- /dev/null +++ b/crates/lakebed-iceberg/src/lib.rs @@ -0,0 +1,89 @@ +//! Apache Iceberg table metadata, manifests and commits, with no I/O of its own: everything is read +//! and written through a [`Host`](lakebed_core::Host) the application implements, so the crate +//! builds for `wasm32-unknown-unknown` (browsers, Cloudflare Workers) as well as native targets. +//! +//! - Reading (always): table metadata of format versions 1 to 3 ([`Metadata`]), schemas and +//! partition specs, manifest lists and manifests with data and delete files ([`read_manifest`]), +//! column and partition values ([`Datum`]), and loading a table from its location ([`Table`]). +//! - `write`: format v2 manifests and manifest lists, metadata updates, and the commit protocol of +//! file-system tables (`commit`). +//! - `arrow`: the schema as an Arrow schema with field ids. +//! - `parquet-stats`: a data file's Iceberg statistics from its Parquet footer. +//! +//! ``` +//! use lakebed_iceberg::{Datum, PrimitiveType}; +//! +//! let bound = Datum::from_bytes(&PrimitiveType::Long, &42i64.to_le_bytes())?; +//! assert_eq!(bound, Datum::Long(42)); +//! # Ok::<(), lakebed_core::Error>(()) +//! ``` + +#[cfg(feature = "arrow")] +pub mod arrow; +mod avro; +#[cfg(feature = "write")] +pub mod commit; +mod datum; +pub mod manifest; +pub mod metadata; +mod partition; +#[cfg(feature = "parquet-stats")] +pub mod stats; +pub mod table; +mod types; + +#[cfg(feature = "write")] +pub use commit::{Change, CommitHooks, CommitInfo, CommitOptions, Extra, NoHooks, commit}; +pub use datum::{Datum, from_be, minimal_be}; +pub use lakebed_core::{Error, Result}; +pub use manifest::{ + Content, DataFile, Entry, FieldSummary, ManifestContent, ManifestFile, Status, read_manifest, read_manifest_list, +}; +pub use metadata::{Metadata, Snapshot}; +pub use partition::{PartitionField, PartitionFieldType, PartitionSpec, Transform}; +pub use table::Table; +pub use types::{NestedField, PrimitiveType, Schema, Type}; + +/// Snapshot summary keys of Iceberg's standard counters. +pub mod summary { + /// Data files added by the snapshot. + pub const ADDED_DATA_FILES: &str = "added-data-files"; + /// Rows added by the snapshot. + pub const ADDED_RECORDS: &str = "added-records"; + /// Bytes added by the snapshot. + pub const ADDED_FILES_SIZE: &str = "added-files-size"; + /// Data files removed by the snapshot. + pub const DELETED_DATA_FILES: &str = "deleted-data-files"; + /// Rows removed by the snapshot. + pub const DELETED_RECORDS: &str = "deleted-records"; + /// Bytes removed by the snapshot. + pub const REMOVED_FILES_SIZE: &str = "removed-files-size"; + /// Data files in the table after the snapshot. + pub const TOTAL_DATA_FILES: &str = "total-data-files"; + /// Rows in the table after the snapshot. + pub const TOTAL_RECORDS: &str = "total-records"; + /// Bytes in the table after the snapshot. + pub const TOTAL_FILES_SIZE: &str = "total-files-size"; + /// Delete files in the table. + pub const TOTAL_DELETE_FILES: &str = "total-delete-files"; + /// Position deletes in the table. + pub const TOTAL_POSITION_DELETES: &str = "total-position-deletes"; + /// Equality deletes in the table. + pub const TOTAL_EQUALITY_DELETES: &str = "total-equality-deletes"; +} + +fn corrupt(msg: impl Into) -> Error { + Error::Corrupt(msg.into()) +} + +/// Runs a future whose host calls complete at once ([`lakebed_core::MemoryHost`]'s do). +#[cfg(test)] +fn block_on(f: F) -> F::Output { + let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); + let mut f = std::pin::pin!(f); + loop { + if let std::task::Poll::Ready(v) = f.as_mut().poll(&mut cx) { + return v; + } + } +} diff --git a/crates/lakebed-iceberg/src/manifest.rs b/crates/lakebed-iceberg/src/manifest.rs new file mode 100644 index 0000000..738cf66 --- /dev/null +++ b/crates/lakebed-iceberg/src/manifest.rs @@ -0,0 +1,942 @@ +//! Manifests and manifest lists: the Avro files that list a snapshot's data and delete files. +//! Reading covers format versions 1 to 3; writing (feature `write`) makes format v2 data +//! manifests in the layout iceberg-rust writes: the same Avro schemas, field ids and header +//! metadata. +//! + +use std::collections::BTreeMap; +use std::sync::Arc; + +use apache_avro::types::Value; +use serde_json::Value as Json; + +use crate::avro::{Header, field, field_of, int, long, records, required, string}; +use crate::datum::{Datum, from_be}; +use crate::partition::{PartitionFieldType, PartitionSpec}; +use crate::types::{PrimitiveType, Schema}; +use crate::{Result, corrupt}; + +/// Whether a manifest entry adds, keeps or removes its file. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Status { + /// Carried over from an earlier snapshot. + Existing = 0, + /// Added by the manifest's snapshot. + Added = 1, + /// Removed by the manifest's snapshot. + Deleted = 2, +} + +/// What a file holds. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum Content { + /// Rows. + #[default] + Data = 0, + /// Deleted row positions (a Parquet delete file, or a Puffin deletion vector in format v3). + PositionDeletes = 1, + /// Deleted rows by the values of `equality_ids` columns. + EqualityDeletes = 2, +} + +/// What a manifest lists. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum ManifestContent { + /// Data files. + #[default] + Data = 0, + /// Delete files. + Deletes = 1, +} + +/// A data or delete file as a manifest records it. Bounds are Iceberg's single-value binary +/// encoding ([`Datum::from_bytes`]), keyed by field id. +#[derive(Clone, Debug, PartialEq)] +pub struct DataFile { + /// What the file holds. + pub content: Content, + /// Absolute URI. + pub path: String, + /// `PARQUET`, `AVRO`, `ORC` or `PUFFIN`. + pub file_format: String, + /// The partition tuple, in the order of the spec's fields (empty for unpartitioned files). + pub partition: Vec>, + /// Rows. + pub record_count: u64, + /// Bytes. + pub file_size: u64, + /// Compressed bytes per field id. + pub column_sizes: BTreeMap, + /// Values per field id. + pub value_counts: BTreeMap, + /// Nulls per field id. + pub null_counts: BTreeMap, + /// NaNs per field id. + pub nan_counts: BTreeMap, + /// Smallest value per field id. + pub lower_bounds: BTreeMap>, + /// Largest value per field id. + pub upper_bounds: BTreeMap>, + /// Row group offsets. + pub split_offsets: Option>, + /// For equality deletes: the columns that identify deleted rows. + pub equality_ids: Option>, + /// The table sort order the file follows. + pub sort_order_id: Option, + /// The row id of the file's first row (format v3). + pub first_row_id: Option, + /// For position deletes and deletion vectors: the one data file they apply to. + pub referenced_data_file: Option, + /// For deletion vectors: the vector's offset in the Puffin file. + pub content_offset: Option, + /// For deletion vectors: the vector's length. + pub content_size: Option, +} + +impl Default for DataFile { + fn default() -> Self { + Self { + content: Content::Data, + path: String::new(), + file_format: "PARQUET".into(), + partition: vec![], + record_count: 0, + file_size: 0, + column_sizes: BTreeMap::new(), + value_counts: BTreeMap::new(), + null_counts: BTreeMap::new(), + nan_counts: BTreeMap::new(), + lower_bounds: BTreeMap::new(), + upper_bounds: BTreeMap::new(), + split_offsets: None, + equality_ids: None, + sort_order_id: None, + first_row_id: None, + referenced_data_file: None, + content_offset: None, + content_size: None, + } + } +} + +/// A file in a snapshot, with the sequence numbers that order its rows against other files'. +#[derive(Clone, Debug, PartialEq)] +pub struct Entry { + /// Added, existing or deleted. + pub status: Status, + /// The snapshot that added (or deleted) the file. + pub snapshot_id: i64, + /// Data sequence number: orders the file's rows against other files' rows and deletes. + pub sequence_number: i64, + /// Sequence number of the commit that added the file. + pub file_sequence_number: i64, + /// The file. + pub file: Arc, +} + +/// An entry of a manifest list. The default (sequence numbers and snapshot 0) suits reading a +/// manifest's files without the list. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ManifestFile { + /// Absolute URI of the manifest. + pub path: String, + /// Its size in bytes. + pub length: i64, + /// The partition spec its files use. + pub partition_spec_id: i32, + /// Data or delete files. + pub content: ManifestContent, + /// Sequence number of the snapshot that wrote it. + pub sequence_number: i64, + /// Smallest data sequence number of its files. + pub min_sequence_number: i64, + /// The snapshot that wrote it. + pub added_snapshot_id: i64, + /// Files added. + pub added_files: i32, + /// Files carried over. + pub existing_files: i32, + /// Files removed. + pub deleted_files: i32, + /// Rows in added files. + pub added_rows: i64, + /// Rows in existing files. + pub existing_rows: i64, + /// Rows in deleted files. + pub deleted_rows: i64, + /// The range of each partition field's values (`None` for unpartitioned manifests). + pub partitions: Option>, + /// The row id of the manifest's first row (format v3). + pub first_row_id: Option, +} + +/// The range of one partition field's values in a manifest. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FieldSummary { + /// Some file has no value for the field. + pub contains_null: bool, + /// Some file's value is NaN (`None`: not recorded). + pub contains_nan: Option, + /// Lowest value, single-value serialized. + pub lower: Option>, + /// Highest value, single-value serialized. + pub upper: Option>, +} + +/// A sequence number the manifest list assigns when the snapshot commits. +pub const UNASSIGNED: i64 = -1; + +// --- reading ------------------------------------------------------------------------------------- + +fn u64_of(v: Option) -> u64 { + v.and_then(|v| u64::try_from(v).ok()).unwrap_or(0) +} + +/// The manifests of a manifest list (format v1 to v3). +pub fn read_manifest_list(bytes: &[u8]) -> Result> { + let (records, _) = records(bytes)?; + records + .iter() + .map(|r| { + let partitions = match field(r, "partitions") { + Some(Value::Array(items)) => Some(items.iter().map(field_summary).collect()), + _ => None, + }; + let count = |names: &[&str]| int(field_of(r, names)).unwrap_or(0); + let rows = |name: &str| long(field(r, name)).unwrap_or(0); + Ok(ManifestFile { + path: string(field(r, "manifest_path")) + .ok_or_else(|| corrupt("manifest list entry without a path"))? + .to_string(), + length: required(long(field(r, "manifest_length")), "length")?, + partition_spec_id: int(field(r, "partition_spec_id")).unwrap_or(0), + content: match long(field(r, "content")) { + None | Some(0) => ManifestContent::Data, + Some(1) => ManifestContent::Deletes, + Some(other) => return Err(corrupt(format!("manifest content {other}"))), + }, + sequence_number: long(field(r, "sequence_number")).unwrap_or(0), + min_sequence_number: long(field(r, "min_sequence_number")).unwrap_or(0), + added_snapshot_id: long(field(r, "added_snapshot_id")).unwrap_or(0), + // Format v1 names the counts `*_data_files_count`. + added_files: count(&["added_files_count", "added_data_files_count"]), + existing_files: count(&["existing_files_count", "existing_data_files_count"]), + deleted_files: count(&["deleted_files_count", "deleted_data_files_count"]), + added_rows: rows("added_rows_count"), + existing_rows: rows("existing_rows_count"), + deleted_rows: rows("deleted_rows_count"), + partitions, + first_row_id: long(field(r, "first_row_id")), + }) + }) + .collect() +} + +fn field_summary(p: &Value) -> FieldSummary { + let bytes = |name: &str| match field(p, name) { + Some(Value::Bytes(b)) => Some(b.clone()), + _ => None, + }; + FieldSummary { + contains_null: matches!(field(p, "contains_null"), Some(Value::Boolean(true))), + contains_nan: match field(p, "contains_nan") { + Some(Value::Boolean(b)) => Some(*b), + _ => None, + }, + lower: bytes("lower_bound"), + upper: bytes("upper_bound"), + } +} + +fn long_map(v: Option<&Value>) -> BTreeMap { + let Some(Value::Array(items)) = v else { return BTreeMap::new() }; + items + .iter() + .filter_map(|kv| Some((int(field(kv, "key"))?, u64::try_from(long(field(kv, "value"))?).ok()?))) + .collect() +} + +fn bytes_map(v: Option<&Value>) -> BTreeMap> { + let Some(Value::Array(items)) = v else { return BTreeMap::new() }; + items + .iter() + .filter_map(|kv| match field(kv, "value") { + Some(Value::Bytes(b)) => Some((int(field(kv, "key"))?, b.clone())), + _ => None, + }) + .collect() +} + +fn longs(v: Option<&Value>) -> Option> { + match v { + Some(Value::Array(items)) => Some(items.iter().filter_map(|x| long(Some(x))).collect()), + _ => None, + } +} + +/// The partition field types a manifest's header records (its `schema` and `partition-spec`), if +/// it records them and they parse. +fn header_partition_type(metadata: &Header) -> Option> { + let json = |key: &str| serde_json::from_slice::(metadata.get(key)?).ok(); + let schema = Schema::from_json(&json("schema")?).ok()?; + let spec_id = std::str::from_utf8(metadata.get("partition-spec-id")?).ok()?.parse().ok()?; + let spec = PartitionSpec::from_fields(spec_id, &json("partition-spec")?).ok()?; + spec.partition_type(&schema).ok() +} + +/// A partition value. `ty` is the field's type when the manifest header gives it; without it the +/// Avro type decides. +fn partition_datum(v: &Value, ty: Option<&PrimitiveType>) -> Result> { + use PrimitiveType as P; + Ok(Some(match (v, ty) { + (Value::Null, _) => return Ok(None), + (Value::Boolean(b), _) => Datum::Boolean(*b), + (Value::Int(n) | Value::Date(n), Some(P::Date)) | (Value::Date(n), None) => Datum::Date(*n), + (Value::Int(n), _) => Datum::Int(*n), + (Value::Long(n) | Value::TimeMicros(n), Some(P::Time)) | (Value::TimeMicros(n), None) => Datum::Time(*n), + (Value::Long(n) | Value::TimestampMicros(n) | Value::LocalTimestampMicros(n), Some(P::Timestamp)) + | (Value::LocalTimestampMicros(n), None) => Datum::Timestamp(*n), + (Value::Long(n) | Value::TimestampMicros(n), Some(P::Timestamptz)) | (Value::TimestampMicros(n), None) => { + Datum::Timestamptz(*n) + } + (Value::Long(n) | Value::TimestampNanos(n) | Value::LocalTimestampNanos(n), Some(P::TimestampNs)) + | (Value::LocalTimestampNanos(n), None) => Datum::TimestampNs(*n), + (Value::Long(n) | Value::TimestampNanos(n), Some(P::TimestamptzNs)) | (Value::TimestampNanos(n), None) => { + Datum::TimestamptzNs(*n) + } + (Value::Long(n), _) => Datum::Long(*n), + (Value::Float(x), _) => Datum::Float(*x), + (Value::Double(x), _) => Datum::Double(*x), + (Value::String(s), _) => Datum::String(s.clone()), + (Value::Bytes(b) | Value::Fixed(_, b), Some(P::Decimal { precision, scale })) if b.len() <= 16 => { + Datum::Decimal { unscaled: from_be(b), precision: *precision, scale: *scale } + } + (Value::Decimal(d), Some(P::Decimal { precision, scale })) => { + let b = Vec::::try_from(d).map_err(crate::avro::error)?; + if b.len() > 16 { + return Err(corrupt("a decimal partition value of more than 16 bytes")); + } + Datum::Decimal { unscaled: from_be(&b), precision: *precision, scale: *scale } + } + (Value::Uuid(u), _) => Datum::Uuid(*u.as_bytes()), + (Value::Fixed(16, b), Some(P::Uuid)) => Datum::Uuid(b.as_slice().try_into().map_err(crate::avro::error)?), + (Value::Fixed(_, b), _) => Datum::Fixed(b.clone()), + (Value::Bytes(b), _) => Datum::Binary(b.clone()), + (other, _) => return Err(corrupt(format!("partition value {other:?}"))), + })) +} + +/// The entries of `manifest` (format v1 to v3), with the snapshot ids and sequence numbers they +/// inherit from it filled in. +pub fn read_manifest(manifest: &ManifestFile, bytes: &[u8]) -> Result> { + let (records, metadata) = records(bytes)?; + let types = header_partition_type(&metadata); + records + .iter() + .map(|r| { + let status = match int(field(r, "status")) { + Some(0) => Status::Existing, + Some(1) => Status::Added, + Some(2) => Status::Deleted, + other => return Err(corrupt(format!("manifest entry status {other:?}"))), + }; + let df = required(field(r, "data_file"), "data_file")?; + let partition = match field(df, "partition") { + Some(Value::Record(fields)) => fields + .iter() + .enumerate() + .map(|(i, (_, v))| { + let ty = types.as_ref().and_then(|t| t.get(i)).map(|t| &t.ty); + partition_datum(crate::avro::strip_union(v), ty) + }) + .collect::>()?, + _ => vec![], + }; + let file = DataFile { + content: match long(field(df, "content")) { + None | Some(0) => Content::Data, + Some(1) => Content::PositionDeletes, + Some(2) => Content::EqualityDeletes, + Some(other) => return Err(corrupt(format!("data file content {other}"))), + }, + path: string(field(df, "file_path")).ok_or_else(|| corrupt("data file without a path"))?.to_string(), + file_format: string(field(df, "file_format")).unwrap_or("PARQUET").to_string(), + partition, + record_count: u64_of(Some(required(long(field(df, "record_count")), "record_count")?)), + file_size: u64_of(Some(required(long(field(df, "file_size_in_bytes")), "file size")?)), + column_sizes: long_map(field(df, "column_sizes")), + value_counts: long_map(field(df, "value_counts")), + null_counts: long_map(field(df, "null_value_counts")), + nan_counts: long_map(field(df, "nan_value_counts")), + lower_bounds: bytes_map(field(df, "lower_bounds")), + upper_bounds: bytes_map(field(df, "upper_bounds")), + split_offsets: longs(field(df, "split_offsets")), + equality_ids: longs(field(df, "equality_ids")) + .map(|ids| ids.into_iter().filter_map(|id| i32::try_from(id).ok()).collect()), + sort_order_id: int(field(df, "sort_order_id")), + first_row_id: long(field(df, "first_row_id")), + referenced_data_file: string(field(df, "referenced_data_file")).map(str::to_string), + content_offset: long(field(df, "content_offset")), + content_size: long(field(df, "content_size_in_bytes")), + }; + // Entries without sequence numbers inherit the manifest's (Iceberg's rule; format v1 + // manifests have none and the list gives 0). + let inherit = |v: Option| v.unwrap_or(manifest.sequence_number); + Ok(Entry { + status, + snapshot_id: long(field(r, "snapshot_id")).unwrap_or(manifest.added_snapshot_id), + sequence_number: inherit(long(field(r, "sequence_number"))), + file_sequence_number: inherit(long(field(r, "file_sequence_number"))), + file: Arc::new(file), + }) + }) + .collect() +} + +// --- writing ------------------------------------------------------------------------------------- + +#[cfg(feature = "write")] +pub use write::{ManifestContext, ManifestWrite, manifest_file, write_manifest, write_manifest_list}; + +#[cfg(feature = "write")] +mod write { + use std::collections::BTreeMap; + use std::fmt::Write as _; + use std::sync::Arc; + + use apache_avro::types::Value; + use apache_avro::{Codec, Schema as AvroSchema, Writer}; + + use super::{Content, DataFile, FieldSummary, ManifestContent, ManifestFile, Status, UNASSIGNED}; + use crate::avro::{error, null, some}; + use crate::datum::Datum; + use crate::partition::PartitionFieldType; + use crate::types::PrimitiveType; + use crate::{Error, Result}; + + /// What a manifest's header records about the table. + #[derive(Clone, Debug)] + pub struct ManifestContext<'a> { + /// The current schema as Iceberg JSON. + pub schema_json: &'a str, + /// Its id. + pub schema_id: i32, + /// The partition spec's fields as Iceberg JSON (`[{"source-id":…,"field-id":…,…}]`). + pub spec_fields_json: &'a str, + /// The spec's id. + pub spec_id: i32, + /// The spec's fields with the types of their values ([`crate::PartitionSpec::partition_type`]). + pub partition_type: &'a [PartitionFieldType], + } + + /// An entry to write into a manifest: `None` fields are inherited from the manifest list. + #[derive(Clone, Debug)] + pub struct ManifestWrite { + /// Added, existing or deleted. + pub status: Status, + /// The snapshot, or `None` to inherit the manifest's. + pub snapshot_id: Option, + /// Data sequence number, or `None` to inherit. + pub sequence_number: Option, + /// File sequence number, or `None` to inherit. + pub file_sequence_number: Option, + /// The file. + pub file: Arc, + } + + /// A name Avro accepts: Iceberg's convention replaces a leading digit with `_` and the digit, and + /// other characters Avro doesn't allow with `_x` and their code in hex. + fn avro_name(name: &str) -> String { + let mut out = String::with_capacity(name.len()); + for (i, c) in name.chars().enumerate() { + match c { + 'a'..='z' | 'A'..='Z' | '_' => out.push(c), + '0'..='9' if i > 0 => out.push(c), + '0'..='9' => { + out.push('_'); + out.push(c); + } + _ => { + let _ = write!(out, "_x{:X}", u32::from(c)); + } + } + } + out + } + + /// The Avro type of partition values of type `ty`. + fn avro_type(ty: &PrimitiveType) -> Result { + let logical = |base: &str, logical: &str, utc: Option| { + let utc = utc.map_or_else(String::new, |u| format!(r#", "adjust-to-utc": {u}"#)); + format!(r#"{{"type": "{base}", "logicalType": "{logical}"{utc}}}"#) + }; + Ok(match ty { + PrimitiveType::Boolean => r#""boolean""#.into(), + PrimitiveType::Int => r#""int""#.into(), + PrimitiveType::Long => r#""long""#.into(), + PrimitiveType::Float => r#""float""#.into(), + PrimitiveType::Double => r#""double""#.into(), + PrimitiveType::String => r#""string""#.into(), + PrimitiveType::Binary => r#""bytes""#.into(), + PrimitiveType::Date => logical("int", "date", None), + PrimitiveType::Time => logical("long", "time-micros", None), + PrimitiveType::Timestamp => logical("long", "timestamp-micros", Some(false)), + PrimitiveType::Timestamptz => logical("long", "timestamp-micros", Some(true)), + PrimitiveType::TimestampNs => logical("long", "timestamp-nanos", Some(false)), + PrimitiveType::TimestamptzNs => logical("long", "timestamp-nanos", Some(true)), + PrimitiveType::Decimal { .. } | PrimitiveType::Uuid | PrimitiveType::Fixed(_) => { + return Err(Error::Unsupported(format!("writing {ty} partition values"))); + } + }) + } + + /// A partition value as Avro, checked against the field's type. + fn avro_value(d: &Datum, ty: &PrimitiveType) -> Result { + if d.primitive_type() != *ty { + return Err(Error::Invalid(format!("a {} partition value for a {ty} field", d.primitive_type()))); + } + Ok(match d { + Datum::Boolean(b) => Value::Boolean(*b), + Datum::Int(v) => Value::Int(*v), + Datum::Date(v) => Value::Date(*v), + Datum::Long(v) => Value::Long(*v), + Datum::Time(v) => Value::TimeMicros(*v), + Datum::Timestamp(v) | Datum::Timestamptz(v) => Value::TimestampMicros(*v), + Datum::TimestampNs(v) | Datum::TimestamptzNs(v) => Value::TimestampNanos(*v), + Datum::Float(v) => Value::Float(*v), + Datum::Double(v) => Value::Double(*v), + Datum::String(s) => Value::String(s.clone()), + Datum::Binary(b) => Value::Bytes(b.clone()), + Datum::Decimal { .. } | Datum::Uuid(_) | Datum::Fixed(_) => { + return Err(Error::Unsupported(format!("writing {ty} partition values"))); + } + }) + } + + /// The Avro schema of manifest entries for a table partitioned as `partition`. + fn manifest_schema(partition: &[PartitionFieldType]) -> Result { + let partition_fields = partition + .iter() + .map(|p| { + Ok(format!( + r#"{{"name": "{}", "type": ["null", {}], "default": null, "field-id": {}}}"#, + avro_name(&p.name), + avro_type(&p.ty)?, + p.field_id + )) + }) + .collect::>>()? + .join(", "); + let map = |name: &str, key: i32, value: i32, ty: &str| { + format!( + r#"{{"name": "{name}", "type": ["null", {{"type": "array", "items": {{"type": "record", "name": "k{key}_v{value}", "fields": [{{"name": "key", "type": "int", "field-id": {key}}}, {{"name": "value", "type": "{ty}", "field-id": {value}}}]}}, "logicalType": "map"}}], "default": null"# + ) + }; + let json = format!( + r#"{{"type": "record", "name": "manifest_entry", "fields": [ + {{"name": "status", "type": "int", "field-id": 0}}, + {{"name": "snapshot_id", "type": ["null", "long"], "default": null, "field-id": 1}}, + {{"name": "sequence_number", "type": ["null", "long"], "default": null, "field-id": 3}}, + {{"name": "file_sequence_number", "type": ["null", "long"], "default": null, "field-id": 4}}, + {{"name": "data_file", "type": {{"type": "record", "name": "r2", "fields": [ + {{"name": "content", "type": "int", "default": 0, "field-id": 134}}, + {{"name": "file_path", "type": "string", "field-id": 100}}, + {{"name": "file_format", "type": "string", "field-id": 101}}, + {{"name": "partition", "type": {{"type": "record", "name": "r102", "fields": [{partition_fields}]}}, "field-id": 102}}, + {{"name": "record_count", "type": "long", "field-id": 103}}, + {{"name": "file_size_in_bytes", "type": "long", "field-id": 104}}, + {}, "field-id": 108}}, + {}, "field-id": 109}}, + {}, "field-id": 110}}, + {}, "field-id": 137}}, + {}, "field-id": 125}}, + {}, "field-id": 128}}, + {{"name": "key_metadata", "type": ["null", "bytes"], "default": null, "field-id": 131}}, + {{"name": "split_offsets", "type": ["null", {{"type": "array", "items": "long", "element-id": 133}}], "default": null, "field-id": 132}}, + {{"name": "equality_ids", "type": ["null", {{"type": "array", "items": "int", "element-id": 136}}], "default": null, "field-id": 135}}, + {{"name": "sort_order_id", "type": ["null", "int"], "default": null, "field-id": 140}}, + {{"name": "first_row_id", "type": ["null", "long"], "default": null, "field-id": 142}}, + {{"name": "referenced_data_file", "type": ["null", "string"], "default": null, "field-id": 143}}, + {{"name": "content_offset", "type": ["null", "long"], "default": null, "field-id": 144}}, + {{"name": "content_size_in_bytes", "type": ["null", "long"], "default": null, "field-id": 145}} + ]}}, "field-id": 2}} + ]}}"#, + map("column_sizes", 117, 118, "long"), + map("value_counts", 119, 120, "long"), + map("null_value_counts", 121, 122, "long"), + map("nan_value_counts", 138, 139, "long"), + map("lower_bounds", 126, 127, "bytes"), + map("upper_bounds", 129, 130, "bytes"), + ); + AvroSchema::parse_str(&json).map_err(error) + } + + const MANIFEST_LIST_SCHEMA: &str = r#"{"type": "record", "name": "manifest_file", "fields": [ + {"name": "manifest_path", "type": "string", "field-id": 500}, + {"name": "manifest_length", "type": "long", "field-id": 501}, + {"name": "partition_spec_id", "type": "int", "field-id": 502}, + {"name": "content", "type": "int", "field-id": 517}, + {"name": "sequence_number", "type": "long", "field-id": 515}, + {"name": "min_sequence_number", "type": "long", "field-id": 516}, + {"name": "added_snapshot_id", "type": "long", "field-id": 503}, + {"name": "added_files_count", "type": "int", "field-id": 504}, + {"name": "existing_files_count", "type": "int", "field-id": 505}, + {"name": "deleted_files_count", "type": "int", "field-id": 506}, + {"name": "added_rows_count", "type": "long", "field-id": 512}, + {"name": "existing_rows_count", "type": "long", "field-id": 513}, + {"name": "deleted_rows_count", "type": "long", "field-id": 514}, + {"name": "partitions", "type": ["null", {"type": "array", "items": {"type": "record", "name": "r508", "fields": [ + {"name": "contains_null", "type": "boolean", "field-id": 509}, + {"name": "contains_nan", "type": ["null", "boolean"], "default": null, "field-id": 518}, + {"name": "lower_bound", "type": ["null", "bytes"], "default": null, "field-id": 510}, + {"name": "upper_bound", "type": ["null", "bytes"], "default": null, "field-id": 511} + ]}, "element-id": 508}], "default": null, "field-id": 507}, + {"name": "key_metadata", "type": ["null", "bytes"], "default": null, "field-id": 519} + ]}"#; + + fn i64_of(n: u64) -> i64 { + i64::try_from(n).unwrap_or(i64::MAX) + } + + fn long_map(m: &BTreeMap) -> Value { + some(Value::Array( + m.iter() + .map(|(k, v)| { + Value::Record(vec![("key".into(), Value::Int(*k)), ("value".into(), Value::Long(i64_of(*v)))]) + }) + .collect(), + )) + } + + fn bytes_map(m: &BTreeMap>) -> Value { + some(Value::Array( + m.iter() + .map(|(k, v)| { + Value::Record(vec![("key".into(), Value::Int(*k)), ("value".into(), Value::Bytes(v.clone()))]) + }) + .collect(), + )) + } + + fn opt(v: Option, f: impl FnOnce(T) -> Value) -> Value { + v.map_or_else(null, |v| some(f(v))) + } + + /// Writes a format v2 data manifest of `entries`. Added entries leave their sequence numbers to + /// be inherited from the manifest list (`None` in `sequence_number` means the commit's own); + /// existing and deleted entries keep theirs. + pub fn write_manifest(ctx: &ManifestContext<'_>, entries: &[ManifestWrite]) -> Result> { + let schema = manifest_schema(ctx.partition_type)?; + let mut w = Writer::with_codec(&schema, vec![], Codec::Null).map_err(error)?; + for (k, v) in [ + ("schema", ctx.schema_json.to_string()), + ("schema-id", ctx.schema_id.to_string()), + ("partition-spec", ctx.spec_fields_json.to_string()), + ("partition-spec-id", ctx.spec_id.to_string()), + ("format-version", "2".to_string()), + ("content", "data".to_string()), + ] { + w.add_user_metadata(k.to_string(), v).map_err(error)?; + } + for e in entries { + let f = &e.file; + if f.content != Content::Data { + return Err(Error::Unsupported("writing delete files".into())); + } + if f.partition.len() != ctx.partition_type.len() { + return Err(Error::Invalid(format!( + "{}: {} partition values for a spec of {} fields", + f.path, + f.partition.len(), + ctx.partition_type.len() + ))); + } + let partition = ctx + .partition_type + .iter() + .zip(&f.partition) + .map(|(p, v)| { + Ok((avro_name(&p.name), opt(v.as_ref().map(|v| avro_value(v, &p.ty)).transpose()?, |v| v))) + }) + .collect::>>()?; + let data_file = Value::Record(vec![ + ("content".into(), Value::Int(f.content as i32)), + ("file_path".into(), Value::String(f.path.clone())), + ("file_format".into(), Value::String(f.file_format.clone())), + ("partition".into(), Value::Record(partition)), + ("record_count".into(), Value::Long(i64_of(f.record_count))), + ("file_size_in_bytes".into(), Value::Long(i64_of(f.file_size))), + ("column_sizes".into(), long_map(&f.column_sizes)), + ("value_counts".into(), long_map(&f.value_counts)), + ("null_value_counts".into(), long_map(&f.null_counts)), + ("nan_value_counts".into(), long_map(&f.nan_counts)), + ("lower_bounds".into(), bytes_map(&f.lower_bounds)), + ("upper_bounds".into(), bytes_map(&f.upper_bounds)), + ("key_metadata".into(), null()), + ( + "split_offsets".into(), + opt(f.split_offsets.as_ref(), |o| Value::Array(o.iter().map(|v| Value::Long(*v)).collect())), + ), + ( + "equality_ids".into(), + opt(f.equality_ids.as_ref(), |ids| Value::Array(ids.iter().map(|v| Value::Int(*v)).collect())), + ), + ("sort_order_id".into(), opt(f.sort_order_id, Value::Int)), + ("first_row_id".into(), opt(f.first_row_id, Value::Long)), + ("referenced_data_file".into(), opt(f.referenced_data_file.clone(), Value::String)), + ("content_offset".into(), opt(f.content_offset, Value::Long)), + ("content_size_in_bytes".into(), opt(f.content_size, Value::Long)), + ]); + w.append_value(Value::Record(vec![ + ("status".into(), Value::Int(e.status as i32)), + ("snapshot_id".into(), opt(e.snapshot_id, Value::Long)), + ("sequence_number".into(), opt(e.sequence_number, Value::Long)), + ("file_sequence_number".into(), opt(e.file_sequence_number, Value::Long)), + ("data_file".into(), data_file), + ])) + .map_err(error)?; + } + w.into_inner().map_err(error) + } + + /// Writes a format v2 manifest list for snapshot `snapshot_id` (child of `parent`) with sequence + /// number `sequence_number`. Manifests with sequence number [`UNASSIGNED`] get the snapshot's + /// (they were written by it). + pub fn write_manifest_list( + manifests: &[ManifestFile], + snapshot_id: i64, + parent: Option, + sequence_number: i64, + ) -> Result> { + let schema = AvroSchema::parse_str(MANIFEST_LIST_SCHEMA).map_err(error)?; + let mut w = Writer::with_codec(&schema, vec![], Codec::Null).map_err(error)?; + let parent = parent.map_or_else(|| "null".into(), |p| p.to_string()); + for (k, v) in [ + ("snapshot-id", snapshot_id.to_string()), + ("parent-snapshot-id", parent), + ("sequence-number", sequence_number.to_string()), + ("format-version", "2".to_string()), + ] { + w.add_user_metadata(k.to_string(), v).map_err(error)?; + } + for m in manifests { + let assign = |v: i64| if v == UNASSIGNED { sequence_number } else { v }; + let partitions = opt(m.partitions.as_ref(), |fields| { + Value::Array( + fields + .iter() + .map(|p| { + Value::Record(vec![ + ("contains_null".into(), Value::Boolean(p.contains_null)), + ("contains_nan".into(), opt(p.contains_nan, Value::Boolean)), + ("lower_bound".into(), opt(p.lower.clone(), Value::Bytes)), + ("upper_bound".into(), opt(p.upper.clone(), Value::Bytes)), + ]) + }) + .collect(), + ) + }); + w.append_value(Value::Record(vec![ + ("manifest_path".into(), Value::String(m.path.clone())), + ("manifest_length".into(), Value::Long(m.length)), + ("partition_spec_id".into(), Value::Int(m.partition_spec_id)), + ("content".into(), Value::Int(m.content as i32)), + ("sequence_number".into(), Value::Long(assign(m.sequence_number))), + ("min_sequence_number".into(), Value::Long(assign(m.min_sequence_number))), + ("added_snapshot_id".into(), Value::Long(m.added_snapshot_id)), + ("added_files_count".into(), Value::Int(m.added_files)), + ("existing_files_count".into(), Value::Int(m.existing_files)), + ("deleted_files_count".into(), Value::Int(m.deleted_files)), + ("added_rows_count".into(), Value::Long(m.added_rows)), + ("existing_rows_count".into(), Value::Long(m.existing_rows)), + ("deleted_rows_count".into(), Value::Long(m.deleted_rows)), + ("partitions".into(), partitions), + ("key_metadata".into(), null()), + ])) + .map_err(error)?; + } + w.into_inner().map_err(error) + } + + /// The manifest-list entry of a data manifest written as `bytes` at `path` with `entries`. + pub fn manifest_file( + path: &str, + bytes: &[u8], + spec_id: i32, + snapshot_id: i64, + entries: &[ManifestWrite], + partition_type: &[PartitionFieldType], + ) -> ManifestFile { + let count = |s: Status| i32::try_from(entries.iter().filter(|e| e.status == s).count()).unwrap_or(i32::MAX); + let rows = |s: Status| i64_of(entries.iter().filter(|e| e.status == s).map(|e| e.file.record_count).sum()); + // Entries that inherit the commit's sequence number sort after every explicit one. + let min_sequence_number = entries.iter().filter_map(|e| e.sequence_number).min().unwrap_or(UNASSIGNED); + let partitions = (!partition_type.is_empty()).then(|| { + partition_type + .iter() + .enumerate() + .map(|(i, p)| { + let values = || entries.iter().map(|e| e.file.partition.get(i).and_then(Option::as_ref)); + let floating = matches!(p.ty, PrimitiveType::Float | PrimitiveType::Double); + let ordered = || values().flatten().filter(|v| !v.is_nan()); + let pick = |better: std::cmp::Ordering| { + ordered().fold(None::<&Datum>, |best, v| match best { + Some(b) if v.partial_cmp(b) != Some(better) => Some(b), + _ => Some(v), + }) + }; + FieldSummary { + contains_null: values().any(|v| v.is_none()), + contains_nan: floating.then(|| values().flatten().any(Datum::is_nan)), + lower: pick(std::cmp::Ordering::Less).map(Datum::to_bytes), + upper: pick(std::cmp::Ordering::Greater).map(Datum::to_bytes), + } + }) + .collect() + }); + ManifestFile { + path: path.to_string(), + length: i64::try_from(bytes.len()).unwrap_or(i64::MAX), + partition_spec_id: spec_id, + content: ManifestContent::Data, + sequence_number: UNASSIGNED, + min_sequence_number, + added_snapshot_id: snapshot_id, + added_files: count(Status::Added), + existing_files: count(Status::Existing), + deleted_files: count(Status::Deleted), + added_rows: rows(Status::Added), + existing_rows: rows(Status::Existing), + deleted_rows: rows(Status::Deleted), + partitions, + first_row_id: None, + } + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn names_become_avro_names() { + assert_eq!(avro_name("time_day"), "time_day"); + assert_eq!(avro_name("1st"), "_1st"); + assert_eq!(avro_name("a-b.c"), "a_x2Db_x2Ec"); + } + } +} + +#[cfg(all(test, feature = "write"))] +mod tests { + use super::*; + + fn file(path: &str, day: Option, device: &str) -> Arc { + Arc::new(DataFile { + path: path.into(), + partition: vec![day.map(Datum::Date), Some(Datum::String(device.into()))], + record_count: 10, + file_size: 100, + column_sizes: BTreeMap::from([(1, 40)]), + value_counts: BTreeMap::from([(1, 10)]), + lower_bounds: BTreeMap::from([(1, 5i64.to_le_bytes().to_vec())]), + upper_bounds: BTreeMap::from([(1, 9i64.to_le_bytes().to_vec())]), + split_offsets: Some(vec![4]), + sort_order_id: Some(1), + ..DataFile::default() + }) + } + + #[test] + fn manifests_round_trip() { + let schema = r#"{"schema-id":0,"type":"struct","fields":[ + {"id":1,"name":"time","required":true,"type":"timestamptz"}, + {"id":2,"name":"device","required":true,"type":"string"}]}"#; + let spec = r#"[{"source-id":1,"field-id":1000,"name":"time_day","transform":"day"}, + {"source-id":2,"field-id":1001,"name":"device","transform":"identity"}]"#; + let types = vec![ + PartitionFieldType { field_id: 1000, name: "time_day".into(), ty: PrimitiveType::Date }, + PartitionFieldType { field_id: 1001, name: "device".into(), ty: PrimitiveType::String }, + ]; + let ctx = ManifestContext { + schema_json: schema, + schema_id: 0, + spec_fields_json: spec, + spec_id: 0, + partition_type: &types, + }; + let entries = vec![ + ManifestWrite { + status: Status::Added, + snapshot_id: Some(7), + sequence_number: None, + file_sequence_number: None, + file: file("a", Some(20_000), "b"), + }, + ManifestWrite { + status: Status::Existing, + snapshot_id: Some(3), + sequence_number: Some(2), + file_sequence_number: Some(2), + file: file("b", None, "a"), + }, + ]; + let bytes = write_manifest(&ctx, &entries).unwrap(); + let mut mf = manifest_file("m.avro", &bytes, 0, 7, &entries, &types); + assert_eq!((mf.added_files, mf.existing_files, mf.added_rows), (1, 1, 10)); + let summaries = mf.partitions.clone().unwrap(); + assert_eq!( + summaries[0], + FieldSummary { + contains_null: true, + contains_nan: None, + lower: Some(20_000i32.to_le_bytes().to_vec()), + upper: Some(20_000i32.to_le_bytes().to_vec()) + } + ); + assert_eq!((summaries[1].lower.as_deref(), summaries[1].upper.as_deref()), (Some(&b"a"[..]), Some(&b"b"[..]))); + + let list = write_manifest_list(std::slice::from_ref(&mf), 7, Some(3), 5).unwrap(); + let read = read_manifest_list(&list).unwrap(); + // The list assigns the snapshot's sequence number; the minimum is the explicit 2. + mf.sequence_number = 5; + assert_eq!(read, [mf.clone()]); + + let read = read_manifest(&mf, &bytes).unwrap(); + assert_eq!((read[0].sequence_number, read[0].snapshot_id, read[0].status), (5, 7, Status::Added)); + assert_eq!((read[1].sequence_number, read[1].snapshot_id, read[1].status), (2, 3, Status::Existing)); + assert_eq!(read[0].file, entries[0].file); + assert_eq!(read[1].file, entries[1].file); + } + + #[test] + fn partition_values_must_fit_the_spec() { + let types = vec![PartitionFieldType { field_id: 1000, name: "d".into(), ty: PrimitiveType::Date }]; + let ctx = ManifestContext { + schema_json: "{}", + schema_id: 0, + spec_fields_json: "[]", + spec_id: 0, + partition_type: &types, + }; + let write = |partition: Vec>| { + let file = Arc::new(DataFile { path: "a".into(), partition, ..DataFile::default() }); + let entry = ManifestWrite { + status: Status::Added, + snapshot_id: None, + sequence_number: None, + file_sequence_number: None, + file, + }; + write_manifest(&ctx, &[entry]) + }; + assert!(write(vec![Some(Datum::Date(1))]).is_ok()); + assert!(matches!(write(vec![Some(Datum::Int(1))]), Err(crate::Error::Invalid(_)))); + assert!(matches!(write(vec![]), Err(crate::Error::Invalid(_)))); + } + + #[test] + fn garbage_is_an_error() { + assert!(read_manifest_list(b"not avro").is_err()); + assert!(read_manifest_list(&[]).is_err()); + } +} diff --git a/crates/lakebed-iceberg/src/metadata.rs b/crates/lakebed-iceberg/src/metadata.rs new file mode 100644 index 0000000..dd7464e --- /dev/null +++ b/crates/lakebed-iceberg/src/metadata.rs @@ -0,0 +1,493 @@ +//! Table metadata (`*.metadata.json`), kept as JSON so fields this crate doesn't model survive a +//! rewrite, with typed views for what readers and writers need. Reading covers format versions 1 +//! to 3; updates (feature `write`) are for format v2, made the way iceberg-rust's metadata builder +//! makes them. +//! + +use std::collections::HashMap; + +use serde_json::Value; + +use crate::partition::{PartitionFieldType, PartitionSpec}; +use crate::types::Schema; +use crate::{Error, Result, corrupt}; + +/// The format version this crate writes. +pub const WRITABLE_FORMAT_VERSION: u8 = 2; + +/// A table's metadata at one version. +#[derive(Clone, Debug, PartialEq)] +pub struct Metadata(Value); + +/// A snapshot: the table's state after one commit. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Snapshot { + /// Its id. + pub snapshot_id: i64, + /// The snapshot it was committed on. + pub parent_snapshot_id: Option, + /// Its sequence number (0 in format v1). + pub sequence_number: i64, + /// When it was made (ms since the epoch). + pub timestamp_ms: i64, + /// Its manifest list (format v1 snapshots may list manifests inline instead). + pub manifest_list: Option, + /// Its summary, including `operation`. + pub summary: HashMap, + /// The schema it was written with. + pub schema_id: Option, +} + +impl Snapshot { + fn from_json(s: &Value) -> Option { + Some(Self { + snapshot_id: s.get("snapshot-id")?.as_i64()?, + parent_snapshot_id: s.get("parent-snapshot-id").and_then(Value::as_i64), + sequence_number: s.get("sequence-number").and_then(Value::as_i64).unwrap_or(0), + timestamp_ms: s.get("timestamp-ms").and_then(Value::as_i64).unwrap_or(0), + manifest_list: s.get("manifest-list").and_then(Value::as_str).map(str::to_string), + summary: strings(s.get("summary")), + schema_id: s.get("schema-id").and_then(Value::as_i64).and_then(|v| i32::try_from(v).ok()), + }) + } + + /// `append`, `replace`, `overwrite` or `delete`. + pub fn operation(&self) -> Option<&str> { + self.summary.get("operation").map(String::as_str) + } +} + +fn strings(v: Option<&Value>) -> HashMap { + let Some(Value::Object(m)) = v else { return HashMap::new() }; + m.iter().filter_map(|(k, v)| Some((k.clone(), v.as_str()?.to_string()))).collect() +} + +fn find_by<'a>(list: Option<&'a Value>, key: &str, id: i64) -> Option<&'a Value> { + list?.as_array()?.iter().find(|v| v.get(key).and_then(Value::as_i64) == Some(id)) +} + +impl Metadata { + /// Parses table metadata of format version 1, 2 or 3. + pub fn parse(bytes: &[u8]) -> Result { + let value: Value = serde_json::from_slice(bytes).map_err(|e| corrupt(format!("table metadata: {e}")))?; + Self::from_json(value) + } + + /// Table metadata from parsed JSON. + pub fn from_json(value: Value) -> Result { + match value.get("format-version").and_then(Value::as_u64) { + Some(1..=3) => Ok(Self(value)), + Some(v) => Err(Error::Unsupported(format!("Iceberg format version {v}"))), + None => Err(corrupt("table metadata without a format version")), + } + } + + /// The format version (1 to 3). + pub fn format_version(&self) -> u8 { + self.0.get("format-version").and_then(Value::as_u64).and_then(|v| u8::try_from(v).ok()).unwrap_or(0) + } + + /// An error unless this crate can write the table (format v2). + pub fn check_writable(&self) -> Result<()> { + match self.format_version() { + WRITABLE_FORMAT_VERSION => Ok(()), + v => Err(Error::Unsupported(format!( + "writing Iceberg format version {v} tables (only version {WRITABLE_FORMAT_VERSION})" + ))), + } + } + + /// The metadata as JSON. + pub fn as_json(&self) -> &Value { + &self.0 + } + + /// The metadata as JSON text. + pub fn to_json(&self) -> String { + self.0.to_string() + } + + /// The table's root location. + pub fn location(&self) -> &str { + self.0.get("location").and_then(Value::as_str).unwrap_or_default() + } + + /// The table's UUID. + pub fn table_uuid(&self) -> Option<&str> { + self.0.get("table-uuid").and_then(Value::as_str) + } + + /// The table properties. + pub fn properties(&self) -> HashMap { + strings(self.0.get("properties")) + } + + /// When the metadata was last updated (ms since the epoch). + pub fn last_updated_ms(&self) -> i64 { + self.0.get("last-updated-ms").and_then(Value::as_i64).unwrap_or(0) + } + + /// The current snapshot's id. + pub fn current_snapshot_id(&self) -> Option { + self.0.get("current-snapshot-id").and_then(Value::as_i64).filter(|&id| id >= 0) + } + + /// Every snapshot, in the order the metadata lists them. + pub fn snapshots(&self) -> Vec { + let Some(Value::Array(list)) = self.0.get("snapshots") else { return vec![] }; + list.iter().filter_map(Snapshot::from_json).collect() + } + + /// The snapshot with id `id`. + pub fn snapshot(&self, id: i64) -> Option { + Snapshot::from_json(find_by(self.0.get("snapshots"), "snapshot-id", id)?) + } + + /// Whether a snapshot with id `id` exists. + pub fn has_snapshot(&self, id: i64) -> bool { + find_by(self.0.get("snapshots"), "snapshot-id", id).is_some() + } + + /// The current snapshot. + pub fn current_snapshot(&self) -> Option { + self.snapshot(self.current_snapshot_id()?) + } + + /// The current snapshot and its ancestors, newest first, as far as they are still listed. + pub fn ancestors(&self) -> Vec { + let mut out: Vec = vec![]; + let mut next = self.current_snapshot_id(); + while let Some(s) = next.and_then(|id| self.snapshot(id)) { + // A cycle would be corrupt metadata; stop rather than loop. + if out.iter().any(|a| a.snapshot_id == s.snapshot_id) { + break; + } + next = s.parent_snapshot_id; + out.push(s); + } + out + } + + /// The current snapshot's summary. + pub fn summary(&self) -> HashMap { + self.current_snapshot().map(|s| s.summary).unwrap_or_default() + } + + /// The current snapshot's manifest list. + pub fn manifest_list(&self) -> Option { + self.current_snapshot()?.manifest_list + } + + /// The sequence number the next snapshot gets. + pub fn next_sequence_number(&self) -> i64 { + self.0.get("last-sequence-number").and_then(Value::as_i64).unwrap_or(0).saturating_add(1) + } + + /// The current schema's id. + pub fn schema_id(&self) -> i32 { + self.0.get("current-schema-id").and_then(Value::as_i64).and_then(|v| i32::try_from(v).ok()).unwrap_or(0) + } + + /// The current schema as Iceberg JSON (`schema` in format v1 metadata without `schemas`). + pub fn schema_json(&self) -> Result<&Value> { + find_by(self.0.get("schemas"), "schema-id", self.schema_id().into()) + .or_else(|| self.0.get("schema")) + .ok_or_else(|| corrupt("table metadata without a current schema")) + } + + /// The current schema. + pub fn schema(&self) -> Result { + Schema::from_json(self.schema_json()?) + } + + /// The default partition spec's id and fields as Iceberg JSON (`partition-spec` in format v1 + /// metadata without `partition-specs`). + pub fn partition_spec_json(&self) -> Result<(i32, &Value)> { + let id = self.0.get("default-spec-id").and_then(Value::as_i64).unwrap_or(0); + let fields = match find_by(self.0.get("partition-specs"), "spec-id", id) { + Some(spec) => spec.get("fields").ok_or_else(|| corrupt("partition spec without fields"))?, + None => self.0.get("partition-spec").ok_or_else(|| corrupt("table metadata without a partition spec"))?, + }; + Ok((i32::try_from(id).map_err(|_| corrupt("partition spec id"))?, fields)) + } + + /// The default partition spec. + pub fn partition_spec(&self) -> Result { + let (id, fields) = self.partition_spec_json()?; + PartitionSpec::from_fields(id, fields) + } + + /// The default partition spec's fields with the types of their values. + pub fn partition_type(&self) -> Result> { + self.partition_spec()?.partition_type(&self.schema()?) + } + + /// The default sort order's id. + pub fn sort_order_id(&self) -> i32 { + self.0.get("default-sort-order-id").and_then(Value::as_i64).and_then(|v| i32::try_from(v).ok()).unwrap_or(0) + } + + /// Entries in the metadata log. + pub fn metadata_log_len(&self) -> usize { + self.0.get("metadata-log").and_then(Value::as_array).map_or(0, Vec::len) + } + + /// The current schema as Arrow, fields carrying their Iceberg field ids. + #[cfg(feature = "arrow")] + pub fn arrow_schema(&self) -> Result { + Ok(std::sync::Arc::new(crate::arrow::schema_to_arrow(&self.schema()?)?)) + } +} + +#[cfg(feature = "write")] +pub use write::NewSnapshot; + +#[cfg(feature = "write")] +mod write { + use std::collections::HashMap; + + use serde_json::{Map, Value, json}; + + use super::Metadata; + use crate::{Result, corrupt}; + + /// How many earlier metadata files the metadata log keeps unless the table sets + /// `write.metadata.previous-versions-max`. + const PREVIOUS_VERSIONS_MAX: usize = 100; + + /// A snapshot to add (see [`Metadata::with_snapshot`]). + #[derive(Clone, Debug)] + pub struct NewSnapshot { + /// Its id. + pub snapshot_id: i64, + /// Its sequence number. + pub sequence_number: i64, + /// When it was made (ms since the epoch). + pub timestamp_ms: i64, + /// Its manifest list. + pub manifest_list: String, + /// `append`, `replace`, `overwrite` or `delete`. + pub operation: String, + /// Summary properties besides the operation. + pub summary: HashMap, + } + + impl NewSnapshot { + fn summary_json(&self) -> Value { + let mut s: Map = self.summary.iter().map(|(k, v)| (k.clone(), json!(v))).collect(); + s.insert("operation".into(), json!(self.operation)); + Value::Object(s) + } + } + + fn push(obj: &mut Map, key: &str, value: Value) { + match obj.get_mut(key) { + Some(Value::Array(a)) => a.push(value), + _ => { + obj.insert(key.into(), Value::Array(vec![value])); + } + } + } + + impl Metadata { + fn object(&mut self) -> Result<&mut Map> { + self.0.as_object_mut().ok_or_else(|| corrupt("table metadata isn't an object")) + } + + /// Records `previous` (the metadata file this version replaces) in the metadata log, + /// trimmed to the table's `write.metadata.previous-versions-max`. + fn log_previous(&mut self, previous: &str, previous_updated_ms: i64) -> Result<()> { + let keep = self + .properties() + .get("write.metadata.previous-versions-max") + .and_then(|v| v.parse::().ok()) + .unwrap_or(PREVIOUS_VERSIONS_MAX) + .max(1); + let obj = self.object()?; + push(obj, "metadata-log", json!({"metadata-file": previous, "timestamp-ms": previous_updated_ms})); + if let Some(Value::Array(log)) = obj.get_mut("metadata-log") + && log.len() > keep + { + log.drain(..log.len() - keep); + } + Ok(()) + } + + /// This metadata with `snapshot` added as the current snapshot of `main`, `previous` (this + /// version's file) recorded in the metadata log, updated at `now_ms`. + pub fn with_snapshot(&self, snapshot: &NewSnapshot, previous: &str, now_ms: i64) -> Result { + self.check_writable()?; + let mut m = self.clone(); + let mut s = json!({ + "snapshot-id": snapshot.snapshot_id, + "sequence-number": snapshot.sequence_number, + "timestamp-ms": snapshot.timestamp_ms, + "manifest-list": snapshot.manifest_list, + "summary": snapshot.summary_json(), + "schema-id": self.schema_id(), + }); + if let (Some(parent), Some(s)) = (self.current_snapshot_id(), s.as_object_mut()) { + s.insert("parent-snapshot-id".into(), json!(parent)); + } + m.log_previous(previous, self.last_updated_ms())?; + let obj = m.object()?; + push(obj, "snapshots", s); + push( + obj, + "snapshot-log", + json!({"snapshot-id": snapshot.snapshot_id, "timestamp-ms": snapshot.timestamp_ms}), + ); + obj.insert("current-snapshot-id".into(), json!(snapshot.snapshot_id)); + obj.insert("last-sequence-number".into(), json!(snapshot.sequence_number)); + obj.insert("last-updated-ms".into(), json!(now_ms)); + let refs = obj.entry("refs").or_insert_with(|| Value::Object(Map::new())); + if let Some(refs) = refs.as_object_mut() { + refs.insert("main".into(), json!({"snapshot-id": snapshot.snapshot_id, "type": "branch"})); + } + Ok(m) + } + + /// This metadata without the snapshots `ids`, as iceberg-rust's builder removes them: refs to + /// them go, and the snapshot log keeps only what follows the last entry of a removed snapshot + /// (so it has no gaps). `previous` is recorded in the metadata log. The current snapshot can't be + /// removed: the table would have none and read as empty. + pub fn without_snapshots(&self, ids: &[i64], previous: &str, now_ms: i64) -> Result { + self.check_writable()?; + if let Some(current) = self.current_snapshot_id().filter(|id| ids.contains(id)) { + return Err(crate::Error::Invalid(format!("removing the current snapshot {current}"))); + } + let mut m = self.clone(); + m.log_previous(previous, self.last_updated_ms())?; + let obj = m.object()?; + let removed = |v: &Value| v.get("snapshot-id").and_then(Value::as_i64).is_some_and(|id| ids.contains(&id)); + if let Some(Value::Array(snapshots)) = obj.get_mut("snapshots") { + snapshots.retain(|s| !removed(s)); + } + if let Some(Value::Object(refs)) = obj.get_mut("refs") { + refs.retain(|_, r| !removed(r)); + } + if let Some(Value::Array(log)) = obj.get_mut("snapshot-log") { + let mut kept = vec![]; + for entry in log.drain(..) { + if removed(&entry) { + kept.clear(); + } else { + kept.push(entry); + } + } + *log = kept; + } + obj.insert("last-updated-ms".into(), json!(now_ms)); + Ok(m) + } + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use crate::types::PrimitiveType; + + fn v2() -> Metadata { + Metadata::from_json(json!({ + "format-version": 2, "table-uuid": "u", "location": "memory://t", "last-sequence-number": 2, + "last-updated-ms": 10, "current-schema-id": 1, + "schemas": [ + {"schema-id": 0, "type": "struct", "fields": []}, + {"schema-id": 1, "type": "struct", "fields": [{"id": 1, "name": "time", "required": true, "type": "timestamptz"}]} + ], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "name": "time_day", "transform": "day"}]}], + "properties": {"write.metadata.previous-versions-max": "2", "n": 1}, + "current-snapshot-id": 2, + "snapshots": [ + {"snapshot-id": 1, "sequence-number": 1, "timestamp-ms": 1, "manifest-list": "l1", "summary": {"operation": "append"}}, + {"snapshot-id": 2, "parent-snapshot-id": 1, "sequence-number": 2, "timestamp-ms": 2, "manifest-list": "l2", "summary": {"operation": "replace"}} + ], + "snapshot-log": [{"snapshot-id": 1, "timestamp-ms": 1}, {"snapshot-id": 2, "timestamp-ms": 2}], + "metadata-log": [], + "unknown-field": {"kept": true} + })) + .unwrap() + } + + #[test] + fn reads_v2() { + let m = v2(); + assert_eq!((m.format_version(), m.location(), m.table_uuid()), (2, "memory://t", Some("u"))); + assert_eq!(m.properties().len(), 1, "non-string properties are skipped"); + assert_eq!(m.schema().unwrap().fields[0].name, "time"); + assert_eq!(m.partition_type().unwrap()[0].ty, PrimitiveType::Date); + assert_eq!(m.current_snapshot().unwrap().operation(), Some("replace")); + let ancestors: Vec = m.ancestors().iter().map(|s| s.snapshot_id).collect(); + assert_eq!(ancestors, [2, 1]); + assert_eq!(m.manifest_list().as_deref(), Some("l2")); + assert_eq!(m.next_sequence_number(), 3); + assert!(m.check_writable().is_ok()); + } + + #[test] + fn reads_v1() { + let m = Metadata::from_json(json!({ + "format-version": 1, "location": "s3://b/t", + "schema": {"type": "struct", "fields": [{"id": 1, "name": "x", "required": false, "type": "long"}]}, + "partition-spec": [{"source-id": 1, "name": "x", "transform": "identity"}], + "snapshots": [] + })) + .unwrap(); + assert_eq!(m.schema().unwrap().fields[0].name, "x"); + assert_eq!(m.partition_spec().unwrap().fields[0].field_id, 1000); + assert_eq!(m.current_snapshot(), None); + assert!(matches!(m.check_writable(), Err(Error::Unsupported(_)))); + } + + #[test] + fn a_sequence_number_at_the_limit_saturates() { + let m = Metadata::from_json(json!({"format-version": 2, "last-sequence-number": i64::MAX})).unwrap(); + assert_eq!(m.next_sequence_number(), i64::MAX); + } + + #[test] + fn rejects_unknown_versions() { + assert!(matches!(Metadata::parse(br#"{"format-version": 4}"#), Err(Error::Unsupported(_)))); + assert!(matches!(Metadata::parse(br#"{"location": "x"}"#), Err(Error::Corrupt(_)))); + assert!(matches!(Metadata::parse(b"{"), Err(Error::Corrupt(_)))); + } + + #[cfg(feature = "write")] + #[test] + fn updates_keep_unknown_fields_and_trim_the_log() { + let m = v2(); + let s = NewSnapshot { + snapshot_id: 3, + sequence_number: 3, + timestamp_ms: 20, + manifest_list: "l3".into(), + operation: "append".into(), + summary: HashMap::from([("k".to_string(), "v".to_string())]), + }; + let m = m.with_snapshot(&s, "v1.metadata.json", 20).unwrap(); + let m = m.with_snapshot(&NewSnapshot { snapshot_id: 4, ..s.clone() }, "v2.metadata.json", 21).unwrap(); + let m = m.with_snapshot(&NewSnapshot { snapshot_id: 5, ..s }, "v3.metadata.json", 22).unwrap(); + assert_eq!(m.current_snapshot_id(), Some(5)); + assert_eq!(m.snapshot(3).unwrap().parent_snapshot_id, Some(2)); + assert_eq!(m.snapshot(3).unwrap().summary["k"], "v"); + assert_eq!(m.metadata_log_len(), 2, "previous-versions-max"); + assert_eq!(m.as_json()["unknown-field"]["kept"], true); + assert_eq!(m.as_json()["refs"]["main"]["snapshot-id"], 5); + + let current = m.without_snapshots(&[1, 5], "v4.metadata.json", 23); + assert!(matches!(current, Err(Error::Invalid(_))), "the current snapshot stays"); + let m = m.without_snapshots(&[1, 2], "v4.metadata.json", 23).unwrap(); + assert!(!m.has_snapshot(1) && !m.has_snapshot(2) && m.has_snapshot(3)); + let log: Vec = m.as_json()["snapshot-log"] + .as_array() + .unwrap() + .iter() + .map(|e| e["snapshot-id"].as_i64().unwrap()) + .collect(); + assert_eq!(log, [3, 4, 5]); + } +} diff --git a/crates/lakebed-iceberg/src/partition.rs b/crates/lakebed-iceberg/src/partition.rs new file mode 100644 index 0000000..c407e04 --- /dev/null +++ b/crates/lakebed-iceberg/src/partition.rs @@ -0,0 +1,228 @@ +//! Partition specs and their transforms. +//! + +use std::fmt; +use std::str::FromStr; + +use serde_json::Value; + +use crate::types::{PrimitiveType, Schema}; +use crate::{Error, Result, corrupt}; + +/// Partition field ids start here when a format v1 spec leaves them out. +const FIRST_FIELD_ID: i32 = 1000; + +/// How a partition value is derived from its source column. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum Transform { + /// The value itself. + Identity, + /// A hash of the value modulo N. + Bucket(u32), + /// The value truncated to width W. + Truncate(u32), + /// Years since 1970. + Year, + /// Months since 1970-01. + Month, + /// Days since 1970-01-01. + Day, + /// Hours since 1970-01-01 00:00. + Hour, + /// Always null. + Void, + /// A transform this version doesn't know, by name. + Other(String), +} + +impl FromStr for Transform { + type Err = Error; + + fn from_str(s: &str) -> Result { + let arg = |prefix: &str| { + s.strip_prefix(prefix) + .and_then(|r| r.strip_suffix(']')) + .map(|n| n.trim().parse::().map_err(|_| corrupt(format!("transform {s}")))) + }; + Ok(match s { + "identity" => Self::Identity, + "year" => Self::Year, + "month" => Self::Month, + "day" => Self::Day, + "hour" => Self::Hour, + "void" => Self::Void, + _ => { + if let Some(n) = arg("bucket[") { + Self::Bucket(n?) + } else if let Some(w) = arg("truncate[") { + Self::Truncate(w?) + } else { + Self::Other(s.to_string()) + } + } + }) + } +} + +impl fmt::Display for Transform { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Identity => f.write_str("identity"), + Self::Bucket(n) => write!(f, "bucket[{n}]"), + Self::Truncate(w) => write!(f, "truncate[{w}]"), + Self::Year => f.write_str("year"), + Self::Month => f.write_str("month"), + Self::Day => f.write_str("day"), + Self::Hour => f.write_str("hour"), + Self::Void => f.write_str("void"), + Self::Other(name) => f.write_str(name), + } + } +} + +impl Transform { + /// The type of the partition values this transform makes from a `source` column, as + /// iceberg-rust types them (`day` gives a `date`); `None` for an unknown transform. + pub fn result_type(&self, source: &PrimitiveType) -> Option { + match self { + Self::Identity | Self::Truncate(_) | Self::Void => Some(source.clone()), + Self::Bucket(_) | Self::Year | Self::Month | Self::Hour => Some(PrimitiveType::Int), + Self::Day => Some(PrimitiveType::Date), + Self::Other(_) => None, + } + } +} + +/// A field of a partition spec. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PartitionField { + /// The source column's field id. + pub source_id: i32, + /// This field's id (1000 and up). + pub field_id: i32, + /// Its name in manifests. + pub name: String, + /// How its values are derived. + pub transform: Transform, +} + +/// A partition spec. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PartitionSpec { + /// Its id. + pub spec_id: i32, + /// Its fields, in the order of the partition tuple. + pub fields: Vec, +} + +impl PartitionSpec { + /// A spec from metadata JSON: `{"spec-id": 0, "fields": [...]}`. + pub fn from_json(spec: &Value) -> Result { + let id = spec.get("spec-id").and_then(Value::as_i64).unwrap_or(0); + let fields = spec.get("fields").ok_or_else(|| corrupt("partition spec without fields"))?; + Self::from_fields(i32::try_from(id).map_err(|_| corrupt("partition spec id"))?, fields) + } + + /// A spec from its fields' JSON (a format v1 table's `partition-spec`, a manifest header's + /// `partition-spec`). + pub fn from_fields(spec_id: i32, fields: &Value) -> Result { + let fields = fields.as_array().ok_or_else(|| corrupt("partition fields aren't a list"))?; + let fields = fields + .iter() + .zip(FIRST_FIELD_ID..) + .map(|(f, default_id)| { + let int = |key: &str| f.get(key).and_then(Value::as_i64).and_then(|v| i32::try_from(v).ok()); + Ok(PartitionField { + source_id: int("source-id").ok_or_else(|| corrupt("partition field without source-id"))?, + field_id: int("field-id").unwrap_or(default_id), + name: f + .get("name") + .and_then(Value::as_str) + .ok_or_else(|| corrupt("partition field without a name"))? + .to_string(), + transform: f + .get("transform") + .and_then(Value::as_str) + .ok_or_else(|| corrupt("partition field without a transform"))? + .parse()?, + }) + }) + .collect::>()?; + Ok(Self { spec_id, fields }) + } + + /// Whether files of this spec all share one partition. + pub fn is_unpartitioned(&self) -> bool { + self.fields.iter().all(|f| f.transform == Transform::Void) + } + + /// Each field with the type of its values, given the table `schema`. + pub fn partition_type(&self, schema: &Schema) -> Result> { + self.fields + .iter() + .map(|f| { + let source = schema + .primitive_by_id(f.source_id) + .ok_or_else(|| corrupt(format!("partition field {} has no primitive source column", f.name)))?; + let ty = f + .transform + .result_type(source) + .ok_or_else(|| Error::Unsupported(format!("partition transform {}", f.transform)))?; + Ok(PartitionFieldType { field_id: f.field_id, name: f.name.clone(), ty }) + }) + .collect() + } +} + +/// A partition field with the type of its values. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PartitionFieldType { + /// The partition field's id. + pub field_id: i32, + /// Its name. + pub name: String, + /// The type of its values. + pub ty: PrimitiveType, +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn transforms_round_trip() { + for name in ["identity", "bucket[16]", "truncate[4]", "year", "month", "day", "hour", "void"] { + assert_eq!(name.parse::().unwrap().to_string(), name); + } + assert_eq!("zorder".parse::().unwrap(), Transform::Other("zorder".into())); + assert!("bucket[x]".parse::().is_err()); + } + + #[test] + fn specs_type_their_fields() { + let schema = Schema::from_json(&json!({"schema-id": 0, "fields": [ + {"id": 1, "name": "time", "required": true, "type": "timestamptz"}, + {"id": 2, "name": "device", "required": true, "type": "string"} + ]})) + .unwrap(); + // A v1 spec without field ids gets 1000 and up. + let spec = PartitionSpec::from_fields( + 0, + &json!([ + {"source-id": 1, "name": "time_day", "transform": "day"}, + {"source-id": 2, "name": "device_bucket", "transform": "bucket[8]"}, + {"source-id": 2, "name": "device", "transform": "identity"} + ]), + ) + .unwrap(); + let types = spec.partition_type(&schema).unwrap(); + let ids: Vec = types.iter().map(|t| t.field_id).collect(); + assert_eq!(ids, [1000, 1001, 1002]); + let tys: Vec<&PrimitiveType> = types.iter().map(|t| &t.ty).collect(); + assert_eq!(tys, [&PrimitiveType::Date, &PrimitiveType::Int, &PrimitiveType::String]); + assert!(!spec.is_unpartitioned()); + assert!(PartitionSpec::from_json(&json!({"spec-id": 1, "fields": []})).unwrap().is_unpartitioned()); + } +} diff --git a/crates/lakebed-iceberg/src/stats.rs b/crates/lakebed-iceberg/src/stats.rs new file mode 100644 index 0000000..a4801bf --- /dev/null +++ b/crates/lakebed-iceberg/src/stats.rs @@ -0,0 +1,247 @@ +//! Iceberg's statistics of a Parquet data file, from its footer (feature `parquet-stats`): the +//! values iceberg-rust's writer records. Columns are matched to fields by the Parquet field ids +//! the writer stored. + +use std::cmp::Ordering; +use std::collections::{BTreeMap, BTreeSet}; + +use parquet::basic::{LogicalType, Type as PhysicalType}; +use parquet::file::metadata::{ParquetMetaData, RowGroupMetaData}; +use parquet::file::statistics::Statistics; +use parquet::schema::types::ColumnDescriptor; + +use crate::datum::{Datum, from_be, minimal_be}; +use crate::manifest::DataFile; + +/// What Iceberg records about a Parquet file. Bounds are Iceberg's single-value binary encoding. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ParquetStats { + /// Rows in the file. + pub record_count: u64, + /// The file's size in bytes. + pub file_size: u64, + /// Compressed bytes per field id. + pub column_sizes: BTreeMap, + /// Values (nulls included) per field id. + pub value_counts: BTreeMap, + /// Nulls per field id. + pub null_counts: BTreeMap, + /// NaNs per field id of a float column (Parquet doesn't record them; the writer counts them). + pub nan_counts: BTreeMap, + /// Smallest value per field id, where Parquet's statistics give an exact one. + pub lower_bounds: BTreeMap>, + /// Largest value per field id, likewise. + pub upper_bounds: BTreeMap>, + /// Offsets of the row groups. + pub split_offsets: Vec, +} + +impl ParquetStats { + /// The statistics of a file of `file_size` bytes with footer `metadata`, and the NaN counts the + /// writer counted. A bound or null count is recorded only when every row group gives it exactly: + /// readers take them as covering the whole file, and one covering part of it would let them skip + /// a file that has matching rows. A row group of nulls only has no bounds and needs none. + pub fn new(metadata: &ParquetMetaData, file_size: u64, nan_counts: BTreeMap) -> Self { + let mut s = Self { + record_count: u64::try_from(metadata.file_metadata().num_rows()).unwrap_or(0), + file_size, + nan_counts, + split_offsets: metadata.row_groups().iter().filter_map(RowGroupMetaData::file_offset).collect(), + ..Self::default() + }; + let (mut no_lower, mut no_upper, mut no_nulls) = (BTreeSet::new(), BTreeSet::new(), BTreeSet::new()); + for group in metadata.row_groups() { + for chunk in group.columns() { + let Some(id) = field_id(chunk.column_descr()) else { continue }; + let values = u64::try_from(chunk.num_values()).unwrap_or(0); + *s.column_sizes.entry(id).or_default() += u64::try_from(chunk.compressed_size()).unwrap_or(0); + *s.value_counts.entry(id).or_default() += values; + let Some(stats) = chunk.statistics() else { + no_lower.insert(id); + no_upper.insert(id); + no_nulls.insert(id); + continue; + }; + match stats.null_count_opt() { + Some(nulls) => *s.null_counts.entry(id).or_default() += nulls, + None => { + no_nulls.insert(id); + } + } + let all_null = stats.null_count_opt() == Some(values); + let descr = chunk.column_descr(); + let min = stats.min_bytes_opt().filter(|_| stats.min_is_exact()).and_then(|b| bound(b, stats, descr)); + match min { + Some(min) => keep(&mut s.lower_bounds, id, min, descr, Ordering::Less), + None if !all_null => { + no_lower.insert(id); + } + None => {} + } + let max = stats.max_bytes_opt().filter(|_| stats.max_is_exact()).and_then(|b| bound(b, stats, descr)); + match max { + Some(max) => keep(&mut s.upper_bounds, id, max, descr, Ordering::Greater), + None if !all_null => { + no_upper.insert(id); + } + None => {} + } + } + } + s.lower_bounds.retain(|id, _| !no_lower.contains(id)); + s.upper_bounds.retain(|id, _| !no_upper.contains(id)); + s.null_counts.retain(|id, _| !no_nulls.contains(id)); + s + } + + /// The data file at `path` with these statistics, in partition `partition`. + pub fn data_file(&self, path: String, partition: Vec>) -> DataFile { + DataFile { + path, + partition, + record_count: self.record_count, + file_size: self.file_size, + column_sizes: self.column_sizes.clone(), + value_counts: self.value_counts.clone(), + null_counts: self.null_counts.clone(), + nan_counts: self.nan_counts.clone(), + lower_bounds: self.lower_bounds.clone(), + upper_bounds: self.upper_bounds.clone(), + split_offsets: Some(self.split_offsets.clone()), + ..DataFile::default() + } + } +} + +/// The Iceberg field id a Parquet column carries. +pub fn field_id(column: &ColumnDescriptor) -> Option { + let info = column.self_type().get_basic_info(); + info.has_id().then(|| info.id()) +} + +/// Keeps `value` for `id` if there is none yet or it compares `wins` against the current one. +fn keep(bounds: &mut BTreeMap>, id: i32, value: Vec, descr: &ColumnDescriptor, wins: Ordering) { + match bounds.get(&id) { + Some(current) if compare(&value, current, descr) != wins => {} + _ => { + bounds.insert(id, value); + } + } +} + +/// A Parquet statistic's plain encoding as an Iceberg bound. They agree for booleans, ints, +/// longs, floats, doubles, dates, times, timestamps, strings, binary, and `fixed[L]`. Decimals +/// become Iceberg's shortest big-endian bytes, whether stored as INT32, INT64, or (big-endian, of +/// their column's width) as a byte array; one wider than 16 bytes has no bound. INT96 has no +/// Iceberg type. +fn bound(plain: &[u8], stats: &Statistics, descr: &ColumnDescriptor) -> Option> { + let decimal = matches!(descr.logical_type_ref(), Some(LogicalType::Decimal { .. })); + match (stats, decimal) { + (Statistics::Int96(_), _) => None, + (Statistics::Int32(_), true) => Some(minimal_be(i128::from(i32::from_le_bytes(plain.try_into().ok()?)))), + (Statistics::Int64(_), true) => Some(minimal_be(i128::from(i64::from_le_bytes(plain.try_into().ok()?)))), + (Statistics::FixedLenByteArray(_) | Statistics::ByteArray(_), true) => { + (!plain.is_empty() && plain.len() <= 16).then(|| minimal_be(from_be(plain))) + } + _ => Some(plain.to_vec()), + } +} + +/// Compares two bounds of a column by value. +fn compare(a: &[u8], b: &[u8], descr: &ColumnDescriptor) -> Ordering { + let decimal = matches!(descr.logical_type_ref(), Some(LogicalType::Decimal { .. })); + if decimal { + return from_be(a).cmp(&from_be(b)); + } + match descr.physical_type() { + PhysicalType::INT32 => le::<4>(a).map(i32::from_le_bytes).cmp(&le::<4>(b).map(i32::from_le_bytes)), + PhysicalType::INT64 => le::<8>(a).map(i64::from_le_bytes).cmp(&le::<8>(b).map(i64::from_le_bytes)), + PhysicalType::FLOAT => { + let (x, y) = (le::<4>(a).map(f32::from_le_bytes), le::<4>(b).map(f32::from_le_bytes)); + x.partial_cmp(&y).unwrap_or(Ordering::Equal) + } + PhysicalType::DOUBLE => { + let (x, y) = (le::<8>(a).map(f64::from_le_bytes), le::<8>(b).map(f64::from_le_bytes)); + x.partial_cmp(&y).unwrap_or(Ordering::Equal) + } + PhysicalType::BOOLEAN | PhysicalType::INT96 | PhysicalType::BYTE_ARRAY | PhysicalType::FIXED_LEN_BYTE_ARRAY => { + a.cmp(b) + } + } +} + +fn le(bytes: &[u8]) -> Option<[u8; N]> { + bytes.try_into().ok() +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use arrow_array::{ArrayRef, Decimal128Array, Int64Array, RecordBatch}; + use arrow_schema::{DataType, Field, Schema}; + use parquet::arrow::ArrowWriter; + use parquet::file::properties::WriterProperties; + + use super::*; + + fn field(name: &str, ty: DataType, id: i32) -> Field { + Field::new(name, ty, true).with_metadata(HashMap::from([("PARQUET:field_id".to_string(), id.to_string())])) + } + + /// A file of `columns` (field ids 1, 2, …) in row groups of `rows` rows; its footer. + fn write(columns: Vec<(Field, ArrayRef)>, rows: usize) -> ParquetMetaData { + let (fields, arrays): (Vec, Vec) = columns.into_iter().unzip(); + let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), arrays).unwrap(); + let props = WriterProperties::builder().set_max_row_group_row_count(Some(rows)).build(); + let mut writer = ArrowWriter::try_new(vec![], batch.schema(), Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap() + } + + #[test] + fn bounds_need_every_row_group() { + let metadata = write( + vec![ + (field("v", DataType::Int64, 1), Arc::new(Int64Array::from(vec![1, 2, 3, 4]))), + (field("n", DataType::Int64, 2), Arc::new(Int64Array::from(vec![Some(5), None, None, None]))), + ], + 2, + ); + let s = ParquetStats::new(&metadata, 0, BTreeMap::new()); + assert_eq!( + (s.lower_bounds[&1].as_slice(), s.upper_bounds[&1].as_slice()), + (&1i64.to_le_bytes()[..], &4i64.to_le_bytes()[..]) + ); + // The second row group is all nulls: it has no bounds and doesn't need any. + assert_eq!( + (s.lower_bounds[&2].as_slice(), s.upper_bounds[&2].as_slice()), + (&5i64.to_le_bytes()[..], &5i64.to_le_bytes()[..]) + ); + assert_eq!((s.null_counts[&1], s.null_counts[&2]), (0, 3)); + + // Without statistics for one row group, the others' bounds and nulls don't cover the file. + let mut builder = metadata.into_builder(); + let mut groups = builder.take_row_groups(); + let stripped = groups[1].columns().iter().map(|c| c.clone().into_builder().clear_statistics().build().unwrap()); + groups[1] = groups[1].clone().into_builder().set_column_metadata(stripped.collect()).build().unwrap(); + let s = ParquetStats::new(&builder.set_row_groups(groups).build(), 0, BTreeMap::new()); + assert!(s.lower_bounds.is_empty() && s.upper_bounds.is_empty() && s.null_counts.is_empty(), "{s:?}"); + assert_eq!(s.value_counts[&1], 4, "counts don't depend on statistics"); + } + + #[test] + fn decimal_byte_array_bounds_are_minimal() { + // decimal(20, 2) is a 9-byte fixed array in Parquet; Iceberg keeps the shortest bytes. + let values = Decimal128Array::from(vec![-150, 12_345]).with_precision_and_scale(20, 2).unwrap(); + let metadata = write(vec![(field("d", DataType::Decimal128(20, 2), 1), Arc::new(values))], 10); + let s = ParquetStats::new(&metadata, 0, BTreeMap::new()); + assert_eq!(s.lower_bounds[&1], minimal_be(-150)); + assert_eq!(s.upper_bounds[&1], minimal_be(12_345)); + assert_eq!( + Datum::from_bytes(&crate::PrimitiveType::Decimal { precision: 20, scale: 2 }, &s.lower_bounds[&1]).unwrap(), + Datum::Decimal { unscaled: -150, precision: 20, scale: 2 } + ); + } +} diff --git a/crates/lakebed-iceberg/src/table.rs b/crates/lakebed-iceberg/src/table.rs new file mode 100644 index 0000000..7cadb70 --- /dev/null +++ b/crates/lakebed-iceberg/src/table.rs @@ -0,0 +1,162 @@ +//! A table at one version, found from its location in a store: metadata files are +//! `/metadata/v.metadata.json`, as file-system ("Hadoop") tables and Timelake name +//! them, with `version-hint.text` naming the newest for stores that can't list. Tables a catalog +//! names with other file names load with [`Table::from_metadata_file`]. + +use lakebed_core::Host; + +use crate::manifest::{Entry, ManifestFile, Status, read_manifest, read_manifest_list}; +use crate::metadata::Metadata; +use crate::{Error, Result}; + +/// `/metadata/v.metadata.json`. +pub fn metadata_uri(location: &str, version: u64) -> String { + format!("{location}/metadata/v{version}.metadata.json") +} + +/// `/metadata/version-hint.text`. +pub fn hint_uri(location: &str) -> String { + format!("{location}/metadata/version-hint.text") +} + +/// The version in a metadata file's name: `v12.metadata.json` and `00012-.metadata.json` +/// are version 12. +pub fn metadata_version(uri: &str) -> Option { + let name = uri.rsplit('/').next()?.strip_suffix(".metadata.json")?; + match name.strip_prefix('v') { + Some(n) => n.parse().ok(), + None => name.split_once('-')?.0.parse().ok(), + } +} + +/// A table at one committed version. +#[derive(Clone, Debug)] +pub struct Table { + /// The table's root URI, without a trailing slash. + pub location: String, + /// The metadata version. + pub version: u64, + /// The metadata file this version was read from (the metadata log records it once a commit + /// replaces it). + pub metadata_file: String, + /// The metadata. + pub metadata: Metadata, + /// The live files (data and delete files), when known: read by [`Table::live_files`], or + /// derived by a commit. + pub files: Option>, +} + +impl Table { + /// A table at `version` with `metadata` from `v.metadata.json`, its files not read + /// yet. + pub fn new(location: impl Into, version: u64, metadata: Metadata) -> Self { + let location = location.into(); + Self { metadata_file: metadata_uri(&location, version), location, version, metadata, files: None } + } + + /// The newest metadata version of the table at `location`: the highest listed, or what + /// `version-hint.text` says where nothing is listed. + pub async fn latest_version(host: &H, location: &str) -> Result { + let listed = host.list(&format!("{location}/metadata/v")).await?; + if let Some(newest) = listed.iter().filter_map(|o| metadata_version(&o.uri)).max() { + return Ok(newest); + } + let hint = host.get(&hint_uri(location)).await?.ok_or_else(|| Error::NotFound(location.to_string()))?; + std::str::from_utf8(&hint) + .ok() + .and_then(|h| h.trim().parse().ok()) + .ok_or_else(|| crate::corrupt(format!("{}: not a version", hint_uri(location)))) + } + + /// The newest version of the table at `location`. + pub async fn load(host: &H, location: &str) -> Result { + let version = Self::latest_version(host, location).await?; + Self::load_version(host, location, version).await + } + + /// Version `version` of the table at `location`. + pub async fn load_version(host: &H, location: &str, version: u64) -> Result { + let uri = metadata_uri(location, version); + let bytes = host.get(&uri).await?.ok_or_else(|| Error::NotFound(uri.clone()))?; + Ok(Self::new(location, version, Metadata::parse(&bytes)?)) + } + + /// The table whose metadata file is `uri` (a catalog's pointer), at the location the metadata + /// records. + pub async fn from_metadata_file(host: &H, uri: &str) -> Result { + let bytes = host.get(uri).await?.ok_or_else(|| Error::NotFound(uri.to_string()))?; + let metadata = Metadata::parse(&bytes)?; + let location = metadata.location().trim_end_matches('/').to_string(); + let table = Self::new(location, metadata_version(uri).unwrap_or(0), metadata); + Ok(Self { metadata_file: uri.to_string(), ..table }) + } + + /// The current snapshot's manifests. + pub async fn manifests(&self, host: &H) -> Result> { + let Some(list) = self.metadata.manifest_list() else { + if self.metadata.current_snapshot().is_some() { + return Err(Error::Unsupported("format v1 snapshots without a manifest list".into())); + } + return Ok(vec![]); + }; + let bytes = host.get(&list).await?.ok_or_else(|| Error::NotFound(list.clone()))?; + read_manifest_list(&bytes) + } + + /// The entries of `manifest`. + pub async fn manifest(host: &H, manifest: &ManifestFile) -> Result> { + let bytes = host.get(&manifest.path).await?.ok_or_else(|| Error::NotFound(manifest.path.clone()))?; + read_manifest(manifest, &bytes) + } + + /// The live files (not deleted by the current snapshot), read from the manifests unless known. + pub async fn live_files(&mut self, host: &H) -> Result<&[Entry]> { + if self.files.is_none() { + let manifests = self.manifests(host).await?; + let read = futures_util::future::try_join_all(manifests.iter().map(|m| Self::manifest(host, m))).await?; + let files = read.into_iter().flatten().filter(|e| e.status != Status::Deleted).collect(); + self.files = Some(files); + } + Ok(self.files.as_deref().unwrap_or_default()) + } +} + +#[cfg(test)] +mod tests { + use lakebed_core::MemoryHost; + + use super::*; + use crate::block_on; + + #[test] + fn metadata_file_names() { + assert_eq!(metadata_version("s3://b/t/metadata/v12.metadata.json"), Some(12)); + assert_eq!(metadata_version("s3://b/t/metadata/00012-1b2c.metadata.json"), Some(12)); + assert_eq!(metadata_version("s3://b/t/metadata/version-hint.text"), None); + assert_eq!(metadata_version("s3://b/t/metadata/vx.metadata.json"), None); + } + + #[test] + fn loads_the_newest_version() { + let host = MemoryHost::new(0); + let meta = |n: u8| format!(r#"{{"format-version": 2, "location": "memory://t", "last-sequence-number": {n}}}"#); + block_on(async { + assert!(matches!(Table::load(&host, "memory://t").await, Err(Error::NotFound(_)))); + // Nothing listed: the hint names the version. + host.put(&hint_uri("memory://t"), b"2\n".to_vec()).await?; + assert_eq!(Table::latest_version(&host, "memory://t").await?, 2); + host.put(&metadata_uri("memory://t", 2), meta(2).into_bytes()).await?; + assert_eq!(Table::load(&host, "memory://t").await?.version, 2); + host.put(&metadata_uri("memory://t", 10), meta(10).into_bytes()).await?; + let mut t = Table::load(&host, "memory://t").await?; + assert_eq!((t.version, t.metadata.next_sequence_number()), (10, 11)); + assert!(t.live_files(&host).await?.is_empty(), "no snapshot, no files"); + let uri = "memory://t/metadata/00002-a.metadata.json"; + host.put(uri, meta(2).into_bytes()).await?; + let t = Table::from_metadata_file(&host, uri).await?; + assert_eq!((t.location.as_str(), t.version, t.metadata_file.as_str()), ("memory://t", 2, uri)); + Ok::<_, Error>(()) + }) + .unwrap(); + } +} diff --git a/crates/lakebed-iceberg/src/types.rs b/crates/lakebed-iceberg/src/types.rs new file mode 100644 index 0000000..f260851 --- /dev/null +++ b/crates/lakebed-iceberg/src/types.rs @@ -0,0 +1,317 @@ +//! Iceberg's types and schemas, read from table metadata JSON. +//! + +use std::fmt; +use std::str::FromStr; + +use serde_json::Value; + +use crate::{Error, Result, corrupt}; + +/// A primitive type. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum PrimitiveType { + /// `boolean`. + Boolean, + /// `int`: 32-bit signed. + Int, + /// `long`: 64-bit signed. + Long, + /// `float`: 32-bit IEEE 754. + Float, + /// `double`: 64-bit IEEE 754. + Double, + /// `decimal(P,S)`. + Decimal { + /// Digits in all. + precision: u8, + /// Digits after the point. + scale: u8, + }, + /// `date`: days since 1970-01-01. + Date, + /// `time`: microseconds since midnight. + Time, + /// `timestamp`: microseconds since the epoch, without a time zone. + Timestamp, + /// `timestamptz`: microseconds since the epoch, UTC. + Timestamptz, + /// `timestamp_ns`: nanoseconds since the epoch, without a time zone (format v3). + TimestampNs, + /// `timestamptz_ns`: nanoseconds since the epoch, UTC (format v3). + TimestamptzNs, + /// `string`: UTF-8. + String, + /// `uuid`. + Uuid, + /// `fixed[L]`: `L` bytes. + Fixed(u64), + /// `binary`: any number of bytes. + Binary, +} + +impl FromStr for PrimitiveType { + type Err = Error; + + fn from_str(s: &str) -> Result { + Ok(match s { + "boolean" => Self::Boolean, + "int" => Self::Int, + "long" => Self::Long, + "float" => Self::Float, + "double" => Self::Double, + "date" => Self::Date, + "time" => Self::Time, + "timestamp" => Self::Timestamp, + "timestamptz" => Self::Timestamptz, + "timestamp_ns" => Self::TimestampNs, + "timestamptz_ns" => Self::TimestamptzNs, + "string" => Self::String, + "uuid" => Self::Uuid, + "binary" => Self::Binary, + other => { + if let Some(n) = other.strip_prefix("fixed[").and_then(|s| s.strip_suffix(']')) { + Self::Fixed(n.trim().parse().map_err(|_| corrupt(format!("type {other}")))?) + } else if let Some(ps) = other.strip_prefix("decimal(").and_then(|s| s.strip_suffix(')')) { + let (p, s) = ps.split_once(',').ok_or_else(|| corrupt(format!("type {other}")))?; + let precision = p.trim().parse().map_err(|_| corrupt(format!("type {other}")))?; + let scale = s.trim().parse().map_err(|_| corrupt(format!("type {other}")))?; + Self::Decimal { precision, scale } + } else { + return Err(Error::Unsupported(format!("type {other}"))); + } + } + }) + } +} + +impl fmt::Display for PrimitiveType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Boolean => f.write_str("boolean"), + Self::Int => f.write_str("int"), + Self::Long => f.write_str("long"), + Self::Float => f.write_str("float"), + Self::Double => f.write_str("double"), + Self::Decimal { precision, scale } => write!(f, "decimal({precision}, {scale})"), + Self::Date => f.write_str("date"), + Self::Time => f.write_str("time"), + Self::Timestamp => f.write_str("timestamp"), + Self::Timestamptz => f.write_str("timestamptz"), + Self::TimestampNs => f.write_str("timestamp_ns"), + Self::TimestamptzNs => f.write_str("timestamptz_ns"), + Self::String => f.write_str("string"), + Self::Uuid => f.write_str("uuid"), + Self::Fixed(n) => write!(f, "fixed[{n}]"), + Self::Binary => f.write_str("binary"), + } + } +} + +/// Any type: a primitive, a nested type, or one this version doesn't model. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Type { + /// A primitive. + Primitive(PrimitiveType), + /// A struct of fields. + Struct(Vec), + /// A list. + List { + /// The element's field id. + element_id: i32, + /// Whether elements can't be null. + element_required: bool, + /// The element type. + element: Box, + }, + /// A map. + Map { + /// The key's field id. + key_id: i32, + /// The key type. + key: Box, + /// The value's field id. + value_id: i32, + /// Whether values can't be null. + value_required: bool, + /// The value type. + value: Box, + }, + /// A type this version doesn't model (format v3's `variant`, `geometry(…)`, `unknown`, …), by + /// its name. Columns of such types can be listed but not decoded. + Other(String), +} + +impl Type { + /// The type of a field's `type` JSON value. + pub fn from_json(t: &Value) -> Result { + if let Some(name) = t.as_str() { + return Ok(match name.parse() { + Ok(p) => Self::Primitive(p), + Err(Error::Unsupported(_)) => Self::Other(name.to_string()), + Err(e) => return Err(e), + }); + } + let id = |key: &str| { + t.get(key) + .and_then(Value::as_i64) + .and_then(|v| i32::try_from(v).ok()) + .ok_or_else(|| corrupt(format!("type without {key}"))) + }; + let child = |key: &str| Self::from_json(t.get(key).ok_or_else(|| corrupt(format!("type without {key}")))?); + let flag = |key: &str| t.get(key).and_then(Value::as_bool).unwrap_or(false); + match t.get("type").and_then(Value::as_str) { + Some("struct") => Ok(Self::Struct(fields(t)?)), + Some("list") => Ok(Self::List { + element_id: id("element-id")?, + element_required: flag("element-required"), + element: Box::new(child("element")?), + }), + Some("map") => Ok(Self::Map { + key_id: id("key-id")?, + key: Box::new(child("key")?), + value_id: id("value-id")?, + value_required: flag("value-required"), + value: Box::new(child("value")?), + }), + other => Err(corrupt(format!("type {other:?}"))), + } + } + + /// The primitive type of the field with id `id` inside this type. + fn primitive_by_id(&self, id: i32) -> Option<&PrimitiveType> { + match self { + Self::Struct(fields) => fields.iter().find_map(|f| f.primitive_by_id(id)), + Self::List { element_id, element, .. } => { + if *element_id == id { + element.as_primitive() + } else { + element.primitive_by_id(id) + } + } + Self::Map { key_id, key, value_id, value, .. } => { + if *key_id == id { + key.as_primitive() + } else if *value_id == id { + value.as_primitive() + } else { + key.primitive_by_id(id).or_else(|| value.primitive_by_id(id)) + } + } + Self::Primitive(_) | Self::Other(_) => None, + } + } + + /// The primitive, if this is one. + pub fn as_primitive(&self) -> Option<&PrimitiveType> { + match self { + Self::Primitive(p) => Some(p), + _ => None, + } + } +} + +/// A field of a struct or schema. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NestedField { + /// Its id, unique in the schema. + pub id: i32, + /// Its name. + pub name: String, + /// Whether values can't be null. + pub required: bool, + /// Its type. + pub ty: Type, + /// Its documentation, if any. + pub doc: Option, +} + +impl NestedField { + fn from_json(f: &Value) -> Result { + let name = f.get("name").and_then(Value::as_str).ok_or_else(|| corrupt("field without a name"))?; + let id = f.get("id").and_then(Value::as_i64).and_then(|v| i32::try_from(v).ok()); + Ok(Self { + id: id.ok_or_else(|| corrupt(format!("field {name} without an id")))?, + name: name.to_string(), + required: f.get("required").and_then(Value::as_bool).unwrap_or(false), + ty: Type::from_json(f.get("type").ok_or_else(|| corrupt(format!("field {name} without a type")))?)?, + doc: f.get("doc").and_then(Value::as_str).map(str::to_string), + }) + } + + fn primitive_by_id(&self, id: i32) -> Option<&PrimitiveType> { + if self.id == id { self.ty.as_primitive() } else { self.ty.primitive_by_id(id) } + } +} + +fn fields(t: &Value) -> Result> { + let fields = t.get("fields").and_then(Value::as_array).ok_or_else(|| corrupt("struct without fields"))?; + fields.iter().map(NestedField::from_json).collect() +} + +/// A table schema. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Schema { + /// Its id (0 for format v1 tables, which have one schema). + pub schema_id: i32, + /// Top-level fields. + pub fields: Vec, +} + +impl Schema { + /// A schema from its metadata JSON (`{"schema-id": 0, "type": "struct", "fields": [...]}`). + pub fn from_json(schema: &Value) -> Result { + let id = schema.get("schema-id").and_then(Value::as_i64).unwrap_or(0); + Ok(Self { schema_id: i32::try_from(id).map_err(|_| corrupt("schema id"))?, fields: fields(schema)? }) + } + + /// The top-level field named `name`. + pub fn field_by_name(&self, name: &str) -> Option<&NestedField> { + self.fields.iter().find(|f| f.name == name) + } + + /// The primitive type of the field (at any depth) with id `id`: the type of its bounds in + /// manifests. + pub fn primitive_by_id(&self, id: i32) -> Option<&PrimitiveType> { + self.fields.iter().find_map(|f| f.primitive_by_id(id)) + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn type_names_round_trip() { + for name in ["boolean", "decimal(10, 2)", "fixed[16]", "timestamptz_ns", "uuid", "binary"] { + assert_eq!(name.parse::().unwrap().to_string(), name); + } + assert_eq!("decimal(9,3)".parse::().unwrap(), PrimitiveType::Decimal { precision: 9, scale: 3 }); + assert!(matches!("variant".parse::(), Err(Error::Unsupported(_)))); + assert!(matches!("fixed[x]".parse::(), Err(Error::Corrupt(_)))); + } + + #[test] + fn schemas_find_nested_field_types() { + let schema = Schema::from_json(&json!({ + "schema-id": 3, "type": "struct", "fields": [ + {"id": 1, "name": "time", "required": true, "type": "timestamptz"}, + {"id": 2, "name": "tags", "required": false, "type": + {"type": "map", "key-id": 4, "key": "string", "value-id": 5, "value-required": false, "value": "long"}}, + {"id": 3, "name": "points", "required": false, "type": + {"type": "list", "element-id": 6, "element-required": true, "element": + {"type": "struct", "fields": [{"id": 7, "name": "x", "required": true, "type": "double"}]}}}, + {"id": 8, "name": "v", "required": false, "type": "variant"} + ] + })) + .unwrap(); + assert_eq!(schema.schema_id, 3); + assert_eq!(schema.primitive_by_id(1), Some(&PrimitiveType::Timestamptz)); + assert_eq!(schema.primitive_by_id(5), Some(&PrimitiveType::Long)); + assert_eq!(schema.primitive_by_id(7), Some(&PrimitiveType::Double)); + assert_eq!(schema.primitive_by_id(6), None, "a struct element isn't primitive"); + assert_eq!(schema.field_by_name("v").unwrap().ty, Type::Other("variant".into())); + } +} diff --git a/crates/lakebed-iceberg/tests/iceberg_rust.rs b/crates/lakebed-iceberg/tests/iceberg_rust.rs new file mode 100644 index 0000000..2f90eda --- /dev/null +++ b/crates/lakebed-iceberg/tests/iceberg_rust.rs @@ -0,0 +1,296 @@ +//! lakebed-iceberg and iceberg-rust read each other's tables: a table iceberg-rust creates and +//! appends to, lakebed reads and commits to; iceberg-rust reads lakebed's commit and appends on top +//! of it; and lakebed computes the statistics iceberg-rust's Parquet writer records. Everything +//! runs in memory. +#![cfg(all(feature = "write", feature = "arrow", feature = "parquet-stats"))] +#![expect(clippy::unwrap_used, reason = "test code")] + +use std::collections::{BTreeMap, HashMap}; +use std::future::{Future, ready}; +use std::sync::Arc; + +use arrow_array::{Float64Array, RecordBatch, StringArray, TimestampMicrosecondArray}; +use bytes::Bytes; +use futures::TryStreamExt; +use iceberg::io::FileIO; +use iceberg::memory::MemoryCatalogBuilder; +use iceberg::spec::{ + DataFile as IceDataFile, FormatVersion, Literal, Manifest, ManifestList, NestedField, PrimitiveType as IceType, + Schema as IceSchema, Struct, Transform as IceTransform, Type as IceFieldType, UnboundPartitionSpec, +}; +use iceberg::transaction::{ApplyTransactionAction, Transaction}; +use iceberg::writer::file_writer::{FileWriter as _, FileWriterBuilder, ParquetWriterBuilder}; +use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, NamespaceIdent, TableCreation, TableIdent}; +use lakebed_core::{Error, Host, Listed, MemoryHost, MemoryUpload}; +use lakebed_iceberg::stats::ParquetStats; +use lakebed_iceberg::{Change, CommitOptions, DataFile, Datum, Entry, NoHooks, Status, Table, commit}; +use parquet::arrow::ArrowWriter; +use parquet::file::metadata::ParquetMetaDataReader; +use parquet::file::properties::WriterProperties; + +/// Microseconds at 2026-01-01T00:00:00Z. +const T0: i64 = 1_767_225_600_000_000; +const DAY_US: i64 = 86_400_000_000; + +/// A [`Host`] over iceberg-rust's `FileIO`, so both see the same objects. Clock and randomness come +/// from a [`MemoryHost`]; listing isn't needed (tables load from their metadata file). +struct IoHost { + io: FileIO, + memory: MemoryHost, +} + +fn store(e: impl std::fmt::Display) -> Error { + Error::Store(e.to_string()) +} + +impl Host for IoHost { + type Upload = MemoryUpload; + + async fn get(&self, uri: &str) -> lakebed_core::Result>> { + if !self.io.exists(uri).await.map_err(store)? { + return Ok(None); + } + Ok(Some(self.io.new_input(uri).map_err(store)?.read().await.map_err(store)?.to_vec())) + } + + async fn get_range(&self, uri: &str, start: u64, len: u64) -> lakebed_core::Result> { + let bytes = self.get(uri).await?.ok_or_else(|| Error::NotFound(uri.to_string()))?; + let start = usize::try_from(start).map_err(store)?.min(bytes.len()); + let end = start.saturating_add(usize::try_from(len).map_err(store)?).min(bytes.len()); + Ok(bytes[start..end].to_vec()) + } + + async fn exists(&self, uri: &str) -> lakebed_core::Result { + self.io.exists(uri).await.map_err(store) + } + + async fn put(&self, uri: &str, bytes: Vec) -> lakebed_core::Result<()> { + self.io.new_output(uri).map_err(store)?.write(Bytes::from(bytes)).await.map_err(store) + } + + /// The tests run one writer at a time, so checking first is enough. + async fn put_if_absent(&self, uri: &str, bytes: Vec) -> lakebed_core::Result { + if self.exists(uri).await? { + return Ok(false); + } + self.put(uri, bytes).await.map(|()| true) + } + + fn upload(&self, _: &str) -> impl Future> { + ready(Err(Error::Unsupported("uploads".into()))) + } + + fn list(&self, _: &str) -> impl Future>> { + ready(Err(Error::Unsupported("listing".into()))) + } + + async fn delete(&self, uris: &[String]) -> lakebed_core::Result<()> { + for uri in uris { + self.io.delete(uri).await.map_err(store)?; + } + Ok(()) + } + + fn now_ms(&self) -> i64 { + self.memory.now_ms() + } + + fn random(&self) -> u64 { + self.memory.random() + } + + fn sleep(&self, ms: u64) -> impl Future { + self.memory.sleep(ms) + } +} + +fn ice_schema() -> IceSchema { + IceSchema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "time", IceFieldType::Primitive(IceType::Timestamptz)).into(), + NestedField::required(2, "device", IceFieldType::Primitive(IceType::String)).into(), + NestedField::optional(3, "value", IceFieldType::Primitive(IceType::Double)).into(), + ]) + .build() + .unwrap() +} + +/// Rows of one day: `n` readings from `day` days after `T0`, every seventh value NaN. +fn batch(arrow: &arrow_schema::SchemaRef, day: i64, n: i64) -> RecordBatch { + let start = T0 + day * DAY_US; + let time = + TimestampMicrosecondArray::from_iter_values((0..n).map(|i| start + i * 1_000_000)).with_timezone("+00:00"); + let device = StringArray::from_iter_values((0..n).map(|i| format!("d{}", i % 5))); + let value: Float64Array = (0..n) + .map(|i| { + if i % 7 == 0 { + Some(f64::NAN) + } else if i % 11 == 0 { + None + } else { + Some(f64::from(i32::try_from(i).unwrap())) + } + }) + .collect(); + RecordBatch::try_new(arrow.clone(), vec![Arc::new(time), Arc::new(device), Arc::new(value)]).unwrap() +} + +fn nan_counts(batch: &RecordBatch) -> BTreeMap { + let values = batch.column(2).as_any().downcast_ref::().unwrap(); + BTreeMap::from([(3, values.iter().filter(|v| v.is_some_and(f64::is_nan)).count() as u64)]) +} + +/// Writes `batch` with iceberg-rust's Parquet writer; returns its data file. +async fn write_theirs(table: &iceberg::table::Table, batch: &RecordBatch, name: &str, day: i32) -> IceDataFile { + let builder = ParquetWriterBuilder::new(WriterProperties::default(), table.metadata().current_schema().clone()); + let uri = format!("{}/data/{name}.parquet", table.metadata().location()); + let mut writer = builder.build(table.file_io().new_output(&uri).unwrap()).await.unwrap(); + writer.write(batch).await.unwrap(); + let mut files = writer.close().await.unwrap(); + files.remove(0).partition(Struct::from_iter([Some(Literal::int(day))])).partition_spec_id(0).build().unwrap() +} + +/// Writes `batch` with arrow-rs's Parquet writer; returns lakebed's data file. +async fn write_ours(host: &IoHost, table: &Table, batch: &RecordBatch, name: &str, day: i32) -> DataFile { + let mut bytes = vec![]; + let mut writer = ArrowWriter::try_new(&mut bytes, batch.schema(), None).unwrap(); + writer.write(batch).unwrap(); + let footer = writer.close().unwrap(); + let stats = ParquetStats::new(&footer, bytes.len() as u64, nan_counts(batch)); + let uri = format!("{}/data/{name}.parquet", table.location); + host.put(&uri, bytes).await.unwrap(); + stats.data_file(uri, vec![Some(Datum::Date(day))]) +} + +async fn append_theirs( + catalog: &MemoryCatalog, + table: &iceberg::table::Table, + file: IceDataFile, +) -> iceberg::table::Table { + let tx = Transaction::new(table); + let tx = tx.fast_append().add_data_files([file]).apply(tx).unwrap(); + tx.commit(catalog).await.unwrap() +} + +/// The data file iceberg-rust records, as lakebed records it. +fn as_ours(f: &IceDataFile) -> DataFile { + let bytes = + |m: &HashMap| m.iter().map(|(k, d)| (*k, d.to_bytes().unwrap().to_vec())).collect(); + let counts = |m: &HashMap| m.iter().map(|(k, v)| (*k, *v)).collect(); + let partition = f + .partition() + .iter() + .map(|v| match v { + Some(Literal::Primitive(iceberg::spec::PrimitiveLiteral::Int(d))) => Some(Datum::Date(*d)), + other => panic!("partition value {other:?}"), + }) + .collect(); + DataFile { + path: f.file_path().to_string(), + partition, + record_count: f.record_count(), + file_size: f.file_size_in_bytes(), + column_sizes: counts(f.column_sizes()), + value_counts: counts(f.value_counts()), + null_counts: counts(f.null_value_counts()), + nan_counts: counts(f.nan_value_counts()), + lower_bounds: bytes(f.lower_bounds()), + upper_bounds: bytes(f.upper_bounds()), + split_offsets: f.split_offsets().map(<[i64]>::to_vec), + sort_order_id: f.sort_order_id(), + ..DataFile::default() + } +} + +/// Files iceberg-rust plans for a scan of the table at `metadata_file`. +async fn planned(io: &FileIO, metadata_file: &str) -> Vec { + let ident = TableIdent::from_strs(["ns", "static"]).unwrap(); + let table = iceberg::table::StaticTable::from_metadata_file(metadata_file, ident, io.clone()).await.unwrap(); + let tasks: Vec<_> = table.scan().build().unwrap().plan_files().await.unwrap().try_collect().await.unwrap(); + let mut paths: Vec = tasks.into_iter().map(|t| t.data_file_path).collect(); + paths.sort(); + paths +} + +async fn live(host: &IoHost, metadata_file: &str) -> (Table, Vec) { + let mut table = Table::from_metadata_file(host, metadata_file).await.unwrap(); + let mut files = table.live_files(host).await.unwrap().to_vec(); + files.sort_by(|a, b| a.file.path.cmp(&b.file.path)); + (table, files) +} + +#[tokio::test] +async fn tables_round_trip_between_lakebed_and_iceberg_rust() { + let catalog = MemoryCatalogBuilder::default() + .load("memory", HashMap::from([("warehouse".to_string(), "memory://wh".to_string())])) + .await + .unwrap(); + let ns = NamespaceIdent::new("ns".into()); + catalog.create_namespace(&ns, HashMap::new()).await.unwrap(); + let mut creation = TableCreation::builder() + .name("t".into()) + .location("memory://wh/ns/t".into()) + .schema(ice_schema()) + .format_version(FormatVersion::V2) + .build(); + creation.partition_spec = + Some(UnboundPartitionSpec::builder().add_partition_field(1, "time_day", IceTransform::Day).unwrap().build()); + let table = catalog.create_table(&ns, creation).await.unwrap(); + let io = table.file_io().clone(); + let host = IoHost { io: io.clone(), memory: MemoryHost::new(T0 / 1000) }; + let arrow = Arc::new(iceberg::arrow::schema_to_arrow_schema(table.metadata().current_schema()).unwrap()); + + // iceberg-rust appends; lakebed reads the file as iceberg-rust records it. + let first = batch(&arrow, 0, 5_000); + let theirs = write_theirs(&table, &first, "theirs-1", 20_454).await; + let table = append_theirs(&catalog, &table, theirs.clone()).await; + let (base, files) = live(&host, table.metadata_location().unwrap()).await; + assert_eq!(files.len(), 1); + assert_eq!(*files[0].file, as_ours(&theirs)); + assert_eq!((files[0].status, files[0].sequence_number, files[0].file_sequence_number), (Status::Added, 1, 1)); + + // lakebed computes the statistics iceberg-rust's writer recorded, from the file's footer. + let bytes = io.new_input(theirs.file_path()).unwrap().read().await.unwrap(); + let footer = ParquetMetaDataReader::new().parse_and_finish(&bytes).unwrap(); + let stats = ParquetStats::new(&footer, bytes.len() as u64, nan_counts(&first)); + assert_eq!(stats.data_file(theirs.file_path().into(), vec![Some(Datum::Date(20_454))]), as_ours(&theirs)); + + // lakebed commits on top; iceberg-rust reads the new version and lakebed's manifests. + let ours = write_ours(&host, &base, &batch(&arrow, 1, 3_000), "ours-1", 20_455).await; + let (committed, info) = commit( + &host, + base, + &Change::Append { files: vec![ours.clone()] }, + &HashMap::new(), + &CommitOptions::default(), + &mut NoHooks, + ) + .await + .unwrap(); + assert_eq!(info.version, 2); + let paths = planned(&io, &committed.metadata_file).await; + assert_eq!(paths, [ours.path.clone(), theirs.file_path().to_string()]); + let list = io.new_input(committed.metadata.manifest_list().unwrap()).unwrap().read().await.unwrap(); + let list = ManifestList::parse_with_version(&list, FormatVersion::V2).unwrap(); + let added = list.entries().iter().find(|m| m.added_snapshot_id == info.snapshot_id).unwrap(); + assert_eq!((added.sequence_number, added.added_files_count), (2, Some(1))); + let manifest = io.new_input(&added.manifest_path).unwrap().read().await.unwrap(); + let (entries, _) = Manifest::parse_avro(&manifest).unwrap().into_parts(); + assert_eq!(as_ours(&entries[0].data_file), ours); + + // iceberg-rust appends on top of lakebed's version; lakebed reads all three files. Its memory + // catalog only names metadata files `NNNNN-.metadata.json`, so it gets a copy so named. + let ident = TableIdent::from_strs(["ns", "registered"]).unwrap(); + let copy = + committed.metadata_file.replace("v2.metadata.json", "00002-1d5e8c1a-0000-4000-8000-000000000000.metadata.json"); + host.put(©, committed.metadata.to_json().into_bytes()).await.unwrap(); + let registered = catalog.register_table(&ident, copy).await.unwrap(); + let again = write_theirs(®istered, &batch(&arrow, 2, 1_000), "theirs-2", 20_456).await; + let registered = append_theirs(&catalog, ®istered, again.clone()).await; + let (_, files) = live(&host, registered.metadata_location().unwrap()).await; + let seen: Vec<(&str, i64)> = + files.iter().map(|e| (e.file.path.rsplit('/').next().unwrap(), e.sequence_number)).collect(); + assert_eq!(seen, [("ours-1.parquet", 2), ("theirs-1.parquet", 1), ("theirs-2.parquet", 3)]); + assert_eq!(*files[2].file, as_ours(&again)); +} diff --git a/crates/lakebed/Cargo.toml b/crates/lakebed/Cargo.toml new file mode 100644 index 0000000..766e29c --- /dev/null +++ b/crates/lakebed/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "lakebed" +description = "Table formats (Apache Iceberg first) read and written without I/O of their own, for WebAssembly and native" +readme = "../../README.md" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true + +[features] +default = ["iceberg"] +# Apache Iceberg (lakebed-iceberg). +iceberg = ["dep:lakebed-iceberg"] +# Writing Iceberg tables. +iceberg-write = ["iceberg", "lakebed-iceberg/write"] +# Iceberg schemas as Arrow schemas. +iceberg-arrow = ["iceberg", "lakebed-iceberg/arrow"] +# Iceberg statistics from Parquet footers. +iceberg-parquet-stats = ["iceberg", "lakebed-iceberg/parquet-stats"] + +[dependencies] +lakebed-core.workspace = true +lakebed-iceberg = { workspace = true, optional = true } + +[package.metadata.docs.rs] +all-features = true + +[lints] +workspace = true diff --git a/crates/lakebed/src/lib.rs b/crates/lakebed/src/lib.rs new file mode 100644 index 0000000..1380306 --- /dev/null +++ b/crates/lakebed/src/lib.rs @@ -0,0 +1,11 @@ +//! Table formats read and written without I/O of their own, for WebAssembly and native: one crate +//! per format, re-exported here behind a feature each. The I/O boundary is [`core::Host`]. +//! +//! | Feature | Crate | +//! |---|---| +//! | `iceberg` (default) | [`iceberg`]: Apache Iceberg (`lakebed-iceberg`) | +//! | `iceberg-write`, `iceberg-arrow`, `iceberg-parquet-stats` | its optional parts | + +pub use lakebed_core as core; +#[cfg(feature = "iceberg")] +pub use lakebed_iceberg as iceberg; diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..7533fe2 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.99" +profile = "minimal" +components = ["rustfmt", "clippy"] diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000..f0ce655 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,3 @@ +style_edition = "2024" +max_width = 120 +use_small_heuristics = "Max" diff --git a/spec/0001-host.md b/spec/0001-host.md new file mode 100644 index 0000000..426269b --- /dev/null +++ b/spec/0001-host.md @@ -0,0 +1,50 @@ +--- +number: 1 +title: The host boundary +status: done +updated: 2026-10-20 +versions: [unreleased] +related: [2, 3] +issues: [] +--- + +# 1. The host boundary + +**Done** · updated 2026-10-20 · versions unreleased · milestone: [0.1](milestones/v0-1.md) + +## Summary + +Format crates do no I/O of their own. They call a `Host` (`lakebed-core`) that the application implements over its +object store, clock, and randomness, so the same code runs in a browser, a Cloudflare Worker, a CLI, or a server, and +tests run against memory. + +## Design + +- **`Host`:** `get` (the object or `None`), `get_range`, `exists`, `put`, `put_if_absent` (the commit primitive: + `false` when an object is there), `upload` (a multipart upload: parts in order, then complete or abort), `list` (by + prefix), `delete` (missing objects are not an error), `now_ms`, `random`, and `sleep`. Paths are absolute URIs, as + table metadata records them. +- **Futures aren't `Send`.** Methods return `impl Future`, without a `Send` bound, so a host can wrap JavaScript + promises (wasm-bindgen). Native hosts are free to be `Send`. +- **`MemoryHost`:** objects in memory, a clock that only moves when told (or when `sleep` is called), and a seeded + random sequence, so tests are deterministic. +- **`Error`:** one error type for all crates, with the variant saying what a caller can do: `Invalid`, `Unsupported`, + `Conflict` (retry on the newer version), `CommitUnknown` (the commit may have landed: delete nothing it wrote, load + the table again), `NotFound`, `Store`, and `Corrupt`. It is `non_exhaustive`. +- **Identifiers:** `uuid_v4` and `uuid_v7` from the host's randomness and clock, for file names. + +## Decisions + +| ID | Decision | Status | +|---|---|---| +| D1 | Format crates do no I/O: everything goes through a `Host` trait the application implements (the idea of delta-kernel-rs's `Engine`) | Accepted | +| D2 | Host futures aren't `Send`, so JavaScript hosts work; native applications lose nothing | Accepted | +| D3 | One error type across crates, whose variants say what a caller can do | Accepted | + +## Open questions and issues + +None open. + +## History + +- 2026-10-20: built, from Timelake's `timelake_format::host`. diff --git a/spec/0002-formats.md b/spec/0002-formats.md new file mode 100644 index 0000000..979a0b2 --- /dev/null +++ b/spec/0002-formats.md @@ -0,0 +1,50 @@ +--- +number: 2 +title: One crate per format +status: in-progress +updated: 2026-10-20 +versions: [unreleased] +related: [1, 3] +issues: [6] +--- + +# 2. One crate per format + +**In progress** · updated 2026-10-20 · versions unreleased · milestone: [0.1](milestones/v0-1.md) + +## Summary + +Each table format is its own crate, `lakebed-`, depending on `lakebed-core` and nothing of another format's. +The `lakebed` crate re-exports them behind a feature each. Iceberg is first (spec 3), because Timelake writes it and +lakeglass reads it; the formats lakeglass explores come next. + +## Design + +- **Layout:** `crates/lakebed-core` (spec 1), `crates/lakebed-`, and `crates/lakebed`, whose features are the + format names (`iceberg`) and their parts (`iceberg-write`, `iceberg-arrow`, `iceberg-parquet-stats`). +- **Formats as they are.** A format crate exposes its format's own model (Iceberg's snapshots and manifests, Delta's + log, Hudi's timeline) without a shared abstraction that would hide the differences. A common view, where an + application needs one (lakeglass's table explorer), is built on top of the format crates, not inside them. +- **Reading first.** A format crate reads every version of its format that is in use, and writes what an application + needs, starting with the version most readers support. +- **Interop is tested** against the format's reference implementation: it reads what we write, and we read what it + writes, offline. +- **Building for WebAssembly:** every crate builds for `wasm32-unknown-unknown` and is checked for `wasm32-wasip2`. +- **Next formats,** in the order lakeglass needs them: Delta Lake, then Apache Hudi, Apache Paimon, Lance, and + DuckLake. Each is proposed as its own spec before it is built. + +## Decisions + +| ID | Decision | Status | +|---|---|---| +| D4 | One repository with a crate per format and an umbrella crate with a feature each, not one crate with every format | Accepted | +| D5 | Format crates show their format as it is; a common view lives above them | Accepted | +| D6 | Arrow and Parquet follow the major version iceberg-rust's newest release uses, so applications using both build one copy | Accepted | + +## Open questions and issues + +- [#6](https://github.com/polythings/lakebed/issues/6) Which format comes after Iceberg, and in what order? + +## History + +- 2026-10-20: proposed and the layout built, with Iceberg as the first format. diff --git a/spec/0003-iceberg.md b/spec/0003-iceberg.md new file mode 100644 index 0000000..0999099 --- /dev/null +++ b/spec/0003-iceberg.md @@ -0,0 +1,86 @@ +--- +number: 3 +title: Apache Iceberg +status: in-progress +updated: 2026-10-20 +versions: [unreleased] +related: [1, 2] +issues: [1, 2, 3, 4, 5] +--- + +# 3. Apache Iceberg + +**In progress** · updated 2026-10-20 · versions unreleased · milestone: [0.1](milestones/v0-1.md) + +## Summary + +`lakebed-iceberg` reads Iceberg tables of format versions 1 to 3 and writes format v2 tables, without I/O of its own +(spec 1). Timelake's WASM reader and writer use it, and lakeglass's table explorer will. It started as Timelake's +shared writer, made general. + +## Design + +**Built:** + +- **Types and values.** Iceberg's primitive types, nested types (struct, list, map), and types this version doesn't + model (format v3's `variant`, `geometry`, …) kept by name so such tables still read. `Datum` holds a value of any + primitive type, with Iceberg's single-value binary encoding both ways; a `long` or `double` bound stored in 4 bytes + (written before the column was promoted) widens. +- **Schemas and partition specs**, from metadata JSON: field lookup by id at any depth, transforms (`identity`, + `bucket[N]`, `truncate[W]`, `year`, `month`, `day`, `hour`, `void`, and unknown ones by name), and each partition + field's result type (`day` gives a `date`, as iceberg-rust types it). Format v1 specs without field ids get 1000 + and up. +- **Table metadata** (`Metadata`): kept as JSON, so fields this crate doesn't model survive a rewrite; typed views of + snapshots, ancestors, the current schema and spec (including format v1's `schema` and `partition-spec`), and + properties. Format versions above 3 are refused. +- **Manifest lists and manifests,** format v1 to v3: data and delete files (position, equality, and v3 deletion + vectors' `referenced_data_file`, `content_offset`, and `content_size_in_bytes`), column statistics, and partition + values typed from the manifest header's schema and spec. Entries inherit snapshot ids and sequence numbers from + their manifest, as the spec says. Format v1's `*_data_files_count` names are read. +- **Tables** (`Table`): the newest `v.metadata.json` by listing, or `version-hint.text` where nothing is listed; + any metadata file a catalog names (`from_metadata_file`); manifests and live files. +- **Writing (feature `write`):** format v2 data manifests and manifest lists in the layout iceberg-rust writes, and + metadata updates (`with_snapshot`, `without_snapshots`) made as iceberg-rust's builder makes them, with the metadata + log trimmed to `write.metadata.previous-versions-max`; the current snapshot can't be removed. Partition values of type uuid, fixed, and decimal are refused + for now. +- **Commits (feature `write`):** upload a snapshot's files, then write `v.metadata.json` only if absent; on a + lost race, reload and try again on the newer version, deleting the lost attempt's files; when the outcome is + unknown (a conditional put that failed and can't be read back), delete nothing. Appends, replacements that keep data + sequence numbers (compaction, one sequence number per added file), and deletions; manifests merge once there would be more than `merge_manifests_at`. + Iceberg's summary counters are kept. A writer whose clock is behind the table's last update dates its commit at that + update, so the snapshot log stays in order (iceberg-rust refuses tables whose log isn't). +- **Options and hooks:** `CommitOptions` (`max_attempts` 10, `backoff_ms` 5–50 times the attempt, `merge_manifests_at` + 8, `write_version_hint` on) and `CommitHooks` (`reload`, `skip` for idempotent retries, `extra` for files and + summary properties of the application's own). Timelake's last-value index, batch ids, and head use them. +- **Arrow (feature `arrow`):** the schema as an Arrow schema with field ids under `PARQUET:field_id`. +- **Statistics (feature `parquet-stats`):** a data file's counts, bounds, and split offsets from its Parquet footer, as + iceberg-rust's writer records them; decimals become Iceberg's shortest big-endian bytes however Parquet stores them. + A bound or null count is recorded only when every row group gives it exactly (a row group of nulls only needs no + bounds): readers take them as covering the whole file, and a partial one would let them skip matching rows. +- **Interop tests:** iceberg-rust creates a table and appends; lakebed reads it and commits; iceberg-rust reads that + version, parses lakebed's manifests, and appends on top; lakebed reads all of it. lakebed's statistics of + iceberg-rust's Parquet file equal what iceberg-rust recorded. + +**Not yet:** applying partition transforms to values, planning files from filters, writing format v3, writing delete +files, and catalogs (REST). Each has an issue. + +## Decisions + +| ID | Decision | Status | +|---|---|---| +| D7 | Read format v1 to v3; write format v2 first, the version every reader supports | Accepted | +| D8 | Table metadata stays JSON with typed views, so unknown fields survive rewrites | Accepted | +| D9 | Commits are extended through hooks (`CommitHooks`) rather than by copying the protocol | Accepted | + +## Open questions and issues + +- [#1](https://github.com/polythings/lakebed/issues/1) Apply partition transforms to values +- [#2](https://github.com/polythings/lakebed/issues/2) Plan files and row groups from filters +- [#3](https://github.com/polythings/lakebed/issues/3) Write uuid, fixed and decimal partition values +- [#4](https://github.com/polythings/lakebed/issues/4) Write format v3 tables +- [#5](https://github.com/polythings/lakebed/issues/5) Iceberg REST catalogs: here or in lakeglass? + +## History + +- 2026-10-20: built from Timelake's shared writer (`timelake-format`), made general: any partition spec, format v1 to + v3 reads, delete files, hooks, and options. diff --git a/spec/README.md b/spec/README.md new file mode 100644 index 0000000..d261de6 --- /dev/null +++ b/spec/README.md @@ -0,0 +1,43 @@ +# Specs + +lakebed's design, laid out like Timelake's and lakeglass's. Start here, people and agents alike. + +- **Specs** (`NNNN-name.md`): one feature each, with the same template ([TEMPLATE.md](TEMPLATE.md)). A spec says what + the feature does and why, what is still open (as GitHub issues), how far it is built, and the releases that changed + it. +- **Milestones** ([`milestones/`](milestones/README.md)): goals made of specs. The roadmap is the list of milestones. + +## Statuses + +| Status | Meaning | +|---|---| +| Done | Built and in effect; the spec describes how it works now. | +| In progress | Partly built: the spec says which parts are, and the open parts have issues. | +| Planned | Agreed or proposed, not built yet. | +| Dropped | Discussed and not done; kept so the reasons stay on record. | + +## Writing and changing a spec + +- **When:** anything beyond a small fix gets a spec, or a change to one, agreed before it is built. +- **New spec:** copy [TEMPLATE.md](TEMPLATE.md) to `NNNN-short-name.md` with the next free number (numbers are never + reused), and add it to a milestone and to the index below. +- **Front matter:** `number`, `title`, `status` (`done`, `in-progress`, `planned`, `dropped`), `updated`, `versions` + (the releases that changed it; `unreleased` until a release names it), and optionally `related` and `issues`. +- **Open questions** are GitHub issues (label `question`), listed under `issues`, not paragraphs in the spec. +- **Same commit:** when work lands, its spec's status, versions, and history change with the code and the changelog. + +## Index + +3 specs: 2 in progress, 1 done. Decisions D1–D9. + +| # | Spec | Status | Updated | +|---:|---|---|---| +| 1 | [The host boundary](0001-host.md) | Done | 2026-10-20 | +| 2 | [One crate per format](0002-formats.md) | In progress | 2026-10-20 | +| 3 | [Apache Iceberg](0003-iceberg.md) | In progress | 2026-10-20 | + +## Milestones + +| Milestone | Status | +|---|---| +| [0.1: Iceberg for Timelake and lakeglass](milestones/v0-1.md) | In progress | diff --git a/spec/TEMPLATE.md b/spec/TEMPLATE.md new file mode 100644 index 0000000..321ab22 --- /dev/null +++ b/spec/TEMPLATE.md @@ -0,0 +1,36 @@ +--- +number: 0 +title: Short title +status: planned +updated: YYYY-MM-DD +versions: [] +related: [] +issues: [] +--- + +# 0. Short title + +**Planned** · updated YYYY-MM-DD · milestone: [Name](milestones/slug.md) + +## Summary + +Two to four sentences: what this feature does, for whom, and why. + +## Design + +How it works, as precisely as needed to build or review it: types, options, interfaces, edge cases. Cite other specs +as "spec N", linked to their files. + +## Decisions + +| ID | Decision | Status | +|---|---|---| +| D0 | One sentence | Proposed | + +## Open questions and issues + +- [#N](https://github.com/polythings/lakebed/issues/N) Each open question or task is an issue. + +## History + +- YYYY-MM-DD: proposed. diff --git a/spec/milestones/README.md b/spec/milestones/README.md new file mode 100644 index 0000000..5f6ca11 --- /dev/null +++ b/spec/milestones/README.md @@ -0,0 +1,7 @@ +# Milestones + +The roadmap: goals made of specs. A milestone is done, in progress, planned, or exploring (an idea being looked into). + +| Milestone | Status | Specs | +|---|---|---| +| [0.1: Iceberg for Timelake and lakeglass](v0-1.md) | In progress | 1, 2, 3 | diff --git a/spec/milestones/v0-1.md b/spec/milestones/v0-1.md new file mode 100644 index 0000000..a249c4b --- /dev/null +++ b/spec/milestones/v0-1.md @@ -0,0 +1,15 @@ +# 0.1: Iceberg for Timelake and lakeglass + +**In progress** + +The first release: the host boundary and the Iceberg crate, enough for Timelake's WASM reader and writer to use +lakebed instead of their own Iceberg code, and for lakeglass to read Iceberg tables in the browser. + +| # | Spec | Status | +|---:|---|---| +| 1 | [The host boundary](../0001-host.md) | Done | +| 2 | [One crate per format](../0002-formats.md) | In progress | +| 3 | [Apache Iceberg](../0003-iceberg.md) | In progress | + +**Done when:** the crates are on crates.io ([#7](https://github.com/polythings/lakebed/issues/7)), Timelake depends +on them by version, and lakeglass's Iceberg reader is built on them.