From 316c89b54aa3f1bb7fd2b35460b89ef624b22933 Mon Sep 17 00:00:00 2001 From: Oleksandr Litus Date: Tue, 20 Oct 2026 17:56:56 -0400 Subject: [PATCH 1/2] Iceberg: lakebed 0.1 with lakebed-core and lakebed-iceberg The I/O boundary (Host) and Apache Iceberg without I/O of its own, for WebAssembly and native: metadata, schemas, partition specs, manifests and manifest lists read for format v1 to v3; format v2 manifests, metadata updates and file-system commits written, with options and hooks; Arrow schemas and Parquet statistics behind features. Started as Timelake's shared writer. The tests commit to one table with lakebed and iceberg-rust in turn. Co-Authored-By: Claude Opus 5.5 --- .github/dependabot.yml | 24 + .github/workflows/ci.yml | 54 + .gitignore | 3 + .pre-commit-config.yaml | 38 + AGENTS.md | 30 + CHANGELOG.md | 20 + CLAUDE.md | 1 + CONTRIBUTING.md | 85 + Cargo.lock | 3309 ++++++++++++++++++ Cargo.toml | 43 + NOTICE | 6 + PRIOR_ART.md | 22 + README.md | 56 + clippy.toml | 2 + crates/lakebed-core/Cargo.toml | 17 + crates/lakebed-core/README.md | 12 + crates/lakebed-core/src/error.rs | 30 + crates/lakebed-core/src/host.rs | 55 + crates/lakebed-core/src/lib.rs | 31 + crates/lakebed-core/src/memory.rs | 138 + crates/lakebed-core/src/uuid.rs | 55 + crates/lakebed-iceberg/Cargo.toml | 43 + crates/lakebed-iceberg/README.md | 15 + crates/lakebed-iceberg/src/arrow.rs | 103 + crates/lakebed-iceberg/src/avro.rs | 86 + crates/lakebed-iceberg/src/commit.rs | 656 ++++ crates/lakebed-iceberg/src/datum.rs | 287 ++ crates/lakebed-iceberg/src/lib.rs | 89 + crates/lakebed-iceberg/src/manifest.rs | 942 +++++ crates/lakebed-iceberg/src/metadata.rs | 481 +++ crates/lakebed-iceberg/src/partition.rs | 228 ++ crates/lakebed-iceberg/src/stats.rs | 148 + crates/lakebed-iceberg/src/table.rs | 162 + crates/lakebed-iceberg/src/types.rs | 317 ++ crates/lakebed-iceberg/tests/iceberg_rust.rs | 296 ++ crates/lakebed/Cargo.toml | 32 + crates/lakebed/src/lib.rs | 11 + rust-toolchain.toml | 4 + rustfmt.toml | 3 + spec/0001-host.md | 50 + spec/0002-formats.md | 50 + spec/0003-iceberg.md | 84 + spec/README.md | 43 + spec/TEMPLATE.md | 36 + spec/milestones/README.md | 7 + spec/milestones/v0-1.md | 15 + 46 files changed, 8219 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 AGENTS.md create mode 100644 CHANGELOG.md create mode 100644 CLAUDE.md create mode 100644 CONTRIBUTING.md create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 NOTICE create mode 100644 PRIOR_ART.md create mode 100644 README.md create mode 100644 clippy.toml create mode 100644 crates/lakebed-core/Cargo.toml create mode 100644 crates/lakebed-core/README.md create mode 100644 crates/lakebed-core/src/error.rs create mode 100644 crates/lakebed-core/src/host.rs create mode 100644 crates/lakebed-core/src/lib.rs create mode 100644 crates/lakebed-core/src/memory.rs create mode 100644 crates/lakebed-core/src/uuid.rs create mode 100644 crates/lakebed-iceberg/Cargo.toml create mode 100644 crates/lakebed-iceberg/README.md create mode 100644 crates/lakebed-iceberg/src/arrow.rs create mode 100644 crates/lakebed-iceberg/src/avro.rs create mode 100644 crates/lakebed-iceberg/src/commit.rs create mode 100644 crates/lakebed-iceberg/src/datum.rs create mode 100644 crates/lakebed-iceberg/src/lib.rs create mode 100644 crates/lakebed-iceberg/src/manifest.rs create mode 100644 crates/lakebed-iceberg/src/metadata.rs create mode 100644 crates/lakebed-iceberg/src/partition.rs create mode 100644 crates/lakebed-iceberg/src/stats.rs create mode 100644 crates/lakebed-iceberg/src/table.rs create mode 100644 crates/lakebed-iceberg/src/types.rs create mode 100644 crates/lakebed-iceberg/tests/iceberg_rust.rs create mode 100644 crates/lakebed/Cargo.toml create mode 100644 crates/lakebed/src/lib.rs create mode 100644 rust-toolchain.toml create mode 100644 rustfmt.toml create mode 100644 spec/0001-host.md create mode 100644 spec/0002-formats.md create mode 100644 spec/0003-iceberg.md create mode 100644 spec/README.md create mode 100644 spec/TEMPLATE.md create mode 100644 spec/milestones/README.md create mode 100644 spec/milestones/v0-1.md diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..97a08c9 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,24 @@ +# Dependabot keeps dependencies current: one pull request a week per ecosystem with minor and patch updates, and one +# per major update. A new version is proposed once it's 7 days old (security updates at once). +version: 2 + +updates: + - package-ecosystem: github-actions + directory: / + schedule: { interval: weekly } + cooldown: { default-days: 7 } + groups: + actions: { update-types: [minor, patch] } + + - package-ecosystem: cargo + directory: / + schedule: { interval: weekly } + cooldown: { default-days: 7 } + groups: + cargo: { update-types: [minor, patch] } + ignore: + # iceberg (pinned with =, a test dependency) fixes the Arrow and Parquet majors: move them with it, by hand. + - dependency-name: "arrow-*" + update-types: [version-update:semver-major] + - dependency-name: parquet + update-types: [version-update:semver-major] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..99d007e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,54 @@ +# Checks every push to main and every pull request. Needs no secrets. +name: CI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.event_name == 'pull_request' && github.ref || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 + +jobs: + native: + name: Native (format, lint, tests) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: rustup toolchain install # from rust-toolchain.toml + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + - run: cargo fmt --all --check + - run: cargo clippy --workspace --all-features --all-targets -- -D warnings + - run: cargo test --workspace --all-features + # Each feature builds on its own, and with none, without warnings. + - name: Features one at a time + env: + RUSTFLAGS: -D warnings + run: | + cargo check -p lakebed-iceberg --no-default-features + cargo check -p lakebed-iceberg --features write + cargo check -p lakebed-iceberg --features arrow + cargo check -p lakebed-iceberg --features parquet-stats + + wasm: + name: WebAssembly (browser and WASI targets) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: rustup toolchain install # from rust-toolchain.toml + - run: rustup target add wasm32-unknown-unknown wasm32-wasip2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + - run: cargo clippy --workspace --all-features --target wasm32-unknown-unknown -- -D warnings + - run: cargo build --workspace --all-features --target wasm32-wasip2 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2c393a2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/target +.env* +.DS_Store diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..409e128 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,38 @@ +# Git hooks, run by prek (https://prek.j178.dev): `uv tool install prek && prek install --install-hooks`. +# On commit: formatters and fast checks on the staged files. On push: clippy, natively and for wasm32. +# Everything: `prek run --all-files --hook-stage pre-push`. CI runs the same checks plus the tests. +default_install_hook_types: [pre-commit, pre-push] + +repos: + - repo: builtin + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-merge-conflict + - id: check-yaml + - id: check-toml + - id: check-added-large-files + args: [--maxkb=1024] + - id: detect-private-key + + - repo: local + hooks: + - id: rustfmt + name: rustfmt + entry: rustfmt --edition 2024 + language: system + types: [rust] + - id: clippy + name: clippy + entry: cargo clippy --workspace --all-features --all-targets --quiet -- -D warnings + language: system + types: [rust] + pass_filenames: false + stages: [pre-push] + - id: clippy-wasm + name: clippy (wasm32) + entry: cargo clippy --workspace --all-features --target wasm32-unknown-unknown --quiet -- -D warnings + language: system + types: [rust] + pass_filenames: false + stages: [pre-push] diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..5a7647d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,30 @@ +# Agent instructions + +lakebed: table formats (Apache Iceberg first) read and written in Rust without I/O of their own, for WebAssembly and +native targets. One crate per format (`crates/lakebed-`) on a shared I/O boundary (`crates/lakebed-core`). +Timelake and lakeglass use it. + +## Before changing code + +- Follow [CONTRIBUTING.md](CONTRIBUTING.md). It is the code guide, and it applies to you. +- For anything beyond a small fix, check the specs ([spec/README.md](spec/README.md)). Agree on the design with the user + before building something new, and record the decision in the spec's Decisions table. + +## Rules + +- **Done means checked.** Before you report work as finished, run `prek run --all-files --hook-stage pre-push` and + `cargo test --workspace --all-features`. Report failures as failures. +- **Fix the code, not the checks.** Don't loosen lint or format configs. Don't add `#[allow]`. Use + `#[expect(..., reason = "...")]` only when the tool is genuinely wrong. +- **Keep things in sync:** a change in behaviour updates its spec (status, design, history), the tests, and + `CHANGELOG.md` (under `Unreleased`) in the same commit. +- **Everything builds for WASM.** Every crate builds for `wasm32-unknown-unknown`: no threads, no blocking I/O, no + `std::time`, no C dependencies. All I/O goes through the `Host` trait. CI checks it. +- **Readers don't trust files.** Bytes from a store can be anything: check lengths, return errors, never panic. +- **Interop is tested.** What a format crate writes must read in that format's reference implementation and the other + way round; the tests show it. +- **Every behaviour a caller might want to change is an option**, named for its mechanism. +- **Ideas, not code.** Never copy or translate another project's code. Credit ideas in `PRIOR_ART.md` + (CONTRIBUTING.md, "Credit"). +- **Tests run offline.** No network, cloud accounts, or real buckets in tests. +- **Never commit secrets** or `.env*` files. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..bb1a983 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,20 @@ +# Changelog + +All notable changes to lakebed. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and +versions follow [Semantic Versioning](https://semver.org/). + +## Unreleased + +### Added + +- `lakebed-core`: the `Host` trait for object stores, clocks, and randomness; `MemoryHost`; one `Error` type. +- `lakebed-iceberg`: Iceberg table metadata, schemas, partition specs, manifest lists, and manifests read for format + versions 1 to 3, including delete files and deletion vector references; values in Iceberg's single-value encoding. +- `lakebed-iceberg` feature `write`: format v2 manifests, manifest lists, and metadata updates, and commits to + file-system tables with options (`CommitOptions`) and hooks (`CommitHooks`). A writer whose clock is behind the + table's last update dates its commit at that update, so the snapshot log stays in order. +- `lakebed-iceberg` features `arrow` (schemas as Arrow schemas with field ids) and `parquet-stats` (a data file's + Iceberg statistics from its Parquet footer). +- `lakebed`: re-exports the format crates behind a feature each. + +The Iceberg code started as Timelake's shared writer (`timelake-format`), made general. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..43c994c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..c3fbea5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,85 @@ +# Contributing to lakebed + +This is the code guide for everyone who changes this repository, people and AI agents alike. Formatters and linters +enforce the mechanical rules (see [Checks](#checks)). This guide covers what they can't: how code should read. +[spec/](spec/README.md) holds the design. + +## Principles + +1. **Readable first.** Someone new should understand a function from its name, its signature, and one pass through + the body. +2. **One crate per format.** A format's crate depends on `lakebed-core` and on nothing of another format's, so adding + one never touches the others. Formats are shown as they are: no shared model flattens their differences. +3. **No I/O of our own.** Crates read and write through `lakebed_core::Host`. They never open sockets, files, or + threads, so they build for `wasm32-unknown-unknown`. +4. **One definition of each thing.** Before writing a helper, constant, or type, look for an existing one. +5. **Behaviour, spec, and tests change together.** A change users can see updates its spec, adds or updates tests, and + adds a line to [CHANGELOG.md](CHANGELOG.md). +6. **Behaviour is configurable.** Where callers could reasonably want something else (retries, limits, what to write), + it is an option with a default, named for its mechanism (`merge_manifests_at`, not `fast`). + +## Comments and docs + +- **Public items get a short doc comment**: what it does, and anything a caller must know (errors, units, + invariants). One to three lines is usually enough. +- **Comments explain why, not what.** Explain a non-obvious choice, a workaround, or a rule from a format's spec, with + a link to the spec section. +- **No long comments.** If an explanation needs a page, it belongs in a spec. Link to it from the code. + +## Credit + +lakebed learns from other projects' ideas, never from copying their code. [PRIOR_ART.md](PRIOR_ART.md) lists the +ideas we took. + +- **Never copy code**, whatever its license: not by pasting, and not by translating it into another language. +- **Credit ideas from open-source projects** with a row in PRIOR_ART.md and a short comment where the idea is + implemented. +- **Don't name open-core, dual-licensed, or source-available projects, or commercial products, as a source.** + +## Rust + +- **Errors:** `lakebed_core::Error`, with the variant that says what a caller can do about it (`NotFound`, `Conflict`, + `CommitUnknown`, `Unsupported`, `Corrupt`). No `unwrap()` outside tests. +- **Types over strings:** enums and newtypes instead of strings and magic numbers. Parse once at the boundary. +- **Don't trust files.** Check lengths and offsets before using them, cap allocations, and return an error instead of + panicking on a malformed file. Tests feed readers truncated and garbage input. +- **Async:** host futures aren't `Send` (JavaScript promises aren't). Never block inside async code. +- **Suppressing a lint:** `#[expect(lint, reason = "...")]` with a real reason. Never `#[allow]`. + +## Tests + +- **A bug fix starts with a test that fails**, then the fix. +- **Tests run offline**, against `MemoryHost` or a host over the reference implementation's in-memory storage. +- **Interop:** writers are tested against the format's reference reader, and readers against the reference writer. + +## Checks + +Install once: `uv tool install prek && prek install --install-hooks`. Hooks then run on commit and on push. Before you +push, or before calling work done: + +```bash +prek run --all-files --hook-stage pre-push # rustfmt, clippy (native and wasm32) +cargo test --workspace --all-features # unit, doc, and interop tests +``` + +## Dependencies + +Keep dependencies on their newest versions; check the latest release when adding one. Arrow and Parquet follow the +major version iceberg-rust's newest release uses (the interop tests build both). Actions are pinned to a commit with +the version in a comment (`uses: actions/checkout@ # v7.0.1`). + +## Commits + +- **Subject:** a short imperative sentence with the area first, for example `Core: …`, `Iceberg: …`, or `Spec: …`. + Add a body when the why isn't obvious. +- **Scope:** one logical change per commit. + +## Changelog + +[CHANGELOG.md](CHANGELOG.md) is for users, not a list of commits. Anything a user can notice gets one line under +`## Unreleased`, in `Added`, `Changed`, `Deprecated`, `Removed`, `Fixed`, or `Security`, in the same commit as the +change. + +## License + +Contributions are licensed under the Apache License 2.0, like the rest of the project (see [LICENSE](LICENSE)). diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..963b662 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,3309 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "apache-avro" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" +dependencies = [ + "bigdecimal", + "bon", + "crc32fast", + "digest 0.10.7", + "log", + "miniz_oxide 0.8.9", + "num-bigint", + "quad-rand", + "rand 0.9.5", + "regex-lite", + "serde", + "serde_bytes", + "serde_json", + "snap", + "strum 0.27.2", + "strum_macros 0.27.2", + "thiserror", + "uuid", + "zstd", +] + +[[package]] +name = "apache-avro" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c" +dependencies = [ + "bigdecimal", + "bon", + "digest 0.11.3", + "log", + "miniz_oxide 0.9.1", + "num-bigint", + "ouroboros", + "quad-rand", + "rand 0.10.3", + "regex-lite", + "serde", + "serde_bytes", + "serde_json", + "strum 0.28.0", + "thiserror", + "uuid", +] + +[[package]] +name = "array-init" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" + +[[package]] +name = "arrow-arith" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64 0.22.1", + "chrono", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-data" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ipc" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", +] + +[[package]] +name = "arrow-ord" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-schema" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" + +[[package]] +name = "arrow-select" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "as-any" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063" + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bigdecimal" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fa3f3d8cbf4dffcfe4991de61d012bef509a409ecbe9dd41049bfe32b4d4653" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "bimap" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bnum" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f781dba93de3a5ef6dc5b17c9958b208f6f3f021623b360fb605ea51ce443f10" +dependencies = [ + "serde", + "serde-big-array", +] + +[[package]] +name = "bon" +version = "3.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "214f5df094ce551a10a30ffb9c70243d61f121a3d985a6495933e181dee6a7d2" +dependencies = [ + "bon-macros", +] + +[[package]] +name = "bon-macros" +version = "3.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2706da6c749998cc555d04184909608956a69ad3f8ab1a076fbc867b4a3c3ce5" +dependencies = [ + "darling 0.24.1", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877bf67ac8e52449eb2f9e98a344ea163beb70dd1f9eebe3cc5357dcfeaee4df" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core 0.24.1", + "darling_macro 0.24.1", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.9", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core 0.24.1", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt-parser" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "dissimilar" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeda16ab4059c5fd2a83f2b9c9e9c981327b18aa8e3b313f7e6563799d4f093e" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "expect-test" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63af43ff4431e848fb47472a920f14fa71c24de13255a5692e93d4e90302acb0" +dependencies = [ + "dissimilar", + "once_cell", +] + +[[package]] +name = "fastnum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020d1b59a944bc239d79903fbac2eda2365138b44890c27979562f6592059dcd" +dependencies = [ + "bnum", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags 2.13.2", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide 0.9.1", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", +] + +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "iceberg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e50fb53c7480f414911ab76b6e14c0042d54ad5c30f6ac2bf2f52a487a25716" +dependencies = [ + "aes-gcm", + "anyhow", + "apache-avro 0.21.0", + "array-init", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "arrow-string", + "as-any", + "async-trait", + "backon", + "base64 0.22.1", + "bimap", + "bytes", + "chrono", + "derive_builder", + "expect-test", + "fastnum", + "flate2", + "fnv", + "futures", + "itertools", + "moka", + "murmur3", + "once_cell", + "ordered-float 4.6.0", + "parquet", + "rand 0.9.5", + "reqwest", + "roaring", + "serde", + "serde_bytes", + "serde_derive", + "serde_json", + "serde_repr", + "serde_with", + "strum 0.27.2", + "tokio", + "tracing", + "typed-builder", + "typetag", + "url", + "uuid", + "zeroize", + "zstd", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + +[[package]] +name = "inventory" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c" +dependencies = [ + "rustversion", +] + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lakebed" +version = "0.1.0" +dependencies = [ + "lakebed-core", + "lakebed-iceberg", +] + +[[package]] +name = "lakebed-core" +version = "0.1.0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "lakebed-iceberg" +version = "0.1.0" +dependencies = [ + "apache-avro 0.22.0", + "arrow-array", + "arrow-schema", + "bytes", + "futures", + "futures-util", + "iceberg", + "lakebed-core", + "parquet", + "serde_json", + "tokio", +] + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "murmur3" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ouroboros" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "parquet" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64 0.22.1", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.17.1", + "lz4_flex", + "num-bigint", + "num-integer", + "num-traits", + "paste", + "ring", + "seq-macro", + "simdutf8", + "snap", + "thrift", + "tokio", + "twox-hash", + "zstd", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" +dependencies = [ + "proc-macro2", + "syn 3.0.9", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "version_check", + "yansi", +] + +[[package]] +name = "quad-rand" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "roaring" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1" +dependencies = [ + "bytemuck", + "byteorder", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9adc193c780ef8f159aee8b61e2d5801aaa555e6eb0947fe45530ec506296f" +dependencies = [ + "base64 0.23.1", + "bs58", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.2", + "jiff", + "schemars 0.9.0", + "schemars 1.2.3", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49" +dependencies = [ + "darling 0.24.1", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "snap" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros 0.27.2", +] + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros 0.28.0", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "ordered-float 2.10.1", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokio" +version = "1.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c168a474b95bfecfe908a65d4bdffc523144b649a74c09c286afb35d53e0bc" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.13.2", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "twox-hash" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2" + +[[package]] +name = "typed-builder" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd9d30e3a08026c78f246b173243cf07b3696d274debd26680773b6773c2afc7" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "typetag" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" +dependencies = [ + "js-sys", + "tokio", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.2+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..c32eb9e --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,43 @@ +[workspace] +resolver = "2" +members = ["crates/lakebed-core", "crates/lakebed-iceberg", "crates/lakebed"] + +[workspace.package] +version = "0.1.0" +edition = "2024" +rust-version = "1.99" +license = "Apache-2.0" +repository = "https://github.com/polythings/lakebed" +keywords = ["lakehouse", "iceberg", "wasm", "table-format"] +categories = ["database-implementations", "wasm"] + +[workspace.dependencies] +lakebed-core = { path = "crates/lakebed-core", version = "0.1.0" } +lakebed-iceberg = { path = "crates/lakebed-iceberg", version = "0.1.0", default-features = false } +thiserror = "2" +serde_json = "1" +apache-avro = { version = "0.22", default-features = false } +futures-util = { version = "0.3", default-features = false, features = ["alloc"] } +# Arrow and Parquet stay on the major that iceberg-rust's newest release (0.10.1) uses, so applications using both +# (Timelake's engine) and the interop tests build one copy of each. +arrow-schema = "58" +parquet = { version = "58", default-features = false } +iceberg = "=0.10.1" +tokio = { version = "1", features = ["macros", "rt"] } + +[workspace.lints.rust] +unsafe_code = "forbid" +unreachable_pub = "warn" +missing_debug_implementations = "warn" +missing_docs = "warn" +rust_2018_idioms = { level = "warn", priority = -1 } + +[workspace.lints.clippy] +pedantic = { level = "warn", priority = -1 } +missing_errors_doc = "allow" +missing_panics_doc = "allow" +must_use_candidate = "allow" +allow_attributes = "warn" +dbg_macro = "warn" +todo = "warn" +unwrap_used = "warn" diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..0ce932b --- /dev/null +++ b/NOTICE @@ -0,0 +1,6 @@ +lakebed +Copyright 2026 Oleksandr Litus and the lakebed contributors + +This product is licensed under the Apache License, Version 2.0 (see LICENSE). + +Ideas lakebed borrowed from other projects are credited in PRIOR_ART.md. diff --git a/PRIOR_ART.md b/PRIOR_ART.md new file mode 100644 index 0000000..7622547 --- /dev/null +++ b/PRIOR_ART.md @@ -0,0 +1,22 @@ +# Prior art + +lakebed stands on other open-source projects and learns from more. This file credits the ideas we took. We take ideas, +never code: every line of lakebed is our own. How we credit is in [CONTRIBUTING.md](CONTRIBUTING.md#credit). + +## Foundations + +| Project | License | What lakebed uses | +|---|---|---| +| [Apache Iceberg](https://iceberg.apache.org) | Apache-2.0 | The table format and its specification | +| [iceberg-rust](https://github.com/apache/iceberg-rust) | Apache-2.0 | The reference the interop tests check against (a test dependency only) | +| [Apache Avro](https://avro.apache.org) (`apache-avro`) | Apache-2.0 | Reading and writing manifests and manifest lists | +| [arrow-rs](https://github.com/apache/arrow-rs) (`arrow-schema`, `parquet`) | Apache-2.0 | Arrow schemas; Parquet footers for statistics | + +## Ideas + +| Idea | From | License of the source | Where in lakebed | +|---|---|---|---| +| A format library with no I/O of its own: the application supplies storage through a trait the library calls | [delta-kernel-rs](https://github.com/delta-io/delta-kernel-rs) (its `Engine` trait) | Apache-2.0 | `lakebed-core`'s `Host` | +| One crate per backend under a small shared core, re-exported by an umbrella crate with a feature each | [arrow-rs](https://github.com/apache/arrow-rs), [Apache OpenDAL](https://github.com/apache/opendal) | Apache-2.0 | The workspace layout | +| Commits to tables without a catalog: `v.metadata.json` written only if absent, `version-hint.text` for readers | Iceberg's file-system ("Hadoop") tables | Apache-2.0 | `lakebed_iceberg::commit` | +| Avro-safe names for partition fields (`_x` and the character's code in hex) | Iceberg's Java implementation | Apache-2.0 | Manifest writing | diff --git a/README.md b/README.md new file mode 100644 index 0000000..c4a52da --- /dev/null +++ b/README.md @@ -0,0 +1,56 @@ +# lakebed + +Table formats read and written without I/O of their own, in Rust, for WebAssembly and native targets. Each format's +metadata (Apache Iceberg first) is parsed and written by plain code that asks a `Host` for bytes: the application +implements the host over its object store, so the same crate runs in a browser, a Cloudflare Worker, a CLI, or a +server. + +lakebed is what [Timelake](https://github.com/polythings/timelake) and [lakeglass](https://github.com/polythings/lakeglass) +share. The official Rust crates for these formats build on tokio and HTTP clients, which don't compile to +`wasm32-unknown-unknown`. + +## Crates + +| Crate | What it does | +|---|---| +| [`lakebed-core`](crates/lakebed-core) | The `Host` trait (ranged reads, listing, conditional writes, clock, randomness), errors, an in-memory host | +| [`lakebed-iceberg`](crates/lakebed-iceberg) | Apache Iceberg: metadata, manifests, and commits | +| [`lakebed`](crates/lakebed) | Re-exports the format crates, one feature each | + +More formats get their own crate as they are added (spec 2). + +## Iceberg + +| Part | Status | +|---|---| +| Table metadata, format v1 to v3: schemas, partition specs, snapshots, properties | Read | +| Manifest lists and manifests: data files, delete files, deletion vector references, column statistics | Read | +| Format v2 manifests, manifest lists, and metadata updates | Write (feature `write`) | +| Commits to file-system tables (`v.metadata.json` with a conditional put), with hooks | Write (feature `write`) | +| Schema as an Arrow schema with field ids | Feature `arrow` | +| A data file's Iceberg statistics from its Parquet footer | Feature `parquet-stats` | + +What lakebed writes, iceberg-rust reads, and the other way round: the tests commit to one table with both in turn. + +```rust +use lakebed_iceberg::Table; + +async fn files(host: &H) -> lakebed_core::Result { + let mut table = Table::load(host, "s3://bucket/warehouse/db/events").await?; + Ok(table.live_files(host).await?.len()) +} +``` + +## Development + +```bash +cargo test --workspace --all-features # tests, offline +cargo build --workspace --all-features --target wasm32-unknown-unknown # the WebAssembly build +prek run --all-files --hook-stage pre-push # formatters and linters +``` + +[CONTRIBUTING.md](CONTRIBUTING.md) is the code guide, and [spec/](spec/README.md) holds the design. + +## License + +Apache License 2.0 ([LICENSE](LICENSE)). diff --git a/clippy.toml b/clippy.toml new file mode 100644 index 0000000..731d87f --- /dev/null +++ b/clippy.toml @@ -0,0 +1,2 @@ +allow-unwrap-in-tests = true +doc-valid-idents = ["WebAssembly", ".."] diff --git a/crates/lakebed-core/Cargo.toml b/crates/lakebed-core/Cargo.toml new file mode 100644 index 0000000..b414a77 --- /dev/null +++ b/crates/lakebed-core/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "lakebed-core" +description = "The I/O boundary shared by lakebed's table format crates: a host trait for object stores, clocks, and randomness" +readme = "README.md" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true + +[dependencies] +thiserror.workspace = true + +[lints] +workspace = true diff --git a/crates/lakebed-core/README.md b/crates/lakebed-core/README.md new file mode 100644 index 0000000..9d74b43 --- /dev/null +++ b/crates/lakebed-core/README.md @@ -0,0 +1,12 @@ +# lakebed-core + +The I/O boundary of [lakebed](https://github.com/polythings/lakebed)'s table format crates. Format crates do no I/O of +their own: they call a `Host` that the application implements over its object store, clock, and randomness, so they +build for `wasm32-unknown-unknown` and native targets alike. + +- `Host`: get, ranged get, exists, put, put-if-absent (the commit primitive), multipart upload, list, delete, the time, + random numbers, and waiting. Its futures aren't `Send`, so a host can wrap JavaScript promises. +- `MemoryHost`: a host in memory with a fixed clock and seeded randomness, for tests. +- `Error`: one error type, whose variant says what a caller can do (`NotFound`, `Conflict`, `CommitUnknown`, …). + +Apache License 2.0. diff --git a/crates/lakebed-core/src/error.rs b/crates/lakebed-core/src/error.rs new file mode 100644 index 0000000..662fd96 --- /dev/null +++ b/crates/lakebed-core/src/error.rs @@ -0,0 +1,30 @@ +/// An error reading or writing a table. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + /// The request doesn't fit the table, or a value is out of range. + #[error("invalid: {0}")] + Invalid(String), + /// The table uses something this version can read but not write, or not read at all. + #[error("unsupported: {0}")] + Unsupported(String), + /// Another writer changed the table in a way this commit can't build on. + #[error("conflict: {0}")] + Conflict(String), + /// A commit whose outcome can't be told: its metadata may be in the table. Nothing it wrote + /// may be deleted; load the table again to see whether it committed. + #[error("commit outcome unknown: {0}")] + CommitUnknown(String), + /// An object the table needs is missing. + #[error("not found: {0}")] + NotFound(String), + /// The store failed. + #[error("store: {0}")] + Store(String), + /// A file isn't what its format says it should be. + #[error("unexpected file contents: {0}")] + Corrupt(String), +} + +/// `Result` with [`Error`]. +pub type Result = std::result::Result; diff --git a/crates/lakebed-core/src/host.rs b/crates/lakebed-core/src/host.rs new file mode 100644 index 0000000..4151c36 --- /dev/null +++ b/crates/lakebed-core/src/host.rs @@ -0,0 +1,55 @@ +use std::future::Future; + +use crate::Result; + +/// An object found by [`Host::list`]. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Listed { + /// Absolute URI. + pub uri: String, + /// Size in bytes. + pub size: u64, +} + +/// What a table format needs from where it runs: an object store, a clock, randomness, and a way +/// to wait. Paths are absolute URIs, as table metadata records them. +/// +/// The futures aren't required to be `Send`, so hosts can wrap JavaScript promises. +pub trait Host { + /// A multipart upload in progress. + type Upload: Upload; + + /// The object at `uri`, or `None` if there is none. + fn get(&self, uri: &str) -> impl Future>>>; + /// `len` bytes of the object at `uri` from `start`. + fn get_range(&self, uri: &str, start: u64, len: u64) -> impl Future>>; + /// Whether an object exists at `uri`. + fn exists(&self, uri: &str) -> impl Future>; + /// Writes (or replaces) the object at `uri`. + fn put(&self, uri: &str, bytes: Vec) -> impl Future>; + /// Writes the object at `uri` unless one exists; `false` if one did. The commit primitive. + fn put_if_absent(&self, uri: &str, bytes: Vec) -> impl Future>; + /// Starts a multipart upload to `uri`. + fn upload(&self, uri: &str) -> impl Future>; + /// Objects whose URI starts with `prefix`. + fn list(&self, prefix: &str) -> impl Future>>; + /// Deletes objects; missing ones are not an error. + fn delete(&self, uris: &[String]) -> impl Future>; + /// Milliseconds since the Unix epoch. + fn now_ms(&self) -> i64; + /// A random number. + fn random(&self) -> u64; + /// Waits `ms` milliseconds. + fn sleep(&self, ms: u64) -> impl Future; +} + +/// A multipart upload: parts in order, then [`Upload::complete`]. +pub trait Upload { + /// Sends the next part. Every part but the last must be at least the store's minimum (5 MiB on + /// S3 and R2). + fn put_part(&mut self, bytes: Vec) -> impl Future>; + /// Completes the upload: the object appears. + fn complete(self) -> impl Future>; + /// Drops the parts sent. + fn abort(self) -> impl Future; +} diff --git a/crates/lakebed-core/src/lib.rs b/crates/lakebed-core/src/lib.rs new file mode 100644 index 0000000..5975f62 --- /dev/null +++ b/crates/lakebed-core/src/lib.rs @@ -0,0 +1,31 @@ +//! The I/O boundary shared by lakebed's table format crates. Format crates do no I/O of their own: +//! they read and write through a [`Host`] the application implements over its object store (an +//! `object_store` client natively, `fetch` or an R2 binding in WebAssembly), so they build for +//! `wasm32-unknown-unknown` and native targets alike. +//! +//! ``` +//! use lakebed_core::{Host, MemoryHost}; +//! +//! # block_on(async { +//! let host = MemoryHost::new(0); +//! host.put("memory://t/a", b"x".to_vec()).await?; +//! assert!(host.exists("memory://t/a").await?); +//! # Ok::<(), lakebed_core::Error>(()) +//! # }).unwrap(); +//! # // MemoryHost's futures are ready at once, so one poll runs them. +//! # fn block_on(f: F) -> F::Output { +//! # let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); +//! # let mut f = std::pin::pin!(f); +//! # loop { if let std::task::Poll::Ready(v) = f.as_mut().poll(&mut cx) { return v; } } +//! # } +//! ``` + +mod error; +mod host; +mod memory; +mod uuid; + +pub use error::{Error, Result}; +pub use host::{Host, Listed, Upload}; +pub use memory::{MemoryHost, MemoryUpload}; +pub use uuid::{uuid_v4, uuid_v7}; diff --git a/crates/lakebed-core/src/memory.rs b/crates/lakebed-core/src/memory.rs new file mode 100644 index 0000000..919ebce --- /dev/null +++ b/crates/lakebed-core/src/memory.rs @@ -0,0 +1,138 @@ +use std::collections::BTreeMap; +use std::future::{Future, ready}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +use crate::{Error, Host, Listed, Result, Upload}; + +/// A [`Host`] in memory, with a fixed clock and a seeded random sequence: for tests and examples. +/// Clones share the same objects. +#[derive(Clone, Debug, Default)] +pub struct MemoryHost { + objects: Arc>>>, + clock: Arc>, + seed: Arc>, +} + +impl MemoryHost { + /// An empty store at time `now_ms`. + pub fn new(now_ms: i64) -> Self { + let host = Self::default(); + *lock(&host.clock) = now_ms; + host + } + + /// Moves the clock forward. + pub fn advance(&self, ms: i64) { + *lock(&self.clock) += ms; + } + + /// The URIs stored, sorted. + pub fn uris(&self) -> Vec { + self.objects().keys().cloned().collect() + } + + fn objects(&self) -> MutexGuard<'_, BTreeMap>> { + lock(&self.objects) + } +} + +fn lock(m: &Mutex) -> MutexGuard<'_, T> { + m.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// An upload to a [`MemoryHost`]. +#[derive(Debug)] +pub struct MemoryUpload { + host: MemoryHost, + uri: String, + bytes: Vec, +} + +impl Upload for MemoryUpload { + fn put_part(&mut self, bytes: Vec) -> impl Future> { + self.bytes.extend(bytes); + ready(Ok(())) + } + + fn complete(self) -> impl Future> { + self.host.objects().insert(self.uri, self.bytes); + ready(Ok(())) + } + + fn abort(self) -> impl Future { + ready(()) + } +} + +impl Host for MemoryHost { + type Upload = MemoryUpload; + + fn get(&self, uri: &str) -> impl Future>>> { + ready(Ok(self.objects().get(uri).cloned())) + } + + fn get_range(&self, uri: &str, start: u64, len: u64) -> impl Future>> { + let objects = self.objects(); + ready(objects.get(uri).ok_or_else(|| Error::NotFound(uri.to_string())).map(|bytes| { + let start = usize::try_from(start).unwrap_or(usize::MAX).min(bytes.len()); + let end = start.saturating_add(usize::try_from(len).unwrap_or(usize::MAX)).min(bytes.len()); + bytes[start..end].to_vec() + })) + } + + fn exists(&self, uri: &str) -> impl Future> { + ready(Ok(self.objects().contains_key(uri))) + } + + fn put(&self, uri: &str, bytes: Vec) -> impl Future> { + self.objects().insert(uri.to_string(), bytes); + ready(Ok(())) + } + + fn put_if_absent(&self, uri: &str, bytes: Vec) -> impl Future> { + let mut objects = self.objects(); + let absent = !objects.contains_key(uri); + if absent { + objects.insert(uri.to_string(), bytes); + } + ready(Ok(absent)) + } + + fn upload(&self, uri: &str) -> impl Future> { + ready(Ok(MemoryUpload { host: self.clone(), uri: uri.to_string(), bytes: vec![] })) + } + + fn list(&self, prefix: &str) -> impl Future>> { + let listed = self + .objects() + .range(prefix.to_string()..) + .take_while(|(k, _)| k.starts_with(prefix)) + .map(|(k, v)| Listed { uri: k.clone(), size: v.len() as u64 }) + .collect(); + ready(Ok(listed)) + } + + fn delete(&self, uris: &[String]) -> impl Future> { + let mut objects = self.objects(); + for uri in uris { + objects.remove(uri); + } + ready(Ok(())) + } + + fn now_ms(&self) -> i64 { + *lock(&self.clock) + } + + fn random(&self) -> u64 { + // A linear congruential generator (Knuth's MMIX constants), with the high bits folded in. + let mut seed = lock(&self.seed); + *seed = seed.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407); + *seed ^ (*seed >> 29) + } + + fn sleep(&self, ms: u64) -> impl Future { + self.advance(i64::try_from(ms).unwrap_or(i64::MAX)); + ready(()) + } +} diff --git a/crates/lakebed-core/src/uuid.rs b/crates/lakebed-core/src/uuid.rs new file mode 100644 index 0000000..320b951 --- /dev/null +++ b/crates/lakebed-core/src/uuid.rs @@ -0,0 +1,55 @@ +use std::fmt::Write as _; + +use crate::Host; + +/// A random UUID (version 4) from `host`. +pub fn uuid_v4(host: &impl Host) -> String { + let mut b = [0u8; 16]; + b[..8].copy_from_slice(&host.random().to_be_bytes()); + b[8..].copy_from_slice(&host.random().to_be_bytes()); + b[6] = (b[6] & 0x0f) | 0x40; + b[8] = (b[8] & 0x3f) | 0x80; + format_uuid(&b) +} + +/// A time-ordered UUID (version 7) from `host`'s clock and randomness: names made with it sort by +/// creation time. +pub fn uuid_v7(host: &impl Host) -> String { + let mut b = [0u8; 16]; + let ms = u64::try_from(host.now_ms()).unwrap_or(0); + b[..6].copy_from_slice(&ms.to_be_bytes()[2..]); + b[6..14].copy_from_slice(&host.random().to_be_bytes()); + b[14..].copy_from_slice(&host.random().to_be_bytes()[..2]); + b[6] = (b[6] & 0x0f) | 0x70; + b[8] = (b[8] & 0x3f) | 0x80; + format_uuid(&b) +} + +fn format_uuid(b: &[u8; 16]) -> String { + let hex = b.iter().fold(String::with_capacity(32), |mut s, x| { + let _ = write!(s, "{x:02x}"); + s + }); + format!("{}-{}-{}-{}-{}", &hex[..8], &hex[8..12], &hex[12..16], &hex[16..20], &hex[20..]) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::MemoryHost; + + #[test] + fn uuids_are_well_formed() { + let host = MemoryHost::new(1_767_225_600_000); + let v4 = uuid_v4(&host); + let v7 = uuid_v7(&host); + for (u, version) in [(&v4, '4'), (&v7, '7')] { + assert_eq!(u.len(), 36); + assert_eq!(u.chars().nth(14), Some(version)); + assert!(matches!(u.chars().nth(19), Some('8' | '9' | 'a' | 'b'))); + } + // Version 7 starts with the time, so later ones sort later. + host.advance(1); + assert!(uuid_v7(&host) > v7); + } +} diff --git a/crates/lakebed-iceberg/Cargo.toml b/crates/lakebed-iceberg/Cargo.toml new file mode 100644 index 0000000..ba6b738 --- /dev/null +++ b/crates/lakebed-iceberg/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "lakebed-iceberg" +description = "Apache Iceberg table metadata, manifests and commits without I/O of its own, for WebAssembly and native" +readme = "README.md" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true + +[features] +default = [] +# Writing: manifests, manifest lists and table metadata updates, and the commit protocol over a host. +write = [] +# The table schema as an Arrow schema, with Iceberg field ids in the field metadata. +arrow = ["dep:arrow-schema"] +# Iceberg's statistics of a data file (counts and bounds per column) from its Parquet footer. +parquet-stats = ["dep:parquet"] + +[dependencies] +lakebed-core.workspace = true +apache-avro.workspace = true +serde_json.workspace = true +futures-util.workspace = true +arrow-schema = { workspace = true, optional = true } +parquet = { workspace = true, optional = true } + +# The interop tests check that iceberg-rust reads what this crate writes and the other way round. +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +iceberg.workspace = true +tokio.workspace = true +parquet = { workspace = true, features = ["arrow"] } +arrow-array = "58" +bytes = "1" +futures = "0.3" + +[package.metadata.docs.rs] +all-features = true + +[lints] +workspace = true diff --git a/crates/lakebed-iceberg/README.md b/crates/lakebed-iceberg/README.md new file mode 100644 index 0000000..cb493a8 --- /dev/null +++ b/crates/lakebed-iceberg/README.md @@ -0,0 +1,15 @@ +# lakebed-iceberg + +[Apache Iceberg](https://iceberg.apache.org) tables read and written without I/O of their own, through a +[`lakebed_core::Host`](../lakebed-core), so they work in WebAssembly (browsers, Cloudflare Workers) as well as natively. + +| Feature | What it adds | +|---|---| +| (always) | Table metadata of format v1 to v3, schemas, partition specs, snapshots; manifest lists and manifests with data and delete files; values in Iceberg's single-value encoding; loading a table from its location or a metadata file | +| `write` | Format v2 manifests, manifest lists, and metadata updates; commits to file-system tables with `CommitOptions` and `CommitHooks` | +| `arrow` | The schema as an Arrow schema, with field ids | +| `parquet-stats` | A data file's Iceberg statistics (counts, bounds, split offsets) from its Parquet footer | + +The tests commit to one table with lakebed and iceberg-rust in turn, each reading what the other wrote. + +Apache License 2.0. diff --git a/crates/lakebed-iceberg/src/arrow.rs b/crates/lakebed-iceberg/src/arrow.rs new file mode 100644 index 0000000..1dd9ba2 --- /dev/null +++ b/crates/lakebed-iceberg/src/arrow.rs @@ -0,0 +1,103 @@ +//! Iceberg schemas as Arrow schemas (feature `arrow`), with each field's Iceberg id in its +//! metadata under [`FIELD_ID_KEY`], as Parquet writers and iceberg-rust's `schema_to_arrow_schema` +//! carry it. + +use std::collections::HashMap; +use std::sync::Arc; + +use arrow_schema::{DataType, Field, Fields, TimeUnit}; + +use crate::types::{NestedField, PrimitiveType, Schema, Type}; +use crate::{Error, Result}; + +/// The Arrow field metadata key carrying an Iceberg field id. +pub const FIELD_ID_KEY: &str = "PARQUET:field_id"; + +/// The time zone of `timestamptz` columns in Arrow. +pub const UTC: &str = "+00:00"; + +/// `schema` as an Arrow schema. +pub fn schema_to_arrow(schema: &Schema) -> Result { + Ok(arrow_schema::Schema::new(fields(&schema.fields)?)) +} + +fn fields(fields: &[NestedField]) -> Result> { + fields.iter().map(|f| field(&f.name, f.id, !f.required, &f.ty)).collect() +} + +fn field(name: &str, id: i32, nullable: bool, ty: &Type) -> Result { + let metadata = HashMap::from([(FIELD_ID_KEY.to_string(), id.to_string())]); + Ok(Field::new(name, data_type(ty)?, nullable).with_metadata(metadata)) +} + +/// The Arrow type of an Iceberg type. +pub fn data_type(ty: &Type) -> Result { + Ok(match ty { + Type::Primitive(p) => primitive(p), + Type::Struct(f) => DataType::Struct(Fields::from(fields(f)?)), + Type::List { element_id, element_required, element } => { + DataType::List(Arc::new(field("element", *element_id, !element_required, element)?)) + } + Type::Map { key_id, key, value_id, value_required, value } => { + let entries = Field::new( + "key_value", + DataType::Struct(Fields::from(vec![ + field("key", *key_id, false, key)?, + field("value", *value_id, !value_required, value)?, + ])), + false, + ); + DataType::Map(Arc::new(entries), false) + } + Type::Other(name) => return Err(Error::Unsupported(format!("{name} columns in Arrow"))), + }) +} + +fn primitive(p: &PrimitiveType) -> DataType { + match p { + PrimitiveType::Boolean => DataType::Boolean, + PrimitiveType::Int => DataType::Int32, + PrimitiveType::Long => DataType::Int64, + PrimitiveType::Float => DataType::Float32, + PrimitiveType::Double => DataType::Float64, + PrimitiveType::Decimal { precision, scale } => { + DataType::Decimal128(*precision, i8::try_from(*scale).unwrap_or(i8::MAX)) + } + PrimitiveType::Date => DataType::Date32, + PrimitiveType::Time => DataType::Time64(TimeUnit::Microsecond), + PrimitiveType::Timestamp => DataType::Timestamp(TimeUnit::Microsecond, None), + PrimitiveType::Timestamptz => DataType::Timestamp(TimeUnit::Microsecond, Some(UTC.into())), + PrimitiveType::TimestampNs => DataType::Timestamp(TimeUnit::Nanosecond, None), + PrimitiveType::TimestamptzNs => DataType::Timestamp(TimeUnit::Nanosecond, Some(UTC.into())), + PrimitiveType::String => DataType::Utf8, + PrimitiveType::Uuid => DataType::FixedSizeBinary(16), + PrimitiveType::Fixed(n) => DataType::FixedSizeBinary(i32::try_from(*n).unwrap_or(i32::MAX)), + PrimitiveType::Binary => DataType::LargeBinary, + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn fields_carry_ids() { + let schema = Schema::from_json(&json!({"fields": [ + {"id": 1, "name": "time", "required": true, "type": "timestamptz"}, + {"id": 2, "name": "tags", "required": false, "type": + {"type": "map", "key-id": 3, "key": "string", "value-id": 4, "value-required": true, "value": "decimal(10,2)"}} + ]})) + .unwrap(); + let arrow = schema_to_arrow(&schema).unwrap(); + let time = arrow.field(0); + assert_eq!(time.data_type(), &DataType::Timestamp(TimeUnit::Microsecond, Some(UTC.into()))); + assert!(!time.is_nullable()); + assert_eq!(time.metadata()[FIELD_ID_KEY], "1"); + let DataType::Map(entries, _) = arrow.field(1).data_type() else { panic!("not a map") }; + let DataType::Struct(kv) = entries.data_type() else { panic!("not a struct") }; + assert_eq!(kv[1].metadata()[FIELD_ID_KEY], "4"); + assert_eq!(kv[1].data_type(), &DataType::Decimal128(10, 2)); + } +} diff --git a/crates/lakebed-iceberg/src/avro.rs b/crates/lakebed-iceberg/src/avro.rs new file mode 100644 index 0000000..08544c5 --- /dev/null +++ b/crates/lakebed-iceberg/src/avro.rs @@ -0,0 +1,86 @@ +//! Reading Avro records by field name, as manifests and manifest lists are read: optional fields +//! are unions with null, and fields a format version lacks are absent. + +use std::collections::HashMap; + +use apache_avro::Reader; +use apache_avro::types::Value; + +use crate::{Error, Result, corrupt}; + +pub(crate) fn error(e: impl std::fmt::Display) -> Error { + corrupt(format!("avro: {e}")) +} + +/// An Avro container file's header metadata. +pub(crate) type Header = HashMap>; + +/// The records of an Avro container file, and its header metadata. +pub(crate) fn records(bytes: &[u8]) -> Result<(Vec, Header)> { + let reader = Reader::new(bytes).map_err(error)?; + let metadata = reader.user_metadata().clone(); + let records = reader.map(|v| v.map_err(error)).collect::>()?; + Ok((records, metadata)) +} + +/// The value inside an optional field's union. +pub(crate) fn strip_union(v: &Value) -> &Value { + match v { + Value::Union(_, inner) => strip_union(inner), + other => other, + } +} + +/// Field `name` of `record`, or `None` if it is absent or null. +pub(crate) fn field<'a>(record: &'a Value, name: &str) -> Option<&'a Value> { + match strip_union(record) { + Value::Record(fields) => { + fields.iter().find(|(n, _)| n == name).map(|(_, v)| strip_union(v)).filter(|v| **v != Value::Null) + } + _ => None, + } +} + +/// The first of `names` that `record` has (fields renamed between format versions). +pub(crate) fn field_of<'a>(record: &'a Value, names: &[&str]) -> Option<&'a Value> { + names.iter().find_map(|n| field(record, n)) +} + +pub(crate) fn long(v: Option<&Value>) -> Option { + match v? { + Value::Long(n) => Some(*n), + Value::Int(n) | Value::Date(n) => Some(i64::from(*n)), + _ => None, + } +} + +pub(crate) fn int(v: Option<&Value>) -> Option { + match v? { + Value::Int(n) | Value::Date(n) => Some(*n), + Value::Long(n) => i32::try_from(*n).ok(), + _ => None, + } +} + +pub(crate) fn string(v: Option<&Value>) -> Option<&str> { + match v? { + Value::String(s) => Some(s), + _ => None, + } +} + +pub(crate) fn required(v: Option, what: &str) -> Result { + v.ok_or_else(|| corrupt(format!("manifest without {what}"))) +} + +/// An optional Avro value. +#[cfg(feature = "write")] +pub(crate) fn some(v: Value) -> Value { + Value::Union(1, Box::new(v)) +} + +/// A null optional Avro value. +#[cfg(feature = "write")] +pub(crate) fn null() -> Value { + Value::Union(0, Box::new(Value::Null)) +} diff --git a/crates/lakebed-iceberg/src/commit.rs b/crates/lakebed-iceberg/src/commit.rs new file mode 100644 index 0000000..89c8c40 --- /dev/null +++ b/crates/lakebed-iceberg/src/commit.rs @@ -0,0 +1,656 @@ +//! Commits to file-system tables (feature `write`): every file of a snapshot (manifests, manifest +//! list, and whatever the caller adds) is uploaded, then `v.metadata.json` is written only if +//! absent. When another writer got there first, the commit reloads the table and tries again on top +//! of it. `version-hint.text` is written after the commit; readers don't rely on it. +//! +//! Applications extend a commit through [`CommitHooks`]: files of their own written with the +//! snapshot, summary properties, a check that skips a commit already made, and how to reload. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::fmt; +use std::future::Future; +use std::hash::BuildHasher; +use std::sync::Arc; + +use lakebed_core::{Host, uuid_v4}; + +use crate::manifest::{ + DataFile, Entry, ManifestContext, ManifestFile, ManifestWrite, Status, manifest_file, write_manifest, + write_manifest_list, +}; +use crate::metadata::NewSnapshot; +use crate::table::{Table, hint_uri, metadata_uri}; +use crate::{Error, Result, summary}; + +/// What a commit changes. +#[derive(Clone, Debug)] +pub enum Change { + /// Add data files. + Append { + /// The files. + files: Vec, + }, + /// Remove files and add files with the same rows (compaction). Each added file keeps the data + /// sequence number in `sequence_numbers`, so its rows keep their order against other files'. + Replace { + /// Files removed. + removed: Vec, + /// Files added. + added: Vec, + /// Data sequence number of each added file. + sequence_numbers: Vec, + }, + /// Remove files (retention). + Delete { + /// Files removed. + removed: Vec, + }, +} + +impl Change { + /// Files the change adds. + pub fn added(&self) -> &[DataFile] { + match self { + Self::Append { files } => files, + Self::Replace { added, .. } => added, + Self::Delete { .. } => &[], + } + } + + /// Files the change removes. + pub fn removed(&self) -> &[Entry] { + match self { + Self::Append { .. } => &[], + Self::Replace { removed, .. } | Self::Delete { removed } => removed, + } + } + + fn data_sequence(&self, i: usize) -> Option { + match self { + Self::Replace { sequence_numbers, .. } => sequence_numbers.get(i).copied(), + _ => None, + } + } + + /// The snapshot operation: `append`, `replace` or `delete`. + pub fn operation(&self) -> &'static str { + match self { + Self::Append { .. } => "append", + Self::Replace { .. } => "replace", + Self::Delete { .. } => "delete", + } + } +} + +/// How a commit behaves. The defaults are the ones Timelake's engine uses. +#[derive(Clone, Debug)] +pub struct CommitOptions { + /// Attempts before a commit that keeps losing the race gives up with [`Error::Conflict`]. + pub max_attempts: u64, + /// The wait before attempt `n` is a random number of milliseconds in this range, times `n`. + pub backoff_ms: (u64, u64), + /// An append merges the table's manifests into one once there would be more than this many. + pub merge_manifests_at: usize, + /// Write `version-hint.text` after each commit, for readers of stores that can't list. + pub write_version_hint: bool, +} + +impl Default for CommitOptions { + fn default() -> Self { + Self { max_attempts: 10, backoff_ms: (5, 50), merge_manifests_at: 8, write_version_hint: true } + } +} + +/// How a commit went. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CommitInfo { + /// The new snapshot (when skipped, the current one). + pub snapshot_id: i64, + /// The table version after the commit. + pub version: u64, + /// Attempts it took. + pub attempts: u64, + /// [`CommitHooks::skip`] found the change already committed; nothing was written. + pub skipped: bool, +} + +/// What an application adds to a snapshot. +#[derive(Clone, Debug, Default)] +pub struct Extra { + /// Files to upload with the snapshot's manifests: `(uri, bytes)`. A commit that loses the race + /// deletes them. + pub files: Vec<(String, Vec)>, + /// Summary properties. They override the caller's; Iceberg's counters override them. + pub summary: HashMap, +} + +/// An application's part in a commit. Every method has a default that does nothing extra. +pub trait CommitHooks { + /// The error the hooks (and so the commit) return. + type Error: From + fmt::Debug; + + /// The newest version of the table, after another writer won the race. + fn reload(&mut self, host: &H, location: &str) -> impl Future> { + async move { Ok(Table::load(host, location).await?) } + } + + /// Whether `base` already holds this change (an idempotent append retried), so nothing is + /// committed. Checked before every attempt. + fn skip(&mut self, _base: &Table) -> bool { + false + } + + /// Files and summary properties to add to snapshot `snapshot_id` on top of `base`. Called once + /// per attempt. + fn extra( + &mut self, + _host: &H, + _base: &Table, + _snapshot_id: i64, + ) -> impl Future> { + async { Ok(Extra::default()) } + } +} + +/// Hooks that add nothing. +#[derive(Clone, Copy, Debug, Default)] +pub struct NoHooks; + +impl CommitHooks for NoHooks { + type Error = Error; +} + +/// Commits `change` on top of `base` (or the newest version, after a lost race), with `props` in +/// the snapshot summary. The returned table knows its files when `base` did. +pub async fn commit, S: BuildHasher>( + host: &H, + base: Table, + change: &Change, + props: &HashMap, + options: &CommitOptions, + hooks: &mut K, +) -> Result<(Table, CommitInfo), K::Error> { + base.metadata.check_writable()?; + let mut base = base; + for attempt in 1..=options.max_attempts { + if attempt > 1 { + let (lo, hi) = options.backoff_ms; + host.sleep((lo + host.random() % hi.saturating_sub(lo).max(1)) * attempt).await; + base = hooks.reload(host, &base.location).await?; + } + if hooks.skip(&base) { + let info = CommitInfo { + snapshot_id: base.metadata.current_snapshot_id().unwrap_or_default(), + version: base.version, + attempts: attempt, + skipped: true, + }; + return Ok((base, info)); + } + if let Some((table, snapshot_id)) = attempt_commit(host, &base, change, props, options, hooks).await? { + let info = CommitInfo { snapshot_id, version: table.version, attempts: attempt, skipped: false }; + return Ok((table, info)); + } + } + Err(Error::Conflict(format!("{}: gave up after {} attempts", base.location, options.max_attempts)).into()) +} + +/// Writes table metadata `bytes` at `uri` unless a version is there; `false` if another writer's +/// is. +/// +/// A conditional PUT that reports failure may have landed all the same: a retry after a first try +/// that landed gets "already exists", and a timeout says nothing. The version is then read back: +/// our own metadata there is the commit, another writer's means it lost. If that can't be told, +/// the error is [`Error::CommitUnknown`], and callers must not delete what the metadata references. +pub async fn put_metadata(host: &H, uri: &str, bytes: Vec) -> Result { + let unknown = |e: &dyn fmt::Display| Error::CommitUnknown(format!("{uri}: {e}")); + match host.put_if_absent(uri, bytes.clone()).await { + Ok(true) => Ok(true), + // Ours (an earlier try landed) or another writer's. Absent now, it was never ours: our + // version would still be there. + Ok(false) => match host.get(uri).await { + Ok(found) => Ok(found.is_some_and(|found| found == bytes)), + Err(e) => Err(unknown(&e)), + }, + // Absent now, the PUT may still land. + Err(e) => match host.get(uri).await { + Ok(Some(found)) => Ok(found == bytes), + Ok(None) | Err(_) => Err(unknown(&e)), + }, + } +} + +/// A snapshot id not in the table yet. +fn new_snapshot_id(host: &impl Host, base: &Table) -> i64 { + loop { + let id = i64::try_from(host.random() >> 1).unwrap_or(1).max(1); + if !base.metadata.has_snapshot(id) { + return id; + } + } +} + +/// One attempt: writes the snapshot's files and publishes it. `None` if another writer committed +/// this version first (this attempt's files are deleted). With an unknown outcome, nothing is +/// deleted. +async fn attempt_commit, S: BuildHasher>( + host: &H, + base: &Table, + change: &Change, + props: &HashMap, + options: &CommitOptions, + hooks: &mut K, +) -> Result, K::Error> { + let snapshot_id = new_snapshot_id(host, base); + let sequence_number = base.metadata.next_sequence_number(); + let mut uploads: Vec<(String, Vec)> = vec![]; + let mut unknown = false; + let result = async { + let manifests = write_manifests(host, base, change, snapshot_id, options, &mut uploads).await?; + let extra = hooks.extra(host, base, snapshot_id).await?; + uploads.extend(extra.files); + let list_uri = format!("{}/metadata/snap-{snapshot_id}-{}.avro", base.location, uuid_v4(host)); + let parent = base.metadata.current_snapshot_id(); + let list = write_manifest_list(&manifests, snapshot_id, parent, sequence_number)?; + uploads.push((list_uri.clone(), list)); + let puts = uploads.iter().map(|(uri, bytes)| host.put(uri, bytes.clone())); + futures_util::future::try_join_all(puts).await?; + + let mut summary: HashMap = props.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); + summary.extend(extra.summary); + summary.extend(counters(change, &base.metadata.summary())); + // Snapshot and metadata logs must stay in time order, so a writer whose clock is behind the + // last one's dates its commit at the table's last update. + let now = host.now_ms().max(base.metadata.last_updated_ms()); + let snapshot = NewSnapshot { + snapshot_id, + sequence_number, + timestamp_ms: now, + manifest_list: list_uri, + operation: change.operation().into(), + summary, + }; + let metadata = base.metadata.with_snapshot(&snapshot, &base.metadata_file, now)?; + let version = base.version + 1; + let ours = put_metadata(host, &metadata_uri(&base.location, version), metadata.to_json().into_bytes()) + .await + .inspect_err(|e| unknown = matches!(e, Error::CommitUnknown(_)))?; + Ok::<_, K::Error>(ours.then_some((metadata, version))) + } + .await; + let (metadata, version) = match result { + Ok(Some(committed)) => committed, + Ok(None) => { + delete_uploads(host, uploads).await; + return Ok(None); + } + Err(e) => { + if !unknown { + delete_uploads(host, uploads).await; + } + return Err(e); + } + }; + + let files = base.files.as_ref().map(|files| { + let removed: HashSet<&str> = change.removed().iter().map(|e| e.file.path.as_str()).collect(); + let mut next: Vec = change + .added() + .iter() + .enumerate() + .map(|(i, f)| Entry { + status: Status::Added, + snapshot_id, + sequence_number: change.data_sequence(i).unwrap_or(sequence_number), + file_sequence_number: sequence_number, + file: Arc::new(f.clone()), + }) + .collect(); + next.extend(files.iter().filter(|e| !removed.contains(e.file.path.as_str())).cloned()); + next + }); + let table = Table { files, ..Table::new(base.location.clone(), version, metadata) }; + if options.write_version_hint { + let _ = host.put(&hint_uri(&table.location), version.to_string().into_bytes()).await; + } + Ok(Some((table, snapshot_id))) +} + +async fn delete_uploads(host: &impl Host, uploads: Vec<(String, Vec)>) { + let written: Vec = uploads.into_iter().map(|(uri, _)| uri).collect(); + let _ = host.delete(&written).await; +} + +/// Writes a manifest of the added files, carries the base snapshot's manifests over, and rewrites +/// the ones holding removed files (or all of them once there are too many). +async fn write_manifests( + host: &H, + base: &Table, + change: &Change, + snapshot_id: i64, + options: &CommitOptions, + uploads: &mut Vec<(String, Vec)>, +) -> Result> { + let metadata = &base.metadata; + let schema_json = metadata.schema_json()?.to_string(); + let (spec_id, spec_fields) = metadata.partition_spec_json()?; + let spec_fields_json = spec_fields.to_string(); + let partition_type = metadata.partition_type()?; + let ctx = ManifestContext { + schema_json: &schema_json, + schema_id: metadata.schema_id(), + spec_fields_json: &spec_fields_json, + spec_id, + partition_type: &partition_type, + }; + let mut write = |kind: &str, entries: &[ManifestWrite]| -> Result { + let uri = format!("{}/metadata/{}-{kind}-m.avro", base.location, uuid_v4(host)); + let bytes = write_manifest(&ctx, entries)?; + let mf = manifest_file(&uri, &bytes, spec_id, snapshot_id, entries, &partition_type); + uploads.push((uri, bytes)); + Ok(mf) + }; + let mut new = vec![]; + let added = change.added(); + if !added.is_empty() { + let entries: Vec = added + .iter() + .enumerate() + .map(|(i, f)| ManifestWrite { + status: Status::Added, + snapshot_id: Some(snapshot_id), + sequence_number: change.data_sequence(i), + file_sequence_number: None, + file: Arc::new(f.clone()), + }) + .collect(); + new.push(write("added", &entries)?); + } + + let current = base.manifests(host).await?; + let removed: HashSet<&str> = change.removed().iter().map(|e| e.file.path.as_str()).collect(); + let merge = removed.is_empty() && current.len() + new.len() > options.merge_manifests_at; + if removed.is_empty() && !merge { + new.extend(current); + return Ok(new); + } + let loaded = futures_util::future::try_join_all(current.iter().map(|m| Table::manifest(host, m))).await?; + let (mut kept, mut deleted, mut carried) = (vec![], vec![], vec![]); + for (manifest, entries) in current.into_iter().zip(loaded) { + let live: Vec<&Entry> = entries.iter().filter(|e| e.status != Status::Deleted).collect(); + let touched = live.iter().any(|e| removed.contains(e.file.path.as_str())); + // Delete manifests and other specs' manifests are carried as they are. + let mergeable = manifest.content == crate::ManifestContent::Data && manifest.partition_spec_id == spec_id; + if !mergeable || (!merge && !touched) { + if touched { + return Err(Error::Unsupported(format!( + "{}: removing files of an older partition spec", + base.location + ))); + } + carried.push(manifest); + continue; + } + for e in live { + let (status, snapshot) = if removed.contains(e.file.path.as_str()) { + (Status::Deleted, snapshot_id) + } else { + (Status::Existing, e.snapshot_id) + }; + let w = ManifestWrite { + status, + snapshot_id: Some(snapshot), + sequence_number: Some(e.sequence_number), + file_sequence_number: Some(e.file_sequence_number), + file: Arc::clone(&e.file), + }; + if status == Status::Deleted { deleted.push(w) } else { kept.push(w) } + } + } + if deleted.len() != removed.len() { + return Err(Error::Conflict(format!( + "{}: {} of {} files to remove are no longer in the table (concurrent maintenance?)", + base.location, + removed.len() - deleted.len(), + removed.len() + ))); + } + if !kept.is_empty() { + new.push(write("merged", &kept)?); + } + if !deleted.is_empty() { + new.push(write("deleted", &deleted)?); + } + new.extend(carried); + Ok(new) +} + +/// Iceberg's standard snapshot summary counters after `change` on a table whose current summary is +/// `base`. Delete-file totals carry over: commits here don't add or remove delete files. +pub fn counters(change: &Change, base: &HashMap) -> BTreeMap { + let total = |k: &str| base.get(k).and_then(|v| v.parse::().ok()).unwrap_or(0); + let (added, removed) = (change.added(), change.removed()); + let added_rows: u64 = added.iter().map(|f| f.record_count).sum(); + let added_bytes: u64 = added.iter().map(|f| f.file_size).sum(); + let removed_rows: u64 = removed.iter().map(|e| e.file.record_count).sum(); + let removed_bytes: u64 = removed.iter().map(|e| e.file.file_size).sum(); + let (added_files, removed_files) = (added.len() as u64, removed.len() as u64); + let mut out = BTreeMap::new(); + for (k, v) in [ + (summary::ADDED_DATA_FILES, added_files), + (summary::ADDED_RECORDS, added_rows), + (summary::ADDED_FILES_SIZE, added_bytes), + (summary::DELETED_DATA_FILES, removed_files), + (summary::DELETED_RECORDS, removed_rows), + (summary::REMOVED_FILES_SIZE, removed_bytes), + ] { + if v != 0 { + out.insert(k.to_string(), v.to_string()); + } + } + for (k, v) in [ + (summary::TOTAL_DATA_FILES, (total(summary::TOTAL_DATA_FILES) + added_files).saturating_sub(removed_files)), + (summary::TOTAL_RECORDS, (total(summary::TOTAL_RECORDS) + added_rows).saturating_sub(removed_rows)), + (summary::TOTAL_FILES_SIZE, (total(summary::TOTAL_FILES_SIZE) + added_bytes).saturating_sub(removed_bytes)), + (summary::TOTAL_DELETE_FILES, total(summary::TOTAL_DELETE_FILES)), + (summary::TOTAL_POSITION_DELETES, total(summary::TOTAL_POSITION_DELETES)), + (summary::TOTAL_EQUALITY_DELETES, total(summary::TOTAL_EQUALITY_DELETES)), + ] { + out.insert(k.to_string(), v.to_string()); + } + out +} + +#[cfg(test)] +mod tests { + use lakebed_core::MemoryHost; + use serde_json::json; + + use super::*; + use crate::{Datum, Metadata, block_on}; + + const LOCATION: &str = "memory://wh/t"; + + fn create(host: &MemoryHost, format_version: u8) -> Table { + let metadata = json!({ + "format-version": format_version, "table-uuid": "u", "location": LOCATION, "last-sequence-number": 0, + "last-updated-ms": 0, "last-column-id": 2, "current-schema-id": 0, + "schemas": [{"schema-id": 0, "type": "struct", "fields": [ + {"id": 1, "name": "time", "required": true, "type": "timestamptz"}, + {"id": 2, "name": "device", "required": true, "type": "string"}]}], + "default-spec-id": 0, "last-partition-id": 1000, + "partition-specs": [{"spec-id": 0, "fields": [ + {"source-id": 1, "field-id": 1000, "name": "time_day", "transform": "day"}]}], + "default-sort-order-id": 0, "sort-orders": [{"order-id": 0, "fields": []}], + "properties": {}, "current-snapshot-id": -1, "snapshots": [], "snapshot-log": [], "metadata-log": [] + }); + let table = Table::new(LOCATION, 1, Metadata::from_json(metadata).unwrap()); + block_on(host.put(&table.metadata_file, table.metadata.to_json().into_bytes())).unwrap(); + Table { files: Some(vec![]), ..table } + } + + fn file(name: &str, day: i32, rows: u64) -> DataFile { + DataFile { + path: format!("{LOCATION}/data/{name}.parquet"), + partition: vec![Some(Datum::Date(day))], + record_count: rows, + file_size: rows * 10, + ..DataFile::default() + } + } + + fn append(files: Vec) -> Change { + Change::Append { files } + } + + fn try_run>( + host: &MemoryHost, + base: Table, + change: &Change, + hooks: &mut K, + ) -> Result<(Table, CommitInfo), K::Error> { + block_on(commit(host, base, change, &HashMap::new(), &CommitOptions::default(), hooks)) + } + + fn run>( + host: &MemoryHost, + base: Table, + change: &Change, + hooks: &mut K, + ) -> (Table, CommitInfo) { + try_run(host, base, change, hooks).unwrap() + } + + fn files_named(t: &Table, name: &str) -> Vec { + t.files.iter().flatten().filter(|e| e.file.path.ends_with(name)).cloned().collect() + } + + /// The live files of the newest version, read back from the store: (name, data sequence + /// number, file sequence number). + fn live(host: &MemoryHost) -> Vec<(String, i64, i64)> { + let mut t = block_on(Table::load(host, LOCATION)).unwrap(); + let mut files: Vec<_> = block_on(t.live_files(host)) + .unwrap() + .iter() + .map(|e| { + ( + e.file.path.rsplit('/').next().unwrap_or_default().to_string(), + e.sequence_number, + e.file_sequence_number, + ) + }) + .collect(); + files.sort(); + files + } + + #[test] + fn append_replace_and_delete() { + let host = MemoryHost::new(1_000); + let t = create(&host, 2); + let (t, info) = run(&host, t, &append(vec![file("a", 1, 10), file("b", 2, 20)]), &mut NoHooks); + assert_eq!((info.version, info.attempts, info.skipped), (2, 1, false)); + assert_eq!(live(&host), [("a.parquet".into(), 1, 1), ("b.parquet".into(), 1, 1)]); + + // Compaction keeps the data sequence number of what it replaces. + let change = Change::Replace { + removed: files_named(&t, "a.parquet"), + added: vec![file("c", 1, 10)], + sequence_numbers: vec![1], + }; + let (t, _) = run(&host, t, &change, &mut NoHooks); + assert_eq!(live(&host), [("b.parquet".into(), 1, 1), ("c.parquet".into(), 1, 2)]); + assert_eq!(t.files.as_ref().map(Vec::len), Some(2), "the commit derives the files"); + + let (t, _) = run(&host, t.clone(), &Change::Delete { removed: files_named(&t, "b.parquet") }, &mut NoHooks); + assert_eq!(live(&host), [("c.parquet".into(), 1, 2)]); + let summary = t.metadata.summary(); + assert_eq!((summary["operation"].as_str(), summary[crate::summary::TOTAL_RECORDS].as_str()), ("delete", "10")); + assert_eq!(summary[crate::summary::DELETED_DATA_FILES], "1"); + assert_eq!(t.metadata.metadata_log_len(), 3); + assert_eq!(block_on(host.get(&hint_uri(LOCATION))).unwrap().as_deref(), Some(&b"4"[..])); + } + + #[test] + fn a_lost_race_retries_on_the_newer_version() { + let host = MemoryHost::new(1_000); + let stale = create(&host, 2); + let (_, first) = run(&host, stale.clone(), &append(vec![file("a", 1, 10)]), &mut NoHooks); + let (_, second) = run(&host, stale, &append(vec![file("b", 1, 10)]), &mut NoHooks); + assert_eq!((first.version, second.version, second.attempts), (2, 3, 2)); + assert_eq!(live(&host).len(), 2); + // The lost attempt's manifests and list were deleted: every list is a snapshot's. + let lists = host.uris().iter().filter(|u| u.contains("/snap-")).count(); + assert_eq!(lists, 2); + } + + #[test] + fn removing_files_that_are_gone_is_a_conflict() { + let host = MemoryHost::new(1_000); + let (t, _) = run(&host, create(&host, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); + let removed = t.files.clone().unwrap_or_default(); + let (t, _) = run(&host, t, &Change::Delete { removed: removed.clone() }, &mut NoHooks); + let err = try_run(&host, t, &Change::Delete { removed }, &mut NoHooks); + assert!(matches!(err, Err(Error::Conflict(_))), "{err:?}"); + } + + /// Hooks that skip a batch already committed, and add a file and summary properties. + struct Batch(&'static str); + + impl CommitHooks for Batch { + type Error = Error; + + fn skip(&mut self, base: &Table) -> bool { + base.metadata.ancestors().iter().any(|s| s.summary.get("batch").map(String::as_str) == Some(self.0)) + } + + fn extra(&mut self, _: &MemoryHost, base: &Table, snapshot_id: i64) -> impl Future> { + let uri = format!("{}/metadata/extra-{snapshot_id}", base.location); + let summary = + HashMap::from([("batch".to_string(), self.0.to_string()), ("extra".to_string(), uri.clone())]); + std::future::ready(Ok(Extra { files: vec![(uri, b"x".to_vec())], summary })) + } + } + + #[test] + fn hooks_skip_and_extend_commits() { + let host = MemoryHost::new(1_000); + let (t, info) = run(&host, create(&host, 2), &append(vec![file("a", 1, 10)]), &mut Batch("b1")); + let extra = t.metadata.summary()["extra"].clone(); + assert!(block_on(host.exists(&extra)).unwrap()); + let (t, again) = run(&host, t, &append(vec![file("a", 1, 10)]), &mut Batch("b1")); + assert!(again.skipped); + assert_eq!((again.version, again.snapshot_id), (info.version, info.snapshot_id)); + assert_eq!(t.version, 2); + } + + #[test] + fn many_manifests_merge() { + let host = MemoryHost::new(1_000); + let mut t = create(&host, 2); + for i in 0..9 { + t = run(&host, t, &append(vec![file(&format!("f{i}"), i, 1)]), &mut NoHooks).0; + } + assert_eq!(block_on(t.manifests(&host)).unwrap().len(), 2, "eight merged into one, plus the new one"); + assert_eq!(live(&host).len(), 9); + } + + #[test] + fn a_clock_behind_the_table_keeps_the_logs_in_order() { + let host = MemoryHost::new(5_000); + let (t, _) = run(&host, create(&host, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); + let late = MemoryHost::new(1_000); + for uri in host.uris() { + block_on(late.put(&uri, block_on(host.get(&uri)).unwrap().unwrap())).unwrap(); + } + let (t, _) = run(&late, t, &append(vec![file("b", 1, 10)]), &mut NoHooks); + assert_eq!((t.metadata.current_snapshot().unwrap().timestamp_ms, t.metadata.last_updated_ms()), (5_000, 5_000)); + } + + #[test] + fn only_v2_tables_are_written() { + let host = MemoryHost::new(1_000); + let err = try_run(&host, create(&host, 1), &append(vec![]), &mut NoHooks); + assert!(matches!(err, Err(Error::Unsupported(_)))); + } +} diff --git a/crates/lakebed-iceberg/src/datum.rs b/crates/lakebed-iceberg/src/datum.rs new file mode 100644 index 0000000..975bf55 --- /dev/null +++ b/crates/lakebed-iceberg/src/datum.rs @@ -0,0 +1,287 @@ +//! Typed values, and Iceberg's single-value binary serialization, which manifests use for column +//! bounds and manifest lists for partition bounds. +//! + +use std::cmp::Ordering; + +use crate::types::PrimitiveType; +use crate::{Result, corrupt}; + +/// A value of a primitive type. Dates, times and timestamps keep Iceberg's integer encoding. +#[derive(Clone, Debug, PartialEq)] +pub enum Datum { + /// `boolean`. + Boolean(bool), + /// `int`. + Int(i32), + /// `long`. + Long(i64), + /// `float`. + Float(f32), + /// `double`. + Double(f64), + /// `decimal(P,S)`: the unscaled value. + Decimal { + /// The value times 10^`scale`. + unscaled: i128, + /// Digits in all. + precision: u8, + /// Digits after the point. + scale: u8, + }, + /// `date`: days since 1970-01-01. + Date(i32), + /// `time`: microseconds since midnight. + Time(i64), + /// `timestamp`: microseconds since the epoch. + Timestamp(i64), + /// `timestamptz`: microseconds since the epoch, UTC. + Timestamptz(i64), + /// `timestamp_ns`: nanoseconds since the epoch. + TimestampNs(i64), + /// `timestamptz_ns`: nanoseconds since the epoch, UTC. + TimestamptzNs(i64), + /// `string`. + String(String), + /// `uuid`, big-endian. + Uuid([u8; 16]), + /// `fixed[L]`. + Fixed(Vec), + /// `binary`. + Binary(Vec), +} + +fn sized(bytes: &[u8], ty: &PrimitiveType) -> Result<[u8; N]> { + bytes.try_into().map_err(|_| corrupt(format!("a {ty} value of {} bytes", bytes.len()))) +} + +impl Datum { + /// Decodes Iceberg's single-value serialization of a `ty` value. A `long` or `double` stored in + /// 4 bytes was written before the column was promoted from `int` or `float`, and is widened. + pub fn from_bytes(ty: &PrimitiveType, bytes: &[u8]) -> Result { + Ok(match ty { + PrimitiveType::Boolean => match bytes { + [b] => Self::Boolean(*b != 0), + _ => return Err(corrupt(format!("a boolean value of {} bytes", bytes.len()))), + }, + PrimitiveType::Int => Self::Int(i32::from_le_bytes(sized(bytes, ty)?)), + PrimitiveType::Long if bytes.len() == 4 => Self::Long(i32::from_le_bytes(sized(bytes, ty)?).into()), + PrimitiveType::Long => Self::Long(i64::from_le_bytes(sized(bytes, ty)?)), + PrimitiveType::Float => Self::Float(f32::from_le_bytes(sized(bytes, ty)?)), + PrimitiveType::Double if bytes.len() == 4 => Self::Double(f32::from_le_bytes(sized(bytes, ty)?).into()), + PrimitiveType::Double => Self::Double(f64::from_le_bytes(sized(bytes, ty)?)), + PrimitiveType::Decimal { precision, scale } => { + if bytes.is_empty() || bytes.len() > 16 { + return Err(corrupt(format!("a decimal value of {} bytes", bytes.len()))); + } + Self::Decimal { unscaled: from_be(bytes), precision: *precision, scale: *scale } + } + PrimitiveType::Date => Self::Date(i32::from_le_bytes(sized(bytes, ty)?)), + PrimitiveType::Time => Self::Time(i64::from_le_bytes(sized(bytes, ty)?)), + PrimitiveType::Timestamp => Self::Timestamp(i64::from_le_bytes(sized(bytes, ty)?)), + PrimitiveType::Timestamptz => Self::Timestamptz(i64::from_le_bytes(sized(bytes, ty)?)), + PrimitiveType::TimestampNs => Self::TimestampNs(i64::from_le_bytes(sized(bytes, ty)?)), + PrimitiveType::TimestamptzNs => Self::TimestamptzNs(i64::from_le_bytes(sized(bytes, ty)?)), + PrimitiveType::String => { + Self::String(String::from_utf8(bytes.to_vec()).map_err(|_| corrupt("a string value that isn't UTF-8"))?) + } + PrimitiveType::Uuid => Self::Uuid(sized(bytes, ty)?), + PrimitiveType::Fixed(_) => Self::Fixed(bytes.to_vec()), + PrimitiveType::Binary => Self::Binary(bytes.to_vec()), + }) + } + + /// Iceberg's single-value serialization of this value. + pub fn to_bytes(&self) -> Vec { + match self { + Self::Boolean(b) => vec![u8::from(*b)], + Self::Int(v) | Self::Date(v) => v.to_le_bytes().to_vec(), + Self::Long(v) + | Self::Time(v) + | Self::Timestamp(v) + | Self::Timestamptz(v) + | Self::TimestampNs(v) + | Self::TimestamptzNs(v) => v.to_le_bytes().to_vec(), + Self::Float(v) => v.to_le_bytes().to_vec(), + Self::Double(v) => v.to_le_bytes().to_vec(), + Self::Decimal { unscaled, .. } => minimal_be(*unscaled), + Self::String(s) => s.as_bytes().to_vec(), + Self::Uuid(u) => u.to_vec(), + Self::Fixed(b) | Self::Binary(b) => b.clone(), + } + } + + /// The value's type. + pub fn primitive_type(&self) -> PrimitiveType { + match self { + Self::Boolean(_) => PrimitiveType::Boolean, + Self::Int(_) => PrimitiveType::Int, + Self::Long(_) => PrimitiveType::Long, + Self::Float(_) => PrimitiveType::Float, + Self::Double(_) => PrimitiveType::Double, + Self::Decimal { precision, scale, .. } => PrimitiveType::Decimal { precision: *precision, scale: *scale }, + Self::Date(_) => PrimitiveType::Date, + Self::Time(_) => PrimitiveType::Time, + Self::Timestamp(_) => PrimitiveType::Timestamp, + Self::Timestamptz(_) => PrimitiveType::Timestamptz, + Self::TimestampNs(_) => PrimitiveType::TimestampNs, + Self::TimestamptzNs(_) => PrimitiveType::TimestamptzNs, + Self::String(_) => PrimitiveType::String, + Self::Uuid(_) => PrimitiveType::Uuid, + Self::Fixed(b) => PrimitiveType::Fixed(b.len() as u64), + Self::Binary(_) => PrimitiveType::Binary, + } + } + + /// The value as an integer, for the integer-encoded types (`int`, `long`, dates, times, and + /// timestamps). + pub fn as_i64(&self) -> Option { + match self { + Self::Int(v) | Self::Date(v) => Some((*v).into()), + Self::Long(v) + | Self::Time(v) + | Self::Timestamp(v) + | Self::Timestamptz(v) + | Self::TimestampNs(v) + | Self::TimestamptzNs(v) => Some(*v), + _ => None, + } + } + + /// Whether this is a float or double NaN. + pub fn is_nan(&self) -> bool { + match self { + Self::Float(v) => v.is_nan(), + Self::Double(v) => v.is_nan(), + _ => false, + } + } +} + +/// Values of the same type compare by value (strings and bytes lexicographically, as Iceberg +/// orders them); values of different types don't compare. +impl PartialOrd for Datum { + fn partial_cmp(&self, other: &Self) -> Option { + match (self, other) { + (Self::Boolean(a), Self::Boolean(b)) => a.partial_cmp(b), + (Self::Int(a), Self::Int(b)) | (Self::Date(a), Self::Date(b)) => a.partial_cmp(b), + (Self::Long(a), Self::Long(b)) + | (Self::Time(a), Self::Time(b)) + | (Self::Timestamp(a), Self::Timestamp(b)) + | (Self::Timestamptz(a), Self::Timestamptz(b)) + | (Self::TimestampNs(a), Self::TimestampNs(b)) + | (Self::TimestamptzNs(a), Self::TimestamptzNs(b)) => a.partial_cmp(b), + (Self::Float(a), Self::Float(b)) => a.partial_cmp(b), + (Self::Double(a), Self::Double(b)) => a.partial_cmp(b), + (Self::Decimal { unscaled: a, scale: sa, .. }, Self::Decimal { unscaled: b, scale: sb, .. }) + if sa == sb => + { + a.partial_cmp(b) + } + (Self::String(a), Self::String(b)) => a.partial_cmp(b), + (Self::Uuid(a), Self::Uuid(b)) => a.partial_cmp(b), + (Self::Fixed(a), Self::Fixed(b)) | (Self::Binary(a), Self::Binary(b)) => a.partial_cmp(b), + _ => None, + } + } +} + +/// The shortest big-endian two's complement bytes of `v` (Iceberg's decimal encoding). +pub fn minimal_be(v: i128) -> Vec { + let bytes = v.to_be_bytes(); + let mut start = 0; + while start < bytes.len() - 1 { + let (b, next) = (bytes[start], bytes[start + 1]); + if (b == 0 && next & 0x80 == 0) || (b == 0xff && next & 0x80 != 0) { + start += 1; + } else { + break; + } + } + bytes[start..].to_vec() +} + +/// A big-endian two's complement value of at most 16 bytes, as an integer. +pub fn from_be(bytes: &[u8]) -> i128 { + let negative = bytes.first().is_some_and(|b| b & 0x80 != 0); + let mut v: i128 = if negative { -1 } else { 0 }; + for &b in &bytes[bytes.len().saturating_sub(16)..] { + v = (v << 8) | i128::from(b); + } + v +} + +#[cfg(test)] +mod tests { + use super::*; + + fn round_trip(d: &Datum) { + assert_eq!(&Datum::from_bytes(&d.primitive_type(), &d.to_bytes()).unwrap(), d); + } + + #[test] + fn values_round_trip() { + for d in [ + Datum::Boolean(true), + Datum::Int(-7), + Datum::Long(1 << 40), + Datum::Float(0.5), + Datum::Double(-1.25), + Datum::Decimal { unscaled: -12_345, precision: 10, scale: 2 }, + Datum::Date(-1), + Datum::Time(3_600_000_000), + Datum::Timestamptz(1_700_000_000_000_000), + Datum::TimestamptzNs(1_700_000_000_000_000_001), + Datum::String("truck_1".into()), + Datum::Uuid([7; 16]), + Datum::Fixed(vec![1, 2, 3]), + Datum::Binary(vec![]), + ] { + round_trip(&d); + } + } + + #[test] + fn encodings_follow_the_spec() { + assert_eq!(Datum::Int(7).to_bytes(), 7i32.to_le_bytes()); + assert_eq!(Datum::Boolean(false).to_bytes(), [0]); + assert_eq!(Datum::Decimal { unscaled: 128, precision: 5, scale: 0 }.to_bytes(), [0x00, 0x80]); + assert_eq!(Datum::String("ab".into()).to_bytes(), b"ab"); + } + + #[test] + fn promoted_columns_widen() { + assert_eq!(Datum::from_bytes(&PrimitiveType::Long, &(-3i32).to_le_bytes()).unwrap(), Datum::Long(-3)); + assert_eq!(Datum::from_bytes(&PrimitiveType::Double, &0.5f32.to_le_bytes()).unwrap(), Datum::Double(0.5)); + } + + #[test] + fn malformed_values_are_errors() { + assert!(Datum::from_bytes(&PrimitiveType::Long, &[1, 2, 3]).is_err()); + assert!(Datum::from_bytes(&PrimitiveType::Int, &[]).is_err()); + assert!(Datum::from_bytes(&PrimitiveType::Boolean, &[]).is_err()); + assert!(Datum::from_bytes(&PrimitiveType::String, &[0xff]).is_err()); + assert!(Datum::from_bytes(&PrimitiveType::Uuid, &[0; 15]).is_err()); + let decimal = PrimitiveType::Decimal { precision: 38, scale: 0 }; + assert!(Datum::from_bytes(&decimal, &[0; 17]).is_err()); + } + + #[test] + fn only_same_types_compare() { + assert!(Datum::Int(1) < Datum::Int(2)); + assert!(Datum::String("a".into()) < Datum::String("b".into())); + assert_eq!(Datum::Int(1).partial_cmp(&Datum::Long(2)), None); + assert_eq!(Datum::Double(f64::NAN).partial_cmp(&Datum::Double(1.0)), None); + } + + #[test] + fn decimal_bytes_are_minimal() { + assert_eq!(minimal_be(0), [0]); + assert_eq!(minimal_be(127), [0x7f]); + assert_eq!(minimal_be(128), [0x00, 0x80]); + assert_eq!(minimal_be(-1), [0xff]); + assert_eq!(minimal_be(-129), [0xff, 0x7f]); + assert_eq!(from_be(&minimal_be(-129)), -129); + assert_eq!(from_be(&minimal_be(1 << 40)), 1 << 40); + } +} diff --git a/crates/lakebed-iceberg/src/lib.rs b/crates/lakebed-iceberg/src/lib.rs new file mode 100644 index 0000000..446b6fb --- /dev/null +++ b/crates/lakebed-iceberg/src/lib.rs @@ -0,0 +1,89 @@ +//! Apache Iceberg table metadata, manifests and commits, with no I/O of its own: everything is read +//! and written through a [`Host`](lakebed_core::Host) the application implements, so the crate +//! builds for `wasm32-unknown-unknown` (browsers, Cloudflare Workers) as well as native targets. +//! +//! - Reading (always): table metadata of format versions 1 to 3 ([`Metadata`]), schemas and +//! partition specs, manifest lists and manifests with data and delete files ([`read_manifest`]), +//! column and partition values ([`Datum`]), and loading a table from its location ([`Table`]). +//! - `write`: format v2 manifests and manifest lists, metadata updates, and the commit protocol of +//! file-system tables (`commit`). +//! - `arrow`: the schema as an Arrow schema with field ids. +//! - `parquet-stats`: a data file's Iceberg statistics from its Parquet footer. +//! +//! ``` +//! use lakebed_iceberg::{Datum, PrimitiveType}; +//! +//! let bound = Datum::from_bytes(&PrimitiveType::Long, &42i64.to_le_bytes())?; +//! assert_eq!(bound, Datum::Long(42)); +//! # Ok::<(), lakebed_core::Error>(()) +//! ``` + +#[cfg(feature = "arrow")] +pub mod arrow; +mod avro; +#[cfg(feature = "write")] +pub mod commit; +mod datum; +pub mod manifest; +pub mod metadata; +mod partition; +#[cfg(feature = "parquet-stats")] +pub mod stats; +pub mod table; +mod types; + +#[cfg(feature = "write")] +pub use commit::{Change, CommitHooks, CommitInfo, CommitOptions, Extra, NoHooks, commit}; +pub use datum::{Datum, from_be, minimal_be}; +pub use lakebed_core::{Error, Result}; +pub use manifest::{ + Content, DataFile, Entry, FieldSummary, ManifestContent, ManifestFile, Status, read_manifest, read_manifest_list, +}; +pub use metadata::{Metadata, Snapshot}; +pub use partition::{PartitionField, PartitionFieldType, PartitionSpec, Transform}; +pub use table::Table; +pub use types::{NestedField, PrimitiveType, Schema, Type}; + +/// Snapshot summary keys of Iceberg's standard counters. +pub mod summary { + /// Data files added by the snapshot. + pub const ADDED_DATA_FILES: &str = "added-data-files"; + /// Rows added by the snapshot. + pub const ADDED_RECORDS: &str = "added-records"; + /// Bytes added by the snapshot. + pub const ADDED_FILES_SIZE: &str = "added-files-size"; + /// Data files removed by the snapshot. + pub const DELETED_DATA_FILES: &str = "deleted-data-files"; + /// Rows removed by the snapshot. + pub const DELETED_RECORDS: &str = "deleted-records"; + /// Bytes removed by the snapshot. + pub const REMOVED_FILES_SIZE: &str = "removed-files-size"; + /// Data files in the table after the snapshot. + pub const TOTAL_DATA_FILES: &str = "total-data-files"; + /// Rows in the table after the snapshot. + pub const TOTAL_RECORDS: &str = "total-records"; + /// Bytes in the table after the snapshot. + pub const TOTAL_FILES_SIZE: &str = "total-files-size"; + /// Delete files in the table. + pub const TOTAL_DELETE_FILES: &str = "total-delete-files"; + /// Position deletes in the table. + pub const TOTAL_POSITION_DELETES: &str = "total-position-deletes"; + /// Equality deletes in the table. + pub const TOTAL_EQUALITY_DELETES: &str = "total-equality-deletes"; +} + +fn corrupt(msg: impl Into) -> Error { + Error::Corrupt(msg.into()) +} + +/// Runs a future whose host calls complete at once ([`lakebed_core::MemoryHost`]'s do). +#[cfg(test)] +fn block_on(f: F) -> F::Output { + let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); + let mut f = std::pin::pin!(f); + loop { + if let std::task::Poll::Ready(v) = f.as_mut().poll(&mut cx) { + return v; + } + } +} diff --git a/crates/lakebed-iceberg/src/manifest.rs b/crates/lakebed-iceberg/src/manifest.rs new file mode 100644 index 0000000..738cf66 --- /dev/null +++ b/crates/lakebed-iceberg/src/manifest.rs @@ -0,0 +1,942 @@ +//! Manifests and manifest lists: the Avro files that list a snapshot's data and delete files. +//! Reading covers format versions 1 to 3; writing (feature `write`) makes format v2 data +//! manifests in the layout iceberg-rust writes: the same Avro schemas, field ids and header +//! metadata. +//! + +use std::collections::BTreeMap; +use std::sync::Arc; + +use apache_avro::types::Value; +use serde_json::Value as Json; + +use crate::avro::{Header, field, field_of, int, long, records, required, string}; +use crate::datum::{Datum, from_be}; +use crate::partition::{PartitionFieldType, PartitionSpec}; +use crate::types::{PrimitiveType, Schema}; +use crate::{Result, corrupt}; + +/// Whether a manifest entry adds, keeps or removes its file. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Status { + /// Carried over from an earlier snapshot. + Existing = 0, + /// Added by the manifest's snapshot. + Added = 1, + /// Removed by the manifest's snapshot. + Deleted = 2, +} + +/// What a file holds. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum Content { + /// Rows. + #[default] + Data = 0, + /// Deleted row positions (a Parquet delete file, or a Puffin deletion vector in format v3). + PositionDeletes = 1, + /// Deleted rows by the values of `equality_ids` columns. + EqualityDeletes = 2, +} + +/// What a manifest lists. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum ManifestContent { + /// Data files. + #[default] + Data = 0, + /// Delete files. + Deletes = 1, +} + +/// A data or delete file as a manifest records it. Bounds are Iceberg's single-value binary +/// encoding ([`Datum::from_bytes`]), keyed by field id. +#[derive(Clone, Debug, PartialEq)] +pub struct DataFile { + /// What the file holds. + pub content: Content, + /// Absolute URI. + pub path: String, + /// `PARQUET`, `AVRO`, `ORC` or `PUFFIN`. + pub file_format: String, + /// The partition tuple, in the order of the spec's fields (empty for unpartitioned files). + pub partition: Vec>, + /// Rows. + pub record_count: u64, + /// Bytes. + pub file_size: u64, + /// Compressed bytes per field id. + pub column_sizes: BTreeMap, + /// Values per field id. + pub value_counts: BTreeMap, + /// Nulls per field id. + pub null_counts: BTreeMap, + /// NaNs per field id. + pub nan_counts: BTreeMap, + /// Smallest value per field id. + pub lower_bounds: BTreeMap>, + /// Largest value per field id. + pub upper_bounds: BTreeMap>, + /// Row group offsets. + pub split_offsets: Option>, + /// For equality deletes: the columns that identify deleted rows. + pub equality_ids: Option>, + /// The table sort order the file follows. + pub sort_order_id: Option, + /// The row id of the file's first row (format v3). + pub first_row_id: Option, + /// For position deletes and deletion vectors: the one data file they apply to. + pub referenced_data_file: Option, + /// For deletion vectors: the vector's offset in the Puffin file. + pub content_offset: Option, + /// For deletion vectors: the vector's length. + pub content_size: Option, +} + +impl Default for DataFile { + fn default() -> Self { + Self { + content: Content::Data, + path: String::new(), + file_format: "PARQUET".into(), + partition: vec![], + record_count: 0, + file_size: 0, + column_sizes: BTreeMap::new(), + value_counts: BTreeMap::new(), + null_counts: BTreeMap::new(), + nan_counts: BTreeMap::new(), + lower_bounds: BTreeMap::new(), + upper_bounds: BTreeMap::new(), + split_offsets: None, + equality_ids: None, + sort_order_id: None, + first_row_id: None, + referenced_data_file: None, + content_offset: None, + content_size: None, + } + } +} + +/// A file in a snapshot, with the sequence numbers that order its rows against other files'. +#[derive(Clone, Debug, PartialEq)] +pub struct Entry { + /// Added, existing or deleted. + pub status: Status, + /// The snapshot that added (or deleted) the file. + pub snapshot_id: i64, + /// Data sequence number: orders the file's rows against other files' rows and deletes. + pub sequence_number: i64, + /// Sequence number of the commit that added the file. + pub file_sequence_number: i64, + /// The file. + pub file: Arc, +} + +/// An entry of a manifest list. The default (sequence numbers and snapshot 0) suits reading a +/// manifest's files without the list. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ManifestFile { + /// Absolute URI of the manifest. + pub path: String, + /// Its size in bytes. + pub length: i64, + /// The partition spec its files use. + pub partition_spec_id: i32, + /// Data or delete files. + pub content: ManifestContent, + /// Sequence number of the snapshot that wrote it. + pub sequence_number: i64, + /// Smallest data sequence number of its files. + pub min_sequence_number: i64, + /// The snapshot that wrote it. + pub added_snapshot_id: i64, + /// Files added. + pub added_files: i32, + /// Files carried over. + pub existing_files: i32, + /// Files removed. + pub deleted_files: i32, + /// Rows in added files. + pub added_rows: i64, + /// Rows in existing files. + pub existing_rows: i64, + /// Rows in deleted files. + pub deleted_rows: i64, + /// The range of each partition field's values (`None` for unpartitioned manifests). + pub partitions: Option>, + /// The row id of the manifest's first row (format v3). + pub first_row_id: Option, +} + +/// The range of one partition field's values in a manifest. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FieldSummary { + /// Some file has no value for the field. + pub contains_null: bool, + /// Some file's value is NaN (`None`: not recorded). + pub contains_nan: Option, + /// Lowest value, single-value serialized. + pub lower: Option>, + /// Highest value, single-value serialized. + pub upper: Option>, +} + +/// A sequence number the manifest list assigns when the snapshot commits. +pub const UNASSIGNED: i64 = -1; + +// --- reading ------------------------------------------------------------------------------------- + +fn u64_of(v: Option) -> u64 { + v.and_then(|v| u64::try_from(v).ok()).unwrap_or(0) +} + +/// The manifests of a manifest list (format v1 to v3). +pub fn read_manifest_list(bytes: &[u8]) -> Result> { + let (records, _) = records(bytes)?; + records + .iter() + .map(|r| { + let partitions = match field(r, "partitions") { + Some(Value::Array(items)) => Some(items.iter().map(field_summary).collect()), + _ => None, + }; + let count = |names: &[&str]| int(field_of(r, names)).unwrap_or(0); + let rows = |name: &str| long(field(r, name)).unwrap_or(0); + Ok(ManifestFile { + path: string(field(r, "manifest_path")) + .ok_or_else(|| corrupt("manifest list entry without a path"))? + .to_string(), + length: required(long(field(r, "manifest_length")), "length")?, + partition_spec_id: int(field(r, "partition_spec_id")).unwrap_or(0), + content: match long(field(r, "content")) { + None | Some(0) => ManifestContent::Data, + Some(1) => ManifestContent::Deletes, + Some(other) => return Err(corrupt(format!("manifest content {other}"))), + }, + sequence_number: long(field(r, "sequence_number")).unwrap_or(0), + min_sequence_number: long(field(r, "min_sequence_number")).unwrap_or(0), + added_snapshot_id: long(field(r, "added_snapshot_id")).unwrap_or(0), + // Format v1 names the counts `*_data_files_count`. + added_files: count(&["added_files_count", "added_data_files_count"]), + existing_files: count(&["existing_files_count", "existing_data_files_count"]), + deleted_files: count(&["deleted_files_count", "deleted_data_files_count"]), + added_rows: rows("added_rows_count"), + existing_rows: rows("existing_rows_count"), + deleted_rows: rows("deleted_rows_count"), + partitions, + first_row_id: long(field(r, "first_row_id")), + }) + }) + .collect() +} + +fn field_summary(p: &Value) -> FieldSummary { + let bytes = |name: &str| match field(p, name) { + Some(Value::Bytes(b)) => Some(b.clone()), + _ => None, + }; + FieldSummary { + contains_null: matches!(field(p, "contains_null"), Some(Value::Boolean(true))), + contains_nan: match field(p, "contains_nan") { + Some(Value::Boolean(b)) => Some(*b), + _ => None, + }, + lower: bytes("lower_bound"), + upper: bytes("upper_bound"), + } +} + +fn long_map(v: Option<&Value>) -> BTreeMap { + let Some(Value::Array(items)) = v else { return BTreeMap::new() }; + items + .iter() + .filter_map(|kv| Some((int(field(kv, "key"))?, u64::try_from(long(field(kv, "value"))?).ok()?))) + .collect() +} + +fn bytes_map(v: Option<&Value>) -> BTreeMap> { + let Some(Value::Array(items)) = v else { return BTreeMap::new() }; + items + .iter() + .filter_map(|kv| match field(kv, "value") { + Some(Value::Bytes(b)) => Some((int(field(kv, "key"))?, b.clone())), + _ => None, + }) + .collect() +} + +fn longs(v: Option<&Value>) -> Option> { + match v { + Some(Value::Array(items)) => Some(items.iter().filter_map(|x| long(Some(x))).collect()), + _ => None, + } +} + +/// The partition field types a manifest's header records (its `schema` and `partition-spec`), if +/// it records them and they parse. +fn header_partition_type(metadata: &Header) -> Option> { + let json = |key: &str| serde_json::from_slice::(metadata.get(key)?).ok(); + let schema = Schema::from_json(&json("schema")?).ok()?; + let spec_id = std::str::from_utf8(metadata.get("partition-spec-id")?).ok()?.parse().ok()?; + let spec = PartitionSpec::from_fields(spec_id, &json("partition-spec")?).ok()?; + spec.partition_type(&schema).ok() +} + +/// A partition value. `ty` is the field's type when the manifest header gives it; without it the +/// Avro type decides. +fn partition_datum(v: &Value, ty: Option<&PrimitiveType>) -> Result> { + use PrimitiveType as P; + Ok(Some(match (v, ty) { + (Value::Null, _) => return Ok(None), + (Value::Boolean(b), _) => Datum::Boolean(*b), + (Value::Int(n) | Value::Date(n), Some(P::Date)) | (Value::Date(n), None) => Datum::Date(*n), + (Value::Int(n), _) => Datum::Int(*n), + (Value::Long(n) | Value::TimeMicros(n), Some(P::Time)) | (Value::TimeMicros(n), None) => Datum::Time(*n), + (Value::Long(n) | Value::TimestampMicros(n) | Value::LocalTimestampMicros(n), Some(P::Timestamp)) + | (Value::LocalTimestampMicros(n), None) => Datum::Timestamp(*n), + (Value::Long(n) | Value::TimestampMicros(n), Some(P::Timestamptz)) | (Value::TimestampMicros(n), None) => { + Datum::Timestamptz(*n) + } + (Value::Long(n) | Value::TimestampNanos(n) | Value::LocalTimestampNanos(n), Some(P::TimestampNs)) + | (Value::LocalTimestampNanos(n), None) => Datum::TimestampNs(*n), + (Value::Long(n) | Value::TimestampNanos(n), Some(P::TimestamptzNs)) | (Value::TimestampNanos(n), None) => { + Datum::TimestamptzNs(*n) + } + (Value::Long(n), _) => Datum::Long(*n), + (Value::Float(x), _) => Datum::Float(*x), + (Value::Double(x), _) => Datum::Double(*x), + (Value::String(s), _) => Datum::String(s.clone()), + (Value::Bytes(b) | Value::Fixed(_, b), Some(P::Decimal { precision, scale })) if b.len() <= 16 => { + Datum::Decimal { unscaled: from_be(b), precision: *precision, scale: *scale } + } + (Value::Decimal(d), Some(P::Decimal { precision, scale })) => { + let b = Vec::::try_from(d).map_err(crate::avro::error)?; + if b.len() > 16 { + return Err(corrupt("a decimal partition value of more than 16 bytes")); + } + Datum::Decimal { unscaled: from_be(&b), precision: *precision, scale: *scale } + } + (Value::Uuid(u), _) => Datum::Uuid(*u.as_bytes()), + (Value::Fixed(16, b), Some(P::Uuid)) => Datum::Uuid(b.as_slice().try_into().map_err(crate::avro::error)?), + (Value::Fixed(_, b), _) => Datum::Fixed(b.clone()), + (Value::Bytes(b), _) => Datum::Binary(b.clone()), + (other, _) => return Err(corrupt(format!("partition value {other:?}"))), + })) +} + +/// The entries of `manifest` (format v1 to v3), with the snapshot ids and sequence numbers they +/// inherit from it filled in. +pub fn read_manifest(manifest: &ManifestFile, bytes: &[u8]) -> Result> { + let (records, metadata) = records(bytes)?; + let types = header_partition_type(&metadata); + records + .iter() + .map(|r| { + let status = match int(field(r, "status")) { + Some(0) => Status::Existing, + Some(1) => Status::Added, + Some(2) => Status::Deleted, + other => return Err(corrupt(format!("manifest entry status {other:?}"))), + }; + let df = required(field(r, "data_file"), "data_file")?; + let partition = match field(df, "partition") { + Some(Value::Record(fields)) => fields + .iter() + .enumerate() + .map(|(i, (_, v))| { + let ty = types.as_ref().and_then(|t| t.get(i)).map(|t| &t.ty); + partition_datum(crate::avro::strip_union(v), ty) + }) + .collect::>()?, + _ => vec![], + }; + let file = DataFile { + content: match long(field(df, "content")) { + None | Some(0) => Content::Data, + Some(1) => Content::PositionDeletes, + Some(2) => Content::EqualityDeletes, + Some(other) => return Err(corrupt(format!("data file content {other}"))), + }, + path: string(field(df, "file_path")).ok_or_else(|| corrupt("data file without a path"))?.to_string(), + file_format: string(field(df, "file_format")).unwrap_or("PARQUET").to_string(), + partition, + record_count: u64_of(Some(required(long(field(df, "record_count")), "record_count")?)), + file_size: u64_of(Some(required(long(field(df, "file_size_in_bytes")), "file size")?)), + column_sizes: long_map(field(df, "column_sizes")), + value_counts: long_map(field(df, "value_counts")), + null_counts: long_map(field(df, "null_value_counts")), + nan_counts: long_map(field(df, "nan_value_counts")), + lower_bounds: bytes_map(field(df, "lower_bounds")), + upper_bounds: bytes_map(field(df, "upper_bounds")), + split_offsets: longs(field(df, "split_offsets")), + equality_ids: longs(field(df, "equality_ids")) + .map(|ids| ids.into_iter().filter_map(|id| i32::try_from(id).ok()).collect()), + sort_order_id: int(field(df, "sort_order_id")), + first_row_id: long(field(df, "first_row_id")), + referenced_data_file: string(field(df, "referenced_data_file")).map(str::to_string), + content_offset: long(field(df, "content_offset")), + content_size: long(field(df, "content_size_in_bytes")), + }; + // Entries without sequence numbers inherit the manifest's (Iceberg's rule; format v1 + // manifests have none and the list gives 0). + let inherit = |v: Option| v.unwrap_or(manifest.sequence_number); + Ok(Entry { + status, + snapshot_id: long(field(r, "snapshot_id")).unwrap_or(manifest.added_snapshot_id), + sequence_number: inherit(long(field(r, "sequence_number"))), + file_sequence_number: inherit(long(field(r, "file_sequence_number"))), + file: Arc::new(file), + }) + }) + .collect() +} + +// --- writing ------------------------------------------------------------------------------------- + +#[cfg(feature = "write")] +pub use write::{ManifestContext, ManifestWrite, manifest_file, write_manifest, write_manifest_list}; + +#[cfg(feature = "write")] +mod write { + use std::collections::BTreeMap; + use std::fmt::Write as _; + use std::sync::Arc; + + use apache_avro::types::Value; + use apache_avro::{Codec, Schema as AvroSchema, Writer}; + + use super::{Content, DataFile, FieldSummary, ManifestContent, ManifestFile, Status, UNASSIGNED}; + use crate::avro::{error, null, some}; + use crate::datum::Datum; + use crate::partition::PartitionFieldType; + use crate::types::PrimitiveType; + use crate::{Error, Result}; + + /// What a manifest's header records about the table. + #[derive(Clone, Debug)] + pub struct ManifestContext<'a> { + /// The current schema as Iceberg JSON. + pub schema_json: &'a str, + /// Its id. + pub schema_id: i32, + /// The partition spec's fields as Iceberg JSON (`[{"source-id":…,"field-id":…,…}]`). + pub spec_fields_json: &'a str, + /// The spec's id. + pub spec_id: i32, + /// The spec's fields with the types of their values ([`crate::PartitionSpec::partition_type`]). + pub partition_type: &'a [PartitionFieldType], + } + + /// An entry to write into a manifest: `None` fields are inherited from the manifest list. + #[derive(Clone, Debug)] + pub struct ManifestWrite { + /// Added, existing or deleted. + pub status: Status, + /// The snapshot, or `None` to inherit the manifest's. + pub snapshot_id: Option, + /// Data sequence number, or `None` to inherit. + pub sequence_number: Option, + /// File sequence number, or `None` to inherit. + pub file_sequence_number: Option, + /// The file. + pub file: Arc, + } + + /// A name Avro accepts: Iceberg's convention replaces a leading digit with `_` and the digit, and + /// other characters Avro doesn't allow with `_x` and their code in hex. + fn avro_name(name: &str) -> String { + let mut out = String::with_capacity(name.len()); + for (i, c) in name.chars().enumerate() { + match c { + 'a'..='z' | 'A'..='Z' | '_' => out.push(c), + '0'..='9' if i > 0 => out.push(c), + '0'..='9' => { + out.push('_'); + out.push(c); + } + _ => { + let _ = write!(out, "_x{:X}", u32::from(c)); + } + } + } + out + } + + /// The Avro type of partition values of type `ty`. + fn avro_type(ty: &PrimitiveType) -> Result { + let logical = |base: &str, logical: &str, utc: Option| { + let utc = utc.map_or_else(String::new, |u| format!(r#", "adjust-to-utc": {u}"#)); + format!(r#"{{"type": "{base}", "logicalType": "{logical}"{utc}}}"#) + }; + Ok(match ty { + PrimitiveType::Boolean => r#""boolean""#.into(), + PrimitiveType::Int => r#""int""#.into(), + PrimitiveType::Long => r#""long""#.into(), + PrimitiveType::Float => r#""float""#.into(), + PrimitiveType::Double => r#""double""#.into(), + PrimitiveType::String => r#""string""#.into(), + PrimitiveType::Binary => r#""bytes""#.into(), + PrimitiveType::Date => logical("int", "date", None), + PrimitiveType::Time => logical("long", "time-micros", None), + PrimitiveType::Timestamp => logical("long", "timestamp-micros", Some(false)), + PrimitiveType::Timestamptz => logical("long", "timestamp-micros", Some(true)), + PrimitiveType::TimestampNs => logical("long", "timestamp-nanos", Some(false)), + PrimitiveType::TimestamptzNs => logical("long", "timestamp-nanos", Some(true)), + PrimitiveType::Decimal { .. } | PrimitiveType::Uuid | PrimitiveType::Fixed(_) => { + return Err(Error::Unsupported(format!("writing {ty} partition values"))); + } + }) + } + + /// A partition value as Avro, checked against the field's type. + fn avro_value(d: &Datum, ty: &PrimitiveType) -> Result { + if d.primitive_type() != *ty { + return Err(Error::Invalid(format!("a {} partition value for a {ty} field", d.primitive_type()))); + } + Ok(match d { + Datum::Boolean(b) => Value::Boolean(*b), + Datum::Int(v) => Value::Int(*v), + Datum::Date(v) => Value::Date(*v), + Datum::Long(v) => Value::Long(*v), + Datum::Time(v) => Value::TimeMicros(*v), + Datum::Timestamp(v) | Datum::Timestamptz(v) => Value::TimestampMicros(*v), + Datum::TimestampNs(v) | Datum::TimestamptzNs(v) => Value::TimestampNanos(*v), + Datum::Float(v) => Value::Float(*v), + Datum::Double(v) => Value::Double(*v), + Datum::String(s) => Value::String(s.clone()), + Datum::Binary(b) => Value::Bytes(b.clone()), + Datum::Decimal { .. } | Datum::Uuid(_) | Datum::Fixed(_) => { + return Err(Error::Unsupported(format!("writing {ty} partition values"))); + } + }) + } + + /// The Avro schema of manifest entries for a table partitioned as `partition`. + fn manifest_schema(partition: &[PartitionFieldType]) -> Result { + let partition_fields = partition + .iter() + .map(|p| { + Ok(format!( + r#"{{"name": "{}", "type": ["null", {}], "default": null, "field-id": {}}}"#, + avro_name(&p.name), + avro_type(&p.ty)?, + p.field_id + )) + }) + .collect::>>()? + .join(", "); + let map = |name: &str, key: i32, value: i32, ty: &str| { + format!( + r#"{{"name": "{name}", "type": ["null", {{"type": "array", "items": {{"type": "record", "name": "k{key}_v{value}", "fields": [{{"name": "key", "type": "int", "field-id": {key}}}, {{"name": "value", "type": "{ty}", "field-id": {value}}}]}}, "logicalType": "map"}}], "default": null"# + ) + }; + let json = format!( + r#"{{"type": "record", "name": "manifest_entry", "fields": [ + {{"name": "status", "type": "int", "field-id": 0}}, + {{"name": "snapshot_id", "type": ["null", "long"], "default": null, "field-id": 1}}, + {{"name": "sequence_number", "type": ["null", "long"], "default": null, "field-id": 3}}, + {{"name": "file_sequence_number", "type": ["null", "long"], "default": null, "field-id": 4}}, + {{"name": "data_file", "type": {{"type": "record", "name": "r2", "fields": [ + {{"name": "content", "type": "int", "default": 0, "field-id": 134}}, + {{"name": "file_path", "type": "string", "field-id": 100}}, + {{"name": "file_format", "type": "string", "field-id": 101}}, + {{"name": "partition", "type": {{"type": "record", "name": "r102", "fields": [{partition_fields}]}}, "field-id": 102}}, + {{"name": "record_count", "type": "long", "field-id": 103}}, + {{"name": "file_size_in_bytes", "type": "long", "field-id": 104}}, + {}, "field-id": 108}}, + {}, "field-id": 109}}, + {}, "field-id": 110}}, + {}, "field-id": 137}}, + {}, "field-id": 125}}, + {}, "field-id": 128}}, + {{"name": "key_metadata", "type": ["null", "bytes"], "default": null, "field-id": 131}}, + {{"name": "split_offsets", "type": ["null", {{"type": "array", "items": "long", "element-id": 133}}], "default": null, "field-id": 132}}, + {{"name": "equality_ids", "type": ["null", {{"type": "array", "items": "int", "element-id": 136}}], "default": null, "field-id": 135}}, + {{"name": "sort_order_id", "type": ["null", "int"], "default": null, "field-id": 140}}, + {{"name": "first_row_id", "type": ["null", "long"], "default": null, "field-id": 142}}, + {{"name": "referenced_data_file", "type": ["null", "string"], "default": null, "field-id": 143}}, + {{"name": "content_offset", "type": ["null", "long"], "default": null, "field-id": 144}}, + {{"name": "content_size_in_bytes", "type": ["null", "long"], "default": null, "field-id": 145}} + ]}}, "field-id": 2}} + ]}}"#, + map("column_sizes", 117, 118, "long"), + map("value_counts", 119, 120, "long"), + map("null_value_counts", 121, 122, "long"), + map("nan_value_counts", 138, 139, "long"), + map("lower_bounds", 126, 127, "bytes"), + map("upper_bounds", 129, 130, "bytes"), + ); + AvroSchema::parse_str(&json).map_err(error) + } + + const MANIFEST_LIST_SCHEMA: &str = r#"{"type": "record", "name": "manifest_file", "fields": [ + {"name": "manifest_path", "type": "string", "field-id": 500}, + {"name": "manifest_length", "type": "long", "field-id": 501}, + {"name": "partition_spec_id", "type": "int", "field-id": 502}, + {"name": "content", "type": "int", "field-id": 517}, + {"name": "sequence_number", "type": "long", "field-id": 515}, + {"name": "min_sequence_number", "type": "long", "field-id": 516}, + {"name": "added_snapshot_id", "type": "long", "field-id": 503}, + {"name": "added_files_count", "type": "int", "field-id": 504}, + {"name": "existing_files_count", "type": "int", "field-id": 505}, + {"name": "deleted_files_count", "type": "int", "field-id": 506}, + {"name": "added_rows_count", "type": "long", "field-id": 512}, + {"name": "existing_rows_count", "type": "long", "field-id": 513}, + {"name": "deleted_rows_count", "type": "long", "field-id": 514}, + {"name": "partitions", "type": ["null", {"type": "array", "items": {"type": "record", "name": "r508", "fields": [ + {"name": "contains_null", "type": "boolean", "field-id": 509}, + {"name": "contains_nan", "type": ["null", "boolean"], "default": null, "field-id": 518}, + {"name": "lower_bound", "type": ["null", "bytes"], "default": null, "field-id": 510}, + {"name": "upper_bound", "type": ["null", "bytes"], "default": null, "field-id": 511} + ]}, "element-id": 508}], "default": null, "field-id": 507}, + {"name": "key_metadata", "type": ["null", "bytes"], "default": null, "field-id": 519} + ]}"#; + + fn i64_of(n: u64) -> i64 { + i64::try_from(n).unwrap_or(i64::MAX) + } + + fn long_map(m: &BTreeMap) -> Value { + some(Value::Array( + m.iter() + .map(|(k, v)| { + Value::Record(vec![("key".into(), Value::Int(*k)), ("value".into(), Value::Long(i64_of(*v)))]) + }) + .collect(), + )) + } + + fn bytes_map(m: &BTreeMap>) -> Value { + some(Value::Array( + m.iter() + .map(|(k, v)| { + Value::Record(vec![("key".into(), Value::Int(*k)), ("value".into(), Value::Bytes(v.clone()))]) + }) + .collect(), + )) + } + + fn opt(v: Option, f: impl FnOnce(T) -> Value) -> Value { + v.map_or_else(null, |v| some(f(v))) + } + + /// Writes a format v2 data manifest of `entries`. Added entries leave their sequence numbers to + /// be inherited from the manifest list (`None` in `sequence_number` means the commit's own); + /// existing and deleted entries keep theirs. + pub fn write_manifest(ctx: &ManifestContext<'_>, entries: &[ManifestWrite]) -> Result> { + let schema = manifest_schema(ctx.partition_type)?; + let mut w = Writer::with_codec(&schema, vec![], Codec::Null).map_err(error)?; + for (k, v) in [ + ("schema", ctx.schema_json.to_string()), + ("schema-id", ctx.schema_id.to_string()), + ("partition-spec", ctx.spec_fields_json.to_string()), + ("partition-spec-id", ctx.spec_id.to_string()), + ("format-version", "2".to_string()), + ("content", "data".to_string()), + ] { + w.add_user_metadata(k.to_string(), v).map_err(error)?; + } + for e in entries { + let f = &e.file; + if f.content != Content::Data { + return Err(Error::Unsupported("writing delete files".into())); + } + if f.partition.len() != ctx.partition_type.len() { + return Err(Error::Invalid(format!( + "{}: {} partition values for a spec of {} fields", + f.path, + f.partition.len(), + ctx.partition_type.len() + ))); + } + let partition = ctx + .partition_type + .iter() + .zip(&f.partition) + .map(|(p, v)| { + Ok((avro_name(&p.name), opt(v.as_ref().map(|v| avro_value(v, &p.ty)).transpose()?, |v| v))) + }) + .collect::>>()?; + let data_file = Value::Record(vec![ + ("content".into(), Value::Int(f.content as i32)), + ("file_path".into(), Value::String(f.path.clone())), + ("file_format".into(), Value::String(f.file_format.clone())), + ("partition".into(), Value::Record(partition)), + ("record_count".into(), Value::Long(i64_of(f.record_count))), + ("file_size_in_bytes".into(), Value::Long(i64_of(f.file_size))), + ("column_sizes".into(), long_map(&f.column_sizes)), + ("value_counts".into(), long_map(&f.value_counts)), + ("null_value_counts".into(), long_map(&f.null_counts)), + ("nan_value_counts".into(), long_map(&f.nan_counts)), + ("lower_bounds".into(), bytes_map(&f.lower_bounds)), + ("upper_bounds".into(), bytes_map(&f.upper_bounds)), + ("key_metadata".into(), null()), + ( + "split_offsets".into(), + opt(f.split_offsets.as_ref(), |o| Value::Array(o.iter().map(|v| Value::Long(*v)).collect())), + ), + ( + "equality_ids".into(), + opt(f.equality_ids.as_ref(), |ids| Value::Array(ids.iter().map(|v| Value::Int(*v)).collect())), + ), + ("sort_order_id".into(), opt(f.sort_order_id, Value::Int)), + ("first_row_id".into(), opt(f.first_row_id, Value::Long)), + ("referenced_data_file".into(), opt(f.referenced_data_file.clone(), Value::String)), + ("content_offset".into(), opt(f.content_offset, Value::Long)), + ("content_size_in_bytes".into(), opt(f.content_size, Value::Long)), + ]); + w.append_value(Value::Record(vec![ + ("status".into(), Value::Int(e.status as i32)), + ("snapshot_id".into(), opt(e.snapshot_id, Value::Long)), + ("sequence_number".into(), opt(e.sequence_number, Value::Long)), + ("file_sequence_number".into(), opt(e.file_sequence_number, Value::Long)), + ("data_file".into(), data_file), + ])) + .map_err(error)?; + } + w.into_inner().map_err(error) + } + + /// Writes a format v2 manifest list for snapshot `snapshot_id` (child of `parent`) with sequence + /// number `sequence_number`. Manifests with sequence number [`UNASSIGNED`] get the snapshot's + /// (they were written by it). + pub fn write_manifest_list( + manifests: &[ManifestFile], + snapshot_id: i64, + parent: Option, + sequence_number: i64, + ) -> Result> { + let schema = AvroSchema::parse_str(MANIFEST_LIST_SCHEMA).map_err(error)?; + let mut w = Writer::with_codec(&schema, vec![], Codec::Null).map_err(error)?; + let parent = parent.map_or_else(|| "null".into(), |p| p.to_string()); + for (k, v) in [ + ("snapshot-id", snapshot_id.to_string()), + ("parent-snapshot-id", parent), + ("sequence-number", sequence_number.to_string()), + ("format-version", "2".to_string()), + ] { + w.add_user_metadata(k.to_string(), v).map_err(error)?; + } + for m in manifests { + let assign = |v: i64| if v == UNASSIGNED { sequence_number } else { v }; + let partitions = opt(m.partitions.as_ref(), |fields| { + Value::Array( + fields + .iter() + .map(|p| { + Value::Record(vec![ + ("contains_null".into(), Value::Boolean(p.contains_null)), + ("contains_nan".into(), opt(p.contains_nan, Value::Boolean)), + ("lower_bound".into(), opt(p.lower.clone(), Value::Bytes)), + ("upper_bound".into(), opt(p.upper.clone(), Value::Bytes)), + ]) + }) + .collect(), + ) + }); + w.append_value(Value::Record(vec![ + ("manifest_path".into(), Value::String(m.path.clone())), + ("manifest_length".into(), Value::Long(m.length)), + ("partition_spec_id".into(), Value::Int(m.partition_spec_id)), + ("content".into(), Value::Int(m.content as i32)), + ("sequence_number".into(), Value::Long(assign(m.sequence_number))), + ("min_sequence_number".into(), Value::Long(assign(m.min_sequence_number))), + ("added_snapshot_id".into(), Value::Long(m.added_snapshot_id)), + ("added_files_count".into(), Value::Int(m.added_files)), + ("existing_files_count".into(), Value::Int(m.existing_files)), + ("deleted_files_count".into(), Value::Int(m.deleted_files)), + ("added_rows_count".into(), Value::Long(m.added_rows)), + ("existing_rows_count".into(), Value::Long(m.existing_rows)), + ("deleted_rows_count".into(), Value::Long(m.deleted_rows)), + ("partitions".into(), partitions), + ("key_metadata".into(), null()), + ])) + .map_err(error)?; + } + w.into_inner().map_err(error) + } + + /// The manifest-list entry of a data manifest written as `bytes` at `path` with `entries`. + pub fn manifest_file( + path: &str, + bytes: &[u8], + spec_id: i32, + snapshot_id: i64, + entries: &[ManifestWrite], + partition_type: &[PartitionFieldType], + ) -> ManifestFile { + let count = |s: Status| i32::try_from(entries.iter().filter(|e| e.status == s).count()).unwrap_or(i32::MAX); + let rows = |s: Status| i64_of(entries.iter().filter(|e| e.status == s).map(|e| e.file.record_count).sum()); + // Entries that inherit the commit's sequence number sort after every explicit one. + let min_sequence_number = entries.iter().filter_map(|e| e.sequence_number).min().unwrap_or(UNASSIGNED); + let partitions = (!partition_type.is_empty()).then(|| { + partition_type + .iter() + .enumerate() + .map(|(i, p)| { + let values = || entries.iter().map(|e| e.file.partition.get(i).and_then(Option::as_ref)); + let floating = matches!(p.ty, PrimitiveType::Float | PrimitiveType::Double); + let ordered = || values().flatten().filter(|v| !v.is_nan()); + let pick = |better: std::cmp::Ordering| { + ordered().fold(None::<&Datum>, |best, v| match best { + Some(b) if v.partial_cmp(b) != Some(better) => Some(b), + _ => Some(v), + }) + }; + FieldSummary { + contains_null: values().any(|v| v.is_none()), + contains_nan: floating.then(|| values().flatten().any(Datum::is_nan)), + lower: pick(std::cmp::Ordering::Less).map(Datum::to_bytes), + upper: pick(std::cmp::Ordering::Greater).map(Datum::to_bytes), + } + }) + .collect() + }); + ManifestFile { + path: path.to_string(), + length: i64::try_from(bytes.len()).unwrap_or(i64::MAX), + partition_spec_id: spec_id, + content: ManifestContent::Data, + sequence_number: UNASSIGNED, + min_sequence_number, + added_snapshot_id: snapshot_id, + added_files: count(Status::Added), + existing_files: count(Status::Existing), + deleted_files: count(Status::Deleted), + added_rows: rows(Status::Added), + existing_rows: rows(Status::Existing), + deleted_rows: rows(Status::Deleted), + partitions, + first_row_id: None, + } + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn names_become_avro_names() { + assert_eq!(avro_name("time_day"), "time_day"); + assert_eq!(avro_name("1st"), "_1st"); + assert_eq!(avro_name("a-b.c"), "a_x2Db_x2Ec"); + } + } +} + +#[cfg(all(test, feature = "write"))] +mod tests { + use super::*; + + fn file(path: &str, day: Option, device: &str) -> Arc { + Arc::new(DataFile { + path: path.into(), + partition: vec![day.map(Datum::Date), Some(Datum::String(device.into()))], + record_count: 10, + file_size: 100, + column_sizes: BTreeMap::from([(1, 40)]), + value_counts: BTreeMap::from([(1, 10)]), + lower_bounds: BTreeMap::from([(1, 5i64.to_le_bytes().to_vec())]), + upper_bounds: BTreeMap::from([(1, 9i64.to_le_bytes().to_vec())]), + split_offsets: Some(vec![4]), + sort_order_id: Some(1), + ..DataFile::default() + }) + } + + #[test] + fn manifests_round_trip() { + let schema = r#"{"schema-id":0,"type":"struct","fields":[ + {"id":1,"name":"time","required":true,"type":"timestamptz"}, + {"id":2,"name":"device","required":true,"type":"string"}]}"#; + let spec = r#"[{"source-id":1,"field-id":1000,"name":"time_day","transform":"day"}, + {"source-id":2,"field-id":1001,"name":"device","transform":"identity"}]"#; + let types = vec![ + PartitionFieldType { field_id: 1000, name: "time_day".into(), ty: PrimitiveType::Date }, + PartitionFieldType { field_id: 1001, name: "device".into(), ty: PrimitiveType::String }, + ]; + let ctx = ManifestContext { + schema_json: schema, + schema_id: 0, + spec_fields_json: spec, + spec_id: 0, + partition_type: &types, + }; + let entries = vec![ + ManifestWrite { + status: Status::Added, + snapshot_id: Some(7), + sequence_number: None, + file_sequence_number: None, + file: file("a", Some(20_000), "b"), + }, + ManifestWrite { + status: Status::Existing, + snapshot_id: Some(3), + sequence_number: Some(2), + file_sequence_number: Some(2), + file: file("b", None, "a"), + }, + ]; + let bytes = write_manifest(&ctx, &entries).unwrap(); + let mut mf = manifest_file("m.avro", &bytes, 0, 7, &entries, &types); + assert_eq!((mf.added_files, mf.existing_files, mf.added_rows), (1, 1, 10)); + let summaries = mf.partitions.clone().unwrap(); + assert_eq!( + summaries[0], + FieldSummary { + contains_null: true, + contains_nan: None, + lower: Some(20_000i32.to_le_bytes().to_vec()), + upper: Some(20_000i32.to_le_bytes().to_vec()) + } + ); + assert_eq!((summaries[1].lower.as_deref(), summaries[1].upper.as_deref()), (Some(&b"a"[..]), Some(&b"b"[..]))); + + let list = write_manifest_list(std::slice::from_ref(&mf), 7, Some(3), 5).unwrap(); + let read = read_manifest_list(&list).unwrap(); + // The list assigns the snapshot's sequence number; the minimum is the explicit 2. + mf.sequence_number = 5; + assert_eq!(read, [mf.clone()]); + + let read = read_manifest(&mf, &bytes).unwrap(); + assert_eq!((read[0].sequence_number, read[0].snapshot_id, read[0].status), (5, 7, Status::Added)); + assert_eq!((read[1].sequence_number, read[1].snapshot_id, read[1].status), (2, 3, Status::Existing)); + assert_eq!(read[0].file, entries[0].file); + assert_eq!(read[1].file, entries[1].file); + } + + #[test] + fn partition_values_must_fit_the_spec() { + let types = vec![PartitionFieldType { field_id: 1000, name: "d".into(), ty: PrimitiveType::Date }]; + let ctx = ManifestContext { + schema_json: "{}", + schema_id: 0, + spec_fields_json: "[]", + spec_id: 0, + partition_type: &types, + }; + let write = |partition: Vec>| { + let file = Arc::new(DataFile { path: "a".into(), partition, ..DataFile::default() }); + let entry = ManifestWrite { + status: Status::Added, + snapshot_id: None, + sequence_number: None, + file_sequence_number: None, + file, + }; + write_manifest(&ctx, &[entry]) + }; + assert!(write(vec![Some(Datum::Date(1))]).is_ok()); + assert!(matches!(write(vec![Some(Datum::Int(1))]), Err(crate::Error::Invalid(_)))); + assert!(matches!(write(vec![]), Err(crate::Error::Invalid(_)))); + } + + #[test] + fn garbage_is_an_error() { + assert!(read_manifest_list(b"not avro").is_err()); + assert!(read_manifest_list(&[]).is_err()); + } +} diff --git a/crates/lakebed-iceberg/src/metadata.rs b/crates/lakebed-iceberg/src/metadata.rs new file mode 100644 index 0000000..1e66712 --- /dev/null +++ b/crates/lakebed-iceberg/src/metadata.rs @@ -0,0 +1,481 @@ +//! Table metadata (`*.metadata.json`), kept as JSON so fields this crate doesn't model survive a +//! rewrite, with typed views for what readers and writers need. Reading covers format versions 1 +//! to 3; updates (feature `write`) are for format v2, made the way iceberg-rust's metadata builder +//! makes them. +//! + +use std::collections::HashMap; + +use serde_json::Value; + +use crate::partition::{PartitionFieldType, PartitionSpec}; +use crate::types::Schema; +use crate::{Error, Result, corrupt}; + +/// The format version this crate writes. +pub const WRITABLE_FORMAT_VERSION: u8 = 2; + +/// A table's metadata at one version. +#[derive(Clone, Debug, PartialEq)] +pub struct Metadata(Value); + +/// A snapshot: the table's state after one commit. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Snapshot { + /// Its id. + pub snapshot_id: i64, + /// The snapshot it was committed on. + pub parent_snapshot_id: Option, + /// Its sequence number (0 in format v1). + pub sequence_number: i64, + /// When it was made (ms since the epoch). + pub timestamp_ms: i64, + /// Its manifest list (format v1 snapshots may list manifests inline instead). + pub manifest_list: Option, + /// Its summary, including `operation`. + pub summary: HashMap, + /// The schema it was written with. + pub schema_id: Option, +} + +impl Snapshot { + fn from_json(s: &Value) -> Option { + Some(Self { + snapshot_id: s.get("snapshot-id")?.as_i64()?, + parent_snapshot_id: s.get("parent-snapshot-id").and_then(Value::as_i64), + sequence_number: s.get("sequence-number").and_then(Value::as_i64).unwrap_or(0), + timestamp_ms: s.get("timestamp-ms").and_then(Value::as_i64).unwrap_or(0), + manifest_list: s.get("manifest-list").and_then(Value::as_str).map(str::to_string), + summary: strings(s.get("summary")), + schema_id: s.get("schema-id").and_then(Value::as_i64).and_then(|v| i32::try_from(v).ok()), + }) + } + + /// `append`, `replace`, `overwrite` or `delete`. + pub fn operation(&self) -> Option<&str> { + self.summary.get("operation").map(String::as_str) + } +} + +fn strings(v: Option<&Value>) -> HashMap { + let Some(Value::Object(m)) = v else { return HashMap::new() }; + m.iter().filter_map(|(k, v)| Some((k.clone(), v.as_str()?.to_string()))).collect() +} + +fn find_by<'a>(list: Option<&'a Value>, key: &str, id: i64) -> Option<&'a Value> { + list?.as_array()?.iter().find(|v| v.get(key).and_then(Value::as_i64) == Some(id)) +} + +impl Metadata { + /// Parses table metadata of format version 1, 2 or 3. + pub fn parse(bytes: &[u8]) -> Result { + let value: Value = serde_json::from_slice(bytes).map_err(|e| corrupt(format!("table metadata: {e}")))?; + Self::from_json(value) + } + + /// Table metadata from parsed JSON. + pub fn from_json(value: Value) -> Result { + match value.get("format-version").and_then(Value::as_u64) { + Some(1..=3) => Ok(Self(value)), + Some(v) => Err(Error::Unsupported(format!("Iceberg format version {v}"))), + None => Err(corrupt("table metadata without a format version")), + } + } + + /// The format version (1 to 3). + pub fn format_version(&self) -> u8 { + self.0.get("format-version").and_then(Value::as_u64).and_then(|v| u8::try_from(v).ok()).unwrap_or(0) + } + + /// An error unless this crate can write the table (format v2). + pub fn check_writable(&self) -> Result<()> { + match self.format_version() { + WRITABLE_FORMAT_VERSION => Ok(()), + v => Err(Error::Unsupported(format!( + "writing Iceberg format version {v} tables (only version {WRITABLE_FORMAT_VERSION})" + ))), + } + } + + /// The metadata as JSON. + pub fn as_json(&self) -> &Value { + &self.0 + } + + /// The metadata as JSON text. + pub fn to_json(&self) -> String { + self.0.to_string() + } + + /// The table's root location. + pub fn location(&self) -> &str { + self.0.get("location").and_then(Value::as_str).unwrap_or_default() + } + + /// The table's UUID. + pub fn table_uuid(&self) -> Option<&str> { + self.0.get("table-uuid").and_then(Value::as_str) + } + + /// The table properties. + pub fn properties(&self) -> HashMap { + strings(self.0.get("properties")) + } + + /// When the metadata was last updated (ms since the epoch). + pub fn last_updated_ms(&self) -> i64 { + self.0.get("last-updated-ms").and_then(Value::as_i64).unwrap_or(0) + } + + /// The current snapshot's id. + pub fn current_snapshot_id(&self) -> Option { + self.0.get("current-snapshot-id").and_then(Value::as_i64).filter(|&id| id >= 0) + } + + /// Every snapshot, in the order the metadata lists them. + pub fn snapshots(&self) -> Vec { + let Some(Value::Array(list)) = self.0.get("snapshots") else { return vec![] }; + list.iter().filter_map(Snapshot::from_json).collect() + } + + /// The snapshot with id `id`. + pub fn snapshot(&self, id: i64) -> Option { + Snapshot::from_json(find_by(self.0.get("snapshots"), "snapshot-id", id)?) + } + + /// Whether a snapshot with id `id` exists. + pub fn has_snapshot(&self, id: i64) -> bool { + find_by(self.0.get("snapshots"), "snapshot-id", id).is_some() + } + + /// The current snapshot. + pub fn current_snapshot(&self) -> Option { + self.snapshot(self.current_snapshot_id()?) + } + + /// The current snapshot and its ancestors, newest first, as far as they are still listed. + pub fn ancestors(&self) -> Vec { + let mut out: Vec = vec![]; + let mut next = self.current_snapshot_id(); + while let Some(s) = next.and_then(|id| self.snapshot(id)) { + // A cycle would be corrupt metadata; stop rather than loop. + if out.iter().any(|a| a.snapshot_id == s.snapshot_id) { + break; + } + next = s.parent_snapshot_id; + out.push(s); + } + out + } + + /// The current snapshot's summary. + pub fn summary(&self) -> HashMap { + self.current_snapshot().map(|s| s.summary).unwrap_or_default() + } + + /// The current snapshot's manifest list. + pub fn manifest_list(&self) -> Option { + self.current_snapshot()?.manifest_list + } + + /// The sequence number the next snapshot gets. + pub fn next_sequence_number(&self) -> i64 { + self.0.get("last-sequence-number").and_then(Value::as_i64).unwrap_or(0) + 1 + } + + /// The current schema's id. + pub fn schema_id(&self) -> i32 { + self.0.get("current-schema-id").and_then(Value::as_i64).and_then(|v| i32::try_from(v).ok()).unwrap_or(0) + } + + /// The current schema as Iceberg JSON (`schema` in format v1 metadata without `schemas`). + pub fn schema_json(&self) -> Result<&Value> { + find_by(self.0.get("schemas"), "schema-id", self.schema_id().into()) + .or_else(|| self.0.get("schema")) + .ok_or_else(|| corrupt("table metadata without a current schema")) + } + + /// The current schema. + pub fn schema(&self) -> Result { + Schema::from_json(self.schema_json()?) + } + + /// The default partition spec's id and fields as Iceberg JSON (`partition-spec` in format v1 + /// metadata without `partition-specs`). + pub fn partition_spec_json(&self) -> Result<(i32, &Value)> { + let id = self.0.get("default-spec-id").and_then(Value::as_i64).unwrap_or(0); + let fields = match find_by(self.0.get("partition-specs"), "spec-id", id) { + Some(spec) => spec.get("fields").ok_or_else(|| corrupt("partition spec without fields"))?, + None => self.0.get("partition-spec").ok_or_else(|| corrupt("table metadata without a partition spec"))?, + }; + Ok((i32::try_from(id).map_err(|_| corrupt("partition spec id"))?, fields)) + } + + /// The default partition spec. + pub fn partition_spec(&self) -> Result { + let (id, fields) = self.partition_spec_json()?; + PartitionSpec::from_fields(id, fields) + } + + /// The default partition spec's fields with the types of their values. + pub fn partition_type(&self) -> Result> { + self.partition_spec()?.partition_type(&self.schema()?) + } + + /// The default sort order's id. + pub fn sort_order_id(&self) -> i32 { + self.0.get("default-sort-order-id").and_then(Value::as_i64).and_then(|v| i32::try_from(v).ok()).unwrap_or(0) + } + + /// Entries in the metadata log. + pub fn metadata_log_len(&self) -> usize { + self.0.get("metadata-log").and_then(Value::as_array).map_or(0, Vec::len) + } + + /// The current schema as Arrow, fields carrying their Iceberg field ids. + #[cfg(feature = "arrow")] + pub fn arrow_schema(&self) -> Result { + Ok(std::sync::Arc::new(crate::arrow::schema_to_arrow(&self.schema()?)?)) + } +} + +#[cfg(feature = "write")] +pub use write::NewSnapshot; + +#[cfg(feature = "write")] +mod write { + use std::collections::HashMap; + + use serde_json::{Map, Value, json}; + + use super::Metadata; + use crate::{Result, corrupt}; + + /// How many earlier metadata files the metadata log keeps unless the table sets + /// `write.metadata.previous-versions-max`. + const PREVIOUS_VERSIONS_MAX: usize = 100; + + /// A snapshot to add (see [`Metadata::with_snapshot`]). + #[derive(Clone, Debug)] + pub struct NewSnapshot { + /// Its id. + pub snapshot_id: i64, + /// Its sequence number. + pub sequence_number: i64, + /// When it was made (ms since the epoch). + pub timestamp_ms: i64, + /// Its manifest list. + pub manifest_list: String, + /// `append`, `replace`, `overwrite` or `delete`. + pub operation: String, + /// Summary properties besides the operation. + pub summary: HashMap, + } + + impl NewSnapshot { + fn summary_json(&self) -> Value { + let mut s: Map = self.summary.iter().map(|(k, v)| (k.clone(), json!(v))).collect(); + s.insert("operation".into(), json!(self.operation)); + Value::Object(s) + } + } + + fn push(obj: &mut Map, key: &str, value: Value) { + match obj.get_mut(key) { + Some(Value::Array(a)) => a.push(value), + _ => { + obj.insert(key.into(), Value::Array(vec![value])); + } + } + } + + impl Metadata { + fn object(&mut self) -> Result<&mut Map> { + self.0.as_object_mut().ok_or_else(|| corrupt("table metadata isn't an object")) + } + + /// Records `previous` (the metadata file this version replaces) in the metadata log, + /// trimmed to the table's `write.metadata.previous-versions-max`. + fn log_previous(&mut self, previous: &str, previous_updated_ms: i64) -> Result<()> { + let keep = self + .properties() + .get("write.metadata.previous-versions-max") + .and_then(|v| v.parse::().ok()) + .unwrap_or(PREVIOUS_VERSIONS_MAX) + .max(1); + let obj = self.object()?; + push(obj, "metadata-log", json!({"metadata-file": previous, "timestamp-ms": previous_updated_ms})); + if let Some(Value::Array(log)) = obj.get_mut("metadata-log") + && log.len() > keep + { + log.drain(..log.len() - keep); + } + Ok(()) + } + + /// This metadata with `snapshot` added as the current snapshot of `main`, `previous` (this + /// version's file) recorded in the metadata log, updated at `now_ms`. + pub fn with_snapshot(&self, snapshot: &NewSnapshot, previous: &str, now_ms: i64) -> Result { + self.check_writable()?; + let mut m = self.clone(); + let mut s = json!({ + "snapshot-id": snapshot.snapshot_id, + "sequence-number": snapshot.sequence_number, + "timestamp-ms": snapshot.timestamp_ms, + "manifest-list": snapshot.manifest_list, + "summary": snapshot.summary_json(), + "schema-id": self.schema_id(), + }); + if let (Some(parent), Some(s)) = (self.current_snapshot_id(), s.as_object_mut()) { + s.insert("parent-snapshot-id".into(), json!(parent)); + } + m.log_previous(previous, self.last_updated_ms())?; + let obj = m.object()?; + push(obj, "snapshots", s); + push( + obj, + "snapshot-log", + json!({"snapshot-id": snapshot.snapshot_id, "timestamp-ms": snapshot.timestamp_ms}), + ); + obj.insert("current-snapshot-id".into(), json!(snapshot.snapshot_id)); + obj.insert("last-sequence-number".into(), json!(snapshot.sequence_number)); + obj.insert("last-updated-ms".into(), json!(now_ms)); + let refs = obj.entry("refs").or_insert_with(|| Value::Object(Map::new())); + if let Some(refs) = refs.as_object_mut() { + refs.insert("main".into(), json!({"snapshot-id": snapshot.snapshot_id, "type": "branch"})); + } + Ok(m) + } + + /// This metadata without the snapshots `ids`, as iceberg-rust's builder removes them: refs to + /// them go, and the snapshot log keeps only what follows the last entry of a removed snapshot + /// (so it has no gaps). `previous` is recorded in the metadata log. + pub fn without_snapshots(&self, ids: &[i64], previous: &str, now_ms: i64) -> Result { + self.check_writable()?; + let mut m = self.clone(); + m.log_previous(previous, self.last_updated_ms())?; + let obj = m.object()?; + let removed = |v: &Value| v.get("snapshot-id").and_then(Value::as_i64).is_some_and(|id| ids.contains(&id)); + if let Some(Value::Array(snapshots)) = obj.get_mut("snapshots") { + snapshots.retain(|s| !removed(s)); + } + if let Some(Value::Object(refs)) = obj.get_mut("refs") { + refs.retain(|_, r| !removed(r)); + } + if let Some(Value::Array(log)) = obj.get_mut("snapshot-log") { + let mut kept = vec![]; + for entry in log.drain(..) { + if removed(&entry) { + kept.clear(); + } else { + kept.push(entry); + } + } + *log = kept; + } + obj.insert("last-updated-ms".into(), json!(now_ms)); + Ok(m) + } + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use crate::types::PrimitiveType; + + fn v2() -> Metadata { + Metadata::from_json(json!({ + "format-version": 2, "table-uuid": "u", "location": "memory://t", "last-sequence-number": 2, + "last-updated-ms": 10, "current-schema-id": 1, + "schemas": [ + {"schema-id": 0, "type": "struct", "fields": []}, + {"schema-id": 1, "type": "struct", "fields": [{"id": 1, "name": "time", "required": true, "type": "timestamptz"}]} + ], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "name": "time_day", "transform": "day"}]}], + "properties": {"write.metadata.previous-versions-max": "2", "n": 1}, + "current-snapshot-id": 2, + "snapshots": [ + {"snapshot-id": 1, "sequence-number": 1, "timestamp-ms": 1, "manifest-list": "l1", "summary": {"operation": "append"}}, + {"snapshot-id": 2, "parent-snapshot-id": 1, "sequence-number": 2, "timestamp-ms": 2, "manifest-list": "l2", "summary": {"operation": "replace"}} + ], + "snapshot-log": [{"snapshot-id": 1, "timestamp-ms": 1}, {"snapshot-id": 2, "timestamp-ms": 2}], + "metadata-log": [], + "unknown-field": {"kept": true} + })) + .unwrap() + } + + #[test] + fn reads_v2() { + let m = v2(); + assert_eq!((m.format_version(), m.location(), m.table_uuid()), (2, "memory://t", Some("u"))); + assert_eq!(m.properties().len(), 1, "non-string properties are skipped"); + assert_eq!(m.schema().unwrap().fields[0].name, "time"); + assert_eq!(m.partition_type().unwrap()[0].ty, PrimitiveType::Date); + assert_eq!(m.current_snapshot().unwrap().operation(), Some("replace")); + let ancestors: Vec = m.ancestors().iter().map(|s| s.snapshot_id).collect(); + assert_eq!(ancestors, [2, 1]); + assert_eq!(m.manifest_list().as_deref(), Some("l2")); + assert_eq!(m.next_sequence_number(), 3); + assert!(m.check_writable().is_ok()); + } + + #[test] + fn reads_v1() { + let m = Metadata::from_json(json!({ + "format-version": 1, "location": "s3://b/t", + "schema": {"type": "struct", "fields": [{"id": 1, "name": "x", "required": false, "type": "long"}]}, + "partition-spec": [{"source-id": 1, "name": "x", "transform": "identity"}], + "snapshots": [] + })) + .unwrap(); + assert_eq!(m.schema().unwrap().fields[0].name, "x"); + assert_eq!(m.partition_spec().unwrap().fields[0].field_id, 1000); + assert_eq!(m.current_snapshot(), None); + assert!(matches!(m.check_writable(), Err(Error::Unsupported(_)))); + } + + #[test] + fn rejects_unknown_versions() { + assert!(matches!(Metadata::parse(br#"{"format-version": 4}"#), Err(Error::Unsupported(_)))); + assert!(matches!(Metadata::parse(br#"{"location": "x"}"#), Err(Error::Corrupt(_)))); + assert!(matches!(Metadata::parse(b"{"), Err(Error::Corrupt(_)))); + } + + #[cfg(feature = "write")] + #[test] + fn updates_keep_unknown_fields_and_trim_the_log() { + let m = v2(); + let s = NewSnapshot { + snapshot_id: 3, + sequence_number: 3, + timestamp_ms: 20, + manifest_list: "l3".into(), + operation: "append".into(), + summary: HashMap::from([("k".to_string(), "v".to_string())]), + }; + let m = m.with_snapshot(&s, "v1.metadata.json", 20).unwrap(); + let m = m.with_snapshot(&NewSnapshot { snapshot_id: 4, ..s.clone() }, "v2.metadata.json", 21).unwrap(); + let m = m.with_snapshot(&NewSnapshot { snapshot_id: 5, ..s }, "v3.metadata.json", 22).unwrap(); + assert_eq!(m.current_snapshot_id(), Some(5)); + assert_eq!(m.snapshot(3).unwrap().parent_snapshot_id, Some(2)); + assert_eq!(m.snapshot(3).unwrap().summary["k"], "v"); + assert_eq!(m.metadata_log_len(), 2, "previous-versions-max"); + assert_eq!(m.as_json()["unknown-field"]["kept"], true); + assert_eq!(m.as_json()["refs"]["main"]["snapshot-id"], 5); + + let m = m.without_snapshots(&[1, 2], "v4.metadata.json", 23).unwrap(); + assert!(!m.has_snapshot(1) && !m.has_snapshot(2) && m.has_snapshot(3)); + let log: Vec = m.as_json()["snapshot-log"] + .as_array() + .unwrap() + .iter() + .map(|e| e["snapshot-id"].as_i64().unwrap()) + .collect(); + assert_eq!(log, [3, 4, 5]); + } +} diff --git a/crates/lakebed-iceberg/src/partition.rs b/crates/lakebed-iceberg/src/partition.rs new file mode 100644 index 0000000..c407e04 --- /dev/null +++ b/crates/lakebed-iceberg/src/partition.rs @@ -0,0 +1,228 @@ +//! Partition specs and their transforms. +//! + +use std::fmt; +use std::str::FromStr; + +use serde_json::Value; + +use crate::types::{PrimitiveType, Schema}; +use crate::{Error, Result, corrupt}; + +/// Partition field ids start here when a format v1 spec leaves them out. +const FIRST_FIELD_ID: i32 = 1000; + +/// How a partition value is derived from its source column. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum Transform { + /// The value itself. + Identity, + /// A hash of the value modulo N. + Bucket(u32), + /// The value truncated to width W. + Truncate(u32), + /// Years since 1970. + Year, + /// Months since 1970-01. + Month, + /// Days since 1970-01-01. + Day, + /// Hours since 1970-01-01 00:00. + Hour, + /// Always null. + Void, + /// A transform this version doesn't know, by name. + Other(String), +} + +impl FromStr for Transform { + type Err = Error; + + fn from_str(s: &str) -> Result { + let arg = |prefix: &str| { + s.strip_prefix(prefix) + .and_then(|r| r.strip_suffix(']')) + .map(|n| n.trim().parse::().map_err(|_| corrupt(format!("transform {s}")))) + }; + Ok(match s { + "identity" => Self::Identity, + "year" => Self::Year, + "month" => Self::Month, + "day" => Self::Day, + "hour" => Self::Hour, + "void" => Self::Void, + _ => { + if let Some(n) = arg("bucket[") { + Self::Bucket(n?) + } else if let Some(w) = arg("truncate[") { + Self::Truncate(w?) + } else { + Self::Other(s.to_string()) + } + } + }) + } +} + +impl fmt::Display for Transform { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Identity => f.write_str("identity"), + Self::Bucket(n) => write!(f, "bucket[{n}]"), + Self::Truncate(w) => write!(f, "truncate[{w}]"), + Self::Year => f.write_str("year"), + Self::Month => f.write_str("month"), + Self::Day => f.write_str("day"), + Self::Hour => f.write_str("hour"), + Self::Void => f.write_str("void"), + Self::Other(name) => f.write_str(name), + } + } +} + +impl Transform { + /// The type of the partition values this transform makes from a `source` column, as + /// iceberg-rust types them (`day` gives a `date`); `None` for an unknown transform. + pub fn result_type(&self, source: &PrimitiveType) -> Option { + match self { + Self::Identity | Self::Truncate(_) | Self::Void => Some(source.clone()), + Self::Bucket(_) | Self::Year | Self::Month | Self::Hour => Some(PrimitiveType::Int), + Self::Day => Some(PrimitiveType::Date), + Self::Other(_) => None, + } + } +} + +/// A field of a partition spec. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PartitionField { + /// The source column's field id. + pub source_id: i32, + /// This field's id (1000 and up). + pub field_id: i32, + /// Its name in manifests. + pub name: String, + /// How its values are derived. + pub transform: Transform, +} + +/// A partition spec. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PartitionSpec { + /// Its id. + pub spec_id: i32, + /// Its fields, in the order of the partition tuple. + pub fields: Vec, +} + +impl PartitionSpec { + /// A spec from metadata JSON: `{"spec-id": 0, "fields": [...]}`. + pub fn from_json(spec: &Value) -> Result { + let id = spec.get("spec-id").and_then(Value::as_i64).unwrap_or(0); + let fields = spec.get("fields").ok_or_else(|| corrupt("partition spec without fields"))?; + Self::from_fields(i32::try_from(id).map_err(|_| corrupt("partition spec id"))?, fields) + } + + /// A spec from its fields' JSON (a format v1 table's `partition-spec`, a manifest header's + /// `partition-spec`). + pub fn from_fields(spec_id: i32, fields: &Value) -> Result { + let fields = fields.as_array().ok_or_else(|| corrupt("partition fields aren't a list"))?; + let fields = fields + .iter() + .zip(FIRST_FIELD_ID..) + .map(|(f, default_id)| { + let int = |key: &str| f.get(key).and_then(Value::as_i64).and_then(|v| i32::try_from(v).ok()); + Ok(PartitionField { + source_id: int("source-id").ok_or_else(|| corrupt("partition field without source-id"))?, + field_id: int("field-id").unwrap_or(default_id), + name: f + .get("name") + .and_then(Value::as_str) + .ok_or_else(|| corrupt("partition field without a name"))? + .to_string(), + transform: f + .get("transform") + .and_then(Value::as_str) + .ok_or_else(|| corrupt("partition field without a transform"))? + .parse()?, + }) + }) + .collect::>()?; + Ok(Self { spec_id, fields }) + } + + /// Whether files of this spec all share one partition. + pub fn is_unpartitioned(&self) -> bool { + self.fields.iter().all(|f| f.transform == Transform::Void) + } + + /// Each field with the type of its values, given the table `schema`. + pub fn partition_type(&self, schema: &Schema) -> Result> { + self.fields + .iter() + .map(|f| { + let source = schema + .primitive_by_id(f.source_id) + .ok_or_else(|| corrupt(format!("partition field {} has no primitive source column", f.name)))?; + let ty = f + .transform + .result_type(source) + .ok_or_else(|| Error::Unsupported(format!("partition transform {}", f.transform)))?; + Ok(PartitionFieldType { field_id: f.field_id, name: f.name.clone(), ty }) + }) + .collect() + } +} + +/// A partition field with the type of its values. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PartitionFieldType { + /// The partition field's id. + pub field_id: i32, + /// Its name. + pub name: String, + /// The type of its values. + pub ty: PrimitiveType, +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn transforms_round_trip() { + for name in ["identity", "bucket[16]", "truncate[4]", "year", "month", "day", "hour", "void"] { + assert_eq!(name.parse::().unwrap().to_string(), name); + } + assert_eq!("zorder".parse::().unwrap(), Transform::Other("zorder".into())); + assert!("bucket[x]".parse::().is_err()); + } + + #[test] + fn specs_type_their_fields() { + let schema = Schema::from_json(&json!({"schema-id": 0, "fields": [ + {"id": 1, "name": "time", "required": true, "type": "timestamptz"}, + {"id": 2, "name": "device", "required": true, "type": "string"} + ]})) + .unwrap(); + // A v1 spec without field ids gets 1000 and up. + let spec = PartitionSpec::from_fields( + 0, + &json!([ + {"source-id": 1, "name": "time_day", "transform": "day"}, + {"source-id": 2, "name": "device_bucket", "transform": "bucket[8]"}, + {"source-id": 2, "name": "device", "transform": "identity"} + ]), + ) + .unwrap(); + let types = spec.partition_type(&schema).unwrap(); + let ids: Vec = types.iter().map(|t| t.field_id).collect(); + assert_eq!(ids, [1000, 1001, 1002]); + let tys: Vec<&PrimitiveType> = types.iter().map(|t| &t.ty).collect(); + assert_eq!(tys, [&PrimitiveType::Date, &PrimitiveType::Int, &PrimitiveType::String]); + assert!(!spec.is_unpartitioned()); + assert!(PartitionSpec::from_json(&json!({"spec-id": 1, "fields": []})).unwrap().is_unpartitioned()); + } +} diff --git a/crates/lakebed-iceberg/src/stats.rs b/crates/lakebed-iceberg/src/stats.rs new file mode 100644 index 0000000..7d913a9 --- /dev/null +++ b/crates/lakebed-iceberg/src/stats.rs @@ -0,0 +1,148 @@ +//! Iceberg's statistics of a Parquet data file, from its footer (feature `parquet-stats`): the +//! values iceberg-rust's writer records. Columns are matched to fields by the Parquet field ids +//! the writer stored. + +use std::cmp::Ordering; +use std::collections::BTreeMap; + +use parquet::basic::{LogicalType, Type as PhysicalType}; +use parquet::file::metadata::{ParquetMetaData, RowGroupMetaData}; +use parquet::file::statistics::Statistics; +use parquet::schema::types::ColumnDescriptor; + +use crate::datum::{Datum, from_be, minimal_be}; +use crate::manifest::DataFile; + +/// What Iceberg records about a Parquet file. Bounds are Iceberg's single-value binary encoding. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ParquetStats { + /// Rows in the file. + pub record_count: u64, + /// The file's size in bytes. + pub file_size: u64, + /// Compressed bytes per field id. + pub column_sizes: BTreeMap, + /// Values (nulls included) per field id. + pub value_counts: BTreeMap, + /// Nulls per field id. + pub null_counts: BTreeMap, + /// NaNs per field id of a float column (Parquet doesn't record them; the writer counts them). + pub nan_counts: BTreeMap, + /// Smallest value per field id, where Parquet's statistics give an exact one. + pub lower_bounds: BTreeMap>, + /// Largest value per field id, likewise. + pub upper_bounds: BTreeMap>, + /// Offsets of the row groups. + pub split_offsets: Vec, +} + +impl ParquetStats { + /// The statistics of a file of `file_size` bytes with footer `metadata`, and the NaN counts the + /// writer counted. + pub fn new(metadata: &ParquetMetaData, file_size: u64, nan_counts: BTreeMap) -> Self { + let mut s = Self { + record_count: u64::try_from(metadata.file_metadata().num_rows()).unwrap_or(0), + file_size, + nan_counts, + split_offsets: metadata.row_groups().iter().filter_map(RowGroupMetaData::file_offset).collect(), + ..Self::default() + }; + for group in metadata.row_groups() { + for chunk in group.columns() { + let Some(id) = field_id(chunk.column_descr()) else { continue }; + *s.column_sizes.entry(id).or_default() += u64::try_from(chunk.compressed_size()).unwrap_or(0); + *s.value_counts.entry(id).or_default() += u64::try_from(chunk.num_values()).unwrap_or(0); + let Some(stats) = chunk.statistics() else { continue }; + if let Some(nulls) = stats.null_count_opt() { + *s.null_counts.entry(id).or_default() += nulls; + } + let descr = chunk.column_descr(); + if stats.min_is_exact() + && let Some(min) = stats.min_bytes_opt().and_then(|b| bound(b, stats, descr)) + { + keep(&mut s.lower_bounds, id, min, descr, Ordering::Less); + } + if stats.max_is_exact() + && let Some(max) = stats.max_bytes_opt().and_then(|b| bound(b, stats, descr)) + { + keep(&mut s.upper_bounds, id, max, descr, Ordering::Greater); + } + } + } + s + } + + /// The data file at `path` with these statistics, in partition `partition`. + pub fn data_file(&self, path: String, partition: Vec>) -> DataFile { + DataFile { + path, + partition, + record_count: self.record_count, + file_size: self.file_size, + column_sizes: self.column_sizes.clone(), + value_counts: self.value_counts.clone(), + null_counts: self.null_counts.clone(), + nan_counts: self.nan_counts.clone(), + lower_bounds: self.lower_bounds.clone(), + upper_bounds: self.upper_bounds.clone(), + split_offsets: Some(self.split_offsets.clone()), + ..DataFile::default() + } + } +} + +/// The Iceberg field id a Parquet column carries. +pub fn field_id(column: &ColumnDescriptor) -> Option { + let info = column.self_type().get_basic_info(); + info.has_id().then(|| info.id()) +} + +/// Keeps `value` for `id` if there is none yet or it compares `wins` against the current one. +fn keep(bounds: &mut BTreeMap>, id: i32, value: Vec, descr: &ColumnDescriptor, wins: Ordering) { + match bounds.get(&id) { + Some(current) if compare(&value, current, descr) != wins => {} + _ => { + bounds.insert(id, value); + } + } +} + +/// A Parquet statistic's plain encoding as an Iceberg bound. They agree for booleans, ints, +/// longs, floats, doubles, dates, times, timestamps, strings, binary, and fixed; decimals stored as +/// INT32 or INT64 become big-endian bytes. INT96 has no Iceberg type. +fn bound(plain: &[u8], stats: &Statistics, descr: &ColumnDescriptor) -> Option> { + let decimal = matches!(descr.logical_type_ref(), Some(LogicalType::Decimal { .. })); + match (stats, decimal) { + (Statistics::Int96(_), _) => None, + (Statistics::Int32(_), true) => Some(minimal_be(i128::from(i32::from_le_bytes(plain.try_into().ok()?)))), + (Statistics::Int64(_), true) => Some(minimal_be(i128::from(i64::from_le_bytes(plain.try_into().ok()?)))), + _ => Some(plain.to_vec()), + } +} + +/// Compares two bounds of a column by value. +fn compare(a: &[u8], b: &[u8], descr: &ColumnDescriptor) -> Ordering { + let decimal = matches!(descr.logical_type_ref(), Some(LogicalType::Decimal { .. })); + if decimal { + return from_be(a).cmp(&from_be(b)); + } + match descr.physical_type() { + PhysicalType::INT32 => le::<4>(a).map(i32::from_le_bytes).cmp(&le::<4>(b).map(i32::from_le_bytes)), + PhysicalType::INT64 => le::<8>(a).map(i64::from_le_bytes).cmp(&le::<8>(b).map(i64::from_le_bytes)), + PhysicalType::FLOAT => { + let (x, y) = (le::<4>(a).map(f32::from_le_bytes), le::<4>(b).map(f32::from_le_bytes)); + x.partial_cmp(&y).unwrap_or(Ordering::Equal) + } + PhysicalType::DOUBLE => { + let (x, y) = (le::<8>(a).map(f64::from_le_bytes), le::<8>(b).map(f64::from_le_bytes)); + x.partial_cmp(&y).unwrap_or(Ordering::Equal) + } + PhysicalType::BOOLEAN | PhysicalType::INT96 | PhysicalType::BYTE_ARRAY | PhysicalType::FIXED_LEN_BYTE_ARRAY => { + a.cmp(b) + } + } +} + +fn le(bytes: &[u8]) -> Option<[u8; N]> { + bytes.try_into().ok() +} diff --git a/crates/lakebed-iceberg/src/table.rs b/crates/lakebed-iceberg/src/table.rs new file mode 100644 index 0000000..7cadb70 --- /dev/null +++ b/crates/lakebed-iceberg/src/table.rs @@ -0,0 +1,162 @@ +//! A table at one version, found from its location in a store: metadata files are +//! `/metadata/v.metadata.json`, as file-system ("Hadoop") tables and Timelake name +//! them, with `version-hint.text` naming the newest for stores that can't list. Tables a catalog +//! names with other file names load with [`Table::from_metadata_file`]. + +use lakebed_core::Host; + +use crate::manifest::{Entry, ManifestFile, Status, read_manifest, read_manifest_list}; +use crate::metadata::Metadata; +use crate::{Error, Result}; + +/// `/metadata/v.metadata.json`. +pub fn metadata_uri(location: &str, version: u64) -> String { + format!("{location}/metadata/v{version}.metadata.json") +} + +/// `/metadata/version-hint.text`. +pub fn hint_uri(location: &str) -> String { + format!("{location}/metadata/version-hint.text") +} + +/// The version in a metadata file's name: `v12.metadata.json` and `00012-.metadata.json` +/// are version 12. +pub fn metadata_version(uri: &str) -> Option { + let name = uri.rsplit('/').next()?.strip_suffix(".metadata.json")?; + match name.strip_prefix('v') { + Some(n) => n.parse().ok(), + None => name.split_once('-')?.0.parse().ok(), + } +} + +/// A table at one committed version. +#[derive(Clone, Debug)] +pub struct Table { + /// The table's root URI, without a trailing slash. + pub location: String, + /// The metadata version. + pub version: u64, + /// The metadata file this version was read from (the metadata log records it once a commit + /// replaces it). + pub metadata_file: String, + /// The metadata. + pub metadata: Metadata, + /// The live files (data and delete files), when known: read by [`Table::live_files`], or + /// derived by a commit. + pub files: Option>, +} + +impl Table { + /// A table at `version` with `metadata` from `v.metadata.json`, its files not read + /// yet. + pub fn new(location: impl Into, version: u64, metadata: Metadata) -> Self { + let location = location.into(); + Self { metadata_file: metadata_uri(&location, version), location, version, metadata, files: None } + } + + /// The newest metadata version of the table at `location`: the highest listed, or what + /// `version-hint.text` says where nothing is listed. + pub async fn latest_version(host: &H, location: &str) -> Result { + let listed = host.list(&format!("{location}/metadata/v")).await?; + if let Some(newest) = listed.iter().filter_map(|o| metadata_version(&o.uri)).max() { + return Ok(newest); + } + let hint = host.get(&hint_uri(location)).await?.ok_or_else(|| Error::NotFound(location.to_string()))?; + std::str::from_utf8(&hint) + .ok() + .and_then(|h| h.trim().parse().ok()) + .ok_or_else(|| crate::corrupt(format!("{}: not a version", hint_uri(location)))) + } + + /// The newest version of the table at `location`. + pub async fn load(host: &H, location: &str) -> Result { + let version = Self::latest_version(host, location).await?; + Self::load_version(host, location, version).await + } + + /// Version `version` of the table at `location`. + pub async fn load_version(host: &H, location: &str, version: u64) -> Result { + let uri = metadata_uri(location, version); + let bytes = host.get(&uri).await?.ok_or_else(|| Error::NotFound(uri.clone()))?; + Ok(Self::new(location, version, Metadata::parse(&bytes)?)) + } + + /// The table whose metadata file is `uri` (a catalog's pointer), at the location the metadata + /// records. + pub async fn from_metadata_file(host: &H, uri: &str) -> Result { + let bytes = host.get(uri).await?.ok_or_else(|| Error::NotFound(uri.to_string()))?; + let metadata = Metadata::parse(&bytes)?; + let location = metadata.location().trim_end_matches('/').to_string(); + let table = Self::new(location, metadata_version(uri).unwrap_or(0), metadata); + Ok(Self { metadata_file: uri.to_string(), ..table }) + } + + /// The current snapshot's manifests. + pub async fn manifests(&self, host: &H) -> Result> { + let Some(list) = self.metadata.manifest_list() else { + if self.metadata.current_snapshot().is_some() { + return Err(Error::Unsupported("format v1 snapshots without a manifest list".into())); + } + return Ok(vec![]); + }; + let bytes = host.get(&list).await?.ok_or_else(|| Error::NotFound(list.clone()))?; + read_manifest_list(&bytes) + } + + /// The entries of `manifest`. + pub async fn manifest(host: &H, manifest: &ManifestFile) -> Result> { + let bytes = host.get(&manifest.path).await?.ok_or_else(|| Error::NotFound(manifest.path.clone()))?; + read_manifest(manifest, &bytes) + } + + /// The live files (not deleted by the current snapshot), read from the manifests unless known. + pub async fn live_files(&mut self, host: &H) -> Result<&[Entry]> { + if self.files.is_none() { + let manifests = self.manifests(host).await?; + let read = futures_util::future::try_join_all(manifests.iter().map(|m| Self::manifest(host, m))).await?; + let files = read.into_iter().flatten().filter(|e| e.status != Status::Deleted).collect(); + self.files = Some(files); + } + Ok(self.files.as_deref().unwrap_or_default()) + } +} + +#[cfg(test)] +mod tests { + use lakebed_core::MemoryHost; + + use super::*; + use crate::block_on; + + #[test] + fn metadata_file_names() { + assert_eq!(metadata_version("s3://b/t/metadata/v12.metadata.json"), Some(12)); + assert_eq!(metadata_version("s3://b/t/metadata/00012-1b2c.metadata.json"), Some(12)); + assert_eq!(metadata_version("s3://b/t/metadata/version-hint.text"), None); + assert_eq!(metadata_version("s3://b/t/metadata/vx.metadata.json"), None); + } + + #[test] + fn loads_the_newest_version() { + let host = MemoryHost::new(0); + let meta = |n: u8| format!(r#"{{"format-version": 2, "location": "memory://t", "last-sequence-number": {n}}}"#); + block_on(async { + assert!(matches!(Table::load(&host, "memory://t").await, Err(Error::NotFound(_)))); + // Nothing listed: the hint names the version. + host.put(&hint_uri("memory://t"), b"2\n".to_vec()).await?; + assert_eq!(Table::latest_version(&host, "memory://t").await?, 2); + host.put(&metadata_uri("memory://t", 2), meta(2).into_bytes()).await?; + assert_eq!(Table::load(&host, "memory://t").await?.version, 2); + host.put(&metadata_uri("memory://t", 10), meta(10).into_bytes()).await?; + let mut t = Table::load(&host, "memory://t").await?; + assert_eq!((t.version, t.metadata.next_sequence_number()), (10, 11)); + assert!(t.live_files(&host).await?.is_empty(), "no snapshot, no files"); + let uri = "memory://t/metadata/00002-a.metadata.json"; + host.put(uri, meta(2).into_bytes()).await?; + let t = Table::from_metadata_file(&host, uri).await?; + assert_eq!((t.location.as_str(), t.version, t.metadata_file.as_str()), ("memory://t", 2, uri)); + Ok::<_, Error>(()) + }) + .unwrap(); + } +} diff --git a/crates/lakebed-iceberg/src/types.rs b/crates/lakebed-iceberg/src/types.rs new file mode 100644 index 0000000..f260851 --- /dev/null +++ b/crates/lakebed-iceberg/src/types.rs @@ -0,0 +1,317 @@ +//! Iceberg's types and schemas, read from table metadata JSON. +//! + +use std::fmt; +use std::str::FromStr; + +use serde_json::Value; + +use crate::{Error, Result, corrupt}; + +/// A primitive type. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum PrimitiveType { + /// `boolean`. + Boolean, + /// `int`: 32-bit signed. + Int, + /// `long`: 64-bit signed. + Long, + /// `float`: 32-bit IEEE 754. + Float, + /// `double`: 64-bit IEEE 754. + Double, + /// `decimal(P,S)`. + Decimal { + /// Digits in all. + precision: u8, + /// Digits after the point. + scale: u8, + }, + /// `date`: days since 1970-01-01. + Date, + /// `time`: microseconds since midnight. + Time, + /// `timestamp`: microseconds since the epoch, without a time zone. + Timestamp, + /// `timestamptz`: microseconds since the epoch, UTC. + Timestamptz, + /// `timestamp_ns`: nanoseconds since the epoch, without a time zone (format v3). + TimestampNs, + /// `timestamptz_ns`: nanoseconds since the epoch, UTC (format v3). + TimestamptzNs, + /// `string`: UTF-8. + String, + /// `uuid`. + Uuid, + /// `fixed[L]`: `L` bytes. + Fixed(u64), + /// `binary`: any number of bytes. + Binary, +} + +impl FromStr for PrimitiveType { + type Err = Error; + + fn from_str(s: &str) -> Result { + Ok(match s { + "boolean" => Self::Boolean, + "int" => Self::Int, + "long" => Self::Long, + "float" => Self::Float, + "double" => Self::Double, + "date" => Self::Date, + "time" => Self::Time, + "timestamp" => Self::Timestamp, + "timestamptz" => Self::Timestamptz, + "timestamp_ns" => Self::TimestampNs, + "timestamptz_ns" => Self::TimestamptzNs, + "string" => Self::String, + "uuid" => Self::Uuid, + "binary" => Self::Binary, + other => { + if let Some(n) = other.strip_prefix("fixed[").and_then(|s| s.strip_suffix(']')) { + Self::Fixed(n.trim().parse().map_err(|_| corrupt(format!("type {other}")))?) + } else if let Some(ps) = other.strip_prefix("decimal(").and_then(|s| s.strip_suffix(')')) { + let (p, s) = ps.split_once(',').ok_or_else(|| corrupt(format!("type {other}")))?; + let precision = p.trim().parse().map_err(|_| corrupt(format!("type {other}")))?; + let scale = s.trim().parse().map_err(|_| corrupt(format!("type {other}")))?; + Self::Decimal { precision, scale } + } else { + return Err(Error::Unsupported(format!("type {other}"))); + } + } + }) + } +} + +impl fmt::Display for PrimitiveType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Boolean => f.write_str("boolean"), + Self::Int => f.write_str("int"), + Self::Long => f.write_str("long"), + Self::Float => f.write_str("float"), + Self::Double => f.write_str("double"), + Self::Decimal { precision, scale } => write!(f, "decimal({precision}, {scale})"), + Self::Date => f.write_str("date"), + Self::Time => f.write_str("time"), + Self::Timestamp => f.write_str("timestamp"), + Self::Timestamptz => f.write_str("timestamptz"), + Self::TimestampNs => f.write_str("timestamp_ns"), + Self::TimestamptzNs => f.write_str("timestamptz_ns"), + Self::String => f.write_str("string"), + Self::Uuid => f.write_str("uuid"), + Self::Fixed(n) => write!(f, "fixed[{n}]"), + Self::Binary => f.write_str("binary"), + } + } +} + +/// Any type: a primitive, a nested type, or one this version doesn't model. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Type { + /// A primitive. + Primitive(PrimitiveType), + /// A struct of fields. + Struct(Vec), + /// A list. + List { + /// The element's field id. + element_id: i32, + /// Whether elements can't be null. + element_required: bool, + /// The element type. + element: Box, + }, + /// A map. + Map { + /// The key's field id. + key_id: i32, + /// The key type. + key: Box, + /// The value's field id. + value_id: i32, + /// Whether values can't be null. + value_required: bool, + /// The value type. + value: Box, + }, + /// A type this version doesn't model (format v3's `variant`, `geometry(…)`, `unknown`, …), by + /// its name. Columns of such types can be listed but not decoded. + Other(String), +} + +impl Type { + /// The type of a field's `type` JSON value. + pub fn from_json(t: &Value) -> Result { + if let Some(name) = t.as_str() { + return Ok(match name.parse() { + Ok(p) => Self::Primitive(p), + Err(Error::Unsupported(_)) => Self::Other(name.to_string()), + Err(e) => return Err(e), + }); + } + let id = |key: &str| { + t.get(key) + .and_then(Value::as_i64) + .and_then(|v| i32::try_from(v).ok()) + .ok_or_else(|| corrupt(format!("type without {key}"))) + }; + let child = |key: &str| Self::from_json(t.get(key).ok_or_else(|| corrupt(format!("type without {key}")))?); + let flag = |key: &str| t.get(key).and_then(Value::as_bool).unwrap_or(false); + match t.get("type").and_then(Value::as_str) { + Some("struct") => Ok(Self::Struct(fields(t)?)), + Some("list") => Ok(Self::List { + element_id: id("element-id")?, + element_required: flag("element-required"), + element: Box::new(child("element")?), + }), + Some("map") => Ok(Self::Map { + key_id: id("key-id")?, + key: Box::new(child("key")?), + value_id: id("value-id")?, + value_required: flag("value-required"), + value: Box::new(child("value")?), + }), + other => Err(corrupt(format!("type {other:?}"))), + } + } + + /// The primitive type of the field with id `id` inside this type. + fn primitive_by_id(&self, id: i32) -> Option<&PrimitiveType> { + match self { + Self::Struct(fields) => fields.iter().find_map(|f| f.primitive_by_id(id)), + Self::List { element_id, element, .. } => { + if *element_id == id { + element.as_primitive() + } else { + element.primitive_by_id(id) + } + } + Self::Map { key_id, key, value_id, value, .. } => { + if *key_id == id { + key.as_primitive() + } else if *value_id == id { + value.as_primitive() + } else { + key.primitive_by_id(id).or_else(|| value.primitive_by_id(id)) + } + } + Self::Primitive(_) | Self::Other(_) => None, + } + } + + /// The primitive, if this is one. + pub fn as_primitive(&self) -> Option<&PrimitiveType> { + match self { + Self::Primitive(p) => Some(p), + _ => None, + } + } +} + +/// A field of a struct or schema. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NestedField { + /// Its id, unique in the schema. + pub id: i32, + /// Its name. + pub name: String, + /// Whether values can't be null. + pub required: bool, + /// Its type. + pub ty: Type, + /// Its documentation, if any. + pub doc: Option, +} + +impl NestedField { + fn from_json(f: &Value) -> Result { + let name = f.get("name").and_then(Value::as_str).ok_or_else(|| corrupt("field without a name"))?; + let id = f.get("id").and_then(Value::as_i64).and_then(|v| i32::try_from(v).ok()); + Ok(Self { + id: id.ok_or_else(|| corrupt(format!("field {name} without an id")))?, + name: name.to_string(), + required: f.get("required").and_then(Value::as_bool).unwrap_or(false), + ty: Type::from_json(f.get("type").ok_or_else(|| corrupt(format!("field {name} without a type")))?)?, + doc: f.get("doc").and_then(Value::as_str).map(str::to_string), + }) + } + + fn primitive_by_id(&self, id: i32) -> Option<&PrimitiveType> { + if self.id == id { self.ty.as_primitive() } else { self.ty.primitive_by_id(id) } + } +} + +fn fields(t: &Value) -> Result> { + let fields = t.get("fields").and_then(Value::as_array).ok_or_else(|| corrupt("struct without fields"))?; + fields.iter().map(NestedField::from_json).collect() +} + +/// A table schema. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Schema { + /// Its id (0 for format v1 tables, which have one schema). + pub schema_id: i32, + /// Top-level fields. + pub fields: Vec, +} + +impl Schema { + /// A schema from its metadata JSON (`{"schema-id": 0, "type": "struct", "fields": [...]}`). + pub fn from_json(schema: &Value) -> Result { + let id = schema.get("schema-id").and_then(Value::as_i64).unwrap_or(0); + Ok(Self { schema_id: i32::try_from(id).map_err(|_| corrupt("schema id"))?, fields: fields(schema)? }) + } + + /// The top-level field named `name`. + pub fn field_by_name(&self, name: &str) -> Option<&NestedField> { + self.fields.iter().find(|f| f.name == name) + } + + /// The primitive type of the field (at any depth) with id `id`: the type of its bounds in + /// manifests. + pub fn primitive_by_id(&self, id: i32) -> Option<&PrimitiveType> { + self.fields.iter().find_map(|f| f.primitive_by_id(id)) + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn type_names_round_trip() { + for name in ["boolean", "decimal(10, 2)", "fixed[16]", "timestamptz_ns", "uuid", "binary"] { + assert_eq!(name.parse::().unwrap().to_string(), name); + } + assert_eq!("decimal(9,3)".parse::().unwrap(), PrimitiveType::Decimal { precision: 9, scale: 3 }); + assert!(matches!("variant".parse::(), Err(Error::Unsupported(_)))); + assert!(matches!("fixed[x]".parse::(), Err(Error::Corrupt(_)))); + } + + #[test] + fn schemas_find_nested_field_types() { + let schema = Schema::from_json(&json!({ + "schema-id": 3, "type": "struct", "fields": [ + {"id": 1, "name": "time", "required": true, "type": "timestamptz"}, + {"id": 2, "name": "tags", "required": false, "type": + {"type": "map", "key-id": 4, "key": "string", "value-id": 5, "value-required": false, "value": "long"}}, + {"id": 3, "name": "points", "required": false, "type": + {"type": "list", "element-id": 6, "element-required": true, "element": + {"type": "struct", "fields": [{"id": 7, "name": "x", "required": true, "type": "double"}]}}}, + {"id": 8, "name": "v", "required": false, "type": "variant"} + ] + })) + .unwrap(); + assert_eq!(schema.schema_id, 3); + assert_eq!(schema.primitive_by_id(1), Some(&PrimitiveType::Timestamptz)); + assert_eq!(schema.primitive_by_id(5), Some(&PrimitiveType::Long)); + assert_eq!(schema.primitive_by_id(7), Some(&PrimitiveType::Double)); + assert_eq!(schema.primitive_by_id(6), None, "a struct element isn't primitive"); + assert_eq!(schema.field_by_name("v").unwrap().ty, Type::Other("variant".into())); + } +} diff --git a/crates/lakebed-iceberg/tests/iceberg_rust.rs b/crates/lakebed-iceberg/tests/iceberg_rust.rs new file mode 100644 index 0000000..2f90eda --- /dev/null +++ b/crates/lakebed-iceberg/tests/iceberg_rust.rs @@ -0,0 +1,296 @@ +//! lakebed-iceberg and iceberg-rust read each other's tables: a table iceberg-rust creates and +//! appends to, lakebed reads and commits to; iceberg-rust reads lakebed's commit and appends on top +//! of it; and lakebed computes the statistics iceberg-rust's Parquet writer records. Everything +//! runs in memory. +#![cfg(all(feature = "write", feature = "arrow", feature = "parquet-stats"))] +#![expect(clippy::unwrap_used, reason = "test code")] + +use std::collections::{BTreeMap, HashMap}; +use std::future::{Future, ready}; +use std::sync::Arc; + +use arrow_array::{Float64Array, RecordBatch, StringArray, TimestampMicrosecondArray}; +use bytes::Bytes; +use futures::TryStreamExt; +use iceberg::io::FileIO; +use iceberg::memory::MemoryCatalogBuilder; +use iceberg::spec::{ + DataFile as IceDataFile, FormatVersion, Literal, Manifest, ManifestList, NestedField, PrimitiveType as IceType, + Schema as IceSchema, Struct, Transform as IceTransform, Type as IceFieldType, UnboundPartitionSpec, +}; +use iceberg::transaction::{ApplyTransactionAction, Transaction}; +use iceberg::writer::file_writer::{FileWriter as _, FileWriterBuilder, ParquetWriterBuilder}; +use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, NamespaceIdent, TableCreation, TableIdent}; +use lakebed_core::{Error, Host, Listed, MemoryHost, MemoryUpload}; +use lakebed_iceberg::stats::ParquetStats; +use lakebed_iceberg::{Change, CommitOptions, DataFile, Datum, Entry, NoHooks, Status, Table, commit}; +use parquet::arrow::ArrowWriter; +use parquet::file::metadata::ParquetMetaDataReader; +use parquet::file::properties::WriterProperties; + +/// Microseconds at 2026-01-01T00:00:00Z. +const T0: i64 = 1_767_225_600_000_000; +const DAY_US: i64 = 86_400_000_000; + +/// A [`Host`] over iceberg-rust's `FileIO`, so both see the same objects. Clock and randomness come +/// from a [`MemoryHost`]; listing isn't needed (tables load from their metadata file). +struct IoHost { + io: FileIO, + memory: MemoryHost, +} + +fn store(e: impl std::fmt::Display) -> Error { + Error::Store(e.to_string()) +} + +impl Host for IoHost { + type Upload = MemoryUpload; + + async fn get(&self, uri: &str) -> lakebed_core::Result>> { + if !self.io.exists(uri).await.map_err(store)? { + return Ok(None); + } + Ok(Some(self.io.new_input(uri).map_err(store)?.read().await.map_err(store)?.to_vec())) + } + + async fn get_range(&self, uri: &str, start: u64, len: u64) -> lakebed_core::Result> { + let bytes = self.get(uri).await?.ok_or_else(|| Error::NotFound(uri.to_string()))?; + let start = usize::try_from(start).map_err(store)?.min(bytes.len()); + let end = start.saturating_add(usize::try_from(len).map_err(store)?).min(bytes.len()); + Ok(bytes[start..end].to_vec()) + } + + async fn exists(&self, uri: &str) -> lakebed_core::Result { + self.io.exists(uri).await.map_err(store) + } + + async fn put(&self, uri: &str, bytes: Vec) -> lakebed_core::Result<()> { + self.io.new_output(uri).map_err(store)?.write(Bytes::from(bytes)).await.map_err(store) + } + + /// The tests run one writer at a time, so checking first is enough. + async fn put_if_absent(&self, uri: &str, bytes: Vec) -> lakebed_core::Result { + if self.exists(uri).await? { + return Ok(false); + } + self.put(uri, bytes).await.map(|()| true) + } + + fn upload(&self, _: &str) -> impl Future> { + ready(Err(Error::Unsupported("uploads".into()))) + } + + fn list(&self, _: &str) -> impl Future>> { + ready(Err(Error::Unsupported("listing".into()))) + } + + async fn delete(&self, uris: &[String]) -> lakebed_core::Result<()> { + for uri in uris { + self.io.delete(uri).await.map_err(store)?; + } + Ok(()) + } + + fn now_ms(&self) -> i64 { + self.memory.now_ms() + } + + fn random(&self) -> u64 { + self.memory.random() + } + + fn sleep(&self, ms: u64) -> impl Future { + self.memory.sleep(ms) + } +} + +fn ice_schema() -> IceSchema { + IceSchema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "time", IceFieldType::Primitive(IceType::Timestamptz)).into(), + NestedField::required(2, "device", IceFieldType::Primitive(IceType::String)).into(), + NestedField::optional(3, "value", IceFieldType::Primitive(IceType::Double)).into(), + ]) + .build() + .unwrap() +} + +/// Rows of one day: `n` readings from `day` days after `T0`, every seventh value NaN. +fn batch(arrow: &arrow_schema::SchemaRef, day: i64, n: i64) -> RecordBatch { + let start = T0 + day * DAY_US; + let time = + TimestampMicrosecondArray::from_iter_values((0..n).map(|i| start + i * 1_000_000)).with_timezone("+00:00"); + let device = StringArray::from_iter_values((0..n).map(|i| format!("d{}", i % 5))); + let value: Float64Array = (0..n) + .map(|i| { + if i % 7 == 0 { + Some(f64::NAN) + } else if i % 11 == 0 { + None + } else { + Some(f64::from(i32::try_from(i).unwrap())) + } + }) + .collect(); + RecordBatch::try_new(arrow.clone(), vec![Arc::new(time), Arc::new(device), Arc::new(value)]).unwrap() +} + +fn nan_counts(batch: &RecordBatch) -> BTreeMap { + let values = batch.column(2).as_any().downcast_ref::().unwrap(); + BTreeMap::from([(3, values.iter().filter(|v| v.is_some_and(f64::is_nan)).count() as u64)]) +} + +/// Writes `batch` with iceberg-rust's Parquet writer; returns its data file. +async fn write_theirs(table: &iceberg::table::Table, batch: &RecordBatch, name: &str, day: i32) -> IceDataFile { + let builder = ParquetWriterBuilder::new(WriterProperties::default(), table.metadata().current_schema().clone()); + let uri = format!("{}/data/{name}.parquet", table.metadata().location()); + let mut writer = builder.build(table.file_io().new_output(&uri).unwrap()).await.unwrap(); + writer.write(batch).await.unwrap(); + let mut files = writer.close().await.unwrap(); + files.remove(0).partition(Struct::from_iter([Some(Literal::int(day))])).partition_spec_id(0).build().unwrap() +} + +/// Writes `batch` with arrow-rs's Parquet writer; returns lakebed's data file. +async fn write_ours(host: &IoHost, table: &Table, batch: &RecordBatch, name: &str, day: i32) -> DataFile { + let mut bytes = vec![]; + let mut writer = ArrowWriter::try_new(&mut bytes, batch.schema(), None).unwrap(); + writer.write(batch).unwrap(); + let footer = writer.close().unwrap(); + let stats = ParquetStats::new(&footer, bytes.len() as u64, nan_counts(batch)); + let uri = format!("{}/data/{name}.parquet", table.location); + host.put(&uri, bytes).await.unwrap(); + stats.data_file(uri, vec![Some(Datum::Date(day))]) +} + +async fn append_theirs( + catalog: &MemoryCatalog, + table: &iceberg::table::Table, + file: IceDataFile, +) -> iceberg::table::Table { + let tx = Transaction::new(table); + let tx = tx.fast_append().add_data_files([file]).apply(tx).unwrap(); + tx.commit(catalog).await.unwrap() +} + +/// The data file iceberg-rust records, as lakebed records it. +fn as_ours(f: &IceDataFile) -> DataFile { + let bytes = + |m: &HashMap| m.iter().map(|(k, d)| (*k, d.to_bytes().unwrap().to_vec())).collect(); + let counts = |m: &HashMap| m.iter().map(|(k, v)| (*k, *v)).collect(); + let partition = f + .partition() + .iter() + .map(|v| match v { + Some(Literal::Primitive(iceberg::spec::PrimitiveLiteral::Int(d))) => Some(Datum::Date(*d)), + other => panic!("partition value {other:?}"), + }) + .collect(); + DataFile { + path: f.file_path().to_string(), + partition, + record_count: f.record_count(), + file_size: f.file_size_in_bytes(), + column_sizes: counts(f.column_sizes()), + value_counts: counts(f.value_counts()), + null_counts: counts(f.null_value_counts()), + nan_counts: counts(f.nan_value_counts()), + lower_bounds: bytes(f.lower_bounds()), + upper_bounds: bytes(f.upper_bounds()), + split_offsets: f.split_offsets().map(<[i64]>::to_vec), + sort_order_id: f.sort_order_id(), + ..DataFile::default() + } +} + +/// Files iceberg-rust plans for a scan of the table at `metadata_file`. +async fn planned(io: &FileIO, metadata_file: &str) -> Vec { + let ident = TableIdent::from_strs(["ns", "static"]).unwrap(); + let table = iceberg::table::StaticTable::from_metadata_file(metadata_file, ident, io.clone()).await.unwrap(); + let tasks: Vec<_> = table.scan().build().unwrap().plan_files().await.unwrap().try_collect().await.unwrap(); + let mut paths: Vec = tasks.into_iter().map(|t| t.data_file_path).collect(); + paths.sort(); + paths +} + +async fn live(host: &IoHost, metadata_file: &str) -> (Table, Vec) { + let mut table = Table::from_metadata_file(host, metadata_file).await.unwrap(); + let mut files = table.live_files(host).await.unwrap().to_vec(); + files.sort_by(|a, b| a.file.path.cmp(&b.file.path)); + (table, files) +} + +#[tokio::test] +async fn tables_round_trip_between_lakebed_and_iceberg_rust() { + let catalog = MemoryCatalogBuilder::default() + .load("memory", HashMap::from([("warehouse".to_string(), "memory://wh".to_string())])) + .await + .unwrap(); + let ns = NamespaceIdent::new("ns".into()); + catalog.create_namespace(&ns, HashMap::new()).await.unwrap(); + let mut creation = TableCreation::builder() + .name("t".into()) + .location("memory://wh/ns/t".into()) + .schema(ice_schema()) + .format_version(FormatVersion::V2) + .build(); + creation.partition_spec = + Some(UnboundPartitionSpec::builder().add_partition_field(1, "time_day", IceTransform::Day).unwrap().build()); + let table = catalog.create_table(&ns, creation).await.unwrap(); + let io = table.file_io().clone(); + let host = IoHost { io: io.clone(), memory: MemoryHost::new(T0 / 1000) }; + let arrow = Arc::new(iceberg::arrow::schema_to_arrow_schema(table.metadata().current_schema()).unwrap()); + + // iceberg-rust appends; lakebed reads the file as iceberg-rust records it. + let first = batch(&arrow, 0, 5_000); + let theirs = write_theirs(&table, &first, "theirs-1", 20_454).await; + let table = append_theirs(&catalog, &table, theirs.clone()).await; + let (base, files) = live(&host, table.metadata_location().unwrap()).await; + assert_eq!(files.len(), 1); + assert_eq!(*files[0].file, as_ours(&theirs)); + assert_eq!((files[0].status, files[0].sequence_number, files[0].file_sequence_number), (Status::Added, 1, 1)); + + // lakebed computes the statistics iceberg-rust's writer recorded, from the file's footer. + let bytes = io.new_input(theirs.file_path()).unwrap().read().await.unwrap(); + let footer = ParquetMetaDataReader::new().parse_and_finish(&bytes).unwrap(); + let stats = ParquetStats::new(&footer, bytes.len() as u64, nan_counts(&first)); + assert_eq!(stats.data_file(theirs.file_path().into(), vec![Some(Datum::Date(20_454))]), as_ours(&theirs)); + + // lakebed commits on top; iceberg-rust reads the new version and lakebed's manifests. + let ours = write_ours(&host, &base, &batch(&arrow, 1, 3_000), "ours-1", 20_455).await; + let (committed, info) = commit( + &host, + base, + &Change::Append { files: vec![ours.clone()] }, + &HashMap::new(), + &CommitOptions::default(), + &mut NoHooks, + ) + .await + .unwrap(); + assert_eq!(info.version, 2); + let paths = planned(&io, &committed.metadata_file).await; + assert_eq!(paths, [ours.path.clone(), theirs.file_path().to_string()]); + let list = io.new_input(committed.metadata.manifest_list().unwrap()).unwrap().read().await.unwrap(); + let list = ManifestList::parse_with_version(&list, FormatVersion::V2).unwrap(); + let added = list.entries().iter().find(|m| m.added_snapshot_id == info.snapshot_id).unwrap(); + assert_eq!((added.sequence_number, added.added_files_count), (2, Some(1))); + let manifest = io.new_input(&added.manifest_path).unwrap().read().await.unwrap(); + let (entries, _) = Manifest::parse_avro(&manifest).unwrap().into_parts(); + assert_eq!(as_ours(&entries[0].data_file), ours); + + // iceberg-rust appends on top of lakebed's version; lakebed reads all three files. Its memory + // catalog only names metadata files `NNNNN-.metadata.json`, so it gets a copy so named. + let ident = TableIdent::from_strs(["ns", "registered"]).unwrap(); + let copy = + committed.metadata_file.replace("v2.metadata.json", "00002-1d5e8c1a-0000-4000-8000-000000000000.metadata.json"); + host.put(©, committed.metadata.to_json().into_bytes()).await.unwrap(); + let registered = catalog.register_table(&ident, copy).await.unwrap(); + let again = write_theirs(®istered, &batch(&arrow, 2, 1_000), "theirs-2", 20_456).await; + let registered = append_theirs(&catalog, ®istered, again.clone()).await; + let (_, files) = live(&host, registered.metadata_location().unwrap()).await; + let seen: Vec<(&str, i64)> = + files.iter().map(|e| (e.file.path.rsplit('/').next().unwrap(), e.sequence_number)).collect(); + assert_eq!(seen, [("ours-1.parquet", 2), ("theirs-1.parquet", 1), ("theirs-2.parquet", 3)]); + assert_eq!(*files[2].file, as_ours(&again)); +} diff --git a/crates/lakebed/Cargo.toml b/crates/lakebed/Cargo.toml new file mode 100644 index 0000000..766e29c --- /dev/null +++ b/crates/lakebed/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "lakebed" +description = "Table formats (Apache Iceberg first) read and written without I/O of their own, for WebAssembly and native" +readme = "../../README.md" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true + +[features] +default = ["iceberg"] +# Apache Iceberg (lakebed-iceberg). +iceberg = ["dep:lakebed-iceberg"] +# Writing Iceberg tables. +iceberg-write = ["iceberg", "lakebed-iceberg/write"] +# Iceberg schemas as Arrow schemas. +iceberg-arrow = ["iceberg", "lakebed-iceberg/arrow"] +# Iceberg statistics from Parquet footers. +iceberg-parquet-stats = ["iceberg", "lakebed-iceberg/parquet-stats"] + +[dependencies] +lakebed-core.workspace = true +lakebed-iceberg = { workspace = true, optional = true } + +[package.metadata.docs.rs] +all-features = true + +[lints] +workspace = true diff --git a/crates/lakebed/src/lib.rs b/crates/lakebed/src/lib.rs new file mode 100644 index 0000000..1380306 --- /dev/null +++ b/crates/lakebed/src/lib.rs @@ -0,0 +1,11 @@ +//! Table formats read and written without I/O of their own, for WebAssembly and native: one crate +//! per format, re-exported here behind a feature each. The I/O boundary is [`core::Host`]. +//! +//! | Feature | Crate | +//! |---|---| +//! | `iceberg` (default) | [`iceberg`]: Apache Iceberg (`lakebed-iceberg`) | +//! | `iceberg-write`, `iceberg-arrow`, `iceberg-parquet-stats` | its optional parts | + +pub use lakebed_core as core; +#[cfg(feature = "iceberg")] +pub use lakebed_iceberg as iceberg; diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..7533fe2 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.99" +profile = "minimal" +components = ["rustfmt", "clippy"] diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000..f0ce655 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,3 @@ +style_edition = "2024" +max_width = 120 +use_small_heuristics = "Max" diff --git a/spec/0001-host.md b/spec/0001-host.md new file mode 100644 index 0000000..426269b --- /dev/null +++ b/spec/0001-host.md @@ -0,0 +1,50 @@ +--- +number: 1 +title: The host boundary +status: done +updated: 2026-10-20 +versions: [unreleased] +related: [2, 3] +issues: [] +--- + +# 1. The host boundary + +**Done** · updated 2026-10-20 · versions unreleased · milestone: [0.1](milestones/v0-1.md) + +## Summary + +Format crates do no I/O of their own. They call a `Host` (`lakebed-core`) that the application implements over its +object store, clock, and randomness, so the same code runs in a browser, a Cloudflare Worker, a CLI, or a server, and +tests run against memory. + +## Design + +- **`Host`:** `get` (the object or `None`), `get_range`, `exists`, `put`, `put_if_absent` (the commit primitive: + `false` when an object is there), `upload` (a multipart upload: parts in order, then complete or abort), `list` (by + prefix), `delete` (missing objects are not an error), `now_ms`, `random`, and `sleep`. Paths are absolute URIs, as + table metadata records them. +- **Futures aren't `Send`.** Methods return `impl Future`, without a `Send` bound, so a host can wrap JavaScript + promises (wasm-bindgen). Native hosts are free to be `Send`. +- **`MemoryHost`:** objects in memory, a clock that only moves when told (or when `sleep` is called), and a seeded + random sequence, so tests are deterministic. +- **`Error`:** one error type for all crates, with the variant saying what a caller can do: `Invalid`, `Unsupported`, + `Conflict` (retry on the newer version), `CommitUnknown` (the commit may have landed: delete nothing it wrote, load + the table again), `NotFound`, `Store`, and `Corrupt`. It is `non_exhaustive`. +- **Identifiers:** `uuid_v4` and `uuid_v7` from the host's randomness and clock, for file names. + +## Decisions + +| ID | Decision | Status | +|---|---|---| +| D1 | Format crates do no I/O: everything goes through a `Host` trait the application implements (the idea of delta-kernel-rs's `Engine`) | Accepted | +| D2 | Host futures aren't `Send`, so JavaScript hosts work; native applications lose nothing | Accepted | +| D3 | One error type across crates, whose variants say what a caller can do | Accepted | + +## Open questions and issues + +None open. + +## History + +- 2026-10-20: built, from Timelake's `timelake_format::host`. diff --git a/spec/0002-formats.md b/spec/0002-formats.md new file mode 100644 index 0000000..979a0b2 --- /dev/null +++ b/spec/0002-formats.md @@ -0,0 +1,50 @@ +--- +number: 2 +title: One crate per format +status: in-progress +updated: 2026-10-20 +versions: [unreleased] +related: [1, 3] +issues: [6] +--- + +# 2. One crate per format + +**In progress** · updated 2026-10-20 · versions unreleased · milestone: [0.1](milestones/v0-1.md) + +## Summary + +Each table format is its own crate, `lakebed-`, depending on `lakebed-core` and nothing of another format's. +The `lakebed` crate re-exports them behind a feature each. Iceberg is first (spec 3), because Timelake writes it and +lakeglass reads it; the formats lakeglass explores come next. + +## Design + +- **Layout:** `crates/lakebed-core` (spec 1), `crates/lakebed-`, and `crates/lakebed`, whose features are the + format names (`iceberg`) and their parts (`iceberg-write`, `iceberg-arrow`, `iceberg-parquet-stats`). +- **Formats as they are.** A format crate exposes its format's own model (Iceberg's snapshots and manifests, Delta's + log, Hudi's timeline) without a shared abstraction that would hide the differences. A common view, where an + application needs one (lakeglass's table explorer), is built on top of the format crates, not inside them. +- **Reading first.** A format crate reads every version of its format that is in use, and writes what an application + needs, starting with the version most readers support. +- **Interop is tested** against the format's reference implementation: it reads what we write, and we read what it + writes, offline. +- **Building for WebAssembly:** every crate builds for `wasm32-unknown-unknown` and is checked for `wasm32-wasip2`. +- **Next formats,** in the order lakeglass needs them: Delta Lake, then Apache Hudi, Apache Paimon, Lance, and + DuckLake. Each is proposed as its own spec before it is built. + +## Decisions + +| ID | Decision | Status | +|---|---|---| +| D4 | One repository with a crate per format and an umbrella crate with a feature each, not one crate with every format | Accepted | +| D5 | Format crates show their format as it is; a common view lives above them | Accepted | +| D6 | Arrow and Parquet follow the major version iceberg-rust's newest release uses, so applications using both build one copy | Accepted | + +## Open questions and issues + +- [#6](https://github.com/polythings/lakebed/issues/6) Which format comes after Iceberg, and in what order? + +## History + +- 2026-10-20: proposed and the layout built, with Iceberg as the first format. diff --git a/spec/0003-iceberg.md b/spec/0003-iceberg.md new file mode 100644 index 0000000..91b2dc8 --- /dev/null +++ b/spec/0003-iceberg.md @@ -0,0 +1,84 @@ +--- +number: 3 +title: Apache Iceberg +status: in-progress +updated: 2026-10-20 +versions: [unreleased] +related: [1, 2] +issues: [1, 2, 3, 4, 5] +--- + +# 3. Apache Iceberg + +**In progress** · updated 2026-10-20 · versions unreleased · milestone: [0.1](milestones/v0-1.md) + +## Summary + +`lakebed-iceberg` reads Iceberg tables of format versions 1 to 3 and writes format v2 tables, without I/O of its own +(spec 1). Timelake's WASM reader and writer use it, and lakeglass's table explorer will. It started as Timelake's +shared writer, made general. + +## Design + +**Built:** + +- **Types and values.** Iceberg's primitive types, nested types (struct, list, map), and types this version doesn't + model (format v3's `variant`, `geometry`, …) kept by name so such tables still read. `Datum` holds a value of any + primitive type, with Iceberg's single-value binary encoding both ways; a `long` or `double` bound stored in 4 bytes + (written before the column was promoted) widens. +- **Schemas and partition specs**, from metadata JSON: field lookup by id at any depth, transforms (`identity`, + `bucket[N]`, `truncate[W]`, `year`, `month`, `day`, `hour`, `void`, and unknown ones by name), and each partition + field's result type (`day` gives a `date`, as iceberg-rust types it). Format v1 specs without field ids get 1000 + and up. +- **Table metadata** (`Metadata`): kept as JSON, so fields this crate doesn't model survive a rewrite; typed views of + snapshots, ancestors, the current schema and spec (including format v1's `schema` and `partition-spec`), and + properties. Format versions above 3 are refused. +- **Manifest lists and manifests,** format v1 to v3: data and delete files (position, equality, and v3 deletion + vectors' `referenced_data_file`, `content_offset`, and `content_size_in_bytes`), column statistics, and partition + values typed from the manifest header's schema and spec. Entries inherit snapshot ids and sequence numbers from + their manifest, as the spec says. Format v1's `*_data_files_count` names are read. +- **Tables** (`Table`): the newest `v.metadata.json` by listing, or `version-hint.text` where nothing is listed; + any metadata file a catalog names (`from_metadata_file`); manifests and live files. +- **Writing (feature `write`):** format v2 data manifests and manifest lists in the layout iceberg-rust writes, and + metadata updates (`with_snapshot`, `without_snapshots`) made as iceberg-rust's builder makes them, with the metadata + log trimmed to `write.metadata.previous-versions-max`. Partition values of type uuid, fixed, and decimal are refused + for now. +- **Commits (feature `write`):** upload a snapshot's files, then write `v.metadata.json` only if absent; on a + lost race, reload and try again on the newer version, deleting the lost attempt's files; when the outcome is + unknown (a conditional put that failed and can't be read back), delete nothing. Appends, replacements that keep data + sequence numbers (compaction), and deletions; manifests merge once there would be more than `merge_manifests_at`. + Iceberg's summary counters are kept. A writer whose clock is behind the table's last update dates its commit at that + update, so the snapshot log stays in order (iceberg-rust refuses tables whose log isn't). +- **Options and hooks:** `CommitOptions` (`max_attempts` 10, `backoff_ms` 5–50 times the attempt, `merge_manifests_at` + 8, `write_version_hint` on) and `CommitHooks` (`reload`, `skip` for idempotent retries, `extra` for files and + summary properties of the application's own). Timelake's last-value index, batch ids, and head use them. +- **Arrow (feature `arrow`):** the schema as an Arrow schema with field ids under `PARQUET:field_id`. +- **Statistics (feature `parquet-stats`):** a data file's counts, bounds, and split offsets from its Parquet footer, as + iceberg-rust's writer records them; decimals stored as INT32 or INT64 become big-endian bounds. +- **Interop tests:** iceberg-rust creates a table and appends; lakebed reads it and commits; iceberg-rust reads that + version, parses lakebed's manifests, and appends on top; lakebed reads all of it. lakebed's statistics of + iceberg-rust's Parquet file equal what iceberg-rust recorded. + +**Not yet:** applying partition transforms to values, planning files from filters, writing format v3, writing delete +files, and catalogs (REST). Each has an issue. + +## Decisions + +| ID | Decision | Status | +|---|---|---| +| D7 | Read format v1 to v3; write format v2 first, the version every reader supports | Accepted | +| D8 | Table metadata stays JSON with typed views, so unknown fields survive rewrites | Accepted | +| D9 | Commits are extended through hooks (`CommitHooks`) rather than by copying the protocol | Accepted | + +## Open questions and issues + +- [#1](https://github.com/polythings/lakebed/issues/1) Apply partition transforms to values +- [#2](https://github.com/polythings/lakebed/issues/2) Plan files and row groups from filters +- [#3](https://github.com/polythings/lakebed/issues/3) Write uuid, fixed and decimal partition values +- [#4](https://github.com/polythings/lakebed/issues/4) Write format v3 tables +- [#5](https://github.com/polythings/lakebed/issues/5) Iceberg REST catalogs: here or in lakeglass? + +## History + +- 2026-10-20: built from Timelake's shared writer (`timelake-format`), made general: any partition spec, format v1 to + v3 reads, delete files, hooks, and options. diff --git a/spec/README.md b/spec/README.md new file mode 100644 index 0000000..d261de6 --- /dev/null +++ b/spec/README.md @@ -0,0 +1,43 @@ +# Specs + +lakebed's design, laid out like Timelake's and lakeglass's. Start here, people and agents alike. + +- **Specs** (`NNNN-name.md`): one feature each, with the same template ([TEMPLATE.md](TEMPLATE.md)). A spec says what + the feature does and why, what is still open (as GitHub issues), how far it is built, and the releases that changed + it. +- **Milestones** ([`milestones/`](milestones/README.md)): goals made of specs. The roadmap is the list of milestones. + +## Statuses + +| Status | Meaning | +|---|---| +| Done | Built and in effect; the spec describes how it works now. | +| In progress | Partly built: the spec says which parts are, and the open parts have issues. | +| Planned | Agreed or proposed, not built yet. | +| Dropped | Discussed and not done; kept so the reasons stay on record. | + +## Writing and changing a spec + +- **When:** anything beyond a small fix gets a spec, or a change to one, agreed before it is built. +- **New spec:** copy [TEMPLATE.md](TEMPLATE.md) to `NNNN-short-name.md` with the next free number (numbers are never + reused), and add it to a milestone and to the index below. +- **Front matter:** `number`, `title`, `status` (`done`, `in-progress`, `planned`, `dropped`), `updated`, `versions` + (the releases that changed it; `unreleased` until a release names it), and optionally `related` and `issues`. +- **Open questions** are GitHub issues (label `question`), listed under `issues`, not paragraphs in the spec. +- **Same commit:** when work lands, its spec's status, versions, and history change with the code and the changelog. + +## Index + +3 specs: 2 in progress, 1 done. Decisions D1–D9. + +| # | Spec | Status | Updated | +|---:|---|---|---| +| 1 | [The host boundary](0001-host.md) | Done | 2026-10-20 | +| 2 | [One crate per format](0002-formats.md) | In progress | 2026-10-20 | +| 3 | [Apache Iceberg](0003-iceberg.md) | In progress | 2026-10-20 | + +## Milestones + +| Milestone | Status | +|---|---| +| [0.1: Iceberg for Timelake and lakeglass](milestones/v0-1.md) | In progress | diff --git a/spec/TEMPLATE.md b/spec/TEMPLATE.md new file mode 100644 index 0000000..321ab22 --- /dev/null +++ b/spec/TEMPLATE.md @@ -0,0 +1,36 @@ +--- +number: 0 +title: Short title +status: planned +updated: YYYY-MM-DD +versions: [] +related: [] +issues: [] +--- + +# 0. Short title + +**Planned** · updated YYYY-MM-DD · milestone: [Name](milestones/slug.md) + +## Summary + +Two to four sentences: what this feature does, for whom, and why. + +## Design + +How it works, as precisely as needed to build or review it: types, options, interfaces, edge cases. Cite other specs +as "spec N", linked to their files. + +## Decisions + +| ID | Decision | Status | +|---|---|---| +| D0 | One sentence | Proposed | + +## Open questions and issues + +- [#N](https://github.com/polythings/lakebed/issues/N) Each open question or task is an issue. + +## History + +- YYYY-MM-DD: proposed. diff --git a/spec/milestones/README.md b/spec/milestones/README.md new file mode 100644 index 0000000..5f6ca11 --- /dev/null +++ b/spec/milestones/README.md @@ -0,0 +1,7 @@ +# Milestones + +The roadmap: goals made of specs. A milestone is done, in progress, planned, or exploring (an idea being looked into). + +| Milestone | Status | Specs | +|---|---|---| +| [0.1: Iceberg for Timelake and lakeglass](v0-1.md) | In progress | 1, 2, 3 | diff --git a/spec/milestones/v0-1.md b/spec/milestones/v0-1.md new file mode 100644 index 0000000..a249c4b --- /dev/null +++ b/spec/milestones/v0-1.md @@ -0,0 +1,15 @@ +# 0.1: Iceberg for Timelake and lakeglass + +**In progress** + +The first release: the host boundary and the Iceberg crate, enough for Timelake's WASM reader and writer to use +lakebed instead of their own Iceberg code, and for lakeglass to read Iceberg tables in the browser. + +| # | Spec | Status | +|---:|---|---| +| 1 | [The host boundary](../0001-host.md) | Done | +| 2 | [One crate per format](../0002-formats.md) | In progress | +| 3 | [Apache Iceberg](../0003-iceberg.md) | In progress | + +**Done when:** the crates are on crates.io ([#7](https://github.com/polythings/lakebed/issues/7)), Timelake depends +on them by version, and lakeglass's Iceberg reader is built on them. From ec633b4ef76a0b7bf662968c4414c3b9bdec5ab5 Mon Sep 17 00:00:00 2001 From: Oleksandr Litus Date: Tue, 20 Oct 2026 21:36:53 -0400 Subject: [PATCH 2/2] Iceberg: fixes from review File statistics record a bound or null count only when every row group gives it exactly (a row group of nulls needs no bounds), so a partial one can't make a reader skip matching rows; decimals stored as byte arrays get Iceberg's shortest bytes; a replace needs one sequence number per added file; the current snapshot can't be removed; the next sequence number saturates. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +- crates/lakebed-iceberg/Cargo.toml | 1 + crates/lakebed-iceberg/src/commit.rs | 17 ++++ crates/lakebed-iceberg/src/metadata.rs | 16 ++- crates/lakebed-iceberg/src/stats.rs | 131 ++++++++++++++++++++++--- spec/0003-iceberg.md | 8 +- 6 files changed, 154 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb1a983..6905974 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,8 @@ versions follow [Semantic Versioning](https://semver.org/). file-system tables with options (`CommitOptions`) and hooks (`CommitHooks`). A writer whose clock is behind the table's last update dates its commit at that update, so the snapshot log stays in order. - `lakebed-iceberg` features `arrow` (schemas as Arrow schemas with field ids) and `parquet-stats` (a data file's - Iceberg statistics from its Parquet footer). + Iceberg statistics from its Parquet footer; a bound or null count only when every row group gives it exactly, and + decimals as Iceberg's shortest big-endian bytes). - `lakebed`: re-exports the format crates behind a feature each. The Iceberg code started as Timelake's shared writer (`timelake-format`), made general. diff --git a/crates/lakebed-iceberg/Cargo.toml b/crates/lakebed-iceberg/Cargo.toml index ba6b738..946ec88 100644 --- a/crates/lakebed-iceberg/Cargo.toml +++ b/crates/lakebed-iceberg/Cargo.toml @@ -33,6 +33,7 @@ iceberg.workspace = true tokio.workspace = true parquet = { workspace = true, features = ["arrow"] } arrow-array = "58" +arrow-schema.workspace = true bytes = "1" futures = "0.3" diff --git a/crates/lakebed-iceberg/src/commit.rs b/crates/lakebed-iceberg/src/commit.rs index 89c8c40..eb3d396 100644 --- a/crates/lakebed-iceberg/src/commit.rs +++ b/crates/lakebed-iceberg/src/commit.rs @@ -171,6 +171,12 @@ pub async fn commit, S: BuildHasher>( hooks: &mut K, ) -> Result<(Table, CommitInfo), K::Error> { base.metadata.check_writable()?; + if let Change::Replace { added, sequence_numbers, .. } = change + && added.len() != sequence_numbers.len() + { + let (files, numbers) = (added.len(), sequence_numbers.len()); + return Err(Error::Invalid(format!("replace: {files} added files but {numbers} sequence numbers")).into()); + } let mut base = base; for attempt in 1..=options.max_attempts { if attempt > 1 { @@ -647,6 +653,17 @@ mod tests { assert_eq!((t.metadata.current_snapshot().unwrap().timestamp_ms, t.metadata.last_updated_ms()), (5_000, 5_000)); } + #[test] + fn a_replace_needs_a_sequence_number_per_added_file() { + let host = MemoryHost::new(1_000); + let (t, _) = run(&host, create(&host, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); + let removed = files_named(&t, "a.parquet"); + let change = + Change::Replace { removed, added: vec![file("b", 1, 5), file("c", 1, 5)], sequence_numbers: vec![1] }; + let err = try_run(&host, t, &change, &mut NoHooks); + assert!(matches!(err, Err(Error::Invalid(_))), "{err:?}"); + } + #[test] fn only_v2_tables_are_written() { let host = MemoryHost::new(1_000); diff --git a/crates/lakebed-iceberg/src/metadata.rs b/crates/lakebed-iceberg/src/metadata.rs index 1e66712..dd7464e 100644 --- a/crates/lakebed-iceberg/src/metadata.rs +++ b/crates/lakebed-iceberg/src/metadata.rs @@ -180,7 +180,7 @@ impl Metadata { /// The sequence number the next snapshot gets. pub fn next_sequence_number(&self) -> i64 { - self.0.get("last-sequence-number").and_then(Value::as_i64).unwrap_or(0) + 1 + self.0.get("last-sequence-number").and_then(Value::as_i64).unwrap_or(0).saturating_add(1) } /// The current schema's id. @@ -349,9 +349,13 @@ mod write { /// This metadata without the snapshots `ids`, as iceberg-rust's builder removes them: refs to /// them go, and the snapshot log keeps only what follows the last entry of a removed snapshot - /// (so it has no gaps). `previous` is recorded in the metadata log. + /// (so it has no gaps). `previous` is recorded in the metadata log. The current snapshot can't be + /// removed: the table would have none and read as empty. pub fn without_snapshots(&self, ids: &[i64], previous: &str, now_ms: i64) -> Result { self.check_writable()?; + if let Some(current) = self.current_snapshot_id().filter(|id| ids.contains(id)) { + return Err(crate::Error::Invalid(format!("removing the current snapshot {current}"))); + } let mut m = self.clone(); m.log_previous(previous, self.last_updated_ms())?; let obj = m.object()?; @@ -439,6 +443,12 @@ mod tests { assert!(matches!(m.check_writable(), Err(Error::Unsupported(_)))); } + #[test] + fn a_sequence_number_at_the_limit_saturates() { + let m = Metadata::from_json(json!({"format-version": 2, "last-sequence-number": i64::MAX})).unwrap(); + assert_eq!(m.next_sequence_number(), i64::MAX); + } + #[test] fn rejects_unknown_versions() { assert!(matches!(Metadata::parse(br#"{"format-version": 4}"#), Err(Error::Unsupported(_)))); @@ -468,6 +478,8 @@ mod tests { assert_eq!(m.as_json()["unknown-field"]["kept"], true); assert_eq!(m.as_json()["refs"]["main"]["snapshot-id"], 5); + let current = m.without_snapshots(&[1, 5], "v4.metadata.json", 23); + assert!(matches!(current, Err(Error::Invalid(_))), "the current snapshot stays"); let m = m.without_snapshots(&[1, 2], "v4.metadata.json", 23).unwrap(); assert!(!m.has_snapshot(1) && !m.has_snapshot(2) && m.has_snapshot(3)); let log: Vec = m.as_json()["snapshot-log"] diff --git a/crates/lakebed-iceberg/src/stats.rs b/crates/lakebed-iceberg/src/stats.rs index 7d913a9..a4801bf 100644 --- a/crates/lakebed-iceberg/src/stats.rs +++ b/crates/lakebed-iceberg/src/stats.rs @@ -3,7 +3,7 @@ //! the writer stored. use std::cmp::Ordering; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use parquet::basic::{LogicalType, Type as PhysicalType}; use parquet::file::metadata::{ParquetMetaData, RowGroupMetaData}; @@ -38,7 +38,9 @@ pub struct ParquetStats { impl ParquetStats { /// The statistics of a file of `file_size` bytes with footer `metadata`, and the NaN counts the - /// writer counted. + /// writer counted. A bound or null count is recorded only when every row group gives it exactly: + /// readers take them as covering the whole file, and one covering part of it would let them skip + /// a file that has matching rows. A row group of nulls only has no bounds and needs none. pub fn new(metadata: &ParquetMetaData, file_size: u64, nan_counts: BTreeMap) -> Self { let mut s = Self { record_count: u64::try_from(metadata.file_metadata().num_rows()).unwrap_or(0), @@ -47,28 +49,48 @@ impl ParquetStats { split_offsets: metadata.row_groups().iter().filter_map(RowGroupMetaData::file_offset).collect(), ..Self::default() }; + let (mut no_lower, mut no_upper, mut no_nulls) = (BTreeSet::new(), BTreeSet::new(), BTreeSet::new()); for group in metadata.row_groups() { for chunk in group.columns() { let Some(id) = field_id(chunk.column_descr()) else { continue }; + let values = u64::try_from(chunk.num_values()).unwrap_or(0); *s.column_sizes.entry(id).or_default() += u64::try_from(chunk.compressed_size()).unwrap_or(0); - *s.value_counts.entry(id).or_default() += u64::try_from(chunk.num_values()).unwrap_or(0); - let Some(stats) = chunk.statistics() else { continue }; - if let Some(nulls) = stats.null_count_opt() { - *s.null_counts.entry(id).or_default() += nulls; + *s.value_counts.entry(id).or_default() += values; + let Some(stats) = chunk.statistics() else { + no_lower.insert(id); + no_upper.insert(id); + no_nulls.insert(id); + continue; + }; + match stats.null_count_opt() { + Some(nulls) => *s.null_counts.entry(id).or_default() += nulls, + None => { + no_nulls.insert(id); + } } + let all_null = stats.null_count_opt() == Some(values); let descr = chunk.column_descr(); - if stats.min_is_exact() - && let Some(min) = stats.min_bytes_opt().and_then(|b| bound(b, stats, descr)) - { - keep(&mut s.lower_bounds, id, min, descr, Ordering::Less); + let min = stats.min_bytes_opt().filter(|_| stats.min_is_exact()).and_then(|b| bound(b, stats, descr)); + match min { + Some(min) => keep(&mut s.lower_bounds, id, min, descr, Ordering::Less), + None if !all_null => { + no_lower.insert(id); + } + None => {} } - if stats.max_is_exact() - && let Some(max) = stats.max_bytes_opt().and_then(|b| bound(b, stats, descr)) - { - keep(&mut s.upper_bounds, id, max, descr, Ordering::Greater); + let max = stats.max_bytes_opt().filter(|_| stats.max_is_exact()).and_then(|b| bound(b, stats, descr)); + match max { + Some(max) => keep(&mut s.upper_bounds, id, max, descr, Ordering::Greater), + None if !all_null => { + no_upper.insert(id); + } + None => {} } } } + s.lower_bounds.retain(|id, _| !no_lower.contains(id)); + s.upper_bounds.retain(|id, _| !no_upper.contains(id)); + s.null_counts.retain(|id, _| !no_nulls.contains(id)); s } @@ -108,14 +130,19 @@ fn keep(bounds: &mut BTreeMap>, id: i32, value: Vec, descr: &Co } /// A Parquet statistic's plain encoding as an Iceberg bound. They agree for booleans, ints, -/// longs, floats, doubles, dates, times, timestamps, strings, binary, and fixed; decimals stored as -/// INT32 or INT64 become big-endian bytes. INT96 has no Iceberg type. +/// longs, floats, doubles, dates, times, timestamps, strings, binary, and `fixed[L]`. Decimals +/// become Iceberg's shortest big-endian bytes, whether stored as INT32, INT64, or (big-endian, of +/// their column's width) as a byte array; one wider than 16 bytes has no bound. INT96 has no +/// Iceberg type. fn bound(plain: &[u8], stats: &Statistics, descr: &ColumnDescriptor) -> Option> { let decimal = matches!(descr.logical_type_ref(), Some(LogicalType::Decimal { .. })); match (stats, decimal) { (Statistics::Int96(_), _) => None, (Statistics::Int32(_), true) => Some(minimal_be(i128::from(i32::from_le_bytes(plain.try_into().ok()?)))), (Statistics::Int64(_), true) => Some(minimal_be(i128::from(i64::from_le_bytes(plain.try_into().ok()?)))), + (Statistics::FixedLenByteArray(_) | Statistics::ByteArray(_), true) => { + (!plain.is_empty() && plain.len() <= 16).then(|| minimal_be(from_be(plain))) + } _ => Some(plain.to_vec()), } } @@ -146,3 +173,75 @@ fn compare(a: &[u8], b: &[u8], descr: &ColumnDescriptor) -> Ordering { fn le(bytes: &[u8]) -> Option<[u8; N]> { bytes.try_into().ok() } + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use arrow_array::{ArrayRef, Decimal128Array, Int64Array, RecordBatch}; + use arrow_schema::{DataType, Field, Schema}; + use parquet::arrow::ArrowWriter; + use parquet::file::properties::WriterProperties; + + use super::*; + + fn field(name: &str, ty: DataType, id: i32) -> Field { + Field::new(name, ty, true).with_metadata(HashMap::from([("PARQUET:field_id".to_string(), id.to_string())])) + } + + /// A file of `columns` (field ids 1, 2, …) in row groups of `rows` rows; its footer. + fn write(columns: Vec<(Field, ArrayRef)>, rows: usize) -> ParquetMetaData { + let (fields, arrays): (Vec, Vec) = columns.into_iter().unzip(); + let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), arrays).unwrap(); + let props = WriterProperties::builder().set_max_row_group_row_count(Some(rows)).build(); + let mut writer = ArrowWriter::try_new(vec![], batch.schema(), Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap() + } + + #[test] + fn bounds_need_every_row_group() { + let metadata = write( + vec![ + (field("v", DataType::Int64, 1), Arc::new(Int64Array::from(vec![1, 2, 3, 4]))), + (field("n", DataType::Int64, 2), Arc::new(Int64Array::from(vec![Some(5), None, None, None]))), + ], + 2, + ); + let s = ParquetStats::new(&metadata, 0, BTreeMap::new()); + assert_eq!( + (s.lower_bounds[&1].as_slice(), s.upper_bounds[&1].as_slice()), + (&1i64.to_le_bytes()[..], &4i64.to_le_bytes()[..]) + ); + // The second row group is all nulls: it has no bounds and doesn't need any. + assert_eq!( + (s.lower_bounds[&2].as_slice(), s.upper_bounds[&2].as_slice()), + (&5i64.to_le_bytes()[..], &5i64.to_le_bytes()[..]) + ); + assert_eq!((s.null_counts[&1], s.null_counts[&2]), (0, 3)); + + // Without statistics for one row group, the others' bounds and nulls don't cover the file. + let mut builder = metadata.into_builder(); + let mut groups = builder.take_row_groups(); + let stripped = groups[1].columns().iter().map(|c| c.clone().into_builder().clear_statistics().build().unwrap()); + groups[1] = groups[1].clone().into_builder().set_column_metadata(stripped.collect()).build().unwrap(); + let s = ParquetStats::new(&builder.set_row_groups(groups).build(), 0, BTreeMap::new()); + assert!(s.lower_bounds.is_empty() && s.upper_bounds.is_empty() && s.null_counts.is_empty(), "{s:?}"); + assert_eq!(s.value_counts[&1], 4, "counts don't depend on statistics"); + } + + #[test] + fn decimal_byte_array_bounds_are_minimal() { + // decimal(20, 2) is a 9-byte fixed array in Parquet; Iceberg keeps the shortest bytes. + let values = Decimal128Array::from(vec![-150, 12_345]).with_precision_and_scale(20, 2).unwrap(); + let metadata = write(vec![(field("d", DataType::Decimal128(20, 2), 1), Arc::new(values))], 10); + let s = ParquetStats::new(&metadata, 0, BTreeMap::new()); + assert_eq!(s.lower_bounds[&1], minimal_be(-150)); + assert_eq!(s.upper_bounds[&1], minimal_be(12_345)); + assert_eq!( + Datum::from_bytes(&crate::PrimitiveType::Decimal { precision: 20, scale: 2 }, &s.lower_bounds[&1]).unwrap(), + Datum::Decimal { unscaled: -150, precision: 20, scale: 2 } + ); + } +} diff --git a/spec/0003-iceberg.md b/spec/0003-iceberg.md index 91b2dc8..0999099 100644 --- a/spec/0003-iceberg.md +++ b/spec/0003-iceberg.md @@ -41,12 +41,12 @@ shared writer, made general. any metadata file a catalog names (`from_metadata_file`); manifests and live files. - **Writing (feature `write`):** format v2 data manifests and manifest lists in the layout iceberg-rust writes, and metadata updates (`with_snapshot`, `without_snapshots`) made as iceberg-rust's builder makes them, with the metadata - log trimmed to `write.metadata.previous-versions-max`. Partition values of type uuid, fixed, and decimal are refused + log trimmed to `write.metadata.previous-versions-max`; the current snapshot can't be removed. Partition values of type uuid, fixed, and decimal are refused for now. - **Commits (feature `write`):** upload a snapshot's files, then write `v.metadata.json` only if absent; on a lost race, reload and try again on the newer version, deleting the lost attempt's files; when the outcome is unknown (a conditional put that failed and can't be read back), delete nothing. Appends, replacements that keep data - sequence numbers (compaction), and deletions; manifests merge once there would be more than `merge_manifests_at`. + sequence numbers (compaction, one sequence number per added file), and deletions; manifests merge once there would be more than `merge_manifests_at`. Iceberg's summary counters are kept. A writer whose clock is behind the table's last update dates its commit at that update, so the snapshot log stays in order (iceberg-rust refuses tables whose log isn't). - **Options and hooks:** `CommitOptions` (`max_attempts` 10, `backoff_ms` 5–50 times the attempt, `merge_manifests_at` @@ -54,7 +54,9 @@ shared writer, made general. summary properties of the application's own). Timelake's last-value index, batch ids, and head use them. - **Arrow (feature `arrow`):** the schema as an Arrow schema with field ids under `PARQUET:field_id`. - **Statistics (feature `parquet-stats`):** a data file's counts, bounds, and split offsets from its Parquet footer, as - iceberg-rust's writer records them; decimals stored as INT32 or INT64 become big-endian bounds. + iceberg-rust's writer records them; decimals become Iceberg's shortest big-endian bytes however Parquet stores them. + A bound or null count is recorded only when every row group gives it exactly (a row group of nulls only needs no + bounds): readers take them as covering the whole file, and a partial one would let them skip matching rows. - **Interop tests:** iceberg-rust creates a table and appends; lakebed reads it and commits; iceberg-rust reads that version, parses lakebed's manifests, and appends on top; lakebed reads all of it. lakebed's statistics of iceberg-rust's Parquet file equal what iceberg-rust recorded.