From a5db84b2fd591d591d20d3047055983a2274bdbb Mon Sep 17 00:00:00 2001 From: Oleksandr Litus Date: Tue, 20 Oct 2026 22:09:44 -0400 Subject: [PATCH 01/14] Rename the project to lakekit lakebed was hard to read. The crates are now lakekit-core, lakekit-iceberg, and lakekit, and the repository is polythings/lakekit (GitHub redirects the old name). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 8 ++-- AGENTS.md | 4 +- CHANGELOG.md | 12 +++--- CONTRIBUTING.md | 10 ++--- Cargo.lock | 12 +++--- Cargo.toml | 8 ++-- NOTICE | 6 +-- PRIOR_ART.md | 12 +++--- README.md | 16 +++---- .../{lakebed-core => lakekit-core}/Cargo.toml | 4 +- .../{lakebed-core => lakekit-core}/README.md | 4 +- .../src/error.rs | 0 .../src/host.rs | 0 .../{lakebed-core => lakekit-core}/src/lib.rs | 6 +-- .../src/memory.rs | 0 .../src/uuid.rs | 0 .../Cargo.toml | 4 +- .../README.md | 6 +-- .../src/arrow.rs | 0 .../src/avro.rs | 0 .../src/commit.rs | 4 +- .../src/datum.rs | 0 .../src/lib.rs | 10 ++--- .../src/manifest.rs | 0 .../src/metadata.rs | 0 .../src/partition.rs | 0 .../src/stats.rs | 0 .../src/table.rs | 4 +- .../src/types.rs | 0 .../tests/iceberg_rust.rs | 42 +++++++++---------- crates/{lakebed => lakekit}/Cargo.toml | 16 +++---- crates/{lakebed => lakekit}/src/lib.rs | 6 +-- spec/0001-host.md | 2 +- spec/0002-formats.md | 10 +++-- spec/0003-iceberg.md | 16 +++---- spec/README.md | 2 +- spec/TEMPLATE.md | 2 +- spec/milestones/v0-1.md | 4 +- 38 files changed, 116 insertions(+), 114 deletions(-) rename crates/{lakebed-core => lakekit-core}/Cargo.toml (80%) rename crates/{lakebed-core => lakekit-core}/README.md (82%) rename crates/{lakebed-core => lakekit-core}/src/error.rs (100%) rename crates/{lakebed-core => lakekit-core}/src/host.rs (100%) rename crates/{lakebed-core => lakekit-core}/src/lib.rs (87%) rename crates/{lakebed-core => lakekit-core}/src/memory.rs (100%) rename crates/{lakebed-core => lakekit-core}/src/uuid.rs (100%) rename crates/{lakebed-iceberg => lakekit-iceberg}/Cargo.toml (96%) rename crates/{lakebed-iceberg => lakekit-iceberg}/README.md (84%) rename crates/{lakebed-iceberg => lakekit-iceberg}/src/arrow.rs (100%) rename crates/{lakebed-iceberg => lakekit-iceberg}/src/avro.rs (100%) rename crates/{lakebed-iceberg => lakekit-iceberg}/src/commit.rs (99%) rename crates/{lakebed-iceberg => lakekit-iceberg}/src/datum.rs (100%) rename crates/{lakebed-iceberg => lakekit-iceberg}/src/lib.rs (92%) rename crates/{lakebed-iceberg => lakekit-iceberg}/src/manifest.rs (100%) rename crates/{lakebed-iceberg => lakekit-iceberg}/src/metadata.rs (100%) rename crates/{lakebed-iceberg => lakekit-iceberg}/src/partition.rs (100%) rename crates/{lakebed-iceberg => lakekit-iceberg}/src/stats.rs (100%) rename crates/{lakebed-iceberg => lakekit-iceberg}/src/table.rs (99%) rename crates/{lakebed-iceberg => lakekit-iceberg}/src/types.rs (100%) rename crates/{lakebed-iceberg => lakekit-iceberg}/tests/iceberg_rust.rs (90%) rename crates/{lakebed => lakekit}/Cargo.toml (62%) rename crates/{lakebed => lakekit}/src/lib.rs (70%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99d007e..d111120 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,10 +35,10 @@ jobs: env: RUSTFLAGS: -D warnings run: | - cargo check -p lakebed-iceberg --no-default-features - cargo check -p lakebed-iceberg --features write - cargo check -p lakebed-iceberg --features arrow - cargo check -p lakebed-iceberg --features parquet-stats + cargo check -p lakekit-iceberg --no-default-features + cargo check -p lakekit-iceberg --features write + cargo check -p lakekit-iceberg --features arrow + cargo check -p lakekit-iceberg --features parquet-stats wasm: name: WebAssembly (browser and WASI targets) diff --git a/AGENTS.md b/AGENTS.md index 5a7647d..34c8930 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,7 @@ # Agent instructions -lakebed: table formats (Apache Iceberg first) read and written in Rust without I/O of their own, for WebAssembly and -native targets. One crate per format (`crates/lakebed-`) on a shared I/O boundary (`crates/lakebed-core`). +lakekit: table formats (Apache Iceberg first) read and written in Rust without I/O of their own, for WebAssembly and +native targets. One crate per format (`crates/lakekit-`) on a shared I/O boundary (`crates/lakekit-core`). Timelake and lakeglass use it. ## Before changing code diff --git a/CHANGELOG.md b/CHANGELOG.md index 6905974..3cb0344 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,21 +1,21 @@ # Changelog -All notable changes to lakebed. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and +All notable changes to lakekit. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and versions follow [Semantic Versioning](https://semver.org/). ## Unreleased ### Added -- `lakebed-core`: the `Host` trait for object stores, clocks, and randomness; `MemoryHost`; one `Error` type. -- `lakebed-iceberg`: Iceberg table metadata, schemas, partition specs, manifest lists, and manifests read for format +- `lakekit-core`: the `Host` trait for object stores, clocks, and randomness; `MemoryHost`; one `Error` type. +- `lakekit-iceberg`: Iceberg table metadata, schemas, partition specs, manifest lists, and manifests read for format versions 1 to 3, including delete files and deletion vector references; values in Iceberg's single-value encoding. -- `lakebed-iceberg` feature `write`: format v2 manifests, manifest lists, and metadata updates, and commits to +- `lakekit-iceberg` feature `write`: format v2 manifests, manifest lists, and metadata updates, and commits to file-system tables with options (`CommitOptions`) and hooks (`CommitHooks`). A writer whose clock is behind the table's last update dates its commit at that update, so the snapshot log stays in order. -- `lakebed-iceberg` features `arrow` (schemas as Arrow schemas with field ids) and `parquet-stats` (a data file's +- `lakekit-iceberg` features `arrow` (schemas as Arrow schemas with field ids) and `parquet-stats` (a data file's Iceberg statistics from its Parquet footer; a bound or null count only when every row group gives it exactly, and decimals as Iceberg's shortest big-endian bytes). -- `lakebed`: re-exports the format crates behind a feature each. +- `lakekit`: re-exports the format crates behind a feature each. The Iceberg code started as Timelake's shared writer (`timelake-format`), made general. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c3fbea5..c15535a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Contributing to lakebed +# Contributing to lakekit This is the code guide for everyone who changes this repository, people and AI agents alike. Formatters and linters enforce the mechanical rules (see [Checks](#checks)). This guide covers what they can't: how code should read. @@ -8,9 +8,9 @@ enforce the mechanical rules (see [Checks](#checks)). This guide covers what the 1. **Readable first.** Someone new should understand a function from its name, its signature, and one pass through the body. -2. **One crate per format.** A format's crate depends on `lakebed-core` and on nothing of another format's, so adding +2. **One crate per format.** A format's crate depends on `lakekit-core` and on nothing of another format's, so adding one never touches the others. Formats are shown as they are: no shared model flattens their differences. -3. **No I/O of our own.** Crates read and write through `lakebed_core::Host`. They never open sockets, files, or +3. **No I/O of our own.** Crates read and write through `lakekit_core::Host`. They never open sockets, files, or threads, so they build for `wasm32-unknown-unknown`. 4. **One definition of each thing.** Before writing a helper, constant, or type, look for an existing one. 5. **Behaviour, spec, and tests change together.** A change users can see updates its spec, adds or updates tests, and @@ -28,7 +28,7 @@ enforce the mechanical rules (see [Checks](#checks)). This guide covers what the ## Credit -lakebed learns from other projects' ideas, never from copying their code. [PRIOR_ART.md](PRIOR_ART.md) lists the +lakekit learns from other projects' ideas, never from copying their code. [PRIOR_ART.md](PRIOR_ART.md) lists the ideas we took. - **Never copy code**, whatever its license: not by pasting, and not by translating it into another language. @@ -38,7 +38,7 @@ ideas we took. ## Rust -- **Errors:** `lakebed_core::Error`, with the variant that says what a caller can do about it (`NotFound`, `Conflict`, +- **Errors:** `lakekit_core::Error`, with the variant that says what a caller can do about it (`NotFound`, `Conflict`, `CommitUnknown`, `Unsupported`, `Corrupt`). No `unwrap()` outside tests. - **Types over strings:** enums and newtypes instead of strings and magic numbers. Parse once at the boundary. - **Don't trust files.** Check lengths and offsets before using them, cap allocations, and return an error instead of diff --git a/Cargo.lock b/Cargo.lock index 963b662..0466307 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1596,22 +1596,22 @@ dependencies = [ ] [[package]] -name = "lakebed" +name = "lakekit" version = "0.1.0" dependencies = [ - "lakebed-core", - "lakebed-iceberg", + "lakekit-core", + "lakekit-iceberg", ] [[package]] -name = "lakebed-core" +name = "lakekit-core" version = "0.1.0" dependencies = [ "thiserror", ] [[package]] -name = "lakebed-iceberg" +name = "lakekit-iceberg" version = "0.1.0" dependencies = [ "apache-avro 0.22.0", @@ -1621,7 +1621,7 @@ dependencies = [ "futures", "futures-util", "iceberg", - "lakebed-core", + "lakekit-core", "parquet", "serde_json", "tokio", diff --git a/Cargo.toml b/Cargo.toml index c32eb9e..b7cdc73 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,19 +1,19 @@ [workspace] resolver = "2" -members = ["crates/lakebed-core", "crates/lakebed-iceberg", "crates/lakebed"] +members = ["crates/lakekit-core", "crates/lakekit-iceberg", "crates/lakekit"] [workspace.package] version = "0.1.0" edition = "2024" rust-version = "1.99" license = "Apache-2.0" -repository = "https://github.com/polythings/lakebed" +repository = "https://github.com/polythings/lakekit" keywords = ["lakehouse", "iceberg", "wasm", "table-format"] categories = ["database-implementations", "wasm"] [workspace.dependencies] -lakebed-core = { path = "crates/lakebed-core", version = "0.1.0" } -lakebed-iceberg = { path = "crates/lakebed-iceberg", version = "0.1.0", default-features = false } +lakekit-core = { path = "crates/lakekit-core", version = "0.1.0" } +lakekit-iceberg = { path = "crates/lakekit-iceberg", version = "0.1.0", default-features = false } thiserror = "2" serde_json = "1" apache-avro = { version = "0.22", default-features = false } diff --git a/NOTICE b/NOTICE index 0ce932b..e3cf2bc 100644 --- a/NOTICE +++ b/NOTICE @@ -1,6 +1,6 @@ -lakebed -Copyright 2026 Oleksandr Litus and the lakebed contributors +lakekit +Copyright 2026 Oleksandr Litus and the lakekit contributors This product is licensed under the Apache License, Version 2.0 (see LICENSE). -Ideas lakebed borrowed from other projects are credited in PRIOR_ART.md. +Ideas lakekit borrowed from other projects are credited in PRIOR_ART.md. diff --git a/PRIOR_ART.md b/PRIOR_ART.md index 7622547..da7a0db 100644 --- a/PRIOR_ART.md +++ b/PRIOR_ART.md @@ -1,11 +1,11 @@ # Prior art -lakebed stands on other open-source projects and learns from more. This file credits the ideas we took. We take ideas, -never code: every line of lakebed is our own. How we credit is in [CONTRIBUTING.md](CONTRIBUTING.md#credit). +lakekit stands on other open-source projects and learns from more. This file credits the ideas we took. We take ideas, +never code: every line of lakekit is our own. How we credit is in [CONTRIBUTING.md](CONTRIBUTING.md#credit). ## Foundations -| Project | License | What lakebed uses | +| Project | License | What lakekit uses | |---|---|---| | [Apache Iceberg](https://iceberg.apache.org) | Apache-2.0 | The table format and its specification | | [iceberg-rust](https://github.com/apache/iceberg-rust) | Apache-2.0 | The reference the interop tests check against (a test dependency only) | @@ -14,9 +14,9 @@ never code: every line of lakebed is our own. How we credit is in [CONTRIBUTING. ## Ideas -| Idea | From | License of the source | Where in lakebed | +| Idea | From | License of the source | Where in lakekit | |---|---|---|---| -| A format library with no I/O of its own: the application supplies storage through a trait the library calls | [delta-kernel-rs](https://github.com/delta-io/delta-kernel-rs) (its `Engine` trait) | Apache-2.0 | `lakebed-core`'s `Host` | +| A format library with no I/O of its own: the application supplies storage through a trait the library calls | [delta-kernel-rs](https://github.com/delta-io/delta-kernel-rs) (its `Engine` trait) | Apache-2.0 | `lakekit-core`'s `Host` | | One crate per backend under a small shared core, re-exported by an umbrella crate with a feature each | [arrow-rs](https://github.com/apache/arrow-rs), [Apache OpenDAL](https://github.com/apache/opendal) | Apache-2.0 | The workspace layout | -| Commits to tables without a catalog: `v.metadata.json` written only if absent, `version-hint.text` for readers | Iceberg's file-system ("Hadoop") tables | Apache-2.0 | `lakebed_iceberg::commit` | +| Commits to tables without a catalog: `v.metadata.json` written only if absent, `version-hint.text` for readers | Iceberg's file-system ("Hadoop") tables | Apache-2.0 | `lakekit_iceberg::commit` | | Avro-safe names for partition fields (`_x` and the character's code in hex) | Iceberg's Java implementation | Apache-2.0 | Manifest writing | diff --git a/README.md b/README.md index c4a52da..13a5876 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -# lakebed +# lakekit Table formats read and written without I/O of their own, in Rust, for WebAssembly and native targets. Each format's metadata (Apache Iceberg first) is parsed and written by plain code that asks a `Host` for bytes: the application implements the host over its object store, so the same crate runs in a browser, a Cloudflare Worker, a CLI, or a server. -lakebed is what [Timelake](https://github.com/polythings/timelake) and [lakeglass](https://github.com/polythings/lakeglass) +lakekit is what [Timelake](https://github.com/polythings/timelake) and [lakeglass](https://github.com/polythings/lakeglass) share. The official Rust crates for these formats build on tokio and HTTP clients, which don't compile to `wasm32-unknown-unknown`. @@ -13,9 +13,9 @@ share. The official Rust crates for these formats build on tokio and HTTP client | Crate | What it does | |---|---| -| [`lakebed-core`](crates/lakebed-core) | The `Host` trait (ranged reads, listing, conditional writes, clock, randomness), errors, an in-memory host | -| [`lakebed-iceberg`](crates/lakebed-iceberg) | Apache Iceberg: metadata, manifests, and commits | -| [`lakebed`](crates/lakebed) | Re-exports the format crates, one feature each | +| [`lakekit-core`](crates/lakekit-core) | The `Host` trait (ranged reads, listing, conditional writes, clock, randomness), errors, an in-memory host | +| [`lakekit-iceberg`](crates/lakekit-iceberg) | Apache Iceberg: metadata, manifests, and commits | +| [`lakekit`](crates/lakekit) | Re-exports the format crates, one feature each | More formats get their own crate as they are added (spec 2). @@ -30,12 +30,12 @@ More formats get their own crate as they are added (spec 2). | Schema as an Arrow schema with field ids | Feature `arrow` | | A data file's Iceberg statistics from its Parquet footer | Feature `parquet-stats` | -What lakebed writes, iceberg-rust reads, and the other way round: the tests commit to one table with both in turn. +What lakekit writes, iceberg-rust reads, and the other way round: the tests commit to one table with both in turn. ```rust -use lakebed_iceberg::Table; +use lakekit_iceberg::Table; -async fn files(host: &H) -> lakebed_core::Result { +async fn files(host: &H) -> lakekit_core::Result { let mut table = Table::load(host, "s3://bucket/warehouse/db/events").await?; Ok(table.live_files(host).await?.len()) } diff --git a/crates/lakebed-core/Cargo.toml b/crates/lakekit-core/Cargo.toml similarity index 80% rename from crates/lakebed-core/Cargo.toml rename to crates/lakekit-core/Cargo.toml index b414a77..ec137de 100644 --- a/crates/lakebed-core/Cargo.toml +++ b/crates/lakekit-core/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "lakebed-core" -description = "The I/O boundary shared by lakebed's table format crates: a host trait for object stores, clocks, and randomness" +name = "lakekit-core" +description = "The I/O boundary shared by lakekit's table format crates: a host trait for object stores, clocks, and randomness" readme = "README.md" version.workspace = true edition.workspace = true diff --git a/crates/lakebed-core/README.md b/crates/lakekit-core/README.md similarity index 82% rename from crates/lakebed-core/README.md rename to crates/lakekit-core/README.md index 9d74b43..734e18c 100644 --- a/crates/lakebed-core/README.md +++ b/crates/lakekit-core/README.md @@ -1,6 +1,6 @@ -# lakebed-core +# lakekit-core -The I/O boundary of [lakebed](https://github.com/polythings/lakebed)'s table format crates. Format crates do no I/O of +The I/O boundary of [lakekit](https://github.com/polythings/lakekit)'s table format crates. Format crates do no I/O of their own: they call a `Host` that the application implements over its object store, clock, and randomness, so they build for `wasm32-unknown-unknown` and native targets alike. diff --git a/crates/lakebed-core/src/error.rs b/crates/lakekit-core/src/error.rs similarity index 100% rename from crates/lakebed-core/src/error.rs rename to crates/lakekit-core/src/error.rs diff --git a/crates/lakebed-core/src/host.rs b/crates/lakekit-core/src/host.rs similarity index 100% rename from crates/lakebed-core/src/host.rs rename to crates/lakekit-core/src/host.rs diff --git a/crates/lakebed-core/src/lib.rs b/crates/lakekit-core/src/lib.rs similarity index 87% rename from crates/lakebed-core/src/lib.rs rename to crates/lakekit-core/src/lib.rs index 5975f62..ddc231a 100644 --- a/crates/lakebed-core/src/lib.rs +++ b/crates/lakekit-core/src/lib.rs @@ -1,16 +1,16 @@ -//! The I/O boundary shared by lakebed's table format crates. Format crates do no I/O of their own: +//! The I/O boundary shared by lakekit's table format crates. Format crates do no I/O of their own: //! they read and write through a [`Host`] the application implements over its object store (an //! `object_store` client natively, `fetch` or an R2 binding in WebAssembly), so they build for //! `wasm32-unknown-unknown` and native targets alike. //! //! ``` -//! use lakebed_core::{Host, MemoryHost}; +//! use lakekit_core::{Host, MemoryHost}; //! //! # block_on(async { //! let host = MemoryHost::new(0); //! host.put("memory://t/a", b"x".to_vec()).await?; //! assert!(host.exists("memory://t/a").await?); -//! # Ok::<(), lakebed_core::Error>(()) +//! # Ok::<(), lakekit_core::Error>(()) //! # }).unwrap(); //! # // MemoryHost's futures are ready at once, so one poll runs them. //! # fn block_on(f: F) -> F::Output { diff --git a/crates/lakebed-core/src/memory.rs b/crates/lakekit-core/src/memory.rs similarity index 100% rename from crates/lakebed-core/src/memory.rs rename to crates/lakekit-core/src/memory.rs diff --git a/crates/lakebed-core/src/uuid.rs b/crates/lakekit-core/src/uuid.rs similarity index 100% rename from crates/lakebed-core/src/uuid.rs rename to crates/lakekit-core/src/uuid.rs diff --git a/crates/lakebed-iceberg/Cargo.toml b/crates/lakekit-iceberg/Cargo.toml similarity index 96% rename from crates/lakebed-iceberg/Cargo.toml rename to crates/lakekit-iceberg/Cargo.toml index 946ec88..b6f0d96 100644 --- a/crates/lakebed-iceberg/Cargo.toml +++ b/crates/lakekit-iceberg/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "lakebed-iceberg" +name = "lakekit-iceberg" description = "Apache Iceberg table metadata, manifests and commits without I/O of its own, for WebAssembly and native" readme = "README.md" version.workspace = true @@ -20,7 +20,7 @@ arrow = ["dep:arrow-schema"] parquet-stats = ["dep:parquet"] [dependencies] -lakebed-core.workspace = true +lakekit-core.workspace = true apache-avro.workspace = true serde_json.workspace = true futures-util.workspace = true diff --git a/crates/lakebed-iceberg/README.md b/crates/lakekit-iceberg/README.md similarity index 84% rename from crates/lakebed-iceberg/README.md rename to crates/lakekit-iceberg/README.md index cb493a8..efd7e77 100644 --- a/crates/lakebed-iceberg/README.md +++ b/crates/lakekit-iceberg/README.md @@ -1,7 +1,7 @@ -# lakebed-iceberg +# lakekit-iceberg [Apache Iceberg](https://iceberg.apache.org) tables read and written without I/O of their own, through a -[`lakebed_core::Host`](../lakebed-core), so they work in WebAssembly (browsers, Cloudflare Workers) as well as natively. +[`lakekit_core::Host`](../lakekit-core), so they work in WebAssembly (browsers, Cloudflare Workers) as well as natively. | Feature | What it adds | |---|---| @@ -10,6 +10,6 @@ | `arrow` | The schema as an Arrow schema, with field ids | | `parquet-stats` | A data file's Iceberg statistics (counts, bounds, split offsets) from its Parquet footer | -The tests commit to one table with lakebed and iceberg-rust in turn, each reading what the other wrote. +The tests commit to one table with lakekit and iceberg-rust in turn, each reading what the other wrote. Apache License 2.0. diff --git a/crates/lakebed-iceberg/src/arrow.rs b/crates/lakekit-iceberg/src/arrow.rs similarity index 100% rename from crates/lakebed-iceberg/src/arrow.rs rename to crates/lakekit-iceberg/src/arrow.rs diff --git a/crates/lakebed-iceberg/src/avro.rs b/crates/lakekit-iceberg/src/avro.rs similarity index 100% rename from crates/lakebed-iceberg/src/avro.rs rename to crates/lakekit-iceberg/src/avro.rs diff --git a/crates/lakebed-iceberg/src/commit.rs b/crates/lakekit-iceberg/src/commit.rs similarity index 99% rename from crates/lakebed-iceberg/src/commit.rs rename to crates/lakekit-iceberg/src/commit.rs index eb3d396..506fc8b 100644 --- a/crates/lakebed-iceberg/src/commit.rs +++ b/crates/lakekit-iceberg/src/commit.rs @@ -12,7 +12,7 @@ use std::future::Future; use std::hash::BuildHasher; use std::sync::Arc; -use lakebed_core::{Host, uuid_v4}; +use lakekit_core::{Host, uuid_v4}; use crate::manifest::{ DataFile, Entry, ManifestContext, ManifestFile, ManifestWrite, Status, manifest_file, write_manifest, @@ -469,7 +469,7 @@ pub fn counters(change: &Change, base: &HashMap(()) +//! # Ok::<(), lakekit_core::Error>(()) //! ``` #[cfg(feature = "arrow")] @@ -35,7 +35,7 @@ mod types; #[cfg(feature = "write")] pub use commit::{Change, CommitHooks, CommitInfo, CommitOptions, Extra, NoHooks, commit}; pub use datum::{Datum, from_be, minimal_be}; -pub use lakebed_core::{Error, Result}; +pub use lakekit_core::{Error, Result}; pub use manifest::{ Content, DataFile, Entry, FieldSummary, ManifestContent, ManifestFile, Status, read_manifest, read_manifest_list, }; @@ -76,7 +76,7 @@ fn corrupt(msg: impl Into) -> Error { Error::Corrupt(msg.into()) } -/// Runs a future whose host calls complete at once ([`lakebed_core::MemoryHost`]'s do). +/// Runs a future whose host calls complete at once ([`lakekit_core::MemoryHost`]'s do). #[cfg(test)] fn block_on(f: F) -> F::Output { let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); diff --git a/crates/lakebed-iceberg/src/manifest.rs b/crates/lakekit-iceberg/src/manifest.rs similarity index 100% rename from crates/lakebed-iceberg/src/manifest.rs rename to crates/lakekit-iceberg/src/manifest.rs diff --git a/crates/lakebed-iceberg/src/metadata.rs b/crates/lakekit-iceberg/src/metadata.rs similarity index 100% rename from crates/lakebed-iceberg/src/metadata.rs rename to crates/lakekit-iceberg/src/metadata.rs diff --git a/crates/lakebed-iceberg/src/partition.rs b/crates/lakekit-iceberg/src/partition.rs similarity index 100% rename from crates/lakebed-iceberg/src/partition.rs rename to crates/lakekit-iceberg/src/partition.rs diff --git a/crates/lakebed-iceberg/src/stats.rs b/crates/lakekit-iceberg/src/stats.rs similarity index 100% rename from crates/lakebed-iceberg/src/stats.rs rename to crates/lakekit-iceberg/src/stats.rs diff --git a/crates/lakebed-iceberg/src/table.rs b/crates/lakekit-iceberg/src/table.rs similarity index 99% rename from crates/lakebed-iceberg/src/table.rs rename to crates/lakekit-iceberg/src/table.rs index 7cadb70..a2a7df2 100644 --- a/crates/lakebed-iceberg/src/table.rs +++ b/crates/lakekit-iceberg/src/table.rs @@ -3,7 +3,7 @@ //! them, with `version-hint.text` naming the newest for stores that can't list. Tables a catalog //! names with other file names load with [`Table::from_metadata_file`]. -use lakebed_core::Host; +use lakekit_core::Host; use crate::manifest::{Entry, ManifestFile, Status, read_manifest, read_manifest_list}; use crate::metadata::Metadata; @@ -123,7 +123,7 @@ impl Table { #[cfg(test)] mod tests { - use lakebed_core::MemoryHost; + use lakekit_core::MemoryHost; use super::*; use crate::block_on; diff --git a/crates/lakebed-iceberg/src/types.rs b/crates/lakekit-iceberg/src/types.rs similarity index 100% rename from crates/lakebed-iceberg/src/types.rs rename to crates/lakekit-iceberg/src/types.rs diff --git a/crates/lakebed-iceberg/tests/iceberg_rust.rs b/crates/lakekit-iceberg/tests/iceberg_rust.rs similarity index 90% rename from crates/lakebed-iceberg/tests/iceberg_rust.rs rename to crates/lakekit-iceberg/tests/iceberg_rust.rs index 2f90eda..88878ed 100644 --- a/crates/lakebed-iceberg/tests/iceberg_rust.rs +++ b/crates/lakekit-iceberg/tests/iceberg_rust.rs @@ -1,6 +1,6 @@ -//! lakebed-iceberg and iceberg-rust read each other's tables: a table iceberg-rust creates and -//! appends to, lakebed reads and commits to; iceberg-rust reads lakebed's commit and appends on top -//! of it; and lakebed computes the statistics iceberg-rust's Parquet writer records. Everything +//! lakekit-iceberg and iceberg-rust read each other's tables: a table iceberg-rust creates and +//! appends to, lakekit reads and commits to; iceberg-rust reads lakekit's commit and appends on top +//! of it; and lakekit computes the statistics iceberg-rust's Parquet writer records. Everything //! runs in memory. #![cfg(all(feature = "write", feature = "arrow", feature = "parquet-stats"))] #![expect(clippy::unwrap_used, reason = "test code")] @@ -21,9 +21,9 @@ use iceberg::spec::{ use iceberg::transaction::{ApplyTransactionAction, Transaction}; use iceberg::writer::file_writer::{FileWriter as _, FileWriterBuilder, ParquetWriterBuilder}; use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, NamespaceIdent, TableCreation, TableIdent}; -use lakebed_core::{Error, Host, Listed, MemoryHost, MemoryUpload}; -use lakebed_iceberg::stats::ParquetStats; -use lakebed_iceberg::{Change, CommitOptions, DataFile, Datum, Entry, NoHooks, Status, Table, commit}; +use lakekit_core::{Error, Host, Listed, MemoryHost, MemoryUpload}; +use lakekit_iceberg::stats::ParquetStats; +use lakekit_iceberg::{Change, CommitOptions, DataFile, Datum, Entry, NoHooks, Status, Table, commit}; use parquet::arrow::ArrowWriter; use parquet::file::metadata::ParquetMetaDataReader; use parquet::file::properties::WriterProperties; @@ -46,45 +46,45 @@ fn store(e: impl std::fmt::Display) -> Error { impl Host for IoHost { type Upload = MemoryUpload; - async fn get(&self, uri: &str) -> lakebed_core::Result>> { + async fn get(&self, uri: &str) -> lakekit_core::Result>> { if !self.io.exists(uri).await.map_err(store)? { return Ok(None); } Ok(Some(self.io.new_input(uri).map_err(store)?.read().await.map_err(store)?.to_vec())) } - async fn get_range(&self, uri: &str, start: u64, len: u64) -> lakebed_core::Result> { + async fn get_range(&self, uri: &str, start: u64, len: u64) -> lakekit_core::Result> { let bytes = self.get(uri).await?.ok_or_else(|| Error::NotFound(uri.to_string()))?; let start = usize::try_from(start).map_err(store)?.min(bytes.len()); let end = start.saturating_add(usize::try_from(len).map_err(store)?).min(bytes.len()); Ok(bytes[start..end].to_vec()) } - async fn exists(&self, uri: &str) -> lakebed_core::Result { + async fn exists(&self, uri: &str) -> lakekit_core::Result { self.io.exists(uri).await.map_err(store) } - async fn put(&self, uri: &str, bytes: Vec) -> lakebed_core::Result<()> { + async fn put(&self, uri: &str, bytes: Vec) -> lakekit_core::Result<()> { self.io.new_output(uri).map_err(store)?.write(Bytes::from(bytes)).await.map_err(store) } /// The tests run one writer at a time, so checking first is enough. - async fn put_if_absent(&self, uri: &str, bytes: Vec) -> lakebed_core::Result { + async fn put_if_absent(&self, uri: &str, bytes: Vec) -> lakekit_core::Result { if self.exists(uri).await? { return Ok(false); } self.put(uri, bytes).await.map(|()| true) } - fn upload(&self, _: &str) -> impl Future> { + fn upload(&self, _: &str) -> impl Future> { ready(Err(Error::Unsupported("uploads".into()))) } - fn list(&self, _: &str) -> impl Future>> { + fn list(&self, _: &str) -> impl Future>> { ready(Err(Error::Unsupported("listing".into()))) } - async fn delete(&self, uris: &[String]) -> lakebed_core::Result<()> { + async fn delete(&self, uris: &[String]) -> lakekit_core::Result<()> { for uri in uris { self.io.delete(uri).await.map_err(store)?; } @@ -151,7 +151,7 @@ async fn write_theirs(table: &iceberg::table::Table, batch: &RecordBatch, name: files.remove(0).partition(Struct::from_iter([Some(Literal::int(day))])).partition_spec_id(0).build().unwrap() } -/// Writes `batch` with arrow-rs's Parquet writer; returns lakebed's data file. +/// Writes `batch` with arrow-rs's Parquet writer; returns lakekit's data file. async fn write_ours(host: &IoHost, table: &Table, batch: &RecordBatch, name: &str, day: i32) -> DataFile { let mut bytes = vec![]; let mut writer = ArrowWriter::try_new(&mut bytes, batch.schema(), None).unwrap(); @@ -173,7 +173,7 @@ async fn append_theirs( tx.commit(catalog).await.unwrap() } -/// The data file iceberg-rust records, as lakebed records it. +/// The data file iceberg-rust records, as lakekit records it. fn as_ours(f: &IceDataFile) -> DataFile { let bytes = |m: &HashMap| m.iter().map(|(k, d)| (*k, d.to_bytes().unwrap().to_vec())).collect(); @@ -221,7 +221,7 @@ async fn live(host: &IoHost, metadata_file: &str) -> (Table, Vec) { } #[tokio::test] -async fn tables_round_trip_between_lakebed_and_iceberg_rust() { +async fn tables_round_trip_between_lakekit_and_iceberg_rust() { let catalog = MemoryCatalogBuilder::default() .load("memory", HashMap::from([("warehouse".to_string(), "memory://wh".to_string())])) .await @@ -241,7 +241,7 @@ async fn tables_round_trip_between_lakebed_and_iceberg_rust() { let host = IoHost { io: io.clone(), memory: MemoryHost::new(T0 / 1000) }; let arrow = Arc::new(iceberg::arrow::schema_to_arrow_schema(table.metadata().current_schema()).unwrap()); - // iceberg-rust appends; lakebed reads the file as iceberg-rust records it. + // iceberg-rust appends; lakekit reads the file as iceberg-rust records it. let first = batch(&arrow, 0, 5_000); let theirs = write_theirs(&table, &first, "theirs-1", 20_454).await; let table = append_theirs(&catalog, &table, theirs.clone()).await; @@ -250,13 +250,13 @@ async fn tables_round_trip_between_lakebed_and_iceberg_rust() { assert_eq!(*files[0].file, as_ours(&theirs)); assert_eq!((files[0].status, files[0].sequence_number, files[0].file_sequence_number), (Status::Added, 1, 1)); - // lakebed computes the statistics iceberg-rust's writer recorded, from the file's footer. + // lakekit computes the statistics iceberg-rust's writer recorded, from the file's footer. let bytes = io.new_input(theirs.file_path()).unwrap().read().await.unwrap(); let footer = ParquetMetaDataReader::new().parse_and_finish(&bytes).unwrap(); let stats = ParquetStats::new(&footer, bytes.len() as u64, nan_counts(&first)); assert_eq!(stats.data_file(theirs.file_path().into(), vec![Some(Datum::Date(20_454))]), as_ours(&theirs)); - // lakebed commits on top; iceberg-rust reads the new version and lakebed's manifests. + // lakekit commits on top; iceberg-rust reads the new version and lakekit's manifests. let ours = write_ours(&host, &base, &batch(&arrow, 1, 3_000), "ours-1", 20_455).await; let (committed, info) = commit( &host, @@ -279,7 +279,7 @@ async fn tables_round_trip_between_lakebed_and_iceberg_rust() { let (entries, _) = Manifest::parse_avro(&manifest).unwrap().into_parts(); assert_eq!(as_ours(&entries[0].data_file), ours); - // iceberg-rust appends on top of lakebed's version; lakebed reads all three files. Its memory + // iceberg-rust appends on top of lakekit's version; lakekit reads all three files. Its memory // catalog only names metadata files `NNNNN-.metadata.json`, so it gets a copy so named. let ident = TableIdent::from_strs(["ns", "registered"]).unwrap(); let copy = diff --git a/crates/lakebed/Cargo.toml b/crates/lakekit/Cargo.toml similarity index 62% rename from crates/lakebed/Cargo.toml rename to crates/lakekit/Cargo.toml index 766e29c..31e61f7 100644 --- a/crates/lakebed/Cargo.toml +++ b/crates/lakekit/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "lakebed" +name = "lakekit" description = "Table formats (Apache Iceberg first) read and written without I/O of their own, for WebAssembly and native" readme = "../../README.md" version.workspace = true @@ -12,18 +12,18 @@ categories.workspace = true [features] default = ["iceberg"] -# Apache Iceberg (lakebed-iceberg). -iceberg = ["dep:lakebed-iceberg"] +# Apache Iceberg (lakekit-iceberg). +iceberg = ["dep:lakekit-iceberg"] # Writing Iceberg tables. -iceberg-write = ["iceberg", "lakebed-iceberg/write"] +iceberg-write = ["iceberg", "lakekit-iceberg/write"] # Iceberg schemas as Arrow schemas. -iceberg-arrow = ["iceberg", "lakebed-iceberg/arrow"] +iceberg-arrow = ["iceberg", "lakekit-iceberg/arrow"] # Iceberg statistics from Parquet footers. -iceberg-parquet-stats = ["iceberg", "lakebed-iceberg/parquet-stats"] +iceberg-parquet-stats = ["iceberg", "lakekit-iceberg/parquet-stats"] [dependencies] -lakebed-core.workspace = true -lakebed-iceberg = { workspace = true, optional = true } +lakekit-core.workspace = true +lakekit-iceberg = { workspace = true, optional = true } [package.metadata.docs.rs] all-features = true diff --git a/crates/lakebed/src/lib.rs b/crates/lakekit/src/lib.rs similarity index 70% rename from crates/lakebed/src/lib.rs rename to crates/lakekit/src/lib.rs index 1380306..50feaf8 100644 --- a/crates/lakebed/src/lib.rs +++ b/crates/lakekit/src/lib.rs @@ -3,9 +3,9 @@ //! //! | Feature | Crate | //! |---|---| -//! | `iceberg` (default) | [`iceberg`]: Apache Iceberg (`lakebed-iceberg`) | +//! | `iceberg` (default) | [`iceberg`]: Apache Iceberg (`lakekit-iceberg`) | //! | `iceberg-write`, `iceberg-arrow`, `iceberg-parquet-stats` | its optional parts | -pub use lakebed_core as core; +pub use lakekit_core as core; #[cfg(feature = "iceberg")] -pub use lakebed_iceberg as iceberg; +pub use lakekit_iceberg as iceberg; diff --git a/spec/0001-host.md b/spec/0001-host.md index 426269b..416ff6d 100644 --- a/spec/0001-host.md +++ b/spec/0001-host.md @@ -14,7 +14,7 @@ issues: [] ## Summary -Format crates do no I/O of their own. They call a `Host` (`lakebed-core`) that the application implements over its +Format crates do no I/O of their own. They call a `Host` (`lakekit-core`) that the application implements over its object store, clock, and randomness, so the same code runs in a browser, a Cloudflare Worker, a CLI, or a server, and tests run against memory. diff --git a/spec/0002-formats.md b/spec/0002-formats.md index 979a0b2..7af34e9 100644 --- a/spec/0002-formats.md +++ b/spec/0002-formats.md @@ -14,13 +14,13 @@ issues: [6] ## Summary -Each table format is its own crate, `lakebed-`, depending on `lakebed-core` and nothing of another format's. -The `lakebed` crate re-exports them behind a feature each. Iceberg is first (spec 3), because Timelake writes it and +Each table format is its own crate, `lakekit-`, depending on `lakekit-core` and nothing of another format's. +The `lakekit` crate re-exports them behind a feature each. Iceberg is first (spec 3), because Timelake writes it and lakeglass reads it; the formats lakeglass explores come next. ## Design -- **Layout:** `crates/lakebed-core` (spec 1), `crates/lakebed-`, and `crates/lakebed`, whose features are the +- **Layout:** `crates/lakekit-core` (spec 1), `crates/lakekit-`, and `crates/lakekit`, whose features are the format names (`iceberg`) and their parts (`iceberg-write`, `iceberg-arrow`, `iceberg-parquet-stats`). - **Formats as they are.** A format crate exposes its format's own model (Iceberg's snapshots and manifests, Delta's log, Hudi's timeline) without a shared abstraction that would hide the differences. A common view, where an @@ -43,8 +43,10 @@ lakeglass reads it; the formats lakeglass explores come next. ## Open questions and issues -- [#6](https://github.com/polythings/lakebed/issues/6) Which format comes after Iceberg, and in what order? +- [#6](https://github.com/polythings/lakekit/issues/6) Which format comes after Iceberg, and in what order? ## History - 2026-10-20: proposed and the layout built, with Iceberg as the first format. +- 2026-10-21: the project renamed from lakebed to lakekit, an easier name to read; crates `lakekit-core`, + `lakekit-iceberg`, and `lakekit`. diff --git a/spec/0003-iceberg.md b/spec/0003-iceberg.md index 0999099..21f4254 100644 --- a/spec/0003-iceberg.md +++ b/spec/0003-iceberg.md @@ -14,7 +14,7 @@ issues: [1, 2, 3, 4, 5] ## Summary -`lakebed-iceberg` reads Iceberg tables of format versions 1 to 3 and writes format v2 tables, without I/O of its own +`lakekit-iceberg` reads Iceberg tables of format versions 1 to 3 and writes format v2 tables, without I/O of its own (spec 1). Timelake's WASM reader and writer use it, and lakeglass's table explorer will. It started as Timelake's shared writer, made general. @@ -57,8 +57,8 @@ shared writer, made general. iceberg-rust's writer records them; decimals become Iceberg's shortest big-endian bytes however Parquet stores them. A bound or null count is recorded only when every row group gives it exactly (a row group of nulls only needs no bounds): readers take them as covering the whole file, and a partial one would let them skip matching rows. -- **Interop tests:** iceberg-rust creates a table and appends; lakebed reads it and commits; iceberg-rust reads that - version, parses lakebed's manifests, and appends on top; lakebed reads all of it. lakebed's statistics of +- **Interop tests:** iceberg-rust creates a table and appends; lakekit reads it and commits; iceberg-rust reads that + version, parses lakekit's manifests, and appends on top; lakekit reads all of it. lakekit's statistics of iceberg-rust's Parquet file equal what iceberg-rust recorded. **Not yet:** applying partition transforms to values, planning files from filters, writing format v3, writing delete @@ -74,11 +74,11 @@ files, and catalogs (REST). Each has an issue. ## Open questions and issues -- [#1](https://github.com/polythings/lakebed/issues/1) Apply partition transforms to values -- [#2](https://github.com/polythings/lakebed/issues/2) Plan files and row groups from filters -- [#3](https://github.com/polythings/lakebed/issues/3) Write uuid, fixed and decimal partition values -- [#4](https://github.com/polythings/lakebed/issues/4) Write format v3 tables -- [#5](https://github.com/polythings/lakebed/issues/5) Iceberg REST catalogs: here or in lakeglass? +- [#1](https://github.com/polythings/lakekit/issues/1) Apply partition transforms to values +- [#2](https://github.com/polythings/lakekit/issues/2) Plan files and row groups from filters +- [#3](https://github.com/polythings/lakekit/issues/3) Write uuid, fixed and decimal partition values +- [#4](https://github.com/polythings/lakekit/issues/4) Write format v3 tables +- [#5](https://github.com/polythings/lakekit/issues/5) Iceberg REST catalogs: here or in lakeglass? ## History diff --git a/spec/README.md b/spec/README.md index d261de6..b973c5c 100644 --- a/spec/README.md +++ b/spec/README.md @@ -1,6 +1,6 @@ # Specs -lakebed's design, laid out like Timelake's and lakeglass's. Start here, people and agents alike. +lakekit's design, laid out like Timelake's and lakeglass's. Start here, people and agents alike. - **Specs** (`NNNN-name.md`): one feature each, with the same template ([TEMPLATE.md](TEMPLATE.md)). A spec says what the feature does and why, what is still open (as GitHub issues), how far it is built, and the releases that changed diff --git a/spec/TEMPLATE.md b/spec/TEMPLATE.md index 321ab22..e67bf2a 100644 --- a/spec/TEMPLATE.md +++ b/spec/TEMPLATE.md @@ -29,7 +29,7 @@ as "spec N", linked to their files. ## Open questions and issues -- [#N](https://github.com/polythings/lakebed/issues/N) Each open question or task is an issue. +- [#N](https://github.com/polythings/lakekit/issues/N) Each open question or task is an issue. ## History diff --git a/spec/milestones/v0-1.md b/spec/milestones/v0-1.md index a249c4b..83e8246 100644 --- a/spec/milestones/v0-1.md +++ b/spec/milestones/v0-1.md @@ -3,7 +3,7 @@ **In progress** The first release: the host boundary and the Iceberg crate, enough for Timelake's WASM reader and writer to use -lakebed instead of their own Iceberg code, and for lakeglass to read Iceberg tables in the browser. +lakekit instead of their own Iceberg code, and for lakeglass to read Iceberg tables in the browser. | # | Spec | Status | |---:|---|---| @@ -11,5 +11,5 @@ lakebed instead of their own Iceberg code, and for lakeglass to read Iceberg tab | 2 | [One crate per format](../0002-formats.md) | In progress | | 3 | [Apache Iceberg](../0003-iceberg.md) | In progress | -**Done when:** the crates are on crates.io ([#7](https://github.com/polythings/lakebed/issues/7)), Timelake depends +**Done when:** the crates are on crates.io ([#7](https://github.com/polythings/lakekit/issues/7)), Timelake depends on them by version, and lakeglass's Iceberg reader is built on them. From 44a577d4746dbd1e93d2b09451a09d5ece397425 Mon Sep 17 00:00:00 2001 From: Oleksandr Litus Date: Tue, 20 Oct 2026 22:40:55 -0400 Subject: [PATCH 02/14] Changelog: the rename from lakebed Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cb0344..b642f9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,4 +18,10 @@ versions follow [Semantic Versioning](https://semver.org/). decimals as Iceberg's shortest big-endian bytes). - `lakekit`: re-exports the format crates behind a feature each. +### Changed + +- The project is named lakekit, and its crates `lakekit-core`, `lakekit-iceberg`, and `lakekit`; it was lakebed + (`lakebed-core`, …) until 2026-10-21. Nothing was published under the old names; GitHub redirects the old + repository URL. + The Iceberg code started as Timelake's shared writer (`timelake-format`), made general. From ba5b89c59642d8619f3857e8fd22110888d7ded7 Mon Sep 17 00:00:00 2001 From: Oleksandr Litus Date: Tue, 20 Oct 2026 22:46:26 -0400 Subject: [PATCH 03/14] Core: Store and Env instead of Host; an object_store adapter The I/O boundary is two traits: Store for objects and Env for the clock, randomness, and waiting (spec 1, D10), with WithEnv to pass both, MemoryStore for tests, and TokioEnv (feature tokio) for native programs. UUIDs come from the uuid crate's builder, fed by the Env. lakekit-object-store is a Store over any object_store client (spec 4); it enables no backends itself. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 3 + AGENTS.md | 2 +- CHANGELOG.md | 7 +- CONTRIBUTING.md | 7 +- Cargo.lock | 160 +++++++++- Cargo.toml | 5 +- PRIOR_ART.md | 4 +- README.md | 25 +- crates/lakekit-core/Cargo.toml | 18 +- crates/lakekit-core/README.md | 15 +- crates/lakekit-core/src/env.rs | 121 ++++++++ crates/lakekit-core/src/ids.rs | 42 +++ crates/lakekit-core/src/lib.rs | 36 ++- crates/lakekit-core/src/memory.rs | 29 +- crates/lakekit-core/src/{host.rs => store.rs} | 18 +- crates/lakekit-core/src/uuid.rs | 55 ---- crates/lakekit-iceberg/README.md | 3 +- crates/lakekit-iceberg/src/commit.rs | 176 +++++------ crates/lakekit-iceberg/src/lib.rs | 7 +- crates/lakekit-iceberg/src/table.rs | 60 ++-- crates/lakekit-iceberg/tests/iceberg_rust.rs | 48 ++- crates/lakekit-object-store/Cargo.toml | 32 ++ crates/lakekit-object-store/README.md | 24 ++ crates/lakekit-object-store/src/lib.rs | 280 ++++++++++++++++++ crates/lakekit/Cargo.toml | 5 + crates/lakekit/src/lib.rs | 7 +- spec/0001-host.md | 50 ---- spec/0001-store-env.md | 60 ++++ spec/0002-formats.md | 11 +- spec/0004-store-adapters.md | 56 ++++ spec/README.md | 7 +- spec/milestones/README.md | 2 +- spec/milestones/v0-1.md | 8 +- 33 files changed, 1054 insertions(+), 329 deletions(-) create mode 100644 crates/lakekit-core/src/env.rs create mode 100644 crates/lakekit-core/src/ids.rs rename crates/lakekit-core/src/{host.rs => store.rs} (75%) delete mode 100644 crates/lakekit-core/src/uuid.rs create mode 100644 crates/lakekit-object-store/Cargo.toml create mode 100644 crates/lakekit-object-store/README.md create mode 100644 crates/lakekit-object-store/src/lib.rs delete mode 100644 spec/0001-host.md create mode 100644 spec/0001-store-env.md create mode 100644 spec/0004-store-adapters.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d111120..bbb31c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,9 @@ jobs: cargo check -p lakekit-iceberg --features write cargo check -p lakekit-iceberg --features arrow cargo check -p lakekit-iceberg --features parquet-stats + cargo check -p lakekit-core --no-default-features + cargo check -p lakekit-core --features tokio + cargo check -p lakekit-object-store wasm: name: WebAssembly (browser and WASI targets) diff --git a/AGENTS.md b/AGENTS.md index 34c8930..dd7cd55 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,7 +19,7 @@ Timelake and lakeglass use it. - **Keep things in sync:** a change in behaviour updates its spec (status, design, history), the tests, and `CHANGELOG.md` (under `Unreleased`) in the same commit. - **Everything builds for WASM.** Every crate builds for `wasm32-unknown-unknown`: no threads, no blocking I/O, no - `std::time`, no C dependencies. All I/O goes through the `Host` trait. CI checks it. + `std::time`, no C dependencies. All I/O goes through the `Store` and `Env` traits. CI checks it. - **Readers don't trust files.** Bytes from a store can be anything: check lengths, return errors, never panic. - **Interop is tested.** What a format crate writes must read in that format's reference implementation and the other way round; the tests show it. diff --git a/CHANGELOG.md b/CHANGELOG.md index b642f9d..74aaea0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,10 @@ versions follow [Semantic Versioning](https://semver.org/). ### Added -- `lakekit-core`: the `Host` trait for object stores, clocks, and randomness; `MemoryHost`; one `Error` type. +- `lakekit-core`: the `Store` trait for objects and the `Env` trait for the clock, randomness, and waiting; + `WithEnv` to pass both as one; `MemoryStore`; `TokioEnv` (feature `tokio`); one `Error` type. +- `lakekit-object-store`: a `Store` over an `object_store` client (S3, GCS, Azure, local files, memory), from a client + or a URL. - `lakekit-iceberg`: Iceberg table metadata, schemas, partition specs, manifest lists, and manifests read for format versions 1 to 3, including delete files and deletion vector references; values in Iceberg's single-value encoding. - `lakekit-iceberg` feature `write`: format v2 manifests, manifest lists, and metadata updates, and commits to @@ -23,5 +26,7 @@ versions follow [Semantic Versioning](https://semver.org/). - The project is named lakekit, and its crates `lakekit-core`, `lakekit-iceberg`, and `lakekit`; it was lakebed (`lakebed-core`, …) until 2026-10-21. Nothing was published under the old names; GitHub redirects the old repository URL. +- The I/O boundary is two traits, `Store` (objects) and `Env` (clock, randomness, waiting), instead of one `Host`; + `MemoryHost` is `MemoryStore`. UUIDs come from the `uuid` crate. The Iceberg code started as Timelake's shared writer (`timelake-format`), made general. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c15535a..3a22c61 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,7 +10,8 @@ enforce the mechanical rules (see [Checks](#checks)). This guide covers what the the body. 2. **One crate per format.** A format's crate depends on `lakekit-core` and on nothing of another format's, so adding one never touches the others. Formats are shown as they are: no shared model flattens their differences. -3. **No I/O of our own.** Crates read and write through `lakekit_core::Host`. They never open sockets, files, or +3. **No I/O of our own.** Crates read and write through `lakekit_core::Store` and take the time and randomness from + `lakekit_core::Env`. They never open sockets, files, or threads, so they build for `wasm32-unknown-unknown`. 4. **One definition of each thing.** Before writing a helper, constant, or type, look for an existing one. 5. **Behaviour, spec, and tests change together.** A change users can see updates its spec, adds or updates tests, and @@ -43,13 +44,13 @@ ideas we took. - **Types over strings:** enums and newtypes instead of strings and magic numbers. Parse once at the boundary. - **Don't trust files.** Check lengths and offsets before using them, cap allocations, and return an error instead of panicking on a malformed file. Tests feed readers truncated and garbage input. -- **Async:** host futures aren't `Send` (JavaScript promises aren't). Never block inside async code. +- **Async:** store and environment futures aren't `Send` (JavaScript promises aren't). Never block inside async code. - **Suppressing a lint:** `#[expect(lint, reason = "...")]` with a real reason. Never `#[allow]`. ## Tests - **A bug fix starts with a test that fails**, then the fix. -- **Tests run offline**, against `MemoryHost` or a host over the reference implementation's in-memory storage. +- **Tests run offline**, against `MemoryStore` or a store over the reference implementation's in-memory storage. - **Interop:** writers are tested against the format's reference reader, and readers against the reference writer. ## Checks diff --git a/Cargo.lock b/Cargo.lock index 0466307..2a03c5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -528,6 +528,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + [[package]] name = "chacha20" version = "0.10.2" @@ -881,6 +887,16 @@ dependencies = [ "typeid", ] +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "event-listener" version = "5.4.2" @@ -1209,6 +1225,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + [[package]] name = "hybrid-array" version = "0.4.15" @@ -1316,7 +1338,7 @@ dependencies = [ "flate2", "fnv", "futures", - "itertools", + "itertools 0.13.0", "moka", "murmur3", "once_cell", @@ -1514,6 +1536,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -1601,13 +1632,17 @@ version = "0.1.0" dependencies = [ "lakekit-core", "lakekit-iceberg", + "lakekit-object-store", ] [[package]] name = "lakekit-core" version = "0.1.0" dependencies = [ + "getrandom 0.4.3", "thiserror", + "tokio", + "uuid", ] [[package]] @@ -1627,6 +1662,19 @@ dependencies = [ "tokio", ] +[[package]] +name = "lakekit-object-store" +version = "0.1.0" +dependencies = [ + "futures-util", + "lakekit-core", + "lakekit-iceberg", + "object_store", + "tempfile", + "tokio", + "url", +] + [[package]] name = "lexical-core" version = "1.0.6" @@ -1696,6 +1744,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.3" @@ -1788,6 +1842,18 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "num-bigint" version = "0.4.8" @@ -1833,6 +1899,34 @@ dependencies = [ "libm", ] +[[package]] +name = "object_store" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures-channel", + "futures-core", + "futures-util", + "http", + "humantime", + "itertools 0.15.0", + "nix", + "parking_lot", + "percent-encoding", + "thiserror", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", + "windows-sys 0.61.2", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -2270,6 +2364,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustversion" version = "1.0.23" @@ -2282,6 +2389,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schemars" version = "0.9.0" @@ -2602,6 +2718,19 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "2.0.21" @@ -2916,6 +3045,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.2" @@ -3006,6 +3145,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "windows-core" version = "0.62.2" diff --git a/Cargo.toml b/Cargo.toml index b7cdc73..ebff389 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "2" -members = ["crates/lakekit-core", "crates/lakekit-iceberg", "crates/lakekit"] +members = ["crates/lakekit-core", "crates/lakekit-iceberg", "crates/lakekit-object-store", "crates/lakekit"] [workspace.package] version = "0.1.0" @@ -14,6 +14,9 @@ categories = ["database-implementations", "wasm"] [workspace.dependencies] lakekit-core = { path = "crates/lakekit-core", version = "0.1.0" } lakekit-iceberg = { path = "crates/lakekit-iceberg", version = "0.1.0", default-features = false } +lakekit-object-store = { path = "crates/lakekit-object-store", version = "0.1.0" } +object_store = { version = "0.14", default-features = false } +url = "2" thiserror = "2" serde_json = "1" apache-avro = { version = "0.22", default-features = false } diff --git a/PRIOR_ART.md b/PRIOR_ART.md index da7a0db..4d68642 100644 --- a/PRIOR_ART.md +++ b/PRIOR_ART.md @@ -11,12 +11,14 @@ never code: every line of lakekit is our own. How we credit is in [CONTRIBUTING. | [iceberg-rust](https://github.com/apache/iceberg-rust) | Apache-2.0 | The reference the interop tests check against (a test dependency only) | | [Apache Avro](https://avro.apache.org) (`apache-avro`) | Apache-2.0 | Reading and writing manifests and manifest lists | | [arrow-rs](https://github.com/apache/arrow-rs) (`arrow-schema`, `parquet`) | Apache-2.0 | Arrow schemas; Parquet footers for statistics | +| [object_store](https://github.com/apache/arrow-rs-object-store) | Apache-2.0 / MIT | The stores `lakekit-object-store` adapts | +| [uuid](https://github.com/uuid-rs/uuid) | Apache-2.0 / MIT | UUIDs for file names, from an `Env`'s randomness and clock | ## Ideas | Idea | From | License of the source | Where in lakekit | |---|---|---|---| -| A format library with no I/O of its own: the application supplies storage through a trait the library calls | [delta-kernel-rs](https://github.com/delta-io/delta-kernel-rs) (its `Engine` trait) | Apache-2.0 | `lakekit-core`'s `Host` | +| A format library with no I/O of its own: the application supplies storage through a trait the library calls | [delta-kernel-rs](https://github.com/delta-io/delta-kernel-rs) (its `Engine` trait) | Apache-2.0 | `lakekit-core`'s `Store` and `Env` | | One crate per backend under a small shared core, re-exported by an umbrella crate with a feature each | [arrow-rs](https://github.com/apache/arrow-rs), [Apache OpenDAL](https://github.com/apache/opendal) | Apache-2.0 | The workspace layout | | Commits to tables without a catalog: `v.metadata.json` written only if absent, `version-hint.text` for readers | Iceberg's file-system ("Hadoop") tables | Apache-2.0 | `lakekit_iceberg::commit` | | Avro-safe names for partition fields (`_x` and the character's code in hex) | Iceberg's Java implementation | Apache-2.0 | Manifest writing | diff --git a/README.md b/README.md index 13a5876..6f503c9 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # lakekit Table formats read and written without I/O of their own, in Rust, for WebAssembly and native targets. Each format's -metadata (Apache Iceberg first) is parsed and written by plain code that asks a `Host` for bytes: the application -implements the host over its object store, so the same crate runs in a browser, a Cloudflare Worker, a CLI, or a -server. +metadata (Apache Iceberg first) is parsed and written by plain code that asks a `Store` for bytes and an `Env` for the +time and randomness: the application supplies both, so the same crate runs in a browser, a Cloudflare Worker, a CLI, +or a server. lakekit is what [Timelake](https://github.com/polythings/timelake) and [lakeglass](https://github.com/polythings/lakeglass) share. The official Rust crates for these formats build on tokio and HTTP clients, which don't compile to @@ -13,11 +13,12 @@ share. The official Rust crates for these formats build on tokio and HTTP client | Crate | What it does | |---|---| -| [`lakekit-core`](crates/lakekit-core) | The `Host` trait (ranged reads, listing, conditional writes, clock, randomness), errors, an in-memory host | +| [`lakekit-core`](crates/lakekit-core) | The `Store` trait (ranged reads, listing, conditional writes) and the `Env` trait (clock, randomness), errors, an in-memory store, a tokio environment | | [`lakekit-iceberg`](crates/lakekit-iceberg) | Apache Iceberg: metadata, manifests, and commits | -| [`lakekit`](crates/lakekit) | Re-exports the format crates, one feature each | +| [`lakekit-object-store`](crates/lakekit-object-store) | A `Store` over an `object_store` client: S3, GCS, Azure, local files, memory | +| [`lakekit`](crates/lakekit) | Re-exports the crates, one feature each | -More formats get their own crate as they are added (spec 2). +More formats get their own crate as they are added (spec 2), and more stores their own adapter (spec 4). ## Iceberg @@ -33,14 +34,20 @@ More formats get their own crate as they are added (spec 2). What lakekit writes, iceberg-rust reads, and the other way round: the tests commit to one table with both in turn. ```rust +use std::sync::Arc; + use lakekit_iceberg::Table; +use lakekit_object_store::ObjectStoreAdapter; -async fn files(host: &H) -> lakekit_core::Result { - let mut table = Table::load(host, "s3://bucket/warehouse/db/events").await?; - Ok(table.live_files(host).await?.len()) +async fn files(s3: Arc) -> lakekit_core::Result { + let store = ObjectStoreAdapter::new(s3, "s3://bucket"); + let mut table = Table::load(&store, "s3://bucket/warehouse/db/events").await?; + Ok(table.live_files(&store).await?.len()) } ``` +Commits also need an `Env`: `WithEnv::new(store, TokioEnv)` natively, the application's own in WebAssembly. + ## Development ```bash diff --git a/crates/lakekit-core/Cargo.toml b/crates/lakekit-core/Cargo.toml index ec137de..08fd50a 100644 --- a/crates/lakekit-core/Cargo.toml +++ b/crates/lakekit-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lakekit-core" -description = "The I/O boundary shared by lakekit's table format crates: a host trait for object stores, clocks, and randomness" +description = "The I/O boundary shared by lakekit's table format crates: a store trait for objects and an environment trait for clocks and randomness" readme = "README.md" version.workspace = true edition.workspace = true @@ -10,8 +10,24 @@ repository.workspace = true keywords.workspace = true categories.workspace = true +[features] +default = [] +# `TokioEnv`: the system clock, the operating system's randomness, and tokio's timer. Native targets only. +tokio = ["dep:tokio", "dep:getrandom"] + [dependencies] thiserror.workspace = true +uuid = { version = "1", default-features = false } + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +tokio = { workspace = true, features = ["time"], optional = true } +getrandom = { version = "0.4", optional = true } + +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt", "time"] } + +[package.metadata.docs.rs] +all-features = true [lints] workspace = true diff --git a/crates/lakekit-core/README.md b/crates/lakekit-core/README.md index 734e18c..bbc79d0 100644 --- a/crates/lakekit-core/README.md +++ b/crates/lakekit-core/README.md @@ -1,12 +1,17 @@ # lakekit-core The I/O boundary of [lakekit](https://github.com/polythings/lakekit)'s table format crates. Format crates do no I/O of -their own: they call a `Host` that the application implements over its object store, clock, and randomness, so they -build for `wasm32-unknown-unknown` and native targets alike. +their own: they read and write through a `Store` and take the time and randomness from an `Env`, both supplied by the +application, so they build for `wasm32-unknown-unknown` and native targets alike. -- `Host`: get, ranged get, exists, put, put-if-absent (the commit primitive), multipart upload, list, delete, the time, - random numbers, and waiting. Its futures aren't `Send`, so a host can wrap JavaScript promises. -- `MemoryHost`: a host in memory with a fixed clock and seeded randomness, for tests. +- `Store`: get, ranged get, exists, put, put-if-absent (the commit primitive), multipart upload, list, delete. Its + futures aren't `Send`, so a store can wrap JavaScript promises. `lakekit-object-store` implements it over + `object_store`. +- `Env`: the time, random numbers, and waiting. `TokioEnv` (feature `tokio`, native only) uses the system clock, the + operating system's randomness, and tokio's timer. +- `WithEnv`: a store and an environment as one value, for APIs that need both (commits). +- `MemoryStore`: both in memory, with a clock that moves only when told and seeded randomness, for tests. +- `uuid_v4`, `uuid_v7`: identifiers from an environment's randomness and clock (with the `uuid` crate). - `Error`: one error type, whose variant says what a caller can do (`NotFound`, `Conflict`, `CommitUnknown`, …). Apache License 2.0. diff --git a/crates/lakekit-core/src/env.rs b/crates/lakekit-core/src/env.rs new file mode 100644 index 0000000..40d7995 --- /dev/null +++ b/crates/lakekit-core/src/env.rs @@ -0,0 +1,121 @@ +use std::future::Future; + +use crate::{Listed, Result, Store}; + +/// What a writer needs from where it runs besides its store: the time, randomness (snapshot ids and +/// file names), and a way to wait (between commit attempts). +/// +/// The future isn't required to be `Send`, so an environment can wrap JavaScript timers. +pub trait Env { + /// Milliseconds since the Unix epoch. + fn now_ms(&self) -> i64; + /// A random number. + fn random(&self) -> u64; + /// Waits `ms` milliseconds. + fn sleep(&self, ms: u64) -> impl Future; +} + +/// A [`Store`] and an [`Env`] as one value, for APIs that need both (commits): +/// `WithEnv::new(ObjectStoreAdapter::new(…), TokioEnv)`. +#[derive(Clone, Debug, Default)] +pub struct WithEnv { + /// The store. + pub store: S, + /// The environment. + pub env: E, +} + +impl WithEnv { + /// `store` with `env`. + pub fn new(store: S, env: E) -> Self { + Self { store, env } + } +} + +impl Store for WithEnv { + type Upload = S::Upload; + + fn get(&self, uri: &str) -> impl Future>>> { + self.store.get(uri) + } + + fn get_range(&self, uri: &str, start: u64, len: u64) -> impl Future>> { + self.store.get_range(uri, start, len) + } + + fn exists(&self, uri: &str) -> impl Future> { + self.store.exists(uri) + } + + fn put(&self, uri: &str, bytes: Vec) -> impl Future> { + self.store.put(uri, bytes) + } + + fn put_if_absent(&self, uri: &str, bytes: Vec) -> impl Future> { + self.store.put_if_absent(uri, bytes) + } + + fn upload(&self, uri: &str) -> impl Future> { + self.store.upload(uri) + } + + fn list(&self, prefix: &str) -> impl Future>> { + self.store.list(prefix) + } + + fn delete(&self, uris: &[String]) -> impl Future> { + self.store.delete(uris) + } +} + +impl Env for WithEnv { + fn now_ms(&self) -> i64 { + self.env.now_ms() + } + + fn random(&self) -> u64 { + self.env.random() + } + + fn sleep(&self, ms: u64) -> impl Future { + self.env.sleep(ms) + } +} + +/// The environment of a native program on tokio (feature `tokio`): the system clock, the operating +/// system's randomness, and tokio's timer. +#[cfg(all(feature = "tokio", not(target_arch = "wasm32")))] +#[derive(Clone, Copy, Debug, Default)] +pub struct TokioEnv; + +#[cfg(all(feature = "tokio", not(target_arch = "wasm32")))] +impl Env for TokioEnv { + fn now_ms(&self) -> i64 { + let since = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap_or_default(); + i64::try_from(since.as_millis()).unwrap_or(i64::MAX) + } + + fn random(&self) -> u64 { + // The operating system's generator only fails when it isn't available at all. + getrandom::u64().expect("the operating system's random number generator") + } + + fn sleep(&self, ms: u64) -> impl Future { + tokio::time::sleep(std::time::Duration::from_millis(ms)) + } +} + +#[cfg(all(test, feature = "tokio", not(target_arch = "wasm32")))] +mod tests { + use super::*; + + #[tokio::test] + async fn the_tokio_env_tells_time_and_waits() { + let env = TokioEnv; + let before = env.now_ms(); + assert!(before > 1_700_000_000_000, "after November 2023"); + env.sleep(5).await; + assert!(env.now_ms() >= before + 5); + assert_ne!(env.random(), env.random()); + } +} diff --git a/crates/lakekit-core/src/ids.rs b/crates/lakekit-core/src/ids.rs new file mode 100644 index 0000000..77eb0df --- /dev/null +++ b/crates/lakekit-core/src/ids.rs @@ -0,0 +1,42 @@ +use uuid::Builder; + +use crate::Env; + +/// A random UUID (version 4) from `env`'s randomness. +pub fn uuid_v4(env: &impl Env) -> String { + let mut bytes = [0u8; 16]; + bytes[..8].copy_from_slice(&env.random().to_be_bytes()); + bytes[8..].copy_from_slice(&env.random().to_be_bytes()); + Builder::from_random_bytes(bytes).into_uuid().to_string() +} + +/// A time-ordered UUID (version 7) from `env`'s clock and randomness: names made with it sort by +/// creation time. +pub fn uuid_v7(env: &impl Env) -> String { + let millis = u64::try_from(env.now_ms()).unwrap_or(0); + let mut random = [0u8; 10]; + random[..8].copy_from_slice(&env.random().to_be_bytes()); + random[8..].copy_from_slice(&env.random().to_be_bytes()[..2]); + Builder::from_unix_timestamp_millis(millis, &random).into_uuid().to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::MemoryStore; + + #[test] + fn uuids_are_well_formed() { + let env = MemoryStore::new(1_767_225_600_000); + let v4 = uuid_v4(&env); + let v7 = uuid_v7(&env); + for (u, version) in [(&v4, '4'), (&v7, '7')] { + assert_eq!(u.len(), 36); + assert_eq!(u.chars().nth(14), Some(version)); + assert!(matches!(u.chars().nth(19), Some('8' | '9' | 'a' | 'b'))); + } + // Version 7 starts with the time, so later ones sort later. + env.advance(1); + assert!(uuid_v7(&env) > v7); + } +} diff --git a/crates/lakekit-core/src/lib.rs b/crates/lakekit-core/src/lib.rs index ddc231a..faf6daa 100644 --- a/crates/lakekit-core/src/lib.rs +++ b/crates/lakekit-core/src/lib.rs @@ -1,18 +1,24 @@ //! The I/O boundary shared by lakekit's table format crates. Format crates do no I/O of their own: -//! they read and write through a [`Host`] the application implements over its object store (an -//! `object_store` client natively, `fetch` or an R2 binding in WebAssembly), so they build for -//! `wasm32-unknown-unknown` and native targets alike. +//! they read and write through a [`Store`] the application implements over its objects (an +//! `object_store` client natively, `fetch` or an R2 binding in WebAssembly), and take the time and +//! randomness from an [`Env`], so they build for `wasm32-unknown-unknown` and native targets alike. +//! +//! - [`Store`]: get, ranged get, put, put-if-absent (the commit primitive), multipart upload, list, +//! delete. +//! - [`Env`]: the time, randomness, and waiting. `TokioEnv` (feature `tokio`) is a native one. +//! - [`WithEnv`]: a store and an environment as one value, for APIs that need both. +//! - [`MemoryStore`]: both, in memory, for tests. //! //! ``` -//! use lakekit_core::{Host, MemoryHost}; +//! use lakekit_core::{MemoryStore, Store}; //! //! # block_on(async { -//! let host = MemoryHost::new(0); -//! host.put("memory://t/a", b"x".to_vec()).await?; -//! assert!(host.exists("memory://t/a").await?); +//! let store = MemoryStore::new(0); +//! store.put("memory://t/a", b"x".to_vec()).await?; +//! assert!(store.exists("memory://t/a").await?); //! # Ok::<(), lakekit_core::Error>(()) //! # }).unwrap(); -//! # // MemoryHost's futures are ready at once, so one poll runs them. +//! # // MemoryStore's futures are ready at once, so one poll runs them. //! # fn block_on(f: F) -> F::Output { //! # let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); //! # let mut f = std::pin::pin!(f); @@ -20,12 +26,16 @@ //! # } //! ``` +mod env; mod error; -mod host; +mod ids; mod memory; -mod uuid; +mod store; +#[cfg(all(feature = "tokio", not(target_arch = "wasm32")))] +pub use env::TokioEnv; +pub use env::{Env, WithEnv}; pub use error::{Error, Result}; -pub use host::{Host, Listed, Upload}; -pub use memory::{MemoryHost, MemoryUpload}; -pub use uuid::{uuid_v4, uuid_v7}; +pub use ids::{uuid_v4, uuid_v7}; +pub use memory::{MemoryStore, MemoryUpload}; +pub use store::{Listed, Store, Upload}; diff --git a/crates/lakekit-core/src/memory.rs b/crates/lakekit-core/src/memory.rs index 919ebce..8b6b425 100644 --- a/crates/lakekit-core/src/memory.rs +++ b/crates/lakekit-core/src/memory.rs @@ -2,23 +2,24 @@ use std::collections::BTreeMap; use std::future::{Future, ready}; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; -use crate::{Error, Host, Listed, Result, Upload}; +use crate::{Env, Error, Listed, Result, Store, Upload}; -/// A [`Host`] in memory, with a fixed clock and a seeded random sequence: for tests and examples. -/// Clones share the same objects. +/// A [`Store`] in memory that is also an [`Env`] with a clock that moves only when told (or when +/// `sleep` is called) and a seeded random sequence: for tests and examples. Clones share the same +/// objects. #[derive(Clone, Debug, Default)] -pub struct MemoryHost { +pub struct MemoryStore { objects: Arc>>>, clock: Arc>, seed: Arc>, } -impl MemoryHost { +impl MemoryStore { /// An empty store at time `now_ms`. pub fn new(now_ms: i64) -> Self { - let host = Self::default(); - *lock(&host.clock) = now_ms; - host + let store = Self::default(); + *lock(&store.clock) = now_ms; + store } /// Moves the clock forward. @@ -40,10 +41,10 @@ fn lock(m: &Mutex) -> MutexGuard<'_, T> { m.lock().unwrap_or_else(PoisonError::into_inner) } -/// An upload to a [`MemoryHost`]. +/// An upload to a [`MemoryStore`]. #[derive(Debug)] pub struct MemoryUpload { - host: MemoryHost, + store: MemoryStore, uri: String, bytes: Vec, } @@ -55,7 +56,7 @@ impl Upload for MemoryUpload { } fn complete(self) -> impl Future> { - self.host.objects().insert(self.uri, self.bytes); + self.store.objects().insert(self.uri, self.bytes); ready(Ok(())) } @@ -64,7 +65,7 @@ impl Upload for MemoryUpload { } } -impl Host for MemoryHost { +impl Store for MemoryStore { type Upload = MemoryUpload; fn get(&self, uri: &str) -> impl Future>>> { @@ -99,7 +100,7 @@ impl Host for MemoryHost { } fn upload(&self, uri: &str) -> impl Future> { - ready(Ok(MemoryUpload { host: self.clone(), uri: uri.to_string(), bytes: vec![] })) + ready(Ok(MemoryUpload { store: self.clone(), uri: uri.to_string(), bytes: vec![] })) } fn list(&self, prefix: &str) -> impl Future>> { @@ -119,7 +120,9 @@ impl Host for MemoryHost { } ready(Ok(())) } +} +impl Env for MemoryStore { fn now_ms(&self) -> i64 { *lock(&self.clock) } diff --git a/crates/lakekit-core/src/host.rs b/crates/lakekit-core/src/store.rs similarity index 75% rename from crates/lakekit-core/src/host.rs rename to crates/lakekit-core/src/store.rs index 4151c36..8b4f2a8 100644 --- a/crates/lakekit-core/src/host.rs +++ b/crates/lakekit-core/src/store.rs @@ -2,7 +2,7 @@ use std::future::Future; use crate::Result; -/// An object found by [`Host::list`]. +/// An object found by [`Store::list`]. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Listed { /// Absolute URI. @@ -11,11 +11,11 @@ pub struct Listed { pub size: u64, } -/// What a table format needs from where it runs: an object store, a clock, randomness, and a way -/// to wait. Paths are absolute URIs, as table metadata records them. +/// Where a table's objects live: an object store, a directory, memory, or a JavaScript binding. +/// Paths are absolute URIs, as table metadata records them. /// -/// The futures aren't required to be `Send`, so hosts can wrap JavaScript promises. -pub trait Host { +/// The futures aren't required to be `Send`, so a store can wrap JavaScript promises. +pub trait Store { /// A multipart upload in progress. type Upload: Upload; @@ -31,16 +31,10 @@ pub trait Host { fn put_if_absent(&self, uri: &str, bytes: Vec) -> impl Future>; /// Starts a multipart upload to `uri`. fn upload(&self, uri: &str) -> impl Future>; - /// Objects whose URI starts with `prefix`. + /// Objects whose URI starts with `prefix` (a string prefix, not only a directory). fn list(&self, prefix: &str) -> impl Future>>; /// Deletes objects; missing ones are not an error. fn delete(&self, uris: &[String]) -> impl Future>; - /// Milliseconds since the Unix epoch. - fn now_ms(&self) -> i64; - /// A random number. - fn random(&self) -> u64; - /// Waits `ms` milliseconds. - fn sleep(&self, ms: u64) -> impl Future; } /// A multipart upload: parts in order, then [`Upload::complete`]. diff --git a/crates/lakekit-core/src/uuid.rs b/crates/lakekit-core/src/uuid.rs deleted file mode 100644 index 320b951..0000000 --- a/crates/lakekit-core/src/uuid.rs +++ /dev/null @@ -1,55 +0,0 @@ -use std::fmt::Write as _; - -use crate::Host; - -/// A random UUID (version 4) from `host`. -pub fn uuid_v4(host: &impl Host) -> String { - let mut b = [0u8; 16]; - b[..8].copy_from_slice(&host.random().to_be_bytes()); - b[8..].copy_from_slice(&host.random().to_be_bytes()); - b[6] = (b[6] & 0x0f) | 0x40; - b[8] = (b[8] & 0x3f) | 0x80; - format_uuid(&b) -} - -/// A time-ordered UUID (version 7) from `host`'s clock and randomness: names made with it sort by -/// creation time. -pub fn uuid_v7(host: &impl Host) -> String { - let mut b = [0u8; 16]; - let ms = u64::try_from(host.now_ms()).unwrap_or(0); - b[..6].copy_from_slice(&ms.to_be_bytes()[2..]); - b[6..14].copy_from_slice(&host.random().to_be_bytes()); - b[14..].copy_from_slice(&host.random().to_be_bytes()[..2]); - b[6] = (b[6] & 0x0f) | 0x70; - b[8] = (b[8] & 0x3f) | 0x80; - format_uuid(&b) -} - -fn format_uuid(b: &[u8; 16]) -> String { - let hex = b.iter().fold(String::with_capacity(32), |mut s, x| { - let _ = write!(s, "{x:02x}"); - s - }); - format!("{}-{}-{}-{}-{}", &hex[..8], &hex[8..12], &hex[12..16], &hex[16..20], &hex[20..]) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::MemoryHost; - - #[test] - fn uuids_are_well_formed() { - let host = MemoryHost::new(1_767_225_600_000); - let v4 = uuid_v4(&host); - let v7 = uuid_v7(&host); - for (u, version) in [(&v4, '4'), (&v7, '7')] { - assert_eq!(u.len(), 36); - assert_eq!(u.chars().nth(14), Some(version)); - assert!(matches!(u.chars().nth(19), Some('8' | '9' | 'a' | 'b'))); - } - // Version 7 starts with the time, so later ones sort later. - host.advance(1); - assert!(uuid_v7(&host) > v7); - } -} diff --git a/crates/lakekit-iceberg/README.md b/crates/lakekit-iceberg/README.md index efd7e77..6ca48f2 100644 --- a/crates/lakekit-iceberg/README.md +++ b/crates/lakekit-iceberg/README.md @@ -1,7 +1,8 @@ # lakekit-iceberg [Apache Iceberg](https://iceberg.apache.org) tables read and written without I/O of their own, through a -[`lakekit_core::Host`](../lakekit-core), so they work in WebAssembly (browsers, Cloudflare Workers) as well as natively. +[`lakekit_core::Store`](../lakekit-core) (and, for commits, an `Env`), so they work in WebAssembly (browsers, +Cloudflare Workers) as well as natively. | Feature | What it adds | |---|---| diff --git a/crates/lakekit-iceberg/src/commit.rs b/crates/lakekit-iceberg/src/commit.rs index 506fc8b..181adda 100644 --- a/crates/lakekit-iceberg/src/commit.rs +++ b/crates/lakekit-iceberg/src/commit.rs @@ -12,7 +12,7 @@ use std::future::Future; use std::hash::BuildHasher; use std::sync::Arc; -use lakekit_core::{Host, uuid_v4}; +use lakekit_core::{Env, Store, uuid_v4}; use crate::manifest::{ DataFile, Entry, ManifestContext, ManifestFile, ManifestWrite, Status, manifest_file, write_manifest, @@ -125,13 +125,13 @@ pub struct Extra { } /// An application's part in a commit. Every method has a default that does nothing extra. -pub trait CommitHooks { +pub trait CommitHooks { /// The error the hooks (and so the commit) return. type Error: From + fmt::Debug; /// The newest version of the table, after another writer won the race. - fn reload(&mut self, host: &H, location: &str) -> impl Future> { - async move { Ok(Table::load(host, location).await?) } + fn reload(&mut self, store: &H, location: &str) -> impl Future> { + async move { Ok(Table::load(store, location).await?) } } /// Whether `base` already holds this change (an idempotent append retried), so nothing is @@ -144,7 +144,7 @@ pub trait CommitHooks { /// per attempt. fn extra( &mut self, - _host: &H, + _store: &H, _base: &Table, _snapshot_id: i64, ) -> impl Future> { @@ -156,14 +156,14 @@ pub trait CommitHooks { #[derive(Clone, Copy, Debug, Default)] pub struct NoHooks; -impl CommitHooks for NoHooks { +impl CommitHooks for NoHooks { type Error = Error; } /// Commits `change` on top of `base` (or the newest version, after a lost race), with `props` in /// the snapshot summary. The returned table knows its files when `base` did. -pub async fn commit, S: BuildHasher>( - host: &H, +pub async fn commit, S: BuildHasher>( + store: &H, base: Table, change: &Change, props: &HashMap, @@ -181,8 +181,8 @@ pub async fn commit, S: BuildHasher>( for attempt in 1..=options.max_attempts { if attempt > 1 { let (lo, hi) = options.backoff_ms; - host.sleep((lo + host.random() % hi.saturating_sub(lo).max(1)) * attempt).await; - base = hooks.reload(host, &base.location).await?; + store.sleep((lo + store.random() % hi.saturating_sub(lo).max(1)) * attempt).await; + base = hooks.reload(store, &base.location).await?; } if hooks.skip(&base) { let info = CommitInfo { @@ -193,7 +193,7 @@ pub async fn commit, S: BuildHasher>( }; return Ok((base, info)); } - if let Some((table, snapshot_id)) = attempt_commit(host, &base, change, props, options, hooks).await? { + if let Some((table, snapshot_id)) = attempt_commit(store, &base, change, props, options, hooks).await? { let info = CommitInfo { snapshot_id, version: table.version, attempts: attempt, skipped: false }; return Ok((table, info)); } @@ -208,18 +208,18 @@ pub async fn commit, S: BuildHasher>( /// that landed gets "already exists", and a timeout says nothing. The version is then read back: /// our own metadata there is the commit, another writer's means it lost. If that can't be told, /// the error is [`Error::CommitUnknown`], and callers must not delete what the metadata references. -pub async fn put_metadata(host: &H, uri: &str, bytes: Vec) -> Result { +pub async fn put_metadata(store: &S, uri: &str, bytes: Vec) -> Result { let unknown = |e: &dyn fmt::Display| Error::CommitUnknown(format!("{uri}: {e}")); - match host.put_if_absent(uri, bytes.clone()).await { + match store.put_if_absent(uri, bytes.clone()).await { Ok(true) => Ok(true), // Ours (an earlier try landed) or another writer's. Absent now, it was never ours: our // version would still be there. - Ok(false) => match host.get(uri).await { + Ok(false) => match store.get(uri).await { Ok(found) => Ok(found.is_some_and(|found| found == bytes)), Err(e) => Err(unknown(&e)), }, // Absent now, the PUT may still land. - Err(e) => match host.get(uri).await { + Err(e) => match store.get(uri).await { Ok(Some(found)) => Ok(found == bytes), Ok(None) | Err(_) => Err(unknown(&e)), }, @@ -227,9 +227,9 @@ pub async fn put_metadata(host: &H, uri: &str, bytes: Vec) -> Resul } /// A snapshot id not in the table yet. -fn new_snapshot_id(host: &impl Host, base: &Table) -> i64 { +fn new_snapshot_id(env: &impl Env, base: &Table) -> i64 { loop { - let id = i64::try_from(host.random() >> 1).unwrap_or(1).max(1); + let id = i64::try_from(env.random() >> 1).unwrap_or(1).max(1); if !base.metadata.has_snapshot(id) { return id; } @@ -239,27 +239,27 @@ fn new_snapshot_id(host: &impl Host, base: &Table) -> i64 { /// One attempt: writes the snapshot's files and publishes it. `None` if another writer committed /// this version first (this attempt's files are deleted). With an unknown outcome, nothing is /// deleted. -async fn attempt_commit, S: BuildHasher>( - host: &H, +async fn attempt_commit, S: BuildHasher>( + store: &H, base: &Table, change: &Change, props: &HashMap, options: &CommitOptions, hooks: &mut K, ) -> Result, K::Error> { - let snapshot_id = new_snapshot_id(host, base); + let snapshot_id = new_snapshot_id(store, base); let sequence_number = base.metadata.next_sequence_number(); let mut uploads: Vec<(String, Vec)> = vec![]; let mut unknown = false; let result = async { - let manifests = write_manifests(host, base, change, snapshot_id, options, &mut uploads).await?; - let extra = hooks.extra(host, base, snapshot_id).await?; + let manifests = write_manifests(store, base, change, snapshot_id, options, &mut uploads).await?; + let extra = hooks.extra(store, base, snapshot_id).await?; uploads.extend(extra.files); - let list_uri = format!("{}/metadata/snap-{snapshot_id}-{}.avro", base.location, uuid_v4(host)); + let list_uri = format!("{}/metadata/snap-{snapshot_id}-{}.avro", base.location, uuid_v4(store)); let parent = base.metadata.current_snapshot_id(); let list = write_manifest_list(&manifests, snapshot_id, parent, sequence_number)?; uploads.push((list_uri.clone(), list)); - let puts = uploads.iter().map(|(uri, bytes)| host.put(uri, bytes.clone())); + let puts = uploads.iter().map(|(uri, bytes)| store.put(uri, bytes.clone())); futures_util::future::try_join_all(puts).await?; let mut summary: HashMap = props.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); @@ -267,7 +267,7 @@ async fn attempt_commit, S: BuildHasher>( summary.extend(counters(change, &base.metadata.summary())); // Snapshot and metadata logs must stay in time order, so a writer whose clock is behind the // last one's dates its commit at the table's last update. - let now = host.now_ms().max(base.metadata.last_updated_ms()); + let now = store.now_ms().max(base.metadata.last_updated_ms()); let snapshot = NewSnapshot { snapshot_id, sequence_number, @@ -278,7 +278,7 @@ async fn attempt_commit, S: BuildHasher>( }; let metadata = base.metadata.with_snapshot(&snapshot, &base.metadata_file, now)?; let version = base.version + 1; - let ours = put_metadata(host, &metadata_uri(&base.location, version), metadata.to_json().into_bytes()) + let ours = put_metadata(store, &metadata_uri(&base.location, version), metadata.to_json().into_bytes()) .await .inspect_err(|e| unknown = matches!(e, Error::CommitUnknown(_)))?; Ok::<_, K::Error>(ours.then_some((metadata, version))) @@ -287,12 +287,12 @@ async fn attempt_commit, S: BuildHasher>( let (metadata, version) = match result { Ok(Some(committed)) => committed, Ok(None) => { - delete_uploads(host, uploads).await; + delete_uploads(store, uploads).await; return Ok(None); } Err(e) => { if !unknown { - delete_uploads(host, uploads).await; + delete_uploads(store, uploads).await; } return Err(e); } @@ -317,20 +317,20 @@ async fn attempt_commit, S: BuildHasher>( }); let table = Table { files, ..Table::new(base.location.clone(), version, metadata) }; if options.write_version_hint { - let _ = host.put(&hint_uri(&table.location), version.to_string().into_bytes()).await; + let _ = store.put(&hint_uri(&table.location), version.to_string().into_bytes()).await; } Ok(Some((table, snapshot_id))) } -async fn delete_uploads(host: &impl Host, uploads: Vec<(String, Vec)>) { +async fn delete_uploads(store: &impl Store, uploads: Vec<(String, Vec)>) { let written: Vec = uploads.into_iter().map(|(uri, _)| uri).collect(); - let _ = host.delete(&written).await; + let _ = store.delete(&written).await; } /// Writes a manifest of the added files, carries the base snapshot's manifests over, and rewrites /// the ones holding removed files (or all of them once there are too many). -async fn write_manifests( - host: &H, +async fn write_manifests( + store: &H, base: &Table, change: &Change, snapshot_id: i64, @@ -350,7 +350,7 @@ async fn write_manifests( partition_type: &partition_type, }; let mut write = |kind: &str, entries: &[ManifestWrite]| -> Result { - let uri = format!("{}/metadata/{}-{kind}-m.avro", base.location, uuid_v4(host)); + let uri = format!("{}/metadata/{}-{kind}-m.avro", base.location, uuid_v4(store)); let bytes = write_manifest(&ctx, entries)?; let mf = manifest_file(&uri, &bytes, spec_id, snapshot_id, entries, &partition_type); uploads.push((uri, bytes)); @@ -373,14 +373,14 @@ async fn write_manifests( new.push(write("added", &entries)?); } - let current = base.manifests(host).await?; + let current = base.manifests(store).await?; let removed: HashSet<&str> = change.removed().iter().map(|e| e.file.path.as_str()).collect(); let merge = removed.is_empty() && current.len() + new.len() > options.merge_manifests_at; if removed.is_empty() && !merge { new.extend(current); return Ok(new); } - let loaded = futures_util::future::try_join_all(current.iter().map(|m| Table::manifest(host, m))).await?; + let loaded = futures_util::future::try_join_all(current.iter().map(|m| Table::manifest(store, m))).await?; let (mut kept, mut deleted, mut carried) = (vec![], vec![], vec![]); for (manifest, entries) in current.into_iter().zip(loaded) { let live: Vec<&Entry> = entries.iter().filter(|e| e.status != Status::Deleted).collect(); @@ -469,7 +469,7 @@ pub fn counters(change: &Change, base: &HashMap Table { + fn create(store: &MemoryStore, format_version: u8) -> Table { let metadata = json!({ "format-version": format_version, "table-uuid": "u", "location": LOCATION, "last-sequence-number": 0, "last-updated-ms": 0, "last-column-id": 2, "current-schema-id": 0, @@ -491,7 +491,7 @@ mod tests { "properties": {}, "current-snapshot-id": -1, "snapshots": [], "snapshot-log": [], "metadata-log": [] }); let table = Table::new(LOCATION, 1, Metadata::from_json(metadata).unwrap()); - block_on(host.put(&table.metadata_file, table.metadata.to_json().into_bytes())).unwrap(); + block_on(store.put(&table.metadata_file, table.metadata.to_json().into_bytes())).unwrap(); Table { files: Some(vec![]), ..table } } @@ -509,22 +509,22 @@ mod tests { Change::Append { files } } - fn try_run>( - host: &MemoryHost, + fn try_run>( + store: &MemoryStore, base: Table, change: &Change, hooks: &mut K, ) -> Result<(Table, CommitInfo), K::Error> { - block_on(commit(host, base, change, &HashMap::new(), &CommitOptions::default(), hooks)) + block_on(commit(store, base, change, &HashMap::new(), &CommitOptions::default(), hooks)) } - fn run>( - host: &MemoryHost, + fn run>( + store: &MemoryStore, base: Table, change: &Change, hooks: &mut K, ) -> (Table, CommitInfo) { - try_run(host, base, change, hooks).unwrap() + try_run(store, base, change, hooks).unwrap() } fn files_named(t: &Table, name: &str) -> Vec { @@ -533,9 +533,9 @@ mod tests { /// The live files of the newest version, read back from the store: (name, data sequence /// number, file sequence number). - fn live(host: &MemoryHost) -> Vec<(String, i64, i64)> { - let mut t = block_on(Table::load(host, LOCATION)).unwrap(); - let mut files: Vec<_> = block_on(t.live_files(host)) + fn live(store: &MemoryStore) -> Vec<(String, i64, i64)> { + let mut t = block_on(Table::load(store, LOCATION)).unwrap(); + let mut files: Vec<_> = block_on(t.live_files(store)) .unwrap() .iter() .map(|e| { @@ -552,11 +552,11 @@ mod tests { #[test] fn append_replace_and_delete() { - let host = MemoryHost::new(1_000); - let t = create(&host, 2); - let (t, info) = run(&host, t, &append(vec![file("a", 1, 10), file("b", 2, 20)]), &mut NoHooks); + let store = MemoryStore::new(1_000); + let t = create(&store, 2); + let (t, info) = run(&store, t, &append(vec![file("a", 1, 10), file("b", 2, 20)]), &mut NoHooks); assert_eq!((info.version, info.attempts, info.skipped), (2, 1, false)); - assert_eq!(live(&host), [("a.parquet".into(), 1, 1), ("b.parquet".into(), 1, 1)]); + assert_eq!(live(&store), [("a.parquet".into(), 1, 1), ("b.parquet".into(), 1, 1)]); // Compaction keeps the data sequence number of what it replaces. let change = Change::Replace { @@ -564,53 +564,53 @@ mod tests { added: vec![file("c", 1, 10)], sequence_numbers: vec![1], }; - let (t, _) = run(&host, t, &change, &mut NoHooks); - assert_eq!(live(&host), [("b.parquet".into(), 1, 1), ("c.parquet".into(), 1, 2)]); + let (t, _) = run(&store, t, &change, &mut NoHooks); + assert_eq!(live(&store), [("b.parquet".into(), 1, 1), ("c.parquet".into(), 1, 2)]); assert_eq!(t.files.as_ref().map(Vec::len), Some(2), "the commit derives the files"); - let (t, _) = run(&host, t.clone(), &Change::Delete { removed: files_named(&t, "b.parquet") }, &mut NoHooks); - assert_eq!(live(&host), [("c.parquet".into(), 1, 2)]); + let (t, _) = run(&store, t.clone(), &Change::Delete { removed: files_named(&t, "b.parquet") }, &mut NoHooks); + assert_eq!(live(&store), [("c.parquet".into(), 1, 2)]); let summary = t.metadata.summary(); assert_eq!((summary["operation"].as_str(), summary[crate::summary::TOTAL_RECORDS].as_str()), ("delete", "10")); assert_eq!(summary[crate::summary::DELETED_DATA_FILES], "1"); assert_eq!(t.metadata.metadata_log_len(), 3); - assert_eq!(block_on(host.get(&hint_uri(LOCATION))).unwrap().as_deref(), Some(&b"4"[..])); + assert_eq!(block_on(store.get(&hint_uri(LOCATION))).unwrap().as_deref(), Some(&b"4"[..])); } #[test] fn a_lost_race_retries_on_the_newer_version() { - let host = MemoryHost::new(1_000); - let stale = create(&host, 2); - let (_, first) = run(&host, stale.clone(), &append(vec![file("a", 1, 10)]), &mut NoHooks); - let (_, second) = run(&host, stale, &append(vec![file("b", 1, 10)]), &mut NoHooks); + let store = MemoryStore::new(1_000); + let stale = create(&store, 2); + let (_, first) = run(&store, stale.clone(), &append(vec![file("a", 1, 10)]), &mut NoHooks); + let (_, second) = run(&store, stale, &append(vec![file("b", 1, 10)]), &mut NoHooks); assert_eq!((first.version, second.version, second.attempts), (2, 3, 2)); - assert_eq!(live(&host).len(), 2); + assert_eq!(live(&store).len(), 2); // The lost attempt's manifests and list were deleted: every list is a snapshot's. - let lists = host.uris().iter().filter(|u| u.contains("/snap-")).count(); + let lists = store.uris().iter().filter(|u| u.contains("/snap-")).count(); assert_eq!(lists, 2); } #[test] fn removing_files_that_are_gone_is_a_conflict() { - let host = MemoryHost::new(1_000); - let (t, _) = run(&host, create(&host, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); + let store = MemoryStore::new(1_000); + let (t, _) = run(&store, create(&store, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); let removed = t.files.clone().unwrap_or_default(); - let (t, _) = run(&host, t, &Change::Delete { removed: removed.clone() }, &mut NoHooks); - let err = try_run(&host, t, &Change::Delete { removed }, &mut NoHooks); + let (t, _) = run(&store, t, &Change::Delete { removed: removed.clone() }, &mut NoHooks); + let err = try_run(&store, t, &Change::Delete { removed }, &mut NoHooks); assert!(matches!(err, Err(Error::Conflict(_))), "{err:?}"); } /// Hooks that skip a batch already committed, and add a file and summary properties. struct Batch(&'static str); - impl CommitHooks for Batch { + impl CommitHooks for Batch { type Error = Error; fn skip(&mut self, base: &Table) -> bool { base.metadata.ancestors().iter().any(|s| s.summary.get("batch").map(String::as_str) == Some(self.0)) } - fn extra(&mut self, _: &MemoryHost, base: &Table, snapshot_id: i64) -> impl Future> { + fn extra(&mut self, _: &MemoryStore, base: &Table, snapshot_id: i64) -> impl Future> { let uri = format!("{}/metadata/extra-{snapshot_id}", base.location); let summary = HashMap::from([("batch".to_string(), self.0.to_string()), ("extra".to_string(), uri.clone())]); @@ -620,11 +620,11 @@ mod tests { #[test] fn hooks_skip_and_extend_commits() { - let host = MemoryHost::new(1_000); - let (t, info) = run(&host, create(&host, 2), &append(vec![file("a", 1, 10)]), &mut Batch("b1")); + let store = MemoryStore::new(1_000); + let (t, info) = run(&store, create(&store, 2), &append(vec![file("a", 1, 10)]), &mut Batch("b1")); let extra = t.metadata.summary()["extra"].clone(); - assert!(block_on(host.exists(&extra)).unwrap()); - let (t, again) = run(&host, t, &append(vec![file("a", 1, 10)]), &mut Batch("b1")); + assert!(block_on(store.exists(&extra)).unwrap()); + let (t, again) = run(&store, t, &append(vec![file("a", 1, 10)]), &mut Batch("b1")); assert!(again.skipped); assert_eq!((again.version, again.snapshot_id), (info.version, info.snapshot_id)); assert_eq!(t.version, 2); @@ -632,22 +632,22 @@ mod tests { #[test] fn many_manifests_merge() { - let host = MemoryHost::new(1_000); - let mut t = create(&host, 2); + let store = MemoryStore::new(1_000); + let mut t = create(&store, 2); for i in 0..9 { - t = run(&host, t, &append(vec![file(&format!("f{i}"), i, 1)]), &mut NoHooks).0; + t = run(&store, t, &append(vec![file(&format!("f{i}"), i, 1)]), &mut NoHooks).0; } - assert_eq!(block_on(t.manifests(&host)).unwrap().len(), 2, "eight merged into one, plus the new one"); - assert_eq!(live(&host).len(), 9); + assert_eq!(block_on(t.manifests(&store)).unwrap().len(), 2, "eight merged into one, plus the new one"); + assert_eq!(live(&store).len(), 9); } #[test] fn a_clock_behind_the_table_keeps_the_logs_in_order() { - let host = MemoryHost::new(5_000); - let (t, _) = run(&host, create(&host, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); - let late = MemoryHost::new(1_000); - for uri in host.uris() { - block_on(late.put(&uri, block_on(host.get(&uri)).unwrap().unwrap())).unwrap(); + let store = MemoryStore::new(5_000); + let (t, _) = run(&store, create(&store, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); + let late = MemoryStore::new(1_000); + for uri in store.uris() { + block_on(late.put(&uri, block_on(store.get(&uri)).unwrap().unwrap())).unwrap(); } let (t, _) = run(&late, t, &append(vec![file("b", 1, 10)]), &mut NoHooks); assert_eq!((t.metadata.current_snapshot().unwrap().timestamp_ms, t.metadata.last_updated_ms()), (5_000, 5_000)); @@ -655,19 +655,19 @@ mod tests { #[test] fn a_replace_needs_a_sequence_number_per_added_file() { - let host = MemoryHost::new(1_000); - let (t, _) = run(&host, create(&host, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); + let store = MemoryStore::new(1_000); + let (t, _) = run(&store, create(&store, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); let removed = files_named(&t, "a.parquet"); let change = Change::Replace { removed, added: vec![file("b", 1, 5), file("c", 1, 5)], sequence_numbers: vec![1] }; - let err = try_run(&host, t, &change, &mut NoHooks); + let err = try_run(&store, t, &change, &mut NoHooks); assert!(matches!(err, Err(Error::Invalid(_))), "{err:?}"); } #[test] fn only_v2_tables_are_written() { - let host = MemoryHost::new(1_000); - let err = try_run(&host, create(&host, 1), &append(vec![]), &mut NoHooks); + let store = MemoryStore::new(1_000); + let err = try_run(&store, create(&store, 1), &append(vec![]), &mut NoHooks); assert!(matches!(err, Err(Error::Unsupported(_)))); } } diff --git a/crates/lakekit-iceberg/src/lib.rs b/crates/lakekit-iceberg/src/lib.rs index da23ca7..8d7604a 100644 --- a/crates/lakekit-iceberg/src/lib.rs +++ b/crates/lakekit-iceberg/src/lib.rs @@ -1,6 +1,7 @@ //! Apache Iceberg table metadata, manifests and commits, with no I/O of its own: everything is read -//! and written through a [`Host`](lakekit_core::Host) the application implements, so the crate -//! builds for `wasm32-unknown-unknown` (browsers, Cloudflare Workers) as well as native targets. +//! and written through a [`Store`](lakekit_core::Store) the application supplies, and commits take +//! the time and randomness from an [`Env`](lakekit_core::Env), so the crate builds for +//! `wasm32-unknown-unknown` (browsers, Cloudflare Workers) as well as native targets. //! //! - Reading (always): table metadata of format versions 1 to 3 ([`Metadata`]), schemas and //! partition specs, manifest lists and manifests with data and delete files ([`read_manifest`]), @@ -76,7 +77,7 @@ fn corrupt(msg: impl Into) -> Error { Error::Corrupt(msg.into()) } -/// Runs a future whose host calls complete at once ([`lakekit_core::MemoryHost`]'s do). +/// Runs a future whose store calls complete at once ([`lakekit_core::MemoryStore`]'s do). #[cfg(test)] fn block_on(f: F) -> F::Output { let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); diff --git a/crates/lakekit-iceberg/src/table.rs b/crates/lakekit-iceberg/src/table.rs index a2a7df2..6f5c49b 100644 --- a/crates/lakekit-iceberg/src/table.rs +++ b/crates/lakekit-iceberg/src/table.rs @@ -3,7 +3,7 @@ //! them, with `version-hint.text` naming the newest for stores that can't list. Tables a catalog //! names with other file names load with [`Table::from_metadata_file`]. -use lakekit_core::Host; +use lakekit_core::Store; use crate::manifest::{Entry, ManifestFile, Status, read_manifest, read_manifest_list}; use crate::metadata::Metadata; @@ -56,12 +56,12 @@ impl Table { /// The newest metadata version of the table at `location`: the highest listed, or what /// `version-hint.text` says where nothing is listed. - pub async fn latest_version(host: &H, location: &str) -> Result { - let listed = host.list(&format!("{location}/metadata/v")).await?; + pub async fn latest_version(store: &S, location: &str) -> Result { + let listed = store.list(&format!("{location}/metadata/v")).await?; if let Some(newest) = listed.iter().filter_map(|o| metadata_version(&o.uri)).max() { return Ok(newest); } - let hint = host.get(&hint_uri(location)).await?.ok_or_else(|| Error::NotFound(location.to_string()))?; + let hint = store.get(&hint_uri(location)).await?.ok_or_else(|| Error::NotFound(location.to_string()))?; std::str::from_utf8(&hint) .ok() .and_then(|h| h.trim().parse().ok()) @@ -69,22 +69,22 @@ impl Table { } /// The newest version of the table at `location`. - pub async fn load(host: &H, location: &str) -> Result { - let version = Self::latest_version(host, location).await?; - Self::load_version(host, location, version).await + pub async fn load(store: &S, location: &str) -> Result { + let version = Self::latest_version(store, location).await?; + Self::load_version(store, location, version).await } /// Version `version` of the table at `location`. - pub async fn load_version(host: &H, location: &str, version: u64) -> Result { + pub async fn load_version(store: &S, location: &str, version: u64) -> Result { let uri = metadata_uri(location, version); - let bytes = host.get(&uri).await?.ok_or_else(|| Error::NotFound(uri.clone()))?; + let bytes = store.get(&uri).await?.ok_or_else(|| Error::NotFound(uri.clone()))?; Ok(Self::new(location, version, Metadata::parse(&bytes)?)) } /// The table whose metadata file is `uri` (a catalog's pointer), at the location the metadata /// records. - pub async fn from_metadata_file(host: &H, uri: &str) -> Result { - let bytes = host.get(uri).await?.ok_or_else(|| Error::NotFound(uri.to_string()))?; + pub async fn from_metadata_file(store: &S, uri: &str) -> Result { + let bytes = store.get(uri).await?.ok_or_else(|| Error::NotFound(uri.to_string()))?; let metadata = Metadata::parse(&bytes)?; let location = metadata.location().trim_end_matches('/').to_string(); let table = Self::new(location, metadata_version(uri).unwrap_or(0), metadata); @@ -92,28 +92,28 @@ impl Table { } /// The current snapshot's manifests. - pub async fn manifests(&self, host: &H) -> Result> { + pub async fn manifests(&self, store: &S) -> Result> { let Some(list) = self.metadata.manifest_list() else { if self.metadata.current_snapshot().is_some() { return Err(Error::Unsupported("format v1 snapshots without a manifest list".into())); } return Ok(vec![]); }; - let bytes = host.get(&list).await?.ok_or_else(|| Error::NotFound(list.clone()))?; + let bytes = store.get(&list).await?.ok_or_else(|| Error::NotFound(list.clone()))?; read_manifest_list(&bytes) } /// The entries of `manifest`. - pub async fn manifest(host: &H, manifest: &ManifestFile) -> Result> { - let bytes = host.get(&manifest.path).await?.ok_or_else(|| Error::NotFound(manifest.path.clone()))?; + pub async fn manifest(store: &S, manifest: &ManifestFile) -> Result> { + let bytes = store.get(&manifest.path).await?.ok_or_else(|| Error::NotFound(manifest.path.clone()))?; read_manifest(manifest, &bytes) } /// The live files (not deleted by the current snapshot), read from the manifests unless known. - pub async fn live_files(&mut self, host: &H) -> Result<&[Entry]> { + pub async fn live_files(&mut self, store: &S) -> Result<&[Entry]> { if self.files.is_none() { - let manifests = self.manifests(host).await?; - let read = futures_util::future::try_join_all(manifests.iter().map(|m| Self::manifest(host, m))).await?; + let manifests = self.manifests(store).await?; + let read = futures_util::future::try_join_all(manifests.iter().map(|m| Self::manifest(store, m))).await?; let files = read.into_iter().flatten().filter(|e| e.status != Status::Deleted).collect(); self.files = Some(files); } @@ -123,7 +123,7 @@ impl Table { #[cfg(test)] mod tests { - use lakekit_core::MemoryHost; + use lakekit_core::MemoryStore; use super::*; use crate::block_on; @@ -138,22 +138,22 @@ mod tests { #[test] fn loads_the_newest_version() { - let host = MemoryHost::new(0); + let store = MemoryStore::new(0); let meta = |n: u8| format!(r#"{{"format-version": 2, "location": "memory://t", "last-sequence-number": {n}}}"#); block_on(async { - assert!(matches!(Table::load(&host, "memory://t").await, Err(Error::NotFound(_)))); + assert!(matches!(Table::load(&store, "memory://t").await, Err(Error::NotFound(_)))); // Nothing listed: the hint names the version. - host.put(&hint_uri("memory://t"), b"2\n".to_vec()).await?; - assert_eq!(Table::latest_version(&host, "memory://t").await?, 2); - host.put(&metadata_uri("memory://t", 2), meta(2).into_bytes()).await?; - assert_eq!(Table::load(&host, "memory://t").await?.version, 2); - host.put(&metadata_uri("memory://t", 10), meta(10).into_bytes()).await?; - let mut t = Table::load(&host, "memory://t").await?; + store.put(&hint_uri("memory://t"), b"2\n".to_vec()).await?; + assert_eq!(Table::latest_version(&store, "memory://t").await?, 2); + store.put(&metadata_uri("memory://t", 2), meta(2).into_bytes()).await?; + assert_eq!(Table::load(&store, "memory://t").await?.version, 2); + store.put(&metadata_uri("memory://t", 10), meta(10).into_bytes()).await?; + let mut t = Table::load(&store, "memory://t").await?; assert_eq!((t.version, t.metadata.next_sequence_number()), (10, 11)); - assert!(t.live_files(&host).await?.is_empty(), "no snapshot, no files"); + assert!(t.live_files(&store).await?.is_empty(), "no snapshot, no files"); let uri = "memory://t/metadata/00002-a.metadata.json"; - host.put(uri, meta(2).into_bytes()).await?; - let t = Table::from_metadata_file(&host, uri).await?; + store.put(uri, meta(2).into_bytes()).await?; + let t = Table::from_metadata_file(&store, uri).await?; assert_eq!((t.location.as_str(), t.version, t.metadata_file.as_str()), ("memory://t", 2, uri)); Ok::<_, Error>(()) }) diff --git a/crates/lakekit-iceberg/tests/iceberg_rust.rs b/crates/lakekit-iceberg/tests/iceberg_rust.rs index 88878ed..f14f5c0 100644 --- a/crates/lakekit-iceberg/tests/iceberg_rust.rs +++ b/crates/lakekit-iceberg/tests/iceberg_rust.rs @@ -21,7 +21,7 @@ use iceberg::spec::{ use iceberg::transaction::{ApplyTransactionAction, Transaction}; use iceberg::writer::file_writer::{FileWriter as _, FileWriterBuilder, ParquetWriterBuilder}; use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, NamespaceIdent, TableCreation, TableIdent}; -use lakekit_core::{Error, Host, Listed, MemoryHost, MemoryUpload}; +use lakekit_core::{Error, Listed, MemoryStore, MemoryUpload, Store, WithEnv}; use lakekit_iceberg::stats::ParquetStats; use lakekit_iceberg::{Change, CommitOptions, DataFile, Datum, Entry, NoHooks, Status, Table, commit}; use parquet::arrow::ArrowWriter; @@ -32,18 +32,20 @@ use parquet::file::properties::WriterProperties; const T0: i64 = 1_767_225_600_000_000; const DAY_US: i64 = 86_400_000_000; -/// A [`Host`] over iceberg-rust's `FileIO`, so both see the same objects. Clock and randomness come -/// from a [`MemoryHost`]; listing isn't needed (tables load from their metadata file). -struct IoHost { +/// A [`Store`] over iceberg-rust's `FileIO`, so both see the same objects. Listing isn't needed +/// (tables load from their metadata file). +struct IoStore { io: FileIO, - memory: MemoryHost, } +/// The store with a [`MemoryStore`]'s clock and randomness. +type TestStore = WithEnv; + fn store(e: impl std::fmt::Display) -> Error { Error::Store(e.to_string()) } -impl Host for IoHost { +impl Store for IoStore { type Upload = MemoryUpload; async fn get(&self, uri: &str) -> lakekit_core::Result>> { @@ -90,18 +92,6 @@ impl Host for IoHost { } Ok(()) } - - fn now_ms(&self) -> i64 { - self.memory.now_ms() - } - - fn random(&self) -> u64 { - self.memory.random() - } - - fn sleep(&self, ms: u64) -> impl Future { - self.memory.sleep(ms) - } } fn ice_schema() -> IceSchema { @@ -152,14 +142,14 @@ async fn write_theirs(table: &iceberg::table::Table, batch: &RecordBatch, name: } /// Writes `batch` with arrow-rs's Parquet writer; returns lakekit's data file. -async fn write_ours(host: &IoHost, table: &Table, batch: &RecordBatch, name: &str, day: i32) -> DataFile { +async fn write_ours(store: &TestStore, table: &Table, batch: &RecordBatch, name: &str, day: i32) -> DataFile { let mut bytes = vec![]; let mut writer = ArrowWriter::try_new(&mut bytes, batch.schema(), None).unwrap(); writer.write(batch).unwrap(); let footer = writer.close().unwrap(); let stats = ParquetStats::new(&footer, bytes.len() as u64, nan_counts(batch)); let uri = format!("{}/data/{name}.parquet", table.location); - host.put(&uri, bytes).await.unwrap(); + store.put(&uri, bytes).await.unwrap(); stats.data_file(uri, vec![Some(Datum::Date(day))]) } @@ -213,9 +203,9 @@ async fn planned(io: &FileIO, metadata_file: &str) -> Vec { paths } -async fn live(host: &IoHost, metadata_file: &str) -> (Table, Vec) { - let mut table = Table::from_metadata_file(host, metadata_file).await.unwrap(); - let mut files = table.live_files(host).await.unwrap().to_vec(); +async fn live(store: &TestStore, metadata_file: &str) -> (Table, Vec) { + let mut table = Table::from_metadata_file(store, metadata_file).await.unwrap(); + let mut files = table.live_files(store).await.unwrap().to_vec(); files.sort_by(|a, b| a.file.path.cmp(&b.file.path)); (table, files) } @@ -238,14 +228,14 @@ async fn tables_round_trip_between_lakekit_and_iceberg_rust() { Some(UnboundPartitionSpec::builder().add_partition_field(1, "time_day", IceTransform::Day).unwrap().build()); let table = catalog.create_table(&ns, creation).await.unwrap(); let io = table.file_io().clone(); - let host = IoHost { io: io.clone(), memory: MemoryHost::new(T0 / 1000) }; + let store = WithEnv::new(IoStore { io: io.clone() }, MemoryStore::new(T0 / 1000)); let arrow = Arc::new(iceberg::arrow::schema_to_arrow_schema(table.metadata().current_schema()).unwrap()); // iceberg-rust appends; lakekit reads the file as iceberg-rust records it. let first = batch(&arrow, 0, 5_000); let theirs = write_theirs(&table, &first, "theirs-1", 20_454).await; let table = append_theirs(&catalog, &table, theirs.clone()).await; - let (base, files) = live(&host, table.metadata_location().unwrap()).await; + let (base, files) = live(&store, table.metadata_location().unwrap()).await; assert_eq!(files.len(), 1); assert_eq!(*files[0].file, as_ours(&theirs)); assert_eq!((files[0].status, files[0].sequence_number, files[0].file_sequence_number), (Status::Added, 1, 1)); @@ -257,9 +247,9 @@ async fn tables_round_trip_between_lakekit_and_iceberg_rust() { assert_eq!(stats.data_file(theirs.file_path().into(), vec![Some(Datum::Date(20_454))]), as_ours(&theirs)); // lakekit commits on top; iceberg-rust reads the new version and lakekit's manifests. - let ours = write_ours(&host, &base, &batch(&arrow, 1, 3_000), "ours-1", 20_455).await; + let ours = write_ours(&store, &base, &batch(&arrow, 1, 3_000), "ours-1", 20_455).await; let (committed, info) = commit( - &host, + &store, base, &Change::Append { files: vec![ours.clone()] }, &HashMap::new(), @@ -284,11 +274,11 @@ async fn tables_round_trip_between_lakekit_and_iceberg_rust() { let ident = TableIdent::from_strs(["ns", "registered"]).unwrap(); let copy = committed.metadata_file.replace("v2.metadata.json", "00002-1d5e8c1a-0000-4000-8000-000000000000.metadata.json"); - host.put(©, committed.metadata.to_json().into_bytes()).await.unwrap(); + store.put(©, committed.metadata.to_json().into_bytes()).await.unwrap(); let registered = catalog.register_table(&ident, copy).await.unwrap(); let again = write_theirs(®istered, &batch(&arrow, 2, 1_000), "theirs-2", 20_456).await; let registered = append_theirs(&catalog, ®istered, again.clone()).await; - let (_, files) = live(&host, registered.metadata_location().unwrap()).await; + let (_, files) = live(&store, registered.metadata_location().unwrap()).await; let seen: Vec<(&str, i64)> = files.iter().map(|e| (e.file.path.rsplit('/').next().unwrap(), e.sequence_number)).collect(); assert_eq!(seen, [("ours-1.parquet", 2), ("theirs-1.parquet", 1), ("theirs-2.parquet", 3)]); diff --git a/crates/lakekit-object-store/Cargo.toml b/crates/lakekit-object-store/Cargo.toml new file mode 100644 index 0000000..ea81147 --- /dev/null +++ b/crates/lakekit-object-store/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "lakekit-object-store" +description = "A lakekit store over an object_store client: S3, GCS, Azure, local files, memory, and more" +readme = "README.md" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true + +[dependencies] +lakekit-core.workspace = true +# Without features: applications enable the backends they need (`aws`, `gcp`, `azure`, `http`, `fs`, or the `*-base` +# variants for WebAssembly) on their own `object_store` dependency. +object_store.workspace = true +futures-util.workspace = true +url.workspace = true + +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +lakekit-core = { workspace = true, features = ["tokio"] } +lakekit-iceberg = { workspace = true, features = ["write"] } +object_store = { workspace = true, features = ["fs"] } +tempfile = "3" +tokio.workspace = true + +[package.metadata.docs.rs] +all-features = true + +[lints] +workspace = true diff --git a/crates/lakekit-object-store/README.md b/crates/lakekit-object-store/README.md new file mode 100644 index 0000000..1e5caeb --- /dev/null +++ b/crates/lakekit-object-store/README.md @@ -0,0 +1,24 @@ +# lakekit-object-store + +A [lakekit](https://github.com/polythings/lakekit) `Store` over an [`object_store`](https://crates.io/crates/object_store) +client: S3, GCS, Azure, local files, memory, and anything else that implements `ObjectStore`. + +```rust +use std::sync::Arc; + +use lakekit_object_store::ObjectStoreAdapter; +use object_store::memory::InMemory; + +let store = ObjectStoreAdapter::new(Arc::new(InMemory::new()), "memory://"); +// Or from a URL, with the backend enabled on `object_store` (here `aws`): +// let store = ObjectStoreAdapter::from_url("s3://bucket/warehouse", [("aws_region", "us-east-2")])?; +``` + +- lakekit names objects by absolute URI; the adapter maps them to paths under its root (`s3://bucket/`, or + `file:///` for local files). +- Put-if-absent, the commit primitive, is `PutMode::Create`; another writer's object reads as `false`. +- Listing is by string prefix (lakekit's `…/metadata/v`), done by listing the prefix's directory. +- The crate enables no `object_store` backends: turn on the ones you need (`aws`, `gcp`, `azure`, `http`, `fs`, or the + `*-base` variants for WebAssembly) on your own `object_store` dependency. + +Apache License 2.0. diff --git a/crates/lakekit-object-store/src/lib.rs b/crates/lakekit-object-store/src/lib.rs new file mode 100644 index 0000000..3ee3058 --- /dev/null +++ b/crates/lakekit-object-store/src/lib.rs @@ -0,0 +1,280 @@ +//! A lakekit [`Store`] over an [`object_store`] client: S3, GCS, Azure, local files, memory, and +//! anything else that implements `ObjectStore`. Applications enable the backends they need on their +//! own `object_store` dependency and hand the client to [`ObjectStoreAdapter::new`], or build one +//! from a URL with [`ObjectStoreAdapter::from_url`]. +//! +//! lakekit names objects by absolute URI (`s3://bucket/warehouse/t/metadata/v1.metadata.json`); +//! the adapter maps them to paths under its root (`s3://bucket`). +//! +//! ``` +//! use std::sync::Arc; +//! +//! use lakekit_object_store::ObjectStoreAdapter; +//! use object_store::memory::InMemory; +//! +//! let store = ObjectStoreAdapter::new(Arc::new(InMemory::new()), "memory://"); +//! ``` + +use std::future::Future; +use std::sync::Arc; + +use futures_util::TryStreamExt; +use lakekit_core::{Error, Listed, Result, Store, Upload}; +use object_store::path::Path; +use object_store::{MultipartUpload, ObjectStore, ObjectStoreExt, PutMode, PutOptions, PutPayload}; +use url::Url; + +/// A [`Store`] over an `object_store` client whose objects have URIs under `root`. +#[derive(Clone, Debug)] +pub struct ObjectStoreAdapter { + store: Arc, + root: String, +} + +fn store_error(e: object_store::Error) -> Error { + match e { + object_store::Error::NotFound { path, .. } => Error::NotFound(path), + object_store::Error::NotImplemented { operation, implementer } => { + Error::Unsupported(format!("{implementer} has no {operation}")) + } + other => Error::Store(other.to_string()), + } +} + +impl ObjectStoreAdapter { + /// `store`, whose objects have URIs under `root`: the object at path `a/b` is `/a/b`. The + /// root is the store's scheme and bucket (`s3://bucket`, `gs://bucket`, `memory://`), or + /// `file:///` for a local file system rooted at `/`. + pub fn new(store: Arc, root: impl Into) -> Self { + let mut root = root.into(); + if !root.ends_with('/') { + root.push('/'); + } + Self { store, root } + } + + /// The store for `url`'s scheme and bucket (`s3://bucket/…`), configured with `options` + /// (`object_store`'s keys, such as `aws_region`). Its backend must be enabled on + /// `object_store`. + pub fn from_url(url: &str, options: I) -> Result + where + I: IntoIterator, + K: AsRef, + V: Into, + { + let parsed = Url::parse(url).map_err(|e| Error::Invalid(format!("{url}: {e}")))?; + let root = match parsed.host_str().filter(|h| !h.is_empty()) { + Some(host) => format!("{}://{host}/", parsed.scheme()), + // `file:///data/t`: a file system rooted at `/`. + None => format!("{}:///", parsed.scheme()), + }; + let root_url = Url::parse(&root).map_err(|e| Error::Invalid(format!("{root}: {e}")))?; + let (store, _) = object_store::parse_url_opts(&root_url, options).map_err(store_error)?; + Ok(Self::new(Arc::from(store), root)) + } + + /// The path of the object at `uri`. + fn path(&self, uri: &str) -> Result { + let rest = uri + .strip_prefix(&self.root) + .ok_or_else(|| Error::Invalid(format!("{uri} is outside this store ({})", self.root)))?; + Path::parse(rest).map_err(|e| Error::Invalid(format!("{uri}: {e}"))) + } + + /// The URI of the object at `path`. + fn uri(&self, path: &Path) -> String { + format!("{}{path}", self.root) + } +} + +/// A multipart upload through `object_store`. +#[derive(Debug)] +pub struct ObjectStoreUpload(Box); + +impl Upload for ObjectStoreUpload { + async fn put_part(&mut self, bytes: Vec) -> Result<()> { + self.0.put_part(PutPayload::from(bytes)).await.map_err(store_error) + } + + async fn complete(mut self) -> Result<()> { + self.0.complete().await.map(drop).map_err(store_error) + } + + async fn abort(mut self) { + let _ = self.0.abort().await; + } +} + +impl Store for ObjectStoreAdapter { + type Upload = ObjectStoreUpload; + + async fn get(&self, uri: &str) -> Result>> { + match self.store.get(&self.path(uri)?).await { + Ok(found) => Ok(Some(found.bytes().await.map_err(store_error)?.to_vec())), + Err(object_store::Error::NotFound { .. }) => Ok(None), + Err(e) => Err(store_error(e)), + } + } + + async fn get_range(&self, uri: &str, start: u64, len: u64) -> Result> { + let range = start..start.saturating_add(len); + Ok(self.store.get_range(&self.path(uri)?, range).await.map_err(store_error)?.to_vec()) + } + + async fn exists(&self, uri: &str) -> Result { + match self.store.head(&self.path(uri)?).await { + Ok(_) => Ok(true), + Err(object_store::Error::NotFound { .. }) => Ok(false), + Err(e) => Err(store_error(e)), + } + } + + async fn put(&self, uri: &str, bytes: Vec) -> Result<()> { + self.store.put(&self.path(uri)?, PutPayload::from(bytes)).await.map(drop).map_err(store_error) + } + + async fn put_if_absent(&self, uri: &str, bytes: Vec) -> Result { + let options = PutOptions { mode: PutMode::Create, ..PutOptions::default() }; + match self.store.put_opts(&self.path(uri)?, PutPayload::from(bytes), options).await { + Ok(_) => Ok(true), + Err(object_store::Error::AlreadyExists { .. } | object_store::Error::Precondition { .. }) => Ok(false), + Err(e) => Err(store_error(e)), + } + } + + async fn upload(&self, uri: &str) -> Result { + Ok(ObjectStoreUpload(self.store.put_multipart(&self.path(uri)?).await.map_err(store_error)?)) + } + + fn list(&self, prefix: &str) -> impl Future>> { + // `object_store` lists by directory; lakekit by string prefix (`…/metadata/v`). List the + // directory the prefix is in and keep what starts with it. + let relative = prefix.strip_prefix(&self.root).map(|rest| rest.rsplit_once('/').map_or("", |(dir, _)| dir)); + async move { + let relative = + relative.ok_or_else(|| Error::Invalid(format!("{prefix} is outside this store ({})", self.root)))?; + let dir = Path::parse(relative).map_err(|e| Error::Invalid(format!("{prefix}: {e}")))?; + let dir = (!relative.is_empty()).then_some(&dir); + let objects: Vec<_> = self.store.list(dir).try_collect().await.map_err(store_error)?; + Ok(objects + .into_iter() + .map(|o| Listed { uri: self.uri(&o.location), size: o.size }) + .filter(|o| o.uri.starts_with(prefix)) + .collect()) + } + } + + async fn delete(&self, uris: &[String]) -> Result<()> { + let deletes = uris.iter().map(|uri| async move { + match self.store.delete(&self.path(uri)?).await { + Ok(()) | Err(object_store::Error::NotFound { .. }) => Ok(()), + Err(e) => Err(store_error(e)), + } + }); + futures_util::future::try_join_all(deletes).await.map(drop) + } +} + +#[cfg(all(test, not(target_arch = "wasm32")))] +mod tests { + use std::collections::HashMap; + + use lakekit_core::{TokioEnv, WithEnv}; + use lakekit_iceberg::{Change, CommitOptions, DataFile, Datum, Metadata, NoHooks, Table, commit}; + use object_store::memory::InMemory; + + use super::*; + + fn adapter() -> ObjectStoreAdapter { + ObjectStoreAdapter::new(Arc::new(InMemory::new()), "memory://") + } + + #[tokio::test] + async fn objects_round_trip() { + let store = adapter(); + assert_eq!(store.get("memory://t/a").await.unwrap(), None); + assert!(!store.exists("memory://t/a").await.unwrap()); + store.put("memory://t/a", b"hello".to_vec()).await.unwrap(); + assert_eq!(store.get("memory://t/a").await.unwrap().as_deref(), Some(&b"hello"[..])); + assert_eq!(store.get_range("memory://t/a", 1, 3).await.unwrap(), b"ell"); + assert!(store.put_if_absent("memory://t/b", b"1".to_vec()).await.unwrap()); + assert!(!store.put_if_absent("memory://t/b", b"2".to_vec()).await.unwrap(), "the first write wins"); + assert_eq!(store.get("memory://t/b").await.unwrap().as_deref(), Some(&b"1"[..])); + + let mut upload = store.upload("memory://t/c").await.unwrap(); + upload.put_part(b"ab".to_vec()).await.unwrap(); + upload.put_part(b"cd".to_vec()).await.unwrap(); + upload.complete().await.unwrap(); + assert_eq!(store.get("memory://t/c").await.unwrap().as_deref(), Some(&b"abcd"[..])); + + store.delete(&["memory://t/a".into(), "memory://t/missing".into()]).await.unwrap(); + assert!(!store.exists("memory://t/a").await.unwrap()); + assert!(matches!(store.get("s3://elsewhere/x").await, Err(Error::Invalid(_)))); + } + + #[tokio::test] + async fn lists_by_string_prefix() { + let store = adapter(); + for name in ["v1.metadata.json", "v2.metadata.json", "version-hint.text", "snap-1.avro"] { + store.put(&format!("memory://t/metadata/{name}"), vec![0; 3]).await.unwrap(); + } + store.put("memory://t/data/v9.parquet", vec![]).await.unwrap(); + let mut listed: Vec = + store.list("memory://t/metadata/v").await.unwrap().into_iter().map(|o| o.uri).collect(); + listed.sort(); + assert_eq!( + listed, + [ + "memory://t/metadata/v1.metadata.json", + "memory://t/metadata/v2.metadata.json", + "memory://t/metadata/version-hint.text" + ] + ); + assert_eq!(store.list("memory://t/metadata/s").await.unwrap()[0].size, 3); + assert_eq!(store.list("memory://t").await.unwrap().len(), 5, "a prefix right under the root"); + } + + #[tokio::test] + async fn local_files_have_file_uris() { + let dir = tempfile::tempdir().unwrap(); + let store = ObjectStoreAdapter::from_url("file:///", std::iter::empty::<(&str, String)>()).unwrap(); + let uri = format!("file://{}/t/a.json", dir.path().display()); + store.put(&uri, b"{}".to_vec()).await.unwrap(); + assert!(dir.path().join("t/a.json").exists()); + let listed = store.list(&format!("file://{}/t/", dir.path().display())).await.unwrap(); + assert_eq!(listed[0].uri, uri); + } + + /// An Iceberg table committed and read back through `object_store`. + #[tokio::test] + async fn iceberg_commits_through_object_store() { + let store = WithEnv::new(adapter(), TokioEnv); + let location = "memory://warehouse/t"; + let metadata = Metadata::parse( + br#"{"format-version": 2, "table-uuid": "u", "location": "memory://warehouse/t", "last-sequence-number": 0, + "last-updated-ms": 0, "last-column-id": 1, "current-schema-id": 0, + "schemas": [{"schema-id": 0, "type": "struct", "fields": [{"id": 1, "name": "x", "required": true, "type": "long"}]}], + "default-spec-id": 0, "partition-specs": [{"spec-id": 0, "fields": []}], "last-partition-id": 999, + "default-sort-order-id": 0, "sort-orders": [{"order-id": 0, "fields": []}], "properties": {}, + "current-snapshot-id": -1, "snapshots": [], "snapshot-log": [], "metadata-log": []}"#, + ) + .unwrap(); + let table = Table::new(location, 1, metadata); + store.put(&table.metadata_file, table.metadata.to_json().into_bytes()).await.unwrap(); + let file = DataFile { + path: format!("{location}/data/a.parquet"), + partition: Vec::>::new(), + record_count: 3, + file_size: 30, + ..DataFile::default() + }; + let change = Change::Append { files: vec![file] }; + let (_, info) = + commit(&store, table, &change, &HashMap::new(), &CommitOptions::default(), &mut NoHooks).await.unwrap(); + assert_eq!(info.version, 2); + + let mut loaded = Table::load(&store, location).await.unwrap(); + assert_eq!(loaded.version, 2); + assert_eq!(loaded.live_files(&store).await.unwrap()[0].file.record_count, 3); + } +} diff --git a/crates/lakekit/Cargo.toml b/crates/lakekit/Cargo.toml index 31e61f7..6513e1c 100644 --- a/crates/lakekit/Cargo.toml +++ b/crates/lakekit/Cargo.toml @@ -20,10 +20,15 @@ iceberg-write = ["iceberg", "lakekit-iceberg/write"] iceberg-arrow = ["iceberg", "lakekit-iceberg/arrow"] # Iceberg statistics from Parquet footers. iceberg-parquet-stats = ["iceberg", "lakekit-iceberg/parquet-stats"] +# A store over an object_store client (lakekit-object-store). +object-store = ["dep:lakekit-object-store"] +# TokioEnv, a native environment (lakekit-core/tokio). +tokio = ["lakekit-core/tokio"] [dependencies] lakekit-core.workspace = true lakekit-iceberg = { workspace = true, optional = true } +lakekit-object-store = { workspace = true, optional = true } [package.metadata.docs.rs] all-features = true diff --git a/crates/lakekit/src/lib.rs b/crates/lakekit/src/lib.rs index 50feaf8..8baf29b 100644 --- a/crates/lakekit/src/lib.rs +++ b/crates/lakekit/src/lib.rs @@ -1,11 +1,16 @@ //! Table formats read and written without I/O of their own, for WebAssembly and native: one crate -//! per format, re-exported here behind a feature each. The I/O boundary is [`core::Host`]. +//! per format and per store adapter, re-exported here behind a feature each. The I/O boundary is +//! [`core::Store`] and [`core::Env`]. //! //! | Feature | Crate | //! |---|---| //! | `iceberg` (default) | [`iceberg`]: Apache Iceberg (`lakekit-iceberg`) | //! | `iceberg-write`, `iceberg-arrow`, `iceberg-parquet-stats` | its optional parts | +//! | `object-store` | `object_store`: a store over an `object_store` client (`lakekit-object-store`) | +//! | `tokio` | `core::TokioEnv`, a native environment | pub use lakekit_core as core; #[cfg(feature = "iceberg")] pub use lakekit_iceberg as iceberg; +#[cfg(feature = "object-store")] +pub use lakekit_object_store as object_store; diff --git a/spec/0001-host.md b/spec/0001-host.md deleted file mode 100644 index 416ff6d..0000000 --- a/spec/0001-host.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -number: 1 -title: The host boundary -status: done -updated: 2026-10-20 -versions: [unreleased] -related: [2, 3] -issues: [] ---- - -# 1. The host boundary - -**Done** · updated 2026-10-20 · versions unreleased · milestone: [0.1](milestones/v0-1.md) - -## Summary - -Format crates do no I/O of their own. They call a `Host` (`lakekit-core`) that the application implements over its -object store, clock, and randomness, so the same code runs in a browser, a Cloudflare Worker, a CLI, or a server, and -tests run against memory. - -## Design - -- **`Host`:** `get` (the object or `None`), `get_range`, `exists`, `put`, `put_if_absent` (the commit primitive: - `false` when an object is there), `upload` (a multipart upload: parts in order, then complete or abort), `list` (by - prefix), `delete` (missing objects are not an error), `now_ms`, `random`, and `sleep`. Paths are absolute URIs, as - table metadata records them. -- **Futures aren't `Send`.** Methods return `impl Future`, without a `Send` bound, so a host can wrap JavaScript - promises (wasm-bindgen). Native hosts are free to be `Send`. -- **`MemoryHost`:** objects in memory, a clock that only moves when told (or when `sleep` is called), and a seeded - random sequence, so tests are deterministic. -- **`Error`:** one error type for all crates, with the variant saying what a caller can do: `Invalid`, `Unsupported`, - `Conflict` (retry on the newer version), `CommitUnknown` (the commit may have landed: delete nothing it wrote, load - the table again), `NotFound`, `Store`, and `Corrupt`. It is `non_exhaustive`. -- **Identifiers:** `uuid_v4` and `uuid_v7` from the host's randomness and clock, for file names. - -## Decisions - -| ID | Decision | Status | -|---|---|---| -| D1 | Format crates do no I/O: everything goes through a `Host` trait the application implements (the idea of delta-kernel-rs's `Engine`) | Accepted | -| D2 | Host futures aren't `Send`, so JavaScript hosts work; native applications lose nothing | Accepted | -| D3 | One error type across crates, whose variants say what a caller can do | Accepted | - -## Open questions and issues - -None open. - -## History - -- 2026-10-20: built, from Timelake's `timelake_format::host`. diff --git a/spec/0001-store-env.md b/spec/0001-store-env.md new file mode 100644 index 0000000..9ee9a3e --- /dev/null +++ b/spec/0001-store-env.md @@ -0,0 +1,60 @@ +--- +number: 1 +title: Store and environment +status: done +updated: 2026-10-21 +versions: [unreleased] +related: [2, 3, 4] +issues: [] +--- + +# 1. Store and environment + +**Done** · updated 2026-10-21 · versions unreleased · milestone: [0.1](milestones/v0-1.md) + +## Summary + +Format crates do no I/O of their own. They read and write objects through a `Store` and take the time and randomness +from an `Env` (`lakekit-core`), both supplied by the application, so the same code runs in a browser, a Cloudflare +Worker, a CLI, or a server, and tests run against memory. + +## Design + +- **`Store`:** `get` (the object or `None`), `get_range`, `exists`, `put`, `put_if_absent` (the commit primitive: + `false` when an object is there), `upload` (a multipart upload: parts in order, then complete or abort), `list` (by + string prefix, not only by directory), and `delete` (missing objects are not an error). Paths are absolute URIs, as + table metadata records them. Adapters implement it over store libraries ([spec 4](0004-store-adapters.md)). +- **`Env`:** `now_ms`, `random`, and `sleep`: what a writer needs besides its store (snapshot ids, file names, commit + timestamps, the wait between commit attempts). Readers need only a `Store`. +- **`WithEnv`:** a store and an environment as one value (`WithEnv::new(store, env)`), for APIs that need both, such as + `lakekit_iceberg::commit`. A JavaScript host that is both implements both traits on one type. +- **`TokioEnv`** (feature `tokio`, native targets only): the system clock, the operating system's randomness, and + tokio's timer. +- **Futures aren't `Send`.** Methods return `impl Future`, without a `Send` bound, so a store can wrap JavaScript + promises (wasm-bindgen). Native stores are free to be `Send`. +- **`MemoryStore`:** a store in memory that is also an environment, with a clock that only moves when told (or when + `sleep` is called) and a seeded random sequence, so tests are deterministic. +- **`Error`:** one error type for all crates, with the variant saying what a caller can do: `Invalid`, `Unsupported`, + `Conflict` (retry on the newer version), `CommitUnknown` (the commit may have landed: delete nothing it wrote, load + the table again), `NotFound`, `Store`, and `Corrupt`. It is `non_exhaustive`. +- **Identifiers:** `uuid_v4` and `uuid_v7`, built by the `uuid` crate from an environment's randomness and clock (its + builder needs neither its own randomness nor `std::time`, so it works in WebAssembly). + +## Decisions + +| ID | Decision | Status | +|---|---|---| +| D1 | Format crates do no I/O: everything goes through traits the application implements (the idea of delta-kernel-rs's `Engine`) | Accepted | +| D2 | Store and environment futures aren't `Send`, so JavaScript hosts work; native applications lose nothing | Accepted | +| D3 | One error type across crates, whose variants say what a caller can do | Accepted | +| D10 | Two small traits, `Store` for objects and `Env` for the clock, randomness, and waiting, rather than one `Host`: names that say what they are, readers that need only a store, and store adapters that don't decide how time is told | Accepted | + +## Open questions and issues + +None open. + +## History + +- 2026-10-20: built, from Timelake's `timelake_format::host`. +- 2026-10-21: one `Host` trait split into `Store` and `Env` (D10); `TokioEnv` and `WithEnv` added; UUIDs from the + `uuid` crate. The spec was "The host boundary". diff --git a/spec/0002-formats.md b/spec/0002-formats.md index 7af34e9..1551192 100644 --- a/spec/0002-formats.md +++ b/spec/0002-formats.md @@ -2,15 +2,15 @@ number: 2 title: One crate per format status: in-progress -updated: 2026-10-20 +updated: 2026-10-21 versions: [unreleased] -related: [1, 3] -issues: [6] +related: [1, 3, 4] +issues: [6, 11] --- # 2. One crate per format -**In progress** · updated 2026-10-20 · versions unreleased · milestone: [0.1](milestones/v0-1.md) +**In progress** · updated 2026-10-21 · versions unreleased · milestone: [0.1](milestones/v0-1.md) ## Summary @@ -32,6 +32,8 @@ lakeglass reads it; the formats lakeglass explores come next. - **Building for WebAssembly:** every crate builds for `wasm32-unknown-unknown` and is checked for `wasm32-wasip2`. - **Next formats,** in the order lakeglass needs them: Delta Lake, then Apache Hudi, Apache Paimon, Lance, and DuckLake. Each is proposed as its own spec before it is built. +- **Engines** get adapters the same way, a crate each, if they are built: a DataFusion table provider per format is + an open question ([#11](https://github.com/polythings/lakekit/issues/11)). ## Decisions @@ -44,6 +46,7 @@ lakeglass reads it; the formats lakeglass explores come next. ## Open questions and issues - [#6](https://github.com/polythings/lakekit/issues/6) Which format comes after Iceberg, and in what order? +- [#11](https://github.com/polythings/lakekit/issues/11) DataFusion table providers (lakekit-datafusion)? ## History diff --git a/spec/0004-store-adapters.md b/spec/0004-store-adapters.md new file mode 100644 index 0000000..88efab5 --- /dev/null +++ b/spec/0004-store-adapters.md @@ -0,0 +1,56 @@ +--- +number: 4 +title: Store adapters +status: in-progress +updated: 2026-10-21 +versions: [unreleased] +related: [1] +issues: [10] +--- + +# 4. Store adapters + +**In progress** · updated 2026-10-21 · versions unreleased · milestone: [0.1](milestones/v0-1.md) + +## Summary + +Ready-made `Store`s ([spec 1](0001-store-env.md)) over the store libraries applications already use, so a native +application doesn't write its own. Each adapter is a crate of its own; the `Store` trait stays the boundary, because +some stores aren't these libraries (a Cloudflare R2 binding reached from JavaScript) and applications wrap their store +(lakeglass counts every request). + +## Design + +**`lakekit-object-store`** (built): `ObjectStoreAdapter` over an `Arc` (0.14). + +- **Names:** lakekit uses absolute URIs; the adapter has a root (`s3://bucket/`, `memory://`, `file:///` for a local + file system) and maps `` to the store's path ``. A URI outside the root is `Error::Invalid`. +- **Operations:** `get` (missing is `None`), `get_range`, `exists` (a `head`), `put`, `put_if_absent` as + `PutMode::Create` (`AlreadyExists` or `Precondition` is `false`), multipart uploads, `delete` (missing is fine, + deletes run concurrently). `list` is by string prefix: the adapter lists the prefix's directory and keeps what + starts with the prefix, since `object_store` lists whole path segments. +- **Construction:** `new(store, root)`, or `from_url(url, options)` through `object_store::parse_url_opts` for the + URL's scheme and bucket. +- **Backends are the application's:** the crate enables no `object_store` features. Applications turn on `aws`, + `gcp`, `azure`, `http`, or `fs` (or the `*-base` variants, which build for WebAssembly) on their own dependency, so + the adapter builds for `wasm32-unknown-unknown` and pulls in nothing unasked. +- **Tests:** objects, conditional writes, multipart, prefix listing, and local files through the adapter, and an + Iceberg commit loaded back through it. + +**OpenDAL** ([#10](https://github.com/polythings/lakekit/issues/10)): the same over an OpenDAL `Operator`, once an +application needs it. + +## Decisions + +| ID | Decision | Status | +|---|---|---| +| D11 | Store libraries are adapters, one crate each, over the `Store` trait; lakekit's crates never depend on them | Accepted | +| D12 | An adapter enables none of its library's backends; the application chooses them | Accepted | + +## Open questions and issues + +- [#10](https://github.com/polythings/lakekit/issues/10) Store adapter for OpenDAL (lakekit-opendal) + +## History + +- 2026-10-21: proposed, and `lakekit-object-store` built. diff --git a/spec/README.md b/spec/README.md index b973c5c..a0a827e 100644 --- a/spec/README.md +++ b/spec/README.md @@ -28,13 +28,14 @@ lakekit's design, laid out like Timelake's and lakeglass's. Start here, people a ## Index -3 specs: 2 in progress, 1 done. Decisions D1–D9. +4 specs: 3 in progress, 1 done. Decisions D1–D12. | # | Spec | Status | Updated | |---:|---|---|---| -| 1 | [The host boundary](0001-host.md) | Done | 2026-10-20 | -| 2 | [One crate per format](0002-formats.md) | In progress | 2026-10-20 | +| 1 | [Store and environment](0001-store-env.md) | Done | 2026-10-21 | +| 2 | [One crate per format](0002-formats.md) | In progress | 2026-10-21 | | 3 | [Apache Iceberg](0003-iceberg.md) | In progress | 2026-10-20 | +| 4 | [Store adapters](0004-store-adapters.md) | In progress | 2026-10-21 | ## Milestones diff --git a/spec/milestones/README.md b/spec/milestones/README.md index 5f6ca11..3a2f70c 100644 --- a/spec/milestones/README.md +++ b/spec/milestones/README.md @@ -4,4 +4,4 @@ The roadmap: goals made of specs. A milestone is done, in progress, planned, or | Milestone | Status | Specs | |---|---|---| -| [0.1: Iceberg for Timelake and lakeglass](v0-1.md) | In progress | 1, 2, 3 | +| [0.1: Iceberg for Timelake and lakeglass](v0-1.md) | In progress | 1, 2, 3, 4 | diff --git a/spec/milestones/v0-1.md b/spec/milestones/v0-1.md index 83e8246..eeffcc9 100644 --- a/spec/milestones/v0-1.md +++ b/spec/milestones/v0-1.md @@ -2,14 +2,16 @@ **In progress** -The first release: the host boundary and the Iceberg crate, enough for Timelake's WASM reader and writer to use -lakekit instead of their own Iceberg code, and for lakeglass to read Iceberg tables in the browser. +The first release: the store and environment traits, the Iceberg crate, and the `object_store` adapter, enough for +Timelake's WASM reader and writer to use lakekit instead of their own Iceberg code, and for lakeglass to read Iceberg +tables in the browser. | # | Spec | Status | |---:|---|---| -| 1 | [The host boundary](../0001-host.md) | Done | +| 1 | [Store and environment](../0001-store-env.md) | Done | | 2 | [One crate per format](../0002-formats.md) | In progress | | 3 | [Apache Iceberg](../0003-iceberg.md) | In progress | +| 4 | [Store adapters](../0004-store-adapters.md) | In progress | **Done when:** the crates are on crates.io ([#7](https://github.com/polythings/lakekit/issues/7)), Timelake depends on them by version, and lakeglass's Iceberg reader is built on them. From 6e863d8b3ecc7bc0255dd97928bb0b442a421478 Mon Sep 17 00:00:00 2001 From: Oleksandr Litus Date: Tue, 20 Oct 2026 23:09:26 -0400 Subject: [PATCH 04/14] Core: Clock and Random instead of Env The time and randomness are two traits, as ZIO's services are (spec 1, D13): Clock (now_ms, sleep) and Random (next_u64). Services passes a store, a clock, and a random source as one value; the native ones are TokioClock (feature tokio) and OsRandom (feature getrandom). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 1 + AGENTS.md | 2 +- CHANGELOG.md | 8 +- CONTRIBUTING.md | 6 +- PRIOR_ART.md | 5 +- README.md | 11 +- crates/lakekit-core/Cargo.toml | 8 +- crates/lakekit-core/README.md | 14 +-- crates/lakekit-core/src/clock.rs | 42 +++++++ crates/lakekit-core/src/env.rs | 121 ------------------- crates/lakekit-core/src/ids.rs | 36 +++--- crates/lakekit-core/src/lib.rs | 24 ++-- crates/lakekit-core/src/memory.rs | 22 ++-- crates/lakekit-core/src/random.rs | 28 +++++ crates/lakekit-core/src/services.rs | 75 ++++++++++++ crates/lakekit-iceberg/README.md | 4 +- crates/lakekit-iceberg/src/commit.rs | 18 +-- crates/lakekit-iceberg/src/lib.rs | 3 +- crates/lakekit-iceberg/tests/iceberg_rust.rs | 9 +- crates/lakekit-object-store/Cargo.toml | 2 +- crates/lakekit-object-store/src/lib.rs | 4 +- crates/lakekit/Cargo.toml | 4 +- crates/lakekit/src/lib.rs | 4 +- spec/0001-store-clock-random.md | 62 ++++++++++ spec/0001-store-env.md | 60 --------- spec/0003-iceberg.md | 8 +- spec/0004-store-adapters.md | 8 +- spec/README.md | 4 +- spec/milestones/v0-1.md | 4 +- 29 files changed, 321 insertions(+), 276 deletions(-) create mode 100644 crates/lakekit-core/src/clock.rs delete mode 100644 crates/lakekit-core/src/env.rs create mode 100644 crates/lakekit-core/src/random.rs create mode 100644 crates/lakekit-core/src/services.rs create mode 100644 spec/0001-store-clock-random.md delete mode 100644 spec/0001-store-env.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bbb31c0..61f818b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,7 @@ jobs: cargo check -p lakekit-iceberg --features parquet-stats cargo check -p lakekit-core --no-default-features cargo check -p lakekit-core --features tokio + cargo check -p lakekit-core --features getrandom cargo check -p lakekit-object-store wasm: diff --git a/AGENTS.md b/AGENTS.md index dd7cd55..cccc979 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,7 +19,7 @@ Timelake and lakeglass use it. - **Keep things in sync:** a change in behaviour updates its spec (status, design, history), the tests, and `CHANGELOG.md` (under `Unreleased`) in the same commit. - **Everything builds for WASM.** Every crate builds for `wasm32-unknown-unknown`: no threads, no blocking I/O, no - `std::time`, no C dependencies. All I/O goes through the `Store` and `Env` traits. CI checks it. + `std::time`, no C dependencies. All I/O goes through the `Store`, `Clock`, and `Random` traits. CI checks it. - **Readers don't trust files.** Bytes from a store can be anything: check lengths, return errors, never panic. - **Interop is tested.** What a format crate writes must read in that format's reference implementation and the other way round; the tests show it. diff --git a/CHANGELOG.md b/CHANGELOG.md index 74aaea0..28e9889 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,8 @@ versions follow [Semantic Versioning](https://semver.org/). ### Added -- `lakekit-core`: the `Store` trait for objects and the `Env` trait for the clock, randomness, and waiting; - `WithEnv` to pass both as one; `MemoryStore`; `TokioEnv` (feature `tokio`); one `Error` type. +- `lakekit-core`: the `Store`, `Clock`, and `Random` traits; `Services` to pass the three as one; `MemoryStore`; + `TokioClock` (feature `tokio`) and `OsRandom` (feature `getrandom`); one `Error` type. - `lakekit-object-store`: a `Store` over an `object_store` client (S3, GCS, Azure, local files, memory), from a client or a URL. - `lakekit-iceberg`: Iceberg table metadata, schemas, partition specs, manifest lists, and manifests read for format @@ -26,7 +26,7 @@ versions follow [Semantic Versioning](https://semver.org/). - The project is named lakekit, and its crates `lakekit-core`, `lakekit-iceberg`, and `lakekit`; it was lakebed (`lakebed-core`, …) until 2026-10-21. Nothing was published under the old names; GitHub redirects the old repository URL. -- The I/O boundary is two traits, `Store` (objects) and `Env` (clock, randomness, waiting), instead of one `Host`; - `MemoryHost` is `MemoryStore`. UUIDs come from the `uuid` crate. +- The I/O boundary is three traits, `Store` (objects), `Clock` (the time and a timer), and `Random` (random numbers), + instead of one `Host`; `MemoryHost` is `MemoryStore`. UUIDs come from the `uuid` crate. The Iceberg code started as Timelake's shared writer (`timelake-format`), made general. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3a22c61..953cfe4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,8 +10,8 @@ enforce the mechanical rules (see [Checks](#checks)). This guide covers what the the body. 2. **One crate per format.** A format's crate depends on `lakekit-core` and on nothing of another format's, so adding one never touches the others. Formats are shown as they are: no shared model flattens their differences. -3. **No I/O of our own.** Crates read and write through `lakekit_core::Store` and take the time and randomness from - `lakekit_core::Env`. They never open sockets, files, or +3. **No I/O of our own.** Crates read and write through `lakekit_core::Store` and take the time and random numbers + from `Clock` and `Random`. They never open sockets, files, or threads, so they build for `wasm32-unknown-unknown`. 4. **One definition of each thing.** Before writing a helper, constant, or type, look for an existing one. 5. **Behaviour, spec, and tests change together.** A change users can see updates its spec, adds or updates tests, and @@ -44,7 +44,7 @@ ideas we took. - **Types over strings:** enums and newtypes instead of strings and magic numbers. Parse once at the boundary. - **Don't trust files.** Check lengths and offsets before using them, cap allocations, and return an error instead of panicking on a malformed file. Tests feed readers truncated and garbage input. -- **Async:** store and environment futures aren't `Send` (JavaScript promises aren't). Never block inside async code. +- **Async:** store and clock futures aren't `Send` (JavaScript promises aren't). Never block inside async code. - **Suppressing a lint:** `#[expect(lint, reason = "...")]` with a real reason. Never `#[allow]`. ## Tests diff --git a/PRIOR_ART.md b/PRIOR_ART.md index 4d68642..ca23a01 100644 --- a/PRIOR_ART.md +++ b/PRIOR_ART.md @@ -12,13 +12,14 @@ never code: every line of lakekit is our own. How we credit is in [CONTRIBUTING. | [Apache Avro](https://avro.apache.org) (`apache-avro`) | Apache-2.0 | Reading and writing manifests and manifest lists | | [arrow-rs](https://github.com/apache/arrow-rs) (`arrow-schema`, `parquet`) | Apache-2.0 | Arrow schemas; Parquet footers for statistics | | [object_store](https://github.com/apache/arrow-rs-object-store) | Apache-2.0 / MIT | The stores `lakekit-object-store` adapts | -| [uuid](https://github.com/uuid-rs/uuid) | Apache-2.0 / MIT | UUIDs for file names, from an `Env`'s randomness and clock | +| [uuid](https://github.com/uuid-rs/uuid) | Apache-2.0 / MIT | UUIDs for file names, from a `Random` and a `Clock` | ## Ideas | Idea | From | License of the source | Where in lakekit | |---|---|---|---| -| A format library with no I/O of its own: the application supplies storage through a trait the library calls | [delta-kernel-rs](https://github.com/delta-io/delta-kernel-rs) (its `Engine` trait) | Apache-2.0 | `lakekit-core`'s `Store` and `Env` | +| A format library with no I/O of its own: the application supplies storage through a trait the library calls | [delta-kernel-rs](https://github.com/delta-io/delta-kernel-rs) (its `Engine` trait) | Apache-2.0 | `lakekit-core`'s `Store`, `Clock`, and `Random` | | One crate per backend under a small shared core, re-exported by an umbrella crate with a feature each | [arrow-rs](https://github.com/apache/arrow-rs), [Apache OpenDAL](https://github.com/apache/opendal) | Apache-2.0 | The workspace layout | | Commits to tables without a catalog: `v.metadata.json` written only if absent, `version-hint.text` for readers | Iceberg's file-system ("Hadoop") tables | Apache-2.0 | `lakekit_iceberg::commit` | +| The time and randomness as separate, swappable services rather than ambient globals | [ZIO](https://github.com/zio/zio) (`Clock`, `Random`) | Apache-2.0 | `lakekit-core`'s `Clock` and `Random` | | Avro-safe names for partition fields (`_x` and the character's code in hex) | Iceberg's Java implementation | Apache-2.0 | Manifest writing | diff --git a/README.md b/README.md index 6f503c9..b507981 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # lakekit Table formats read and written without I/O of their own, in Rust, for WebAssembly and native targets. Each format's -metadata (Apache Iceberg first) is parsed and written by plain code that asks a `Store` for bytes and an `Env` for the -time and randomness: the application supplies both, so the same crate runs in a browser, a Cloudflare Worker, a CLI, -or a server. +metadata (Apache Iceberg first) is parsed and written by plain code that asks a `Store` for bytes, a `Clock` for the +time, and a `Random` for random numbers: the application supplies them, so the same crate runs in a browser, a +Cloudflare Worker, a CLI, or a server. lakekit is what [Timelake](https://github.com/polythings/timelake) and [lakeglass](https://github.com/polythings/lakeglass) share. The official Rust crates for these formats build on tokio and HTTP clients, which don't compile to @@ -13,7 +13,7 @@ share. The official Rust crates for these formats build on tokio and HTTP client | Crate | What it does | |---|---| -| [`lakekit-core`](crates/lakekit-core) | The `Store` trait (ranged reads, listing, conditional writes) and the `Env` trait (clock, randomness), errors, an in-memory store, a tokio environment | +| [`lakekit-core`](crates/lakekit-core) | The `Store`, `Clock`, and `Random` traits, errors, an in-memory store, a tokio clock, OS randomness | | [`lakekit-iceberg`](crates/lakekit-iceberg) | Apache Iceberg: metadata, manifests, and commits | | [`lakekit-object-store`](crates/lakekit-object-store) | A `Store` over an `object_store` client: S3, GCS, Azure, local files, memory | | [`lakekit`](crates/lakekit) | Re-exports the crates, one feature each | @@ -46,7 +46,8 @@ async fn files(s3: Arc) -> lakekit_core::Result i64; + /// Waits `ms` milliseconds. + fn sleep(&self, ms: u64) -> impl Future; +} + +/// The system clock and tokio's timer (feature `tokio`, native targets only). +#[cfg(all(feature = "tokio", not(target_arch = "wasm32")))] +#[derive(Clone, Copy, Debug, Default)] +pub struct TokioClock; + +#[cfg(all(feature = "tokio", not(target_arch = "wasm32")))] +impl Clock for TokioClock { + fn now_ms(&self) -> i64 { + let since = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap_or_default(); + i64::try_from(since.as_millis()).unwrap_or(i64::MAX) + } + + fn sleep(&self, ms: u64) -> impl Future { + tokio::time::sleep(std::time::Duration::from_millis(ms)) + } +} + +#[cfg(all(test, feature = "tokio", not(target_arch = "wasm32")))] +mod tests { + use super::*; + + #[tokio::test] + async fn the_tokio_clock_tells_time_and_waits() { + let before = TokioClock.now_ms(); + assert!(before > 1_700_000_000_000, "after November 2023"); + TokioClock.sleep(5).await; + assert!(TokioClock.now_ms() >= before + 5); + } +} diff --git a/crates/lakekit-core/src/env.rs b/crates/lakekit-core/src/env.rs deleted file mode 100644 index 40d7995..0000000 --- a/crates/lakekit-core/src/env.rs +++ /dev/null @@ -1,121 +0,0 @@ -use std::future::Future; - -use crate::{Listed, Result, Store}; - -/// What a writer needs from where it runs besides its store: the time, randomness (snapshot ids and -/// file names), and a way to wait (between commit attempts). -/// -/// The future isn't required to be `Send`, so an environment can wrap JavaScript timers. -pub trait Env { - /// Milliseconds since the Unix epoch. - fn now_ms(&self) -> i64; - /// A random number. - fn random(&self) -> u64; - /// Waits `ms` milliseconds. - fn sleep(&self, ms: u64) -> impl Future; -} - -/// A [`Store`] and an [`Env`] as one value, for APIs that need both (commits): -/// `WithEnv::new(ObjectStoreAdapter::new(…), TokioEnv)`. -#[derive(Clone, Debug, Default)] -pub struct WithEnv { - /// The store. - pub store: S, - /// The environment. - pub env: E, -} - -impl WithEnv { - /// `store` with `env`. - pub fn new(store: S, env: E) -> Self { - Self { store, env } - } -} - -impl Store for WithEnv { - type Upload = S::Upload; - - fn get(&self, uri: &str) -> impl Future>>> { - self.store.get(uri) - } - - fn get_range(&self, uri: &str, start: u64, len: u64) -> impl Future>> { - self.store.get_range(uri, start, len) - } - - fn exists(&self, uri: &str) -> impl Future> { - self.store.exists(uri) - } - - fn put(&self, uri: &str, bytes: Vec) -> impl Future> { - self.store.put(uri, bytes) - } - - fn put_if_absent(&self, uri: &str, bytes: Vec) -> impl Future> { - self.store.put_if_absent(uri, bytes) - } - - fn upload(&self, uri: &str) -> impl Future> { - self.store.upload(uri) - } - - fn list(&self, prefix: &str) -> impl Future>> { - self.store.list(prefix) - } - - fn delete(&self, uris: &[String]) -> impl Future> { - self.store.delete(uris) - } -} - -impl Env for WithEnv { - fn now_ms(&self) -> i64 { - self.env.now_ms() - } - - fn random(&self) -> u64 { - self.env.random() - } - - fn sleep(&self, ms: u64) -> impl Future { - self.env.sleep(ms) - } -} - -/// The environment of a native program on tokio (feature `tokio`): the system clock, the operating -/// system's randomness, and tokio's timer. -#[cfg(all(feature = "tokio", not(target_arch = "wasm32")))] -#[derive(Clone, Copy, Debug, Default)] -pub struct TokioEnv; - -#[cfg(all(feature = "tokio", not(target_arch = "wasm32")))] -impl Env for TokioEnv { - fn now_ms(&self) -> i64 { - let since = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap_or_default(); - i64::try_from(since.as_millis()).unwrap_or(i64::MAX) - } - - fn random(&self) -> u64 { - // The operating system's generator only fails when it isn't available at all. - getrandom::u64().expect("the operating system's random number generator") - } - - fn sleep(&self, ms: u64) -> impl Future { - tokio::time::sleep(std::time::Duration::from_millis(ms)) - } -} - -#[cfg(all(test, feature = "tokio", not(target_arch = "wasm32")))] -mod tests { - use super::*; - - #[tokio::test] - async fn the_tokio_env_tells_time_and_waits() { - let env = TokioEnv; - let before = env.now_ms(); - assert!(before > 1_700_000_000_000, "after November 2023"); - env.sleep(5).await; - assert!(env.now_ms() >= before + 5); - assert_ne!(env.random(), env.random()); - } -} diff --git a/crates/lakekit-core/src/ids.rs b/crates/lakekit-core/src/ids.rs index 77eb0df..32e063c 100644 --- a/crates/lakekit-core/src/ids.rs +++ b/crates/lakekit-core/src/ids.rs @@ -1,23 +1,23 @@ use uuid::Builder; -use crate::Env; +use crate::{Clock, Random}; -/// A random UUID (version 4) from `env`'s randomness. -pub fn uuid_v4(env: &impl Env) -> String { +/// A random UUID (version 4) from `random`. +pub fn uuid_v4(random: &impl Random) -> String { let mut bytes = [0u8; 16]; - bytes[..8].copy_from_slice(&env.random().to_be_bytes()); - bytes[8..].copy_from_slice(&env.random().to_be_bytes()); + bytes[..8].copy_from_slice(&random.next_u64().to_be_bytes()); + bytes[8..].copy_from_slice(&random.next_u64().to_be_bytes()); Builder::from_random_bytes(bytes).into_uuid().to_string() } -/// A time-ordered UUID (version 7) from `env`'s clock and randomness: names made with it sort by -/// creation time. -pub fn uuid_v7(env: &impl Env) -> String { - let millis = u64::try_from(env.now_ms()).unwrap_or(0); - let mut random = [0u8; 10]; - random[..8].copy_from_slice(&env.random().to_be_bytes()); - random[8..].copy_from_slice(&env.random().to_be_bytes()[..2]); - Builder::from_unix_timestamp_millis(millis, &random).into_uuid().to_string() +/// A time-ordered UUID (version 7) from `source`'s clock and random numbers: names made with it +/// sort by creation time. +pub fn uuid_v7(source: &(impl Clock + Random)) -> String { + let millis = u64::try_from(source.now_ms()).unwrap_or(0); + let mut bytes = [0u8; 10]; + bytes[..8].copy_from_slice(&source.next_u64().to_be_bytes()); + bytes[8..].copy_from_slice(&source.next_u64().to_be_bytes()[..2]); + Builder::from_unix_timestamp_millis(millis, &bytes).into_uuid().to_string() } #[cfg(test)] @@ -27,16 +27,16 @@ mod tests { #[test] fn uuids_are_well_formed() { - let env = MemoryStore::new(1_767_225_600_000); - let v4 = uuid_v4(&env); - let v7 = uuid_v7(&env); + let source = MemoryStore::new(1_767_225_600_000); + let v4 = uuid_v4(&source); + let v7 = uuid_v7(&source); for (u, version) in [(&v4, '4'), (&v7, '7')] { assert_eq!(u.len(), 36); assert_eq!(u.chars().nth(14), Some(version)); assert!(matches!(u.chars().nth(19), Some('8' | '9' | 'a' | 'b'))); } // Version 7 starts with the time, so later ones sort later. - env.advance(1); - assert!(uuid_v7(&env) > v7); + source.advance(1); + assert!(uuid_v7(&source) > v7); } } diff --git a/crates/lakekit-core/src/lib.rs b/crates/lakekit-core/src/lib.rs index faf6daa..10d0a29 100644 --- a/crates/lakekit-core/src/lib.rs +++ b/crates/lakekit-core/src/lib.rs @@ -1,13 +1,15 @@ //! The I/O boundary shared by lakekit's table format crates. Format crates do no I/O of their own: //! they read and write through a [`Store`] the application implements over its objects (an -//! `object_store` client natively, `fetch` or an R2 binding in WebAssembly), and take the time and -//! randomness from an [`Env`], so they build for `wasm32-unknown-unknown` and native targets alike. +//! `object_store` client natively, `fetch` or an R2 binding in WebAssembly), and take the time from +//! a [`Clock`] and random numbers from a [`Random`], so they build for `wasm32-unknown-unknown` and +//! native targets alike. //! //! - [`Store`]: get, ranged get, put, put-if-absent (the commit primitive), multipart upload, list, //! delete. -//! - [`Env`]: the time, randomness, and waiting. `TokioEnv` (feature `tokio`) is a native one. -//! - [`WithEnv`]: a store and an environment as one value, for APIs that need both. -//! - [`MemoryStore`]: both, in memory, for tests. +//! - [`Clock`]: the time and a timer. `TokioClock` (feature `tokio`) is a native one. +//! - [`Random`]: random numbers. `OsRandom` (feature `getrandom`) is a native one. +//! - [`Services`]: a store, a clock, and a random source as one value, for APIs that need all three. +//! - [`MemoryStore`]: all three, in memory, for tests. //! //! ``` //! use lakekit_core::{MemoryStore, Store}; @@ -26,16 +28,22 @@ //! # } //! ``` -mod env; +mod clock; mod error; mod ids; mod memory; +mod random; +mod services; mod store; +pub use clock::Clock; #[cfg(all(feature = "tokio", not(target_arch = "wasm32")))] -pub use env::TokioEnv; -pub use env::{Env, WithEnv}; +pub use clock::TokioClock; pub use error::{Error, Result}; pub use ids::{uuid_v4, uuid_v7}; pub use memory::{MemoryStore, MemoryUpload}; +#[cfg(all(feature = "getrandom", not(target_arch = "wasm32")))] +pub use random::OsRandom; +pub use random::Random; +pub use services::Services; pub use store::{Listed, Store, Upload}; diff --git a/crates/lakekit-core/src/memory.rs b/crates/lakekit-core/src/memory.rs index 8b6b425..e4fca46 100644 --- a/crates/lakekit-core/src/memory.rs +++ b/crates/lakekit-core/src/memory.rs @@ -2,10 +2,10 @@ use std::collections::BTreeMap; use std::future::{Future, ready}; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; -use crate::{Env, Error, Listed, Result, Store, Upload}; +use crate::{Clock, Error, Listed, Random, Result, Store, Upload}; -/// A [`Store`] in memory that is also an [`Env`] with a clock that moves only when told (or when -/// `sleep` is called) and a seeded random sequence: for tests and examples. Clones share the same +/// A [`Store`] in memory that is also a [`Clock`] that moves only when told (or when `sleep` is +/// called) and a [`Random`] with a seeded sequence: for tests and examples. Clones share the same /// objects. #[derive(Clone, Debug, Default)] pub struct MemoryStore { @@ -122,20 +122,22 @@ impl Store for MemoryStore { } } -impl Env for MemoryStore { +impl Clock for MemoryStore { fn now_ms(&self) -> i64 { *lock(&self.clock) } - fn random(&self) -> u64 { + fn sleep(&self, ms: u64) -> impl Future { + self.advance(i64::try_from(ms).unwrap_or(i64::MAX)); + ready(()) + } +} + +impl Random for MemoryStore { + fn next_u64(&self) -> u64 { // A linear congruential generator (Knuth's MMIX constants), with the high bits folded in. let mut seed = lock(&self.seed); *seed = seed.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407); *seed ^ (*seed >> 29) } - - fn sleep(&self, ms: u64) -> impl Future { - self.advance(i64::try_from(ms).unwrap_or(i64::MAX)); - ready(()) - } } diff --git a/crates/lakekit-core/src/random.rs b/crates/lakekit-core/src/random.rs new file mode 100644 index 0000000..5ea2597 --- /dev/null +++ b/crates/lakekit-core/src/random.rs @@ -0,0 +1,28 @@ +/// Random numbers: snapshot ids, file names, and the jitter between commit attempts. +pub trait Random { + /// A random number. + fn next_u64(&self) -> u64; +} + +/// The operating system's random number generator (feature `getrandom`, native targets only). +#[cfg(all(feature = "getrandom", not(target_arch = "wasm32")))] +#[derive(Clone, Copy, Debug, Default)] +pub struct OsRandom; + +#[cfg(all(feature = "getrandom", not(target_arch = "wasm32")))] +impl Random for OsRandom { + fn next_u64(&self) -> u64 { + // The operating system's generator only fails when it isn't available at all. + getrandom::u64().expect("the operating system's random number generator") + } +} + +#[cfg(all(test, feature = "getrandom", not(target_arch = "wasm32")))] +mod tests { + use super::*; + + #[test] + fn os_random_numbers_differ() { + assert_ne!(OsRandom.next_u64(), OsRandom.next_u64()); + } +} diff --git a/crates/lakekit-core/src/services.rs b/crates/lakekit-core/src/services.rs new file mode 100644 index 0000000..70c8910 --- /dev/null +++ b/crates/lakekit-core/src/services.rs @@ -0,0 +1,75 @@ +use std::future::Future; + +use crate::{Clock, Listed, Random, Result, Store}; + +/// A store, a clock, and a random source as one value, for APIs that need all three (commits): +/// `Services::new(ObjectStoreAdapter::new(…), TokioClock, OsRandom)`. A type that is all three (a +/// JavaScript host, [`MemoryStore`](crate::MemoryStore)) needs no wrapper. +#[derive(Clone, Debug, Default)] +pub struct Services { + /// The store. + pub store: S, + /// The clock. + pub clock: C, + /// The random source. + pub random: R, +} + +impl Services { + /// `store` with `clock` and `random`. + pub fn new(store: S, clock: C, random: R) -> Self { + Self { store, clock, random } + } +} + +impl Store for Services { + type Upload = S::Upload; + + fn get(&self, uri: &str) -> impl Future>>> { + self.store.get(uri) + } + + fn get_range(&self, uri: &str, start: u64, len: u64) -> impl Future>> { + self.store.get_range(uri, start, len) + } + + fn exists(&self, uri: &str) -> impl Future> { + self.store.exists(uri) + } + + fn put(&self, uri: &str, bytes: Vec) -> impl Future> { + self.store.put(uri, bytes) + } + + fn put_if_absent(&self, uri: &str, bytes: Vec) -> impl Future> { + self.store.put_if_absent(uri, bytes) + } + + fn upload(&self, uri: &str) -> impl Future> { + self.store.upload(uri) + } + + fn list(&self, prefix: &str) -> impl Future>> { + self.store.list(prefix) + } + + fn delete(&self, uris: &[String]) -> impl Future> { + self.store.delete(uris) + } +} + +impl Clock for Services { + fn now_ms(&self) -> i64 { + self.clock.now_ms() + } + + fn sleep(&self, ms: u64) -> impl Future { + self.clock.sleep(ms) + } +} + +impl Random for Services { + fn next_u64(&self) -> u64 { + self.random.next_u64() + } +} diff --git a/crates/lakekit-iceberg/README.md b/crates/lakekit-iceberg/README.md index 6ca48f2..766356f 100644 --- a/crates/lakekit-iceberg/README.md +++ b/crates/lakekit-iceberg/README.md @@ -1,8 +1,8 @@ # lakekit-iceberg [Apache Iceberg](https://iceberg.apache.org) tables read and written without I/O of their own, through a -[`lakekit_core::Store`](../lakekit-core) (and, for commits, an `Env`), so they work in WebAssembly (browsers, -Cloudflare Workers) as well as natively. +[`lakekit_core::Store`](../lakekit-core) (and, for commits, a `Clock` and a `Random`), so they work in WebAssembly +(browsers, Cloudflare Workers) as well as natively. | Feature | What it adds | |---|---| diff --git a/crates/lakekit-iceberg/src/commit.rs b/crates/lakekit-iceberg/src/commit.rs index 181adda..c3b455f 100644 --- a/crates/lakekit-iceberg/src/commit.rs +++ b/crates/lakekit-iceberg/src/commit.rs @@ -12,7 +12,7 @@ use std::future::Future; use std::hash::BuildHasher; use std::sync::Arc; -use lakekit_core::{Env, Store, uuid_v4}; +use lakekit_core::{Clock, Random, Store, uuid_v4}; use crate::manifest::{ DataFile, Entry, ManifestContext, ManifestFile, ManifestWrite, Status, manifest_file, write_manifest, @@ -125,7 +125,7 @@ pub struct Extra { } /// An application's part in a commit. Every method has a default that does nothing extra. -pub trait CommitHooks { +pub trait CommitHooks { /// The error the hooks (and so the commit) return. type Error: From + fmt::Debug; @@ -156,13 +156,13 @@ pub trait CommitHooks { #[derive(Clone, Copy, Debug, Default)] pub struct NoHooks; -impl CommitHooks for NoHooks { +impl CommitHooks for NoHooks { type Error = Error; } /// Commits `change` on top of `base` (or the newest version, after a lost race), with `props` in /// the snapshot summary. The returned table knows its files when `base` did. -pub async fn commit, S: BuildHasher>( +pub async fn commit, S: BuildHasher>( store: &H, base: Table, change: &Change, @@ -181,7 +181,7 @@ pub async fn commit, S: BuildHasher>( for attempt in 1..=options.max_attempts { if attempt > 1 { let (lo, hi) = options.backoff_ms; - store.sleep((lo + store.random() % hi.saturating_sub(lo).max(1)) * attempt).await; + store.sleep((lo + store.next_u64() % hi.saturating_sub(lo).max(1)) * attempt).await; base = hooks.reload(store, &base.location).await?; } if hooks.skip(&base) { @@ -227,9 +227,9 @@ pub async fn put_metadata(store: &S, uri: &str, bytes: Vec) -> Res } /// A snapshot id not in the table yet. -fn new_snapshot_id(env: &impl Env, base: &Table) -> i64 { +fn new_snapshot_id(random: &impl Random, base: &Table) -> i64 { loop { - let id = i64::try_from(env.random() >> 1).unwrap_or(1).max(1); + let id = i64::try_from(random.next_u64() >> 1).unwrap_or(1).max(1); if !base.metadata.has_snapshot(id) { return id; } @@ -239,7 +239,7 @@ fn new_snapshot_id(env: &impl Env, base: &Table) -> i64 { /// One attempt: writes the snapshot's files and publishes it. `None` if another writer committed /// this version first (this attempt's files are deleted). With an unknown outcome, nothing is /// deleted. -async fn attempt_commit, S: BuildHasher>( +async fn attempt_commit, S: BuildHasher>( store: &H, base: &Table, change: &Change, @@ -329,7 +329,7 @@ async fn delete_uploads(store: &impl Store, uploads: Vec<(String, Vec)>) { /// Writes a manifest of the added files, carries the base snapshot's manifests over, and rewrites /// the ones holding removed files (or all of them once there are too many). -async fn write_manifests( +async fn write_manifests( store: &H, base: &Table, change: &Change, diff --git a/crates/lakekit-iceberg/src/lib.rs b/crates/lakekit-iceberg/src/lib.rs index 8d7604a..f2f19d4 100644 --- a/crates/lakekit-iceberg/src/lib.rs +++ b/crates/lakekit-iceberg/src/lib.rs @@ -1,6 +1,7 @@ //! Apache Iceberg table metadata, manifests and commits, with no I/O of its own: everything is read //! and written through a [`Store`](lakekit_core::Store) the application supplies, and commits take -//! the time and randomness from an [`Env`](lakekit_core::Env), so the crate builds for +//! the time from a [`Clock`](lakekit_core::Clock) and random numbers from a +//! [`Random`](lakekit_core::Random), so the crate builds for //! `wasm32-unknown-unknown` (browsers, Cloudflare Workers) as well as native targets. //! //! - Reading (always): table metadata of format versions 1 to 3 ([`Metadata`]), schemas and diff --git a/crates/lakekit-iceberg/tests/iceberg_rust.rs b/crates/lakekit-iceberg/tests/iceberg_rust.rs index f14f5c0..370945f 100644 --- a/crates/lakekit-iceberg/tests/iceberg_rust.rs +++ b/crates/lakekit-iceberg/tests/iceberg_rust.rs @@ -21,7 +21,7 @@ use iceberg::spec::{ use iceberg::transaction::{ApplyTransactionAction, Transaction}; use iceberg::writer::file_writer::{FileWriter as _, FileWriterBuilder, ParquetWriterBuilder}; use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, NamespaceIdent, TableCreation, TableIdent}; -use lakekit_core::{Error, Listed, MemoryStore, MemoryUpload, Store, WithEnv}; +use lakekit_core::{Error, Listed, MemoryStore, MemoryUpload, Services, Store}; use lakekit_iceberg::stats::ParquetStats; use lakekit_iceberg::{Change, CommitOptions, DataFile, Datum, Entry, NoHooks, Status, Table, commit}; use parquet::arrow::ArrowWriter; @@ -38,8 +38,8 @@ struct IoStore { io: FileIO, } -/// The store with a [`MemoryStore`]'s clock and randomness. -type TestStore = WithEnv; +/// The store with a [`MemoryStore`]'s clock and random numbers. +type TestStore = Services; fn store(e: impl std::fmt::Display) -> Error { Error::Store(e.to_string()) @@ -228,7 +228,8 @@ async fn tables_round_trip_between_lakekit_and_iceberg_rust() { Some(UnboundPartitionSpec::builder().add_partition_field(1, "time_day", IceTransform::Day).unwrap().build()); let table = catalog.create_table(&ns, creation).await.unwrap(); let io = table.file_io().clone(); - let store = WithEnv::new(IoStore { io: io.clone() }, MemoryStore::new(T0 / 1000)); + let memory = MemoryStore::new(T0 / 1000); + let store = Services::new(IoStore { io: io.clone() }, memory.clone(), memory); let arrow = Arc::new(iceberg::arrow::schema_to_arrow_schema(table.metadata().current_schema()).unwrap()); // iceberg-rust appends; lakekit reads the file as iceberg-rust records it. diff --git a/crates/lakekit-object-store/Cargo.toml b/crates/lakekit-object-store/Cargo.toml index ea81147..67a429b 100644 --- a/crates/lakekit-object-store/Cargo.toml +++ b/crates/lakekit-object-store/Cargo.toml @@ -19,7 +19,7 @@ futures-util.workspace = true url.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] -lakekit-core = { workspace = true, features = ["tokio"] } +lakekit-core = { workspace = true, features = ["tokio", "getrandom"] } lakekit-iceberg = { workspace = true, features = ["write"] } object_store = { workspace = true, features = ["fs"] } tempfile = "3" diff --git a/crates/lakekit-object-store/src/lib.rs b/crates/lakekit-object-store/src/lib.rs index 3ee3058..5d8b992 100644 --- a/crates/lakekit-object-store/src/lib.rs +++ b/crates/lakekit-object-store/src/lib.rs @@ -179,7 +179,7 @@ impl Store for ObjectStoreAdapter { mod tests { use std::collections::HashMap; - use lakekit_core::{TokioEnv, WithEnv}; + use lakekit_core::{OsRandom, Services, TokioClock}; use lakekit_iceberg::{Change, CommitOptions, DataFile, Datum, Metadata, NoHooks, Table, commit}; use object_store::memory::InMemory; @@ -248,7 +248,7 @@ mod tests { /// An Iceberg table committed and read back through `object_store`. #[tokio::test] async fn iceberg_commits_through_object_store() { - let store = WithEnv::new(adapter(), TokioEnv); + let store = Services::new(adapter(), TokioClock, OsRandom); let location = "memory://warehouse/t"; let metadata = Metadata::parse( br#"{"format-version": 2, "table-uuid": "u", "location": "memory://warehouse/t", "last-sequence-number": 0, diff --git a/crates/lakekit/Cargo.toml b/crates/lakekit/Cargo.toml index 6513e1c..2466e36 100644 --- a/crates/lakekit/Cargo.toml +++ b/crates/lakekit/Cargo.toml @@ -22,8 +22,10 @@ iceberg-arrow = ["iceberg", "lakekit-iceberg/arrow"] iceberg-parquet-stats = ["iceberg", "lakekit-iceberg/parquet-stats"] # A store over an object_store client (lakekit-object-store). object-store = ["dep:lakekit-object-store"] -# TokioEnv, a native environment (lakekit-core/tokio). +# TokioClock, a native clock (lakekit-core/tokio). tokio = ["lakekit-core/tokio"] +# OsRandom, the operating system's random numbers (lakekit-core/getrandom). +getrandom = ["lakekit-core/getrandom"] [dependencies] lakekit-core.workspace = true diff --git a/crates/lakekit/src/lib.rs b/crates/lakekit/src/lib.rs index 8baf29b..09b36b5 100644 --- a/crates/lakekit/src/lib.rs +++ b/crates/lakekit/src/lib.rs @@ -1,13 +1,13 @@ //! Table formats read and written without I/O of their own, for WebAssembly and native: one crate //! per format and per store adapter, re-exported here behind a feature each. The I/O boundary is -//! [`core::Store`] and [`core::Env`]. +//! [`core::Store`], [`core::Clock`], and [`core::Random`]. //! //! | Feature | Crate | //! |---|---| //! | `iceberg` (default) | [`iceberg`]: Apache Iceberg (`lakekit-iceberg`) | //! | `iceberg-write`, `iceberg-arrow`, `iceberg-parquet-stats` | its optional parts | //! | `object-store` | `object_store`: a store over an `object_store` client (`lakekit-object-store`) | -//! | `tokio` | `core::TokioEnv`, a native environment | +//! | `tokio`, `getrandom` | `core::TokioClock` and `core::OsRandom`, native ones | pub use lakekit_core as core; #[cfg(feature = "iceberg")] diff --git a/spec/0001-store-clock-random.md b/spec/0001-store-clock-random.md new file mode 100644 index 0000000..ec21d29 --- /dev/null +++ b/spec/0001-store-clock-random.md @@ -0,0 +1,62 @@ +--- +number: 1 +title: Store, Clock, and Random +status: done +updated: 2026-10-21 +versions: [unreleased] +related: [2, 3, 4] +issues: [] +--- + +# 1. Store, Clock, and Random + +**Done** · updated 2026-10-21 · versions unreleased · milestone: [0.1](milestones/v0-1.md) + +## Summary + +Format crates do no I/O of their own. They read and write objects through a `Store`, take the time from a `Clock`, and +take random numbers from a `Random` (`lakekit-core`), all supplied by the application, so the same code runs in a +browser, a Cloudflare Worker, a CLI, or a server, and tests run against memory. + +## Design + +- **`Store`:** `get` (the object or `None`), `get_range`, `exists`, `put`, `put_if_absent` (the commit primitive: + `false` when an object is there), `upload` (a multipart upload: parts in order, then complete or abort), `list` (by + string prefix, not only by directory), and `delete` (missing objects are not an error). Paths are absolute URIs, as + table metadata records them. Adapters implement it over store libraries ([spec 4](0004-store-adapters.md)). +- **`Clock`:** `now_ms` and `sleep`: commit timestamps, time-ordered file names, and the wait between commit attempts. +- **`Random`:** `next_u64`: snapshot ids, file names, and the jitter between commit attempts. +- **Who needs what:** readers need only a `Store`; commits need `Store + Clock + Random`. +- **`Services`:** a store, a clock, and a random source as one value (`Services::new(store, clock, random)`), for APIs + that need all three. A type that is all three (a JavaScript host, `MemoryStore`) needs no wrapper. +- **Native implementations:** `TokioClock` (feature `tokio`: the system clock and tokio's timer) and `OsRandom` + (feature `getrandom`: the operating system's generator), on native targets only. +- **Futures aren't `Send`.** Methods return `impl Future`, without a `Send` bound, so a store or clock can wrap + JavaScript promises (wasm-bindgen). Native ones are free to be `Send`. +- **`MemoryStore`:** a store in memory that is also a clock that only moves when told (or when `sleep` is called) and a + seeded random sequence, so tests are deterministic. +- **`Error`:** one error type for all crates, with the variant saying what a caller can do: `Invalid`, `Unsupported`, + `Conflict` (retry on the newer version), `CommitUnknown` (the commit may have landed: delete nothing it wrote, load + the table again), `NotFound`, `Store`, and `Corrupt`. It is `non_exhaustive`. +- **Identifiers:** `uuid_v4` (from a `Random`) and `uuid_v7` (from a `Clock + Random`), built by the `uuid` crate (its + builder needs neither its own randomness nor `std::time`, so it works in WebAssembly). + +## Decisions + +| ID | Decision | Status | +|---|---|---| +| D1 | Format crates do no I/O: everything goes through traits the application implements (the idea of delta-kernel-rs's `Engine`) | Accepted | +| D2 | Store and clock futures aren't `Send`, so JavaScript hosts work; native applications lose nothing | Accepted | +| D3 | One error type across crates, whose variants say what a caller can do | Accepted | +| D10 | Objects are a `Store` of their own, apart from the time and randomness: readers need only a store, and store adapters don't decide how time is told | Accepted | +| D13 | The time and randomness are two traits, `Clock` and `Random`, as ZIO's services are, rather than one environment trait: each says what it is, and each can be swapped on its own | Accepted | + +## Open questions and issues + +None open. + +## History + +- 2026-10-20: built, from Timelake's `timelake_format::host`. +- 2026-10-21: one `Host` trait split into `Store` (D10), `Clock`, and `Random` (D13); `Services`, `TokioClock`, and + `OsRandom` added; UUIDs from the `uuid` crate. The spec was "The host boundary". diff --git a/spec/0001-store-env.md b/spec/0001-store-env.md deleted file mode 100644 index 9ee9a3e..0000000 --- a/spec/0001-store-env.md +++ /dev/null @@ -1,60 +0,0 @@ ---- -number: 1 -title: Store and environment -status: done -updated: 2026-10-21 -versions: [unreleased] -related: [2, 3, 4] -issues: [] ---- - -# 1. Store and environment - -**Done** · updated 2026-10-21 · versions unreleased · milestone: [0.1](milestones/v0-1.md) - -## Summary - -Format crates do no I/O of their own. They read and write objects through a `Store` and take the time and randomness -from an `Env` (`lakekit-core`), both supplied by the application, so the same code runs in a browser, a Cloudflare -Worker, a CLI, or a server, and tests run against memory. - -## Design - -- **`Store`:** `get` (the object or `None`), `get_range`, `exists`, `put`, `put_if_absent` (the commit primitive: - `false` when an object is there), `upload` (a multipart upload: parts in order, then complete or abort), `list` (by - string prefix, not only by directory), and `delete` (missing objects are not an error). Paths are absolute URIs, as - table metadata records them. Adapters implement it over store libraries ([spec 4](0004-store-adapters.md)). -- **`Env`:** `now_ms`, `random`, and `sleep`: what a writer needs besides its store (snapshot ids, file names, commit - timestamps, the wait between commit attempts). Readers need only a `Store`. -- **`WithEnv`:** a store and an environment as one value (`WithEnv::new(store, env)`), for APIs that need both, such as - `lakekit_iceberg::commit`. A JavaScript host that is both implements both traits on one type. -- **`TokioEnv`** (feature `tokio`, native targets only): the system clock, the operating system's randomness, and - tokio's timer. -- **Futures aren't `Send`.** Methods return `impl Future`, without a `Send` bound, so a store can wrap JavaScript - promises (wasm-bindgen). Native stores are free to be `Send`. -- **`MemoryStore`:** a store in memory that is also an environment, with a clock that only moves when told (or when - `sleep` is called) and a seeded random sequence, so tests are deterministic. -- **`Error`:** one error type for all crates, with the variant saying what a caller can do: `Invalid`, `Unsupported`, - `Conflict` (retry on the newer version), `CommitUnknown` (the commit may have landed: delete nothing it wrote, load - the table again), `NotFound`, `Store`, and `Corrupt`. It is `non_exhaustive`. -- **Identifiers:** `uuid_v4` and `uuid_v7`, built by the `uuid` crate from an environment's randomness and clock (its - builder needs neither its own randomness nor `std::time`, so it works in WebAssembly). - -## Decisions - -| ID | Decision | Status | -|---|---|---| -| D1 | Format crates do no I/O: everything goes through traits the application implements (the idea of delta-kernel-rs's `Engine`) | Accepted | -| D2 | Store and environment futures aren't `Send`, so JavaScript hosts work; native applications lose nothing | Accepted | -| D3 | One error type across crates, whose variants say what a caller can do | Accepted | -| D10 | Two small traits, `Store` for objects and `Env` for the clock, randomness, and waiting, rather than one `Host`: names that say what they are, readers that need only a store, and store adapters that don't decide how time is told | Accepted | - -## Open questions and issues - -None open. - -## History - -- 2026-10-20: built, from Timelake's `timelake_format::host`. -- 2026-10-21: one `Host` trait split into `Store` and `Env` (D10); `TokioEnv` and `WithEnv` added; UUIDs from the - `uuid` crate. The spec was "The host boundary". diff --git a/spec/0003-iceberg.md b/spec/0003-iceberg.md index 21f4254..5fd15fb 100644 --- a/spec/0003-iceberg.md +++ b/spec/0003-iceberg.md @@ -41,13 +41,13 @@ shared writer, made general. any metadata file a catalog names (`from_metadata_file`); manifests and live files. - **Writing (feature `write`):** format v2 data manifests and manifest lists in the layout iceberg-rust writes, and metadata updates (`with_snapshot`, `without_snapshots`) made as iceberg-rust's builder makes them, with the metadata - log trimmed to `write.metadata.previous-versions-max`; the current snapshot can't be removed. Partition values of type uuid, fixed, and decimal are refused - for now. + log trimmed to `write.metadata.previous-versions-max`; the current snapshot can't be removed. Partition values of + type uuid, fixed, and decimal are refused for now. - **Commits (feature `write`):** upload a snapshot's files, then write `v.metadata.json` only if absent; on a lost race, reload and try again on the newer version, deleting the lost attempt's files; when the outcome is unknown (a conditional put that failed and can't be read back), delete nothing. Appends, replacements that keep data - sequence numbers (compaction, one sequence number per added file), and deletions; manifests merge once there would be more than `merge_manifests_at`. - Iceberg's summary counters are kept. A writer whose clock is behind the table's last update dates its commit at that + sequence numbers (compaction, one sequence number per added file), and deletions; manifests merge once there would be + more than `merge_manifests_at`. Iceberg's summary counters are kept. A writer whose clock is behind the table's last update dates its commit at that update, so the snapshot log stays in order (iceberg-rust refuses tables whose log isn't). - **Options and hooks:** `CommitOptions` (`max_attempts` 10, `backoff_ms` 5–50 times the attempt, `merge_manifests_at` 8, `write_version_hint` on) and `CommitHooks` (`reload`, `skip` for idempotent retries, `extra` for files and diff --git a/spec/0004-store-adapters.md b/spec/0004-store-adapters.md index 88efab5..367af9d 100644 --- a/spec/0004-store-adapters.md +++ b/spec/0004-store-adapters.md @@ -14,10 +14,10 @@ issues: [10] ## Summary -Ready-made `Store`s ([spec 1](0001-store-env.md)) over the store libraries applications already use, so a native -application doesn't write its own. Each adapter is a crate of its own; the `Store` trait stays the boundary, because -some stores aren't these libraries (a Cloudflare R2 binding reached from JavaScript) and applications wrap their store -(lakeglass counts every request). +Ready-made `Store`s ([spec 1](0001-store-clock-random.md)) over the store libraries applications already use, so a +native application doesn't write its own. Each adapter is a crate of its own; the `Store` trait stays the boundary, +because some stores aren't these libraries (a Cloudflare R2 binding reached from JavaScript) and applications wrap +their store (lakeglass counts every request). ## Design diff --git a/spec/README.md b/spec/README.md index a0a827e..50c346e 100644 --- a/spec/README.md +++ b/spec/README.md @@ -28,11 +28,11 @@ lakekit's design, laid out like Timelake's and lakeglass's. Start here, people a ## Index -4 specs: 3 in progress, 1 done. Decisions D1–D12. +4 specs: 3 in progress, 1 done. Decisions D1–D13. | # | Spec | Status | Updated | |---:|---|---|---| -| 1 | [Store and environment](0001-store-env.md) | Done | 2026-10-21 | +| 1 | [Store, Clock, and Random](0001-store-clock-random.md) | Done | 2026-10-21 | | 2 | [One crate per format](0002-formats.md) | In progress | 2026-10-21 | | 3 | [Apache Iceberg](0003-iceberg.md) | In progress | 2026-10-20 | | 4 | [Store adapters](0004-store-adapters.md) | In progress | 2026-10-21 | diff --git a/spec/milestones/v0-1.md b/spec/milestones/v0-1.md index eeffcc9..dde1ad1 100644 --- a/spec/milestones/v0-1.md +++ b/spec/milestones/v0-1.md @@ -2,13 +2,13 @@ **In progress** -The first release: the store and environment traits, the Iceberg crate, and the `object_store` adapter, enough for +The first release: the store, clock, and random traits, the Iceberg crate, and the `object_store` adapter, enough for Timelake's WASM reader and writer to use lakekit instead of their own Iceberg code, and for lakeglass to read Iceberg tables in the browser. | # | Spec | Status | |---:|---|---| -| 1 | [Store and environment](../0001-store-env.md) | Done | +| 1 | [Store, Clock, and Random](../0001-store-clock-random.md) | Done | | 2 | [One crate per format](../0002-formats.md) | In progress | | 3 | [Apache Iceberg](../0003-iceberg.md) | In progress | | 4 | [Store adapters](../0004-store-adapters.md) | In progress | From aec2b9d5b23d803eefd7d072678c9d4a94047adf Mon Sep 17 00:00:00 2001 From: Oleksandr Litus Date: Tue, 20 Oct 2026 23:59:47 -0400 Subject: [PATCH 05/14] Core: Store, Clock, and Random come from loko The three traits, MemoryStore, the test and native implementations, and the object_store adapter are loko's (polythings/loko), shared by the polythings projects. lakekit-core re-exports them and keeps the error type (a store's error converts into it), the UUID helpers, and Services, now built with loko's derive. MemoryServices bundles a MemoryStore, a TestClock, and a SeededRandom for tests. Clock::sleep takes a Duration. lakekit-object-store is gone; the lakekit crate's object-store feature re-exports loko-object-store, and the Iceberg commit through object_store is a lakekit-iceberg test. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 14 +- CONTRIBUTING.md | 2 +- Cargo.lock | 164 ++++------- Cargo.toml | 10 +- PRIOR_ART.md | 3 +- README.md | 12 +- crates/lakekit-core/Cargo.toml | 19 +- crates/lakekit-core/README.md | 15 +- crates/lakekit-core/src/clock.rs | 42 --- crates/lakekit-core/src/error.rs | 28 ++ crates/lakekit-core/src/ids.rs | 8 +- crates/lakekit-core/src/lib.rs | 38 ++- crates/lakekit-core/src/memory.rs | 143 ---------- crates/lakekit-core/src/random.rs | 28 -- crates/lakekit-core/src/services.rs | 70 ++--- crates/lakekit-core/src/store.rs | 49 ---- crates/lakekit-iceberg/Cargo.toml | 3 + crates/lakekit-iceberg/src/commit.rs | 46 +-- crates/lakekit-iceberg/src/table.rs | 2 +- crates/lakekit-iceberg/tests/iceberg_rust.rs | 28 +- crates/lakekit-iceberg/tests/object_store.rs | 43 +++ crates/lakekit-object-store/Cargo.toml | 32 --- crates/lakekit-object-store/README.md | 24 -- crates/lakekit-object-store/src/lib.rs | 280 ------------------- crates/lakekit/Cargo.toml | 6 +- crates/lakekit/src/lib.rs | 6 +- spec/0001-store-clock-random.md | 25 +- spec/0004-store-adapters.md | 24 +- spec/README.md | 4 +- spec/milestones/v0-1.md | 6 +- 31 files changed, 293 insertions(+), 883 deletions(-) delete mode 100644 crates/lakekit-core/src/clock.rs delete mode 100644 crates/lakekit-core/src/memory.rs delete mode 100644 crates/lakekit-core/src/random.rs delete mode 100644 crates/lakekit-core/src/store.rs create mode 100644 crates/lakekit-iceberg/tests/object_store.rs delete mode 100644 crates/lakekit-object-store/Cargo.toml delete mode 100644 crates/lakekit-object-store/README.md delete mode 100644 crates/lakekit-object-store/src/lib.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 61f818b..1226776 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: cargo check -p lakekit-core --no-default-features cargo check -p lakekit-core --features tokio cargo check -p lakekit-core --features getrandom - cargo check -p lakekit-object-store + cargo check -p lakekit --features object-store wasm: name: WebAssembly (browser and WASI targets) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28e9889..da915fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,11 @@ versions follow [Semantic Versioning](https://semver.org/). ### Added -- `lakekit-core`: the `Store`, `Clock`, and `Random` traits; `Services` to pass the three as one; `MemoryStore`; - `TokioClock` (feature `tokio`) and `OsRandom` (feature `getrandom`); one `Error` type. -- `lakekit-object-store`: a `Store` over an `object_store` client (S3, GCS, Azure, local files, memory), from a client - or a URL. +- `lakekit-core`: loko's `Store`, `Clock`, and `Random` services with their test and native implementations + (`TokioClock` with feature `tokio`, `OsRandom` with feature `getrandom`); `Services` to pass the three as one; + `MemoryServices` for tests; one `Error` type, which a store's error converts into. +- `lakekit` feature `object-store`: a `Store` over an `object_store` client (S3, GCS, Azure, local files, memory), + from a client or a URL (`loko-object-store`). - `lakekit-iceberg`: Iceberg table metadata, schemas, partition specs, manifest lists, and manifests read for format versions 1 to 3, including delete files and deletion vector references; values in Iceberg's single-value encoding. - `lakekit-iceberg` feature `write`: format v2 manifests, manifest lists, and metadata updates, and commits to @@ -27,6 +28,9 @@ versions follow [Semantic Versioning](https://semver.org/). (`lakebed-core`, …) until 2026-10-21. Nothing was published under the old names; GitHub redirects the old repository URL. - The I/O boundary is three traits, `Store` (objects), `Clock` (the time and a timer), and `Random` (random numbers), - instead of one `Host`; `MemoryHost` is `MemoryStore`. UUIDs come from the `uuid` crate. + instead of one `Host`; `MemoryHost` is `MemoryServices`. UUIDs come from the `uuid` crate. +- The three traits are [loko](https://github.com/polythings/loko)'s, and `Clock::sleep` takes a `Duration`. The + `object_store` adapter is `loko-object-store`, re-exported as `lakekit::object_store`; `lakekit-object-store` is + gone (it was never published). The Iceberg code started as Timelake's shared writer (`timelake-format`), made general. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 953cfe4..e868f58 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,7 +50,7 @@ ideas we took. ## Tests - **A bug fix starts with a test that fails**, then the fix. -- **Tests run offline**, against `MemoryStore` or a store over the reference implementation's in-memory storage. +- **Tests run offline**, against `MemoryServices` or a store over the reference implementation's in-memory storage. - **Interop:** writers are tested against the format's reference reader, and readers against the reference writer. ## Checks diff --git a/Cargo.lock b/Cargo.lock index 2a03c5e..3a2bb36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -528,12 +528,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" -[[package]] -name = "cfg_aliases" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" - [[package]] name = "chacha20" version = "0.10.2" @@ -887,16 +881,6 @@ dependencies = [ "typeid", ] -[[package]] -name = "errno" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - [[package]] name = "event-listener" version = "5.4.2" @@ -1632,16 +1616,18 @@ version = "0.1.0" dependencies = [ "lakekit-core", "lakekit-iceberg", - "lakekit-object-store", + "loko-object-store", ] [[package]] name = "lakekit-core" version = "0.1.0" dependencies = [ - "getrandom 0.4.3", + "loko-clock", + "loko-macros", + "loko-random", + "loko-store", "thiserror", - "tokio", "uuid", ] @@ -1657,24 +1643,13 @@ dependencies = [ "futures-util", "iceberg", "lakekit-core", + "loko-object-store", + "object_store", "parquet", "serde_json", "tokio", ] -[[package]] -name = "lakekit-object-store" -version = "0.1.0" -dependencies = [ - "futures-util", - "lakekit-core", - "lakekit-iceberg", - "object_store", - "tempfile", - "tokio", - "url", -] - [[package]] name = "lexical-core" version = "1.0.6" @@ -1744,12 +1719,6 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" -[[package]] -name = "linux-raw-sys" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" - [[package]] name = "litemap" version = "0.8.3" @@ -1771,6 +1740,54 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "loko-clock" +version = "0.2.0" +source = "git+https://github.com/polythings/loko?rev=3972fdd0f2dc863328c38048583f3066cba939c4#3972fdd0f2dc863328c38048583f3066cba939c4" +dependencies = [ + "loko-macros", + "tokio", +] + +[[package]] +name = "loko-macros" +version = "0.2.0" +source = "git+https://github.com/polythings/loko?rev=3972fdd0f2dc863328c38048583f3066cba939c4#3972fdd0f2dc863328c38048583f3066cba939c4" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "loko-object-store" +version = "0.2.0" +source = "git+https://github.com/polythings/loko?rev=3972fdd0f2dc863328c38048583f3066cba939c4#3972fdd0f2dc863328c38048583f3066cba939c4" +dependencies = [ + "futures-util", + "loko-store", + "object_store", + "url", +] + +[[package]] +name = "loko-random" +version = "0.2.0" +source = "git+https://github.com/polythings/loko?rev=3972fdd0f2dc863328c38048583f3066cba939c4#3972fdd0f2dc863328c38048583f3066cba939c4" +dependencies = [ + "getrandom 0.4.3", + "loko-macros", +] + +[[package]] +name = "loko-store" +version = "0.2.0" +source = "git+https://github.com/polythings/loko?rev=3972fdd0f2dc863328c38048583f3066cba939c4#3972fdd0f2dc863328c38048583f3066cba939c4" +dependencies = [ + "loko-macros", + "thiserror", +] + [[package]] name = "lz4_flex" version = "0.13.1" @@ -1842,18 +1859,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" -[[package]] -name = "nix" -version = "0.31.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" -dependencies = [ - "bitflags 2.13.2", - "cfg-if", - "cfg_aliases", - "libc", -] - [[package]] name = "num-bigint" version = "0.4.8" @@ -1914,17 +1919,12 @@ dependencies = [ "http", "humantime", "itertools 0.15.0", - "nix", "parking_lot", "percent-encoding", "thiserror", - "tokio", - "tracing", "url", - "walkdir", "wasm-bindgen-futures", "web-time", - "windows-sys 0.61.2", ] [[package]] @@ -2364,19 +2364,6 @@ dependencies = [ "semver", ] -[[package]] -name = "rustix" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" -dependencies = [ - "bitflags 2.13.2", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.61.2", -] - [[package]] name = "rustversion" version = "1.0.23" @@ -2389,15 +2376,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "schemars" version = "0.9.0" @@ -2718,19 +2696,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" -[[package]] -name = "tempfile" -version = "3.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" -dependencies = [ - "fastrand", - "getrandom 0.4.3", - "once_cell", - "rustix", - "windows-sys 0.61.2", -] - [[package]] name = "thiserror" version = "2.0.21" @@ -3045,16 +3010,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - [[package]] name = "want" version = "0.3.2" @@ -3155,15 +3110,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "winapi-util" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" -dependencies = [ - "windows-sys 0.61.2", -] - [[package]] name = "windows-core" version = "0.62.2" diff --git a/Cargo.toml b/Cargo.toml index ebff389..22949a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "2" -members = ["crates/lakekit-core", "crates/lakekit-iceberg", "crates/lakekit-object-store", "crates/lakekit"] +members = ["crates/lakekit-core", "crates/lakekit-iceberg", "crates/lakekit"] [workspace.package] version = "0.1.0" @@ -14,7 +14,13 @@ categories = ["database-implementations", "wasm"] [workspace.dependencies] lakekit-core = { path = "crates/lakekit-core", version = "0.1.0" } lakekit-iceberg = { path = "crates/lakekit-iceberg", version = "0.1.0", default-features = false } -lakekit-object-store = { path = "crates/lakekit-object-store", version = "0.1.0" } +# loko provides the Store, Clock, and Random services. From git until loko 0.2.0 is on crates.io +# (polythings/loko#1); publishing uses the version. +loko-store = { git = "https://github.com/polythings/loko", rev = "3972fdd0f2dc863328c38048583f3066cba939c4", version = "0.2.0" } +loko-clock = { git = "https://github.com/polythings/loko", rev = "3972fdd0f2dc863328c38048583f3066cba939c4", version = "0.2.0" } +loko-random = { git = "https://github.com/polythings/loko", rev = "3972fdd0f2dc863328c38048583f3066cba939c4", version = "0.2.0" } +loko-macros = { git = "https://github.com/polythings/loko", rev = "3972fdd0f2dc863328c38048583f3066cba939c4", version = "0.2.0" } +loko-object-store = { git = "https://github.com/polythings/loko", rev = "3972fdd0f2dc863328c38048583f3066cba939c4", version = "0.2.0" } object_store = { version = "0.14", default-features = false } url = "2" thiserror = "2" diff --git a/PRIOR_ART.md b/PRIOR_ART.md index ca23a01..741bdad 100644 --- a/PRIOR_ART.md +++ b/PRIOR_ART.md @@ -11,7 +11,8 @@ never code: every line of lakekit is our own. How we credit is in [CONTRIBUTING. | [iceberg-rust](https://github.com/apache/iceberg-rust) | Apache-2.0 | The reference the interop tests check against (a test dependency only) | | [Apache Avro](https://avro.apache.org) (`apache-avro`) | Apache-2.0 | Reading and writing manifests and manifest lists | | [arrow-rs](https://github.com/apache/arrow-rs) (`arrow-schema`, `parquet`) | Apache-2.0 | Arrow schemas; Parquet footers for statistics | -| [object_store](https://github.com/apache/arrow-rs-object-store) | Apache-2.0 / MIT | The stores `lakekit-object-store` adapts | +| [loko](https://github.com/polythings/loko) | Apache-2.0 | The `Store`, `Clock`, and `Random` services, and the `object_store` adapter | +| [object_store](https://github.com/apache/arrow-rs-object-store) | Apache-2.0 / MIT | The stores `loko-object-store` adapts (re-exported by `lakekit`) | | [uuid](https://github.com/uuid-rs/uuid) | Apache-2.0 / MIT | UUIDs for file names, from a `Random` and a `Clock` | ## Ideas diff --git a/README.md b/README.md index b507981..e884e6b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ Table formats read and written without I/O of their own, in Rust, for WebAssembly and native targets. Each format's metadata (Apache Iceberg first) is parsed and written by plain code that asks a `Store` for bytes, a `Clock` for the -time, and a `Random` for random numbers: the application supplies them, so the same crate runs in a browser, a +time, and a `Random` for random numbers ([loko](https://github.com/polythings/loko)'s services): the application +supplies them, so the same crate runs in a browser, a Cloudflare Worker, a CLI, or a server. lakekit is what [Timelake](https://github.com/polythings/timelake) and [lakeglass](https://github.com/polythings/lakeglass) @@ -13,12 +14,11 @@ share. The official Rust crates for these formats build on tokio and HTTP client | Crate | What it does | |---|---| -| [`lakekit-core`](crates/lakekit-core) | The `Store`, `Clock`, and `Random` traits, errors, an in-memory store, a tokio clock, OS randomness | +| [`lakekit-core`](crates/lakekit-core) | loko's `Store`, `Clock`, and `Random` with their test and native implementations, `Services`, and one error type | | [`lakekit-iceberg`](crates/lakekit-iceberg) | Apache Iceberg: metadata, manifests, and commits | -| [`lakekit-object-store`](crates/lakekit-object-store) | A `Store` over an `object_store` client: S3, GCS, Azure, local files, memory | -| [`lakekit`](crates/lakekit) | Re-exports the crates, one feature each | +| [`lakekit`](crates/lakekit) | Re-exports the crates, one feature each, and `loko-object-store` (a `Store` over an `object_store` client) | -More formats get their own crate as they are added (spec 2), and more stores their own adapter (spec 4). +More formats get their own crate as they are added (spec 2). Store adapters are loko's (spec 4). ## Iceberg @@ -37,7 +37,7 @@ What lakekit writes, iceberg-rust reads, and the other way round: the tests comm use std::sync::Arc; use lakekit_iceberg::Table; -use lakekit_object_store::ObjectStoreAdapter; +use loko_object_store::ObjectStoreAdapter; async fn files(s3: Arc) -> lakekit_core::Result { let store = ObjectStoreAdapter::new(s3, "s3://bucket"); diff --git a/crates/lakekit-core/Cargo.toml b/crates/lakekit-core/Cargo.toml index 99eed81..ec105e6 100644 --- a/crates/lakekit-core/Cargo.toml +++ b/crates/lakekit-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lakekit-core" -description = "The I/O boundary shared by lakekit's table format crates: traits for objects, the clock, and randomness" +description = "The I/O boundary shared by lakekit's table format crates: loko's Store, Clock, and Random, and one error type" readme = "README.md" version.workspace = true edition.workspace = true @@ -13,21 +13,18 @@ categories.workspace = true [features] default = [] # `TokioClock`: the system clock and tokio's timer. Native targets only. -tokio = ["dep:tokio"] -# `OsRandom`: the operating system's random number generator. Native targets only. -getrandom = ["dep:getrandom"] +tokio = ["loko-clock/tokio"] +# `OsRandom`: the operating system's random number generator. Native targets and WASI. +getrandom = ["loko-random/getrandom"] [dependencies] +loko-store.workspace = true +loko-clock.workspace = true +loko-random.workspace = true +loko-macros.workspace = true thiserror.workspace = true uuid = { version = "1", default-features = false } -[target.'cfg(not(target_arch = "wasm32"))'.dependencies] -tokio = { workspace = true, features = ["time"], optional = true } -getrandom = { version = "0.4", optional = true } - -[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] -tokio = { workspace = true, features = ["macros", "rt", "time"] } - [package.metadata.docs.rs] all-features = true diff --git a/crates/lakekit-core/README.md b/crates/lakekit-core/README.md index b6527fe..0c081bf 100644 --- a/crates/lakekit-core/README.md +++ b/crates/lakekit-core/README.md @@ -2,16 +2,19 @@ The I/O boundary of [lakekit](https://github.com/polythings/lakekit)'s table format crates. Format crates do no I/O of their own: they read and write through a `Store`, take the time from a `Clock`, and take random numbers from a -`Random`, all supplied by the application, so they build for `wasm32-unknown-unknown` and native targets alike. +`Random`, all supplied by the application ([loko](https://github.com/polythings/loko)'s services, re-exported here), so they build for `wasm32-unknown-unknown` and native targets alike. - `Store`: get, ranged get, exists, put, put-if-absent (the commit primitive), multipart upload, list, delete. Its - futures aren't `Send`, so a store can wrap JavaScript promises. `lakekit-object-store` implements it over + futures aren't `Send`, so a store can wrap JavaScript promises. `loko-object-store` implements it over `object_store`. -- `Clock`: the time and a timer. `TokioClock` (feature `tokio`, native only) uses the system clock and tokio's timer. -- `Random`: random numbers. `OsRandom` (feature `getrandom`, native only) uses the operating system's generator. +- `Clock`: the time and a timer. `TokioClock` (feature `tokio`, native only) uses the system clock and tokio's timer; + `TestClock` moves only when told. +- `Random`: random numbers. `OsRandom` (feature `getrandom`, native and WASI) uses the operating system's generator; + `SeededRandom` repeats a sequence from a seed. - `Services`: a store, a clock, and a random source as one value, for APIs that need all three (commits). -- `MemoryStore`: all three in memory, with a clock that moves only when told and seeded randomness, for tests. +- `MemoryServices`: a `MemoryStore`, a `TestClock`, and a `SeededRandom`, for tests. - `uuid_v4`, `uuid_v7`: identifiers from a `Random` (and a `Clock`), with the `uuid` crate. -- `Error`: one error type, whose variant says what a caller can do (`NotFound`, `Conflict`, `CommitUnknown`, …). +- `Error`: one error type, whose variant says what a caller can do (`NotFound`, `Conflict`, `CommitUnknown`, …); a + store's error converts into it. Apache License 2.0. diff --git a/crates/lakekit-core/src/clock.rs b/crates/lakekit-core/src/clock.rs deleted file mode 100644 index c21a224..0000000 --- a/crates/lakekit-core/src/clock.rs +++ /dev/null @@ -1,42 +0,0 @@ -use std::future::Future; - -/// The time and a timer: commit timestamps, time-ordered file names, and the wait between commit -/// attempts. -/// -/// The future isn't required to be `Send`, so a clock can wrap JavaScript timers. -pub trait Clock { - /// Milliseconds since the Unix epoch. - fn now_ms(&self) -> i64; - /// Waits `ms` milliseconds. - fn sleep(&self, ms: u64) -> impl Future; -} - -/// The system clock and tokio's timer (feature `tokio`, native targets only). -#[cfg(all(feature = "tokio", not(target_arch = "wasm32")))] -#[derive(Clone, Copy, Debug, Default)] -pub struct TokioClock; - -#[cfg(all(feature = "tokio", not(target_arch = "wasm32")))] -impl Clock for TokioClock { - fn now_ms(&self) -> i64 { - let since = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap_or_default(); - i64::try_from(since.as_millis()).unwrap_or(i64::MAX) - } - - fn sleep(&self, ms: u64) -> impl Future { - tokio::time::sleep(std::time::Duration::from_millis(ms)) - } -} - -#[cfg(all(test, feature = "tokio", not(target_arch = "wasm32")))] -mod tests { - use super::*; - - #[tokio::test] - async fn the_tokio_clock_tells_time_and_waits() { - let before = TokioClock.now_ms(); - assert!(before > 1_700_000_000_000, "after November 2023"); - TokioClock.sleep(5).await; - assert!(TokioClock.now_ms() >= before + 5); - } -} diff --git a/crates/lakekit-core/src/error.rs b/crates/lakekit-core/src/error.rs index 662fd96..8d6441a 100644 --- a/crates/lakekit-core/src/error.rs +++ b/crates/lakekit-core/src/error.rs @@ -28,3 +28,31 @@ pub enum Error { /// `Result` with [`Error`]. pub type Result = std::result::Result; + +impl From for Error { + fn from(e: loko_store::Error) -> Self { + match e { + loko_store::Error::NotFound(m) => Error::NotFound(m), + loko_store::Error::Invalid(m) => Error::Invalid(m), + loko_store::Error::Unsupported(m) => Error::Unsupported(m), + other => Error::Store(match other { + loko_store::Error::Store(m) => m, + other => other.to_string(), + }), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn store_errors_keep_their_kind() { + let from = |e| Error::from(e).to_string(); + assert_eq!(from(loko_store::Error::NotFound("a".into())), "not found: a"); + assert_eq!(from(loko_store::Error::Invalid("b".into())), "invalid: b"); + assert_eq!(from(loko_store::Error::Unsupported("c".into())), "unsupported: c"); + assert_eq!(from(loko_store::Error::Store("d".into())), "store: d"); + } +} diff --git a/crates/lakekit-core/src/ids.rs b/crates/lakekit-core/src/ids.rs index 32e063c..0825893 100644 --- a/crates/lakekit-core/src/ids.rs +++ b/crates/lakekit-core/src/ids.rs @@ -22,12 +22,14 @@ pub fn uuid_v7(source: &(impl Clock + Random)) -> String { #[cfg(test)] mod tests { + use std::time::Duration; + use super::*; - use crate::MemoryStore; + use crate::MemoryServices; #[test] fn uuids_are_well_formed() { - let source = MemoryStore::new(1_767_225_600_000); + let source = MemoryServices::at_ms(1_767_225_600_000); let v4 = uuid_v4(&source); let v7 = uuid_v7(&source); for (u, version) in [(&v4, '4'), (&v7, '7')] { @@ -36,7 +38,7 @@ mod tests { assert!(matches!(u.chars().nth(19), Some('8' | '9' | 'a' | 'b'))); } // Version 7 starts with the time, so later ones sort later. - source.advance(1); + source.clock.advance(Duration::from_millis(1)); assert!(uuid_v7(&source) > v7); } } diff --git a/crates/lakekit-core/src/lib.rs b/crates/lakekit-core/src/lib.rs index 10d0a29..976a206 100644 --- a/crates/lakekit-core/src/lib.rs +++ b/crates/lakekit-core/src/lib.rs @@ -1,21 +1,22 @@ //! The I/O boundary shared by lakekit's table format crates. Format crates do no I/O of their own: -//! they read and write through a [`Store`] the application implements over its objects (an -//! `object_store` client natively, `fetch` or an R2 binding in WebAssembly), and take the time from -//! a [`Clock`] and random numbers from a [`Random`], so they build for `wasm32-unknown-unknown` and -//! native targets alike. +//! they read and write through a [`Store`] the application supplies (an `object_store` client +//! natively, `fetch` or an R2 binding in WebAssembly), and take the time from a [`Clock`] and random +//! numbers from a [`Random`], so they build for `wasm32-unknown-unknown` and native targets alike. +//! The three are [loko](https://github.com/polythings/loko)'s services, re-exported here. //! //! - [`Store`]: get, ranged get, put, put-if-absent (the commit primitive), multipart upload, list, -//! delete. +//! delete. `loko-object-store` implements it over `object_store`. //! - [`Clock`]: the time and a timer. `TokioClock` (feature `tokio`) is a native one. //! - [`Random`]: random numbers. `OsRandom` (feature `getrandom`) is a native one. -//! - [`Services`]: a store, a clock, and a random source as one value, for APIs that need all three. -//! - [`MemoryStore`]: all three, in memory, for tests. +//! - [`Services`]: a store, a clock, and a random source as one value, for APIs that need all three; +//! [`MemoryServices`] is all three in memory, for tests. +//! - [`Error`]: one error type for the format crates, converted from the store's. //! //! ``` //! use lakekit_core::{MemoryStore, Store}; //! //! # block_on(async { -//! let store = MemoryStore::new(0); +//! let store = MemoryStore::new(); //! store.put("memory://t/a", b"x".to_vec()).await?; //! assert!(store.exists("memory://t/a").await?); //! # Ok::<(), lakekit_core::Error>(()) @@ -28,22 +29,17 @@ //! # } //! ``` -mod clock; mod error; mod ids; -mod memory; -mod random; mod services; -mod store; -pub use clock::Clock; -#[cfg(all(feature = "tokio", not(target_arch = "wasm32")))] -pub use clock::TokioClock; pub use error::{Error, Result}; pub use ids::{uuid_v4, uuid_v7}; -pub use memory::{MemoryStore, MemoryUpload}; -#[cfg(all(feature = "getrandom", not(target_arch = "wasm32")))] -pub use random::OsRandom; -pub use random::Random; -pub use services::Services; -pub use store::{Listed, Store, Upload}; +#[cfg(all(feature = "tokio", not(target_arch = "wasm32")))] +pub use loko_clock::TokioClock; +pub use loko_clock::{Clock, HasClock, TestClock}; +#[cfg(all(feature = "getrandom", not(all(target_arch = "wasm32", target_os = "unknown"))))] +pub use loko_random::OsRandom; +pub use loko_random::{HasRandom, Random, SeededRandom}; +pub use loko_store::{Error as StoreError, HasStore, Listed, MemoryStore, MemoryUpload, Store, Upload}; +pub use services::{MemoryServices, Services}; diff --git a/crates/lakekit-core/src/memory.rs b/crates/lakekit-core/src/memory.rs deleted file mode 100644 index e4fca46..0000000 --- a/crates/lakekit-core/src/memory.rs +++ /dev/null @@ -1,143 +0,0 @@ -use std::collections::BTreeMap; -use std::future::{Future, ready}; -use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; - -use crate::{Clock, Error, Listed, Random, Result, Store, Upload}; - -/// A [`Store`] in memory that is also a [`Clock`] that moves only when told (or when `sleep` is -/// called) and a [`Random`] with a seeded sequence: for tests and examples. Clones share the same -/// objects. -#[derive(Clone, Debug, Default)] -pub struct MemoryStore { - objects: Arc>>>, - clock: Arc>, - seed: Arc>, -} - -impl MemoryStore { - /// An empty store at time `now_ms`. - pub fn new(now_ms: i64) -> Self { - let store = Self::default(); - *lock(&store.clock) = now_ms; - store - } - - /// Moves the clock forward. - pub fn advance(&self, ms: i64) { - *lock(&self.clock) += ms; - } - - /// The URIs stored, sorted. - pub fn uris(&self) -> Vec { - self.objects().keys().cloned().collect() - } - - fn objects(&self) -> MutexGuard<'_, BTreeMap>> { - lock(&self.objects) - } -} - -fn lock(m: &Mutex) -> MutexGuard<'_, T> { - m.lock().unwrap_or_else(PoisonError::into_inner) -} - -/// An upload to a [`MemoryStore`]. -#[derive(Debug)] -pub struct MemoryUpload { - store: MemoryStore, - uri: String, - bytes: Vec, -} - -impl Upload for MemoryUpload { - fn put_part(&mut self, bytes: Vec) -> impl Future> { - self.bytes.extend(bytes); - ready(Ok(())) - } - - fn complete(self) -> impl Future> { - self.store.objects().insert(self.uri, self.bytes); - ready(Ok(())) - } - - fn abort(self) -> impl Future { - ready(()) - } -} - -impl Store for MemoryStore { - type Upload = MemoryUpload; - - fn get(&self, uri: &str) -> impl Future>>> { - ready(Ok(self.objects().get(uri).cloned())) - } - - fn get_range(&self, uri: &str, start: u64, len: u64) -> impl Future>> { - let objects = self.objects(); - ready(objects.get(uri).ok_or_else(|| Error::NotFound(uri.to_string())).map(|bytes| { - let start = usize::try_from(start).unwrap_or(usize::MAX).min(bytes.len()); - let end = start.saturating_add(usize::try_from(len).unwrap_or(usize::MAX)).min(bytes.len()); - bytes[start..end].to_vec() - })) - } - - fn exists(&self, uri: &str) -> impl Future> { - ready(Ok(self.objects().contains_key(uri))) - } - - fn put(&self, uri: &str, bytes: Vec) -> impl Future> { - self.objects().insert(uri.to_string(), bytes); - ready(Ok(())) - } - - fn put_if_absent(&self, uri: &str, bytes: Vec) -> impl Future> { - let mut objects = self.objects(); - let absent = !objects.contains_key(uri); - if absent { - objects.insert(uri.to_string(), bytes); - } - ready(Ok(absent)) - } - - fn upload(&self, uri: &str) -> impl Future> { - ready(Ok(MemoryUpload { store: self.clone(), uri: uri.to_string(), bytes: vec![] })) - } - - fn list(&self, prefix: &str) -> impl Future>> { - let listed = self - .objects() - .range(prefix.to_string()..) - .take_while(|(k, _)| k.starts_with(prefix)) - .map(|(k, v)| Listed { uri: k.clone(), size: v.len() as u64 }) - .collect(); - ready(Ok(listed)) - } - - fn delete(&self, uris: &[String]) -> impl Future> { - let mut objects = self.objects(); - for uri in uris { - objects.remove(uri); - } - ready(Ok(())) - } -} - -impl Clock for MemoryStore { - fn now_ms(&self) -> i64 { - *lock(&self.clock) - } - - fn sleep(&self, ms: u64) -> impl Future { - self.advance(i64::try_from(ms).unwrap_or(i64::MAX)); - ready(()) - } -} - -impl Random for MemoryStore { - fn next_u64(&self) -> u64 { - // A linear congruential generator (Knuth's MMIX constants), with the high bits folded in. - let mut seed = lock(&self.seed); - *seed = seed.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407); - *seed ^ (*seed >> 29) - } -} diff --git a/crates/lakekit-core/src/random.rs b/crates/lakekit-core/src/random.rs deleted file mode 100644 index 5ea2597..0000000 --- a/crates/lakekit-core/src/random.rs +++ /dev/null @@ -1,28 +0,0 @@ -/// Random numbers: snapshot ids, file names, and the jitter between commit attempts. -pub trait Random { - /// A random number. - fn next_u64(&self) -> u64; -} - -/// The operating system's random number generator (feature `getrandom`, native targets only). -#[cfg(all(feature = "getrandom", not(target_arch = "wasm32")))] -#[derive(Clone, Copy, Debug, Default)] -pub struct OsRandom; - -#[cfg(all(feature = "getrandom", not(target_arch = "wasm32")))] -impl Random for OsRandom { - fn next_u64(&self) -> u64 { - // The operating system's generator only fails when it isn't available at all. - getrandom::u64().expect("the operating system's random number generator") - } -} - -#[cfg(all(test, feature = "getrandom", not(target_arch = "wasm32")))] -mod tests { - use super::*; - - #[test] - fn os_random_numbers_differ() { - assert_ne!(OsRandom.next_u64(), OsRandom.next_u64()); - } -} diff --git a/crates/lakekit-core/src/services.rs b/crates/lakekit-core/src/services.rs index 70c8910..86377ac 100644 --- a/crates/lakekit-core/src/services.rs +++ b/crates/lakekit-core/src/services.rs @@ -1,17 +1,21 @@ -use std::future::Future; - -use crate::{Clock, Listed, Random, Result, Store}; +use loko_clock::{Clock, HasClock, TestClock}; +use loko_macros::Services; +use loko_random::{HasRandom, Random, SeededRandom}; +use loko_store::{HasStore, MemoryStore, Store}; /// A store, a clock, and a random source as one value, for APIs that need all three (commits): -/// `Services::new(ObjectStoreAdapter::new(…), TokioClock, OsRandom)`. A type that is all three (a -/// JavaScript host, [`MemoryStore`](crate::MemoryStore)) needs no wrapper. -#[derive(Clone, Debug, Default)] +/// `Services::new(ObjectStoreAdapter::new(…), TokioClock, OsRandom)`. Any type that is all three +/// works as well: a JavaScript host, or an application's own `#[derive(loko::Services)]` struct. +#[derive(Clone, Debug, Default, Services)] pub struct Services { /// The store. + #[service(Store)] pub store: S, /// The clock. + #[service(Clock)] pub clock: C, /// The random source. + #[service(Random)] pub random: R, } @@ -22,54 +26,12 @@ impl Services { } } -impl Store for Services { - type Upload = S::Upload; - - fn get(&self, uri: &str) -> impl Future>>> { - self.store.get(uri) - } - - fn get_range(&self, uri: &str, start: u64, len: u64) -> impl Future>> { - self.store.get_range(uri, start, len) - } - - fn exists(&self, uri: &str) -> impl Future> { - self.store.exists(uri) - } - - fn put(&self, uri: &str, bytes: Vec) -> impl Future> { - self.store.put(uri, bytes) - } - - fn put_if_absent(&self, uri: &str, bytes: Vec) -> impl Future> { - self.store.put_if_absent(uri, bytes) - } - - fn upload(&self, uri: &str) -> impl Future> { - self.store.upload(uri) - } - - fn list(&self, prefix: &str) -> impl Future>> { - self.store.list(prefix) - } - - fn delete(&self, uris: &[String]) -> impl Future> { - self.store.delete(uris) - } -} - -impl Clock for Services { - fn now_ms(&self) -> i64 { - self.clock.now_ms() - } - - fn sleep(&self, ms: u64) -> impl Future { - self.clock.sleep(ms) - } -} +/// Everything in memory, for tests: a [`MemoryStore`], a [`TestClock`], and a [`SeededRandom`]. +pub type MemoryServices = Services; -impl Random for Services { - fn next_u64(&self) -> u64 { - self.random.next_u64() +impl MemoryServices { + /// An empty store, the clock at `now_ms` milliseconds after the Unix epoch, and seed 0. + pub fn at_ms(now_ms: i64) -> Self { + Self::new(MemoryStore::new(), TestClock::at_ms(now_ms), SeededRandom::new(0)) } } diff --git a/crates/lakekit-core/src/store.rs b/crates/lakekit-core/src/store.rs deleted file mode 100644 index 8b4f2a8..0000000 --- a/crates/lakekit-core/src/store.rs +++ /dev/null @@ -1,49 +0,0 @@ -use std::future::Future; - -use crate::Result; - -/// An object found by [`Store::list`]. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Listed { - /// Absolute URI. - pub uri: String, - /// Size in bytes. - pub size: u64, -} - -/// Where a table's objects live: an object store, a directory, memory, or a JavaScript binding. -/// Paths are absolute URIs, as table metadata records them. -/// -/// The futures aren't required to be `Send`, so a store can wrap JavaScript promises. -pub trait Store { - /// A multipart upload in progress. - type Upload: Upload; - - /// The object at `uri`, or `None` if there is none. - fn get(&self, uri: &str) -> impl Future>>>; - /// `len` bytes of the object at `uri` from `start`. - fn get_range(&self, uri: &str, start: u64, len: u64) -> impl Future>>; - /// Whether an object exists at `uri`. - fn exists(&self, uri: &str) -> impl Future>; - /// Writes (or replaces) the object at `uri`. - fn put(&self, uri: &str, bytes: Vec) -> impl Future>; - /// Writes the object at `uri` unless one exists; `false` if one did. The commit primitive. - fn put_if_absent(&self, uri: &str, bytes: Vec) -> impl Future>; - /// Starts a multipart upload to `uri`. - fn upload(&self, uri: &str) -> impl Future>; - /// Objects whose URI starts with `prefix` (a string prefix, not only a directory). - fn list(&self, prefix: &str) -> impl Future>>; - /// Deletes objects; missing ones are not an error. - fn delete(&self, uris: &[String]) -> impl Future>; -} - -/// A multipart upload: parts in order, then [`Upload::complete`]. -pub trait Upload { - /// Sends the next part. Every part but the last must be at least the store's minimum (5 MiB on - /// S3 and R2). - fn put_part(&mut self, bytes: Vec) -> impl Future>; - /// Completes the upload: the object appears. - fn complete(self) -> impl Future>; - /// Drops the parts sent. - fn abort(self) -> impl Future; -} diff --git a/crates/lakekit-iceberg/Cargo.toml b/crates/lakekit-iceberg/Cargo.toml index b6f0d96..1f75ec8 100644 --- a/crates/lakekit-iceberg/Cargo.toml +++ b/crates/lakekit-iceberg/Cargo.toml @@ -36,6 +36,9 @@ arrow-array = "58" arrow-schema.workspace = true bytes = "1" futures = "0.3" +lakekit-core = { workspace = true, features = ["tokio", "getrandom"] } +loko-object-store.workspace = true +object_store.workspace = true [package.metadata.docs.rs] all-features = true diff --git a/crates/lakekit-iceberg/src/commit.rs b/crates/lakekit-iceberg/src/commit.rs index c3b455f..0f57fc5 100644 --- a/crates/lakekit-iceberg/src/commit.rs +++ b/crates/lakekit-iceberg/src/commit.rs @@ -11,6 +11,7 @@ use std::fmt; use std::future::Future; use std::hash::BuildHasher; use std::sync::Arc; +use std::time::Duration; use lakekit_core::{Clock, Random, Store, uuid_v4}; @@ -181,7 +182,8 @@ pub async fn commit, S: BuildHasher for attempt in 1..=options.max_attempts { if attempt > 1 { let (lo, hi) = options.backoff_ms; - store.sleep((lo + store.next_u64() % hi.saturating_sub(lo).max(1)) * attempt).await; + let ms = (lo + store.next_u64() % hi.saturating_sub(lo).max(1)) * attempt; + store.sleep(Duration::from_millis(ms)).await; base = hooks.reload(store, &base.location).await?; } if hooks.skip(&base) { @@ -260,7 +262,7 @@ async fn attempt_commit, S: BuildHa let list = write_manifest_list(&manifests, snapshot_id, parent, sequence_number)?; uploads.push((list_uri.clone(), list)); let puts = uploads.iter().map(|(uri, bytes)| store.put(uri, bytes.clone())); - futures_util::future::try_join_all(puts).await?; + futures_util::future::try_join_all(puts).await.map_err(Error::from)?; let mut summary: HashMap = props.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); summary.extend(extra.summary); @@ -469,7 +471,7 @@ pub fn counters(change: &Change, base: &HashMap Table { + fn create(store: &MemoryServices, format_version: u8) -> Table { let metadata = json!({ "format-version": format_version, "table-uuid": "u", "location": LOCATION, "last-sequence-number": 0, "last-updated-ms": 0, "last-column-id": 2, "current-schema-id": 0, @@ -509,8 +511,8 @@ mod tests { Change::Append { files } } - fn try_run>( - store: &MemoryStore, + fn try_run>( + store: &MemoryServices, base: Table, change: &Change, hooks: &mut K, @@ -518,8 +520,8 @@ mod tests { block_on(commit(store, base, change, &HashMap::new(), &CommitOptions::default(), hooks)) } - fn run>( - store: &MemoryStore, + fn run>( + store: &MemoryServices, base: Table, change: &Change, hooks: &mut K, @@ -533,7 +535,7 @@ mod tests { /// The live files of the newest version, read back from the store: (name, data sequence /// number, file sequence number). - fn live(store: &MemoryStore) -> Vec<(String, i64, i64)> { + fn live(store: &MemoryServices) -> Vec<(String, i64, i64)> { let mut t = block_on(Table::load(store, LOCATION)).unwrap(); let mut files: Vec<_> = block_on(t.live_files(store)) .unwrap() @@ -552,7 +554,7 @@ mod tests { #[test] fn append_replace_and_delete() { - let store = MemoryStore::new(1_000); + let store = MemoryServices::at_ms(1_000); let t = create(&store, 2); let (t, info) = run(&store, t, &append(vec![file("a", 1, 10), file("b", 2, 20)]), &mut NoHooks); assert_eq!((info.version, info.attempts, info.skipped), (2, 1, false)); @@ -579,20 +581,20 @@ mod tests { #[test] fn a_lost_race_retries_on_the_newer_version() { - let store = MemoryStore::new(1_000); + let store = MemoryServices::at_ms(1_000); let stale = create(&store, 2); let (_, first) = run(&store, stale.clone(), &append(vec![file("a", 1, 10)]), &mut NoHooks); let (_, second) = run(&store, stale, &append(vec![file("b", 1, 10)]), &mut NoHooks); assert_eq!((first.version, second.version, second.attempts), (2, 3, 2)); assert_eq!(live(&store).len(), 2); // The lost attempt's manifests and list were deleted: every list is a snapshot's. - let lists = store.uris().iter().filter(|u| u.contains("/snap-")).count(); + let lists = store.store.uris().iter().filter(|u| u.contains("/snap-")).count(); assert_eq!(lists, 2); } #[test] fn removing_files_that_are_gone_is_a_conflict() { - let store = MemoryStore::new(1_000); + let store = MemoryServices::at_ms(1_000); let (t, _) = run(&store, create(&store, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); let removed = t.files.clone().unwrap_or_default(); let (t, _) = run(&store, t, &Change::Delete { removed: removed.clone() }, &mut NoHooks); @@ -603,14 +605,14 @@ mod tests { /// Hooks that skip a batch already committed, and add a file and summary properties. struct Batch(&'static str); - impl CommitHooks for Batch { + impl CommitHooks for Batch { type Error = Error; fn skip(&mut self, base: &Table) -> bool { base.metadata.ancestors().iter().any(|s| s.summary.get("batch").map(String::as_str) == Some(self.0)) } - fn extra(&mut self, _: &MemoryStore, base: &Table, snapshot_id: i64) -> impl Future> { + fn extra(&mut self, _: &MemoryServices, base: &Table, snapshot_id: i64) -> impl Future> { let uri = format!("{}/metadata/extra-{snapshot_id}", base.location); let summary = HashMap::from([("batch".to_string(), self.0.to_string()), ("extra".to_string(), uri.clone())]); @@ -620,7 +622,7 @@ mod tests { #[test] fn hooks_skip_and_extend_commits() { - let store = MemoryStore::new(1_000); + let store = MemoryServices::at_ms(1_000); let (t, info) = run(&store, create(&store, 2), &append(vec![file("a", 1, 10)]), &mut Batch("b1")); let extra = t.metadata.summary()["extra"].clone(); assert!(block_on(store.exists(&extra)).unwrap()); @@ -632,7 +634,7 @@ mod tests { #[test] fn many_manifests_merge() { - let store = MemoryStore::new(1_000); + let store = MemoryServices::at_ms(1_000); let mut t = create(&store, 2); for i in 0..9 { t = run(&store, t, &append(vec![file(&format!("f{i}"), i, 1)]), &mut NoHooks).0; @@ -643,10 +645,10 @@ mod tests { #[test] fn a_clock_behind_the_table_keeps_the_logs_in_order() { - let store = MemoryStore::new(5_000); + let store = MemoryServices::at_ms(5_000); let (t, _) = run(&store, create(&store, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); - let late = MemoryStore::new(1_000); - for uri in store.uris() { + let late = MemoryServices::at_ms(1_000); + for uri in store.store.uris() { block_on(late.put(&uri, block_on(store.get(&uri)).unwrap().unwrap())).unwrap(); } let (t, _) = run(&late, t, &append(vec![file("b", 1, 10)]), &mut NoHooks); @@ -655,7 +657,7 @@ mod tests { #[test] fn a_replace_needs_a_sequence_number_per_added_file() { - let store = MemoryStore::new(1_000); + let store = MemoryServices::at_ms(1_000); let (t, _) = run(&store, create(&store, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); let removed = files_named(&t, "a.parquet"); let change = @@ -666,7 +668,7 @@ mod tests { #[test] fn only_v2_tables_are_written() { - let store = MemoryStore::new(1_000); + let store = MemoryServices::at_ms(1_000); let err = try_run(&store, create(&store, 1), &append(vec![]), &mut NoHooks); assert!(matches!(err, Err(Error::Unsupported(_)))); } diff --git a/crates/lakekit-iceberg/src/table.rs b/crates/lakekit-iceberg/src/table.rs index 6f5c49b..2602338 100644 --- a/crates/lakekit-iceberg/src/table.rs +++ b/crates/lakekit-iceberg/src/table.rs @@ -138,7 +138,7 @@ mod tests { #[test] fn loads_the_newest_version() { - let store = MemoryStore::new(0); + let store = MemoryStore::new(); let meta = |n: u8| format!(r#"{{"format-version": 2, "location": "memory://t", "last-sequence-number": {n}}}"#); block_on(async { assert!(matches!(Table::load(&store, "memory://t").await, Err(Error::NotFound(_)))); diff --git a/crates/lakekit-iceberg/tests/iceberg_rust.rs b/crates/lakekit-iceberg/tests/iceberg_rust.rs index 370945f..ad54de1 100644 --- a/crates/lakekit-iceberg/tests/iceberg_rust.rs +++ b/crates/lakekit-iceberg/tests/iceberg_rust.rs @@ -21,7 +21,7 @@ use iceberg::spec::{ use iceberg::transaction::{ApplyTransactionAction, Transaction}; use iceberg::writer::file_writer::{FileWriter as _, FileWriterBuilder, ParquetWriterBuilder}; use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, NamespaceIdent, TableCreation, TableIdent}; -use lakekit_core::{Error, Listed, MemoryStore, MemoryUpload, Services, Store}; +use lakekit_core::{Listed, MemoryUpload, SeededRandom, Services, Store, StoreError as Error, TestClock}; use lakekit_iceberg::stats::ParquetStats; use lakekit_iceberg::{Change, CommitOptions, DataFile, Datum, Entry, NoHooks, Status, Table, commit}; use parquet::arrow::ArrowWriter; @@ -38,8 +38,11 @@ struct IoStore { io: FileIO, } -/// The store with a [`MemoryStore`]'s clock and random numbers. -type TestStore = Services; +/// The store with a test clock and seeded random numbers. +type TestStore = Services; + +/// The store's results. +type Result = std::result::Result; fn store(e: impl std::fmt::Display) -> Error { Error::Store(e.to_string()) @@ -48,45 +51,45 @@ fn store(e: impl std::fmt::Display) -> Error { impl Store for IoStore { type Upload = MemoryUpload; - async fn get(&self, uri: &str) -> lakekit_core::Result>> { + async fn get(&self, uri: &str) -> Result>> { if !self.io.exists(uri).await.map_err(store)? { return Ok(None); } Ok(Some(self.io.new_input(uri).map_err(store)?.read().await.map_err(store)?.to_vec())) } - async fn get_range(&self, uri: &str, start: u64, len: u64) -> lakekit_core::Result> { + async fn get_range(&self, uri: &str, start: u64, len: u64) -> Result> { let bytes = self.get(uri).await?.ok_or_else(|| Error::NotFound(uri.to_string()))?; let start = usize::try_from(start).map_err(store)?.min(bytes.len()); let end = start.saturating_add(usize::try_from(len).map_err(store)?).min(bytes.len()); Ok(bytes[start..end].to_vec()) } - async fn exists(&self, uri: &str) -> lakekit_core::Result { + async fn exists(&self, uri: &str) -> Result { self.io.exists(uri).await.map_err(store) } - async fn put(&self, uri: &str, bytes: Vec) -> lakekit_core::Result<()> { + async fn put(&self, uri: &str, bytes: Vec) -> Result<()> { self.io.new_output(uri).map_err(store)?.write(Bytes::from(bytes)).await.map_err(store) } /// The tests run one writer at a time, so checking first is enough. - async fn put_if_absent(&self, uri: &str, bytes: Vec) -> lakekit_core::Result { + async fn put_if_absent(&self, uri: &str, bytes: Vec) -> Result { if self.exists(uri).await? { return Ok(false); } self.put(uri, bytes).await.map(|()| true) } - fn upload(&self, _: &str) -> impl Future> { + fn upload(&self, _: &str) -> impl Future> { ready(Err(Error::Unsupported("uploads".into()))) } - fn list(&self, _: &str) -> impl Future>> { + fn list(&self, _: &str) -> impl Future>> { ready(Err(Error::Unsupported("listing".into()))) } - async fn delete(&self, uris: &[String]) -> lakekit_core::Result<()> { + async fn delete(&self, uris: &[String]) -> Result<()> { for uri in uris { self.io.delete(uri).await.map_err(store)?; } @@ -228,8 +231,7 @@ async fn tables_round_trip_between_lakekit_and_iceberg_rust() { Some(UnboundPartitionSpec::builder().add_partition_field(1, "time_day", IceTransform::Day).unwrap().build()); let table = catalog.create_table(&ns, creation).await.unwrap(); let io = table.file_io().clone(); - let memory = MemoryStore::new(T0 / 1000); - let store = Services::new(IoStore { io: io.clone() }, memory.clone(), memory); + let store = Services::new(IoStore { io: io.clone() }, TestClock::at_ms(T0 / 1000), SeededRandom::new(0)); let arrow = Arc::new(iceberg::arrow::schema_to_arrow_schema(table.metadata().current_schema()).unwrap()); // iceberg-rust appends; lakekit reads the file as iceberg-rust records it. diff --git a/crates/lakekit-iceberg/tests/object_store.rs b/crates/lakekit-iceberg/tests/object_store.rs new file mode 100644 index 0000000..0bb4efb --- /dev/null +++ b/crates/lakekit-iceberg/tests/object_store.rs @@ -0,0 +1,43 @@ +//! An Iceberg table committed and read back through `object_store` (`loko-object-store`), with the +//! system clock and random numbers. +#![cfg(feature = "write")] + +use std::collections::HashMap; +use std::sync::Arc; + +use lakekit_core::{OsRandom, Services, Store, TokioClock}; +use lakekit_iceberg::{Change, CommitOptions, DataFile, Datum, Metadata, NoHooks, Table, commit}; +use loko_object_store::ObjectStoreAdapter; +use object_store::memory::InMemory; + +#[tokio::test] +async fn iceberg_commits_through_object_store() { + let store = Services::new(ObjectStoreAdapter::new(Arc::new(InMemory::new()), "memory://"), TokioClock, OsRandom); + let location = "memory://warehouse/t"; + let metadata = Metadata::parse( + br#"{"format-version": 2, "table-uuid": "u", "location": "memory://warehouse/t", "last-sequence-number": 0, + "last-updated-ms": 0, "last-column-id": 1, "current-schema-id": 0, + "schemas": [{"schema-id": 0, "type": "struct", "fields": [{"id": 1, "name": "x", "required": true, "type": "long"}]}], + "default-spec-id": 0, "partition-specs": [{"spec-id": 0, "fields": []}], "last-partition-id": 999, + "default-sort-order-id": 0, "sort-orders": [{"order-id": 0, "fields": []}], "properties": {}, + "current-snapshot-id": -1, "snapshots": [], "snapshot-log": [], "metadata-log": []}"#, + ) + .unwrap(); + let table = Table::new(location, 1, metadata); + store.put(&table.metadata_file, table.metadata.to_json().into_bytes()).await.unwrap(); + let file = DataFile { + path: format!("{location}/data/a.parquet"), + partition: Vec::>::new(), + record_count: 3, + file_size: 30, + ..DataFile::default() + }; + let change = Change::Append { files: vec![file] }; + let (_, info) = + commit(&store, table, &change, &HashMap::new(), &CommitOptions::default(), &mut NoHooks).await.unwrap(); + assert_eq!(info.version, 2); + + let mut loaded = Table::load(&store, location).await.unwrap(); + assert_eq!(loaded.version, 2); + assert_eq!(loaded.live_files(&store).await.unwrap()[0].file.record_count, 3); +} diff --git a/crates/lakekit-object-store/Cargo.toml b/crates/lakekit-object-store/Cargo.toml deleted file mode 100644 index 67a429b..0000000 --- a/crates/lakekit-object-store/Cargo.toml +++ /dev/null @@ -1,32 +0,0 @@ -[package] -name = "lakekit-object-store" -description = "A lakekit store over an object_store client: S3, GCS, Azure, local files, memory, and more" -readme = "README.md" -version.workspace = true -edition.workspace = true -rust-version.workspace = true -license.workspace = true -repository.workspace = true -keywords.workspace = true -categories.workspace = true - -[dependencies] -lakekit-core.workspace = true -# Without features: applications enable the backends they need (`aws`, `gcp`, `azure`, `http`, `fs`, or the `*-base` -# variants for WebAssembly) on their own `object_store` dependency. -object_store.workspace = true -futures-util.workspace = true -url.workspace = true - -[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] -lakekit-core = { workspace = true, features = ["tokio", "getrandom"] } -lakekit-iceberg = { workspace = true, features = ["write"] } -object_store = { workspace = true, features = ["fs"] } -tempfile = "3" -tokio.workspace = true - -[package.metadata.docs.rs] -all-features = true - -[lints] -workspace = true diff --git a/crates/lakekit-object-store/README.md b/crates/lakekit-object-store/README.md deleted file mode 100644 index 1e5caeb..0000000 --- a/crates/lakekit-object-store/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# lakekit-object-store - -A [lakekit](https://github.com/polythings/lakekit) `Store` over an [`object_store`](https://crates.io/crates/object_store) -client: S3, GCS, Azure, local files, memory, and anything else that implements `ObjectStore`. - -```rust -use std::sync::Arc; - -use lakekit_object_store::ObjectStoreAdapter; -use object_store::memory::InMemory; - -let store = ObjectStoreAdapter::new(Arc::new(InMemory::new()), "memory://"); -// Or from a URL, with the backend enabled on `object_store` (here `aws`): -// let store = ObjectStoreAdapter::from_url("s3://bucket/warehouse", [("aws_region", "us-east-2")])?; -``` - -- lakekit names objects by absolute URI; the adapter maps them to paths under its root (`s3://bucket/`, or - `file:///` for local files). -- Put-if-absent, the commit primitive, is `PutMode::Create`; another writer's object reads as `false`. -- Listing is by string prefix (lakekit's `…/metadata/v`), done by listing the prefix's directory. -- The crate enables no `object_store` backends: turn on the ones you need (`aws`, `gcp`, `azure`, `http`, `fs`, or the - `*-base` variants for WebAssembly) on your own `object_store` dependency. - -Apache License 2.0. diff --git a/crates/lakekit-object-store/src/lib.rs b/crates/lakekit-object-store/src/lib.rs deleted file mode 100644 index 5d8b992..0000000 --- a/crates/lakekit-object-store/src/lib.rs +++ /dev/null @@ -1,280 +0,0 @@ -//! A lakekit [`Store`] over an [`object_store`] client: S3, GCS, Azure, local files, memory, and -//! anything else that implements `ObjectStore`. Applications enable the backends they need on their -//! own `object_store` dependency and hand the client to [`ObjectStoreAdapter::new`], or build one -//! from a URL with [`ObjectStoreAdapter::from_url`]. -//! -//! lakekit names objects by absolute URI (`s3://bucket/warehouse/t/metadata/v1.metadata.json`); -//! the adapter maps them to paths under its root (`s3://bucket`). -//! -//! ``` -//! use std::sync::Arc; -//! -//! use lakekit_object_store::ObjectStoreAdapter; -//! use object_store::memory::InMemory; -//! -//! let store = ObjectStoreAdapter::new(Arc::new(InMemory::new()), "memory://"); -//! ``` - -use std::future::Future; -use std::sync::Arc; - -use futures_util::TryStreamExt; -use lakekit_core::{Error, Listed, Result, Store, Upload}; -use object_store::path::Path; -use object_store::{MultipartUpload, ObjectStore, ObjectStoreExt, PutMode, PutOptions, PutPayload}; -use url::Url; - -/// A [`Store`] over an `object_store` client whose objects have URIs under `root`. -#[derive(Clone, Debug)] -pub struct ObjectStoreAdapter { - store: Arc, - root: String, -} - -fn store_error(e: object_store::Error) -> Error { - match e { - object_store::Error::NotFound { path, .. } => Error::NotFound(path), - object_store::Error::NotImplemented { operation, implementer } => { - Error::Unsupported(format!("{implementer} has no {operation}")) - } - other => Error::Store(other.to_string()), - } -} - -impl ObjectStoreAdapter { - /// `store`, whose objects have URIs under `root`: the object at path `a/b` is `/a/b`. The - /// root is the store's scheme and bucket (`s3://bucket`, `gs://bucket`, `memory://`), or - /// `file:///` for a local file system rooted at `/`. - pub fn new(store: Arc, root: impl Into) -> Self { - let mut root = root.into(); - if !root.ends_with('/') { - root.push('/'); - } - Self { store, root } - } - - /// The store for `url`'s scheme and bucket (`s3://bucket/…`), configured with `options` - /// (`object_store`'s keys, such as `aws_region`). Its backend must be enabled on - /// `object_store`. - pub fn from_url(url: &str, options: I) -> Result - where - I: IntoIterator, - K: AsRef, - V: Into, - { - let parsed = Url::parse(url).map_err(|e| Error::Invalid(format!("{url}: {e}")))?; - let root = match parsed.host_str().filter(|h| !h.is_empty()) { - Some(host) => format!("{}://{host}/", parsed.scheme()), - // `file:///data/t`: a file system rooted at `/`. - None => format!("{}:///", parsed.scheme()), - }; - let root_url = Url::parse(&root).map_err(|e| Error::Invalid(format!("{root}: {e}")))?; - let (store, _) = object_store::parse_url_opts(&root_url, options).map_err(store_error)?; - Ok(Self::new(Arc::from(store), root)) - } - - /// The path of the object at `uri`. - fn path(&self, uri: &str) -> Result { - let rest = uri - .strip_prefix(&self.root) - .ok_or_else(|| Error::Invalid(format!("{uri} is outside this store ({})", self.root)))?; - Path::parse(rest).map_err(|e| Error::Invalid(format!("{uri}: {e}"))) - } - - /// The URI of the object at `path`. - fn uri(&self, path: &Path) -> String { - format!("{}{path}", self.root) - } -} - -/// A multipart upload through `object_store`. -#[derive(Debug)] -pub struct ObjectStoreUpload(Box); - -impl Upload for ObjectStoreUpload { - async fn put_part(&mut self, bytes: Vec) -> Result<()> { - self.0.put_part(PutPayload::from(bytes)).await.map_err(store_error) - } - - async fn complete(mut self) -> Result<()> { - self.0.complete().await.map(drop).map_err(store_error) - } - - async fn abort(mut self) { - let _ = self.0.abort().await; - } -} - -impl Store for ObjectStoreAdapter { - type Upload = ObjectStoreUpload; - - async fn get(&self, uri: &str) -> Result>> { - match self.store.get(&self.path(uri)?).await { - Ok(found) => Ok(Some(found.bytes().await.map_err(store_error)?.to_vec())), - Err(object_store::Error::NotFound { .. }) => Ok(None), - Err(e) => Err(store_error(e)), - } - } - - async fn get_range(&self, uri: &str, start: u64, len: u64) -> Result> { - let range = start..start.saturating_add(len); - Ok(self.store.get_range(&self.path(uri)?, range).await.map_err(store_error)?.to_vec()) - } - - async fn exists(&self, uri: &str) -> Result { - match self.store.head(&self.path(uri)?).await { - Ok(_) => Ok(true), - Err(object_store::Error::NotFound { .. }) => Ok(false), - Err(e) => Err(store_error(e)), - } - } - - async fn put(&self, uri: &str, bytes: Vec) -> Result<()> { - self.store.put(&self.path(uri)?, PutPayload::from(bytes)).await.map(drop).map_err(store_error) - } - - async fn put_if_absent(&self, uri: &str, bytes: Vec) -> Result { - let options = PutOptions { mode: PutMode::Create, ..PutOptions::default() }; - match self.store.put_opts(&self.path(uri)?, PutPayload::from(bytes), options).await { - Ok(_) => Ok(true), - Err(object_store::Error::AlreadyExists { .. } | object_store::Error::Precondition { .. }) => Ok(false), - Err(e) => Err(store_error(e)), - } - } - - async fn upload(&self, uri: &str) -> Result { - Ok(ObjectStoreUpload(self.store.put_multipart(&self.path(uri)?).await.map_err(store_error)?)) - } - - fn list(&self, prefix: &str) -> impl Future>> { - // `object_store` lists by directory; lakekit by string prefix (`…/metadata/v`). List the - // directory the prefix is in and keep what starts with it. - let relative = prefix.strip_prefix(&self.root).map(|rest| rest.rsplit_once('/').map_or("", |(dir, _)| dir)); - async move { - let relative = - relative.ok_or_else(|| Error::Invalid(format!("{prefix} is outside this store ({})", self.root)))?; - let dir = Path::parse(relative).map_err(|e| Error::Invalid(format!("{prefix}: {e}")))?; - let dir = (!relative.is_empty()).then_some(&dir); - let objects: Vec<_> = self.store.list(dir).try_collect().await.map_err(store_error)?; - Ok(objects - .into_iter() - .map(|o| Listed { uri: self.uri(&o.location), size: o.size }) - .filter(|o| o.uri.starts_with(prefix)) - .collect()) - } - } - - async fn delete(&self, uris: &[String]) -> Result<()> { - let deletes = uris.iter().map(|uri| async move { - match self.store.delete(&self.path(uri)?).await { - Ok(()) | Err(object_store::Error::NotFound { .. }) => Ok(()), - Err(e) => Err(store_error(e)), - } - }); - futures_util::future::try_join_all(deletes).await.map(drop) - } -} - -#[cfg(all(test, not(target_arch = "wasm32")))] -mod tests { - use std::collections::HashMap; - - use lakekit_core::{OsRandom, Services, TokioClock}; - use lakekit_iceberg::{Change, CommitOptions, DataFile, Datum, Metadata, NoHooks, Table, commit}; - use object_store::memory::InMemory; - - use super::*; - - fn adapter() -> ObjectStoreAdapter { - ObjectStoreAdapter::new(Arc::new(InMemory::new()), "memory://") - } - - #[tokio::test] - async fn objects_round_trip() { - let store = adapter(); - assert_eq!(store.get("memory://t/a").await.unwrap(), None); - assert!(!store.exists("memory://t/a").await.unwrap()); - store.put("memory://t/a", b"hello".to_vec()).await.unwrap(); - assert_eq!(store.get("memory://t/a").await.unwrap().as_deref(), Some(&b"hello"[..])); - assert_eq!(store.get_range("memory://t/a", 1, 3).await.unwrap(), b"ell"); - assert!(store.put_if_absent("memory://t/b", b"1".to_vec()).await.unwrap()); - assert!(!store.put_if_absent("memory://t/b", b"2".to_vec()).await.unwrap(), "the first write wins"); - assert_eq!(store.get("memory://t/b").await.unwrap().as_deref(), Some(&b"1"[..])); - - let mut upload = store.upload("memory://t/c").await.unwrap(); - upload.put_part(b"ab".to_vec()).await.unwrap(); - upload.put_part(b"cd".to_vec()).await.unwrap(); - upload.complete().await.unwrap(); - assert_eq!(store.get("memory://t/c").await.unwrap().as_deref(), Some(&b"abcd"[..])); - - store.delete(&["memory://t/a".into(), "memory://t/missing".into()]).await.unwrap(); - assert!(!store.exists("memory://t/a").await.unwrap()); - assert!(matches!(store.get("s3://elsewhere/x").await, Err(Error::Invalid(_)))); - } - - #[tokio::test] - async fn lists_by_string_prefix() { - let store = adapter(); - for name in ["v1.metadata.json", "v2.metadata.json", "version-hint.text", "snap-1.avro"] { - store.put(&format!("memory://t/metadata/{name}"), vec![0; 3]).await.unwrap(); - } - store.put("memory://t/data/v9.parquet", vec![]).await.unwrap(); - let mut listed: Vec = - store.list("memory://t/metadata/v").await.unwrap().into_iter().map(|o| o.uri).collect(); - listed.sort(); - assert_eq!( - listed, - [ - "memory://t/metadata/v1.metadata.json", - "memory://t/metadata/v2.metadata.json", - "memory://t/metadata/version-hint.text" - ] - ); - assert_eq!(store.list("memory://t/metadata/s").await.unwrap()[0].size, 3); - assert_eq!(store.list("memory://t").await.unwrap().len(), 5, "a prefix right under the root"); - } - - #[tokio::test] - async fn local_files_have_file_uris() { - let dir = tempfile::tempdir().unwrap(); - let store = ObjectStoreAdapter::from_url("file:///", std::iter::empty::<(&str, String)>()).unwrap(); - let uri = format!("file://{}/t/a.json", dir.path().display()); - store.put(&uri, b"{}".to_vec()).await.unwrap(); - assert!(dir.path().join("t/a.json").exists()); - let listed = store.list(&format!("file://{}/t/", dir.path().display())).await.unwrap(); - assert_eq!(listed[0].uri, uri); - } - - /// An Iceberg table committed and read back through `object_store`. - #[tokio::test] - async fn iceberg_commits_through_object_store() { - let store = Services::new(adapter(), TokioClock, OsRandom); - let location = "memory://warehouse/t"; - let metadata = Metadata::parse( - br#"{"format-version": 2, "table-uuid": "u", "location": "memory://warehouse/t", "last-sequence-number": 0, - "last-updated-ms": 0, "last-column-id": 1, "current-schema-id": 0, - "schemas": [{"schema-id": 0, "type": "struct", "fields": [{"id": 1, "name": "x", "required": true, "type": "long"}]}], - "default-spec-id": 0, "partition-specs": [{"spec-id": 0, "fields": []}], "last-partition-id": 999, - "default-sort-order-id": 0, "sort-orders": [{"order-id": 0, "fields": []}], "properties": {}, - "current-snapshot-id": -1, "snapshots": [], "snapshot-log": [], "metadata-log": []}"#, - ) - .unwrap(); - let table = Table::new(location, 1, metadata); - store.put(&table.metadata_file, table.metadata.to_json().into_bytes()).await.unwrap(); - let file = DataFile { - path: format!("{location}/data/a.parquet"), - partition: Vec::>::new(), - record_count: 3, - file_size: 30, - ..DataFile::default() - }; - let change = Change::Append { files: vec![file] }; - let (_, info) = - commit(&store, table, &change, &HashMap::new(), &CommitOptions::default(), &mut NoHooks).await.unwrap(); - assert_eq!(info.version, 2); - - let mut loaded = Table::load(&store, location).await.unwrap(); - assert_eq!(loaded.version, 2); - assert_eq!(loaded.live_files(&store).await.unwrap()[0].file.record_count, 3); - } -} diff --git a/crates/lakekit/Cargo.toml b/crates/lakekit/Cargo.toml index 2466e36..686d2fd 100644 --- a/crates/lakekit/Cargo.toml +++ b/crates/lakekit/Cargo.toml @@ -20,8 +20,8 @@ iceberg-write = ["iceberg", "lakekit-iceberg/write"] iceberg-arrow = ["iceberg", "lakekit-iceberg/arrow"] # Iceberg statistics from Parquet footers. iceberg-parquet-stats = ["iceberg", "lakekit-iceberg/parquet-stats"] -# A store over an object_store client (lakekit-object-store). -object-store = ["dep:lakekit-object-store"] +# A store over an object_store client (loko-object-store). +object-store = ["dep:loko-object-store"] # TokioClock, a native clock (lakekit-core/tokio). tokio = ["lakekit-core/tokio"] # OsRandom, the operating system's random numbers (lakekit-core/getrandom). @@ -30,7 +30,7 @@ getrandom = ["lakekit-core/getrandom"] [dependencies] lakekit-core.workspace = true lakekit-iceberg = { workspace = true, optional = true } -lakekit-object-store = { workspace = true, optional = true } +loko-object-store = { workspace = true, optional = true } [package.metadata.docs.rs] all-features = true diff --git a/crates/lakekit/src/lib.rs b/crates/lakekit/src/lib.rs index 09b36b5..cbc7e9c 100644 --- a/crates/lakekit/src/lib.rs +++ b/crates/lakekit/src/lib.rs @@ -1,16 +1,16 @@ //! Table formats read and written without I/O of their own, for WebAssembly and native: one crate -//! per format and per store adapter, re-exported here behind a feature each. The I/O boundary is +//! per format, re-exported here behind a feature each. The I/O boundary is //! [`core::Store`], [`core::Clock`], and [`core::Random`]. //! //! | Feature | Crate | //! |---|---| //! | `iceberg` (default) | [`iceberg`]: Apache Iceberg (`lakekit-iceberg`) | //! | `iceberg-write`, `iceberg-arrow`, `iceberg-parquet-stats` | its optional parts | -//! | `object-store` | `object_store`: a store over an `object_store` client (`lakekit-object-store`) | +//! | `object-store` | `object_store`: a store over an `object_store` client (`loko-object-store`) | //! | `tokio`, `getrandom` | `core::TokioClock` and `core::OsRandom`, native ones | pub use lakekit_core as core; #[cfg(feature = "iceberg")] pub use lakekit_iceberg as iceberg; #[cfg(feature = "object-store")] -pub use lakekit_object_store as object_store; +pub use loko_object_store as object_store; diff --git a/spec/0001-store-clock-random.md b/spec/0001-store-clock-random.md index ec21d29..0565ee4 100644 --- a/spec/0001-store-clock-random.md +++ b/spec/0001-store-clock-random.md @@ -15,7 +15,7 @@ issues: [] ## Summary Format crates do no I/O of their own. They read and write objects through a `Store`, take the time from a `Clock`, and -take random numbers from a `Random` (`lakekit-core`), all supplied by the application, so the same code runs in a +take random numbers from a `Random`, all supplied by the application, so the same code runs in a browser, a Cloudflare Worker, a CLI, or a server, and tests run against memory. ## Design @@ -24,20 +24,26 @@ browser, a Cloudflare Worker, a CLI, or a server, and tests run against memory. `false` when an object is there), `upload` (a multipart upload: parts in order, then complete or abort), `list` (by string prefix, not only by directory), and `delete` (missing objects are not an error). Paths are absolute URIs, as table metadata records them. Adapters implement it over store libraries ([spec 4](0004-store-adapters.md)). -- **`Clock`:** `now_ms` and `sleep`: commit timestamps, time-ordered file names, and the wait between commit attempts. +- **Where they come from:** the three are [loko](https://github.com/polythings/loko)'s services (`loko-store`, + `loko-clock`, `loko-random`), shared by the polythings projects; `lakekit-core` re-exports them with their test and + native implementations, and adds what is lakekit's: the error type, the identifiers, and `Services`. +- **`Clock`:** `now` and `now_ms`, and `sleep(Duration)`: commit timestamps, time-ordered file names, and the wait + between commit attempts. - **`Random`:** `next_u64`: snapshot ids, file names, and the jitter between commit attempts. - **Who needs what:** readers need only a `Store`; commits need `Store + Clock + Random`. - **`Services`:** a store, a clock, and a random source as one value (`Services::new(store, clock, random)`), for APIs - that need all three. A type that is all three (a JavaScript host, `MemoryStore`) needs no wrapper. -- **Native implementations:** `TokioClock` (feature `tokio`: the system clock and tokio's timer) and `OsRandom` - (feature `getrandom`: the operating system's generator), on native targets only. + that need all three, built with loko's `#[derive(Services)]`. Any type that is all three (a JavaScript host, an + application's own bundle) needs no wrapper. +- **Native implementations:** `TokioClock` (feature `tokio`: the system clock and tokio's timer, native targets) and + `OsRandom` (feature `getrandom`: the operating system's generator, native targets and WASI). - **Futures aren't `Send`.** Methods return `impl Future`, without a `Send` bound, so a store or clock can wrap JavaScript promises (wasm-bindgen). Native ones are free to be `Send`. -- **`MemoryStore`:** a store in memory that is also a clock that only moves when told (or when `sleep` is called) and a - seeded random sequence, so tests are deterministic. +- **For tests:** `MemoryServices` is a `MemoryStore`, a `TestClock` that only moves when told (or when `sleep` is + called), and a `SeededRandom`, so tests are deterministic. - **`Error`:** one error type for all crates, with the variant saying what a caller can do: `Invalid`, `Unsupported`, `Conflict` (retry on the newer version), `CommitUnknown` (the commit may have landed: delete nothing it wrote, load - the table again), `NotFound`, `Store`, and `Corrupt`. It is `non_exhaustive`. + the table again), `NotFound`, `Store`, and `Corrupt`. It is `non_exhaustive`. A store's error (`StoreError`) converts + into it, keeping `NotFound`, `Invalid`, and `Unsupported`. - **Identifiers:** `uuid_v4` (from a `Random`) and `uuid_v7` (from a `Clock + Random`), built by the `uuid` crate (its builder needs neither its own randomness nor `std::time`, so it works in WebAssembly). @@ -50,6 +56,7 @@ browser, a Cloudflare Worker, a CLI, or a server, and tests run against memory. | D3 | One error type across crates, whose variants say what a caller can do | Accepted | | D10 | Objects are a `Store` of their own, apart from the time and randomness: readers need only a store, and store adapters don't decide how time is told | Accepted | | D13 | The time and randomness are two traits, `Clock` and `Random`, as ZIO's services are, rather than one environment trait: each says what it is, and each can be swapped on its own | Accepted | +| D14 | `Store`, `Clock`, and `Random` are loko's services, which every polythings project shares, rather than lakekit's own; `lakekit-core` re-exports them and keeps the error type | Accepted | ## Open questions and issues @@ -60,3 +67,5 @@ None open. - 2026-10-20: built, from Timelake's `timelake_format::host`. - 2026-10-21: one `Host` trait split into `Store` (D10), `Clock`, and `Random` (D13); `Services`, `TokioClock`, and `OsRandom` added; UUIDs from the `uuid` crate. The spec was "The host boundary". +- 2026-10-21: the traits come from loko (D14); `Clock::sleep` takes a `Duration`; `MemoryStore` is only a store, and + `MemoryServices` bundles it with a test clock and seeded randomness. diff --git a/spec/0004-store-adapters.md b/spec/0004-store-adapters.md index 367af9d..8b2c8df 100644 --- a/spec/0004-store-adapters.md +++ b/spec/0004-store-adapters.md @@ -1,27 +1,29 @@ --- number: 4 title: Store adapters -status: in-progress +status: done updated: 2026-10-21 versions: [unreleased] related: [1] -issues: [10] +issues: [] --- # 4. Store adapters -**In progress** · updated 2026-10-21 · versions unreleased · milestone: [0.1](milestones/v0-1.md) +**Done** · updated 2026-10-21 · versions unreleased · milestone: [0.1](milestones/v0-1.md) ## Summary Ready-made `Store`s ([spec 1](0001-store-clock-random.md)) over the store libraries applications already use, so a -native application doesn't write its own. Each adapter is a crate of its own; the `Store` trait stays the boundary, +native application doesn't write its own. They are [loko](https://github.com/polythings/loko)'s crates, since `Store` +is loko's service; `lakekit` re-exports `loko-object-store` (feature `object-store`). Each adapter is a crate of its +own; the `Store` trait stays the boundary, because some stores aren't these libraries (a Cloudflare R2 binding reached from JavaScript) and applications wrap their store (lakeglass counts every request). ## Design -**`lakekit-object-store`** (built): `ObjectStoreAdapter` over an `Arc` (0.14). +**`loko-object-store`** (built, as `lakekit-object-store` first): `ObjectStoreAdapter` over an `Arc` (0.14). - **Names:** lakekit uses absolute URIs; the adapter has a root (`s3://bucket/`, `memory://`, `file:///` for a local file system) and maps `` to the store's path ``. A URI outside the root is `Error::Invalid`. @@ -34,11 +36,11 @@ their store (lakeglass counts every request). - **Backends are the application's:** the crate enables no `object_store` features. Applications turn on `aws`, `gcp`, `azure`, `http`, or `fs` (or the `*-base` variants, which build for WebAssembly) on their own dependency, so the adapter builds for `wasm32-unknown-unknown` and pulls in nothing unasked. -- **Tests:** objects, conditional writes, multipart, prefix listing, and local files through the adapter, and an - Iceberg commit loaded back through it. +- **Tests:** objects, conditional writes, multipart, prefix listing, and local files through the adapter (in loko), + and an Iceberg commit loaded back through it (here, `lakekit-iceberg/tests/object_store.rs`). -**OpenDAL** ([#10](https://github.com/polythings/lakekit/issues/10)): the same over an OpenDAL `Operator`, once an -application needs it. +**OpenDAL** ([loko#6](https://github.com/polythings/loko/issues/6)): the same over an OpenDAL `Operator`, once an +application needs it, as a loko crate. ## Decisions @@ -46,11 +48,13 @@ application needs it. |---|---|---| | D11 | Store libraries are adapters, one crate each, over the `Store` trait; lakekit's crates never depend on them | Accepted | | D12 | An adapter enables none of its library's backends; the application chooses them | Accepted | +| D15 | Store adapters are loko's crates, next to the `Store` trait; lakekit uses them | Accepted | ## Open questions and issues -- [#10](https://github.com/polythings/lakekit/issues/10) Store adapter for OpenDAL (lakekit-opendal) +None open here. The OpenDAL adapter is [loko#6](https://github.com/polythings/loko/issues/6). ## History - 2026-10-21: proposed, and `lakekit-object-store` built. +- 2026-10-21: the adapter moved to loko as `loko-object-store` (D15); the OpenDAL issue moved with it. diff --git a/spec/README.md b/spec/README.md index 50c346e..a85d5fe 100644 --- a/spec/README.md +++ b/spec/README.md @@ -28,14 +28,14 @@ lakekit's design, laid out like Timelake's and lakeglass's. Start here, people a ## Index -4 specs: 3 in progress, 1 done. Decisions D1–D13. +4 specs: 2 in progress, 2 done. Decisions D1–D15. | # | Spec | Status | Updated | |---:|---|---|---| | 1 | [Store, Clock, and Random](0001-store-clock-random.md) | Done | 2026-10-21 | | 2 | [One crate per format](0002-formats.md) | In progress | 2026-10-21 | | 3 | [Apache Iceberg](0003-iceberg.md) | In progress | 2026-10-20 | -| 4 | [Store adapters](0004-store-adapters.md) | In progress | 2026-10-21 | +| 4 | [Store adapters](0004-store-adapters.md) | Done | 2026-10-21 | ## Milestones diff --git a/spec/milestones/v0-1.md b/spec/milestones/v0-1.md index dde1ad1..e3e40ba 100644 --- a/spec/milestones/v0-1.md +++ b/spec/milestones/v0-1.md @@ -2,7 +2,7 @@ **In progress** -The first release: the store, clock, and random traits, the Iceberg crate, and the `object_store` adapter, enough for +The first release: the Iceberg crate on loko's store, clock, and random services, and the `object_store` adapter, enough for Timelake's WASM reader and writer to use lakekit instead of their own Iceberg code, and for lakeglass to read Iceberg tables in the browser. @@ -11,7 +11,7 @@ tables in the browser. | 1 | [Store, Clock, and Random](../0001-store-clock-random.md) | Done | | 2 | [One crate per format](../0002-formats.md) | In progress | | 3 | [Apache Iceberg](../0003-iceberg.md) | In progress | -| 4 | [Store adapters](../0004-store-adapters.md) | In progress | +| 4 | [Store adapters](../0004-store-adapters.md) | Done | -**Done when:** the crates are on crates.io ([#7](https://github.com/polythings/lakekit/issues/7)), Timelake depends +**Done when:** loko 0.2.0 ([loko#1](https://github.com/polythings/loko/issues/1)) and these crates are on crates.io ([#7](https://github.com/polythings/lakekit/issues/7)), Timelake depends on them by version, and lakeglass's Iceberg reader is built on them. From 08595e164fdfb6d400b13f24f1b6348ed5b2c97a Mon Sep 17 00:00:00 2001 From: Oleksandr Litus Date: Wed, 21 Oct 2026 00:41:34 -0400 Subject: [PATCH 06/14] Core: Resources, following loko's naming loko calls the things it passes resources and its derive Provides (loko#5). Services and MemoryServices are Resources and MemoryResources, built with #[derive(Provides)], and loko is pinned at the revision that has them. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 ++--- CONTRIBUTING.md | 2 +- Cargo.lock | 10 +++--- Cargo.toml | 12 +++---- PRIOR_ART.md | 2 +- README.md | 6 ++-- crates/lakekit-core/README.md | 6 ++-- crates/lakekit-core/src/ids.rs | 4 +-- crates/lakekit-core/src/lib.rs | 10 +++--- .../src/{services.rs => resources.rs} | 22 ++++++------ crates/lakekit-iceberg/src/commit.rs | 36 +++++++++---------- crates/lakekit-iceberg/tests/iceberg_rust.rs | 6 ++-- crates/lakekit-iceberg/tests/object_store.rs | 4 +-- spec/0001-store-clock-random.md | 18 +++++----- spec/0004-store-adapters.md | 2 +- spec/milestones/v0-1.md | 2 +- 16 files changed, 76 insertions(+), 74 deletions(-) rename crates/lakekit-core/src/{services.rs => resources.rs} (67%) diff --git a/CHANGELOG.md b/CHANGELOG.md index da915fd..898031f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,9 @@ versions follow [Semantic Versioning](https://semver.org/). ### Added -- `lakekit-core`: loko's `Store`, `Clock`, and `Random` services with their test and native implementations - (`TokioClock` with feature `tokio`, `OsRandom` with feature `getrandom`); `Services` to pass the three as one; - `MemoryServices` for tests; one `Error` type, which a store's error converts into. +- `lakekit-core`: loko's `Store`, `Clock`, and `Random` resources with their test and native implementations + (`TokioClock` with feature `tokio`, `OsRandom` with feature `getrandom`); `Resources` to pass the three as one; + `MemoryResources` for tests; one `Error` type, which a store's error converts into. - `lakekit` feature `object-store`: a `Store` over an `object_store` client (S3, GCS, Azure, local files, memory), from a client or a URL (`loko-object-store`). - `lakekit-iceberg`: Iceberg table metadata, schemas, partition specs, manifest lists, and manifests read for format @@ -28,7 +28,7 @@ versions follow [Semantic Versioning](https://semver.org/). (`lakebed-core`, …) until 2026-10-21. Nothing was published under the old names; GitHub redirects the old repository URL. - The I/O boundary is three traits, `Store` (objects), `Clock` (the time and a timer), and `Random` (random numbers), - instead of one `Host`; `MemoryHost` is `MemoryServices`. UUIDs come from the `uuid` crate. + instead of one `Host`; `MemoryHost` is `MemoryResources`. UUIDs come from the `uuid` crate. - The three traits are [loko](https://github.com/polythings/loko)'s, and `Clock::sleep` takes a `Duration`. The `object_store` adapter is `loko-object-store`, re-exported as `lakekit::object_store`; `lakekit-object-store` is gone (it was never published). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e868f58..a248db6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,7 +50,7 @@ ideas we took. ## Tests - **A bug fix starts with a test that fails**, then the fix. -- **Tests run offline**, against `MemoryServices` or a store over the reference implementation's in-memory storage. +- **Tests run offline**, against `MemoryResources` or a store over the reference implementation's in-memory storage. - **Interop:** writers are tested against the format's reference reader, and readers against the reference writer. ## Checks diff --git a/Cargo.lock b/Cargo.lock index 3a2bb36..a714b1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1743,7 +1743,7 @@ checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "loko-clock" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=3972fdd0f2dc863328c38048583f3066cba939c4#3972fdd0f2dc863328c38048583f3066cba939c4" +source = "git+https://github.com/polythings/loko?rev=009756afed9566f22b6b25bb843eb8761a3bc1b3#009756afed9566f22b6b25bb843eb8761a3bc1b3" dependencies = [ "loko-macros", "tokio", @@ -1752,7 +1752,7 @@ dependencies = [ [[package]] name = "loko-macros" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=3972fdd0f2dc863328c38048583f3066cba939c4#3972fdd0f2dc863328c38048583f3066cba939c4" +source = "git+https://github.com/polythings/loko?rev=009756afed9566f22b6b25bb843eb8761a3bc1b3#009756afed9566f22b6b25bb843eb8761a3bc1b3" dependencies = [ "proc-macro2", "quote", @@ -1762,7 +1762,7 @@ dependencies = [ [[package]] name = "loko-object-store" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=3972fdd0f2dc863328c38048583f3066cba939c4#3972fdd0f2dc863328c38048583f3066cba939c4" +source = "git+https://github.com/polythings/loko?rev=009756afed9566f22b6b25bb843eb8761a3bc1b3#009756afed9566f22b6b25bb843eb8761a3bc1b3" dependencies = [ "futures-util", "loko-store", @@ -1773,7 +1773,7 @@ dependencies = [ [[package]] name = "loko-random" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=3972fdd0f2dc863328c38048583f3066cba939c4#3972fdd0f2dc863328c38048583f3066cba939c4" +source = "git+https://github.com/polythings/loko?rev=009756afed9566f22b6b25bb843eb8761a3bc1b3#009756afed9566f22b6b25bb843eb8761a3bc1b3" dependencies = [ "getrandom 0.4.3", "loko-macros", @@ -1782,7 +1782,7 @@ dependencies = [ [[package]] name = "loko-store" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=3972fdd0f2dc863328c38048583f3066cba939c4#3972fdd0f2dc863328c38048583f3066cba939c4" +source = "git+https://github.com/polythings/loko?rev=009756afed9566f22b6b25bb843eb8761a3bc1b3#009756afed9566f22b6b25bb843eb8761a3bc1b3" dependencies = [ "loko-macros", "thiserror", diff --git a/Cargo.toml b/Cargo.toml index 22949a3..fd38e9c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,13 +14,13 @@ categories = ["database-implementations", "wasm"] [workspace.dependencies] lakekit-core = { path = "crates/lakekit-core", version = "0.1.0" } lakekit-iceberg = { path = "crates/lakekit-iceberg", version = "0.1.0", default-features = false } -# loko provides the Store, Clock, and Random services. From git until loko 0.2.0 is on crates.io +# loko provides the Store, Clock, and Random resources. From git until loko 0.2.0 is on crates.io # (polythings/loko#1); publishing uses the version. -loko-store = { git = "https://github.com/polythings/loko", rev = "3972fdd0f2dc863328c38048583f3066cba939c4", version = "0.2.0" } -loko-clock = { git = "https://github.com/polythings/loko", rev = "3972fdd0f2dc863328c38048583f3066cba939c4", version = "0.2.0" } -loko-random = { git = "https://github.com/polythings/loko", rev = "3972fdd0f2dc863328c38048583f3066cba939c4", version = "0.2.0" } -loko-macros = { git = "https://github.com/polythings/loko", rev = "3972fdd0f2dc863328c38048583f3066cba939c4", version = "0.2.0" } -loko-object-store = { git = "https://github.com/polythings/loko", rev = "3972fdd0f2dc863328c38048583f3066cba939c4", version = "0.2.0" } +loko-store = { git = "https://github.com/polythings/loko", rev = "009756afed9566f22b6b25bb843eb8761a3bc1b3", version = "0.2.0" } +loko-clock = { git = "https://github.com/polythings/loko", rev = "009756afed9566f22b6b25bb843eb8761a3bc1b3", version = "0.2.0" } +loko-random = { git = "https://github.com/polythings/loko", rev = "009756afed9566f22b6b25bb843eb8761a3bc1b3", version = "0.2.0" } +loko-macros = { git = "https://github.com/polythings/loko", rev = "009756afed9566f22b6b25bb843eb8761a3bc1b3", version = "0.2.0" } +loko-object-store = { git = "https://github.com/polythings/loko", rev = "009756afed9566f22b6b25bb843eb8761a3bc1b3", version = "0.2.0" } object_store = { version = "0.14", default-features = false } url = "2" thiserror = "2" diff --git a/PRIOR_ART.md b/PRIOR_ART.md index 741bdad..b8effec 100644 --- a/PRIOR_ART.md +++ b/PRIOR_ART.md @@ -11,7 +11,7 @@ never code: every line of lakekit is our own. How we credit is in [CONTRIBUTING. | [iceberg-rust](https://github.com/apache/iceberg-rust) | Apache-2.0 | The reference the interop tests check against (a test dependency only) | | [Apache Avro](https://avro.apache.org) (`apache-avro`) | Apache-2.0 | Reading and writing manifests and manifest lists | | [arrow-rs](https://github.com/apache/arrow-rs) (`arrow-schema`, `parquet`) | Apache-2.0 | Arrow schemas; Parquet footers for statistics | -| [loko](https://github.com/polythings/loko) | Apache-2.0 | The `Store`, `Clock`, and `Random` services, and the `object_store` adapter | +| [loko](https://github.com/polythings/loko) | Apache-2.0 | The `Store`, `Clock`, and `Random` resources, and the `object_store` adapter | | [object_store](https://github.com/apache/arrow-rs-object-store) | Apache-2.0 / MIT | The stores `loko-object-store` adapts (re-exported by `lakekit`) | | [uuid](https://github.com/uuid-rs/uuid) | Apache-2.0 / MIT | UUIDs for file names, from a `Random` and a `Clock` | diff --git a/README.md b/README.md index e884e6b..8a6b3ee 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Table formats read and written without I/O of their own, in Rust, for WebAssembly and native targets. Each format's metadata (Apache Iceberg first) is parsed and written by plain code that asks a `Store` for bytes, a `Clock` for the -time, and a `Random` for random numbers ([loko](https://github.com/polythings/loko)'s services): the application +time, and a `Random` for random numbers ([loko](https://github.com/polythings/loko)'s resources): the application supplies them, so the same crate runs in a browser, a Cloudflare Worker, a CLI, or a server. @@ -14,7 +14,7 @@ share. The official Rust crates for these formats build on tokio and HTTP client | Crate | What it does | |---|---| -| [`lakekit-core`](crates/lakekit-core) | loko's `Store`, `Clock`, and `Random` with their test and native implementations, `Services`, and one error type | +| [`lakekit-core`](crates/lakekit-core) | loko's `Store`, `Clock`, and `Random` with their test and native implementations, `Resources`, and one error type | | [`lakekit-iceberg`](crates/lakekit-iceberg) | Apache Iceberg: metadata, manifests, and commits | | [`lakekit`](crates/lakekit) | Re-exports the crates, one feature each, and `loko-object-store` (a `Store` over an `object_store` client) | @@ -46,7 +46,7 @@ async fn files(s3: Arc) -> lakekit_core::Result { +/// `Resources::new(ObjectStoreAdapter::new(…), TokioClock, OsRandom)`. Any type that is all three +/// works as well: a JavaScript host, or an application's own `#[derive(loko::Provides)]` struct. +#[derive(Clone, Debug, Default, Provides)] +pub struct Resources { /// The store. - #[service(Store)] + #[provides(Store)] pub store: S, /// The clock. - #[service(Clock)] + #[provides(Clock)] pub clock: C, /// The random source. - #[service(Random)] + #[provides(Random)] pub random: R, } -impl Services { +impl Resources { /// `store` with `clock` and `random`. pub fn new(store: S, clock: C, random: R) -> Self { Self { store, clock, random } @@ -27,9 +27,9 @@ impl Services { } /// Everything in memory, for tests: a [`MemoryStore`], a [`TestClock`], and a [`SeededRandom`]. -pub type MemoryServices = Services; +pub type MemoryResources = Resources; -impl MemoryServices { +impl MemoryResources { /// An empty store, the clock at `now_ms` milliseconds after the Unix epoch, and seed 0. pub fn at_ms(now_ms: i64) -> Self { Self::new(MemoryStore::new(), TestClock::at_ms(now_ms), SeededRandom::new(0)) diff --git a/crates/lakekit-iceberg/src/commit.rs b/crates/lakekit-iceberg/src/commit.rs index 0f57fc5..e92b66d 100644 --- a/crates/lakekit-iceberg/src/commit.rs +++ b/crates/lakekit-iceberg/src/commit.rs @@ -471,7 +471,7 @@ pub fn counters(change: &Change, base: &HashMap Table { + fn create(store: &MemoryResources, format_version: u8) -> Table { let metadata = json!({ "format-version": format_version, "table-uuid": "u", "location": LOCATION, "last-sequence-number": 0, "last-updated-ms": 0, "last-column-id": 2, "current-schema-id": 0, @@ -511,8 +511,8 @@ mod tests { Change::Append { files } } - fn try_run>( - store: &MemoryServices, + fn try_run>( + store: &MemoryResources, base: Table, change: &Change, hooks: &mut K, @@ -520,8 +520,8 @@ mod tests { block_on(commit(store, base, change, &HashMap::new(), &CommitOptions::default(), hooks)) } - fn run>( - store: &MemoryServices, + fn run>( + store: &MemoryResources, base: Table, change: &Change, hooks: &mut K, @@ -535,7 +535,7 @@ mod tests { /// The live files of the newest version, read back from the store: (name, data sequence /// number, file sequence number). - fn live(store: &MemoryServices) -> Vec<(String, i64, i64)> { + fn live(store: &MemoryResources) -> Vec<(String, i64, i64)> { let mut t = block_on(Table::load(store, LOCATION)).unwrap(); let mut files: Vec<_> = block_on(t.live_files(store)) .unwrap() @@ -554,7 +554,7 @@ mod tests { #[test] fn append_replace_and_delete() { - let store = MemoryServices::at_ms(1_000); + let store = MemoryResources::at_ms(1_000); let t = create(&store, 2); let (t, info) = run(&store, t, &append(vec![file("a", 1, 10), file("b", 2, 20)]), &mut NoHooks); assert_eq!((info.version, info.attempts, info.skipped), (2, 1, false)); @@ -581,7 +581,7 @@ mod tests { #[test] fn a_lost_race_retries_on_the_newer_version() { - let store = MemoryServices::at_ms(1_000); + let store = MemoryResources::at_ms(1_000); let stale = create(&store, 2); let (_, first) = run(&store, stale.clone(), &append(vec![file("a", 1, 10)]), &mut NoHooks); let (_, second) = run(&store, stale, &append(vec![file("b", 1, 10)]), &mut NoHooks); @@ -594,7 +594,7 @@ mod tests { #[test] fn removing_files_that_are_gone_is_a_conflict() { - let store = MemoryServices::at_ms(1_000); + let store = MemoryResources::at_ms(1_000); let (t, _) = run(&store, create(&store, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); let removed = t.files.clone().unwrap_or_default(); let (t, _) = run(&store, t, &Change::Delete { removed: removed.clone() }, &mut NoHooks); @@ -605,14 +605,14 @@ mod tests { /// Hooks that skip a batch already committed, and add a file and summary properties. struct Batch(&'static str); - impl CommitHooks for Batch { + impl CommitHooks for Batch { type Error = Error; fn skip(&mut self, base: &Table) -> bool { base.metadata.ancestors().iter().any(|s| s.summary.get("batch").map(String::as_str) == Some(self.0)) } - fn extra(&mut self, _: &MemoryServices, base: &Table, snapshot_id: i64) -> impl Future> { + fn extra(&mut self, _: &MemoryResources, base: &Table, snapshot_id: i64) -> impl Future> { let uri = format!("{}/metadata/extra-{snapshot_id}", base.location); let summary = HashMap::from([("batch".to_string(), self.0.to_string()), ("extra".to_string(), uri.clone())]); @@ -622,7 +622,7 @@ mod tests { #[test] fn hooks_skip_and_extend_commits() { - let store = MemoryServices::at_ms(1_000); + let store = MemoryResources::at_ms(1_000); let (t, info) = run(&store, create(&store, 2), &append(vec![file("a", 1, 10)]), &mut Batch("b1")); let extra = t.metadata.summary()["extra"].clone(); assert!(block_on(store.exists(&extra)).unwrap()); @@ -634,7 +634,7 @@ mod tests { #[test] fn many_manifests_merge() { - let store = MemoryServices::at_ms(1_000); + let store = MemoryResources::at_ms(1_000); let mut t = create(&store, 2); for i in 0..9 { t = run(&store, t, &append(vec![file(&format!("f{i}"), i, 1)]), &mut NoHooks).0; @@ -645,9 +645,9 @@ mod tests { #[test] fn a_clock_behind_the_table_keeps_the_logs_in_order() { - let store = MemoryServices::at_ms(5_000); + let store = MemoryResources::at_ms(5_000); let (t, _) = run(&store, create(&store, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); - let late = MemoryServices::at_ms(1_000); + let late = MemoryResources::at_ms(1_000); for uri in store.store.uris() { block_on(late.put(&uri, block_on(store.get(&uri)).unwrap().unwrap())).unwrap(); } @@ -657,7 +657,7 @@ mod tests { #[test] fn a_replace_needs_a_sequence_number_per_added_file() { - let store = MemoryServices::at_ms(1_000); + let store = MemoryResources::at_ms(1_000); let (t, _) = run(&store, create(&store, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); let removed = files_named(&t, "a.parquet"); let change = @@ -668,7 +668,7 @@ mod tests { #[test] fn only_v2_tables_are_written() { - let store = MemoryServices::at_ms(1_000); + let store = MemoryResources::at_ms(1_000); let err = try_run(&store, create(&store, 1), &append(vec![]), &mut NoHooks); assert!(matches!(err, Err(Error::Unsupported(_)))); } diff --git a/crates/lakekit-iceberg/tests/iceberg_rust.rs b/crates/lakekit-iceberg/tests/iceberg_rust.rs index ad54de1..ff9ff0d 100644 --- a/crates/lakekit-iceberg/tests/iceberg_rust.rs +++ b/crates/lakekit-iceberg/tests/iceberg_rust.rs @@ -21,7 +21,7 @@ use iceberg::spec::{ use iceberg::transaction::{ApplyTransactionAction, Transaction}; use iceberg::writer::file_writer::{FileWriter as _, FileWriterBuilder, ParquetWriterBuilder}; use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, NamespaceIdent, TableCreation, TableIdent}; -use lakekit_core::{Listed, MemoryUpload, SeededRandom, Services, Store, StoreError as Error, TestClock}; +use lakekit_core::{Listed, MemoryUpload, SeededRandom, Resources, Store, StoreError as Error, TestClock}; use lakekit_iceberg::stats::ParquetStats; use lakekit_iceberg::{Change, CommitOptions, DataFile, Datum, Entry, NoHooks, Status, Table, commit}; use parquet::arrow::ArrowWriter; @@ -39,7 +39,7 @@ struct IoStore { } /// The store with a test clock and seeded random numbers. -type TestStore = Services; +type TestStore = Resources; /// The store's results. type Result = std::result::Result; @@ -231,7 +231,7 @@ async fn tables_round_trip_between_lakekit_and_iceberg_rust() { Some(UnboundPartitionSpec::builder().add_partition_field(1, "time_day", IceTransform::Day).unwrap().build()); let table = catalog.create_table(&ns, creation).await.unwrap(); let io = table.file_io().clone(); - let store = Services::new(IoStore { io: io.clone() }, TestClock::at_ms(T0 / 1000), SeededRandom::new(0)); + let store = Resources::new(IoStore { io: io.clone() }, TestClock::at_ms(T0 / 1000), SeededRandom::new(0)); let arrow = Arc::new(iceberg::arrow::schema_to_arrow_schema(table.metadata().current_schema()).unwrap()); // iceberg-rust appends; lakekit reads the file as iceberg-rust records it. diff --git a/crates/lakekit-iceberg/tests/object_store.rs b/crates/lakekit-iceberg/tests/object_store.rs index 0bb4efb..b29558f 100644 --- a/crates/lakekit-iceberg/tests/object_store.rs +++ b/crates/lakekit-iceberg/tests/object_store.rs @@ -5,14 +5,14 @@ use std::collections::HashMap; use std::sync::Arc; -use lakekit_core::{OsRandom, Services, Store, TokioClock}; +use lakekit_core::{OsRandom, Resources, Store, TokioClock}; use lakekit_iceberg::{Change, CommitOptions, DataFile, Datum, Metadata, NoHooks, Table, commit}; use loko_object_store::ObjectStoreAdapter; use object_store::memory::InMemory; #[tokio::test] async fn iceberg_commits_through_object_store() { - let store = Services::new(ObjectStoreAdapter::new(Arc::new(InMemory::new()), "memory://"), TokioClock, OsRandom); + let store = Resources::new(ObjectStoreAdapter::new(Arc::new(InMemory::new()), "memory://"), TokioClock, OsRandom); let location = "memory://warehouse/t"; let metadata = Metadata::parse( br#"{"format-version": 2, "table-uuid": "u", "location": "memory://warehouse/t", "last-sequence-number": 0, diff --git a/spec/0001-store-clock-random.md b/spec/0001-store-clock-random.md index 0565ee4..18ffbbb 100644 --- a/spec/0001-store-clock-random.md +++ b/spec/0001-store-clock-random.md @@ -24,21 +24,21 @@ browser, a Cloudflare Worker, a CLI, or a server, and tests run against memory. `false` when an object is there), `upload` (a multipart upload: parts in order, then complete or abort), `list` (by string prefix, not only by directory), and `delete` (missing objects are not an error). Paths are absolute URIs, as table metadata records them. Adapters implement it over store libraries ([spec 4](0004-store-adapters.md)). -- **Where they come from:** the three are [loko](https://github.com/polythings/loko)'s services (`loko-store`, +- **Where they come from:** the three are [loko](https://github.com/polythings/loko)'s resources (`loko-store`, `loko-clock`, `loko-random`), shared by the polythings projects; `lakekit-core` re-exports them with their test and - native implementations, and adds what is lakekit's: the error type, the identifiers, and `Services`. + native implementations, and adds what is lakekit's: the error type, the identifiers, and `Resources`. - **`Clock`:** `now` and `now_ms`, and `sleep(Duration)`: commit timestamps, time-ordered file names, and the wait between commit attempts. - **`Random`:** `next_u64`: snapshot ids, file names, and the jitter between commit attempts. - **Who needs what:** readers need only a `Store`; commits need `Store + Clock + Random`. -- **`Services`:** a store, a clock, and a random source as one value (`Services::new(store, clock, random)`), for APIs - that need all three, built with loko's `#[derive(Services)]`. Any type that is all three (a JavaScript host, an +- **`Resources`:** a store, a clock, and a random source as one value (`Resources::new(store, clock, random)`), for APIs + that need all three, built with loko's `#[derive(Provides)]`. Any type that is all three (a JavaScript host, an application's own bundle) needs no wrapper. - **Native implementations:** `TokioClock` (feature `tokio`: the system clock and tokio's timer, native targets) and `OsRandom` (feature `getrandom`: the operating system's generator, native targets and WASI). - **Futures aren't `Send`.** Methods return `impl Future`, without a `Send` bound, so a store or clock can wrap JavaScript promises (wasm-bindgen). Native ones are free to be `Send`. -- **For tests:** `MemoryServices` is a `MemoryStore`, a `TestClock` that only moves when told (or when `sleep` is +- **For tests:** `MemoryResources` is a `MemoryStore`, a `TestClock` that only moves when told (or when `sleep` is called), and a `SeededRandom`, so tests are deterministic. - **`Error`:** one error type for all crates, with the variant saying what a caller can do: `Invalid`, `Unsupported`, `Conflict` (retry on the newer version), `CommitUnknown` (the commit may have landed: delete nothing it wrote, load @@ -56,7 +56,7 @@ browser, a Cloudflare Worker, a CLI, or a server, and tests run against memory. | D3 | One error type across crates, whose variants say what a caller can do | Accepted | | D10 | Objects are a `Store` of their own, apart from the time and randomness: readers need only a store, and store adapters don't decide how time is told | Accepted | | D13 | The time and randomness are two traits, `Clock` and `Random`, as ZIO's services are, rather than one environment trait: each says what it is, and each can be swapped on its own | Accepted | -| D14 | `Store`, `Clock`, and `Random` are loko's services, which every polythings project shares, rather than lakekit's own; `lakekit-core` re-exports them and keeps the error type | Accepted | +| D14 | `Store`, `Clock`, and `Random` are loko's resources, which every polythings project shares, rather than lakekit's own; `lakekit-core` re-exports them and keeps the error type | Accepted | ## Open questions and issues @@ -65,7 +65,9 @@ None open. ## History - 2026-10-20: built, from Timelake's `timelake_format::host`. -- 2026-10-21: one `Host` trait split into `Store` (D10), `Clock`, and `Random` (D13); `Services`, `TokioClock`, and +- 2026-10-21: one `Host` trait split into `Store` (D10), `Clock`, and `Random` (D13); `Resources`, `TokioClock`, and `OsRandom` added; UUIDs from the `uuid` crate. The spec was "The host boundary". - 2026-10-21: the traits come from loko (D14); `Clock::sleep` takes a `Duration`; `MemoryStore` is only a store, and - `MemoryServices` bundles it with a test clock and seeded randomness. + `MemoryResources` bundles it with a test clock and seeded randomness. +- 2026-10-21: `Services` and `MemoryServices` are `Resources` and `MemoryResources`, as loko calls the things it + passes resources; `Resources` uses loko's `#[derive(Provides)]`. diff --git a/spec/0004-store-adapters.md b/spec/0004-store-adapters.md index 8b2c8df..256e7f0 100644 --- a/spec/0004-store-adapters.md +++ b/spec/0004-store-adapters.md @@ -16,7 +16,7 @@ issues: [] Ready-made `Store`s ([spec 1](0001-store-clock-random.md)) over the store libraries applications already use, so a native application doesn't write its own. They are [loko](https://github.com/polythings/loko)'s crates, since `Store` -is loko's service; `lakekit` re-exports `loko-object-store` (feature `object-store`). Each adapter is a crate of its +is loko's resource; `lakekit` re-exports `loko-object-store` (feature `object-store`). Each adapter is a crate of its own; the `Store` trait stays the boundary, because some stores aren't these libraries (a Cloudflare R2 binding reached from JavaScript) and applications wrap their store (lakeglass counts every request). diff --git a/spec/milestones/v0-1.md b/spec/milestones/v0-1.md index e3e40ba..8541d52 100644 --- a/spec/milestones/v0-1.md +++ b/spec/milestones/v0-1.md @@ -2,7 +2,7 @@ **In progress** -The first release: the Iceberg crate on loko's store, clock, and random services, and the `object_store` adapter, enough for +The first release: the Iceberg crate on loko's store, clock, and random resources, and the `object_store` adapter, enough for Timelake's WASM reader and writer to use lakekit instead of their own Iceberg code, and for lakeglass to read Iceberg tables in the browser. From 04e6f60fb01f137f6bb9bf101f78e6dd20bc43f6 Mon Sep 17 00:00:00 2001 From: Oleksandr Litus Date: Wed, 21 Oct 2026 00:41:58 -0400 Subject: [PATCH 07/14] Iceberg: rustfmt after the Resources rename Co-Authored-By: Claude Opus 5.5 --- crates/lakekit-iceberg/src/commit.rs | 7 ++++++- crates/lakekit-iceberg/tests/iceberg_rust.rs | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/lakekit-iceberg/src/commit.rs b/crates/lakekit-iceberg/src/commit.rs index e92b66d..75d318e 100644 --- a/crates/lakekit-iceberg/src/commit.rs +++ b/crates/lakekit-iceberg/src/commit.rs @@ -612,7 +612,12 @@ mod tests { base.metadata.ancestors().iter().any(|s| s.summary.get("batch").map(String::as_str) == Some(self.0)) } - fn extra(&mut self, _: &MemoryResources, base: &Table, snapshot_id: i64) -> impl Future> { + fn extra( + &mut self, + _: &MemoryResources, + base: &Table, + snapshot_id: i64, + ) -> impl Future> { let uri = format!("{}/metadata/extra-{snapshot_id}", base.location); let summary = HashMap::from([("batch".to_string(), self.0.to_string()), ("extra".to_string(), uri.clone())]); diff --git a/crates/lakekit-iceberg/tests/iceberg_rust.rs b/crates/lakekit-iceberg/tests/iceberg_rust.rs index ff9ff0d..a077295 100644 --- a/crates/lakekit-iceberg/tests/iceberg_rust.rs +++ b/crates/lakekit-iceberg/tests/iceberg_rust.rs @@ -21,7 +21,7 @@ use iceberg::spec::{ use iceberg::transaction::{ApplyTransactionAction, Transaction}; use iceberg::writer::file_writer::{FileWriter as _, FileWriterBuilder, ParquetWriterBuilder}; use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, NamespaceIdent, TableCreation, TableIdent}; -use lakekit_core::{Listed, MemoryUpload, SeededRandom, Resources, Store, StoreError as Error, TestClock}; +use lakekit_core::{Listed, MemoryUpload, Resources, SeededRandom, Store, StoreError as Error, TestClock}; use lakekit_iceberg::stats::ParquetStats; use lakekit_iceberg::{Change, CommitOptions, DataFile, Datum, Entry, NoHooks, Status, Table, commit}; use parquet::arrow::ArrowWriter; From 7468ed2a6d5b59de54205c2d07012977c7d51b06 Mon Sep 17 00:00:00 2001 From: Oleksandr Litus Date: Wed, 21 Oct 2026 02:52:44 -0400 Subject: [PATCH 08/14] Core: draw identifiers and jitter from Random::rng loko's Random now hands out generators (rng()) instead of numbers. UUIDs fill their random bytes from one generator, snapshot ids and commit jitter draw from one, and lakekit-core re-exports Rng, CryptoRng, and SeededRng. loko is pinned at the revision with the change. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 11 ++++++----- Cargo.toml | 10 +++++----- crates/lakekit-core/src/ids.rs | 8 +++----- crates/lakekit-core/src/lib.rs | 2 +- crates/lakekit-iceberg/src/commit.rs | 7 ++++--- spec/0001-store-clock-random.md | 4 +++- 6 files changed, 22 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a714b1c..a434ea2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1743,7 +1743,7 @@ checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "loko-clock" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=009756afed9566f22b6b25bb843eb8761a3bc1b3#009756afed9566f22b6b25bb843eb8761a3bc1b3" +source = "git+https://github.com/polythings/loko?rev=112fb3e81114aef2c6a060eb1c00366cf4927f5b#112fb3e81114aef2c6a060eb1c00366cf4927f5b" dependencies = [ "loko-macros", "tokio", @@ -1752,7 +1752,7 @@ dependencies = [ [[package]] name = "loko-macros" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=009756afed9566f22b6b25bb843eb8761a3bc1b3#009756afed9566f22b6b25bb843eb8761a3bc1b3" +source = "git+https://github.com/polythings/loko?rev=112fb3e81114aef2c6a060eb1c00366cf4927f5b#112fb3e81114aef2c6a060eb1c00366cf4927f5b" dependencies = [ "proc-macro2", "quote", @@ -1762,7 +1762,7 @@ dependencies = [ [[package]] name = "loko-object-store" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=009756afed9566f22b6b25bb843eb8761a3bc1b3#009756afed9566f22b6b25bb843eb8761a3bc1b3" +source = "git+https://github.com/polythings/loko?rev=112fb3e81114aef2c6a060eb1c00366cf4927f5b#112fb3e81114aef2c6a060eb1c00366cf4927f5b" dependencies = [ "futures-util", "loko-store", @@ -1773,16 +1773,17 @@ dependencies = [ [[package]] name = "loko-random" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=009756afed9566f22b6b25bb843eb8761a3bc1b3#009756afed9566f22b6b25bb843eb8761a3bc1b3" +source = "git+https://github.com/polythings/loko?rev=112fb3e81114aef2c6a060eb1c00366cf4927f5b#112fb3e81114aef2c6a060eb1c00366cf4927f5b" dependencies = [ "getrandom 0.4.3", "loko-macros", + "rand_core 0.10.1", ] [[package]] name = "loko-store" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=009756afed9566f22b6b25bb843eb8761a3bc1b3#009756afed9566f22b6b25bb843eb8761a3bc1b3" +source = "git+https://github.com/polythings/loko?rev=112fb3e81114aef2c6a060eb1c00366cf4927f5b#112fb3e81114aef2c6a060eb1c00366cf4927f5b" dependencies = [ "loko-macros", "thiserror", diff --git a/Cargo.toml b/Cargo.toml index fd38e9c..19cd6a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,11 +16,11 @@ lakekit-core = { path = "crates/lakekit-core", version = "0.1.0" } lakekit-iceberg = { path = "crates/lakekit-iceberg", version = "0.1.0", default-features = false } # loko provides the Store, Clock, and Random resources. From git until loko 0.2.0 is on crates.io # (polythings/loko#1); publishing uses the version. -loko-store = { git = "https://github.com/polythings/loko", rev = "009756afed9566f22b6b25bb843eb8761a3bc1b3", version = "0.2.0" } -loko-clock = { git = "https://github.com/polythings/loko", rev = "009756afed9566f22b6b25bb843eb8761a3bc1b3", version = "0.2.0" } -loko-random = { git = "https://github.com/polythings/loko", rev = "009756afed9566f22b6b25bb843eb8761a3bc1b3", version = "0.2.0" } -loko-macros = { git = "https://github.com/polythings/loko", rev = "009756afed9566f22b6b25bb843eb8761a3bc1b3", version = "0.2.0" } -loko-object-store = { git = "https://github.com/polythings/loko", rev = "009756afed9566f22b6b25bb843eb8761a3bc1b3", version = "0.2.0" } +loko-store = { git = "https://github.com/polythings/loko", rev = "112fb3e81114aef2c6a060eb1c00366cf4927f5b", version = "0.2.0" } +loko-clock = { git = "https://github.com/polythings/loko", rev = "112fb3e81114aef2c6a060eb1c00366cf4927f5b", version = "0.2.0" } +loko-random = { git = "https://github.com/polythings/loko", rev = "112fb3e81114aef2c6a060eb1c00366cf4927f5b", version = "0.2.0" } +loko-macros = { git = "https://github.com/polythings/loko", rev = "112fb3e81114aef2c6a060eb1c00366cf4927f5b", version = "0.2.0" } +loko-object-store = { git = "https://github.com/polythings/loko", rev = "112fb3e81114aef2c6a060eb1c00366cf4927f5b", version = "0.2.0" } object_store = { version = "0.14", default-features = false } url = "2" thiserror = "2" diff --git a/crates/lakekit-core/src/ids.rs b/crates/lakekit-core/src/ids.rs index 6a49f24..e75c507 100644 --- a/crates/lakekit-core/src/ids.rs +++ b/crates/lakekit-core/src/ids.rs @@ -1,12 +1,11 @@ use uuid::Builder; -use crate::{Clock, Random}; +use crate::{Clock, Random, Rng}; /// A random UUID (version 4) from `random`. pub fn uuid_v4(random: &impl Random) -> String { let mut bytes = [0u8; 16]; - bytes[..8].copy_from_slice(&random.next_u64().to_be_bytes()); - bytes[8..].copy_from_slice(&random.next_u64().to_be_bytes()); + random.rng().fill_bytes(&mut bytes); Builder::from_random_bytes(bytes).into_uuid().to_string() } @@ -15,8 +14,7 @@ pub fn uuid_v4(random: &impl Random) -> String { pub fn uuid_v7(source: &(impl Clock + Random)) -> String { let millis = u64::try_from(source.now_ms()).unwrap_or(0); let mut bytes = [0u8; 10]; - bytes[..8].copy_from_slice(&source.next_u64().to_be_bytes()); - bytes[8..].copy_from_slice(&source.next_u64().to_be_bytes()[..2]); + source.rng().fill_bytes(&mut bytes); Builder::from_unix_timestamp_millis(millis, &bytes).into_uuid().to_string() } diff --git a/crates/lakekit-core/src/lib.rs b/crates/lakekit-core/src/lib.rs index ed69916..48bd529 100644 --- a/crates/lakekit-core/src/lib.rs +++ b/crates/lakekit-core/src/lib.rs @@ -40,6 +40,6 @@ pub use loko_clock::TokioClock; pub use loko_clock::{Clock, HasClock, TestClock}; #[cfg(all(feature = "getrandom", not(all(target_arch = "wasm32", target_os = "unknown"))))] pub use loko_random::OsRandom; -pub use loko_random::{HasRandom, Random, SeededRandom}; +pub use loko_random::{CryptoRng, HasRandom, Random, Rng, SeededRandom, SeededRng}; pub use loko_store::{Error as StoreError, HasStore, Listed, MemoryStore, MemoryUpload, Store, Upload}; pub use resources::{MemoryResources, Resources}; diff --git a/crates/lakekit-iceberg/src/commit.rs b/crates/lakekit-iceberg/src/commit.rs index 75d318e..ea65342 100644 --- a/crates/lakekit-iceberg/src/commit.rs +++ b/crates/lakekit-iceberg/src/commit.rs @@ -13,7 +13,7 @@ use std::hash::BuildHasher; use std::sync::Arc; use std::time::Duration; -use lakekit_core::{Clock, Random, Store, uuid_v4}; +use lakekit_core::{Clock, Random, Rng, Store, uuid_v4}; use crate::manifest::{ DataFile, Entry, ManifestContext, ManifestFile, ManifestWrite, Status, manifest_file, write_manifest, @@ -182,7 +182,7 @@ pub async fn commit, S: BuildHasher for attempt in 1..=options.max_attempts { if attempt > 1 { let (lo, hi) = options.backoff_ms; - let ms = (lo + store.next_u64() % hi.saturating_sub(lo).max(1)) * attempt; + let ms = (lo + store.rng().next_u64() % hi.saturating_sub(lo).max(1)) * attempt; store.sleep(Duration::from_millis(ms)).await; base = hooks.reload(store, &base.location).await?; } @@ -230,8 +230,9 @@ pub async fn put_metadata(store: &S, uri: &str, bytes: Vec) -> Res /// A snapshot id not in the table yet. fn new_snapshot_id(random: &impl Random, base: &Table) -> i64 { + let mut rng = random.rng(); loop { - let id = i64::try_from(random.next_u64() >> 1).unwrap_or(1).max(1); + let id = i64::try_from(rng.next_u64() >> 1).unwrap_or(1).max(1); if !base.metadata.has_snapshot(id) { return id; } diff --git a/spec/0001-store-clock-random.md b/spec/0001-store-clock-random.md index 18ffbbb..3ffca52 100644 --- a/spec/0001-store-clock-random.md +++ b/spec/0001-store-clock-random.md @@ -29,7 +29,8 @@ browser, a Cloudflare Worker, a CLI, or a server, and tests run against memory. native implementations, and adds what is lakekit's: the error type, the identifiers, and `Resources`. - **`Clock`:** `now` and `now_ms`, and `sleep(Duration)`: commit timestamps, time-ordered file names, and the wait between commit attempts. -- **`Random`:** `next_u64`: snapshot ids, file names, and the jitter between commit attempts. +- **`Random`:** `rng()`, a generator of the caller's own: snapshot ids, file names, and the jitter between commit + attempts. - **Who needs what:** readers need only a `Store`; commits need `Store + Clock + Random`. - **`Resources`:** a store, a clock, and a random source as one value (`Resources::new(store, clock, random)`), for APIs that need all three, built with loko's `#[derive(Provides)]`. Any type that is all three (a JavaScript host, an @@ -71,3 +72,4 @@ None open. `MemoryResources` bundles it with a test clock and seeded randomness. - 2026-10-21: `Services` and `MemoryServices` are `Resources` and `MemoryResources`, as loko calls the things it passes resources; `Resources` uses loko's `#[derive(Provides)]`. +- 2026-10-21: loko's `Random` hands out generators (`rng()`); lakekit draws its identifiers and jitter from them. From eea4a7763d2a66465fece7ec67db8b303316f4f2 Mon Sep 17 00:00:00 2001 From: Oleksandr Litus Date: Wed, 21 Oct 2026 03:25:14 -0400 Subject: [PATCH 09/14] Core: Resources is a #[loko::bundle], following loko MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit loko declares bundles with #[loko::bundle] and #[loko::has(…)] fields instead of #[derive(Provides)]; loko is pinned at that revision. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 10 +++++----- Cargo.toml | 10 +++++----- crates/lakekit-core/src/resources.rs | 12 ++++++------ spec/0001-store-clock-random.md | 4 ++-- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a434ea2..2dc1abb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1743,7 +1743,7 @@ checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "loko-clock" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=112fb3e81114aef2c6a060eb1c00366cf4927f5b#112fb3e81114aef2c6a060eb1c00366cf4927f5b" +source = "git+https://github.com/polythings/loko?rev=4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc#4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc" dependencies = [ "loko-macros", "tokio", @@ -1752,7 +1752,7 @@ dependencies = [ [[package]] name = "loko-macros" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=112fb3e81114aef2c6a060eb1c00366cf4927f5b#112fb3e81114aef2c6a060eb1c00366cf4927f5b" +source = "git+https://github.com/polythings/loko?rev=4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc#4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc" dependencies = [ "proc-macro2", "quote", @@ -1762,7 +1762,7 @@ dependencies = [ [[package]] name = "loko-object-store" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=112fb3e81114aef2c6a060eb1c00366cf4927f5b#112fb3e81114aef2c6a060eb1c00366cf4927f5b" +source = "git+https://github.com/polythings/loko?rev=4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc#4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc" dependencies = [ "futures-util", "loko-store", @@ -1773,7 +1773,7 @@ dependencies = [ [[package]] name = "loko-random" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=112fb3e81114aef2c6a060eb1c00366cf4927f5b#112fb3e81114aef2c6a060eb1c00366cf4927f5b" +source = "git+https://github.com/polythings/loko?rev=4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc#4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc" dependencies = [ "getrandom 0.4.3", "loko-macros", @@ -1783,7 +1783,7 @@ dependencies = [ [[package]] name = "loko-store" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=112fb3e81114aef2c6a060eb1c00366cf4927f5b#112fb3e81114aef2c6a060eb1c00366cf4927f5b" +source = "git+https://github.com/polythings/loko?rev=4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc#4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc" dependencies = [ "loko-macros", "thiserror", diff --git a/Cargo.toml b/Cargo.toml index 19cd6a7..17f0773 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,11 +16,11 @@ lakekit-core = { path = "crates/lakekit-core", version = "0.1.0" } lakekit-iceberg = { path = "crates/lakekit-iceberg", version = "0.1.0", default-features = false } # loko provides the Store, Clock, and Random resources. From git until loko 0.2.0 is on crates.io # (polythings/loko#1); publishing uses the version. -loko-store = { git = "https://github.com/polythings/loko", rev = "112fb3e81114aef2c6a060eb1c00366cf4927f5b", version = "0.2.0" } -loko-clock = { git = "https://github.com/polythings/loko", rev = "112fb3e81114aef2c6a060eb1c00366cf4927f5b", version = "0.2.0" } -loko-random = { git = "https://github.com/polythings/loko", rev = "112fb3e81114aef2c6a060eb1c00366cf4927f5b", version = "0.2.0" } -loko-macros = { git = "https://github.com/polythings/loko", rev = "112fb3e81114aef2c6a060eb1c00366cf4927f5b", version = "0.2.0" } -loko-object-store = { git = "https://github.com/polythings/loko", rev = "112fb3e81114aef2c6a060eb1c00366cf4927f5b", version = "0.2.0" } +loko-store = { git = "https://github.com/polythings/loko", rev = "4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc", version = "0.2.0" } +loko-clock = { git = "https://github.com/polythings/loko", rev = "4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc", version = "0.2.0" } +loko-random = { git = "https://github.com/polythings/loko", rev = "4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc", version = "0.2.0" } +loko-macros = { git = "https://github.com/polythings/loko", rev = "4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc", version = "0.2.0" } +loko-object-store = { git = "https://github.com/polythings/loko", rev = "4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc", version = "0.2.0" } object_store = { version = "0.14", default-features = false } url = "2" thiserror = "2" diff --git a/crates/lakekit-core/src/resources.rs b/crates/lakekit-core/src/resources.rs index 43422e9..a719a8f 100644 --- a/crates/lakekit-core/src/resources.rs +++ b/crates/lakekit-core/src/resources.rs @@ -1,21 +1,21 @@ use loko_clock::{Clock, HasClock, TestClock}; -use loko_macros::Provides; use loko_random::{HasRandom, Random, SeededRandom}; use loko_store::{HasStore, MemoryStore, Store}; /// A store, a clock, and a random source as one value, for APIs that need all three (commits): /// `Resources::new(ObjectStoreAdapter::new(…), TokioClock, OsRandom)`. Any type that is all three -/// works as well: a JavaScript host, or an application's own `#[derive(loko::Provides)]` struct. -#[derive(Clone, Debug, Default, Provides)] +/// works as well: a JavaScript host, or an application's own `#[loko::bundle]` struct. +#[loko_macros::bundle] +#[derive(Clone, Debug, Default)] pub struct Resources { /// The store. - #[provides(Store)] + #[loko_macros::has(Store)] pub store: S, /// The clock. - #[provides(Clock)] + #[loko_macros::has(Clock)] pub clock: C, /// The random source. - #[provides(Random)] + #[loko_macros::has(Random)] pub random: R, } diff --git a/spec/0001-store-clock-random.md b/spec/0001-store-clock-random.md index 3ffca52..2ac51f8 100644 --- a/spec/0001-store-clock-random.md +++ b/spec/0001-store-clock-random.md @@ -33,7 +33,7 @@ browser, a Cloudflare Worker, a CLI, or a server, and tests run against memory. attempts. - **Who needs what:** readers need only a `Store`; commits need `Store + Clock + Random`. - **`Resources`:** a store, a clock, and a random source as one value (`Resources::new(store, clock, random)`), for APIs - that need all three, built with loko's `#[derive(Provides)]`. Any type that is all three (a JavaScript host, an + that need all three, built with loko's `#[loko::bundle]`. Any type that is all three (a JavaScript host, an application's own bundle) needs no wrapper. - **Native implementations:** `TokioClock` (feature `tokio`: the system clock and tokio's timer, native targets) and `OsRandom` (feature `getrandom`: the operating system's generator, native targets and WASI). @@ -71,5 +71,5 @@ None open. - 2026-10-21: the traits come from loko (D14); `Clock::sleep` takes a `Duration`; `MemoryStore` is only a store, and `MemoryResources` bundles it with a test clock and seeded randomness. - 2026-10-21: `Services` and `MemoryServices` are `Resources` and `MemoryResources`, as loko calls the things it - passes resources; `Resources` uses loko's `#[derive(Provides)]`. + passes resources; `Resources` uses loko's `#[loko::bundle]`. - 2026-10-21: loko's `Random` hands out generators (`rng()`); lakekit draws its identifiers and jitter from them. From c052853991e8abaaa90c2fb8be4a045fa761a92e Mon Sep 17 00:00:00 2001 From: Oleksandr Litus Date: Wed, 21 Oct 2026 04:18:03 -0400 Subject: [PATCH 10/14] Core: Resources implements loko's Has traits by hand, without loko-macros Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 25 ++++-------------------- Cargo.toml | 9 ++++----- crates/lakekit-core/Cargo.toml | 1 - crates/lakekit-core/src/resources.rs | 29 ++++++++++++++++++++++++---- spec/0001-store-clock-random.md | 4 ++-- 5 files changed, 35 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2dc1abb..52c0521 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1624,7 +1624,6 @@ name = "lakekit-core" version = "0.1.0" dependencies = [ "loko-clock", - "loko-macros", "loko-random", "loko-store", "thiserror", @@ -1743,26 +1742,15 @@ checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "loko-clock" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc#4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc" +source = "git+https://github.com/polythings/loko?rev=33bfbd045392b957f7310423d55a3813d7736673#33bfbd045392b957f7310423d55a3813d7736673" dependencies = [ - "loko-macros", "tokio", ] -[[package]] -name = "loko-macros" -version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc#4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "loko-object-store" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc#4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc" +source = "git+https://github.com/polythings/loko?rev=33bfbd045392b957f7310423d55a3813d7736673#33bfbd045392b957f7310423d55a3813d7736673" dependencies = [ "futures-util", "loko-store", @@ -1773,21 +1761,16 @@ dependencies = [ [[package]] name = "loko-random" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc#4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc" +source = "git+https://github.com/polythings/loko?rev=33bfbd045392b957f7310423d55a3813d7736673#33bfbd045392b957f7310423d55a3813d7736673" dependencies = [ "getrandom 0.4.3", - "loko-macros", "rand_core 0.10.1", ] [[package]] name = "loko-store" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc#4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc" -dependencies = [ - "loko-macros", - "thiserror", -] +source = "git+https://github.com/polythings/loko?rev=33bfbd045392b957f7310423d55a3813d7736673#33bfbd045392b957f7310423d55a3813d7736673" [[package]] name = "lz4_flex" diff --git a/Cargo.toml b/Cargo.toml index 17f0773..0fb26e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,11 +16,10 @@ lakekit-core = { path = "crates/lakekit-core", version = "0.1.0" } lakekit-iceberg = { path = "crates/lakekit-iceberg", version = "0.1.0", default-features = false } # loko provides the Store, Clock, and Random resources. From git until loko 0.2.0 is on crates.io # (polythings/loko#1); publishing uses the version. -loko-store = { git = "https://github.com/polythings/loko", rev = "4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc", version = "0.2.0" } -loko-clock = { git = "https://github.com/polythings/loko", rev = "4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc", version = "0.2.0" } -loko-random = { git = "https://github.com/polythings/loko", rev = "4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc", version = "0.2.0" } -loko-macros = { git = "https://github.com/polythings/loko", rev = "4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc", version = "0.2.0" } -loko-object-store = { git = "https://github.com/polythings/loko", rev = "4f258adbf5c44c6c6d499ea9440ba1dda84e2ddc", version = "0.2.0" } +loko-store = { git = "https://github.com/polythings/loko", rev = "33bfbd045392b957f7310423d55a3813d7736673", version = "0.2.0" } +loko-clock = { git = "https://github.com/polythings/loko", rev = "33bfbd045392b957f7310423d55a3813d7736673", version = "0.2.0" } +loko-random = { git = "https://github.com/polythings/loko", rev = "33bfbd045392b957f7310423d55a3813d7736673", version = "0.2.0" } +loko-object-store = { git = "https://github.com/polythings/loko", rev = "33bfbd045392b957f7310423d55a3813d7736673", version = "0.2.0" } object_store = { version = "0.14", default-features = false } url = "2" thiserror = "2" diff --git a/crates/lakekit-core/Cargo.toml b/crates/lakekit-core/Cargo.toml index ec105e6..6c3ee67 100644 --- a/crates/lakekit-core/Cargo.toml +++ b/crates/lakekit-core/Cargo.toml @@ -21,7 +21,6 @@ getrandom = ["loko-random/getrandom"] loko-store.workspace = true loko-clock.workspace = true loko-random.workspace = true -loko-macros.workspace = true thiserror.workspace = true uuid = { version = "1", default-features = false } diff --git a/crates/lakekit-core/src/resources.rs b/crates/lakekit-core/src/resources.rs index a719a8f..751f4c7 100644 --- a/crates/lakekit-core/src/resources.rs +++ b/crates/lakekit-core/src/resources.rs @@ -5,20 +5,41 @@ use loko_store::{HasStore, MemoryStore, Store}; /// A store, a clock, and a random source as one value, for APIs that need all three (commits): /// `Resources::new(ObjectStoreAdapter::new(…), TokioClock, OsRandom)`. Any type that is all three /// works as well: a JavaScript host, or an application's own `#[loko::bundle]` struct. -#[loko_macros::bundle] #[derive(Clone, Debug, Default)] pub struct Resources { /// The store. - #[loko_macros::has(Store)] pub store: S, /// The clock. - #[loko_macros::has(Clock)] pub clock: C, /// The random source. - #[loko_macros::has(Random)] pub random: R, } +// The three impls `#[loko::bundle]` would write, by hand, so lakekit needs no macros. +impl HasStore for Resources { + type Store = S; + + fn store(&self) -> &S { + &self.store + } +} + +impl HasClock for Resources { + type Clock = C; + + fn clock(&self) -> &C { + &self.clock + } +} + +impl HasRandom for Resources { + type Random = R; + + fn random(&self) -> &R { + &self.random + } +} + impl Resources { /// `store` with `clock` and `random`. pub fn new(store: S, clock: C, random: R) -> Self { diff --git a/spec/0001-store-clock-random.md b/spec/0001-store-clock-random.md index 2ac51f8..23cc4da 100644 --- a/spec/0001-store-clock-random.md +++ b/spec/0001-store-clock-random.md @@ -33,7 +33,7 @@ browser, a Cloudflare Worker, a CLI, or a server, and tests run against memory. attempts. - **Who needs what:** readers need only a `Store`; commits need `Store + Clock + Random`. - **`Resources`:** a store, a clock, and a random source as one value (`Resources::new(store, clock, random)`), for APIs - that need all three, built with loko's `#[loko::bundle]`. Any type that is all three (a JavaScript host, an + that need all three: it implements loko's `HasStore`, `HasClock`, and `HasRandom`, so it is all three. Any type that is all three (a JavaScript host, an application's own bundle) needs no wrapper. - **Native implementations:** `TokioClock` (feature `tokio`: the system clock and tokio's timer, native targets) and `OsRandom` (feature `getrandom`: the operating system's generator, native targets and WASI). @@ -71,5 +71,5 @@ None open. - 2026-10-21: the traits come from loko (D14); `Clock::sleep` takes a `Duration`; `MemoryStore` is only a store, and `MemoryResources` bundles it with a test clock and seeded randomness. - 2026-10-21: `Services` and `MemoryServices` are `Resources` and `MemoryResources`, as loko calls the things it - passes resources; `Resources` uses loko's `#[loko::bundle]`. + passes resources; `Resources` implements loko's `Has` traits. - 2026-10-21: loko's `Random` hands out generators (`rng()`); lakekit draws its identifiers and jitter from them. From 9883f684dd4448431366c9f192c21f797592ab50 Mon Sep 17 00:00:00 2001 From: Oleksandr Litus Date: Wed, 21 Oct 2026 04:25:55 -0400 Subject: [PATCH 11/14] Core: Services and MemoryServices again, as loko calls them services again Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 ++-- CONTRIBUTING.md | 2 +- Cargo.lock | 8 ++-- Cargo.toml | 10 ++--- PRIOR_ART.md | 2 +- README.md | 6 +-- crates/lakekit-core/README.md | 6 +-- crates/lakekit-core/src/ids.rs | 4 +- crates/lakekit-core/src/lib.rs | 10 ++--- .../src/{resources.rs => services.rs} | 16 ++++---- crates/lakekit-iceberg/src/commit.rs | 41 ++++++++----------- crates/lakekit-iceberg/tests/iceberg_rust.rs | 6 +-- crates/lakekit-iceberg/tests/object_store.rs | 4 +- spec/0001-store-clock-random.md | 18 ++++---- spec/0004-store-adapters.md | 2 +- spec/milestones/v0-1.md | 2 +- 16 files changed, 71 insertions(+), 74 deletions(-) rename crates/lakekit-core/src/{resources.rs => services.rs} (74%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 898031f..da915fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,9 @@ versions follow [Semantic Versioning](https://semver.org/). ### Added -- `lakekit-core`: loko's `Store`, `Clock`, and `Random` resources with their test and native implementations - (`TokioClock` with feature `tokio`, `OsRandom` with feature `getrandom`); `Resources` to pass the three as one; - `MemoryResources` for tests; one `Error` type, which a store's error converts into. +- `lakekit-core`: loko's `Store`, `Clock`, and `Random` services with their test and native implementations + (`TokioClock` with feature `tokio`, `OsRandom` with feature `getrandom`); `Services` to pass the three as one; + `MemoryServices` for tests; one `Error` type, which a store's error converts into. - `lakekit` feature `object-store`: a `Store` over an `object_store` client (S3, GCS, Azure, local files, memory), from a client or a URL (`loko-object-store`). - `lakekit-iceberg`: Iceberg table metadata, schemas, partition specs, manifest lists, and manifests read for format @@ -28,7 +28,7 @@ versions follow [Semantic Versioning](https://semver.org/). (`lakebed-core`, …) until 2026-10-21. Nothing was published under the old names; GitHub redirects the old repository URL. - The I/O boundary is three traits, `Store` (objects), `Clock` (the time and a timer), and `Random` (random numbers), - instead of one `Host`; `MemoryHost` is `MemoryResources`. UUIDs come from the `uuid` crate. + instead of one `Host`; `MemoryHost` is `MemoryServices`. UUIDs come from the `uuid` crate. - The three traits are [loko](https://github.com/polythings/loko)'s, and `Clock::sleep` takes a `Duration`. The `object_store` adapter is `loko-object-store`, re-exported as `lakekit::object_store`; `lakekit-object-store` is gone (it was never published). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a248db6..e868f58 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,7 +50,7 @@ ideas we took. ## Tests - **A bug fix starts with a test that fails**, then the fix. -- **Tests run offline**, against `MemoryResources` or a store over the reference implementation's in-memory storage. +- **Tests run offline**, against `MemoryServices` or a store over the reference implementation's in-memory storage. - **Interop:** writers are tested against the format's reference reader, and readers against the reference writer. ## Checks diff --git a/Cargo.lock b/Cargo.lock index 52c0521..3a9180e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1742,7 +1742,7 @@ checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "loko-clock" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=33bfbd045392b957f7310423d55a3813d7736673#33bfbd045392b957f7310423d55a3813d7736673" +source = "git+https://github.com/polythings/loko?rev=6c79750109b7929e927a582e92570105da625714#6c79750109b7929e927a582e92570105da625714" dependencies = [ "tokio", ] @@ -1750,7 +1750,7 @@ dependencies = [ [[package]] name = "loko-object-store" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=33bfbd045392b957f7310423d55a3813d7736673#33bfbd045392b957f7310423d55a3813d7736673" +source = "git+https://github.com/polythings/loko?rev=6c79750109b7929e927a582e92570105da625714#6c79750109b7929e927a582e92570105da625714" dependencies = [ "futures-util", "loko-store", @@ -1761,7 +1761,7 @@ dependencies = [ [[package]] name = "loko-random" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=33bfbd045392b957f7310423d55a3813d7736673#33bfbd045392b957f7310423d55a3813d7736673" +source = "git+https://github.com/polythings/loko?rev=6c79750109b7929e927a582e92570105da625714#6c79750109b7929e927a582e92570105da625714" dependencies = [ "getrandom 0.4.3", "rand_core 0.10.1", @@ -1770,7 +1770,7 @@ dependencies = [ [[package]] name = "loko-store" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=33bfbd045392b957f7310423d55a3813d7736673#33bfbd045392b957f7310423d55a3813d7736673" +source = "git+https://github.com/polythings/loko?rev=6c79750109b7929e927a582e92570105da625714#6c79750109b7929e927a582e92570105da625714" [[package]] name = "lz4_flex" diff --git a/Cargo.toml b/Cargo.toml index 0fb26e3..abe688d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,12 +14,12 @@ categories = ["database-implementations", "wasm"] [workspace.dependencies] lakekit-core = { path = "crates/lakekit-core", version = "0.1.0" } lakekit-iceberg = { path = "crates/lakekit-iceberg", version = "0.1.0", default-features = false } -# loko provides the Store, Clock, and Random resources. From git until loko 0.2.0 is on crates.io +# loko provides the Store, Clock, and Random services. From git until loko 0.2.0 is on crates.io # (polythings/loko#1); publishing uses the version. -loko-store = { git = "https://github.com/polythings/loko", rev = "33bfbd045392b957f7310423d55a3813d7736673", version = "0.2.0" } -loko-clock = { git = "https://github.com/polythings/loko", rev = "33bfbd045392b957f7310423d55a3813d7736673", version = "0.2.0" } -loko-random = { git = "https://github.com/polythings/loko", rev = "33bfbd045392b957f7310423d55a3813d7736673", version = "0.2.0" } -loko-object-store = { git = "https://github.com/polythings/loko", rev = "33bfbd045392b957f7310423d55a3813d7736673", version = "0.2.0" } +loko-store = { git = "https://github.com/polythings/loko", rev = "6c79750109b7929e927a582e92570105da625714", version = "0.2.0" } +loko-clock = { git = "https://github.com/polythings/loko", rev = "6c79750109b7929e927a582e92570105da625714", version = "0.2.0" } +loko-random = { git = "https://github.com/polythings/loko", rev = "6c79750109b7929e927a582e92570105da625714", version = "0.2.0" } +loko-object-store = { git = "https://github.com/polythings/loko", rev = "6c79750109b7929e927a582e92570105da625714", version = "0.2.0" } object_store = { version = "0.14", default-features = false } url = "2" thiserror = "2" diff --git a/PRIOR_ART.md b/PRIOR_ART.md index b8effec..741bdad 100644 --- a/PRIOR_ART.md +++ b/PRIOR_ART.md @@ -11,7 +11,7 @@ never code: every line of lakekit is our own. How we credit is in [CONTRIBUTING. | [iceberg-rust](https://github.com/apache/iceberg-rust) | Apache-2.0 | The reference the interop tests check against (a test dependency only) | | [Apache Avro](https://avro.apache.org) (`apache-avro`) | Apache-2.0 | Reading and writing manifests and manifest lists | | [arrow-rs](https://github.com/apache/arrow-rs) (`arrow-schema`, `parquet`) | Apache-2.0 | Arrow schemas; Parquet footers for statistics | -| [loko](https://github.com/polythings/loko) | Apache-2.0 | The `Store`, `Clock`, and `Random` resources, and the `object_store` adapter | +| [loko](https://github.com/polythings/loko) | Apache-2.0 | The `Store`, `Clock`, and `Random` services, and the `object_store` adapter | | [object_store](https://github.com/apache/arrow-rs-object-store) | Apache-2.0 / MIT | The stores `loko-object-store` adapts (re-exported by `lakekit`) | | [uuid](https://github.com/uuid-rs/uuid) | Apache-2.0 / MIT | UUIDs for file names, from a `Random` and a `Clock` | diff --git a/README.md b/README.md index 8a6b3ee..e884e6b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Table formats read and written without I/O of their own, in Rust, for WebAssembly and native targets. Each format's metadata (Apache Iceberg first) is parsed and written by plain code that asks a `Store` for bytes, a `Clock` for the -time, and a `Random` for random numbers ([loko](https://github.com/polythings/loko)'s resources): the application +time, and a `Random` for random numbers ([loko](https://github.com/polythings/loko)'s services): the application supplies them, so the same crate runs in a browser, a Cloudflare Worker, a CLI, or a server. @@ -14,7 +14,7 @@ share. The official Rust crates for these formats build on tokio and HTTP client | Crate | What it does | |---|---| -| [`lakekit-core`](crates/lakekit-core) | loko's `Store`, `Clock`, and `Random` with their test and native implementations, `Resources`, and one error type | +| [`lakekit-core`](crates/lakekit-core) | loko's `Store`, `Clock`, and `Random` with their test and native implementations, `Services`, and one error type | | [`lakekit-iceberg`](crates/lakekit-iceberg) | Apache Iceberg: metadata, manifests, and commits | | [`lakekit`](crates/lakekit) | Re-exports the crates, one feature each, and `loko-object-store` (a `Store` over an `object_store` client) | @@ -46,7 +46,7 @@ async fn files(s3: Arc) -> lakekit_core::Result { +pub struct Services { /// The store. pub store: S, /// The clock. @@ -16,7 +16,7 @@ pub struct Resources { } // The three impls `#[loko::bundle]` would write, by hand, so lakekit needs no macros. -impl HasStore for Resources { +impl HasStore for Services { type Store = S; fn store(&self) -> &S { @@ -24,7 +24,7 @@ impl HasStore for Resources { } } -impl HasClock for Resources { +impl HasClock for Services { type Clock = C; fn clock(&self) -> &C { @@ -32,7 +32,7 @@ impl HasClock for Resources { } } -impl HasRandom for Resources { +impl HasRandom for Services { type Random = R; fn random(&self) -> &R { @@ -40,7 +40,7 @@ impl HasRandom for Resources { } } -impl Resources { +impl Services { /// `store` with `clock` and `random`. pub fn new(store: S, clock: C, random: R) -> Self { Self { store, clock, random } @@ -48,9 +48,9 @@ impl Resources { } /// Everything in memory, for tests: a [`MemoryStore`], a [`TestClock`], and a [`SeededRandom`]. -pub type MemoryResources = Resources; +pub type MemoryServices = Services; -impl MemoryResources { +impl MemoryServices { /// An empty store, the clock at `now_ms` milliseconds after the Unix epoch, and seed 0. pub fn at_ms(now_ms: i64) -> Self { Self::new(MemoryStore::new(), TestClock::at_ms(now_ms), SeededRandom::new(0)) diff --git a/crates/lakekit-iceberg/src/commit.rs b/crates/lakekit-iceberg/src/commit.rs index ea65342..d0a2cdf 100644 --- a/crates/lakekit-iceberg/src/commit.rs +++ b/crates/lakekit-iceberg/src/commit.rs @@ -472,7 +472,7 @@ pub fn counters(change: &Change, base: &HashMap Table { + fn create(store: &MemoryServices, format_version: u8) -> Table { let metadata = json!({ "format-version": format_version, "table-uuid": "u", "location": LOCATION, "last-sequence-number": 0, "last-updated-ms": 0, "last-column-id": 2, "current-schema-id": 0, @@ -512,8 +512,8 @@ mod tests { Change::Append { files } } - fn try_run>( - store: &MemoryResources, + fn try_run>( + store: &MemoryServices, base: Table, change: &Change, hooks: &mut K, @@ -521,8 +521,8 @@ mod tests { block_on(commit(store, base, change, &HashMap::new(), &CommitOptions::default(), hooks)) } - fn run>( - store: &MemoryResources, + fn run>( + store: &MemoryServices, base: Table, change: &Change, hooks: &mut K, @@ -536,7 +536,7 @@ mod tests { /// The live files of the newest version, read back from the store: (name, data sequence /// number, file sequence number). - fn live(store: &MemoryResources) -> Vec<(String, i64, i64)> { + fn live(store: &MemoryServices) -> Vec<(String, i64, i64)> { let mut t = block_on(Table::load(store, LOCATION)).unwrap(); let mut files: Vec<_> = block_on(t.live_files(store)) .unwrap() @@ -555,7 +555,7 @@ mod tests { #[test] fn append_replace_and_delete() { - let store = MemoryResources::at_ms(1_000); + let store = MemoryServices::at_ms(1_000); let t = create(&store, 2); let (t, info) = run(&store, t, &append(vec![file("a", 1, 10), file("b", 2, 20)]), &mut NoHooks); assert_eq!((info.version, info.attempts, info.skipped), (2, 1, false)); @@ -582,7 +582,7 @@ mod tests { #[test] fn a_lost_race_retries_on_the_newer_version() { - let store = MemoryResources::at_ms(1_000); + let store = MemoryServices::at_ms(1_000); let stale = create(&store, 2); let (_, first) = run(&store, stale.clone(), &append(vec![file("a", 1, 10)]), &mut NoHooks); let (_, second) = run(&store, stale, &append(vec![file("b", 1, 10)]), &mut NoHooks); @@ -595,7 +595,7 @@ mod tests { #[test] fn removing_files_that_are_gone_is_a_conflict() { - let store = MemoryResources::at_ms(1_000); + let store = MemoryServices::at_ms(1_000); let (t, _) = run(&store, create(&store, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); let removed = t.files.clone().unwrap_or_default(); let (t, _) = run(&store, t, &Change::Delete { removed: removed.clone() }, &mut NoHooks); @@ -606,19 +606,14 @@ mod tests { /// Hooks that skip a batch already committed, and add a file and summary properties. struct Batch(&'static str); - impl CommitHooks for Batch { + impl CommitHooks for Batch { type Error = Error; fn skip(&mut self, base: &Table) -> bool { base.metadata.ancestors().iter().any(|s| s.summary.get("batch").map(String::as_str) == Some(self.0)) } - fn extra( - &mut self, - _: &MemoryResources, - base: &Table, - snapshot_id: i64, - ) -> impl Future> { + fn extra(&mut self, _: &MemoryServices, base: &Table, snapshot_id: i64) -> impl Future> { let uri = format!("{}/metadata/extra-{snapshot_id}", base.location); let summary = HashMap::from([("batch".to_string(), self.0.to_string()), ("extra".to_string(), uri.clone())]); @@ -628,7 +623,7 @@ mod tests { #[test] fn hooks_skip_and_extend_commits() { - let store = MemoryResources::at_ms(1_000); + let store = MemoryServices::at_ms(1_000); let (t, info) = run(&store, create(&store, 2), &append(vec![file("a", 1, 10)]), &mut Batch("b1")); let extra = t.metadata.summary()["extra"].clone(); assert!(block_on(store.exists(&extra)).unwrap()); @@ -640,7 +635,7 @@ mod tests { #[test] fn many_manifests_merge() { - let store = MemoryResources::at_ms(1_000); + let store = MemoryServices::at_ms(1_000); let mut t = create(&store, 2); for i in 0..9 { t = run(&store, t, &append(vec![file(&format!("f{i}"), i, 1)]), &mut NoHooks).0; @@ -651,9 +646,9 @@ mod tests { #[test] fn a_clock_behind_the_table_keeps_the_logs_in_order() { - let store = MemoryResources::at_ms(5_000); + let store = MemoryServices::at_ms(5_000); let (t, _) = run(&store, create(&store, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); - let late = MemoryResources::at_ms(1_000); + let late = MemoryServices::at_ms(1_000); for uri in store.store.uris() { block_on(late.put(&uri, block_on(store.get(&uri)).unwrap().unwrap())).unwrap(); } @@ -663,7 +658,7 @@ mod tests { #[test] fn a_replace_needs_a_sequence_number_per_added_file() { - let store = MemoryResources::at_ms(1_000); + let store = MemoryServices::at_ms(1_000); let (t, _) = run(&store, create(&store, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); let removed = files_named(&t, "a.parquet"); let change = @@ -674,7 +669,7 @@ mod tests { #[test] fn only_v2_tables_are_written() { - let store = MemoryResources::at_ms(1_000); + let store = MemoryServices::at_ms(1_000); let err = try_run(&store, create(&store, 1), &append(vec![]), &mut NoHooks); assert!(matches!(err, Err(Error::Unsupported(_)))); } diff --git a/crates/lakekit-iceberg/tests/iceberg_rust.rs b/crates/lakekit-iceberg/tests/iceberg_rust.rs index a077295..ad54de1 100644 --- a/crates/lakekit-iceberg/tests/iceberg_rust.rs +++ b/crates/lakekit-iceberg/tests/iceberg_rust.rs @@ -21,7 +21,7 @@ use iceberg::spec::{ use iceberg::transaction::{ApplyTransactionAction, Transaction}; use iceberg::writer::file_writer::{FileWriter as _, FileWriterBuilder, ParquetWriterBuilder}; use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, NamespaceIdent, TableCreation, TableIdent}; -use lakekit_core::{Listed, MemoryUpload, Resources, SeededRandom, Store, StoreError as Error, TestClock}; +use lakekit_core::{Listed, MemoryUpload, SeededRandom, Services, Store, StoreError as Error, TestClock}; use lakekit_iceberg::stats::ParquetStats; use lakekit_iceberg::{Change, CommitOptions, DataFile, Datum, Entry, NoHooks, Status, Table, commit}; use parquet::arrow::ArrowWriter; @@ -39,7 +39,7 @@ struct IoStore { } /// The store with a test clock and seeded random numbers. -type TestStore = Resources; +type TestStore = Services; /// The store's results. type Result = std::result::Result; @@ -231,7 +231,7 @@ async fn tables_round_trip_between_lakekit_and_iceberg_rust() { Some(UnboundPartitionSpec::builder().add_partition_field(1, "time_day", IceTransform::Day).unwrap().build()); let table = catalog.create_table(&ns, creation).await.unwrap(); let io = table.file_io().clone(); - let store = Resources::new(IoStore { io: io.clone() }, TestClock::at_ms(T0 / 1000), SeededRandom::new(0)); + let store = Services::new(IoStore { io: io.clone() }, TestClock::at_ms(T0 / 1000), SeededRandom::new(0)); let arrow = Arc::new(iceberg::arrow::schema_to_arrow_schema(table.metadata().current_schema()).unwrap()); // iceberg-rust appends; lakekit reads the file as iceberg-rust records it. diff --git a/crates/lakekit-iceberg/tests/object_store.rs b/crates/lakekit-iceberg/tests/object_store.rs index b29558f..0bb4efb 100644 --- a/crates/lakekit-iceberg/tests/object_store.rs +++ b/crates/lakekit-iceberg/tests/object_store.rs @@ -5,14 +5,14 @@ use std::collections::HashMap; use std::sync::Arc; -use lakekit_core::{OsRandom, Resources, Store, TokioClock}; +use lakekit_core::{OsRandom, Services, Store, TokioClock}; use lakekit_iceberg::{Change, CommitOptions, DataFile, Datum, Metadata, NoHooks, Table, commit}; use loko_object_store::ObjectStoreAdapter; use object_store::memory::InMemory; #[tokio::test] async fn iceberg_commits_through_object_store() { - let store = Resources::new(ObjectStoreAdapter::new(Arc::new(InMemory::new()), "memory://"), TokioClock, OsRandom); + let store = Services::new(ObjectStoreAdapter::new(Arc::new(InMemory::new()), "memory://"), TokioClock, OsRandom); let location = "memory://warehouse/t"; let metadata = Metadata::parse( br#"{"format-version": 2, "table-uuid": "u", "location": "memory://warehouse/t", "last-sequence-number": 0, diff --git a/spec/0001-store-clock-random.md b/spec/0001-store-clock-random.md index 23cc4da..fd5ce91 100644 --- a/spec/0001-store-clock-random.md +++ b/spec/0001-store-clock-random.md @@ -24,22 +24,22 @@ browser, a Cloudflare Worker, a CLI, or a server, and tests run against memory. `false` when an object is there), `upload` (a multipart upload: parts in order, then complete or abort), `list` (by string prefix, not only by directory), and `delete` (missing objects are not an error). Paths are absolute URIs, as table metadata records them. Adapters implement it over store libraries ([spec 4](0004-store-adapters.md)). -- **Where they come from:** the three are [loko](https://github.com/polythings/loko)'s resources (`loko-store`, +- **Where they come from:** the three are [loko](https://github.com/polythings/loko)'s services (`loko-store`, `loko-clock`, `loko-random`), shared by the polythings projects; `lakekit-core` re-exports them with their test and - native implementations, and adds what is lakekit's: the error type, the identifiers, and `Resources`. + native implementations, and adds what is lakekit's: the error type, the identifiers, and `Services`. - **`Clock`:** `now` and `now_ms`, and `sleep(Duration)`: commit timestamps, time-ordered file names, and the wait between commit attempts. - **`Random`:** `rng()`, a generator of the caller's own: snapshot ids, file names, and the jitter between commit attempts. - **Who needs what:** readers need only a `Store`; commits need `Store + Clock + Random`. -- **`Resources`:** a store, a clock, and a random source as one value (`Resources::new(store, clock, random)`), for APIs +- **`Services`:** a store, a clock, and a random source as one value (`Services::new(store, clock, random)`), for APIs that need all three: it implements loko's `HasStore`, `HasClock`, and `HasRandom`, so it is all three. Any type that is all three (a JavaScript host, an application's own bundle) needs no wrapper. - **Native implementations:** `TokioClock` (feature `tokio`: the system clock and tokio's timer, native targets) and `OsRandom` (feature `getrandom`: the operating system's generator, native targets and WASI). - **Futures aren't `Send`.** Methods return `impl Future`, without a `Send` bound, so a store or clock can wrap JavaScript promises (wasm-bindgen). Native ones are free to be `Send`. -- **For tests:** `MemoryResources` is a `MemoryStore`, a `TestClock` that only moves when told (or when `sleep` is +- **For tests:** `MemoryServices` is a `MemoryStore`, a `TestClock` that only moves when told (or when `sleep` is called), and a `SeededRandom`, so tests are deterministic. - **`Error`:** one error type for all crates, with the variant saying what a caller can do: `Invalid`, `Unsupported`, `Conflict` (retry on the newer version), `CommitUnknown` (the commit may have landed: delete nothing it wrote, load @@ -57,7 +57,7 @@ browser, a Cloudflare Worker, a CLI, or a server, and tests run against memory. | D3 | One error type across crates, whose variants say what a caller can do | Accepted | | D10 | Objects are a `Store` of their own, apart from the time and randomness: readers need only a store, and store adapters don't decide how time is told | Accepted | | D13 | The time and randomness are two traits, `Clock` and `Random`, as ZIO's services are, rather than one environment trait: each says what it is, and each can be swapped on its own | Accepted | -| D14 | `Store`, `Clock`, and `Random` are loko's resources, which every polythings project shares, rather than lakekit's own; `lakekit-core` re-exports them and keeps the error type | Accepted | +| D14 | `Store`, `Clock`, and `Random` are loko's services, which every polythings project shares, rather than lakekit's own; `lakekit-core` re-exports them and keeps the error type | Accepted | ## Open questions and issues @@ -66,10 +66,12 @@ None open. ## History - 2026-10-20: built, from Timelake's `timelake_format::host`. -- 2026-10-21: one `Host` trait split into `Store` (D10), `Clock`, and `Random` (D13); `Resources`, `TokioClock`, and +- 2026-10-21: one `Host` trait split into `Store` (D10), `Clock`, and `Random` (D13); `Services`, `TokioClock`, and `OsRandom` added; UUIDs from the `uuid` crate. The spec was "The host boundary". - 2026-10-21: the traits come from loko (D14); `Clock::sleep` takes a `Duration`; `MemoryStore` is only a store, and - `MemoryResources` bundles it with a test clock and seeded randomness. + `MemoryServices` bundles it with a test clock and seeded randomness. - 2026-10-21: `Services` and `MemoryServices` are `Resources` and `MemoryResources`, as loko calls the things it - passes resources; `Resources` implements loko's `Has` traits. + passes services; `Services` implements loko's `Has` traits. - 2026-10-21: loko's `Random` hands out generators (`rng()`); lakekit draws its identifiers and jitter from them. +- 2026-10-21: `Resources` and `MemoryResources` are `Services` and `MemoryServices` again, as loko calls them services + again. diff --git a/spec/0004-store-adapters.md b/spec/0004-store-adapters.md index 256e7f0..8b2c8df 100644 --- a/spec/0004-store-adapters.md +++ b/spec/0004-store-adapters.md @@ -16,7 +16,7 @@ issues: [] Ready-made `Store`s ([spec 1](0001-store-clock-random.md)) over the store libraries applications already use, so a native application doesn't write its own. They are [loko](https://github.com/polythings/loko)'s crates, since `Store` -is loko's resource; `lakekit` re-exports `loko-object-store` (feature `object-store`). Each adapter is a crate of its +is loko's service; `lakekit` re-exports `loko-object-store` (feature `object-store`). Each adapter is a crate of its own; the `Store` trait stays the boundary, because some stores aren't these libraries (a Cloudflare R2 binding reached from JavaScript) and applications wrap their store (lakeglass counts every request). diff --git a/spec/milestones/v0-1.md b/spec/milestones/v0-1.md index 8541d52..e3e40ba 100644 --- a/spec/milestones/v0-1.md +++ b/spec/milestones/v0-1.md @@ -2,7 +2,7 @@ **In progress** -The first release: the Iceberg crate on loko's store, clock, and random resources, and the `object_store` adapter, enough for +The first release: the Iceberg crate on loko's store, clock, and random services, and the `object_store` adapter, enough for Timelake's WASM reader and writer to use lakekit instead of their own Iceberg code, and for lakeglass to read Iceberg tables in the browser. From ee74e3c01a552c22a1c0b93267d03d28706dec85 Mon Sep 17 00:00:00 2001 From: Oleksandr Litus Date: Wed, 21 Oct 2026 10:41:04 -0400 Subject: [PATCH 12/14] Core: Services is three borrowed services, not one value that is all three Commits took one value bounded by Store + Clock + Random, so callers had to glue a store, a clock, and a random source into a struct that implemented all three. Services is now a plain struct of three references with named fields, built from whatever the caller has: Services::new(&s3, &TokioClock, &OsRandom), or Services::new(&host, &host, &host) for one object that is all three. Commit code says which service each call uses (services.clock.sleep, services.random.rng(), services.store.put), and CommitHooks is generic over the three and receives Services too. MemoryServices is gone (the tests keep their own), lakekit-core no longer re-exports loko's Has traits, and uuid_v7 takes a clock and a random source. Spec 1 records D16. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +- CONTRIBUTING.md | 2 +- README.md | 2 +- crates/lakekit-core/README.md | 4 +- crates/lakekit-core/src/ids.rs | 22 +-- crates/lakekit-core/src/lib.rs | 11 +- crates/lakekit-core/src/services.rs | 71 +++------ crates/lakekit-iceberg/src/commit.rs | 145 +++++++++++-------- crates/lakekit-iceberg/tests/iceberg_rust.rs | 12 +- crates/lakekit-iceberg/tests/object_store.rs | 14 +- spec/0001-store-clock-random.md | 15 +- spec/README.md | 2 +- 12 files changed, 158 insertions(+), 146 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da915fd..fbec074 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,8 @@ versions follow [Semantic Versioning](https://semver.org/). ### Added - `lakekit-core`: loko's `Store`, `Clock`, and `Random` services with their test and native implementations - (`TokioClock` with feature `tokio`, `OsRandom` with feature `getrandom`); `Services` to pass the three as one; - `MemoryServices` for tests; one `Error` type, which a store's error converts into. + (`TokioClock` with feature `tokio`, `OsRandom` with feature `getrandom`); `Services`, the three borrowed + together for commits; one `Error` type, which a store's error converts into. - `lakekit` feature `object-store`: a `Store` over an `object_store` client (S3, GCS, Azure, local files, memory), from a client or a URL (`loko-object-store`). - `lakekit-iceberg`: Iceberg table metadata, schemas, partition specs, manifest lists, and manifests read for format diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e868f58..79f6e62 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,7 +50,7 @@ ideas we took. ## Tests - **A bug fix starts with a test that fails**, then the fix. -- **Tests run offline**, against `MemoryServices` or a store over the reference implementation's in-memory storage. +- **Tests run offline**, against a `MemoryStore` with a `TestClock` and a `SeededRandom`, or a store over the reference implementation's in-memory storage. - **Interop:** writers are tested against the format's reference reader, and readers against the reference writer. ## Checks diff --git a/README.md b/README.md index e884e6b..f76a851 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ async fn files(s3: Arc) -> lakekit_core::Result String { Builder::from_random_bytes(bytes).into_uuid().to_string() } -/// A time-ordered UUID (version 7) from `source`'s clock and random numbers: names made with it -/// sort by creation time. -pub fn uuid_v7(source: &(impl Clock + Random)) -> String { - let millis = u64::try_from(source.now_ms()).unwrap_or(0); +/// A time-ordered UUID (version 7) from `clock` and `random`: names made with it sort by creation +/// time. +pub fn uuid_v7(clock: &impl Clock, random: &impl Random) -> String { + let millis = u64::try_from(clock.now_ms()).unwrap_or(0); let mut bytes = [0u8; 10]; - source.rng().fill_bytes(&mut bytes); + random.rng().fill_bytes(&mut bytes); Builder::from_unix_timestamp_millis(millis, &bytes).into_uuid().to_string() } @@ -23,20 +23,20 @@ mod tests { use std::time::Duration; use super::*; - use crate::MemoryServices; + use crate::{SeededRandom, TestClock}; #[test] fn uuids_are_well_formed() { - let source = MemoryServices::at_ms(1_767_225_600_000); - let v4 = uuid_v4(&source); - let v7 = uuid_v7(&source); + let (clock, random) = (TestClock::at_ms(1_767_225_600_000), SeededRandom::new(0)); + let v4 = uuid_v4(&random); + let v7 = uuid_v7(&clock, &random); for (u, version) in [(&v4, '4'), (&v7, '7')] { assert_eq!(u.len(), 36); assert_eq!(u.chars().nth(14), Some(version)); assert!(matches!(u.chars().nth(19), Some('8' | '9' | 'a' | 'b'))); } // Version 7 starts with the time, so later ones sort later. - source.clock.advance(Duration::from_millis(1)); - assert!(uuid_v7(&source) > v7); + clock.advance(Duration::from_millis(1)); + assert!(uuid_v7(&clock, &random) > v7); } } diff --git a/crates/lakekit-core/src/lib.rs b/crates/lakekit-core/src/lib.rs index 5f76651..9b5b22c 100644 --- a/crates/lakekit-core/src/lib.rs +++ b/crates/lakekit-core/src/lib.rs @@ -8,8 +8,7 @@ //! delete. `loko-object-store` implements it over `object_store`. //! - [`Clock`]: the time and a timer. `TokioClock` (feature `tokio`) is a native one. //! - [`Random`]: random numbers. `OsRandom` (feature `getrandom`) is a native one. -//! - [`Services`]: a store, a clock, and a random source as one value, for APIs that need all three; -//! [`MemoryServices`] is all three in memory, for tests. +//! - [`Services`]: a store, a clock, and a random source, borrowed, for APIs that need all three. //! - [`Error`]: one error type for the format crates, converted from the store's. //! //! ``` @@ -37,9 +36,9 @@ pub use error::{Error, Result}; pub use ids::{uuid_v4, uuid_v7}; #[cfg(all(feature = "tokio", not(target_arch = "wasm32")))] pub use loko_clock::TokioClock; -pub use loko_clock::{Clock, HasClock, TestClock}; +pub use loko_clock::{Clock, TestClock}; #[cfg(all(feature = "getrandom", not(all(target_arch = "wasm32", target_os = "unknown"))))] pub use loko_random::OsRandom; -pub use loko_random::{CryptoRng, HasRandom, Random, Rng, SeededRandom, SeededRng}; -pub use loko_store::{Error as StoreError, HasStore, Listed, MemoryStore, MemoryUpload, Store, Upload}; -pub use services::{MemoryServices, Services}; +pub use loko_random::{CryptoRng, Random, Rng, SeededRandom, SeededRng}; +pub use loko_store::{Error as StoreError, Listed, MemoryStore, MemoryUpload, Store, Upload}; +pub use services::Services; diff --git a/crates/lakekit-core/src/services.rs b/crates/lakekit-core/src/services.rs index 771c6e6..0ab4f62 100644 --- a/crates/lakekit-core/src/services.rs +++ b/crates/lakekit-core/src/services.rs @@ -1,58 +1,35 @@ -use loko_clock::{Clock, HasClock, TestClock}; -use loko_random::{HasRandom, Random, SeededRandom}; -use loko_store::{HasStore, MemoryStore, Store}; - -/// A store, a clock, and a random source as one value, for APIs that need all three (commits): -/// `Services::new(ObjectStoreAdapter::new(…), TokioClock, OsRandom)`. Any type that is all three -/// works as well: a JavaScript host, or an application's own `#[loko::bundle]` struct. -#[derive(Clone, Debug, Default)] -pub struct Services { - /// The store. - pub store: S, - /// The clock. - pub clock: C, - /// The random source. - pub random: R, -} - -// The three impls `#[loko::bundle]` would write, by hand, so lakekit needs no macros. -impl HasStore for Services { - type Store = S; - - fn store(&self) -> &S { - &self.store - } -} - -impl HasClock for Services { - type Clock = C; - - fn clock(&self) -> &C { - &self.clock - } +use std::fmt; + +/// The store, clock, and random source an operation such as a commit uses, each borrowed from +/// wherever the caller keeps it: `Services::new(&s3, &TokioClock, &OsRandom)`. One object that is +/// all three (a JavaScript host) goes in all three places: `Services::new(&host, &host, &host)`. +pub struct Services<'a, S, C, R> { + /// Where objects are read and written. + pub store: &'a S, + /// The time, and waiting between attempts. + pub clock: &'a C, + /// Random numbers: snapshot ids, file names, and jitter. + pub random: &'a R, } -impl HasRandom for Services { - type Random = R; - - fn random(&self) -> &R { - &self.random +impl<'a, S, C, R> Services<'a, S, C, R> { + /// `store`, `clock`, and `random`. + pub fn new(store: &'a S, clock: &'a C, random: &'a R) -> Self { + Self { store, clock, random } } } -impl Services { - /// `store` with `clock` and `random`. - pub fn new(store: S, clock: C, random: R) -> Self { - Self { store, clock, random } +// By hand, so `Services` is `Copy` whatever the services are (a derive would ask them to be). +impl Clone for Services<'_, S, C, R> { + fn clone(&self) -> Self { + *self } } -/// Everything in memory, for tests: a [`MemoryStore`], a [`TestClock`], and a [`SeededRandom`]. -pub type MemoryServices = Services; +impl Copy for Services<'_, S, C, R> {} -impl MemoryServices { - /// An empty store, the clock at `now_ms` milliseconds after the Unix epoch, and seed 0. - pub fn at_ms(now_ms: i64) -> Self { - Self::new(MemoryStore::new(), TestClock::at_ms(now_ms), SeededRandom::new(0)) +impl fmt::Debug for Services<'_, S, C, R> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Services").finish_non_exhaustive() } } diff --git a/crates/lakekit-iceberg/src/commit.rs b/crates/lakekit-iceberg/src/commit.rs index d0a2cdf..36a0bb6 100644 --- a/crates/lakekit-iceberg/src/commit.rs +++ b/crates/lakekit-iceberg/src/commit.rs @@ -13,7 +13,7 @@ use std::hash::BuildHasher; use std::sync::Arc; use std::time::Duration; -use lakekit_core::{Clock, Random, Rng, Store, uuid_v4}; +use lakekit_core::{Clock, Random, Rng, Services, Store, uuid_v4}; use crate::manifest::{ DataFile, Entry, ManifestContext, ManifestFile, ManifestWrite, Status, manifest_file, write_manifest, @@ -126,13 +126,17 @@ pub struct Extra { } /// An application's part in a commit. Every method has a default that does nothing extra. -pub trait CommitHooks { +pub trait CommitHooks { /// The error the hooks (and so the commit) return. type Error: From + fmt::Debug; /// The newest version of the table, after another writer won the race. - fn reload(&mut self, store: &H, location: &str) -> impl Future> { - async move { Ok(Table::load(store, location).await?) } + fn reload( + &mut self, + services: Services<'_, S, C, R>, + location: &str, + ) -> impl Future> { + async move { Ok(Table::load(services.store, location).await?) } } /// Whether `base` already holds this change (an idempotent append retried), so nothing is @@ -145,7 +149,7 @@ pub trait CommitHooks { /// per attempt. fn extra( &mut self, - _store: &H, + _services: Services<'_, S, C, R>, _base: &Table, _snapshot_id: i64, ) -> impl Future> { @@ -157,17 +161,17 @@ pub trait CommitHooks { #[derive(Clone, Copy, Debug, Default)] pub struct NoHooks; -impl CommitHooks for NoHooks { +impl CommitHooks for NoHooks { type Error = Error; } /// Commits `change` on top of `base` (or the newest version, after a lost race), with `props` in /// the snapshot summary. The returned table knows its files when `base` did. -pub async fn commit, S: BuildHasher>( - store: &H, +pub async fn commit, P: BuildHasher>( + services: Services<'_, S, C, R>, base: Table, change: &Change, - props: &HashMap, + props: &HashMap, options: &CommitOptions, hooks: &mut K, ) -> Result<(Table, CommitInfo), K::Error> { @@ -182,9 +186,9 @@ pub async fn commit, S: BuildHasher for attempt in 1..=options.max_attempts { if attempt > 1 { let (lo, hi) = options.backoff_ms; - let ms = (lo + store.rng().next_u64() % hi.saturating_sub(lo).max(1)) * attempt; - store.sleep(Duration::from_millis(ms)).await; - base = hooks.reload(store, &base.location).await?; + let ms = (lo + services.random.rng().next_u64() % hi.saturating_sub(lo).max(1)) * attempt; + services.clock.sleep(Duration::from_millis(ms)).await; + base = hooks.reload(services, &base.location).await?; } if hooks.skip(&base) { let info = CommitInfo { @@ -195,7 +199,7 @@ pub async fn commit, S: BuildHasher }; return Ok((base, info)); } - if let Some((table, snapshot_id)) = attempt_commit(store, &base, change, props, options, hooks).await? { + if let Some((table, snapshot_id)) = attempt_commit(services, &base, change, props, options, hooks).await? { let info = CommitInfo { snapshot_id, version: table.version, attempts: attempt, skipped: false }; return Ok((table, info)); } @@ -242,27 +246,27 @@ fn new_snapshot_id(random: &impl Random, base: &Table) -> i64 { /// One attempt: writes the snapshot's files and publishes it. `None` if another writer committed /// this version first (this attempt's files are deleted). With an unknown outcome, nothing is /// deleted. -async fn attempt_commit, S: BuildHasher>( - store: &H, +async fn attempt_commit, P: BuildHasher>( + services: Services<'_, S, C, R>, base: &Table, change: &Change, - props: &HashMap, + props: &HashMap, options: &CommitOptions, hooks: &mut K, ) -> Result, K::Error> { - let snapshot_id = new_snapshot_id(store, base); + let snapshot_id = new_snapshot_id(services.random, base); let sequence_number = base.metadata.next_sequence_number(); let mut uploads: Vec<(String, Vec)> = vec![]; let mut unknown = false; let result = async { - let manifests = write_manifests(store, base, change, snapshot_id, options, &mut uploads).await?; - let extra = hooks.extra(store, base, snapshot_id).await?; + let manifests = write_manifests(services, base, change, snapshot_id, options, &mut uploads).await?; + let extra = hooks.extra(services, base, snapshot_id).await?; uploads.extend(extra.files); - let list_uri = format!("{}/metadata/snap-{snapshot_id}-{}.avro", base.location, uuid_v4(store)); + let list_uri = format!("{}/metadata/snap-{snapshot_id}-{}.avro", base.location, uuid_v4(services.random)); let parent = base.metadata.current_snapshot_id(); let list = write_manifest_list(&manifests, snapshot_id, parent, sequence_number)?; uploads.push((list_uri.clone(), list)); - let puts = uploads.iter().map(|(uri, bytes)| store.put(uri, bytes.clone())); + let puts = uploads.iter().map(|(uri, bytes)| services.store.put(uri, bytes.clone())); futures_util::future::try_join_all(puts).await.map_err(Error::from)?; let mut summary: HashMap = props.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); @@ -270,7 +274,7 @@ async fn attempt_commit, S: BuildHa summary.extend(counters(change, &base.metadata.summary())); // Snapshot and metadata logs must stay in time order, so a writer whose clock is behind the // last one's dates its commit at the table's last update. - let now = store.now_ms().max(base.metadata.last_updated_ms()); + let now = services.clock.now_ms().max(base.metadata.last_updated_ms()); let snapshot = NewSnapshot { snapshot_id, sequence_number, @@ -281,21 +285,22 @@ async fn attempt_commit, S: BuildHa }; let metadata = base.metadata.with_snapshot(&snapshot, &base.metadata_file, now)?; let version = base.version + 1; - let ours = put_metadata(store, &metadata_uri(&base.location, version), metadata.to_json().into_bytes()) - .await - .inspect_err(|e| unknown = matches!(e, Error::CommitUnknown(_)))?; + let ours = + put_metadata(services.store, &metadata_uri(&base.location, version), metadata.to_json().into_bytes()) + .await + .inspect_err(|e| unknown = matches!(e, Error::CommitUnknown(_)))?; Ok::<_, K::Error>(ours.then_some((metadata, version))) } .await; let (metadata, version) = match result { Ok(Some(committed)) => committed, Ok(None) => { - delete_uploads(store, uploads).await; + delete_uploads(services.store, uploads).await; return Ok(None); } Err(e) => { if !unknown { - delete_uploads(store, uploads).await; + delete_uploads(services.store, uploads).await; } return Err(e); } @@ -320,7 +325,7 @@ async fn attempt_commit, S: BuildHa }); let table = Table { files, ..Table::new(base.location.clone(), version, metadata) }; if options.write_version_hint { - let _ = store.put(&hint_uri(&table.location), version.to_string().into_bytes()).await; + let _ = services.store.put(&hint_uri(&table.location), version.to_string().into_bytes()).await; } Ok(Some((table, snapshot_id))) } @@ -332,8 +337,8 @@ async fn delete_uploads(store: &impl Store, uploads: Vec<(String, Vec)>) { /// Writes a manifest of the added files, carries the base snapshot's manifests over, and rewrites /// the ones holding removed files (or all of them once there are too many). -async fn write_manifests( - store: &H, +async fn write_manifests( + services: Services<'_, S, C, R>, base: &Table, change: &Change, snapshot_id: i64, @@ -353,7 +358,7 @@ async fn write_manifests( partition_type: &partition_type, }; let mut write = |kind: &str, entries: &[ManifestWrite]| -> Result { - let uri = format!("{}/metadata/{}-{kind}-m.avro", base.location, uuid_v4(store)); + let uri = format!("{}/metadata/{}-{kind}-m.avro", base.location, uuid_v4(services.random)); let bytes = write_manifest(&ctx, entries)?; let mf = manifest_file(&uri, &bytes, spec_id, snapshot_id, entries, &partition_type); uploads.push((uri, bytes)); @@ -376,14 +381,14 @@ async fn write_manifests( new.push(write("added", &entries)?); } - let current = base.manifests(store).await?; + let current = base.manifests(services.store).await?; let removed: HashSet<&str> = change.removed().iter().map(|e| e.file.path.as_str()).collect(); let merge = removed.is_empty() && current.len() + new.len() > options.merge_manifests_at; if removed.is_empty() && !merge { new.extend(current); return Ok(new); } - let loaded = futures_util::future::try_join_all(current.iter().map(|m| Table::manifest(store, m))).await?; + let loaded = futures_util::future::try_join_all(current.iter().map(|m| Table::manifest(services.store, m))).await?; let (mut kept, mut deleted, mut carried) = (vec![], vec![], vec![]); for (manifest, entries) in current.into_iter().zip(loaded) { let live: Vec<&Entry> = entries.iter().filter(|e| e.status != Status::Deleted).collect(); @@ -472,7 +477,7 @@ pub fn counters(change: &Change, base: &HashMap Table { + /// A store, a clock, and a random source in memory: what a test commit uses. + struct Mem { + store: MemoryStore, + clock: TestClock, + random: SeededRandom, + } + + impl Mem { + fn at_ms(ms: i64) -> Self { + Self { store: MemoryStore::new(), clock: TestClock::at_ms(ms), random: SeededRandom::new(0) } + } + + fn services(&self) -> Services<'_, MemoryStore, TestClock, SeededRandom> { + Services::new(&self.store, &self.clock, &self.random) + } + } + + fn create(store: &Mem, format_version: u8) -> Table { let metadata = json!({ "format-version": format_version, "table-uuid": "u", "location": LOCATION, "last-sequence-number": 0, "last-updated-ms": 0, "last-column-id": 2, "current-schema-id": 0, @@ -494,7 +516,7 @@ mod tests { "properties": {}, "current-snapshot-id": -1, "snapshots": [], "snapshot-log": [], "metadata-log": [] }); let table = Table::new(LOCATION, 1, Metadata::from_json(metadata).unwrap()); - block_on(store.put(&table.metadata_file, table.metadata.to_json().into_bytes())).unwrap(); + block_on(store.store.put(&table.metadata_file, table.metadata.to_json().into_bytes())).unwrap(); Table { files: Some(vec![]), ..table } } @@ -512,17 +534,17 @@ mod tests { Change::Append { files } } - fn try_run>( - store: &MemoryServices, + fn try_run>( + store: &Mem, base: Table, change: &Change, hooks: &mut K, ) -> Result<(Table, CommitInfo), K::Error> { - block_on(commit(store, base, change, &HashMap::new(), &CommitOptions::default(), hooks)) + block_on(commit(store.services(), base, change, &HashMap::new(), &CommitOptions::default(), hooks)) } - fn run>( - store: &MemoryServices, + fn run>( + store: &Mem, base: Table, change: &Change, hooks: &mut K, @@ -536,9 +558,9 @@ mod tests { /// The live files of the newest version, read back from the store: (name, data sequence /// number, file sequence number). - fn live(store: &MemoryServices) -> Vec<(String, i64, i64)> { - let mut t = block_on(Table::load(store, LOCATION)).unwrap(); - let mut files: Vec<_> = block_on(t.live_files(store)) + fn live(store: &Mem) -> Vec<(String, i64, i64)> { + let mut t = block_on(Table::load(&store.store, LOCATION)).unwrap(); + let mut files: Vec<_> = block_on(t.live_files(&store.store)) .unwrap() .iter() .map(|e| { @@ -555,7 +577,7 @@ mod tests { #[test] fn append_replace_and_delete() { - let store = MemoryServices::at_ms(1_000); + let store = Mem::at_ms(1_000); let t = create(&store, 2); let (t, info) = run(&store, t, &append(vec![file("a", 1, 10), file("b", 2, 20)]), &mut NoHooks); assert_eq!((info.version, info.attempts, info.skipped), (2, 1, false)); @@ -577,12 +599,12 @@ mod tests { assert_eq!((summary["operation"].as_str(), summary[crate::summary::TOTAL_RECORDS].as_str()), ("delete", "10")); assert_eq!(summary[crate::summary::DELETED_DATA_FILES], "1"); assert_eq!(t.metadata.metadata_log_len(), 3); - assert_eq!(block_on(store.get(&hint_uri(LOCATION))).unwrap().as_deref(), Some(&b"4"[..])); + assert_eq!(block_on(store.store.get(&hint_uri(LOCATION))).unwrap().as_deref(), Some(&b"4"[..])); } #[test] fn a_lost_race_retries_on_the_newer_version() { - let store = MemoryServices::at_ms(1_000); + let store = Mem::at_ms(1_000); let stale = create(&store, 2); let (_, first) = run(&store, stale.clone(), &append(vec![file("a", 1, 10)]), &mut NoHooks); let (_, second) = run(&store, stale, &append(vec![file("b", 1, 10)]), &mut NoHooks); @@ -595,7 +617,7 @@ mod tests { #[test] fn removing_files_that_are_gone_is_a_conflict() { - let store = MemoryServices::at_ms(1_000); + let store = Mem::at_ms(1_000); let (t, _) = run(&store, create(&store, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); let removed = t.files.clone().unwrap_or_default(); let (t, _) = run(&store, t, &Change::Delete { removed: removed.clone() }, &mut NoHooks); @@ -606,14 +628,19 @@ mod tests { /// Hooks that skip a batch already committed, and add a file and summary properties. struct Batch(&'static str); - impl CommitHooks for Batch { + impl CommitHooks for Batch { type Error = Error; fn skip(&mut self, base: &Table) -> bool { base.metadata.ancestors().iter().any(|s| s.summary.get("batch").map(String::as_str) == Some(self.0)) } - fn extra(&mut self, _: &MemoryServices, base: &Table, snapshot_id: i64) -> impl Future> { + fn extra( + &mut self, + _: Services<'_, MemoryStore, TestClock, SeededRandom>, + base: &Table, + snapshot_id: i64, + ) -> impl Future> { let uri = format!("{}/metadata/extra-{snapshot_id}", base.location); let summary = HashMap::from([("batch".to_string(), self.0.to_string()), ("extra".to_string(), uri.clone())]); @@ -623,10 +650,10 @@ mod tests { #[test] fn hooks_skip_and_extend_commits() { - let store = MemoryServices::at_ms(1_000); + let store = Mem::at_ms(1_000); let (t, info) = run(&store, create(&store, 2), &append(vec![file("a", 1, 10)]), &mut Batch("b1")); let extra = t.metadata.summary()["extra"].clone(); - assert!(block_on(store.exists(&extra)).unwrap()); + assert!(block_on(store.store.exists(&extra)).unwrap()); let (t, again) = run(&store, t, &append(vec![file("a", 1, 10)]), &mut Batch("b1")); assert!(again.skipped); assert_eq!((again.version, again.snapshot_id), (info.version, info.snapshot_id)); @@ -635,22 +662,22 @@ mod tests { #[test] fn many_manifests_merge() { - let store = MemoryServices::at_ms(1_000); + let store = Mem::at_ms(1_000); let mut t = create(&store, 2); for i in 0..9 { t = run(&store, t, &append(vec![file(&format!("f{i}"), i, 1)]), &mut NoHooks).0; } - assert_eq!(block_on(t.manifests(&store)).unwrap().len(), 2, "eight merged into one, plus the new one"); + assert_eq!(block_on(t.manifests(&store.store)).unwrap().len(), 2, "eight merged into one, plus the new one"); assert_eq!(live(&store).len(), 9); } #[test] fn a_clock_behind_the_table_keeps_the_logs_in_order() { - let store = MemoryServices::at_ms(5_000); + let store = Mem::at_ms(5_000); let (t, _) = run(&store, create(&store, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); - let late = MemoryServices::at_ms(1_000); + let late = Mem::at_ms(1_000); for uri in store.store.uris() { - block_on(late.put(&uri, block_on(store.get(&uri)).unwrap().unwrap())).unwrap(); + block_on(late.store.put(&uri, block_on(store.store.get(&uri)).unwrap().unwrap())).unwrap(); } let (t, _) = run(&late, t, &append(vec![file("b", 1, 10)]), &mut NoHooks); assert_eq!((t.metadata.current_snapshot().unwrap().timestamp_ms, t.metadata.last_updated_ms()), (5_000, 5_000)); @@ -658,7 +685,7 @@ mod tests { #[test] fn a_replace_needs_a_sequence_number_per_added_file() { - let store = MemoryServices::at_ms(1_000); + let store = Mem::at_ms(1_000); let (t, _) = run(&store, create(&store, 2), &append(vec![file("a", 1, 10)]), &mut NoHooks); let removed = files_named(&t, "a.parquet"); let change = @@ -669,7 +696,7 @@ mod tests { #[test] fn only_v2_tables_are_written() { - let store = MemoryServices::at_ms(1_000); + let store = Mem::at_ms(1_000); let err = try_run(&store, create(&store, 1), &append(vec![]), &mut NoHooks); assert!(matches!(err, Err(Error::Unsupported(_)))); } diff --git a/crates/lakekit-iceberg/tests/iceberg_rust.rs b/crates/lakekit-iceberg/tests/iceberg_rust.rs index ad54de1..35fd18d 100644 --- a/crates/lakekit-iceberg/tests/iceberg_rust.rs +++ b/crates/lakekit-iceberg/tests/iceberg_rust.rs @@ -38,9 +38,6 @@ struct IoStore { io: FileIO, } -/// The store with a test clock and seeded random numbers. -type TestStore = Services; - /// The store's results. type Result = std::result::Result; @@ -145,7 +142,7 @@ async fn write_theirs(table: &iceberg::table::Table, batch: &RecordBatch, name: } /// Writes `batch` with arrow-rs's Parquet writer; returns lakekit's data file. -async fn write_ours(store: &TestStore, table: &Table, batch: &RecordBatch, name: &str, day: i32) -> DataFile { +async fn write_ours(store: &IoStore, table: &Table, batch: &RecordBatch, name: &str, day: i32) -> DataFile { let mut bytes = vec![]; let mut writer = ArrowWriter::try_new(&mut bytes, batch.schema(), None).unwrap(); writer.write(batch).unwrap(); @@ -206,7 +203,7 @@ async fn planned(io: &FileIO, metadata_file: &str) -> Vec { paths } -async fn live(store: &TestStore, metadata_file: &str) -> (Table, Vec) { +async fn live(store: &IoStore, metadata_file: &str) -> (Table, Vec) { let mut table = Table::from_metadata_file(store, metadata_file).await.unwrap(); let mut files = table.live_files(store).await.unwrap().to_vec(); files.sort_by(|a, b| a.file.path.cmp(&b.file.path)); @@ -231,7 +228,8 @@ async fn tables_round_trip_between_lakekit_and_iceberg_rust() { Some(UnboundPartitionSpec::builder().add_partition_field(1, "time_day", IceTransform::Day).unwrap().build()); let table = catalog.create_table(&ns, creation).await.unwrap(); let io = table.file_io().clone(); - let store = Services::new(IoStore { io: io.clone() }, TestClock::at_ms(T0 / 1000), SeededRandom::new(0)); + let store = IoStore { io: io.clone() }; + let (clock, random) = (TestClock::at_ms(T0 / 1000), SeededRandom::new(0)); let arrow = Arc::new(iceberg::arrow::schema_to_arrow_schema(table.metadata().current_schema()).unwrap()); // iceberg-rust appends; lakekit reads the file as iceberg-rust records it. @@ -252,7 +250,7 @@ async fn tables_round_trip_between_lakekit_and_iceberg_rust() { // lakekit commits on top; iceberg-rust reads the new version and lakekit's manifests. let ours = write_ours(&store, &base, &batch(&arrow, 1, 3_000), "ours-1", 20_455).await; let (committed, info) = commit( - &store, + Services::new(&store, &clock, &random), base, &Change::Append { files: vec![ours.clone()] }, &HashMap::new(), diff --git a/crates/lakekit-iceberg/tests/object_store.rs b/crates/lakekit-iceberg/tests/object_store.rs index 0bb4efb..8cb1bf3 100644 --- a/crates/lakekit-iceberg/tests/object_store.rs +++ b/crates/lakekit-iceberg/tests/object_store.rs @@ -12,7 +12,7 @@ use object_store::memory::InMemory; #[tokio::test] async fn iceberg_commits_through_object_store() { - let store = Services::new(ObjectStoreAdapter::new(Arc::new(InMemory::new()), "memory://"), TokioClock, OsRandom); + let store = ObjectStoreAdapter::new(Arc::new(InMemory::new()), "memory://"); let location = "memory://warehouse/t"; let metadata = Metadata::parse( br#"{"format-version": 2, "table-uuid": "u", "location": "memory://warehouse/t", "last-sequence-number": 0, @@ -33,8 +33,16 @@ async fn iceberg_commits_through_object_store() { ..DataFile::default() }; let change = Change::Append { files: vec![file] }; - let (_, info) = - commit(&store, table, &change, &HashMap::new(), &CommitOptions::default(), &mut NoHooks).await.unwrap(); + let (_, info) = commit( + Services::new(&store, &TokioClock, &OsRandom), + table, + &change, + &HashMap::new(), + &CommitOptions::default(), + &mut NoHooks, + ) + .await + .unwrap(); assert_eq!(info.version, 2); let mut loaded = Table::load(&store, location).await.unwrap(); diff --git a/spec/0001-store-clock-random.md b/spec/0001-store-clock-random.md index fd5ce91..6237888 100644 --- a/spec/0001-store-clock-random.md +++ b/spec/0001-store-clock-random.md @@ -31,16 +31,16 @@ browser, a Cloudflare Worker, a CLI, or a server, and tests run against memory. between commit attempts. - **`Random`:** `rng()`, a generator of the caller's own: snapshot ids, file names, and the jitter between commit attempts. -- **Who needs what:** readers need only a `Store`; commits need `Store + Clock + Random`. -- **`Services`:** a store, a clock, and a random source as one value (`Services::new(store, clock, random)`), for APIs - that need all three: it implements loko's `HasStore`, `HasClock`, and `HasRandom`, so it is all three. Any type that is all three (a JavaScript host, an - application's own bundle) needs no wrapper. +- **Who needs what:** readers take only a `Store`; commits take `Services`, a store, a clock, and a random source. +- **`Services`:** a struct of three references, `store`, `clock`, and `random`, each borrowed from wherever the + caller keeps it: `Services::new(&s3, &TokioClock, &OsRandom)`. One object that is all three (a JavaScript host) goes + in all three places: `Services::new(&host, &host, &host)`. It's `Copy`, and commit hooks receive it too. - **Native implementations:** `TokioClock` (feature `tokio`: the system clock and tokio's timer, native targets) and `OsRandom` (feature `getrandom`: the operating system's generator, native targets and WASI). - **Futures aren't `Send`.** Methods return `impl Future`, without a `Send` bound, so a store or clock can wrap JavaScript promises (wasm-bindgen). Native ones are free to be `Send`. -- **For tests:** `MemoryServices` is a `MemoryStore`, a `TestClock` that only moves when told (or when `sleep` is - called), and a `SeededRandom`, so tests are deterministic. +- **For tests:** a `MemoryStore`, a `TestClock` that only moves when told (or when `sleep` is called), and a + `SeededRandom`, so tests are deterministic. - **`Error`:** one error type for all crates, with the variant saying what a caller can do: `Invalid`, `Unsupported`, `Conflict` (retry on the newer version), `CommitUnknown` (the commit may have landed: delete nothing it wrote, load the table again), `NotFound`, `Store`, and `Corrupt`. It is `non_exhaustive`. A store's error (`StoreError`) converts @@ -58,6 +58,7 @@ browser, a Cloudflare Worker, a CLI, or a server, and tests run against memory. | D10 | Objects are a `Store` of their own, apart from the time and randomness: readers need only a store, and store adapters don't decide how time is told | Accepted | | D13 | The time and randomness are two traits, `Clock` and `Random`, as ZIO's services are, rather than one environment trait: each says what it is, and each can be swapped on its own | Accepted | | D14 | `Store`, `Clock`, and `Random` are loko's services, which every polythings project shares, rather than lakekit's own; `lakekit-core` re-exports them and keeps the error type | Accepted | +| D16 | Commits take `Services`, three references with named fields, rather than one value bounded by `Store + Clock + Random`: callers pass what they have without gluing it into a struct that implements all three, and the code says which service each call uses | Accepted | ## Open questions and issues @@ -75,3 +76,5 @@ None open. - 2026-10-21: loko's `Random` hands out generators (`rng()`); lakekit draws its identifiers and jitter from them. - 2026-10-21: `Resources` and `MemoryResources` are `Services` and `MemoryServices` again, as loko calls them services again. +- 2026-10-21: `Services` is three references with named fields, and commits and hooks take it instead of one value that + is all three (D16); `MemoryServices` is gone, and `uuid_v7` takes a clock and a random source. diff --git a/spec/README.md b/spec/README.md index a85d5fe..0f1a882 100644 --- a/spec/README.md +++ b/spec/README.md @@ -28,7 +28,7 @@ lakekit's design, laid out like Timelake's and lakeglass's. Start here, people a ## Index -4 specs: 2 in progress, 2 done. Decisions D1–D15. +4 specs: 2 in progress, 2 done. Decisions D1–D16. | # | Spec | Status | Updated | |---:|---|---|---| From 61ff41c74cc47ab4d4f619974372a35745158c8c Mon Sep 17 00:00:00 2001 From: Oleksandr Litus Date: Wed, 21 Oct 2026 18:12:06 -0400 Subject: [PATCH 13/14] Core: loko from its merged main (polythings/loko#5) Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 8 ++++---- Cargo.toml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a9180e..9a9c3d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1742,7 +1742,7 @@ checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "loko-clock" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=6c79750109b7929e927a582e92570105da625714#6c79750109b7929e927a582e92570105da625714" +source = "git+https://github.com/polythings/loko?rev=0c425d3bf65cdedbadd628843fe68d3e9ddc419b#0c425d3bf65cdedbadd628843fe68d3e9ddc419b" dependencies = [ "tokio", ] @@ -1750,7 +1750,7 @@ dependencies = [ [[package]] name = "loko-object-store" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=6c79750109b7929e927a582e92570105da625714#6c79750109b7929e927a582e92570105da625714" +source = "git+https://github.com/polythings/loko?rev=0c425d3bf65cdedbadd628843fe68d3e9ddc419b#0c425d3bf65cdedbadd628843fe68d3e9ddc419b" dependencies = [ "futures-util", "loko-store", @@ -1761,7 +1761,7 @@ dependencies = [ [[package]] name = "loko-random" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=6c79750109b7929e927a582e92570105da625714#6c79750109b7929e927a582e92570105da625714" +source = "git+https://github.com/polythings/loko?rev=0c425d3bf65cdedbadd628843fe68d3e9ddc419b#0c425d3bf65cdedbadd628843fe68d3e9ddc419b" dependencies = [ "getrandom 0.4.3", "rand_core 0.10.1", @@ -1770,7 +1770,7 @@ dependencies = [ [[package]] name = "loko-store" version = "0.2.0" -source = "git+https://github.com/polythings/loko?rev=6c79750109b7929e927a582e92570105da625714#6c79750109b7929e927a582e92570105da625714" +source = "git+https://github.com/polythings/loko?rev=0c425d3bf65cdedbadd628843fe68d3e9ddc419b#0c425d3bf65cdedbadd628843fe68d3e9ddc419b" [[package]] name = "lz4_flex" diff --git a/Cargo.toml b/Cargo.toml index abe688d..c228fda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,10 +16,10 @@ lakekit-core = { path = "crates/lakekit-core", version = "0.1.0" } lakekit-iceberg = { path = "crates/lakekit-iceberg", version = "0.1.0", default-features = false } # loko provides the Store, Clock, and Random services. From git until loko 0.2.0 is on crates.io # (polythings/loko#1); publishing uses the version. -loko-store = { git = "https://github.com/polythings/loko", rev = "6c79750109b7929e927a582e92570105da625714", version = "0.2.0" } -loko-clock = { git = "https://github.com/polythings/loko", rev = "6c79750109b7929e927a582e92570105da625714", version = "0.2.0" } -loko-random = { git = "https://github.com/polythings/loko", rev = "6c79750109b7929e927a582e92570105da625714", version = "0.2.0" } -loko-object-store = { git = "https://github.com/polythings/loko", rev = "6c79750109b7929e927a582e92570105da625714", version = "0.2.0" } +loko-store = { git = "https://github.com/polythings/loko", rev = "0c425d3bf65cdedbadd628843fe68d3e9ddc419b", version = "0.2.0" } +loko-clock = { git = "https://github.com/polythings/loko", rev = "0c425d3bf65cdedbadd628843fe68d3e9ddc419b", version = "0.2.0" } +loko-random = { git = "https://github.com/polythings/loko", rev = "0c425d3bf65cdedbadd628843fe68d3e9ddc419b", version = "0.2.0" } +loko-object-store = { git = "https://github.com/polythings/loko", rev = "0c425d3bf65cdedbadd628843fe68d3e9ddc419b", version = "0.2.0" } object_store = { version = "0.14", default-features = false } url = "2" thiserror = "2" From bcfb50401e3774654b1c6adb200ffea654b41125 Mon Sep 17 00:00:00 2001 From: Oleksandr Litus Date: Wed, 21 Oct 2026 18:17:37 -0400 Subject: [PATCH 14/14] Iceberg: core::time::Duration, as the WASM rule asks (review) Co-Authored-By: Claude Opus 5.5 --- crates/lakekit-iceberg/src/commit.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/lakekit-iceberg/src/commit.rs b/crates/lakekit-iceberg/src/commit.rs index 36a0bb6..dc31632 100644 --- a/crates/lakekit-iceberg/src/commit.rs +++ b/crates/lakekit-iceberg/src/commit.rs @@ -6,12 +6,12 @@ //! Applications extend a commit through [`CommitHooks`]: files of their own written with the //! snapshot, summary properties, a check that skips a commit already made, and how to reload. +use core::time::Duration; use std::collections::{BTreeMap, HashMap, HashSet}; use std::fmt; use std::future::Future; use std::hash::BuildHasher; use std::sync::Arc; -use std::time::Duration; use lakekit_core::{Clock, Random, Rng, Services, Store, uuid_v4};