diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..06bdc49 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,18 @@ +# Dependabot keeps dependencies current: one pull request a week per ecosystem with minor and patch updates, and one +# per major update. A new version is proposed once it's 7 days old (security updates at once). +version: 2 + +updates: + - package-ecosystem: github-actions + directory: / + schedule: { interval: weekly } + cooldown: { default-days: 7 } + groups: + actions: { update-types: [minor, patch] } + + - package-ecosystem: cargo + directory: / + schedule: { interval: weekly } + cooldown: { default-days: 7 } + groups: + cargo: { update-types: [minor, patch] } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..1d2c3a1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,84 @@ +# Checks every push to main and every pull request. Needs no secrets. +name: CI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.event_name == 'pull_request' && github.ref || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 + +jobs: + hooks: + name: Hooks (YAML, TOML, private keys, rustfmt) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: rustup toolchain install # rustfmt, for the rustfmt hook + # The commit-stage hooks of .pre-commit-config.yaml; the push-stage ones (clippy) run in the jobs below. + - uses: j178/prek-action@44924fe9ef4a6f88253c4b6b1bf11c689d3ce53d # v3.1.0 + with: + prek-version: 0.5.7 + + native: + name: Native (format, lint, tests) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: rustup toolchain install # from rust-toolchain.toml + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + - run: cargo fmt --all --check + - run: cargo clippy --workspace --all-features --all-targets -- -D warnings + - run: cargo test --workspace --all-features + # Each feature builds on its own, and with none, without warnings. + - name: Features one at a time + env: + RUSTFLAGS: -D warnings + run: | + cargo check -p loko --no-default-features + cargo check -p loko + cargo check -p loko --features tokio + cargo check -p loko --features store + cargo check -p loko --features object-store + cargo check -p loko-clock --no-default-features + cargo check -p loko-clock --features tokio + cargo check -p loko-random --no-default-features + cargo check -p loko-random --features getrandom + cargo check -p loko-env + cargo check -p loko-console + cargo check -p loko-store + cargo check -p loko-object-store + + wasm: + name: WebAssembly (browser and WASI targets) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: rustup toolchain install # from rust-toolchain.toml + - run: rustup target add wasm32-unknown-unknown wasm32-wasip2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + - run: cargo clippy --workspace --all-features --target wasm32-unknown-unknown -- -D warnings + - run: cargo clippy --workspace --all-features --target wasm32-wasip2 -- -D warnings + # The JavaScript implementations on their own, and the browser build without them. + - name: Features one at a time + env: + RUSTFLAGS: -D warnings + run: | + cargo check -p loko --no-default-features --target wasm32-unknown-unknown + cargo check -p loko --no-default-features --features js --target wasm32-unknown-unknown + cargo check -p loko --target wasm32-wasip2 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2c393a2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/target +.env* +.DS_Store diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..7535203 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,45 @@ +# Git hooks, run by prek (https://prek.j178.dev): `uv tool install prek && prek install --install-hooks`. +# On commit: formatters and fast checks on the staged files. On push: clippy, natively and for both wasm32 targets. +# Everything: `prek run --all-files --hook-stage pre-push`. CI runs the same checks plus the tests. +default_install_hook_types: [pre-commit, pre-push] + +repos: + - repo: builtin + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-merge-conflict + - id: check-yaml + - id: check-toml + - id: check-added-large-files + args: [--maxkb=1024] + - id: detect-private-key + + - repo: local + hooks: + - id: rustfmt + name: rustfmt + entry: rustfmt --edition 2024 + language: system + types: [rust] + - id: clippy + name: clippy + entry: cargo clippy --workspace --all-features --all-targets --quiet -- -D warnings + language: system + types: [rust] + pass_filenames: false + stages: [pre-push] + - id: clippy-wasm + name: clippy (wasm32) + entry: cargo clippy --workspace --all-features --target wasm32-unknown-unknown --quiet -- -D warnings + language: system + types: [rust] + pass_filenames: false + stages: [pre-push] + - id: clippy-wasi + name: clippy (wasm32-wasip2) + entry: cargo clippy --workspace --all-features --target wasm32-wasip2 --quiet -- -D warnings + language: system + types: [rust] + pass_filenames: false + stages: [pre-push] diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..a082585 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,32 @@ +# Agent instructions + +loko: services for Rust programs (`Clock`, `Random`, `Env`, `Console`, `Store`) passed to code as traits, with test, +system, and JavaScript implementations. One crate per service (`crates/loko-`), the macros +(`crates/loko-macros`), and the `loko` crate, which re-exports them and adds the platforms. + +## Before changing code + +- Follow [CONTRIBUTING.md](CONTRIBUTING.md). It is the code guide, and it applies to you. +- For anything beyond a small fix, check the specs ([spec/README.md](spec/README.md)). Agree on the design with the user + before building something new, and record the decision in the spec's Decisions table. + +## Rules + +- **Done means checked.** Before you report work as finished, run `prek run --all-files --hook-stage pre-push` and + `cargo test --workspace --all-features`. Report failures as failures. +- **Fix the code, not the checks.** Don't loosen lint or format configs. Don't add `#[allow]`. Use + `#[expect(..., reason = "...")]` only when the tool is genuinely wrong. +- **Keep things in sync:** a change in behaviour updates its spec (status, design, history), the crate's README, the + tests, and `CHANGELOG.md` (under `Unreleased`) in the same commit. +- **The README's examples are doctests** (`ReadmeExamples` in `crates/loko/src/lib.rs`, with features `store` and + `tokio`). Keep them running: `cargo test -p loko --all-features --doc`. +- **Everything builds for WASM.** Every crate builds for `wasm32-unknown-unknown` and `wasm32-wasip2`. JavaScript + implementations are gated to `wasm32-unknown-unknown` and feature `js`; system ones to targets that have a system. + CI checks all three targets. +- **Every service is a `#[loko_macros::service]` trait** whose methods take `&self` and return `impl Future` without a + `Send` bound when they are async. +- **Test implementations are deterministic** and their clones share state. +- **Ideas, not code.** Never copy or translate another project's code. Credit ideas in `PRIOR_ART.md` + (CONTRIBUTING.md, "Credit"). +- **Tests run offline.** No network, cloud accounts, or real buckets in tests. +- **Never commit secrets** or `.env*` files. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..3c71c9d --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,32 @@ +# Changelog + +All notable changes to loko. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and +versions follow [Semantic Versioning](https://semver.org/). + +## Unreleased + +The first release of loko as a services library. It is version 0.2.0 because the `loko` name on crates.io carries an +unrelated 0.1.0 from 2022 (task scripting), by the same owner. + +### Added + +- `loko-clock`: the `Clock` service (`now`, `now_ms`, `sleep`), `TestClock`, `StdClock`, `TokioClock` (feature + `tokio`), `JsClock` (feature `js`), and `OffsetClock`, a clock ahead or behind another. +- `loko-random`: the `Random` service, a source of `rand_core` generators (`rng()`, drawn from with no locks); + `SeededRandom` (`SeededRng`, SplitMix64, for tests), `OsRandom` (feature `getrandom`, `OsRng`), `JsRandom` (feature + `js`, `JsRng`), and `Random` for a `Mutex` around any seedable `rand_core` generator. The OS and JavaScript generators + are `CryptoRng`, so code that makes secrets can ask for `Random`. +- `loko-env`: the `Env` service (`var`, `vars`), `SystemEnv`, and `MapEnv`. +- `loko-console`: the `Console` service (`print_line`, `print_error_line`), `StdConsole`, `TestConsole`, and + `JsConsole` (feature `js`). +- `loko-store`: the `Store` service (objects by URI, with put-if-absent, multipart uploads, and prefix listing), + `MemoryStore`, an error type, and the wrappers `CountingStore` (counts calls), `ReadOnlyStore` (refuses writes), and + `PrefixedStore` (passes on only URIs under a prefix). +- `loko-object-store`: a `Store` over an `object_store` client, from a client or a URL. +- The service crates depend on nothing but `rand_core` (for `loko-random`); the macros are needed only to define + new services. +- `loko-macros`: `#[service]`, which adds a `HasX` trait to a service trait `X` and makes every `HasX` an `X`, and + `#[loko::bundle]`, which makes a struct each service its fields provide, and `#[wraps]`, which makes a wrapper + that replaces some of a service's methods and forwards the rest. +- `loko`: the services under one crate (`loko::clock`, …, `loko::store` with feature `store`), the platforms + `TestPlatform`, `StdPlatform`, `TokioPlatform`, and `JsPlatform`, and a prelude. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..43c994c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/COMPARISON.md b/COMPARISON.md new file mode 100644 index 0000000..22111d8 --- /dev/null +++ b/COMPARISON.md @@ -0,0 +1,160 @@ +# How loko compares + +loko does two things: + +1. **It ships services.** `Clock`, `Random`, `Env`, `Console`, and `Store`, each with an implementation for tests and + one for the system, and JavaScript ones for `Clock`, `Random`, and `Console` (in JavaScript, `Env` is a map and + `Store` the application's own binding), plus platforms that bundle them and wrappers that change them. +2. **It has a small macro layer.** `#[service]`, `#[loko::bundle]`, and `#[wraps]` let your own traits work the + same way. + +Each crate below overlaps with one of those. The comparisons were checked against these versions (October 2026): +delegate 0.13.5, ambassador 0.5.1, fundle 0.4.0, shaku 0.6.3, nject 0.5.1, service-builder 0.3.0, dill 0.17.2, +dependency-injector 2.1.0, ferrous-di 0.2.0, reinhardt-di 0.3.25, tower 0.5.3, and cap-std 4.0.3. + +## In short + +| Crate | What it is | Use it when | +|---|---|---| +| [delegate](https://crates.io/crates/delegate) | A macro that writes forwarding methods you list | You want a few methods forwarded to a field, inherent or trait | +| [ambassador](https://crates.io/crates/ambassador) | A derive that forwards a whole trait to a field or enum variants | A type should implement a trait entirely through a field, or an enum through its variants | +| [fundle](https://crates.io/crates/fundle) | A builder for a struct of dependencies, built in order | Many components need each other while being built | +| [shaku](https://crates.io/crates/shaku) | A compile-time-checked container of trait objects | A server needs singletons and per-request instances, with axum, actix, or Rocket integration | +| [nject](https://crates.io/crates/nject) | Compile-time providers that build types from their fields | You want constructors generated from a provider, including in `no_std` | +| [service-builder](https://crates.io/crates/service-builder) | A builder-pattern macro for structs of services | You want a builder per service struct | +| [dill](https://crates.io/crates/dill), [dependency-injector](https://crates.io/crates/dependency-injector), [ferrous-di](https://crates.io/crates/ferrous-di) | Runtime containers: register, then resolve by type | Implementations are chosen at run time, or you want Java/.NET-style lifetimes | +| [reinhardt-di](https://crates.io/crates/reinhardt-di) | FastAPI-style injection for the Reinhardt web framework | You build on Reinhardt | +| [tower](https://crates.io/crates/tower) | Request-to-response services and middleware layers | You compose network clients and servers | +| [cap-std](https://crates.io/crates/cap-std) | The standard library's file system and network through capability handles | Code must not reach files outside a directory it was given | +| loko | Services with test, system, and JavaScript implementations, and the macros to add your own | Code should get the time, randomness, environment, console, or storage passed in, and run in tests, natively, and in the browser | + +## Forwarding: delegate and ambassador + +These are the closest to loko's macros. + +- **delegate** writes methods that forward to a field: `delegate! { to self.inner { fn get(&self, key: &str) -> + Option; } }`. You list each signature. Methods you don't list you write yourself, so a trait impl can forward + some methods and write others. `async fn` gets an `.await`, and generic methods work. Associated types are yours to + write. +- **ambassador** forwards a whole trait. Mark the trait `#[delegatable_trait]` (or copy a foreign trait's signature + with `#[delegatable_trait_remote]`), and derive `#[delegate(Trait, target = "field")]`. It forwards to enum variants + too, and its tests cover associated types, generic methods, GATs, `async fn`, and `-> impl Future` methods. + Forwarding is all or nothing: overriding some methods is an open request (issues #13 and #67). + +**What loko adds:** +- `#[loko::bundle]` is ambassador's idea. Credited in [PRIOR_ART.md](PRIOR_ART.md). +- `#[service]` also generates a `HasX` trait. Code can bound on it, and get the held service with `.x()`. +- `#[wraps]` forwards everything you don't write, without listing signatures. That combines what delegate and + ambassador each do half of. + +**Use them instead** for forwarding inherent methods (delegate), for enums (ambassador), for traits you can't mark +(ambassador's remote traits), or anywhere you don't need loko's services. + +## Compile-time wiring: fundle, nject, shaku, and service-builder + +These build a graph of components and check at compile time that every dependency is there. + +- **fundle** generates a builder for a struct of dependencies: + `AppState::builder().logger(|_| Logger::new()).database(|x| Database::connect(url, x)).build()`. Each field is set + once, and its closure sees the fields set before it, so construction happens in order. It implements `AsRef` for + each field type that appears once, and components take `impl AsRef`. It hands out concrete types, with no + trait objects and no `Send` requirement, and has async and fallible setters. +- **nject** generates construction from providers: `provider.provide::()` builds a type from its fields, + sharing what the provider holds by reference. It works with generics and with `&dyn` or `Box` traits, has + scopes, and is `no_std`. +- **shaku** declares components (singletons) and providers (a new instance each time) in a `module!`, injects them into + fields as `Arc`, and resolves them with `module.resolve_ref()`. Interfaces must be `Send + Sync + 'static` + with the default `thread_safe` feature (`'static` without it), and must be dyn-compatible. For async methods it + suggests `async-trait`. It has test overrides (`with_component_override`) and integrations for axum, actix, and + Rocket. +- **service-builder** generates `Service::builder().dependency(…).build()`. `build()` returns a `Result`, and a missing + dependency is a `BuildError::MissingDependency` at run time. + +**Where loko differs:** +- **It doesn't build a graph.** A loko bundle is a struct literal you write in `main` or in a test. Construction order + is ordinary code. +- **It decides what code may use, and with which implementation.** Code bounds on traits (`impl Store + Clock`), not on + concrete types (fundle's `AsRef`). +- **No trait objects.** loko never needs `dyn`, so services with `-> impl Future` methods (`Store`, `Clock::sleep`) + work as they are. shaku would need them boxed through `async-trait`. +- **No `Send`, `Sync`, or `'static` requirement,** so JavaScript-backed services work. +- **It ships the services:** test clocks, seeded randomness, an in-memory store, consoles that record. These crates + wire your types; you bring the implementations, test ones included. + +**Use them alongside or instead** when a large application has many components that need each other at +construction (fundle), or when a server needs per-request instances next to singletons and framework integration +(shaku). They combine with loko: fundle can build the struct that `#[loko::bundle]` then exposes as services, and +a shaku component can hold a loko platform. + +## Runtime containers: dill, dependency-injector, ferrous-di, and reinhardt-di + +These register implementations in a container and resolve them by type at run time, as Java and .NET containers do. + +- **dill** has a `Catalog`, `#[component]` constructor injection, `Arc` interfaces, `OneOf`, `AllOf`, + `Maybe`, and `Lazy`, and transient, singleton, and transaction lifetimes. Components are `Send + Sync`. +- **dependency-injector** resolves `get::()` to `Arc` from a lock-free map, with singletons, lazy singletons, + transients, and child scopes that can override their parent's services. Services are `Send + Sync + 'static`. +- **ferrous-di** follows Microsoft.Extensions.DependencyInjection: a `ServiceCollection` of factories, `Arc` and + `Arc`, and singleton, scoped, and transient lifetimes. Everything is `Send + Sync + 'static`; its `async` + feature uses `async-trait` and tokio's multi-threaded runtime. +- **reinhardt-di** is FastAPI-style injection for the Reinhardt web framework: `Depends<…>` in handlers, async + providers, request, singleton, and transient scopes, and test overrides. Injectables are `Send + Sync + 'static`, it + uses `async-trait`, and its README says resolution runs only on native targets. + +**Where loko differs:** nothing is resolved at run time, so a missing service is a compile error, not a startup +failure. There's no registry to configure, and calls are statically dispatched. loko also has no lifetimes or scopes, +and no choosing implementations from configuration at run time. + +**Use them instead** when the implementation is only known at run time (plugins, configuration), or when you want a +container's lifetimes and are fine with trait objects and `Send + Sync`. + +## Middleware: tower + +tower's `Service` is one generic trait, a request in and a future of a response out, and its `Layer` wraps a service in +another. Timeouts, retries, rate limits, and load balancing stack around any client or server. + +**Where loko differs:** +- **Same word, different thing.** A tower `Service` is a request handler. A loko service is something code is + given, such as a clock or a store, as in ZIO and the dependency-injection crates. loko exports no `Service` trait. +- **Many specific traits instead of one generic one.** Wrapping is one feature among several, not the point of the + library. +- **The same stacking idea, applied to services.** loko's wrappers stack the way tower's layers do (credited in + [PRIOR_ART.md](PRIOR_ART.md)), but there's no `Layer` trait yet ([#7](https://github.com/polythings/loko/issues/7)). +- **Complementary:** a tower retry layer could take its backoff from a loko `Clock`, so tests could skip the waits. + +## Capabilities: cap-std + +cap-std gives the standard library's file system and network through handles. A program opens files through a `Dir` +it was given, not by path, so it can't reach outside it. It's a foundation for WASI in Wasmtime. + +**Where loko differs:** the idea is shared, passing access instead of reaching for globals, but the aim isn't. +- **cap-std confines.** It's about security, and it covers files and sockets. +- **loko substitutes.** It's about swapping implementations for tests and platforms, and it covers the time, + randomness, the environment, the console, and object storage. +- **Closest overlap:** `PrefixedStore` (URIs under a prefix, refusing `..`) is the nearest loko comes to cap-std's + confinement. + +## What loko does that the others don't + +- **Ships the services.** A test clock whose `sleep` moves time instead of waiting, seeded randomness, an environment + in a map, a console that records lines, and an in-memory store with conditional writes. Each has a system + counterpart with the same interface, and the clock, randomness, and console have JavaScript ones. The other crates + wire up implementations; none ships these. +- **Async without boxing.** Services return `impl Future`, and loko passes them as generics, never as `dyn`, so + there's no `async-trait` and no heap allocation per call. +- **No `Send`, `Sync`, or `'static` requirement.** Futures that wrap JavaScript promises aren't `Send`. dill, + dependency-injector, ferrous-di, and reinhardt-di require `Send + Sync`; shaku requires it unless `thread_safe` is + turned off. +- **Checked on WebAssembly.** CI builds every crate for `wasm32-unknown-unknown` and `wasm32-wasip2`, and the + JavaScript implementations use the host's clock, timers, `crypto.getRandomValues`, and console. +- **Wrappers that write only what they change,** with `#[wraps]`, and ready-made ones that stack. + +## What loko doesn't do + +- **No container or resolution by type.** Bundles are structs you write. +- **No lifetimes or scopes** (singleton, transient, per request), and no field injection. +- **No choosing implementations at run time** from configuration or plugins. +- **No help building a graph in order,** and no web framework integrations. + +For those, pair loko with the crate that does them, or use that crate on its own. For a program with a handful of +services, chosen once and swapped in tests and in the browser, loko and ordinary Rust are enough. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..43c74f5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,96 @@ +# Contributing to loko + +This is the code guide for everyone who changes this repository, people and AI agents alike. Formatters and linters +enforce the mechanical rules (see [Checks](#checks)). This guide covers what they can't: how code should read. +[spec/](spec/README.md) holds the design. + +## Principles + +1. **Readable first.** Someone new should understand a function from its name, its signature, and one pass through + the body. +2. **One crate per service.** A service's crate holds its trait and its implementations, and depends on no other + service's crate, so a library can take just the service it needs. The `loko` crate re-exports them all and holds + what combines them (the platforms). +3. **Small traits.** A service has the few methods its callers need, and provided methods for what follows from them + (`now_ms` from `now`). Implementations are where the variety goes. +4. **Every target.** Each service has a test implementation that runs anywhere, a system one where there is a system + (native and WASI), and a JavaScript one for `wasm32-unknown-unknown` where it makes sense. +5. **One definition of each thing.** Before writing a helper, constant, or type, look for an existing one. +6. **Behaviour, spec, and tests change together.** A change users can see updates its spec, adds or updates tests, and + adds a line to [CHANGELOG.md](CHANGELOG.md). + +## Comments and docs + +- **Public items get a short doc comment**: what it does, and anything a caller must know (errors, units, which + targets, whether it blocks). One to three lines is usually enough. +- **Comments explain why, not what.** Explain a non-obvious choice or a workaround. +- **No long comments.** If an explanation needs a page, it belongs in a spec. Link to it from the code. + +## Credit + +loko learns from other projects' ideas, never from copying their code. [PRIOR_ART.md](PRIOR_ART.md) lists the ideas +we took. + +- **Never copy code**, whatever its license: not by pasting, and not by translating it into another language. +- **Credit ideas from open-source projects** with a row in PRIOR_ART.md and a short comment where the idea is + implemented. +- **Don't name open-core, dual-licensed, or source-available projects, or commercial products, as a source.** + +## Rust + +- **loko's own services need no macros.** Their crates write out what `#[service]` generates (`HasX`, its + `wrapped_` methods, and the blanket impl) and implement their wrappers' `HasX` by hand, so a library can depend on + `loko-clock` without `syn`. Keep that code in the macro's shape; the tests in `crates/loko/tests` use it with + `#[loko::bundle]` and `#[loko::wraps]`. +- **Services:** a trait marked `#[loko_macros::service]` (or written out, as above), methods on `&self`, async methods returning `impl Future` + without `Send`. Implementations that hold state share it between clones (`Arc>`). +- **State in services:** methods take `&self`, so state that changes sits behind a `Mutex` or an atomic (counters). + Lock only for the synchronous part of a call: never hold a `std::sync::Mutex` guard or a `RefCell` borrow across + `.await` (Clippy's `await_holding_lock` and `await_holding_refcell_ref` catch it). Clones share state. +- **Errors:** what the platform gives (`io::Result` for the console), or the crate's own `non_exhaustive` error enum + (`loko_store::Error`). No `unwrap()` outside tests; a poisoned lock is recovered, not a panic. +- **Layout:** the trait in `lib.rs`, and each implementation in a module named for where it runs: `testing.rs`, + `system.rs` (native and WASI), `js.rs`, or a runtime's (`tokio_clock.rs`). Each module has its own tests. + A wrapper gets a module named for what it does (`offset.rs`, `counting.rs`) and is written with + `#[loko_macros::wraps]`. +- **Targets:** a module that only builds on some targets is gated once, on its `mod` line and its `pub use`: + JavaScript code under `#[cfg(all(feature = "js", target_arch = "wasm32", target_os = "unknown"))]`, system code under + `#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))]`. +- **Suppressing a lint:** `#[expect(lint, reason = "...")]` with a real reason. Never `#[allow]`. + +## Tests + +- **A bug fix starts with a test that fails**, then the fix. +- **Tests run offline** and use the test implementations; a test of a system implementation checks only what holds on + every machine. + +## Checks + +Install once: `uv tool install prek && prek install --install-hooks`. Hooks then run on commit and on push. Before you +push, or before calling work done: + +```bash +prek run --all-files --hook-stage pre-push # rustfmt, clippy (native, wasm32-unknown-unknown, wasm32-wasip2) +cargo test --workspace --all-features # unit and doc tests +``` + +## Dependencies + +Keep dependencies on their newest versions; check the latest release when adding one. Actions are pinned to a commit +with the version in a comment (`uses: actions/checkout@ # v7.0.1`). + +## Commits + +- **Subject:** a short imperative sentence with the area first, for example `Clock: …`, `Macros: …`, or `Spec: …`. + Add a body when the why isn't obvious. +- **Scope:** one logical change per commit. + +## Changelog + +[CHANGELOG.md](CHANGELOG.md) is for users, not a list of commits. Anything a user can notice gets one line under +`## Unreleased`, in `Added`, `Changed`, `Deprecated`, `Removed`, `Fixed`, or `Security`, in the same commit as the +change. + +## License + +Contributions are licensed under the Apache License 2.0, like the rest of the project (see [LICENSE](LICENSE)). diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..4d6b53a --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1086 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877bf67ac8e52449eb2f9e98a344ea163beb70dd1f9eebe3cc5357dcfeaee4df" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-macro", + "futures-sink", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "loko" +version = "0.2.0" +dependencies = [ + "loko-clock", + "loko-console", + "loko-env", + "loko-macros", + "loko-object-store", + "loko-random", + "loko-store", + "tokio", +] + +[[package]] +name = "loko-clock" +version = "0.2.0" +dependencies = [ + "js-sys", + "tokio", + "wasm-bindgen", + "wasm-bindgen-futures", +] + +[[package]] +name = "loko-console" +version = "0.2.0" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "loko-env" +version = "0.2.0" + +[[package]] +name = "loko-macros" +version = "0.2.0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "loko-object-store" +version = "0.2.0" +dependencies = [ + "futures-util", + "loko-store", + "object_store", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "loko-random" +version = "0.2.0" +dependencies = [ + "getrandom", + "js-sys", + "rand_core", + "wasm-bindgen", +] + +[[package]] +name = "loko-store" +version = "0.2.0" + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object_store" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures-channel", + "futures-core", + "futures-util", + "http", + "humantime", + "itertools", + "nix", + "parking_lot", + "percent-encoding", + "thiserror", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", + "windows-sys", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c168a474b95bfecfe908a65d4bdffc523144b649a74c09c286afb35d53e0bc" +dependencies = [ + "bytes", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" +dependencies = [ + "js-sys", + "tokio", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..590d518 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,60 @@ +[workspace] +resolver = "2" +members = [ + "crates/loko-clock", + "crates/loko-random", + "crates/loko-env", + "crates/loko-console", + "crates/loko-store", + "crates/loko-object-store", + "crates/loko-macros", + "crates/loko", +] + +[workspace.package] +version = "0.2.0" +edition = "2024" +rust-version = "1.99" +license = "Apache-2.0" +repository = "https://github.com/polythings/loko" +keywords = ["services", "clock", "random", "testing", "wasm"] +categories = ["development-tools::testing", "wasm", "asynchronous"] + +[workspace.dependencies] +loko-clock = { path = "crates/loko-clock", version = "0.2.0" } +loko-random = { path = "crates/loko-random", version = "0.2.0" } +loko-env = { path = "crates/loko-env", version = "0.2.0" } +loko-console = { path = "crates/loko-console", version = "0.2.0" } +loko-store = { path = "crates/loko-store", version = "0.2.0" } +loko-object-store = { path = "crates/loko-object-store", version = "0.2.0" } +loko-macros = { path = "crates/loko-macros", version = "0.2.0" } +loko = { path = "crates/loko", version = "0.2.0" } +futures-util = { version = "0.3", default-features = false, features = ["alloc"] } +object_store = { version = "0.14", default-features = false } +url = "2" +rand_core = { version = "0.10", default-features = false } +tokio = { version = "1", features = ["time"] } +getrandom = "0.4" +js-sys = "0.3" +wasm-bindgen = "0.2" +wasm-bindgen-futures = "0.4" +proc-macro2 = "1" +quote = "1" +syn = { version = "2", features = ["full", "visit", "visit-mut"] } + +[workspace.lints.rust] +unsafe_code = "forbid" +unreachable_pub = "warn" +missing_debug_implementations = "warn" +missing_docs = "warn" +rust_2018_idioms = { level = "warn", priority = -1 } + +[workspace.lints.clippy] +pedantic = { level = "warn", priority = -1 } +missing_errors_doc = "allow" +missing_panics_doc = "allow" +must_use_candidate = "allow" +allow_attributes = "warn" +dbg_macro = "warn" +todo = "warn" +unwrap_used = "warn" diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..ac52329 --- /dev/null +++ b/NOTICE @@ -0,0 +1,6 @@ +loko +Copyright 2026 Oleksandr Litus and the loko contributors + +This product is licensed under the Apache License, Version 2.0 (see LICENSE). + +Ideas loko borrowed from other projects are credited in PRIOR_ART.md. diff --git a/PRIOR_ART.md b/PRIOR_ART.md new file mode 100644 index 0000000..89d1f34 --- /dev/null +++ b/PRIOR_ART.md @@ -0,0 +1,27 @@ +# Prior art + +loko stands on other open-source projects and learns from more. This file credits the ideas we took. We take ideas, +never code: every line of loko is our own. How we credit is in [CONTRIBUTING.md](CONTRIBUTING.md#credit). + +## Foundations + +| Project | License | What loko uses | +|---|---|---| +| [tokio](https://github.com/tokio-rs/tokio) | MIT | The timer behind `TokioClock` | +| [getrandom](https://github.com/rust-random/getrandom) | Apache-2.0 / MIT | The operating system's generator behind `OsRandom` | +| [rand_core](https://github.com/rust-random/rand) | Apache-2.0 / MIT | The generator traits (`Rng`, `CryptoRng`, `SeedableRng`) of the generators `Random` hands out, and `fork` | +| [wasm-bindgen](https://github.com/wasm-bindgen/wasm-bindgen) (`js-sys`, `wasm-bindgen-futures`) | Apache-2.0 / MIT | The JavaScript implementations | +| [object_store](https://github.com/apache/arrow-rs-object-store) | Apache-2.0 / MIT | The stores `loko-object-store` adapts | +| [syn](https://github.com/dtolnay/syn), [quote](https://github.com/dtolnay/quote) | Apache-2.0 / MIT | The macros | + +## Ideas + +| Idea | From | License of the source | Where in loko | +|---|---|---|---| +| The time, randomness, the console, and the system as services passed to code, each with a test implementation (a clock that moves when told) | [ZIO](https://github.com/zio/zio) (`Clock`, `Random`, `Console`, `System`, `TestClock`) | Apache-2.0 | The service crates and `TestPlatform` | +| An environment of services described by "has a" traits (`HasX`), so code asks for the services it needs and any environment holding them qualifies | [RIO](https://github.com/commercialhaskell/rio) (`HasLogFunc` and the "Has" type class pattern) | MIT | `#[service]`'s `HasX` traits | +| Implementing a trait for a struct by forwarding to a field, chosen with a derive and a field attribute | [ambassador](https://github.com/hobofan/ambassador) | Apache-2.0 / MIT | `#[loko::bundle]` and `#[wraps]` | +| Wrappers that are the trait they wrap, so they stack | [tower](https://github.com/tower-rs/tower) (`Layer`) | MIT | `#[wraps]`, `CountingStore`, `ReadOnlyStore`, `PrefixedStore`, `OffsetClock` | +| SplitMix64: a 64-bit state advanced by a constant and mixed with two multiply-xorshift rounds | Steele, Lea, and Flood, "Fast Splittable Pseudorandom Number Generators" (OOPSLA 2014) | A published algorithm | `SeededRandom` | +| A storage interface with no I/O library of its own, supplied by the application | [delta-kernel-rs](https://github.com/delta-io/delta-kernel-rs) (its `Engine` trait) | Apache-2.0 | `Store` | +| One crate per backend under an umbrella crate with a feature each | [arrow-rs](https://github.com/apache/arrow-rs), [Apache OpenDAL](https://github.com/apache/opendal) | Apache-2.0 | The workspace layout | diff --git a/README.md b/README.md new file mode 100644 index 0000000..2ad082c --- /dev/null +++ b/README.md @@ -0,0 +1,444 @@ +# loko + +Services for Rust programs: the clock, random numbers, environment variables, the console, and object storage, passed +to code as traits instead of reached for as globals. The program passes the real ones, tests pass ones they control, +and the same code runs natively, under WASI, and in the browser. + +## Why + +Code that calls `SystemTime::now()`, sleeps, or draws from a global random generator is hard to test: checking a +one-hour timeout takes an hour, and random retries behave differently on every run. With loko, the code takes the clock +as an argument, and the test decides what time it is: + +```rust +use std::time::{Duration, SystemTime}; + +use loko::Clock; +use loko::clock::TestClock; + +/// Whether a session that started at `started` has expired. +fn expired(clock: &impl Clock, started: SystemTime) -> bool { + clock.now() > started + Duration::from_secs(60 * 60) +} + +let clock = TestClock::at_ms(0); +let started = clock.now(); +assert!(!expired(&clock, started)); +clock.advance(Duration::from_secs(60 * 60 + 1)); // an hour passes, at once +assert!(expired(&clock, started)); +``` + +The program passes `StdClock`, `TokioClock`, or `JsClock` in the browser, and `expired` stays as it is. What you get: + +- **Tests that control time.** A test clock moves only when told, and sleeping on it returns at once, so retries, + timeouts, and polling loops run in tests without waiting. +- **Random numbers that repeat in tests,** and secure ones in programs, with signatures that say which is needed. +- **One codebase for native, WASI, and the browser.** Platforms bundle the right implementations for each, and nothing + requires `Send`, so services backed by JavaScript work. +- **Storage you can swap:** memory in tests, S3, GCS, Azure, or local files in production. +- **Tests that set environment variables and read what was printed,** without touching the process. +- **Your own services and wrappers,** with three small macros, in the same style. + +```toml +[dependencies] +loko = { version = "0.2", features = ["tokio", "store"] } +tokio = { version = "1", features = ["macros", "rt"] } # for the examples' #[tokio::main] +``` + +| Feature | Adds | +|---|---| +| `getrandom` (default) | `OsRandom` and `StdPlatform` (native and WASI) | +| `tokio` | `TokioClock` and `TokioPlatform` (native) | +| `js` | `JsClock`, `JsRandom`, `JsConsole`, and `JsPlatform` (`wasm32-unknown-unknown`) | +| `store` | `loko::store`: the `Store` service, `MemoryStore`, and the store wrappers | +| `object-store` | `loko::object_store`: a `Store` over an `object_store` client (S3, GCS, Azure, local files) | + +## Guide + +### 1. Waiting without waiting + +A retry loop that waits a second between tries. Its test takes no time, because a test platform's sleeps move its +clock instead of blocking. + +```rust +use std::time::Duration; + +use loko::{Clock, TestPlatform}; + +/// Tries `attempt` up to three times, a second apart. +async fn retry(clock: &impl Clock, mut attempt: impl FnMut() -> Result) -> Result { + for _ in 1..3 { + if let Ok(value) = attempt() { + return Ok(value); + } + clock.sleep(Duration::from_secs(1)).await; + } + attempt() +} + +#[tokio::main(flavor = "current_thread")] +async fn main() { + let platform = TestPlatform::new(); // TokioPlatform::default() in a program + let mut calls = 0; + let result = retry(&platform, || { + calls += 1; + if calls < 3 { Err("busy") } else { Ok(calls) } + }) + .await; + assert_eq!(result, Ok(3)); + assert_eq!(platform.now_ms(), 2_000, "two seconds of test time, none of real time"); +} +``` + +### 2. Random numbers that repeat + +A `Random` hands out generators (`rng()`). In tests they come from a seed, so every run makes the same choices. + +```rust +use loko::{Random, Rng, TestPlatform}; + +/// Picks one of `servers` at random. +fn pick<'a>(random: &impl Random, servers: &[&'a str]) -> &'a str { + servers[(random.rng().next_u64() % servers.len() as u64) as usize] +} + +let servers = ["a", "b", "c"]; +assert_eq!(pick(&TestPlatform::new(), &servers), pick(&TestPlatform::new(), &servers)); +``` + +Code that makes secrets asks for a secure generator, `fn token(random: &impl Random)`, so a seeded +one can't be passed to it by mistake. Drawing many numbers from one `rng()` takes no locks. + +### 3. Several services, one platform + +A function can ask for several services at once. A platform is one of each of `Clock`, `Random`, `Env`, and `Console`: +`TestPlatform` for tests, `StdPlatform` or `TokioPlatform` for programs, `JsPlatform` in the browser. + +```rust +use loko::{Console, Env, TestPlatform}; + +fn greet(services: &(impl Env + Console)) { + let name = services.var("USER").unwrap_or_else(|| "stranger".into()); + services.print_line(&format!("Hello, {name}")).unwrap(); +} + +let platform = TestPlatform::new().with_var("USER", "Ada"); +greet(&platform); +assert_eq!(platform.console.lines(), ["Hello, Ada"]); +``` + +### 4. Storage + +`Store` reads and writes objects by URI, including a write that succeeds only if nothing is there yet, for writers +that must not overwrite each other. + +```rust +use loko::store::{MemoryStore, Store}; + +#[tokio::main(flavor = "current_thread")] +async fn main() { + let store = MemoryStore::new(); // ObjectStoreAdapter::from_url("s3://bucket/…", options)? in production + assert!(store.put_if_absent("memory://a.txt", b"first".to_vec()).await.unwrap()); + assert!(!store.put_if_absent("memory://a.txt", b"second".to_vec()).await.unwrap(), "the first write wins"); +} +``` + +### 5. Bundles + +When code needs a platform and a store together, put them in one struct: `#[loko::bundle]` makes it each service its +fields hold, as marked with `#[loko::has(…)]`. + +```rust +use loko::prelude::*; +use loko::store::MemoryStore; + +#[loko::bundle] +struct App { + #[loko::has(Clock, Random, Env, Console)] + platform: P, + #[loko::has(Store)] + store: S, +} + +/// Saves a report named by the time. +async fn save_report(app: &(impl Store + Clock)) -> loko::store::Result { + let uri = format!("memory://reports/{}.txt", app.now_ms()); + app.put(&uri, b"ok".to_vec()).await?; + Ok(uri) +} + +#[tokio::main(flavor = "current_thread")] +async fn main() { + // TokioPlatform and an S3 store in production. + let app = App { platform: TestPlatform::new().at_ms(42), store: MemoryStore::new() }; + assert_eq!(save_report(&app).await.unwrap(), "memory://reports/42.txt"); +} +``` + +`#[loko::has(Clock)]` names `HasClock` next to `Clock`, so bring both in (`loko::prelude::*` does). A library type can +hold its services the same way, as one generic field: `struct Uploader { services: S }` with +`impl Uploader`. + +### 6. Your own service + +`#[loko::service]` makes any trait a service, so your own dependencies (a mailer, a payment client) work like loko's: +a real implementation for the program, a test one that records, and a field in the bundle. + +```rust +use std::sync::Mutex; + +use loko::prelude::*; + +#[loko::service] +trait Mailer { + fn send(&self, to: &str, body: &str); +} + +/// For tests: remembers what was sent. +#[derive(Default)] +struct TestMailer { + sent: Mutex>, +} + +impl Mailer for TestMailer { + fn send(&self, to: &str, body: &str) { + self.sent.lock().unwrap().push(format!("{to}: {body}")); + } +} + +fn report_backup(app: &(impl Mailer + Env)) { + let to = app.var("ADMIN").unwrap_or_else(|| "root".into()); + app.send(&to, "backup finished"); +} + +#[loko::bundle] +struct App { + #[loko::has(Env)] + platform: TestPlatform, + #[loko::has(Mailer)] + mailer: TestMailer, +} + +let app = App { platform: TestPlatform::new().with_var("ADMIN", "ops"), mailer: TestMailer::default() }; +report_backup(&app); +assert_eq!(*app.mailer.sent.lock().unwrap(), ["ops: backup finished"]); +``` + +### 7. Your own implementation and platform + +loko's services are ordinary traits, so you can implement one: write the required methods, and the provided ones +(`now_ms`) come with it. A platform is a bundle with one field per service, which is how loko's own are built. + +```rust +use std::future::{Future, ready}; +use std::time::{Duration, SystemTime}; + +use loko::console::TestConsole; +use loko::env::MapEnv; +use loko::prelude::*; +use loko::random::SeededRandom; + +/// A clock stopped at one moment, for output that must be the same on every run. +struct FrozenClock(SystemTime); + +impl Clock for FrozenClock { + fn now(&self) -> SystemTime { + self.0 + } + + fn sleep(&self, _: Duration) -> impl Future { + ready(()) + } +} + +/// Everything fixed: for snapshot tests. +#[loko::bundle] +struct SnapshotPlatform { + #[loko::has(Clock)] + clock: FrozenClock, + #[loko::has(Random)] + random: SeededRandom, + #[loko::has(Env)] + env: MapEnv, + #[loko::has(Console)] + console: TestConsole, +} + +let platform = SnapshotPlatform { + clock: FrozenClock(loko::clock::from_ms(1_700_000_000_000).unwrap()), + random: SeededRandom::new(1), + env: MapEnv::new(), + console: TestConsole::new(), +}; +assert_eq!(platform.now_ms(), 1_700_000_000_000); +``` + +### 8. Wrappers + +A wrapper holds one implementation and is one itself, changing some methods. `#[loko::wraps(field: Type)]` forwards +every method it doesn't write to the field. This one speeds up any clock, test or real: + +```rust +use std::future::Future; +use std::time::{Duration, SystemTime}; + +use loko::clock::{TestClock, to_ms}; +use loko::prelude::*; + +/// Any clock, running `speed` times faster from when the wrapper is made. +struct Faster { + inner: C, + started: SystemTime, + speed: u32, +} + +#[loko::wraps(inner: C)] +impl Clock for Faster { + fn now(&self) -> SystemTime { + let elapsed = self.inner.now().duration_since(self.started).unwrap_or_default(); + self.started + elapsed * self.speed + } + + // Written too: the inner clock's `now_ms` would leave out the speed-up. + fn now_ms(&self) -> i64 { + to_ms(Clock::now(self)) + } + + fn sleep(&self, duration: Duration) -> impl Future { + self.inner.sleep(duration / self.speed) + } +} + +let real = TestClock::at_ms(0); +let fast = Faster { inner: real.clone(), started: real.now(), speed: 60 }; +real.advance(Duration::from_secs(1)); +assert_eq!(fast.now_ms(), 60_000, "a second of the inner clock is a minute"); +``` + +loko has some ready-made, and they stack: `OffsetClock` (a clock ahead or behind another, for clock-skew tests), +`CountingStore`, `ReadOnlyStore`, and `PrefixedStore`. + +```rust +use loko::store::{CountingStore, MemoryStore, PrefixedStore, ReadOnlyStore, Store}; + +#[tokio::main(flavor = "current_thread")] +async fn main() { + // Counts its calls, never writes, and stays inside one tenant's data. + let store = CountingStore::new(ReadOnlyStore::new(PrefixedStore::new(MemoryStore::new(), "memory://tenant-a/"))); + assert!(store.get("memory://tenant-b/report").await.is_err(), "outside the prefix"); + assert!(store.put("memory://tenant-a/report", vec![]).await.is_err(), "read-only"); + assert_eq!(store.counts().total(), 2); +} +``` + +### 9. Services with state + +Every service method takes `&self`, so one value can be several services (`greet(&platform)` above), clones share +state (a test advances the clock it handed to the code), and services can be used across `.await`. A service whose +state changes keeps it behind a lock or an atomic, as `TestMailer` does. Three rules: + +- **Lock only for the synchronous part.** Never hold a `std::sync::Mutex` guard or a `RefCell` borrow across `.await`: + another task using the service meanwhile would deadlock or panic. Clippy warns about both (`await_holding_lock`, + `await_holding_refcell_ref`). If state must stay locked while waiting, use an async mutex (`tokio::sync::Mutex`). +- **Counters are atomics,** which need no lock. +- **Clones share state.** Cloning a service that holds an `Arc` gives a second handle to the same state. + +### 10. In the browser + +With feature `js` on `wasm32-unknown-unknown`, `JsPlatform` uses `Date.now`, `setTimeout`, `crypto.getRandomValues`, +and the JavaScript console. The `retry` from step 1 runs unchanged: + +```rust,ignore +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +pub async fn fetch_with_retries() -> Result { + let platform = loko::JsPlatform::default(); + retry(&platform, || try_once()).await.map_err(JsValue::from) +} +``` + +## Sans-I/O + +Libraries that do no I/O of their own come in two shapes, and loko fits both: + +- **State machines driven by the caller** (quinn-proto, str0m, smoltcp): the time comes in as an argument, the next + deadline goes out, and the library never sleeps. The program around it waits with a `Clock`, and its tests run + through hours of timers at once with a `TestClock`. +- **Libraries that call traits the application implements** (delta-kernel-rs's `Engine`, rustls's time and randomness + providers): the library runs a chain of dependent steps and calls back for each read, write, or timestamp. That's the + shape loko's services serve: the library takes `&(impl Store + Clock + Random)`, and the application chooses. + +Choose the first for a protocol reacting to packets and timers, the second for reads and writes that depend on each +other. + +## Related crates + +[COMPARISON.md](COMPARISON.md) compares loko with forwarding macros (delegate, ambassador), dependency-injection crates +(fundle, shaku, nject, service-builder, dill, dependency-injector, ferrous-di, reinhardt-di), tower, and cap-std: what +each is for, when to use which, and what loko does differently. + +## Reference + +### Services + +| Service | Methods | For tests | System | JavaScript (feature `js`) | +|---|---|---|---|---| +| [`Clock`](crates/loko-clock) | `now`, `now_ms`, `sleep` | `TestClock` | `StdClock`, `TokioClock` (feature `tokio`) | `JsClock` | +| [`Random`](crates/loko-random) | `rng`, a generator of your own | `SeededRandom` | `OsRandom` (feature `getrandom`) | `JsRandom` | +| [`Env`](crates/loko-env) | `var`, `vars` | `MapEnv` | `SystemEnv` | `MapEnv` | +| [`Console`](crates/loko-console) | `print_line`, `print_error_line` | `TestConsole` | `StdConsole` | `JsConsole` | +| [`Store`](crates/loko-store) (feature `store`) | get, put, put-if-absent, upload, list, delete | `MemoryStore` | [`ObjectStoreAdapter`](crates/loko-object-store) (feature `object-store`) | the application's binding | + +### Platforms + +| Platform | Clock | Random | Env | Console | Where | +|---|---|---|---|---|---| +| `TestPlatform` | `TestClock` | `SeededRandom` | `MapEnv` | `TestConsole` | Tests, anywhere | +| `StdPlatform` | `StdClock` | `OsRandom` | `SystemEnv` | `StdConsole` | Native and WASI, without an async runtime | +| `TokioPlatform` | `TokioClock` | `OsRandom` | `SystemEnv` | `StdConsole` | Native, under tokio (feature `tokio`) | +| `JsPlatform` | `JsClock` | `JsRandom` | `MapEnv` | `JsConsole` | Browsers, Workers, and Node (feature `js`) | + +### Wrappers + +| Wrapper | What it does | +|---|---| +| `OffsetClock` | Another clock, ahead or behind by a fixed amount | +| `CountingStore` | Counts the calls of each kind it passes on | +| `ReadOnlyStore` | Reads through, refuses writes | +| `PrefixedStore` | Passes on only URIs under a prefix, refusing `..` and the rest | + +### Macros + +| Macro | What it does | +|---|---| +| `#[loko::service]` | Makes a trait a service: adds `HasX` and makes every `HasX` an `X` | +| `#[loko::bundle]` | Makes a struct each service its `#[loko::has(…)]` fields provide | +| `#[loko::wraps(field: Type)]` | Makes an `impl X for Wrapper` a wrapper: the methods written replace `X`'s, the rest forward | + +### Crates + +| Crate | What it holds | +|---|---| +| [`loko`](crates/loko) | Everything below, the platforms, and a prelude, with a feature per optional part | +| [`loko-clock`](crates/loko-clock), [`loko-random`](crates/loko-random), [`loko-env`](crates/loko-env), [`loko-console`](crates/loko-console), [`loko-store`](crates/loko-store) | One service each, with its implementations and wrappers | +| [`loko-object-store`](crates/loko-object-store) | A `Store` over an `object_store` client: S3, GCS, Azure, local files, memory | +| [`loko-macros`](crates/loko-macros) | `#[service]`, `#[loko::bundle]`, and `#[wraps]` | + +A library that needs one service can depend on its crate alone. The service crates have no dependencies of their +own (`loko-random` has `rand_core`), and no macros: only `loko` and code defining its own services use `loko-macros`. + +## Development + +```bash +cargo test --workspace --all-features # tests, offline, and this README's examples +cargo build --workspace --all-features --target wasm32-unknown-unknown # browsers and Workers +cargo build --workspace --all-features --target wasm32-wasip2 # WASI +prek run --all-files --hook-stage pre-push # formatters and linters +``` + +[CONTRIBUTING.md](CONTRIBUTING.md) is the code guide, and [spec/](spec/README.md) holds the design. + +## License + +Apache License 2.0 ([LICENSE](LICENSE)). diff --git a/clippy.toml b/clippy.toml new file mode 100644 index 0000000..731d87f --- /dev/null +++ b/clippy.toml @@ -0,0 +1,2 @@ +allow-unwrap-in-tests = true +doc-valid-idents = ["WebAssembly", ".."] diff --git a/crates/loko-clock/Cargo.toml b/crates/loko-clock/Cargo.toml new file mode 100644 index 0000000..a904384 --- /dev/null +++ b/crates/loko-clock/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "loko-clock" +description = "loko's Clock service: the time and a timer, with test, tokio, and JavaScript clocks" +readme = "README.md" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true + +[features] +default = [] +# `TokioClock`: the system clock and tokio's timer (native targets). +tokio = ["dep:tokio"] +# `JsClock`: `Date.now` and `setTimeout` (`wasm32-unknown-unknown` with a JavaScript host). +js = ["dep:js-sys", "dep:wasm-bindgen", "dep:wasm-bindgen-futures"] + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +tokio = { workspace = true, optional = true } + +[target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] +js-sys = { workspace = true, optional = true } +wasm-bindgen = { workspace = true, optional = true } +wasm-bindgen-futures = { workspace = true, optional = true } + +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt"] } + +[package.metadata.docs.rs] +all-features = true + +[lints] +workspace = true diff --git a/crates/loko-clock/README.md b/crates/loko-clock/README.md new file mode 100644 index 0000000..7092a3f --- /dev/null +++ b/crates/loko-clock/README.md @@ -0,0 +1,25 @@ +# loko-clock + +[loko](https://github.com/polythings/loko)'s `Clock` service: the time and a timer, passed to code instead of read +from the system, so tests control time and the same code runs natively, under WASI, and in browsers. + +- `Clock`: `now` (a `SystemTime`), `now_ms` (milliseconds since the Unix epoch), and `sleep(Duration)`. The future + isn't required to be `Send`. +- `TestClock`: moves only when told (`advance`, `set`) or when slept on, so a test of a retry loop runs at once. + Clones share the time. +- `StdClock`: the system clock, with a `sleep` that blocks the thread (native and WASI, no async runtime needed). +- `TokioClock` (feature `tokio`): the system clock and tokio's timer. +- `JsClock` (feature `js`, `wasm32-unknown-unknown`): `Date.now` and `setTimeout`. +- `OffsetClock`: another clock, ahead or behind by a fixed amount, for testing against clock skew. + +```rust +use std::time::Duration; + +use loko_clock::{Clock, TestClock}; + +let clock = TestClock::at_ms(1_000); +clock.advance(Duration::from_secs(1)); +assert_eq!(clock.now_ms(), 2_000); +``` + +Apache License 2.0. diff --git a/crates/loko-clock/src/js.rs b/crates/loko-clock/src/js.rs new file mode 100644 index 0000000..b830e7d --- /dev/null +++ b/crates/loko-clock/src/js.rs @@ -0,0 +1,52 @@ +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use js_sys::{Function, Promise, Reflect}; +use wasm_bindgen::{JsCast, JsValue}; + +use crate::Clock; + +/// `Date.now` and `setTimeout` of the JavaScript host (feature `js`, `wasm32-unknown-unknown`): +/// browsers, Workers, and Node. +#[derive(Clone, Copy, Debug, Default)] +pub struct JsClock; + +impl Clock for JsClock { + fn now(&self) -> SystemTime { + #[expect(clippy::cast_possible_truncation, clippy::cast_sign_loss, reason = "milliseconds since the epoch")] + let ms = js_sys::Date::now() as u64; + UNIX_EPOCH + Duration::from_millis(ms) + } + + async fn sleep(&self, duration: Duration) { + // `setTimeout` holds its delay in a signed 32-bit integer and fires at once above it, so a + // longer sleep waits in steps of at most that. + const STEP_MS: u128 = i32::MAX as u128; + let mut left = duration.as_millis(); + loop { + let step = left.min(STEP_MS); + timeout(u32::try_from(step).unwrap_or(u32::MAX)).await; + left -= step; + if left == 0 { + break; + } + } + } +} + +/// Waits `ms` milliseconds on `setTimeout`, or not at all on a host without timers. +async fn timeout(ms: u32) { + let ms = JsValue::from_f64(f64::from(ms)); + let promise = Promise::new(&mut |resolve, _| { + let global = js_sys::global(); + match Reflect::get(&global, &"setTimeout".into()).and_then(JsCast::dyn_into::) { + Ok(set_timeout) => { + let _ = set_timeout.call2(&global, &resolve, &ms); + } + // A host without timers: resolve at once. + Err(_) => { + let _ = resolve.call0(&JsValue::NULL); + } + } + }); + let _ = wasm_bindgen_futures::JsFuture::from(promise).await; +} diff --git a/crates/loko-clock/src/lib.rs b/crates/loko-clock/src/lib.rs new file mode 100644 index 0000000..098990b --- /dev/null +++ b/crates/loko-clock/src/lib.rs @@ -0,0 +1,121 @@ +//! loko's `Clock` service: the time and a timer, passed to code instead of read from the system, so +//! tests control time and the same code runs natively and in WebAssembly. +//! +//! - [`Clock`]: `now`, `now_ms`, and `sleep`. +//! - [`TestClock`]: a clock that moves only when told (or when slept on), for tests. +//! - `StdClock`: the system clock with a blocking sleep, for programs without an async runtime. +//! - `TokioClock` (feature `tokio`): the system clock and tokio's timer, on native targets. +//! - `JsClock` (feature `js`): `Date.now` and `setTimeout`, in browsers, Workers, and Node. +//! - [`OffsetClock`]: another clock, ahead or behind by a fixed amount. +//! +//! ``` +//! use std::time::Duration; +//! +//! use loko_clock::{Clock, TestClock}; +//! +//! let clock = TestClock::at_ms(1_000); +//! clock.advance(Duration::from_secs(1)); +//! assert_eq!(clock.now_ms(), 2_000); +//! ``` + +use std::future::Future; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +#[cfg(all(feature = "js", all(target_arch = "wasm32", target_os = "unknown")))] +mod js; +mod offset; +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +mod system; +mod testing; +#[cfg(all(feature = "tokio", not(target_arch = "wasm32")))] +mod tokio_clock; + +#[cfg(all(feature = "js", all(target_arch = "wasm32", target_os = "unknown")))] +pub use js::JsClock; +pub use offset::OffsetClock; +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +pub use system::StdClock; +pub use testing::TestClock; +#[cfg(all(feature = "tokio", not(target_arch = "wasm32")))] +pub use tokio_clock::TokioClock; + +/// The time and a timer. +/// +/// The future isn't required to be `Send`, so a clock can wrap JavaScript timers. +pub trait Clock { + /// The current time. + fn now(&self) -> SystemTime; + + /// Milliseconds since the Unix epoch (negative before it). + fn now_ms(&self) -> i64 { + to_ms(self.now()) + } + + /// Waits `duration`. + fn sleep(&self, duration: Duration) -> impl Future; +} + +/// A type that holds a [`Clock`], and so is one: `#[loko::bundle]` implements this for a field, +/// and `#[loko::wraps]` for a wrapper that changes some methods. It's what `#[loko::service]` +/// generates, written out so this crate needs no macros. +pub trait HasClock { + /// The service held. + type Clock: Clock; + + /// The [`Clock`] this holds. + fn clock(&self) -> &Self::Clock; + + /// Forwards [`Clock::now`] to the held service; a `#[loko::wraps]` implementation may replace it. + #[doc(hidden)] + #[inline] + fn wrapped_now(&self) -> SystemTime { + self.clock().now() + } + + /// Forwards [`Clock::now_ms`] to the held service; a `#[loko::wraps]` implementation may replace it. + #[doc(hidden)] + #[inline] + fn wrapped_now_ms(&self) -> i64 { + self.clock().now_ms() + } + + /// Forwards [`Clock::sleep`] to the held service; a `#[loko::wraps]` implementation may replace it. + #[doc(hidden)] + #[inline] + fn wrapped_sleep(&self, duration: Duration) -> impl Future { + self.clock().sleep(duration) + } +} + +impl Clock for H { + #[inline] + fn now(&self) -> SystemTime { + self.wrapped_now() + } + + #[inline] + fn now_ms(&self) -> i64 { + self.wrapped_now_ms() + } + + #[inline] + fn sleep(&self, duration: Duration) -> impl Future { + self.wrapped_sleep(duration) + } +} + +/// Milliseconds from the Unix epoch to `time` (negative before it), saturating. +pub fn to_ms(time: SystemTime) -> i64 { + match time.duration_since(UNIX_EPOCH) { + Ok(since) => i64::try_from(since.as_millis()).unwrap_or(i64::MAX), + Err(before) => i64::try_from(before.duration().as_millis()).map_or(i64::MIN, |ms| -ms), + } +} + +/// The time `ms` milliseconds after the Unix epoch (before it, if negative), or `None` where the +/// platform can't hold it: WebAssembly (WASI and `wasm32-unknown-unknown`) has no times before the +/// epoch. +pub fn from_ms(ms: i64) -> Option { + let magnitude = Duration::from_millis(ms.unsigned_abs()); + if ms >= 0 { UNIX_EPOCH.checked_add(magnitude) } else { UNIX_EPOCH.checked_sub(magnitude) } +} diff --git a/crates/loko-clock/src/offset.rs b/crates/loko-clock/src/offset.rs new file mode 100644 index 0000000..3249fe2 --- /dev/null +++ b/crates/loko-clock/src/offset.rs @@ -0,0 +1,69 @@ +use std::time::{Duration, SystemTime}; + +use crate::{Clock, HasClock, to_ms}; + +/// Another clock, ahead or behind by a fixed amount: what a machine whose clock is off reads, for +/// testing code against clock skew. Sleeping is the other clock's. +#[derive(Clone, Debug, Default)] +pub struct OffsetClock { + inner: C, + offset: Duration, + behind: bool, +} + +impl OffsetClock { + /// `inner`, ahead by `offset`. + pub fn ahead(inner: C, offset: Duration) -> Self { + Self { inner, offset, behind: false } + } + + /// `inner`, behind by `offset`. + pub fn behind(inner: C, offset: Duration) -> Self { + Self { inner, offset, behind: true } + } + + /// The clock this one is offset from. + pub fn inner(&self) -> &C { + &self.inner + } +} + +impl HasClock for OffsetClock { + type Clock = C; + + fn clock(&self) -> &C { + &self.inner + } + + fn wrapped_now(&self) -> SystemTime { + let now = self.inner.now(); + let moved = if self.behind { now.checked_sub(self.offset) } else { now.checked_add(self.offset) }; + // Past what `SystemTime` holds: stay at the other clock's time rather than panic. + moved.unwrap_or(now) + } + + // The other clock's `now_ms` would leave out the offset. + fn wrapped_now_ms(&self) -> i64 { + to_ms(Clock::now(self)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestClock; + + #[test] + fn offset_clocks_follow_the_other_clock() { + let clock = TestClock::at_ms(10_000); + let ahead = OffsetClock::ahead(clock.clone(), Duration::from_secs(2)); + let behind = OffsetClock::behind(clock.clone(), Duration::from_secs(3)); + assert_eq!((ahead.now_ms(), behind.now_ms()), (12_000, 7_000)); + clock.advance(Duration::from_secs(1)); + assert_eq!((ahead.now_ms(), behind.now_ms()), (13_000, 8_000)); + + let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); + assert!(std::pin::pin!(ahead.sleep(Duration::from_secs(5))).as_mut().poll(&mut cx).is_ready()); + assert_eq!(clock.now_ms(), 16_000, "sleeping is the other clock's"); + } +} diff --git a/crates/loko-clock/src/system.rs b/crates/loko-clock/src/system.rs new file mode 100644 index 0000000..a714614 --- /dev/null +++ b/crates/loko-clock/src/system.rs @@ -0,0 +1,35 @@ +use std::future::{Future, ready}; +use std::time::{Duration, SystemTime}; + +use crate::Clock; + +/// The system clock for programs without an async runtime (native targets and WASI). Its `sleep` +/// blocks the calling thread before the future is returned, so in async code use `TokioClock` (or +/// another runtime's clock) instead. +#[derive(Clone, Copy, Debug, Default)] +pub struct StdClock; + +impl Clock for StdClock { + fn now(&self) -> SystemTime { + SystemTime::now() + } + + fn sleep(&self, duration: Duration) -> impl Future { + std::thread::sleep(duration); + ready(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_std_clock_tells_time_and_blocks() { + let before = StdClock.now_ms(); + let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); + let mut sleep = std::pin::pin!(StdClock.sleep(Duration::from_millis(5))); + assert!(sleep.as_mut().poll(&mut cx).is_ready()); + assert!(StdClock.now_ms() >= before + 5); + } +} diff --git a/crates/loko-clock/src/testing.rs b/crates/loko-clock/src/testing.rs new file mode 100644 index 0000000..6b28fec --- /dev/null +++ b/crates/loko-clock/src/testing.rs @@ -0,0 +1,85 @@ +use std::future::{Future, ready}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use crate::{Clock, from_ms}; + +/// A clock that moves only when told, or when slept on (sleeping returns at once, the clock moved +/// forward by the sleep). Clones share the same time. The idea is ZIO's `TestClock`. +#[derive(Clone, Debug)] +pub struct TestClock { + now: Arc>, +} + +impl TestClock { + /// A clock at `now`. + pub fn new(now: SystemTime) -> Self { + Self { now: Arc::new(Mutex::new(now)) } + } + + /// A clock `ms` milliseconds after the Unix epoch. + /// + /// # Panics + /// + /// If the platform can't hold that time ([`from_ms`]): before the epoch on WebAssembly (WASI and + /// `wasm32-unknown-unknown`). + pub fn at_ms(ms: i64) -> Self { + Self::new(from_ms(ms).expect("a time this platform can hold")) + } + + /// Moves the clock forward by `duration`. + pub fn advance(&self, duration: Duration) { + *self.now.lock().unwrap_or_else(PoisonError::into_inner) += duration; + } + + /// Sets the clock to `now`, forward or back. + pub fn set(&self, now: SystemTime) { + *self.now.lock().unwrap_or_else(PoisonError::into_inner) = now; + } +} + +impl Default for TestClock { + /// A clock at the Unix epoch. + fn default() -> Self { + Self::new(UNIX_EPOCH) + } +} + +impl Clock for TestClock { + fn now(&self) -> SystemTime { + *self.now.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn sleep(&self, duration: Duration) -> impl Future { + self.advance(duration); + ready(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_clocks_move_when_told() { + let clock = TestClock::at_ms(5_000); + let other = clock.clone(); + other.advance(Duration::from_millis(250)); + assert_eq!(clock.now_ms(), 5_250, "clones share the time"); + // Times before the epoch exist natively, not on WebAssembly. + if let Some(before) = from_ms(-1_500) { + clock.set(before); + assert_eq!(clock.now_ms(), -1_500); + } + assert_eq!(TestClock::default().now(), UNIX_EPOCH); + } + + #[test] + fn sleeping_moves_a_test_clock() { + let clock = TestClock::at_ms(0); + let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); + let mut sleep = std::pin::pin!(clock.sleep(Duration::from_secs(3))); + assert!(sleep.as_mut().poll(&mut cx).is_ready()); + assert_eq!(clock.now_ms(), 3_000); + } +} diff --git a/crates/loko-clock/src/tokio_clock.rs b/crates/loko-clock/src/tokio_clock.rs new file mode 100644 index 0000000..986a985 --- /dev/null +++ b/crates/loko-clock/src/tokio_clock.rs @@ -0,0 +1,31 @@ +use std::future::Future; +use std::time::{Duration, SystemTime}; + +use crate::Clock; + +/// The system clock and tokio's timer (feature `tokio`, native targets). +#[derive(Clone, Copy, Debug, Default)] +pub struct TokioClock; + +impl Clock for TokioClock { + fn now(&self) -> SystemTime { + SystemTime::now() + } + + fn sleep(&self, duration: Duration) -> impl Future { + tokio::time::sleep(duration) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn the_tokio_clock_tells_time_and_waits() { + let before = TokioClock.now_ms(); + assert!(before > 1_700_000_000_000, "after November 2023"); + TokioClock.sleep(Duration::from_millis(5)).await; + assert!(TokioClock.now_ms() >= before + 5); + } +} diff --git a/crates/loko-console/Cargo.toml b/crates/loko-console/Cargo.toml new file mode 100644 index 0000000..872c5c2 --- /dev/null +++ b/crates/loko-console/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "loko-console" +description = "loko's Console service: lines to standard output and error, captured in tests, or to the JavaScript console" +readme = "README.md" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true + +[features] +default = [] +# `JsConsole`: `console.log` and `console.error` (`wasm32-unknown-unknown` with a JavaScript host). +js = ["dep:js-sys", "dep:wasm-bindgen"] + +[target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] +js-sys = { workspace = true, optional = true } +wasm-bindgen = { workspace = true, optional = true } + +[package.metadata.docs.rs] +all-features = true + +[lints] +workspace = true diff --git a/crates/loko-console/README.md b/crates/loko-console/README.md new file mode 100644 index 0000000..995962e --- /dev/null +++ b/crates/loko-console/README.md @@ -0,0 +1,20 @@ +# loko-console + +[loko](https://github.com/polythings/loko)'s `Console` service: lines to standard output and standard error, passed to +code instead of printed directly, so tests read what was printed and code in a browser logs to its console. + +- `Console`: `print_line` and `print_error_line`. +- `StdConsole`: the process's standard output and error. +- `TestConsole`: keeps the lines (`lines`, `error_lines`). Clones share them. +- `JsConsole` (feature `js`, `wasm32-unknown-unknown`): `console.log` and `console.error`. + +```rust +use loko_console::{Console, TestConsole}; + +let console = TestConsole::new(); +console.print_line("done")?; +assert_eq!(console.lines(), ["done"]); +# Ok::<(), std::io::Error>(()) +``` + +Apache License 2.0. diff --git a/crates/loko-console/src/js.rs b/crates/loko-console/src/js.rs new file mode 100644 index 0000000..505a7b9 --- /dev/null +++ b/crates/loko-console/src/js.rs @@ -0,0 +1,33 @@ +use std::io; + +use js_sys::{Function, Reflect}; +use wasm_bindgen::JsCast; + +use crate::Console; + +/// `console.log` and `console.error` of the JavaScript host (feature `js`, `wasm32-unknown-unknown`): +/// browsers, Workers, and Node. +#[derive(Clone, Copy, Debug, Default)] +pub struct JsConsole; + +impl JsConsole { + fn call(method: &str, line: &str) -> io::Result<()> { + let console = Reflect::get(&js_sys::global(), &"console".into()) + .map_err(|_| io::Error::other("the host has no console"))?; + let f: Function = Reflect::get(&console, &method.into()) + .ok() + .and_then(|f| f.dyn_into().ok()) + .ok_or_else(|| io::Error::other(format!("the console has no {method}")))?; + f.call1(&console, &line.into()).map(drop).map_err(|e| io::Error::other(format!("console.{method}: {e:?}"))) + } +} + +impl Console for JsConsole { + fn print_line(&self, line: &str) -> io::Result<()> { + Self::call("log", line) + } + + fn print_error_line(&self, line: &str) -> io::Result<()> { + Self::call("error", line) + } +} diff --git a/crates/loko-console/src/lib.rs b/crates/loko-console/src/lib.rs new file mode 100644 index 0000000..24ee6ab --- /dev/null +++ b/crates/loko-console/src/lib.rs @@ -0,0 +1,76 @@ +//! loko's `Console` service: lines to standard output and standard error, passed to code instead of +//! printed directly, so tests read what was printed and code in a browser logs to its console. +//! +//! - [`Console`]: `print_line` and `print_error_line`. +//! - `StdConsole`: the process's standard output and error (native targets and WASI). +//! - [`TestConsole`]: keeps the lines, for tests. +//! - `JsConsole` (feature `js`): `console.log` and `console.error`, in browsers, Workers, and Node. +//! +//! ``` +//! use loko_console::{Console, TestConsole}; +//! +//! let console = TestConsole::new(); +//! console.print_line("done")?; +//! assert_eq!(console.lines(), ["done"]); +//! # Ok::<(), std::io::Error>(()) +//! ``` + +use std::io; + +#[cfg(all(feature = "js", all(target_arch = "wasm32", target_os = "unknown")))] +mod js; +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +mod system; +mod testing; + +#[cfg(all(feature = "js", all(target_arch = "wasm32", target_os = "unknown")))] +pub use js::JsConsole; +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +pub use system::StdConsole; +pub use testing::TestConsole; + +/// Lines of text to standard output and standard error. +pub trait Console { + /// Writes `line` and a newline to standard output. + fn print_line(&self, line: &str) -> io::Result<()>; + + /// Writes `line` and a newline to standard error. + fn print_error_line(&self, line: &str) -> io::Result<()>; +} + +/// A type that holds a [`Console`], and so is one: `#[loko::bundle]` implements this for a field, +/// and `#[loko::wraps]` for a wrapper that changes some methods. It's what `#[loko::service]` +/// generates, written out so this crate needs no macros. +pub trait HasConsole { + /// The service held. + type Console: Console; + + /// The [`Console`] this holds. + fn console(&self) -> &Self::Console; + + /// Forwards [`Console::print_line`] to the held service; a `#[loko::wraps]` implementation may replace it. + #[doc(hidden)] + #[inline] + fn wrapped_print_line(&self, line: &str) -> io::Result<()> { + self.console().print_line(line) + } + + /// Forwards [`Console::print_error_line`] to the held service; a `#[loko::wraps]` implementation may replace it. + #[doc(hidden)] + #[inline] + fn wrapped_print_error_line(&self, line: &str) -> io::Result<()> { + self.console().print_error_line(line) + } +} + +impl Console for H { + #[inline] + fn print_line(&self, line: &str) -> io::Result<()> { + self.wrapped_print_line(line) + } + + #[inline] + fn print_error_line(&self, line: &str) -> io::Result<()> { + self.wrapped_print_error_line(line) + } +} diff --git a/crates/loko-console/src/system.rs b/crates/loko-console/src/system.rs new file mode 100644 index 0000000..bf3d288 --- /dev/null +++ b/crates/loko-console/src/system.rs @@ -0,0 +1,27 @@ +use std::io::{self, Write}; + +use crate::Console; + +/// The process's standard output and standard error. +#[derive(Clone, Copy, Debug, Default)] +pub struct StdConsole; + +impl Console for StdConsole { + fn print_line(&self, line: &str) -> io::Result<()> { + writeln!(io::stdout().lock(), "{line}") + } + + fn print_error_line(&self, line: &str) -> io::Result<()> { + writeln!(io::stderr().lock(), "{line}") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_std_console_writes() { + StdConsole.print_line("loko-console test line").unwrap(); + } +} diff --git a/crates/loko-console/src/testing.rs b/crates/loko-console/src/testing.rs new file mode 100644 index 0000000..df33dbb --- /dev/null +++ b/crates/loko-console/src/testing.rs @@ -0,0 +1,60 @@ +use std::io; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +use crate::Console; + +/// Keeps the lines written, for tests. Clones share the lines. +#[derive(Clone, Debug, Default)] +pub struct TestConsole { + out: Arc>>, + err: Arc>>, +} + +fn lock(lines: &Mutex>) -> MutexGuard<'_, Vec> { + lines.lock().unwrap_or_else(PoisonError::into_inner) +} + +impl TestConsole { + /// No lines yet. + pub fn new() -> Self { + Self::default() + } + + /// The lines written to standard output, in order. + pub fn lines(&self) -> Vec { + lock(&self.out).clone() + } + + /// The lines written to standard error, in order. + pub fn error_lines(&self) -> Vec { + lock(&self.err).clone() + } +} + +impl Console for TestConsole { + fn print_line(&self, line: &str) -> io::Result<()> { + lock(&self.out).push(line.to_string()); + Ok(()) + } + + fn print_error_line(&self, line: &str) -> io::Result<()> { + lock(&self.err).push(line.to_string()); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_consoles_keep_lines() { + let console = TestConsole::new(); + let shared = console.clone(); + console.print_line("a").unwrap(); + shared.print_error_line("oops").unwrap(); + console.print_line("b").unwrap(); + assert_eq!(console.lines(), ["a", "b"]); + assert_eq!(console.error_lines(), ["oops"]); + } +} diff --git a/crates/loko-env/Cargo.toml b/crates/loko-env/Cargo.toml new file mode 100644 index 0000000..8390cd6 --- /dev/null +++ b/crates/loko-env/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "loko-env" +description = "loko's Env service: environment variables, from the process or from a map" +readme = "README.md" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true + +[package.metadata.docs.rs] +all-features = true + +[lints] +workspace = true diff --git a/crates/loko-env/README.md b/crates/loko-env/README.md new file mode 100644 index 0000000..7a5a372 --- /dev/null +++ b/crates/loko-env/README.md @@ -0,0 +1,17 @@ +# loko-env + +[loko](https://github.com/polythings/loko)'s `Env` service: environment variables, passed to code instead of read from +the process, so tests set their own and code in a browser (which has none) reads them from a map. + +- `Env`: `var` and `vars`. +- `SystemEnv`: the process's variables. +- `MapEnv`: variables in a map, for tests and for hosts without a process environment. Clones share the map. + +```rust +use loko_env::{Env, MapEnv}; + +let env = MapEnv::new().with("REGION", "us-east-2"); +assert_eq!(env.var("REGION").as_deref(), Some("us-east-2")); +``` + +Apache License 2.0. diff --git a/crates/loko-env/src/lib.rs b/crates/loko-env/src/lib.rs new file mode 100644 index 0000000..0455ccd --- /dev/null +++ b/crates/loko-env/src/lib.rs @@ -0,0 +1,68 @@ +//! loko's `Env` service: environment variables, passed to code instead of read from the process, so +//! tests set their own and code in a browser (which has none) gets them from a map. +//! +//! - [`Env`]: `var` and `vars`. +//! - `SystemEnv`: the process's variables (native targets and WASI). +//! - [`MapEnv`]: variables in a map, for tests and for hosts without a process environment. +//! +//! ``` +//! use loko_env::{Env, MapEnv}; +//! +//! let env = MapEnv::new().with("REGION", "us-east-2"); +//! assert_eq!(env.var("REGION").as_deref(), Some("us-east-2")); +//! assert_eq!(env.var("MISSING"), None); +//! ``` + +mod map; +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +mod system; + +pub use map::MapEnv; +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +pub use system::SystemEnv; + +/// Environment variables. +pub trait Env { + /// The variable `name`, or `None` if it isn't set (or isn't valid Unicode). + fn var(&self, name: &str) -> Option; + + /// Every variable, as `(name, value)` pairs. + fn vars(&self) -> Vec<(String, String)>; +} + +/// A type that holds a [`Env`], and so is one: `#[loko::bundle]` implements this for a field, +/// and `#[loko::wraps]` for a wrapper that changes some methods. It's what `#[loko::service]` +/// generates, written out so this crate needs no macros. +pub trait HasEnv { + /// The service held. + type Env: Env; + + /// The [`Env`] this holds. + fn env(&self) -> &Self::Env; + + /// Forwards [`Env::var`] to the held service; a `#[loko::wraps]` implementation may replace it. + #[doc(hidden)] + #[inline] + fn wrapped_var(&self, name: &str) -> Option { + self.env().var(name) + } + + /// Forwards [`Env::vars`] to the held service; a `#[loko::wraps]` implementation may replace it. + #[doc(hidden)] + #[inline] + fn wrapped_vars(&self) -> Vec<(String, String)> { + self.env().vars() + } +} + +impl Env for H { + #[inline] + fn var(&self, name: &str) -> Option { + self.wrapped_var(name) + } + + #[inline] + fn vars(&self) -> Vec<(String, String)> { + self.wrapped_vars() + } +} diff --git a/crates/loko-env/src/map.rs b/crates/loko-env/src/map.rs new file mode 100644 index 0000000..6c31b57 --- /dev/null +++ b/crates/loko-env/src/map.rs @@ -0,0 +1,72 @@ +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +use crate::Env; + +/// Environment variables in a map. Clones share the map. +#[derive(Clone, Debug, Default)] +pub struct MapEnv { + vars: Arc>>, +} + +impl MapEnv { + /// No variables. + pub fn new() -> Self { + Self::default() + } + + /// These variables and `name` set to `value`. + #[must_use] + pub fn with(self, name: impl Into, value: impl Into) -> Self { + self.set(name, value); + self + } + + /// Sets `name` to `value`. + pub fn set(&self, name: impl Into, value: impl Into) { + self.map().insert(name.into(), value.into()); + } + + /// Unsets `name`. + pub fn remove(&self, name: &str) { + self.map().remove(name); + } + + fn map(&self) -> MutexGuard<'_, BTreeMap> { + self.vars.lock().unwrap_or_else(PoisonError::into_inner) + } +} + +impl, V: Into> FromIterator<(K, V)> for MapEnv { + fn from_iter>(iter: I) -> Self { + let env = Self::new(); + for (k, v) in iter { + env.set(k, v); + } + env + } +} + +impl Env for MapEnv { + fn var(&self, name: &str) -> Option { + self.map().get(name).cloned() + } + + fn vars(&self) -> Vec<(String, String)> { + self.map().iter().map(|(k, v)| (k.clone(), v.clone())).collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn map_envs_set_and_unset() { + let env: MapEnv = [("A", "1"), ("B", "2")].into_iter().collect(); + let shared = env.clone(); + shared.set("C", "3"); + shared.remove("A"); + assert_eq!(env.vars(), [("B".to_string(), "2".to_string()), ("C".to_string(), "3".to_string())]); + } +} diff --git a/crates/loko-env/src/system.rs b/crates/loko-env/src/system.rs new file mode 100644 index 0000000..2c5924a --- /dev/null +++ b/crates/loko-env/src/system.rs @@ -0,0 +1,29 @@ +use crate::Env; + +/// The process's environment variables (`std::env`), on native targets and WASI. Variables whose +/// name or value isn't valid Unicode are left out, as `var` leaves them out. +#[derive(Clone, Copy, Debug, Default)] +pub struct SystemEnv; + +impl Env for SystemEnv { + fn var(&self, name: &str) -> Option { + std::env::var(name).ok() + } + + fn vars(&self) -> Vec<(String, String)> { + // `std::env::vars` panics on the first variable that isn't Unicode. + std::env::vars_os().filter_map(|(k, v)| Some((k.into_string().ok()?, v.into_string().ok()?))).collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_system_env_reads_the_process() { + // PATH is set wherever these tests run. + assert!(SystemEnv.var("PATH").is_some()); + assert!(SystemEnv.vars().iter().any(|(k, _)| k == "PATH")); + } +} diff --git a/crates/loko-macros/Cargo.toml b/crates/loko-macros/Cargo.toml new file mode 100644 index 0000000..d618e27 --- /dev/null +++ b/crates/loko-macros/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "loko-macros" +description = "loko's macros: #[service] makes a trait a service, #[bundle] wires services into a struct, #[wraps] makes wrappers" +readme = "README.md" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true + +[lib] +proc-macro = true + +[dependencies] +proc-macro2.workspace = true +quote.workspace = true +syn.workspace = true + +[lints] +workspace = true diff --git a/crates/loko-macros/README.md b/crates/loko-macros/README.md new file mode 100644 index 0000000..405e546 --- /dev/null +++ b/crates/loko-macros/README.md @@ -0,0 +1,40 @@ +# loko-macros + +[loko](https://github.com/polythings/loko)'s macros. Use them through the `loko` crate. + +- `#[service]` on a trait `X` adds a trait `HasX` (a type that holds an `X`, with an accessor) and implements `X` for + every `HasX`, forwarding each method, associated type, and constant. Methods take `&self`; the trait can't be + generic. +- `#[loko::bundle]` implements `HasX` for each `#[loko::has(X, …)]` on a struct's fields, so the struct is each of + those services. It names `HasX` next to the path given: `#[loko::has(loko::Clock)]` uses `loko::HasClock`, and a bare + `#[loko::has(Clock)]` needs `HasClock` in scope (`loko::prelude::*` brings in both). +- `#[wraps(field: Type)]` on `impl X for Wrapper` makes a wrapper: the methods written replace `X`'s, and the rest + forward to `self.field`. A wrapper keeps the inner service's associated types. + +```rust +use loko::prelude::*; + +#[loko::service] +trait Greeter { + fn greet(&self, name: &str) -> String; +} + +#[loko::bundle] +struct App { + #[loko::has(Clock, Random)] + platform: TestPlatform, + #[loko::has(Greeter)] + greeter: G, +} + +struct Excited(G); + +#[loko::wraps(0: G)] +impl Greeter for Excited { + fn greet(&self, name: &str) -> String { + format!("{}!", self.0.greet(name)) + } +} +``` + +Apache License 2.0. diff --git a/crates/loko-macros/src/lib.rs b/crates/loko-macros/src/lib.rs new file mode 100644 index 0000000..6898657 --- /dev/null +++ b/crates/loko-macros/src/lib.rs @@ -0,0 +1,445 @@ +//! loko's macros, used by its service crates and re-exported by `loko`. +//! +//! - `#[service]` on a trait `X` adds a trait `HasX` (a type that holds an `X`) and implements `X` +//! for every `HasX`. Each method goes through a `HasX` method (`wrapped_`) that forwards +//! to the held service unless the implementation says otherwise. +//! - `#[bundle]` on a struct implements `HasX` for its fields marked `#[loko::has(X, …)]`, so the +//! struct is every service its fields hold. +//! - `#[wraps(field: Type)]` on an `impl X for Wrapper` implements `HasX` instead: the methods +//! written replace the forwarding ones, and the rest forward to the field. +//! +//! The `HasX` traits follow RIO's "Has" type classes, and choosing a field to forward to with an attribute +//! follows ambassador (`PRIOR_ART.md`). + +use proc_macro::TokenStream; +use proc_macro2::{Span, TokenStream as TokenStream2}; +use quote::{ToTokens, format_ident, quote}; +use syn::parse::{Parse, ParseStream}; +use syn::punctuated::Punctuated; +use syn::spanned::Spanned; +use syn::visit_mut::VisitMut; +use syn::{ + Error, FnArg, Ident, ImplItem, ItemImpl, ItemTrait, Member, Pat, PatIdent, Path, ReturnType, Signature, Token, + TraitItem, TraitItemFn, Type, TypePath, parse_macro_input, parse_quote, +}; + +/// Makes a trait a loko service: adds `Has` and implements the trait for every type that +/// has one. Methods must take `&self`, so one value can be several services and clones can share +/// state; an implementation whose state changes keeps it behind a lock or an atomic, never held +/// across `.await`. The trait can't be generic. +#[proc_macro_attribute] +pub fn service(args: TokenStream, input: TokenStream) -> TokenStream { + if !args.is_empty() { + return Error::new(Span::call_site(), "#[service] takes no arguments").to_compile_error().into(); + } + let item = parse_macro_input!(input as ItemTrait); + match expand_service(&item) { + Ok(extra) => quote! { #item #extra }.into(), + Err(e) => { + let e = e.to_compile_error(); + quote! { #item #e }.into() + } + } +} + +/// The accessor of service `name`: `Clock` → `clock`, `ObjectStore` → `object_store`, +/// `HTTPClient` → `http_client`. A word starts at a capital after a lowercase letter or digit, or at +/// the last capital of a run followed by a lowercase letter. +fn accessor(name: &Ident) -> Ident { + let chars: Vec = name.to_string().chars().collect(); + let mut snake = String::new(); + for (i, &c) in chars.iter().enumerate() { + if c.is_uppercase() && i > 0 { + let after_lower = chars[i - 1].is_lowercase() || chars[i - 1].is_ascii_digit(); + let ends_run = chars[i - 1].is_uppercase() && chars.get(i + 1).is_some_and(|n| n.is_lowercase()); + if after_lower || ends_run { + snake.push('_'); + } + } + snake.extend(c.to_lowercase()); + } + Ident::new(&snake, name.span()) +} + +/// The `HasX` method behind method `method` of service `X`. +fn wrapped(method: &Ident) -> Ident { + format_ident!("wrapped_{}", method) +} + +/// `Has`, at the same path: `loko::Clock` → `loko::HasClock`. +fn has_path(service: &Path) -> syn::Result<(Path, Ident)> { + let name = service.segments.last().ok_or_else(|| Error::new(service.span(), "a service path"))?.ident.clone(); + let mut has = service.clone(); + if let Some(last) = has.segments.last_mut() { + last.ident = format_ident!("Has{}", name); + last.arguments = syn::PathArguments::None; + } + Ok((has, name)) +} + +/// Rewrites `Self::Assoc` to the held service's `Assoc`, for signatures moved into `HasX`, where +/// `Self` is the holder. `Self::X` (the `HasX` type itself) stays. +struct HeldAssoc<'a> { + has: &'a Path, + name: &'a Ident, +} + +impl VisitMut for HeldAssoc<'_> { + fn visit_type_path_mut(&mut self, path: &mut TypePath) { + let segments = &path.path.segments; + if path.qself.is_none() + && segments.len() >= 2 + && segments[0].ident == "Self" + && segments[0].arguments.is_none() + && segments[1].ident != *self.name + { + let (has, name) = (self.has, self.name); + let rest: Vec<_> = segments.iter().skip(1).collect(); + *path = parse_quote! { <::#name as #name>::#(#rest)::* }; + } + syn::visit_mut::visit_type_path_mut(self, path); + } +} + +/// Finds `Self` used as a type on its own (`-> Self`, `Option`), not as `Self::Assoc`. +#[derive(Default)] +struct BareSelf(Option); + +impl syn::visit::Visit<'_> for BareSelf { + fn visit_type_path(&mut self, path: &TypePath) { + if path.qself.is_none() && path.path.segments.len() == 1 && path.path.segments[0].ident == "Self" { + self.0.get_or_insert(path.span()); + } + syn::visit::visit_type_path(self, path); + } +} + +/// An error if `sig` uses `Self` as a type: the holder of a service isn't the service held, so a +/// forwarded `Self` would be the wrong type. +fn reject_bare_self(sig: &Signature) -> syn::Result<()> { + let mut found = BareSelf::default(); + for input in sig.inputs.iter().skip(1) { + syn::visit::Visit::visit_fn_arg(&mut found, input); + } + syn::visit::Visit::visit_return_type(&mut found, &sig.output); + match found.0 { + Some(span) => Err(Error::new( + span, + "loko service methods can't take or return `Self`: a type holding the service isn't the service", + )), + None => Ok(()), + } +} + +/// Names the arguments of `sig` (after `&self`) `arg0`, `arg1`, …, so a body can pass them on. +fn name_arguments(sig: &mut Signature) -> syn::Result> { + match sig.inputs.first() { + Some(FnArg::Receiver(r)) if r.reference.is_some() && r.mutability.is_none() => {} + _ => return Err(Error::new(sig.span(), "loko service methods take `&self`")), + } + let mut args = vec![]; + for (i, input) in sig.inputs.iter_mut().skip(1).enumerate() { + let FnArg::Typed(typed) = input else { continue }; + let ident = format_ident!("arg{}", i); + *typed.pat = + Pat::Ident(PatIdent { attrs: vec![], by_ref: None, mutability: None, ident: ident.clone(), subpat: None }); + args.push(ident); + } + Ok(args) +} + +/// The `#[cfg]` and `#[cfg_attr]` attributes of a trait item, which its generated items need too. +fn cfgs(attrs: &[syn::Attribute]) -> Vec<&syn::Attribute> { + attrs.iter().filter(|a| a.path().is_ident("cfg") || a.path().is_ident("cfg_attr")).collect() +} + +/// `sig { call }`, awaiting the call in an async method. +fn method(sig: &Signature, call: &TokenStream2) -> TokenStream2 { + let body = if sig.asyncness.is_some() { + quote! { #call.await } + } else { + call.clone() + }; + match sig.output { + ReturnType::Default => quote! { #[inline] #sig { #body; } }, + ReturnType::Type(..) => quote! { #[inline] #sig { #body } }, + } +} + +fn expand_service(item: &ItemTrait) -> syn::Result { + if !item.generics.params.is_empty() { + return Err(Error::new(item.generics.span(), "a loko service can't be generic")); + } + if item.unsafety.is_some() { + return Err(Error::new(item.span(), "a loko service can't be an unsafe trait")); + } + let vis = &item.vis; + let name = &item.ident; + let has: Path = format_ident!("Has{}", name).into(); + let get = accessor(name); + // A name for the blanket impl's type that a method's own generics won't use. + let holder = format_ident!("Loko{}Holder", name); + let held = quote! { <<#holder as #has>::#name as #name> }; + + let (mut has_items, mut impl_items) = (vec![], vec![]); + for trait_item in &item.items { + match trait_item { + TraitItem::Type(t) => { + let (ident, cfgs) = (&t.ident, cfgs(&t.attrs)); + let (_, ty_generics, where_clause) = t.generics.split_for_impl(); + impl_items + .push(quote! { #(#cfgs)* type #ident #ty_generics = #held::#ident #ty_generics #where_clause; }); + } + TraitItem::Const(c) => { + let (ident, ty, cfgs) = (&c.ident, &c.ty, cfgs(&c.attrs)); + impl_items.push(quote! { #(#cfgs)* const #ident: #ty = #held::#ident; }); + } + TraitItem::Fn(f) => { + let (has_item, impl_item) = forward_fn(f, &has, name, &get, &holder)?; + has_items.push(has_item); + impl_items.push(impl_item); + } + other => return Err(Error::new(other.span(), "loko services can hold types, constants, and methods")), + } + } + + let doc = format!( + "A type that holds a [`{name}`], and so is one: `#[loko::bundle]` implements this for a field, \ + and `#[loko::wraps]` for a wrapper that changes some methods." + ); + let get_doc = format!("The [`{name}`] this holds."); + let supertraits = &item.supertraits; + let bound = if supertraits.is_empty() { + quote! {} + } else { + quote! { + #supertraits } + }; + Ok(quote! { + #[doc = #doc] + #vis trait #has { + /// The service held. + type #name: #name; + #[doc = #get_doc] + fn #get(&self) -> &Self::#name; + #(#has_items)* + } + + #[automatically_derived] + impl<#holder: #has #bound> #name for #holder { + #(#impl_items)* + } + }) +} + +/// For a method of the service: its forwarding method in `HasX` (`wrapped_`, calling the +/// held service), and its method in the blanket impl (calling the `HasX` one). +fn forward_fn( + f: &TraitItemFn, + has: &Path, + name: &Ident, + get: &Ident, + holder: &Ident, +) -> syn::Result<(TokenStream2, TokenStream2)> { + reject_bare_self(&f.sig)?; + let mut sig = f.sig.clone(); + let args = name_arguments(&mut sig)?; + let cfgs = cfgs(&f.attrs); + let ident = sig.ident.clone(); + let wrapped_ident = wrapped(&ident); + + let mut has_sig = sig.clone(); + has_sig.ident = wrapped_ident.clone(); + HeldAssoc { has, name }.visit_signature_mut(&mut has_sig); + let has_call = quote! { <::#name as #name>::#ident(#has::#get(self), #(#args),*) }; + let doc = + format!("Forwards [`{name}::{ident}`] to the held service; a `#[loko::wraps]` implementation may replace it."); + let has_method = method(&has_sig, &has_call); + + let impl_call = quote! { <#holder as #has>::#wrapped_ident(self, #(#args),*) }; + let impl_method = method(&sig, &impl_call); + Ok((quote! { #(#cfgs)* #[doc = #doc] #[doc(hidden)] #has_method }, quote! { #(#cfgs)* #impl_method })) +} + +/// Whether `attr` is `#[loko::has(…)]` (or `#[has(…)]`, or a re-export's path ending in `has`). +fn is_has(attr: &syn::Attribute) -> bool { + attr.path().segments.last().is_some_and(|s| s.ident == "has") +} + +/// Makes a struct a bundle: for each field marked `#[loko::has(Service, …)]`, implements +/// `Has`, so the struct is each of those services, forwarding to the field. +#[proc_macro_attribute] +pub fn bundle(args: TokenStream, input: TokenStream) -> TokenStream { + if !args.is_empty() { + return Error::new(Span::call_site(), "#[bundle] takes no arguments").to_compile_error().into(); + } + let mut item = parse_macro_input!(input as syn::ItemStruct); + match expand_bundle(&mut item) { + Ok(impls) => quote! { #item #impls }.into(), + Err(e) => { + let e = e.to_compile_error(); + quote! { #item #e }.into() + } + } +} + +/// The `Has` impls of `item`'s `#[has(…)]` fields, which it removes from the struct. +fn expand_bundle(item: &mut syn::ItemStruct) -> syn::Result { + let name = item.ident.clone(); + let generics = item.generics.clone(); + let (impl_generics, ty_generics, _) = generics.split_for_impl(); + let mut impls = vec![]; + for (i, field) in item.fields.iter_mut().enumerate() { + let index = syn::Index::from(i); + let access = field.ident.as_ref().map_or_else(|| quote! { #index }, |ident| quote! { #ident }); + let ty = field.ty.clone(); + let (has_attrs, others): (Vec<_>, Vec<_>) = field.attrs.drain(..).partition(is_has); + field.attrs = others; + for attr in has_attrs { + let services = attr.parse_args_with(Punctuated::::parse_terminated)?; + for service in services { + let (has, service_name) = has_path(&service)?; + let get = accessor(&service_name); + // A generic field type must be the service it holds. + let mut generics = generics.clone(); + generics.make_where_clause().predicates.push(parse_quote! { #ty: #service }); + let where_clause = &generics.where_clause; + impls.push(quote! { + #[automatically_derived] + impl #impl_generics #has for #name #ty_generics #where_clause { + type #service_name = #ty; + #[inline] + fn #get(&self) -> &#ty { + &self.#access + } + } + }); + } + } + } + if impls.is_empty() { + return Err(Error::new(name.span(), "no field has #[loko::has(…)]")); + } + Ok(quote! { #(#impls)* }) +} + +/// Marks a field of a `#[loko::bundle]` struct with the services it holds: +/// `#[loko::has(Clock, Random)]`. `#[loko::bundle]` reads and removes it; anywhere else it's an +/// error. +#[proc_macro_attribute] +pub fn has(_args: TokenStream, input: TokenStream) -> TokenStream { + let e = + Error::new(Span::call_site(), "#[loko::has(…)] goes on a field of a #[loko::bundle] struct").to_compile_error(); + let input = TokenStream2::from(input); + quote! { #e #input }.into() +} + +/// `field: Type`, the arguments of `#[wraps]`. +struct WrapsArgs { + field: Member, + ty: Type, +} + +impl Parse for WrapsArgs { + fn parse(input: ParseStream<'_>) -> syn::Result { + let field = input.parse()?; + input.parse::()?; + Ok(Self { field, ty: input.parse()? }) + } +} + +/// Makes `impl Service for Wrapper` a wrapper around a field: `#[wraps(inner: S)]`. The methods +/// written replace the service's; the others forward to `self.inner`. A wrapper keeps the inner +/// service's associated types. +#[proc_macro_attribute] +pub fn wraps(args: TokenStream, input: TokenStream) -> TokenStream { + let args = parse_macro_input!(args as WrapsArgs); + let item = parse_macro_input!(input as ItemImpl); + match expand_wraps(&args, &item) { + Ok(tokens) => tokens.into(), + Err(e) => e.to_compile_error().into(), + } +} + +fn expand_wraps(args: &WrapsArgs, item: &ItemImpl) -> syn::Result { + let Some((None, service, _)) = &item.trait_ else { + return Err(Error::new(item.span(), "#[wraps] goes on an `impl Service for Wrapper`")); + }; + let (has, name) = has_path(service)?; + let get = accessor(&name); + let mut methods = vec![]; + for impl_item in &item.items { + let ImplItem::Fn(f) = impl_item else { + return Err(Error::new( + impl_item.span(), + "a wrapper only replaces methods; it keeps the inner service's types and constants", + )); + }; + let mut f = f.clone(); + f.sig.ident = wrapped(&f.sig.ident); + HeldAssoc { has: &has, name: &name }.visit_signature_mut(&mut f.sig); + methods.push(f.into_token_stream()); + } + let (impl_generics, _, where_clause) = item.generics.split_for_impl(); + let (attrs, self_ty, (field, ty)) = (&item.attrs, &item.self_ty, (&args.field, &args.ty)); + Ok(quote! { + #(#attrs)* + impl #impl_generics #has for #self_ty #where_clause { + type #name = #ty; + #[inline] + fn #get(&self) -> &#ty { + &self.#field + } + #(#methods)* + } + }) +} + +#[cfg(test)] +mod tests { + use syn::parse::Parser; + + use super::*; + + #[test] + fn accessors_are_snake_case() { + assert_eq!(accessor(&format_ident!("Clock")), "clock"); + assert_eq!(accessor(&format_ident!("ObjectStore")), "object_store"); + assert_eq!(accessor(&format_ident!("HTTPClient")), "http_client"); + assert_eq!(accessor(&format_ident!("S3Store")), "s3_store"); + assert_eq!(accessor(&format_ident!("IOError")), "io_error"); + } + + #[test] + fn services_parse() { + let parser = Punctuated::::parse_terminated; + let paths = parser.parse_str("Clock, loko::Random").unwrap(); + assert_eq!(paths.len(), 2); + let (has, name) = has_path(&paths[1]).unwrap(); + assert_eq!( + (has.to_token_stream().to_string(), name.to_string()), + ("loko :: HasRandom".into(), "Random".into()) + ); + } + + #[test] + fn bare_self_is_rejected() { + for sig in + [quote! { fn duplicate(&self) -> Self where Self: Sized }, quote! { fn merge(&self, other: Option) }] + { + assert!(reject_bare_self(&syn::parse2(sig).unwrap()).is_err()); + } + let ok: Signature = parse_quote! { fn upload(&self) -> Result where Self: Sized }; + assert!(reject_bare_self(&ok).is_ok(), "associated types and where clauses are fine"); + } + + #[test] + fn self_types_become_the_held_services() { + let mut sig: Signature = parse_quote! { fn upload(&self) -> Result }; + let has: Path = parse_quote! { HasStore }; + HeldAssoc { has: &has, name: &format_ident!("Store") }.visit_signature_mut(&mut sig); + assert_eq!( + sig.output.to_token_stream().to_string(), + "-> Result < < < Self as HasStore > :: Store as Store > :: Upload >" + ); + } +} diff --git a/crates/loko-object-store/Cargo.toml b/crates/loko-object-store/Cargo.toml new file mode 100644 index 0000000..dcb8bbd --- /dev/null +++ b/crates/loko-object-store/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "loko-object-store" +description = "A loko store over an object_store client: S3, GCS, Azure, local files, memory, and more" +readme = "README.md" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true + +[dependencies] +loko-store.workspace = true +# Without features: applications enable the backends they need (`aws`, `gcp`, `azure`, `http`, `fs`, or the `*-base` +# variants for WebAssembly) on their own `object_store` dependency. +object_store.workspace = true +futures-util.workspace = true +url.workspace = true + +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +object_store = { workspace = true, features = ["fs"] } +tempfile = "3" +tokio = { workspace = true, features = ["macros", "rt"] } + +[package.metadata.docs.rs] +all-features = true + +[lints] +workspace = true diff --git a/crates/loko-object-store/README.md b/crates/loko-object-store/README.md new file mode 100644 index 0000000..89935f2 --- /dev/null +++ b/crates/loko-object-store/README.md @@ -0,0 +1,24 @@ +# loko-object-store + +A [loko](https://github.com/polythings/loko) `Store` over an [`object_store`](https://crates.io/crates/object_store) +client: S3, GCS, Azure, local files, memory, and anything else that implements `ObjectStore`. + +```rust +use std::sync::Arc; + +use loko_object_store::ObjectStoreAdapter; +use object_store::memory::InMemory; + +let store = ObjectStoreAdapter::new(Arc::new(InMemory::new()), "memory://"); +// Or from a URL, with the backend enabled on `object_store` (here `aws`): +// let store = ObjectStoreAdapter::from_url("s3://bucket/warehouse", [("aws_region", "us-east-2")])?; +``` + +- loko names objects by absolute URI; the adapter maps them to paths under its root (`s3://bucket/`, or + `file:///` for local files). +- Put-if-absent, the commit primitive, is `PutMode::Create`; another writer's object reads as `false`. +- Listing is by string prefix (`…/metadata/v`), done by listing the prefix's directory. +- The crate enables no `object_store` backends: turn on the ones you need (`aws`, `gcp`, `azure`, `http`, `fs`, or the + `*-base` variants for WebAssembly) on your own `object_store` dependency. + +Apache License 2.0. diff --git a/crates/loko-object-store/src/lib.rs b/crates/loko-object-store/src/lib.rs new file mode 100644 index 0000000..55923af --- /dev/null +++ b/crates/loko-object-store/src/lib.rs @@ -0,0 +1,358 @@ +//! A loko [`Store`] over an [`object_store`] client: S3, GCS, Azure, local files, memory, and +//! anything else that implements `ObjectStore`. Applications enable the backends they need on their +//! own `object_store` dependency and hand the client to [`ObjectStoreAdapter::new`], or build one +//! from a URL with [`ObjectStoreAdapter::from_url`]. +//! +//! loko names objects by absolute URI (`s3://bucket/warehouse/t/metadata/v1.metadata.json`); +//! the adapter maps them to paths under its root (`s3://bucket`). +//! +//! ``` +//! use std::sync::Arc; +//! +//! use loko_object_store::ObjectStoreAdapter; +//! use object_store::memory::InMemory; +//! +//! let store = ObjectStoreAdapter::new(Arc::new(InMemory::new()), "memory://"); +//! ``` + +use std::fmt; +use std::future::Future; +use std::sync::Arc; + +use futures_util::TryStreamExt; +use loko_store::{Error, Listed, Result, Store, Upload}; +use object_store::path::Path; +use object_store::{MultipartUpload, ObjectStore, ObjectStoreExt, PutMode, PutOptions, PutPayload}; +use url::Url; + +/// A [`Store`] over an `object_store` client whose objects have URIs under `root`. +#[derive(Clone)] +pub struct ObjectStoreAdapter { + store: Arc, + root: String, + /// The root as shown in errors and `Debug`, without a password. + shown: String, +} + +impl fmt::Debug for ObjectStoreAdapter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ObjectStoreAdapter").field("store", &self.store).field("root", &self.shown).finish() + } +} + +fn store_error(e: object_store::Error) -> Error { + match e { + object_store::Error::NotFound { path, .. } => Error::NotFound(path), + object_store::Error::NotImplemented { operation, implementer } => { + Error::Unsupported(format!("{implementer} has no {operation}")) + } + other => Error::Store(other.to_string()), + } +} + +impl ObjectStoreAdapter { + /// `store`, whose objects have URIs under `root`: the object at path `a/b` is `/a/b`. The + /// root is the store's scheme and bucket (`s3://bucket`, `gs://bucket`, `memory://`), or + /// `file:///` for a local file system rooted at `/`. + pub fn new(store: Arc, root: impl Into) -> Self { + let mut root = root.into(); + if !root.ends_with('/') { + root.push('/'); + } + let shown = redacted(&root); + Self { store, root, shown } + } + + /// The store for `url`'s scheme and bucket (`s3://bucket/…`), configured with `options` + /// (`object_store`'s keys, such as `aws_region`). Its backend must be enabled on + /// `object_store`. + pub fn from_url(url: &str, options: I) -> Result + where + I: IntoIterator, + K: AsRef, + V: Into, + { + let parsed = Url::parse(url).map_err(|e| Error::Invalid(format!("{url}: {e}")))?; + let root = root_of(&parsed); + let root_url = Url::parse(&root).map_err(|e| Error::Invalid(format!("{root}: {e}")))?; + let (store, _) = object_store::parse_url_opts(&root_url, options).map_err(store_error)?; + Ok(Self::new(Arc::from(store), root)) + } + + /// The path of the object at `uri`. + fn path(&self, uri: &str) -> Result { + let rest = uri + .strip_prefix(&self.root) + .ok_or_else(|| Error::Invalid(format!("{uri} is outside this store ({})", self.shown)))?; + Path::parse(rest).map_err(|e| Error::Invalid(format!("{uri}: {e}"))) + } + + /// The URI of the object at `path`. + fn uri(&self, path: &Path) -> String { + format!("{}{path}", self.root) + } +} + +/// The root of `url`: everything before the path, so its scheme, userinfo, host, and port +/// (`abfss://filesystem@account.dfs.core.windows.net/`, `http://localhost:9000/`), which the +/// URIs of its objects start with. `file:///data/t` gives `file:///`, a file system rooted at `/`. +fn root_of(url: &Url) -> String { + let Some(host) = url.host_str().filter(|h| !h.is_empty()) else { + return format!("{}:///", url.scheme()); + }; + let userinfo = match (url.username(), url.password()) { + ("", None) => String::new(), + (user, None) => format!("{user}@"), + (user, Some(password)) => format!("{user}:{password}@"), + }; + let port = url.port().map(|p| format!(":{p}")).unwrap_or_default(); + format!("{}://{userinfo}{host}{port}/", url.scheme()) +} + +/// `root` with any password replaced by `***`, for errors and `Debug`. +fn redacted(root: &str) -> String { + let Ok(mut url) = Url::parse(root) else { return root.to_string() }; + if url.password().is_none() || url.set_password(Some("***")).is_err() { + return root.to_string(); + } + url.to_string() +} + +/// An offset for listing directory `dir` that skips objects sorting before `name`, the start of the +/// object names wanted. `object_store` lists the objects after the offset, so the offset is `name` +/// with its last character one lower (`v` → `u`), which still keeps an object named exactly `name`. +/// Only for letters and digits, whose predecessor is a letter or digit that paths keep as it is. +fn list_offset(dir: &str, name: &str) -> Option { + let last = name.chars().next_back()?; + if !last.is_ascii_alphanumeric() || matches!(last, '0' | 'A' | 'a') { + return None; + } + let lower = char::from(u8::try_from(last).ok()? - 1); + let start = &name[..name.len() - 1]; + let offset = if dir.is_empty() { format!("{start}{lower}") } else { format!("{dir}/{start}{lower}") }; + Path::parse(offset).ok() +} + +/// A multipart upload through `object_store`. +#[derive(Debug)] +pub struct ObjectStoreUpload(Box); + +impl Upload for ObjectStoreUpload { + async fn put_part(&mut self, bytes: Vec) -> Result<()> { + self.0.put_part(PutPayload::from(bytes)).await.map_err(store_error) + } + + async fn complete(mut self) -> Result<()> { + self.0.complete().await.map(drop).map_err(store_error) + } + + async fn abort(mut self) { + let _ = self.0.abort().await; + } +} + +impl Store for ObjectStoreAdapter { + type Upload = ObjectStoreUpload; + + async fn get(&self, uri: &str) -> Result>> { + match self.store.get(&self.path(uri)?).await { + Ok(found) => Ok(Some(found.bytes().await.map_err(store_error)?.to_vec())), + Err(object_store::Error::NotFound { .. }) => Ok(None), + Err(e) => Err(store_error(e)), + } + } + + async fn get_range(&self, uri: &str, start: u64, len: u64) -> Result> { + // `object_store` rejects empty ranges and ranges starting past the end, which a `Store` + // answers with no bytes (as `MemoryStore` does) when the object exists. + let path = self.path(uri)?; + if len == 0 { + return self.store.head(&path).await.map(|_| Vec::new()).map_err(store_error); + } + match self.store.get_range(&path, start..start.saturating_add(len)).await { + Ok(bytes) => Ok(bytes.to_vec()), + Err(e @ object_store::Error::NotFound { .. }) => Err(store_error(e)), + Err(e) => match self.store.head(&path).await { + Ok(meta) if start >= meta.size => Ok(Vec::new()), + _ => Err(store_error(e)), + }, + } + } + + async fn exists(&self, uri: &str) -> Result { + match self.store.head(&self.path(uri)?).await { + Ok(_) => Ok(true), + Err(object_store::Error::NotFound { .. }) => Ok(false), + Err(e) => Err(store_error(e)), + } + } + + async fn put(&self, uri: &str, bytes: Vec) -> Result<()> { + self.store.put(&self.path(uri)?, PutPayload::from(bytes)).await.map(drop).map_err(store_error) + } + + async fn put_if_absent(&self, uri: &str, bytes: Vec) -> Result { + let options = PutOptions { mode: PutMode::Create, ..PutOptions::default() }; + match self.store.put_opts(&self.path(uri)?, PutPayload::from(bytes), options).await { + Ok(_) => Ok(true), + Err(object_store::Error::AlreadyExists { .. } | object_store::Error::Precondition { .. }) => Ok(false), + Err(e) => Err(store_error(e)), + } + } + + async fn upload(&self, uri: &str) -> Result { + Ok(ObjectStoreUpload(self.store.put_multipart(&self.path(uri)?).await.map_err(store_error)?)) + } + + fn list(&self, prefix: &str) -> impl Future>> { + // `object_store` lists by directory; loko by string prefix (`…/metadata/v`). List the + // directory the prefix is in, from just before the prefix, and keep what starts with it. + let split = prefix.strip_prefix(&self.root).map(|rest| rest.rsplit_once('/').unwrap_or(("", rest))); + async move { + let (relative, name) = + split.ok_or_else(|| Error::Invalid(format!("{prefix} is outside this store ({})", self.shown)))?; + let dir = Path::parse(relative).map_err(|e| Error::Invalid(format!("{prefix}: {e}")))?; + let dir = (!relative.is_empty()).then_some(&dir); + let listing = match list_offset(relative, name) { + Some(offset) => self.store.list_with_offset(dir, &offset), + None => self.store.list(dir), + }; + let objects: Vec<_> = listing.try_collect().await.map_err(store_error)?; + Ok(objects + .into_iter() + .map(|o| Listed { uri: self.uri(&o.location), size: o.size }) + .filter(|o| o.uri.starts_with(prefix)) + .collect()) + } + } + + async fn delete(&self, uris: &[String]) -> Result<()> { + let deletes = uris.iter().map(|uri| async move { + match self.store.delete(&self.path(uri)?).await { + Ok(()) | Err(object_store::Error::NotFound { .. }) => Ok(()), + Err(e) => Err(store_error(e)), + } + }); + futures_util::future::try_join_all(deletes).await.map(drop) + } +} + +#[cfg(all(test, not(target_arch = "wasm32")))] +mod tests { + use object_store::memory::InMemory; + + use super::*; + + fn adapter() -> ObjectStoreAdapter { + ObjectStoreAdapter::new(Arc::new(InMemory::new()), "memory://") + } + + #[tokio::test] + async fn objects_round_trip() { + let store = adapter(); + assert_eq!(store.get("memory://t/a").await.unwrap(), None); + assert!(!store.exists("memory://t/a").await.unwrap()); + store.put("memory://t/a", b"hello".to_vec()).await.unwrap(); + assert_eq!(store.get("memory://t/a").await.unwrap().as_deref(), Some(&b"hello"[..])); + assert_eq!(store.get_range("memory://t/a", 1, 3).await.unwrap(), b"ell"); + assert!(store.put_if_absent("memory://t/b", b"1".to_vec()).await.unwrap()); + assert!(!store.put_if_absent("memory://t/b", b"2".to_vec()).await.unwrap(), "the first write wins"); + assert_eq!(store.get("memory://t/b").await.unwrap().as_deref(), Some(&b"1"[..])); + + let mut upload = store.upload("memory://t/c").await.unwrap(); + upload.put_part(b"ab".to_vec()).await.unwrap(); + upload.put_part(b"cd".to_vec()).await.unwrap(); + upload.complete().await.unwrap(); + assert_eq!(store.get("memory://t/c").await.unwrap().as_deref(), Some(&b"abcd"[..])); + + store.delete(&["memory://t/a".into(), "memory://t/missing".into()]).await.unwrap(); + assert!(!store.exists("memory://t/a").await.unwrap()); + assert!(matches!(store.get("s3://elsewhere/x").await, Err(Error::Invalid(_)))); + } + + #[tokio::test] + async fn lists_by_string_prefix() { + let store = adapter(); + for name in ["v1.metadata.json", "v2.metadata.json", "version-hint.text", "snap-1.avro"] { + store.put(&format!("memory://t/metadata/{name}"), vec![0; 3]).await.unwrap(); + } + store.put("memory://t/data/v9.parquet", vec![]).await.unwrap(); + let mut listed: Vec = + store.list("memory://t/metadata/v").await.unwrap().into_iter().map(|o| o.uri).collect(); + listed.sort(); + assert_eq!( + listed, + [ + "memory://t/metadata/v1.metadata.json", + "memory://t/metadata/v2.metadata.json", + "memory://t/metadata/version-hint.text" + ] + ); + assert_eq!(store.list("memory://t/metadata/s").await.unwrap()[0].size, 3); + assert_eq!(store.list("memory://t").await.unwrap().len(), 5, "a prefix right under the root"); + } + + #[tokio::test] + async fn empty_and_past_the_end_ranges_are_empty() { + let store = adapter(); + store.put("memory://t/a", b"abc".to_vec()).await.unwrap(); + assert_eq!(store.get_range("memory://t/a", 1, 0).await.unwrap(), b""); + assert_eq!(store.get_range("memory://t/a", 3, 2).await.unwrap(), b""); + assert_eq!(store.get_range("memory://t/a", 9, 2).await.unwrap(), b""); + assert_eq!(store.get_range("memory://t/a", 1, 10).await.unwrap(), b"bc", "cut at the end"); + assert!(matches!(store.get_range("memory://t/missing", 0, 0).await, Err(Error::NotFound(_)))); + assert!(matches!(store.get_range("memory://t/missing", 0, 2).await, Err(Error::NotFound(_)))); + } + + #[tokio::test] + async fn listing_from_an_offset_keeps_the_exact_prefix() { + let store = adapter(); + for name in ["snap-1.avro", "u9", "v", "v1.metadata.json", "w"] { + store.put(&format!("memory://t/metadata/{name}"), vec![]).await.unwrap(); + } + let mut listed: Vec = + store.list("memory://t/metadata/v").await.unwrap().into_iter().map(|o| o.uri).collect(); + listed.sort(); + assert_eq!(listed, ["memory://t/metadata/v", "memory://t/metadata/v1.metadata.json"]); + } + + #[test] + fn offsets_sort_just_before_the_prefix() { + assert_eq!(list_offset("t/metadata", "v").map(|p| p.to_string()).as_deref(), Some("t/metadata/u")); + assert_eq!(list_offset("", "t").map(|p| p.to_string()).as_deref(), Some("s")); + assert_eq!(list_offset("t", "snap-9").map(|p| p.to_string()).as_deref(), Some("t/snap-8")); + assert!(list_offset("t", "").is_none(), "a whole directory"); + assert!(list_offset("t", "a").is_none() && list_offset("t", "x-").is_none(), "no safe predecessor"); + } + + #[test] + fn roots_keep_what_uris_start_with() { + let root = |url: &str| root_of(&Url::parse(url).unwrap()); + assert_eq!(root("http://localhost:9000/bucket/t"), "http://localhost:9000/"); + assert_eq!(root("https://example.com:443/t"), "https://example.com/", "443 is https's own"); + assert_eq!(root("abfss://fs@account.dfs.core.windows.net/t"), "abfss://fs@account.dfs.core.windows.net/"); + assert_eq!(root("https://user:secret@example.com/t"), "https://user:secret@example.com/"); + assert_eq!(root("s3://bucket/warehouse/t"), "s3://bucket/"); + assert_eq!(root("file:///data/t"), "file:///"); + } + + #[tokio::test] + async fn passwords_stay_out_of_errors_and_debug() { + let store = ObjectStoreAdapter::new(Arc::new(InMemory::new()), "memory://user:secret@host/"); + assert!(!format!("{store:?}").contains("secret")); + let Err(Error::Invalid(message)) = store.get("s3://elsewhere/x").await else { panic!("outside the store") }; + assert!(message.contains("user:***@host") && !message.contains("secret"), "{message}"); + assert_eq!(redacted("s3://bucket/"), "s3://bucket/"); + } + + #[tokio::test] + async fn local_files_have_file_uris() { + let dir = tempfile::tempdir().unwrap(); + let store = ObjectStoreAdapter::from_url("file:///", std::iter::empty::<(&str, String)>()).unwrap(); + let uri = format!("file://{}/t/a.json", dir.path().display()); + store.put(&uri, b"{}".to_vec()).await.unwrap(); + assert!(dir.path().join("t/a.json").exists()); + let listed = store.list(&format!("file://{}/t/", dir.path().display())).await.unwrap(); + assert_eq!(listed[0].uri, uri); + } +} diff --git a/crates/loko-random/Cargo.toml b/crates/loko-random/Cargo.toml new file mode 100644 index 0000000..362af68 --- /dev/null +++ b/crates/loko-random/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "loko-random" +description = "loko's Random service: random numbers, with seeded, operating-system, and JavaScript sources" +readme = "README.md" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true + +[features] +default = [] +# `OsRandom`: the operating system's random number generator (native targets and WASI). +getrandom = ["dep:getrandom"] +# `JsRandom`: `crypto.getRandomValues` (`wasm32-unknown-unknown` with a JavaScript host). +js = ["dep:js-sys", "dep:wasm-bindgen"] + +[dependencies] +rand_core.workspace = true + +[target.'cfg(not(all(target_arch = "wasm32", target_os = "unknown")))'.dependencies] +getrandom = { workspace = true, optional = true } + +[target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] +js-sys = { workspace = true, optional = true } +wasm-bindgen = { workspace = true, optional = true } + +[package.metadata.docs.rs] +all-features = true + +[lints] +workspace = true diff --git a/crates/loko-random/README.md b/crates/loko-random/README.md new file mode 100644 index 0000000..b35a90c --- /dev/null +++ b/crates/loko-random/README.md @@ -0,0 +1,28 @@ +# loko-random + +[loko](https://github.com/polythings/loko)'s `Random` service: a source of random number generators, passed to code +instead of drawn from a global, so tests are repeatable and the same code runs natively, under WASI, and in browsers. + +- `Random::rng()` hands out a generator of the caller's own: a `rand_core::Rng`, drawn from with `&mut` and no locks, + and usable with anything in the `rand` ecosystem. +- `SeededRandom`: `SeededRng`s (SplitMix64) forked from one seeded sequence, the same on every run. For tests; not for + secrets. Clones share the sequence. +- `OsRandom` (feature `getrandom`): `OsRng`, the operating system's generator (native and WASI). A `CryptoRng`. +- `JsRandom` (feature `js`, `wasm32-unknown-unknown`): `JsRng`, `crypto.getRandomValues`. A `CryptoRng`. +- A `Mutex` around any seedable `rand_core` generator is a `Random` whose generators are forked from it. + +```rust +use loko_random::{CryptoRng, Random, Rng, SeededRandom}; + +let (a, b) = (SeededRandom::new(7), SeededRandom::new(7)); +assert_eq!(a.rng().next_u64(), b.rng().next_u64()); + +/// Secrets ask for a cryptographic generator, so a SeededRandom can't be passed here by mistake. +fn token(random: &impl Random) -> [u8; 32] { + let mut bytes = [0; 32]; + random.rng().fill_bytes(&mut bytes); + bytes +} +``` + +Apache License 2.0. diff --git a/crates/loko-random/src/js.rs b/crates/loko-random/src/js.rs new file mode 100644 index 0000000..797db4b --- /dev/null +++ b/crates/loko-random/src/js.rs @@ -0,0 +1,66 @@ +use std::convert::Infallible; + +use js_sys::{Function, Reflect, Uint8Array}; +use rand_core::{TryCryptoRng, TryRng}; +use wasm_bindgen::{JsCast, JsValue}; + +use crate::Random; + +/// `crypto.getRandomValues` of the JavaScript host (feature `js`, `wasm32-unknown-unknown`): +/// browsers, Workers, Deno, and Node. +#[derive(Clone, Copy, Debug, Default)] +pub struct JsRandom; + +impl Random for JsRandom { + type Rng = JsRng; + + fn rng(&self) -> JsRng { + JsRng + } +} + +/// Numbers from `crypto.getRandomValues`; fit for secrets. Panics on a host without Web Crypto, +/// rather than handing out numbers that aren't. +#[derive(Clone, Copy, Debug, Default)] +pub struct JsRng; + +fn get_random_values() -> (JsValue, Function) { + let crypto = Reflect::get(&js_sys::global(), &"crypto".into()).ok().filter(|c| !c.is_undefined()); + let function = crypto + .as_ref() + .and_then(|c| Reflect::get(c, &"getRandomValues".into()).ok()) + .and_then(|f| f.dyn_into::().ok()); + match (crypto, function) { + (Some(crypto), Some(function)) => (crypto, function), + _ => panic!("the JavaScript host has no crypto.getRandomValues"), + } +} + +impl TryRng for JsRng { + type Error = Infallible; + + fn try_next_u32(&mut self) -> Result { + let mut bytes = [0u8; 4]; + self.try_fill_bytes(&mut bytes)?; + Ok(u32::from_le_bytes(bytes)) + } + + fn try_next_u64(&mut self) -> Result { + let mut bytes = [0u8; 8]; + self.try_fill_bytes(&mut bytes)?; + Ok(u64::from_le_bytes(bytes)) + } + + fn try_fill_bytes(&mut self, dst: &mut [u8]) -> Result<(), Infallible> { + let (crypto, function) = get_random_values(); + // getRandomValues fills at most 65,536 bytes per call. + for chunk in dst.chunks_mut(65_536) { + let array = Uint8Array::new_with_length(u32::try_from(chunk.len()).unwrap_or(u32::MAX)); + function.call1(&crypto, &array).expect("crypto.getRandomValues"); + array.copy_to(chunk); + } + Ok(()) + } +} + +impl TryCryptoRng for JsRng {} diff --git a/crates/loko-random/src/lib.rs b/crates/loko-random/src/lib.rs new file mode 100644 index 0000000..b14ba63 --- /dev/null +++ b/crates/loko-random/src/lib.rs @@ -0,0 +1,75 @@ +//! loko's `Random` service: a source of random number generators, passed to code instead of drawn +//! from a global, so tests are repeatable and the same code runs natively and in WebAssembly. +//! +//! [`Random::rng`] hands out a generator of the caller's own, a [`rand_core::Rng`] drawn from with +//! `&mut` and no locks, and usable with anything in the `rand` ecosystem. Each implementation chooses +//! the generator: +//! +//! - [`SeededRandom`]: [`SeededRng`]s forked from one seeded sequence, the same on every run; for +//! tests, not for secrets. +//! - `OsRandom` (feature `getrandom`): `OsRng`, the operating system's generator, on native targets +//! and WASI. A [`CryptoRng`]. +//! - `JsRandom` (feature `js`): `JsRng`, `crypto.getRandomValues`, in browsers, Workers, and Node. +//! A [`CryptoRng`]. +//! - A `Mutex` around any seedable `rand_core` generator: generators forked from it. +//! +//! Code that makes secrets asks for a cryptographic generator: `random: &impl Random`. +//! +//! ``` +//! use loko_random::{Random, Rng, SeededRandom}; +//! +//! let (a, b) = (SeededRandom::new(7), SeededRandom::new(7)); +//! let (mut x, mut y) = (a.rng(), b.rng()); +//! assert_eq!(x.next_u64(), y.next_u64(), "the same seed gives the same numbers"); +//! ``` + +#[cfg(all(feature = "js", all(target_arch = "wasm32", target_os = "unknown")))] +mod js; +mod rand; +#[cfg(all(feature = "getrandom", not(all(target_arch = "wasm32", target_os = "unknown"))))] +mod system; +mod testing; + +#[cfg(all(feature = "js", all(target_arch = "wasm32", target_os = "unknown")))] +pub use js::{JsRandom, JsRng}; +pub use rand_core::{self, CryptoRng, Rng, SeedableRng}; +#[cfg(all(feature = "getrandom", not(all(target_arch = "wasm32", target_os = "unknown"))))] +pub use system::{OsRandom, OsRng}; +pub use testing::{SeededRandom, SeededRng}; + +/// A source of random number generators. +pub trait Random { + /// The generator [`rng`](Random::rng) hands out. + type Rng: rand_core::Rng; + + /// A generator of the caller's own: draw from it with `&mut` as much as needed, with no locks. + /// Whether it is fit for secrets depends on the implementation: ask for `Random`. + fn rng(&self) -> Self::Rng; +} + +/// A type that holds a [`Random`], and so is one: `#[loko::bundle]` implements this for a field, +/// and `#[loko::wraps]` for a wrapper that changes some methods. It's what `#[loko::service]` +/// generates, written out so this crate needs no macros. +pub trait HasRandom { + /// The service held. + type Random: Random; + + /// The [`Random`] this holds. + fn random(&self) -> &Self::Random; + + /// Forwards [`Random::rng`] to the held service; a `#[loko::wraps]` implementation may replace it. + #[doc(hidden)] + #[inline] + fn wrapped_rng(&self) -> ::Rng { + self.random().rng() + } +} + +impl Random for H { + type Rng = <::Random as Random>::Rng; + + #[inline] + fn rng(&self) -> Self::Rng { + self.wrapped_rng() + } +} diff --git a/crates/loko-random/src/rand.rs b/crates/loko-random/src/rand.rs new file mode 100644 index 0000000..d29973d --- /dev/null +++ b/crates/loko-random/src/rand.rs @@ -0,0 +1,28 @@ +use std::sync::{Mutex, PoisonError}; + +use rand_core::{Rng, SeedableRng}; + +use crate::Random; + +/// Any seedable `rand_core` generator, behind a mutex so it can be shared: each +/// [`rng`](Random::rng) is a generator forked from it (`SeedableRng::fork`). +impl Random for Mutex { + type Rng = R; + + fn rng(&self) -> R { + self.lock().unwrap_or_else(PoisonError::into_inner).fork() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::SeededRng; + + #[test] + fn a_shared_generator_forks() { + let shared = Mutex::new(SeededRng::new(9)); + let (mut a, mut b) = (shared.rng(), shared.rng()); + assert_ne!(a.next_u64(), b.next_u64(), "each fork is seeded from the next numbers"); + } +} diff --git a/crates/loko-random/src/system.rs b/crates/loko-random/src/system.rs new file mode 100644 index 0000000..1c285d8 --- /dev/null +++ b/crates/loko-random/src/system.rs @@ -0,0 +1,63 @@ +use std::convert::Infallible; + +use rand_core::{TryCryptoRng, TryRng}; + +use crate::Random; + +/// The operating system's random number generator (feature `getrandom`, native targets and WASI). +#[derive(Clone, Copy, Debug, Default)] +pub struct OsRandom; + +impl Random for OsRandom { + type Rng = OsRng; + + fn rng(&self) -> OsRng { + OsRng + } +} + +/// Numbers straight from the operating system's generator; fit for secrets. +#[derive(Clone, Copy, Debug, Default)] +pub struct OsRng; + +// The operating system's generator only fails when it isn't available at all. +const UNAVAILABLE: &str = "the operating system's random number generator"; + +impl TryRng for OsRng { + type Error = Infallible; + + fn try_next_u32(&mut self) -> Result { + Ok(getrandom::u32().expect(UNAVAILABLE)) + } + + fn try_next_u64(&mut self) -> Result { + Ok(getrandom::u64().expect(UNAVAILABLE)) + } + + fn try_fill_bytes(&mut self, dst: &mut [u8]) -> Result<(), Infallible> { + getrandom::fill(dst).expect(UNAVAILABLE); + Ok(()) + } +} + +impl TryCryptoRng for OsRng {} + +#[cfg(test)] +mod tests { + use rand_core::{CryptoRng, Rng}; + + use super::*; + + fn token(random: &impl Random) -> [u8; 32] { + let mut bytes = [0u8; 32]; + random.rng().fill_bytes(&mut bytes); + bytes + } + + #[test] + fn os_numbers_differ_and_are_fit_for_secrets() { + let mut rng = OsRandom.rng(); + assert_ne!(rng.next_u64(), rng.next_u64()); + assert_ne!(token(&OsRandom), token(&OsRandom)); + } +} diff --git a/crates/loko-random/src/testing.rs b/crates/loko-random/src/testing.rs new file mode 100644 index 0000000..85147c3 --- /dev/null +++ b/crates/loko-random/src/testing.rs @@ -0,0 +1,122 @@ +use std::convert::Infallible; +use std::sync::{Arc, Mutex, PoisonError}; + +use rand_core::{SeedableRng, TryRng}; + +use crate::Random; + +/// One step of `SplitMix64` (Steele, Lea, and Flood, 2014): a Weyl sequence with a mixing function. +fn split_mix(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +/// A repeatable source for tests: every [`rng`](Random::rng) is a [`SeededRng`] seeded from one +/// sequence that starts at the seed, so the same seed and the same calls give the same numbers on +/// every run. Not for secrets. Clones share the sequence. +#[derive(Clone, Debug)] +pub struct SeededRandom { + state: Arc>, +} + +impl SeededRandom { + /// The sequence of `seed`. + pub fn new(seed: u64) -> Self { + Self { state: Arc::new(Mutex::new(seed)) } + } +} + +impl Default for SeededRandom { + /// The sequence of seed 0. + fn default() -> Self { + Self::new(0) + } +} + +impl Random for SeededRandom { + type Rng = SeededRng; + + fn rng(&self) -> SeededRng { + SeededRng::new(split_mix(&mut self.state.lock().unwrap_or_else(PoisonError::into_inner))) + } +} + +/// A `SplitMix64` generator: small, fast, and the same on every platform. Not for secrets. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SeededRng { + state: u64, +} + +impl SeededRng { + /// The generator whose state starts at `seed`. + pub fn new(seed: u64) -> Self { + Self { state: seed } + } +} + +impl TryRng for SeededRng { + type Error = Infallible; + + fn try_next_u32(&mut self) -> Result { + Ok((split_mix(&mut self.state) >> 32) as u32) + } + + fn try_next_u64(&mut self) -> Result { + Ok(split_mix(&mut self.state)) + } + + fn try_fill_bytes(&mut self, dst: &mut [u8]) -> Result<(), Infallible> { + for chunk in dst.chunks_mut(8) { + let bytes = split_mix(&mut self.state).to_le_bytes(); + chunk.copy_from_slice(&bytes[..chunk.len()]); + } + Ok(()) + } +} + +impl SeedableRng for SeededRng { + type Seed = [u8; 8]; + + fn from_seed(seed: [u8; 8]) -> Self { + Self::new(u64::from_le_bytes(seed)) + } +} + +#[cfg(test)] +mod tests { + use rand_core::Rng; + + use super::*; + + #[test] + fn split_mix_matches_the_published_sequence() { + // SplitMix64's first output from state 0, as published. + assert_eq!(SeededRng::new(0).next_u64(), 0xE220_A839_7B1D_CDAF); + } + + #[test] + fn seeded_sources_repeat() { + let draw = |random: &SeededRandom| -> Vec { + let mut rng = random.rng(); + (0..5).map(|_| rng.next_u64()).collect() + }; + let (a, b) = (SeededRandom::new(42), SeededRandom::new(42)); + assert_eq!(draw(&a), draw(&b)); + assert_ne!(draw(&a), draw(&a), "each rng() is a new generator"); + assert_ne!(SeededRandom::new(1).rng().next_u64(), SeededRandom::new(2).rng().next_u64()); + assert_eq!(SeededRandom::new(0).rng().next_u64(), 0xA706_DD2F_4D19_7E6F); + } + + #[test] + fn clones_share_the_sequence_and_bytes_cover_odd_lengths() { + let random = SeededRandom::new(3); + let clone = random.clone(); + assert_ne!(random.rng(), clone.rng(), "clones share one sequence"); + let mut bytes = [0u8; 13]; + random.rng().fill_bytes(&mut bytes); + assert!(bytes.iter().any(|&b| b != 0)); + } +} diff --git a/crates/loko-store/Cargo.toml b/crates/loko-store/Cargo.toml new file mode 100644 index 0000000..6cd67e5 --- /dev/null +++ b/crates/loko-store/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "loko-store" +description = "loko's Store service: objects by URI, with conditional writes, multipart uploads, and prefix listing; a store in memory" +readme = "README.md" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true + +[package.metadata.docs.rs] +all-features = true + +[lints] +workspace = true diff --git a/crates/loko-store/README.md b/crates/loko-store/README.md new file mode 100644 index 0000000..6e2dc92 --- /dev/null +++ b/crates/loko-store/README.md @@ -0,0 +1,17 @@ +# loko-store + +[loko](https://github.com/polythings/loko)'s `Store` service: objects by absolute URI, passed to code instead of +reached through a particular client, so the same code works over S3, a local directory, memory, or a JavaScript +binding. + +- `Store`: `get`, `get_range`, `exists`, `put`, `put_if_absent` (a conditional write: `false` when an object is + there), `upload` (multipart: parts in order, then complete or abort), `list` (by string prefix), and `delete` + (missing objects are not an error). The futures aren't required to be `Send`, so a store can wrap JavaScript + promises. +- `MemoryStore`: objects in memory, for tests. Clones share the objects. +- Wrappers, each a `Store` around another: `CountingStore` counts calls, `ReadOnlyStore` refuses writes, and + `PrefixedStore` passes on only URIs under a prefix. They stack. +- `Error`: `NotFound`, `Invalid`, `Unsupported`, and `Store` (the backend's error, as text). +- [`loko-object-store`](https://crates.io/crates/loko-object-store) implements `Store` over an `object_store` client. + +Apache License 2.0. diff --git a/crates/loko-store/src/counting.rs b/crates/loko-store/src/counting.rs new file mode 100644 index 0000000..35c3fc8 --- /dev/null +++ b/crates/loko-store/src/counting.rs @@ -0,0 +1,150 @@ +use std::future::Future; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use crate::{HasStore, Listed, Result, Store}; + +/// How many calls of each kind a [`CountingStore`] passed on. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[non_exhaustive] +pub struct StoreCounts { + /// `get` calls. + pub get: u64, + /// `get_range` calls. + pub get_range: u64, + /// `exists` calls. + pub exists: u64, + /// `put` calls. + pub put: u64, + /// `put_if_absent` calls. + pub put_if_absent: u64, + /// `upload` calls (uploads started). + pub upload: u64, + /// `list` calls. + pub list: u64, + /// `delete` calls (each may delete several objects). + pub delete: u64, +} + +impl StoreCounts { + /// All calls. + pub fn total(&self) -> u64 { + self.get + self.get_range + self.exists + self.put + self.put_if_absent + self.upload + self.list + self.delete + } +} + +/// The calls of each kind, in [`StoreCounts`] order. +#[derive(Debug, Default)] +struct Counters([AtomicU64; 8]); + +/// A store that counts the calls it passes on to another: to show what reading a table costs, or to +/// check in a test that a cache saves requests. Clones share the counts. +#[derive(Clone, Debug, Default)] +pub struct CountingStore { + inner: S, + counters: Arc, +} + +impl CountingStore { + /// Counts the calls to `inner`, from zero. + pub fn new(inner: S) -> Self { + Self { inner, counters: Arc::default() } + } + + /// The calls so far. + pub fn counts(&self) -> StoreCounts { + let [get, get_range, exists, put, put_if_absent, upload, list, delete] = + self.counters.0.each_ref().map(|c| c.load(Ordering::Relaxed)); + StoreCounts { get, get_range, exists, put, put_if_absent, upload, list, delete } + } + + /// Starts the counts again from zero. + pub fn reset(&self) { + for counter in &self.counters.0 { + counter.store(0, Ordering::Relaxed); + } + } + + /// The store counted. + pub fn inner(&self) -> &S { + &self.inner + } + + fn count(&self, kind: usize) { + self.counters.0[kind].fetch_add(1, Ordering::Relaxed); + } +} + +impl HasStore for CountingStore { + type Store = S; + + fn store(&self) -> &S { + &self.inner + } + + fn wrapped_get(&self, uri: &str) -> impl Future>>> { + self.count(0); + self.inner.get(uri) + } + + fn wrapped_get_range(&self, uri: &str, start: u64, len: u64) -> impl Future>> { + self.count(1); + self.inner.get_range(uri, start, len) + } + + fn wrapped_exists(&self, uri: &str) -> impl Future> { + self.count(2); + self.inner.exists(uri) + } + + fn wrapped_put(&self, uri: &str, bytes: Vec) -> impl Future> { + self.count(3); + self.inner.put(uri, bytes) + } + + fn wrapped_put_if_absent(&self, uri: &str, bytes: Vec) -> impl Future> { + self.count(4); + self.inner.put_if_absent(uri, bytes) + } + + fn wrapped_upload(&self, uri: &str) -> impl Future> { + self.count(5); + self.inner.upload(uri) + } + + fn wrapped_list(&self, prefix: &str) -> impl Future>> { + self.count(6); + self.inner.list(prefix) + } + + fn wrapped_delete(&self, uris: &[String]) -> impl Future> { + self.count(7); + self.inner.delete(uris) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{MemoryStore, now}; + + #[test] + fn calls_are_counted_by_kind() { + let store = CountingStore::new(MemoryStore::new()); + now(store.put("memory://t/a", b"abc".to_vec())).unwrap(); + now(store.get("memory://t/a")).unwrap(); + now(store.get_range("memory://t/a", 0, 1)).unwrap(); + now(store.get_range("memory://t/a", 1, 1)).unwrap(); + now(store.list("memory://t/")).unwrap(); + let shared = store.clone(); + assert_eq!( + shared.counts(), + StoreCounts { put: 1, get: 1, get_range: 2, list: 1, ..StoreCounts::default() }, + "clones share the counts" + ); + assert_eq!(store.counts().total(), 5); + store.reset(); + assert_eq!(store.counts(), StoreCounts::default()); + assert_eq!(store.inner().uris(), ["memory://t/a"]); + } +} diff --git a/crates/loko-store/src/lib.rs b/crates/loko-store/src/lib.rs new file mode 100644 index 0000000..c1d41a4 --- /dev/null +++ b/crates/loko-store/src/lib.rs @@ -0,0 +1,80 @@ +//! loko's `Store` service: objects by URI, passed to code instead of reached through a particular +//! client, so the same code works over S3, a local directory, memory, or a JavaScript binding. +//! +//! - [`Store`]: get, ranged get, exists, put, put-if-absent (a conditional write), multipart upload, +//! list by prefix, delete. +//! - [`MemoryStore`]: objects in memory, for tests. +//! - Wrappers, each a `Store` around another: [`CountingStore`] counts requests, [`ReadOnlyStore`] +//! refuses writes, and [`PrefixedStore`] keeps to URIs under a prefix. They stack. +//! - `loko-object-store`: a `Store` over an `object_store` client (S3, GCS, Azure, local files). +//! +//! ``` +//! use loko_store::{MemoryStore, Store}; +//! +//! # block_on(async { +//! let store = MemoryStore::new(); +//! assert!(store.put_if_absent("memory://t/a", b"1".to_vec()).await?); +//! assert!(!store.put_if_absent("memory://t/a", b"2".to_vec()).await?, "the first write wins"); +//! # Ok::<(), loko_store::Error>(()) +//! # }).unwrap(); +//! # // MemoryStore's futures are ready at once, so one poll runs them. +//! # fn block_on(f: F) -> F::Output { +//! # let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); +//! # let mut f = std::pin::pin!(f); +//! # loop { if let std::task::Poll::Ready(v) = f.as_mut().poll(&mut cx) { return v; } } +//! # } +//! ``` + +mod counting; +mod memory; +mod prefixed; +mod read_only; +mod store; + +pub use counting::{CountingStore, StoreCounts}; +pub use memory::{MemoryStore, MemoryUpload}; +pub use prefixed::PrefixedStore; +pub use read_only::ReadOnlyStore; +pub use store::{HasStore, Listed, Store, Upload}; + +/// An error of a store. +#[derive(Debug)] +#[non_exhaustive] +pub enum Error { + /// The object isn't there. + NotFound(String), + /// The request doesn't fit the store (a URI outside it, a bad path). + Invalid(String), + /// The store can't do this (a backend without conditional writes). + Unsupported(String), + /// The store failed. + Store(String), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::NotFound(m) => write!(f, "not found: {m}"), + Self::Invalid(m) => write!(f, "invalid: {m}"), + Self::Unsupported(m) => write!(f, "unsupported: {m}"), + Self::Store(m) => write!(f, "store: {m}"), + } + } +} + +impl std::error::Error for Error {} + +/// `Result` with [`Error`]. +pub type Result = std::result::Result; + +/// Runs a future that is ready at once, as the memory store's are. +#[cfg(test)] +fn now(f: F) -> F::Output { + let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); + let mut f = std::pin::pin!(f); + loop { + if let std::task::Poll::Ready(v) = f.as_mut().poll(&mut cx) { + return v; + } + } +} diff --git a/crates/loko-store/src/memory.rs b/crates/loko-store/src/memory.rs new file mode 100644 index 0000000..1e431c6 --- /dev/null +++ b/crates/loko-store/src/memory.rs @@ -0,0 +1,141 @@ +use std::collections::BTreeMap; +use std::future::{Future, ready}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +use crate::{Error, Listed, Result, Store, Upload}; + +/// A [`Store`] in memory, for tests and examples. Clones share the same objects. +#[derive(Clone, Debug, Default)] +pub struct MemoryStore { + objects: Arc>>>, +} + +impl MemoryStore { + /// An empty store. + pub fn new() -> Self { + Self::default() + } + + /// The URIs stored, sorted. + pub fn uris(&self) -> Vec { + self.objects().keys().cloned().collect() + } + + fn objects(&self) -> MutexGuard<'_, BTreeMap>> { + lock(&self.objects) + } +} + +fn lock(m: &Mutex) -> MutexGuard<'_, T> { + m.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// An upload to a [`MemoryStore`]. +#[derive(Debug)] +pub struct MemoryUpload { + store: MemoryStore, + uri: String, + bytes: Vec, +} + +impl Upload for MemoryUpload { + fn put_part(&mut self, bytes: Vec) -> impl Future> { + self.bytes.extend(bytes); + ready(Ok(())) + } + + fn complete(self) -> impl Future> { + self.store.objects().insert(self.uri, self.bytes); + ready(Ok(())) + } + + fn abort(self) -> impl Future { + ready(()) + } +} + +impl Store for MemoryStore { + type Upload = MemoryUpload; + + fn get(&self, uri: &str) -> impl Future>>> { + ready(Ok(self.objects().get(uri).cloned())) + } + + fn get_range(&self, uri: &str, start: u64, len: u64) -> impl Future>> { + let objects = self.objects(); + ready(objects.get(uri).ok_or_else(|| Error::NotFound(uri.to_string())).map(|bytes| { + let start = usize::try_from(start).unwrap_or(usize::MAX).min(bytes.len()); + let end = start.saturating_add(usize::try_from(len).unwrap_or(usize::MAX)).min(bytes.len()); + bytes[start..end].to_vec() + })) + } + + fn exists(&self, uri: &str) -> impl Future> { + ready(Ok(self.objects().contains_key(uri))) + } + + fn put(&self, uri: &str, bytes: Vec) -> impl Future> { + self.objects().insert(uri.to_string(), bytes); + ready(Ok(())) + } + + fn put_if_absent(&self, uri: &str, bytes: Vec) -> impl Future> { + let mut objects = self.objects(); + let absent = !objects.contains_key(uri); + if absent { + objects.insert(uri.to_string(), bytes); + } + ready(Ok(absent)) + } + + fn upload(&self, uri: &str) -> impl Future> { + ready(Ok(MemoryUpload { store: self.clone(), uri: uri.to_string(), bytes: vec![] })) + } + + fn list(&self, prefix: &str) -> impl Future>> { + let listed = self + .objects() + .range(prefix.to_string()..) + .take_while(|(k, _)| k.starts_with(prefix)) + .map(|(k, v)| Listed { uri: k.clone(), size: v.len() as u64 }) + .collect(); + ready(Ok(listed)) + } + + fn delete(&self, uris: &[String]) -> impl Future> { + let mut objects = self.objects(); + for uri in uris { + objects.remove(uri); + } + ready(Ok(())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::now; + + #[test] + fn objects_round_trip() { + let store = MemoryStore::new(); + assert_eq!(now(store.get("memory://t/a")).unwrap(), None); + now(store.put("memory://t/a", b"hello".to_vec())).unwrap(); + assert_eq!(now(store.get_range("memory://t/a", 1, 3)).unwrap(), b"ell"); + assert!(matches!(now(store.get_range("memory://t/b", 0, 1)), Err(Error::NotFound(_)))); + assert!(now(store.put_if_absent("memory://t/b", b"1".to_vec())).unwrap()); + assert!(!now(store.put_if_absent("memory://t/b", b"2".to_vec())).unwrap()); + + let mut upload = now(store.upload("memory://t/c")).unwrap(); + now(upload.put_part(b"ab".to_vec())).unwrap(); + now(upload.put_part(b"cd".to_vec())).unwrap(); + now(upload.complete()).unwrap(); + assert_eq!(now(store.get("memory://t/c")).unwrap().as_deref(), Some(&b"abcd"[..])); + + let listed: Vec = now(store.list("memory://t/")).unwrap().into_iter().map(|o| o.uri).collect(); + assert_eq!(listed, ["memory://t/a", "memory://t/b", "memory://t/c"]); + now(store.delete(&["memory://t/a".into(), "memory://t/missing".into()])).unwrap(); + assert_eq!(store.uris(), ["memory://t/b", "memory://t/c"]); + assert!(store.clone().uris().len() == 2, "clones share the objects"); + } +} diff --git a/crates/loko-store/src/prefixed.rs b/crates/loko-store/src/prefixed.rs new file mode 100644 index 0000000..35d1166 --- /dev/null +++ b/crates/loko-store/src/prefixed.rs @@ -0,0 +1,109 @@ +use crate::{Error, HasStore, Listed, Result, Store}; + +/// A store that passes on only requests for URIs under a prefix (`s3://bucket/tenant-a/`), and +/// refuses the rest with [`Error::Invalid`]: to keep code to one table or one tenant's data. +/// `.` and `..` path segments are refused, so a URI can't climb out of the prefix. +#[derive(Clone, Debug, Default)] +pub struct PrefixedStore { + inner: S, + prefix: String, +} + +impl PrefixedStore { + /// `inner`, kept to URIs under `prefix`. A `/` is added to the prefix if it has none, so + /// `s3://bucket/a` doesn't let `s3://bucket/ab/…` through. + pub fn new(inner: S, prefix: impl Into) -> Self { + let mut prefix = prefix.into(); + if !prefix.ends_with('/') { + prefix.push('/'); + } + Self { inner, prefix } + } + + /// The prefix requests must be under. + pub fn prefix(&self) -> &str { + &self.prefix + } + + /// The store requests go to. + pub fn inner(&self) -> &S { + &self.inner + } + + fn check(&self, uri: &str) -> Result<()> { + match uri.strip_prefix(&self.prefix) { + Some(rest) if !rest.split('/').any(|segment| segment == "." || segment == "..") => Ok(()), + _ => Err(Error::Invalid(format!("{uri} is outside {}", self.prefix))), + } + } +} + +impl HasStore for PrefixedStore { + type Store = S; + + fn store(&self) -> &S { + &self.inner + } + + async fn wrapped_get(&self, uri: &str) -> Result>> { + self.check(uri)?; + self.inner.get(uri).await + } + + async fn wrapped_get_range(&self, uri: &str, start: u64, len: u64) -> Result> { + self.check(uri)?; + self.inner.get_range(uri, start, len).await + } + + async fn wrapped_exists(&self, uri: &str) -> Result { + self.check(uri)?; + self.inner.exists(uri).await + } + + async fn wrapped_put(&self, uri: &str, bytes: Vec) -> Result<()> { + self.check(uri)?; + self.inner.put(uri, bytes).await + } + + async fn wrapped_put_if_absent(&self, uri: &str, bytes: Vec) -> Result { + self.check(uri)?; + self.inner.put_if_absent(uri, bytes).await + } + + async fn wrapped_upload(&self, uri: &str) -> Result { + self.check(uri)?; + self.inner.upload(uri).await + } + + async fn wrapped_list(&self, prefix: &str) -> Result> { + self.check(prefix)?; + self.inner.list(prefix).await + } + + async fn wrapped_delete(&self, uris: &[String]) -> Result<()> { + uris.iter().try_for_each(|uri| self.check(uri))?; + self.inner.delete(uris).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{MemoryStore, now}; + + #[test] + fn only_uris_under_the_prefix_pass() { + let store = PrefixedStore::new(MemoryStore::new(), "memory://wh/a"); + assert_eq!(store.prefix(), "memory://wh/a/"); + now(store.put("memory://wh/a/t/1", b"x".to_vec())).unwrap(); + assert!(now(store.exists("memory://wh/a/t/1")).unwrap()); + assert_eq!(now(store.list("memory://wh/a/t/")).unwrap().len(), 1); + for outside in ["memory://wh/b/t/1", "memory://wh/ab/t/1", "memory://wh/a/../b/t/1", "memory://wh/a/./t/1"] { + assert!(matches!(now(store.get(outside)), Err(Error::Invalid(_))), "{outside}"); + } + assert!(matches!(now(store.list("memory://wh/")), Err(Error::Invalid(_)))); + let delete = ["memory://wh/a/t/1".to_string(), "memory://wh/b/t/1".to_string()]; + assert!(matches!(now(store.delete(&delete)), Err(Error::Invalid(_)))); + assert_eq!(store.inner().uris(), ["memory://wh/a/t/1"], "a refused delete deletes nothing"); + } +} diff --git a/crates/loko-store/src/read_only.rs b/crates/loko-store/src/read_only.rs new file mode 100644 index 0000000..af4e62c --- /dev/null +++ b/crates/loko-store/src/read_only.rs @@ -0,0 +1,70 @@ +use std::future::{Future, ready}; + +use crate::{Error, HasStore, Result, Store}; + +/// A store that reads through to another and refuses writes with [`Error::Unsupported`]: for a +/// viewer that must not change the tables it opens. +#[derive(Clone, Debug, Default)] +pub struct ReadOnlyStore { + inner: S, +} + +impl ReadOnlyStore { + /// Reads from `inner`, never writes. + pub fn new(inner: S) -> Self { + Self { inner } + } + + /// The store read. + pub fn inner(&self) -> &S { + &self.inner + } +} + +fn refused(uri: &str) -> Error { + Error::Unsupported(format!("{uri}: the store is read-only")) +} + +impl HasStore for ReadOnlyStore { + type Store = S; + + fn store(&self) -> &S { + &self.inner + } + + fn wrapped_put(&self, uri: &str, _bytes: Vec) -> impl Future> { + ready(Err(refused(uri))) + } + + fn wrapped_put_if_absent(&self, uri: &str, _bytes: Vec) -> impl Future> { + ready(Err(refused(uri))) + } + + fn wrapped_upload(&self, uri: &str) -> impl Future> { + ready(Err(refused(uri))) + } + + fn wrapped_delete(&self, uris: &[String]) -> impl Future> { + ready(Err(refused(uris.first().map_or("", String::as_str)))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{MemoryStore, now}; + + #[test] + fn reads_pass_and_writes_are_refused() { + let memory = MemoryStore::new(); + now(memory.put("memory://t/a", b"abc".to_vec())).unwrap(); + let store = ReadOnlyStore::new(memory.clone()); + assert_eq!(now(store.get("memory://t/a")).unwrap().as_deref(), Some(&b"abc"[..])); + assert_eq!(now(store.list("memory://t/")).unwrap().len(), 1); + assert!(matches!(now(store.put("memory://t/b", vec![])), Err(Error::Unsupported(_)))); + assert!(matches!(now(store.put_if_absent("memory://t/b", vec![])), Err(Error::Unsupported(_)))); + assert!(matches!(now(store.upload("memory://t/b")), Err(Error::Unsupported(_)))); + assert!(matches!(now(store.delete(&["memory://t/a".into()])), Err(Error::Unsupported(_)))); + assert_eq!(memory.uris(), ["memory://t/a"], "nothing changed"); + } +} diff --git a/crates/loko-store/src/store.rs b/crates/loko-store/src/store.rs new file mode 100644 index 0000000..75f09dd --- /dev/null +++ b/crates/loko-store/src/store.rs @@ -0,0 +1,160 @@ +use std::future::Future; + +use crate::Result; + +/// An object found by [`Store::list`]. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Listed { + /// Absolute URI. + pub uri: String, + /// Size in bytes. + pub size: u64, +} + +/// Where a table's objects live: an object store, a directory, memory, or a JavaScript binding. +/// Paths are absolute URIs, as table metadata records them. +/// +/// The futures aren't required to be `Send`, so a store can wrap JavaScript promises. +pub trait Store { + /// A multipart upload in progress. + type Upload: Upload; + + /// The object at `uri`, or `None` if there is none. + fn get(&self, uri: &str) -> impl Future>>>; + /// `len` bytes of the object at `uri` from `start`. + fn get_range(&self, uri: &str, start: u64, len: u64) -> impl Future>>; + /// Whether an object exists at `uri`. + fn exists(&self, uri: &str) -> impl Future>; + /// Writes (or replaces) the object at `uri`. + fn put(&self, uri: &str, bytes: Vec) -> impl Future>; + /// Writes the object at `uri` unless one exists; `false` if one did. The commit primitive. + fn put_if_absent(&self, uri: &str, bytes: Vec) -> impl Future>; + /// Starts a multipart upload to `uri`. + fn upload(&self, uri: &str) -> impl Future>; + /// Objects whose URI starts with `prefix` (a string prefix, not only a directory). + fn list(&self, prefix: &str) -> impl Future>>; + /// Deletes objects; missing ones are not an error. + fn delete(&self, uris: &[String]) -> impl Future>; +} + +/// A type that holds a [`Store`], and so is one: `#[loko::bundle]` implements this for a field, +/// and `#[loko::wraps]` for a wrapper that changes some methods. It's what `#[loko::service]` +/// generates, written out so this crate needs no macros. +pub trait HasStore { + /// The service held. + type Store: Store; + + /// The [`Store`] this holds. + fn store(&self) -> &Self::Store; + + /// Forwards [`Store::get`] to the held service; a `#[loko::wraps]` implementation may replace it. + #[doc(hidden)] + #[inline] + fn wrapped_get(&self, uri: &str) -> impl Future>>> { + self.store().get(uri) + } + + /// Forwards [`Store::get_range`] to the held service; a `#[loko::wraps]` implementation may replace it. + #[doc(hidden)] + #[inline] + fn wrapped_get_range(&self, uri: &str, start: u64, len: u64) -> impl Future>> { + self.store().get_range(uri, start, len) + } + + /// Forwards [`Store::exists`] to the held service; a `#[loko::wraps]` implementation may replace it. + #[doc(hidden)] + #[inline] + fn wrapped_exists(&self, uri: &str) -> impl Future> { + self.store().exists(uri) + } + + /// Forwards [`Store::put`] to the held service; a `#[loko::wraps]` implementation may replace it. + #[doc(hidden)] + #[inline] + fn wrapped_put(&self, uri: &str, bytes: Vec) -> impl Future> { + self.store().put(uri, bytes) + } + + /// Forwards [`Store::put_if_absent`] to the held service; a `#[loko::wraps]` implementation may replace it. + #[doc(hidden)] + #[inline] + fn wrapped_put_if_absent(&self, uri: &str, bytes: Vec) -> impl Future> { + self.store().put_if_absent(uri, bytes) + } + + /// Forwards [`Store::upload`] to the held service; a `#[loko::wraps]` implementation may replace it. + #[doc(hidden)] + #[inline] + fn wrapped_upload(&self, uri: &str) -> impl Future::Upload>> { + self.store().upload(uri) + } + + /// Forwards [`Store::list`] to the held service; a `#[loko::wraps]` implementation may replace it. + #[doc(hidden)] + #[inline] + fn wrapped_list(&self, prefix: &str) -> impl Future>> { + self.store().list(prefix) + } + + /// Forwards [`Store::delete`] to the held service; a `#[loko::wraps]` implementation may replace it. + #[doc(hidden)] + #[inline] + fn wrapped_delete(&self, uris: &[String]) -> impl Future> { + self.store().delete(uris) + } +} + +impl Store for H { + type Upload = <::Store as Store>::Upload; + + #[inline] + fn get(&self, uri: &str) -> impl Future>>> { + self.wrapped_get(uri) + } + + #[inline] + fn get_range(&self, uri: &str, start: u64, len: u64) -> impl Future>> { + self.wrapped_get_range(uri, start, len) + } + + #[inline] + fn exists(&self, uri: &str) -> impl Future> { + self.wrapped_exists(uri) + } + + #[inline] + fn put(&self, uri: &str, bytes: Vec) -> impl Future> { + self.wrapped_put(uri, bytes) + } + + #[inline] + fn put_if_absent(&self, uri: &str, bytes: Vec) -> impl Future> { + self.wrapped_put_if_absent(uri, bytes) + } + + #[inline] + fn upload(&self, uri: &str) -> impl Future> { + self.wrapped_upload(uri) + } + + #[inline] + fn list(&self, prefix: &str) -> impl Future>> { + self.wrapped_list(prefix) + } + + #[inline] + fn delete(&self, uris: &[String]) -> impl Future> { + self.wrapped_delete(uris) + } +} + +/// A multipart upload: parts in order, then [`Upload::complete`]. +pub trait Upload { + /// Sends the next part. Every part but the last must be at least the store's minimum (5 MiB on + /// S3 and R2). + fn put_part(&mut self, bytes: Vec) -> impl Future>; + /// Completes the upload: the object appears. + fn complete(self) -> impl Future>; + /// Drops the parts sent. + fn abort(self) -> impl Future; +} diff --git a/crates/loko/Cargo.toml b/crates/loko/Cargo.toml new file mode 100644 index 0000000..d11a3b4 --- /dev/null +++ b/crates/loko/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "loko" +description = "Services passed to code instead of reached for: Clock, Random, Env, Console, and Store, with test, native, and JavaScript implementations" +readme = "README.md" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true + +[features] +default = ["getrandom"] +# `OsRandom` and `StdPlatform` (native targets and WASI). +getrandom = ["loko-random/getrandom"] +# `TokioClock` and `TokioPlatform` (native targets). +tokio = ["getrandom", "loko-clock/tokio"] +# `JsClock`, `JsRandom`, `JsConsole`, and `JsPlatform` (`wasm32-unknown-unknown` with a JavaScript host). +js = ["loko-clock/js", "loko-random/js", "loko-console/js"] +# `loko::store`: the `Store` service and `MemoryStore`. +store = ["dep:loko-store"] +# `loko::object_store`: a `Store` over an `object_store` client. +object-store = ["store", "dep:loko-object-store"] + +[dependencies] +loko-clock.workspace = true +loko-random.workspace = true +loko-env.workspace = true +loko-console.workspace = true +loko-macros.workspace = true +loko-store = { workspace = true, optional = true } +loko-object-store = { workspace = true, optional = true } + +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt"] } + +[package.metadata.docs.rs] +all-features = true + +[lints] +workspace = true diff --git a/crates/loko/README.md b/crates/loko/README.md new file mode 100644 index 0000000..f3b4dda --- /dev/null +++ b/crates/loko/README.md @@ -0,0 +1,35 @@ +# loko + +Services for Rust programs: the time, random numbers, environment variables, the console, and object storage, passed +to code as traits instead of reached for as globals, with test, system, and JavaScript implementations, and platforms +that bundle one of each. + +```rust +use std::time::{Duration, SystemTime}; + +use loko::Clock; +use loko::clock::TestClock; + +/// Whether a session that started at `started` has expired. +fn expired(clock: &impl Clock, started: SystemTime) -> bool { + clock.now() > started + Duration::from_secs(60 * 60) +} + +let clock = TestClock::at_ms(0); // StdClock, TokioClock, or JsClock in a program +let started = clock.now(); +clock.advance(Duration::from_secs(60 * 60 + 1)); // an hour passes, at once +assert!(expired(&clock, started)); +``` + +| Feature | Adds | +|---|---| +| `getrandom` (default) | `OsRandom` and `StdPlatform` (native and WASI) | +| `tokio` | `TokioClock` and `TokioPlatform` (native) | +| `js` | `JsClock`, `JsRandom`, `JsConsole`, and `JsPlatform` (`wasm32-unknown-unknown`) | +| `store` | `loko::store`: the `Store` service and `MemoryStore` | +| `object-store` | `loko::object_store`: a `Store` over an `object_store` client | + +`#[loko::service]` makes your own traits services and `#[loko::bundle]` bundles them with loko's. The +[repository](https://github.com/polythings/loko) has the details. + +Apache License 2.0. diff --git a/crates/loko/src/lib.rs b/crates/loko/src/lib.rs new file mode 100644 index 0000000..a0359b2 --- /dev/null +++ b/crates/loko/src/lib.rs @@ -0,0 +1,100 @@ +//! Services passed to code instead of reached for: the time, randomness, environment variables, +//! the console, and object storage. Code takes `&impl Clock` (or `&(impl Clock + Random)`, …), the +//! program passes a platform, and tests pass a [`TestPlatform`] they control. The same code runs +//! natively and in WebAssembly. +//! +//! - Services: [`clock`], [`random`], [`env`](mod@env), [`console`], and `store` (feature `store`), each a +//! trait with test, native, and JavaScript implementations. The traits are also at the root. +//! - Platforms: [`TestPlatform`], `StdPlatform`, `TokioPlatform`, and `JsPlatform` bundle one of each. +//! - [`service`] makes a trait a service, [`bundle`] makes a struct each service its fields +//! provide, and [`wraps`] makes a wrapper that changes some of a service's methods and forwards +//! the rest. [`prelude`] brings in the services with the `HasClock`-style traits these name. +//! +//! Service methods take `&self`, so one value can be several services and clones can share state. +//! A service whose state changes keeps it behind a lock or an atomic, locked only for the +//! synchronous part of a call: a `std::sync::Mutex` guard or `RefCell` borrow held across `.await` +//! deadlocks or panics when another task uses the service meanwhile. +//! +//! ``` +//! use std::time::Duration; +//! +//! use loko::{Clock, Random, Rng, TestPlatform}; +//! +//! /// An ID from the time and a random suffix. +//! fn id(services: &(impl Clock + Random)) -> String { +//! format!("{}-{:04x}", services.now_ms(), services.rng().next_u64() & 0xffff) +//! } +//! +//! let platform = TestPlatform::default(); +//! platform.clock.advance(Duration::from_secs(1)); +//! assert_eq!(id(&platform), "1000-7e6f"); +//! ``` +//! +//! A program bundles the services it needs, its own among them: +//! +//! ``` +//! # #[cfg(feature = "store")] { +//! use loko::prelude::*; +//! use loko::store::MemoryStore; +//! +//! #[loko::bundle] +//! struct App { +//! #[loko::has(Store)] +//! store: MemoryStore, +//! #[loko::has(Clock, Random)] +//! platform: TestPlatform, +//! } +//! +//! async fn save(app: &(impl Store + Clock)) -> loko::store::Result<()> { +//! app.put(&format!("memory:///logs/{}", app.now_ms()), b"started".to_vec()).await +//! } +//! # } +//! ``` + +pub use loko_clock as clock; +pub use loko_clock::{Clock, HasClock}; +pub use loko_console as console; +pub use loko_console::{Console, HasConsole}; +pub use loko_env as env; +pub use loko_env::{Env, HasEnv}; +pub use loko_macros::{bundle, has, service, wraps}; +#[cfg(feature = "object-store")] +pub use loko_object_store as object_store; +pub use loko_random as random; +pub use loko_random::{CryptoRng, HasRandom, Random, Rng}; +#[cfg(feature = "store")] +pub use loko_store as store; +#[cfg(feature = "store")] +pub use loko_store::{HasStore, Store}; + +mod platform; + +/// The repository README's examples, run as doctests. +#[cfg(all(doctest, feature = "store", feature = "tokio"))] +#[doc = include_str!("../../../README.md")] +pub struct ReadmeExamples; + +/// The services, the `Has` traits that `#[loko::bundle]` and `#[wraps]` name next to them, the +/// macros, and the platforms. +pub mod prelude { + #[cfg(all(feature = "js", all(target_arch = "wasm32", target_os = "unknown")))] + pub use crate::JsPlatform; + #[cfg(all(feature = "getrandom", not(all(target_arch = "wasm32", target_os = "unknown"))))] + pub use crate::StdPlatform; + #[cfg(all(feature = "tokio", not(target_arch = "wasm32")))] + pub use crate::TokioPlatform; + pub use crate::{ + Clock, Console, Env, HasClock, HasConsole, HasEnv, HasRandom, Random, Rng, TestPlatform, bundle, has, service, + wraps, + }; + #[cfg(feature = "store")] + pub use crate::{HasStore, Store}; +} + +#[cfg(all(feature = "js", all(target_arch = "wasm32", target_os = "unknown")))] +pub use platform::JsPlatform; +#[cfg(all(feature = "getrandom", not(all(target_arch = "wasm32", target_os = "unknown"))))] +pub use platform::StdPlatform; +pub use platform::TestPlatform; +#[cfg(all(feature = "tokio", not(target_arch = "wasm32")))] +pub use platform::TokioPlatform; diff --git a/crates/loko/src/platform.rs b/crates/loko/src/platform.rs new file mode 100644 index 0000000..a5989e8 --- /dev/null +++ b/crates/loko/src/platform.rs @@ -0,0 +1,151 @@ +//! Platforms: one of each service, ready to pass. A platform is each service it holds; a program +//! that needs more (a `Store`, its own services) puts a platform in a `#[loko::bundle]` struct. + +use crate::clock::{Clock, TestClock}; +use crate::console::{Console, TestConsole}; +use crate::env::{Env, MapEnv}; +use crate::random::{Random, SeededRandom}; +use crate::{HasClock, HasConsole, HasEnv, HasRandom, bundle}; + +/// Services for tests: a [`TestClock`] at the Unix epoch, a [`SeededRandom`] with seed 0, an empty +/// [`MapEnv`], and a [`TestConsole`] that records lines. Clones share the clock, the generator, and +/// the console; the fields are public so a test can set them up and look at them. +#[bundle] +#[derive(Clone, Debug, Default)] +#[non_exhaustive] +pub struct TestPlatform { + /// The time, moved by the test. + #[has(Clock)] + pub clock: TestClock, + /// Random numbers, the same on every run. + #[has(Random)] + pub random: SeededRandom, + /// Environment variables set by the test. + #[has(Env)] + pub env: MapEnv, + /// The lines printed. + #[has(Console)] + pub console: TestConsole, +} + +impl TestPlatform { + /// The default test platform. + pub fn new() -> Self { + Self::default() + } + + /// The clock set to `ms` milliseconds after the Unix epoch. + #[must_use] + pub fn at_ms(mut self, ms: i64) -> Self { + self.clock = TestClock::at_ms(ms); + self + } + + /// The generator seeded with `seed`. + #[must_use] + pub fn seeded(mut self, seed: u64) -> Self { + self.random = SeededRandom::new(seed); + self + } + + /// Environment variable `name` set to `value`. + #[must_use] + pub fn with_var(mut self, name: impl Into, value: impl Into) -> Self { + self.env = self.env.with(name, value); + self + } +} + +/// The system's services for programs without an async runtime (feature `getrandom`, native +/// targets and WASI): [`StdClock`](crate::clock::StdClock), whose `sleep` blocks the thread, +/// [`OsRandom`](crate::random::OsRandom), [`SystemEnv`](crate::env::SystemEnv), and +/// [`StdConsole`](crate::console::StdConsole). +#[cfg(all(feature = "getrandom", not(all(target_arch = "wasm32", target_os = "unknown"))))] +#[bundle] +#[derive(Clone, Copy, Debug, Default)] +#[non_exhaustive] +pub struct StdPlatform { + /// The system clock. + #[has(Clock)] + pub clock: crate::clock::StdClock, + /// The operating system's generator. + #[has(Random)] + pub random: crate::random::OsRandom, + /// The process's environment variables. + #[has(Env)] + pub env: crate::env::SystemEnv, + /// Standard output and standard error. + #[has(Console)] + pub console: crate::console::StdConsole, +} + +/// The system's services under tokio (feature `tokio`, native targets): [`StdPlatform`] with +/// [`TokioClock`](crate::clock::TokioClock), whose `sleep` is tokio's timer. +#[cfg(all(feature = "tokio", not(target_arch = "wasm32")))] +#[bundle] +#[derive(Clone, Copy, Debug, Default)] +#[non_exhaustive] +pub struct TokioPlatform { + /// The system clock and tokio's timer. + #[has(Clock)] + pub clock: crate::clock::TokioClock, + /// The operating system's generator. + #[has(Random)] + pub random: crate::random::OsRandom, + /// The process's environment variables. + #[has(Env)] + pub env: crate::env::SystemEnv, + /// Standard output and standard error. + #[has(Console)] + pub console: crate::console::StdConsole, +} + +/// The JavaScript host's services (feature `js`, `wasm32-unknown-unknown`): [`JsClock`](crate::clock::JsClock), +/// [`JsRandom`](crate::random::JsRandom), a [`MapEnv`] (JavaScript hosts have no common environment; +/// set the variables the program reads), and [`JsConsole`](crate::console::JsConsole). +#[cfg(all(feature = "js", all(target_arch = "wasm32", target_os = "unknown")))] +#[bundle] +#[derive(Clone, Debug, Default)] +#[non_exhaustive] +pub struct JsPlatform { + /// `Date.now` and `setTimeout`. + #[has(Clock)] + pub clock: crate::clock::JsClock, + /// `crypto.getRandomValues`. + #[has(Random)] + pub random: crate::random::JsRandom, + /// Variables set by the program. + #[has(Env)] + pub env: MapEnv, + /// `console.log` and `console.error`. + #[has(Console)] + pub console: crate::console::JsConsole, +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + use crate::Rng; + + #[test] + fn a_test_platform_is_each_service() { + let platform = TestPlatform::new().at_ms(1_000).seeded(0).with_var("REGION", "us-east-2"); + platform.clock.advance(Duration::from_millis(500)); + assert_eq!(platform.now_ms(), 1_500); + assert_eq!(platform.rng().next_u64(), 0xA706_DD2F_4D19_7E6F); + assert_eq!(platform.var("REGION").as_deref(), Some("us-east-2")); + platform.print_line("hello").unwrap(); + assert_eq!(platform.console.lines(), ["hello"]); + } + + #[cfg(all(feature = "getrandom", not(all(target_arch = "wasm32", target_os = "unknown"))))] + #[test] + fn the_std_platform_reads_the_system() { + let platform = StdPlatform::default(); + assert!(platform.now_ms() > 1_700_000_000_000); + assert_ne!(platform.rng().next_u64(), platform.rng().next_u64()); + assert_eq!(platform.var("LOKO_SURELY_UNSET"), None); + } +} diff --git a/crates/loko/tests/services.rs b/crates/loko/tests/services.rs new file mode 100644 index 0000000..92ce21b --- /dev/null +++ b/crates/loko/tests/services.rs @@ -0,0 +1,200 @@ +//! Services and bundles defined outside loko work like loko's own. + +use std::time::Duration; + +use loko::clock::TestClock; +use loko::prelude::*; + +/// A service of this crate's own. +#[loko::service] +trait Greeter { + /// A greeting for `name`. + fn greet(&self, name: &str) -> String; +} + +/// A greeter that needs other services: the clock for the time of day. +struct ClockGreeter { + clock: C, +} + +impl Greeter for ClockGreeter { + fn greet(&self, name: &str) -> String { + let hour = self.clock.now_ms() / 3_600_000 % 24; + if hour < 12 { format!("Good morning, {name}") } else { format!("Good afternoon, {name}") } + } +} + +/// A bundle of a platform and the greeter, generic over the greeter. +#[loko::bundle] +struct App { + #[loko::has(Clock, Random, Env, Console)] + platform: TestPlatform, + #[loko::has(Greeter)] + greeter: G, +} + +/// Code that needs a mix of loko's services and this crate's. +fn welcome(app: &(impl Greeter + Console + Env)) { + let name = app.var("USER").unwrap_or_else(|| "stranger".into()); + app.print_line(&app.greet(&name)).expect("a test console doesn't fail"); +} + +#[test] +fn own_services_bundle_with_loko_services() { + let platform = TestPlatform::new().with_var("USER", "Ada"); + let greeter = ClockGreeter { clock: platform.clock.clone() }; + let app = App { platform: platform.clone(), greeter }; + welcome(&app); + platform.clock.advance(Duration::from_hours(13)); + welcome(&app); + assert_eq!(platform.console.lines(), ["Good morning, Ada", "Good afternoon, Ada"]); + assert_eq!(app.rng().next_u64(), 0xA706_DD2F_4D19_7E6F, "Random forwards to the platform"); +} + +/// Tuple structs bundle too. +#[loko::bundle] +struct Pair(#[loko::has(Clock)] TestPlatform, #[loko::has(Greeter)] ClockGreeter); + +#[test] +fn tuple_bundles_forward() { + let platform = TestPlatform::new().at_ms(15 * 3_600_000); + let pair = Pair(platform.clone(), ClockGreeter { clock: platform.clock.clone() }); + assert_eq!(pair.now_ms(), 15 * 3_600_000); + assert_eq!(pair.greet("Lin"), "Good afternoon, Lin"); +} + +#[cfg(feature = "store")] +mod store { + use loko::store::{MemoryStore, Upload}; + + use super::*; + + #[loko::bundle] + #[derive(Clone)] + struct Stored { + #[loko::has(Store)] + store: MemoryStore, + #[loko::has(Clock)] + platform: TestPlatform, + } + + /// Async methods, an associated type, and default methods forward. + #[test] + fn async_services_forward() { + let app = Stored { store: MemoryStore::new(), platform: TestPlatform::new().at_ms(42) }; + let run = async { + let uri = format!("memory:///logs/{}", app.now_ms()); + assert!(app.put_if_absent(&uri, b"a".to_vec()).await.unwrap()); + assert!(!app.put_if_absent(&uri, b"b".to_vec()).await.unwrap()); + let mut upload = app.upload("memory:///big").await.unwrap(); + upload.put_part(b"x".to_vec()).await.unwrap(); + upload.complete().await.unwrap(); + app.sleep(Duration::from_secs(1)).await; + let listed: Vec<_> = app.list("memory:///").await.unwrap().into_iter().map(|l| l.uri).collect(); + (listed, app.get(&uri).await.unwrap()) + }; + let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); + let std::task::Poll::Ready((listed, got)) = std::pin::pin!(run).poll(&mut cx) else { + panic!("memory and test services are ready at once"); + }; + assert_eq!(listed, ["memory:///big", "memory:///logs/42"]); + assert_eq!(got.as_deref(), Some(&b"a"[..])); + assert_eq!(app.platform.now_ms(), 1_042); + } +} + +/// A console that shouts its standard output and passes standard error on as it is. +struct Shouting { + inner: C, +} + +#[loko::wraps(inner: C)] +impl Console for Shouting { + fn print_line(&self, line: &str) -> std::io::Result<()> { + self.inner.print_line(&line.to_uppercase()) + } +} + +/// A greeter around another, adding an exclamation mark. +struct Excited(G); + +#[loko::wraps(0: G)] +impl Greeter for Excited { + fn greet(&self, name: &str) -> String { + format!("{}!", self.0.greet(name)) + } +} + +#[test] +fn wrappers_replace_some_methods_and_forward_the_rest() { + let platform = TestPlatform::new(); + let console = Shouting { inner: platform.console.clone() }; + console.print_line("quiet").unwrap(); + console.print_error_line("as it is").unwrap(); + assert_eq!(platform.console.lines(), ["QUIET"]); + assert_eq!(platform.console.error_lines(), ["as it is"]); + + let greeter = Excited(ClockGreeter { clock: platform.clock.clone() }); + assert_eq!(greeter.greet("Ada"), "Good morning, Ada!"); +} + +#[cfg(feature = "store")] +#[test] +fn store_wrappers_stack() { + use loko::store::{CountingStore, MemoryStore, PrefixedStore, ReadOnlyStore}; + + let memory = MemoryStore::new(); + let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); + let mut run = |f: std::pin::Pin<&mut dyn Future>| f.poll(&mut cx).is_ready(); + assert!(run(std::pin::pin!(async { memory.put("memory://wh/a/t", b"x".to_vec()).await.is_ok() }))); + + let counted = CountingStore::new(ReadOnlyStore::new(PrefixedStore::new(memory.clone(), "memory://wh/a/"))); + assert!(run(std::pin::pin!(async { counted.get("memory://wh/a/t").await.unwrap().is_some() }))); + assert!(run(std::pin::pin!(async { counted.get("memory://wh/b/t").await.is_err() }))); + assert!(run(std::pin::pin!(async { counted.put("memory://wh/a/u", vec![]).await.is_err() }))); + assert_eq!((counted.counts().get, counted.counts().put), (2, 1)); + assert_eq!(memory.uris(), ["memory://wh/a/t"]); +} + +/// A service with a generic method (whose `T` the generated code mustn't clash with) and a method +/// compiled out by `#[cfg]` (whose generated items must go with it). +#[loko::service] +trait Log { + /// The line logged for `value`. + fn line(&self, value: T) -> String; + + /// Never compiled. + #[cfg(any())] + fn never(&self); +} + +struct Plain; + +impl Log for Plain { + fn line(&self, value: T) -> String { + format!("log: {value}") + } +} + +#[loko::bundle] +struct Logged(#[loko::has(Log)] Plain); + +#[test] +fn generic_and_cfg_gated_methods_forward() { + assert_eq!(Logged(Plain).line(42), "log: 42"); +} + +/// A derive written above `#[loko::bundle]` works as well as one below it. +#[derive(Clone, Debug)] +#[loko::bundle] +struct DeriveFirst { + #[loko::has(Clock)] + platform: TestPlatform, +} + +#[test] +fn derives_go_above_or_below_the_bundle() { + let bundle = DeriveFirst { platform: TestPlatform::new().at_ms(5) }; + assert_eq!(bundle.clone().now_ms(), 5); + assert!(format!("{bundle:?}").contains("platform")); +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..7533fe2 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.99" +profile = "minimal" +components = ["rustfmt", "clippy"] diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000..f0ce655 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,3 @@ +style_edition = "2024" +max_width = 120 +use_small_heuristics = "Max" diff --git a/spec/0001-services.md b/spec/0001-services.md new file mode 100644 index 0000000..eac833a --- /dev/null +++ b/spec/0001-services.md @@ -0,0 +1,84 @@ +--- +number: 1 +title: Services, bundles, and wrappers +status: done +updated: 2026-10-21 +versions: [unreleased] +related: [2, 3, 4, 5, 6, 7] +issues: [1, 2, 4, 7] +--- + +# 1. Services, bundles, and wrappers + +**Done** · updated 2026-10-21 · versions unreleased · milestone: [0.2](milestones/v0-2.md) + +## Summary + +A service is a trait for something code would otherwise reach for as a global: the time, random numbers, +environment variables, the console, object storage. Code takes the services it needs as `&impl Trait`, the program +passes the system's, and tests pass ones they control. `#[service]`, `#[loko::bundle]`, and `#[wraps]` let +anyone define new services, bundle them with loko's, and wrap one implementation in another. + +## Design + +- **A service** is a trait marked `#[loko_macros::service]` (`#[loko::service]` through the `loko` crate). Its + methods take `&self`; async ones return `impl Future` with no `Send` bound, so an implementation can wrap JavaScript + promises. The trait can't be generic; it can have associated types, constants, and provided methods. +- **`#[service]` on `trait X`** adds: + - `trait HasX { type X: X; fn x(&self) -> &Self::X; … }` (the accessor is the trait's name in snake case), with + a hidden method `wrapped_m` for each method `m` of `X` that forwards to the held service by default, and + - `impl X for T`, whose methods call the `wrapped_` ones, and whose associated types and constants are the + held service's. Provided methods forward too, so an implementation's overrides are kept. +- **`#[loko::bundle]`** on a struct implements `HasX` for each `#[loko::has(X, …)]` on a field, with the field's + type as the held service (and `FieldType: X` as a bound, so generic fields work). Named and tuple fields both work. + A field can provide several services, such as a platform providing `Clock, Random`. +- **Paths:** `#[loko::bundle]` names `HasX` next to the path given: `#[loko::has(loko::Clock)]` uses `loko::HasClock`, and a + bare `#[loko::has(Clock)]` needs `HasClock` in scope. `loko::prelude` brings in each service with its `HasX`. +- **`#[wraps(field: Type)]`** on `impl X for Wrapper` implements `HasX` instead, with `field` (a name or a tuple + index) of type `Type` as the held service. The methods written become the `wrapped_` methods, replacing the + forwarding ones; `Self::Assoc` in their signatures means the held service's. A wrapper keeps the held service's + associated types and constants, and a provided method it doesn't write forwards to the held one, so a wrapper writes + the provided methods that depend on what it changes (`OffsetClock` writes `now_ms` as well as `now`). Wrappers + stack: `CountingStore>`. +- **`&self` and state:** every method takes `&self`, so one value can be several services (`f(r, r)`), clones can share + state, and services can be held across `.await`. An implementation whose state changes keeps it behind a lock or an + atomic, locked only for the synchronous part of a call; a lock or `RefCell` borrow held across `.await` deadlocks or + panics when another task uses the service meanwhile. +- **Composition:** an implementation of one service can hold others (a greeter that holds a clock), and a program's + bundle can mix loko's services with its own. Code asks only for what it uses: `&(impl Store + Clock)`. +- **Crates:** one per service (`loko-clock`, `loko-random`, `loko-env`, `loko-console`, `loko-store`), each with its + trait and implementations; `loko-object-store`; `loko-macros`; and `loko`, which re-exports them + (`loko::clock`, …; the traits also at the root), adds the platforms ([spec 7](0007-platforms.md)), and has a feature + per optional part. A library that needs one service depends on its crate alone. +- **Targets:** every crate builds for native targets, `wasm32-unknown-unknown`, and `wasm32-wasip2`. JavaScript + implementations need feature `js` and `wasm32-unknown-unknown`; system ones are on every target with a system + (native and WASI). + +## Decisions + +| ID | Decision | Status | +|---|---|---| +| D1 | A crate per service, re-exported by `loko`, so a library takes only the service it needs. | Accepted | +| D2 | Futures aren't `Send`: services can wrap JavaScript promises; multi-threaded executors are an open question (#2). | Accepted | +| D3 | `#[service]` adds `HasX` and a blanket `impl X for T`; `#[loko::bundle]` implements `HasX` per field attribute. A service implemented by hand and one held in a bundle look the same to callers. | Accepted | +| D4 | `#[loko::bundle]` names `HasX` next to the path given, and `loko::prelude` brings each service in with its `HasX`, rather than generating macros named after the traits. | Accepted | +| D5 | The crate is `loko`, at 0.2.0: the name's owner already published an unrelated 0.1.0 in 2022. | Accepted | +| D16 | The things passed are called services (`#[service]`), the word ZIO and most dependency-injection libraries use (ferrous-di, dependency-injector, .NET, Angular). tower's `Service` is a different thing, a request handler; loko exports no `Service` trait, so the two don't meet in code. | Accepted | +| D17 | Bundles are declared with `#[loko::bundle]` on the struct and `#[loko::has(X, …)]` on fields: the struct-level name says what the struct is, the field-level one what each field holds. An attribute macro (not a derive) so the field attribute can be the path `loko::has` and the three macros look alike. | Accepted | +| D18 | Wrappers implement `HasX` through `#[wraps]`, overriding hidden `wrapped_` methods, so they write only what they change. The forwarding lives in `HasX`, whose signatures resolve where the trait is defined, rather than in generated macros that would resolve them at the wrapper. | Accepted | +| D19 | loko's own service crates write out what `#[service]` generates instead of using the macro, so they have no proc-macro dependency (and `loko-store` no `thiserror`): `loko-clock`, `loko-env`, and `loko-console` depend on nothing, `loko-random` only on `rand_core`. | Accepted | + +## Open questions and issues + +- [#1](https://github.com/polythings/loko/issues/1) Publish 0.2.0 to crates.io +- [#2](https://github.com/polythings/loko/issues/2) Send futures for multi-threaded executors +- [#4](https://github.com/polythings/loko/issues/4) Which services come next? +- [#7](https://github.com/polythings/loko/issues/7) Layers: stacking wrappers in reading order + +## History + +- 2026-10-20: built for 0.2, as services with `#[derive(Services)]`. +- 2026-10-21: `#[derive(Services)]` became `#[loko::bundle]` with `#[loko::has]` fields (D17); services were called + resources for a day, and are services again (D16); + `#[wraps]` added (D18). +- 2026-10-21: loko's own services written out by hand, without the macros (D19). diff --git a/spec/0002-clock.md b/spec/0002-clock.md new file mode 100644 index 0000000..ae5b106 --- /dev/null +++ b/spec/0002-clock.md @@ -0,0 +1,52 @@ +--- +number: 2 +title: Clock +status: done +updated: 2026-10-21 +versions: [unreleased] +related: [1, 7] +issues: [] +--- + +# 2. Clock + +**Done** · updated 2026-10-21 · versions unreleased · milestone: [0.2](milestones/v0-2.md) + +## Summary + +The time and a timer. Code that stamps records, measures, retries, or waits takes a `Clock`, so tests set the time +and skip the waits. + +## Design + +- **`Clock`:** `now() -> SystemTime`, `now_ms() -> i64` (provided: milliseconds since the Unix epoch, negative before + it, saturating), and `sleep(Duration) -> impl Future`. +- **`from_ms(i64) -> Option`:** the inverse of `now_ms`, `None` where the platform can't hold the time + (before the epoch on WebAssembly: WASI and `wasm32-unknown-unknown`). +- **`TestClock`:** starts where told (`new`, `at_ms`; the Unix epoch by default), moves on `advance` and `set`, and + on `sleep`, which moves it forward by the sleep and returns at once. Clones share the time. +- **`StdClock`:** `SystemTime::now`, and a `sleep` that blocks the thread before returning a ready future: for + programs without an async runtime (native and WASI). +- **`TokioClock`** (feature `tokio`, native): `SystemTime::now` and `tokio::time::sleep`. +- **`OffsetClock`:** a wrapper ([spec 1](0001-services.md)): another clock, `ahead` or `behind` by a fixed + `Duration`, for testing code against clock skew. It writes `now_ms` as well as `now`; sleeping is the other clock's. +- **`to_ms(SystemTime) -> i64`:** what `now_ms` computes, for wrappers that change `now`. +- **`JsClock`** (feature `js`, `wasm32-unknown-unknown`): `Date.now` and a promise around `setTimeout`, in steps of + at most 2³¹ − 1 ms (the longest delay `setTimeout` holds); resolved at once on a host without `setTimeout`. + +## Decisions + +| ID | Decision | Status | +|---|---|---| +| D6 | `now` returns `SystemTime` and `sleep` takes `Duration`, the standard library's types; `now_ms` is provided for the common case. | Accepted | +| D7 | `TestClock`'s `sleep` moves the clock and returns at once, so code that waits runs in tests without waiting. | Accepted | +| D8 | `StdClock` blocks in `sleep`, documented, so programs without a runtime have a system clock; async programs use their runtime's clock. | Accepted | + +## Open questions and issues + +None. + +## History + +- 2026-10-20: built for 0.2. +- 2026-10-21: `OffsetClock` and `to_ms` added. diff --git a/spec/0003-random.md b/spec/0003-random.md new file mode 100644 index 0000000..4c6058c --- /dev/null +++ b/spec/0003-random.md @@ -0,0 +1,52 @@ +--- +number: 3 +title: Random +status: done +updated: 2026-10-21 +versions: [unreleased] +related: [1, 7] +issues: [] +--- + +# 3. Random + +**Done** · updated 2026-10-21 · versions unreleased · milestone: [0.2](milestones/v0-2.md) + +## Summary + +A source of random number generators. Code that makes identifiers, jitter, or samples takes a `Random` and asks it for +a generator of its own, so tests get the same numbers on every run and drawing many numbers takes no locks. + +## Design + +- **`Random`:** `type Rng: rand_core::Rng` and `rng(&self) -> Self::Rng`: a generator of the caller's own, drawn from + with `&mut` and no locks, usable with the whole `rand` ecosystem. Each implementation chooses the generator, and so + whether it's fit for secrets: code that makes secrets asks for `Random`. +- **`SeededRandom`:** one SplitMix64 sequence from a seed (0 by default), shared by clones; each `rng()` is a + `SeededRng` (SplitMix64) seeded from the sequence's next number, so the same seed and the same calls give the same + numbers on every platform. Not for secrets. `SeededRng` is also a `SeedableRng`. +- **`OsRandom`** (feature `getrandom`, native and WASI): `OsRng`, which reads the operating system's generator on + every draw. A `CryptoRng`. +- **`JsRandom`** (feature `js`, `wasm32-unknown-unknown`): `JsRng`, which calls `crypto.getRandomValues` in chunks of + at most 65,536 bytes. A `CryptoRng`; it panics on a host without Web Crypto rather than fall back to numbers that + aren't. +- **`Mutex`** is a `Random` for any `R: SeedableRng + Rng`, so a program's existing generator works as a service; + `rng()` forks it (`SeedableRng::fork`). +- `rand_core`'s `Rng`, `CryptoRng`, and `SeedableRng` are re-exported, and `Rng` is in `loko::prelude`. + +## Decisions + +| ID | Decision | Status | +|---|---|---| +| D9 | `Random` hands out generators (`rng()`) rather than numbers: one lock at most per generator instead of one per number, `&mut` generators as the `rand` ecosystem expects, and the generator's type says whether it's fit for secrets. | Accepted | +| D10 | `SeededRandom` is SplitMix64: small, fast, statistically good enough for tests, and the same on every platform. | Accepted | + +## Open questions and issues + +None. + +## History + +- 2026-10-20: built for 0.2. +- 2026-10-21: `Random` hands out generators (`rng()`, D9) instead of `next_u64` and `fill`; the `rand` feature is gone + (`rand_core` is always a dependency); `JsRandom` no longer falls back to `Math.random`. diff --git a/spec/0004-env.md b/spec/0004-env.md new file mode 100644 index 0000000..51c68fc --- /dev/null +++ b/spec/0004-env.md @@ -0,0 +1,39 @@ +--- +number: 4 +title: Env +status: done +updated: 2026-10-20 +versions: [unreleased] +related: [1, 7] +issues: [3] +--- + +# 4. Env + +**Done** · updated 2026-10-20 · versions unreleased · milestone: [0.2](milestones/v0-2.md) + +## Summary + +Environment variables. Code that reads configuration takes an `Env`, so tests set their own and hosts without a +process environment supply a map. + +## Design + +- **`Env`:** `var(name) -> Option` (`None` when unset or not Unicode) and `vars() -> Vec<(String, String)>`. +- **`SystemEnv`:** `std::env`. On `wasm32-unknown-unknown` nothing is set. +- **`MapEnv`:** variables in a sorted map: `new`, `with` (builder), `set`, `remove`, and `FromIterator`. Clones share + the map. `JsPlatform` uses one ([spec 7](0007-platforms.md)). + +## Decisions + +| ID | Decision | Status | +|---|---|---| +| D11 | JavaScript hosts get a `MapEnv` the application fills, since browsers, Node, Deno, and Workers keep variables in different places (#3). | Accepted | + +## Open questions and issues + +- [#3](https://github.com/polythings/loko/issues/3) Environment variables in JavaScript hosts + +## History + +- 2026-10-20: built for 0.2. diff --git a/spec/0005-console.md b/spec/0005-console.md new file mode 100644 index 0000000..46a088f --- /dev/null +++ b/spec/0005-console.md @@ -0,0 +1,40 @@ +--- +number: 5 +title: Console +status: done +updated: 2026-10-20 +versions: [unreleased] +related: [1, 7] +issues: [] +--- + +# 5. Console + +**Done** · updated 2026-10-20 · versions unreleased · milestone: [0.2](milestones/v0-2.md) + +## Summary + +Lines to standard output and standard error. Code that reports progress or results takes a `Console`, so tests +read what it printed and code in a browser logs to the developer console. + +## Design + +- **`Console`:** `print_line(&str) -> io::Result<()>` and `print_error_line(&str) -> io::Result<()>`; each writes the + line and a newline. +- **`StdConsole`:** the process's standard output and error, locked for the line. +- **`TestConsole`:** keeps the lines (`lines`, `error_lines`). Clones share them. +- **`JsConsole`** (feature `js`, `wasm32-unknown-unknown`): `console.log` and `console.error`. + +## Decisions + +| ID | Decision | Status | +|---|---|---| +| D12 | Lines rather than a byte stream: what programs print to a console, and what JavaScript consoles take. | Accepted | + +## Open questions and issues + +None. + +## History + +- 2026-10-20: built for 0.2. diff --git a/spec/0006-store.md b/spec/0006-store.md new file mode 100644 index 0000000..74ba592 --- /dev/null +++ b/spec/0006-store.md @@ -0,0 +1,56 @@ +--- +number: 6 +title: Store +status: done +updated: 2026-10-21 +versions: [unreleased] +related: [1] +issues: [6] +--- + +# 6. Store + +**Done** · updated 2026-10-21 · versions unreleased · milestone: [0.2](milestones/v0-2.md) + +## Summary + +Objects by absolute URI. Code that reads and writes files in object storage takes a `Store`, so the same code works +over S3, a local directory, memory, or a JavaScript binding. + +## Design + +- **`Store`:** `get` (the object or `None`), `get_range`, `exists`, `put`, `put_if_absent` (a conditional write, + `false` when an object is there: the commit primitive of table formats), `upload` (a multipart `Upload`: parts in + order, then `complete` or `abort`), `list` (by string prefix, not only by directory), and `delete` (missing objects + are not an error). Paths are absolute URIs. +- **`Error`:** `NotFound`, `Invalid`, `Unsupported`, and `Store` (the backend's error, as text); `non_exhaustive`. + Libraries with their own error type convert from it. +- **`MemoryStore`:** objects in memory, for tests. Clones share the objects. +- **Wrappers** ([spec 1](0001-services.md)), each a `Store` around another; they stack: + - `CountingStore`: counts the calls of each kind it passes on (`counts()`, `reset()`); clones share the counts. + - `ReadOnlyStore`: reads through; `put`, `put_if_absent`, `upload`, and `delete` return `Unsupported`. + - `PrefixedStore`: passes on only URIs under its prefix (a `/` is added if missing), refusing the rest and any `.` + or `..` segment with `Invalid`. A `delete` with one URI outside deletes nothing. +- **`ObjectStoreAdapter`** (`loko-object-store`): a `Store` over any `object_store` client, from a client and a root + URI or from a URL with options (the root is everything before the path: scheme, userinfo such as Azure's + filesystem, host, and port; a password shows as `***` in errors and `Debug`). Put-if-absent is + `PutMode::Create`. An empty range, or one starting past the end, reads as no bytes, as in `MemoryStore`. Listing a + prefix lists its directory from an offset just before the prefix, so `…/metadata/v` skips the manifests before it. The crate enables no backends; the application + turns on the ones it needs on its own `object_store` dependency. +- **Not in the platforms:** a program bundles a store with a platform ([spec 7](0007-platforms.md)). + +## Decisions + +| ID | Decision | Status | +|---|---|---| +| D13 | `Store` and the `object_store` adapter are loko crates of their own, so any library can take a store without a storage client; a library with its own error type converts from `loko_store::Error`. | Accepted | + +## Open questions and issues + +- [#6](https://github.com/polythings/loko/issues/6) Store adapter for OpenDAL (loko-opendal) + +## History + +- 2026-10-20: built for 0.2. +- 2026-10-21: review fixes: `from_url` keeps the port, empty and past-the-end ranges read as no bytes, + and listing starts from an offset. Wrappers `CountingStore`, `ReadOnlyStore`, and `PrefixedStore` added. diff --git a/spec/0007-platforms.md b/spec/0007-platforms.md new file mode 100644 index 0000000..0587fb9 --- /dev/null +++ b/spec/0007-platforms.md @@ -0,0 +1,49 @@ +--- +number: 7 +title: Platforms +status: done +updated: 2026-10-20 +versions: [unreleased] +related: [1, 2, 3, 4, 5] +issues: [3] +--- + +# 7. Platforms + +**Done** · updated 2026-10-20 · versions unreleased · milestone: [0.2](milestones/v0-2.md) + +## Summary + +A platform is one of each of `Clock`, `Random`, `Env`, and `Console`, ready to pass: the test services, the +system's, the system's under tokio, or a JavaScript host's. A program that needs more puts a platform in its own +`#[loko::bundle]` bundle. + +## Design + +| Platform | Clock | Random | Env | Console | Targets | +|---|---|---|---|---|---| +| `TestPlatform` | `TestClock` | `SeededRandom` | `MapEnv` | `TestConsole` | All | +| `StdPlatform` | `StdClock` | `OsRandom` | `SystemEnv` | `StdConsole` | Native and WASI (feature `getrandom`, on by default) | +| `TokioPlatform` | `TokioClock` | `OsRandom` | `SystemEnv` | `StdConsole` | Native (feature `tokio`) | +| `JsPlatform` | `JsClock` | `JsRandom` | `MapEnv` | `JsConsole` | `wasm32-unknown-unknown` (feature `js`) | + +- Each platform is a struct with public fields, one per service, built with `#[loko::bundle]`, so it is each + service and a test can set up and inspect the parts (`platform.clock.advance(…)`, `platform.console.lines()`). +- The structs are `non_exhaustive`, so a service added later isn't a breaking change; they are built with `default()` + (and `TestPlatform`'s `at_ms`, `seeded`, and `with_var`). +- `Store` is not in a platform: which store a program uses is its own choice. + +## Decisions + +| ID | Decision | Status | +|---|---|---| +| D14 | Platforms bundle `Clock`, `Random`, `Env`, and `Console`, without `Store`, and are named for where they run (`TestPlatform`, `StdPlatform`, `TokioPlatform`, `JsPlatform`). | Accepted | +| D15 | JavaScript implementations are for `wasm32-unknown-unknown` only; system implementations, and `StdPlatform`, also build for WASI. | Accepted | + +## Open questions and issues + +- [#3](https://github.com/polythings/loko/issues/3) Environment variables in JavaScript hosts + +## History + +- 2026-10-20: built for 0.2. diff --git a/spec/README.md b/spec/README.md new file mode 100644 index 0000000..1a98047 --- /dev/null +++ b/spec/README.md @@ -0,0 +1,48 @@ +# Specs + +loko's design. Start here, people and agents alike. + +- **Specs** (`NNNN-name.md`): one feature each, with the same template ([TEMPLATE.md](TEMPLATE.md)). A spec says what + the feature does and why, what is still open (as GitHub issues), how far it is built, and the releases that changed + it. +- **Milestones** ([`milestones/`](milestones/README.md)): goals made of specs. The roadmap is the list of milestones. + +## Statuses + +| Status | Meaning | +|---|---| +| Done | Built and in effect; the spec describes how it works now. | +| In progress | Partly built: the spec says which parts are, and the open parts have issues. | +| Planned | Agreed or proposed, not built yet. | +| Dropped | Discussed and not done; kept so the reasons stay on record. | + +## Writing and changing a spec + +- **When:** anything beyond a small fix gets a spec, or a change to one, agreed before it is built. A new service is + a new spec. +- **New spec:** copy [TEMPLATE.md](TEMPLATE.md) to `NNNN-short-name.md` with the next free number (numbers are never + reused), and add it to a milestone and to the index below. +- **Front matter:** `number`, `title`, `status` (`done`, `in-progress`, `planned`, `dropped`), `updated`, `versions` + (the releases that changed it; `unreleased` until a release names it), and optionally `related` and `issues`. +- **Open questions** are GitHub issues (label `question`), listed under `issues`, not paragraphs in the spec. +- **Same commit:** when work lands, its spec's status, versions, and history change with the code and the changelog. + +## Index + +7 specs: 7 done. Decisions D1–D19. + +| # | Spec | Status | Updated | +|---:|---|---|---| +| 1 | [Services, bundles, and wrappers](0001-services.md) | Done | 2026-10-21 | +| 2 | [Clock](0002-clock.md) | Done | 2026-10-21 | +| 3 | [Random](0003-random.md) | Done | 2026-10-21 | +| 4 | [Env](0004-env.md) | Done | 2026-10-20 | +| 5 | [Console](0005-console.md) | Done | 2026-10-20 | +| 6 | [Store](0006-store.md) | Done | 2026-10-21 | +| 7 | [Platforms](0007-platforms.md) | Done | 2026-10-20 | + +## Milestones + +| Milestone | Status | +|---|---| +| [0.2: First release](milestones/v0-2.md) | In progress | diff --git a/spec/TEMPLATE.md b/spec/TEMPLATE.md new file mode 100644 index 0000000..1b58a04 --- /dev/null +++ b/spec/TEMPLATE.md @@ -0,0 +1,36 @@ +--- +number: 0 +title: Short title +status: planned +updated: YYYY-MM-DD +versions: [] +related: [] +issues: [] +--- + +# 0. Short title + +**Planned** · updated YYYY-MM-DD · milestone: [Name](milestones/slug.md) + +## Summary + +Two to four sentences: what this feature does, for whom, and why. + +## Design + +How it works, as precisely as needed to build or review it: types, options, interfaces, edge cases. Cite other specs +as "spec N", linked to their files. + +## Decisions + +| ID | Decision | Status | +|---|---|---| +| D0 | One sentence | Proposed | + +## Open questions and issues + +- [#N](https://github.com/polythings/loko/issues/N) Each open question or task is an issue. + +## History + +- YYYY-MM-DD: proposed. diff --git a/spec/milestones/README.md b/spec/milestones/README.md new file mode 100644 index 0000000..3810755 --- /dev/null +++ b/spec/milestones/README.md @@ -0,0 +1,7 @@ +# Milestones + +The roadmap: goals made of specs. A milestone is done, in progress, planned, or exploring (an idea being looked into). + +| Milestone | Status | Specs | +|---|---|---| +| [0.2: First release](v0-2.md) | In progress | 1, 2, 3, 4, 5, 6, 7 | diff --git a/spec/milestones/v0-2.md b/spec/milestones/v0-2.md new file mode 100644 index 0000000..89a64a2 --- /dev/null +++ b/spec/milestones/v0-2.md @@ -0,0 +1,18 @@ +# 0.2: First release + +**In progress** + +The first release as a services library: `Clock`, `Random`, `Env`, `Console`, and `Store` with test, system, and +JavaScript implementations, the platforms, and the macros for new services and bundles. + +| # | Spec | Status | +|---:|---|---| +| 1 | [Services, bundles, and wrappers](../0001-services.md) | Done | +| 2 | [Clock](../0002-clock.md) | Done | +| 3 | [Random](../0003-random.md) | Done | +| 4 | [Env](../0004-env.md) | Done | +| 5 | [Console](../0005-console.md) | Done | +| 6 | [Store](../0006-store.md) | Done | +| 7 | [Platforms](../0007-platforms.md) | Done | + +**Done when:** the crates are on crates.io ([#1](https://github.com/polythings/loko/issues/1)).