From ef7ad53a6b494ee292410531b54b8dbac40eb06c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 19:35:48 +0000 Subject: [PATCH 1/2] lua: native modules come only from the READ-only libs grant Pairs with MicroPerceptron/kore ft/cool-bardeen-f86yix (d929d9a): the store now takes an attesting subtype (EXECUTABLE_IMAGE) only through BIND_EXECUTABLE, at create and on every later byte change, and kore-late resolves native modules itself through the task's READ-only `libs` grant (never a path walk, never the task's own tree), refusing a `libs` edge the task could write beneath (LoadError::LibsWritable). `native/` is re-exported from that kore revision with tools/sync-native.py (it was 750 commits behind kore main; both lockfiles follow). The bridge's `require` keeps its source search (own tree, then libs) and drops the native one: `Loader::load_native(fs, stem)`, plus the LibsWritable refusal text. docs/lua.md states the rule. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BDbTpiJFNzUJDBdjhHtXDm --- Cargo.lock | 2 + crates/lua/src/bridge.rs | 32 +- docs/lua.md | 22 +- native/Cargo.lock | 10 +- native/crates/abi/src/cap.rs | 2 + native/crates/abi/src/guest/aarch64/mod.rs | 6 + native/crates/abi/src/guest/mod.rs | 74 + native/crates/abi/src/guest/riscv64/mod.rs | 6 + .../abi/src/{guest.rs => guest/wire.rs} | 197 +- native/crates/abi/src/guest/x86_64/mod.rs | 29 + native/crates/abi/src/image.rs | 8 +- native/crates/abi/src/iommu_status.rs | 249 ++ native/crates/abi/src/lib.rs | 2 + native/crates/abi/src/machine.rs | 53 +- native/crates/abi/src/monitor.rs | 63 +- native/crates/abi/src/process.rs | 92 +- native/crates/abi/src/processor_status.rs | 282 ++- native/crates/abi/src/session.rs | 288 ++- native/crates/abi/src/stor.rs | 309 +++ native/crates/abi/src/svc.rs | 350 ++- native/crates/abi/src/task.rs | 33 +- native/crates/abi/src/uring.rs | 5 + native/crates/collections/src/boundedvec.rs | 5 +- native/crates/dagfs/Cargo.toml | 4 +- native/crates/dagfs/src/client.rs | 152 +- native/crates/dagfs/src/error.rs | 7 + native/crates/dagfs/src/fs.rs | 49 + native/crates/dagfs/src/graph.rs | 5 +- native/crates/dagfs/src/io.rs | 32 +- native/crates/dagfs/src/lib.rs | 18 +- native/crates/dagfs/src/object.rs | 75 +- native/crates/dagfs/src/protocol.rs | 98 +- native/crates/dagfs/src/stor.rs | 1193 ++++++++++ native/crates/dagfs/src/walk.rs | 15 +- native/crates/dagfs/src/wire.rs | 448 +++- native/crates/driver/Cargo.toml | 2 +- native/crates/driver/src/accel.rs | 118 +- native/crates/driver/src/block/memory.rs | 148 ++ native/crates/driver/src/block/mod.rs | 197 +- native/crates/driver/src/block/region.rs | 99 +- native/crates/driver/src/block/shared.rs | 986 -------- .../driver/src/common/pagetable/alloc.rs | 162 +- .../driver/src/common/pagetable/walker.rs | 4 +- native/crates/driver/src/cookie.rs | 202 +- native/crates/driver/src/gpt.rs | 19 + native/crates/driver/src/host.rs | 324 ++- native/crates/driver/src/lib.rs | 1 + native/crates/driver/src/ring.rs | 28 + native/crates/driver/src/seam.rs | 37 + native/crates/driver/src/stats.rs | 4 + native/crates/driver/src/stor/engine.rs | 2113 +++++++++++++++++ native/crates/driver/src/stor/grant.rs | 214 ++ native/crates/driver/src/stor/lanes.rs | 1003 ++++++++ native/crates/driver/src/stor/mod.rs | 282 +++ native/crates/driver/src/stor/timeline.rs | 178 ++ native/crates/late/src/lib.rs | 144 +- native/crates/late/src/metal.rs | 30 +- native/crates/ld/src/place.rs | 18 +- native/crates/ld/tests/built_images.rs | 139 +- native/crates/ld/tests/link_engine.rs | 16 +- native/crates/ring/src/event.rs | 5 + native/crates/ring/src/lib.rs | 1 + native/crates/ring/src/stor.rs | 686 ++++++ native/crates/rt/Cargo.toml | 1 + native/crates/rt/src/barrier.rs | 3 +- native/crates/rt/src/cap/mod.rs | 28 +- native/crates/rt/src/cap/verbs.rs | 544 ++++- native/crates/rt/src/chan.rs | 143 +- native/crates/rt/src/entry.rs | 14 + native/crates/rt/src/executor/time.rs | 388 ++- native/crates/rt/src/heap/mod.rs | 2 +- native/crates/rt/src/heap/source.rs | 25 +- native/crates/rt/src/pinned.rs | 6 +- native/crates/rt/src/process.rs | 346 ++- native/crates/rt/src/process/interrupt.rs | 142 ++ native/crates/rt/src/service/mod.rs | 14 + native/crates/rt/src/session/builder.rs | 46 +- native/crates/rt/src/session/tests.rs | 59 +- native/crates/rt/src/spmd.rs | 458 +++- native/crates/rt/src/star.rs | 127 +- native/crates/rt/src/svc/client.rs | 222 +- native/crates/rt/src/svc/mod.rs | 57 +- native/crates/rt/src/svc/serve.rs | 32 + native/crates/rt/src/thread.rs | 11 +- native/crates/rt/src/transport/mod.rs | 5 +- native/crates/window/src/client.rs | 30 +- native/crates/window/src/lib.rs | 16 + native/provenance.json | 155 +- native/services/extract/fuzz/Cargo.lock | 1 + 89 files changed, 12482 insertions(+), 1768 deletions(-) create mode 100644 native/crates/abi/src/guest/aarch64/mod.rs create mode 100644 native/crates/abi/src/guest/mod.rs create mode 100644 native/crates/abi/src/guest/riscv64/mod.rs rename native/crates/abi/src/{guest.rs => guest/wire.rs} (67%) create mode 100644 native/crates/abi/src/guest/x86_64/mod.rs create mode 100644 native/crates/abi/src/iommu_status.rs create mode 100644 native/crates/abi/src/stor.rs create mode 100644 native/crates/dagfs/src/stor.rs delete mode 100644 native/crates/driver/src/block/shared.rs create mode 100644 native/crates/driver/src/stor/engine.rs create mode 100644 native/crates/driver/src/stor/grant.rs create mode 100644 native/crates/driver/src/stor/lanes.rs create mode 100644 native/crates/driver/src/stor/mod.rs create mode 100644 native/crates/driver/src/stor/timeline.rs create mode 100644 native/crates/ring/src/stor.rs create mode 100644 native/crates/rt/src/process/interrupt.rs diff --git a/Cargo.lock b/Cargo.lock index 6b72e97..83e2d02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -813,6 +813,7 @@ dependencies = [ "heapless", "kore-abi", "kore-collections", + "kore-ring", "kore-rt", "kore-svc-extract", "zerocopy", @@ -900,6 +901,7 @@ dependencies = [ name = "kore-rt" version = "0.0.1" dependencies = [ + "heapless", "kore-abi", "kore-collections", "kore-hint", diff --git a/crates/lua/src/bridge.rs b/crates/lua/src/bridge.rs index fb0e246..cd071eb 100644 --- a/crates/lua/src/bridge.rs +++ b/crates/lua/src/bridge.rs @@ -49,10 +49,16 @@ //! remote list — or as pointers into a frame the owner keeps alive across //! the whole phase (the eval's script). //! -//! Resolution order (`SEARCH`): the task's own tree, then `libs/`; a source -//! module `.lua` wins over a native `.so`. Native modules are -//! store objects stamped `EXECUTABLE_IMAGE` that `kore-late` checks, places -//! in the JIT arena, and links against this executable's exported C API. +//! Resolution: a source module `.lua` from `SOURCE_SEARCH` — the +//! task's own tree, then `libs/` — wins over a native `.so`, which +//! comes from the READ-only `libs/` view alone. Source runs inside the VM +//! with the VM's authority; native code runs with the task's whole +//! authority, so it is never taken from a tree the task can write (nor can +//! a squatting `.so` there shadow the library). `kore-late` holds +//! that rule: it resolves the library edge itself, refuses a member not +//! stamped `EXECUTABLE_IMAGE` (a stamp only the install authority can +//! make), checks it, places it in the JIT arena, and links it against this +//! executable's exported C API. use alloc::vec::Vec; use core::cell::UnsafeCell; @@ -81,9 +87,10 @@ use crate::{Answer, Extension, OP_EMBEDDER_BASE, Policy}; /// engine sees it. Larger sources belong in the store as several modules. pub const MAX_SOURCE_MODULE_BYTES: usize = 64 * 1024; -/// Where `require` looks, in order: the task's own tree (its cwd view), -/// then the READ-only `libs/` view every task carries. -const SEARCH: &[&[u8]] = &[b"", b"libs"]; +/// Where `require` looks for a *source* module, in order: the task's own +/// tree (its cwd view), then the READ-only `libs/` view every task carries. +/// A native module takes no search list (`Loader::load_native`). +const SOURCE_SEARCH: &[&[u8]] = &[b"", kore_late::LIBS_VIEW.as_bytes()]; /// The prefix a native module's opener carries: `luaopen_`. const OPENER_PREFIX: &[u8] = b"luaopen_"; @@ -1112,7 +1119,9 @@ enum Reply { } async fn resolve(services: &mut Services<'_, '_>, stem: &[u8], policy: Policy) -> Reply { - let search = Search { prefixes: SEARCH }; + let search = Search { + prefixes: SOURCE_SEARCH, + }; let Some(fs) = services.fs().await else { return Reply::Error(c"filesystem face is unavailable"); }; @@ -1143,7 +1152,7 @@ async fn resolve(services: &mut Services<'_, '_>, stem: &[u8], policy: Policy) - let (Some(fs), Some(loader)) = (services.fs.as_mut(), services.loader.as_mut()) else { return Reply::Error(c"filesystem face is unavailable"); }; - let id = match loader.load_native(fs, search, stem).await { + let id = match loader.load_native(fs, stem).await { Ok(id) => id, Err(error) => return Reply::Error(describe(error)), }; @@ -1165,7 +1174,10 @@ async fn resolve(services: &mut Services<'_, '_>, stem: &[u8], policy: Policy) - fn describe(error: LoadError) -> &'static CStr { match error { LoadError::Name => c"module name is not a store name", - LoadError::NotFound => c"no .lua in the task tree and no libs/.so", + LoadError::NotFound => { + c"no .lua in the task tree or libs/, and no libs/.so (native modules load only from libs/)" + } + LoadError::LibsWritable => c"libs/ is writable by this task; native modules load only from the READ-only library view", LoadError::NotExecutableImage => c"libs/ member is not stamped EXECUTABLE_IMAGE", LoadError::TooLarge => c"module exceeds the bounded loader", LoadError::TooMany => c"module table is full", diff --git a/docs/lua.md b/docs/lua.md index ae98dfe..ee8f4f9 100644 --- a/docs/lua.md +++ b/docs/lua.md @@ -542,13 +542,21 @@ remember. ## `require` -`require ""` resolves against the task's own tree first, then `libs/`; a -source module `.lua` wins over a native `.so`. Source modules are -bounded at 64 KiB — larger sources belong in the store as several modules. - -A native module is a store object stamped `EXECUTABLE_IMAGE` that is checked, -placed in the JIT arena, and linked against this executable's exported C API. It -must export `luaopen_`. +`require ""` looks for a source module `.lua` in the task's own +tree first, then `libs/`; a source module wins over a native `.so`. Source +modules run inside the VM with the VM's authority and are bounded at 64 KiB — +larger sources belong in the store as several modules. + +A native module runs with the task's whole authority, so it comes from the +READ-only `libs/` view **alone** — never from a tree the task can write, and a +`.so` squatting there cannot shadow the library. `kore-late` holds that +rule: it enters the `libs` grant directly (not by path, which a `libs` child of +the task's own root would answer first), refuses one the task could write +beneath, and refuses a member unless it is stamped `EXECUTABLE_IMAGE` — a stamp +the store takes, and keeps over later byte changes, only through the install +authority's `BIND_EXECUTABLE`. The member is then checked, placed in the JIT +arena, and linked against this executable's exported C API. It must export +`luaopen_`. ## Errors diff --git a/native/Cargo.lock b/native/Cargo.lock index 6b94cf8..f135334 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -248,6 +248,7 @@ dependencies = [ "heapless", "kore-abi", "kore-collections", + "kore-ring", "kore-rt", "kore-svc-extract", "kore-svc-extract-worker-rust", @@ -314,6 +315,7 @@ dependencies = [ name = "kore-rt" version = "0.0.1" dependencies = [ + "heapless", "kore-abi", "kore-collections", "kore-hint", @@ -598,9 +600,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "virtio-accel-core" -version = "0.3.10" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758d9a32f90f88ae4ae50a7dfb13aa214b20c9a85b797b0eefbe0f77b165840f" +checksum = "6522497e9e97db9c8cb695987397efb222b78f5279870c929730ad5d14f8fe65" dependencies = [ "bitflags", "virtio-accel-transport", @@ -608,9 +610,9 @@ dependencies = [ [[package]] name = "virtio-accel-transport" -version = "0.3.10" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71764cd76e27b5923526e3e1384a317132319d4150f7263abe57fd3f6809afe5" +checksum = "c7747200a37db419a82c884730d5c31d1d80a6b4d14cc3de312064e4793dc289" [[package]] name = "winnow" diff --git a/native/crates/abi/src/cap.rs b/native/crates/abi/src/cap.rs index fbc3f18..2b767f1 100644 --- a/native/crates/abi/src/cap.rs +++ b/native/crates/abi/src/cap.rs @@ -236,6 +236,8 @@ pub enum ResourceKind { /// peer window it sponsored before the epoch may be reclaimed. /// Append-only wire. DeviceMmu, + /// Boot-discovered memory controller; currently kernel-owned. + MemoryController, } /// The stamped capability record as it lives in the shared cap arena — the diff --git a/native/crates/abi/src/guest/aarch64/mod.rs b/native/crates/abi/src/guest/aarch64/mod.rs new file mode 100644 index 0000000..396bd7e --- /dev/null +++ b/native/crates/abi/src/guest/aarch64/mod.rs @@ -0,0 +1,6 @@ +//! AArch64 guest architecture vocabulary. +//! +//! The kernel's EL2 guest backend is not implemented yet. No register +//! indices are exported until that backend defines its wire conversion; +//! in particular, this module must not inherit x86-64's register numbering. +//! Shared records and semantic role positions remain in the parent module. diff --git a/native/crates/abi/src/guest/mod.rs b/native/crates/abi/src/guest/mod.rs new file mode 100644 index 0000000..40c0f1e --- /dev/null +++ b/native/crates/abi/src/guest/mod.rs @@ -0,0 +1,74 @@ +//! Guest exit/control wire records — the **per-vCPU window vocabulary** +//! (#487/#481), frozen here because kernel and hypervisor are two +//! endpoints of one contract and can never drift on a layout. +//! +//! One page per vCPU carries **two SPSC rings** (`kore-ring`'s frozen +//! `SPSC_ABI = 1` format, one ring per direction): +//! +//! ```text +//! offset 0: exit ring (kernel → hypervisor, +//! capacity 2, GuestExitRecord) +//! offset EXIT_RING_BYTES: control ring (hypervisor → kernel, +//! capacity 4, GuestCtrlRecord) +//! ``` +//! +//! The exit capacity is **2 by proof, not by tuning** (#487 §5): the +//! producer is the vCPU run loop, which parks after every publish, so at +//! most one ordinary exit plus one terminal record can ever be +//! outstanding. `Full` on the exit ring is therefore a protocol +//! violation and the kernel's response is the fail-closed poisoned-window +//! retire — never a drop, never a wait. The control capacity (4) is the +//! largest sane batch (`WriteState + Inject + Resume` and headroom); the +//! run loop drains it to empty at its top-of-loop safe point, so `Full` +//! there is ordinary backpressure on an awake producer. +//! +//! Records are **bytes and indices only** — no capability ever rides a +//! ring (the standing doctrine). Every field is any-bit-valid on read: +//! the peer is untrusted, and a record must decode from arbitrary bytes +//! (unknown kinds refuse at the consumer, they do not trap). +//! +//! The register image rides as a fixed `[u64; GUEST_STATE_QWORDS]` in +//! the **executing architecture's own frozen wire order** — the kernel's +//! per-arch `state_to_wire`/`state_from_wire` seam uses the shared architecture +//! layout selected by [`arch`] so the instruction decoder cannot drift from it. +//! The record itself stays architecture-agnostic: the protocol cites +//! registers exclusively through the semantic roles in [`state_index`], +//! which every architecture's wire order is required to satisfy. The +//! array is the contract, never a field list. +//! +//! ## The decode-assist tail (#765) +//! +//! A stage-2 fault names an *address*, not an instruction. Servicing it +//! as MMIO needs the trapped instruction's width, direction and register +//! — facts the hardware hands over in three different shapes: AMD stores +//! the raw instruction bytes in the VMCB (decode assists), Intel stores +//! nothing and the kernel fetches the bytes through the guest's own page +//! tables, and Arm / RISC-V report the access **already decoded** +//! (`ESR_EL2.ISS` / `htinst`). An exit record therefore carries whichever +//! shape the producing architecture has, tagged by [`GuestExitRecord::assist`] +//! (the [`assist`] roster): raw bytes in `insn`/`insn_len`, or a decoded +//! access in `access`/`data`. The consumer's one job is to lift that into +//! a decoded access — running an ISA decoder only when handed bytes — so +//! an architecture whose hardware decodes for it never needs one. No form +//! is mandatory: [`assist::NONE`] is an honest "no help", and the consumer +//! decides fail-closed. + +// Keep architecture selection here, as in the kernel's platform::arch. +// Consumers use guest::arch; an x86 register name cannot leak into a +// build for another ISA through an unconditional module. +#[cfg(target_arch = "aarch64")] +pub mod aarch64; +#[cfg(target_arch = "riscv64")] +pub mod riscv64; +#[cfg(target_arch = "x86_64")] +pub mod x86_64; + +#[cfg(target_arch = "aarch64")] +pub use aarch64 as arch; +#[cfg(target_arch = "riscv64")] +pub use riscv64 as arch; +#[cfg(target_arch = "x86_64")] +pub use x86_64 as arch; + +mod wire; +pub use wire::*; diff --git a/native/crates/abi/src/guest/riscv64/mod.rs b/native/crates/abi/src/guest/riscv64/mod.rs new file mode 100644 index 0000000..6ea1831 --- /dev/null +++ b/native/crates/abi/src/guest/riscv64/mod.rs @@ -0,0 +1,6 @@ +//! RISC-V guest architecture vocabulary. +//! +//! The kernel's H-extension guest backend is not implemented yet. No register +//! indices are exported until that backend defines its wire conversion; +//! in particular, this module must not inherit x86-64's register numbering. +//! Shared records and semantic role positions remain in the parent module. diff --git a/native/crates/abi/src/guest.rs b/native/crates/abi/src/guest/wire.rs similarity index 67% rename from native/crates/abi/src/guest.rs rename to native/crates/abi/src/guest/wire.rs index 2b04a0d..5bf45df 100644 --- a/native/crates/abi/src/guest.rs +++ b/native/crates/abi/src/guest/wire.rs @@ -1,49 +1,15 @@ -//! Guest exit/control wire records — the **per-vCPU window vocabulary** -//! (#487/#481), frozen here because kernel and hypervisor are two -//! endpoints of one contract and can never drift on a layout. -//! -//! One page per vCPU carries **two SPSC rings** (`kore-ring`'s frozen -//! `SPSC_ABI = 1` format, one ring per direction): -//! -//! ```text -//! offset 0: exit ring (kernel → hypervisor, -//! capacity 2, GuestExitRecord) -//! offset EXIT_RING_BYTES: control ring (hypervisor → kernel, -//! capacity 4, GuestCtrlRecord) -//! ``` -//! -//! The exit capacity is **2 by proof, not by tuning** (#487 §5): the -//! producer is the vCPU run loop, which parks after every publish, so at -//! most one ordinary exit plus one terminal record can ever be -//! outstanding. `Full` on the exit ring is therefore a protocol -//! violation and the kernel's response is the fail-closed poisoned-window -//! retire — never a drop, never a wait. The control capacity (4) is the -//! largest sane batch (`WriteState + Inject + Resume` and headroom); the -//! run loop drains it to empty at its top-of-loop safe point, so `Full` -//! there is ordinary backpressure on an awake producer. -//! -//! Records are **bytes and indices only** — no capability ever rides a -//! ring (the standing doctrine). Every field is any-bit-valid on read: -//! the peer is untrusted, and a record must decode from arbitrary bytes -//! (unknown kinds refuse at the consumer, they do not trap). -//! -//! The register image rides as a fixed `[u64; GUEST_STATE_QWORDS]` in -//! the **executing architecture's own frozen wire order** — the kernel's -//! per-arch `state_to_wire`/`state_from_wire` seam owns which physical -//! register sits at which index, and is the only place those names may -//! appear. This object stays architecture-agnostic: the protocol cites -//! registers exclusively through the semantic roles in [`state_index`], -//! which every architecture's wire order is required to satisfy. The -//! array is the contract, never a field list. - +//! Architecture-neutral guest window records and frozen semantic roles. use zerocopy::{FromBytes, Immutable, IntoBytes}; use crate::cap::Rights; /// Quadwords in the guest register image — sized once, for every -/// architecture, by the largest supported image; a smaller image -/// zero-fills its tail. Frozen with the record layout. -pub const GUEST_STATE_QWORDS: usize = 18; +/// architecture, by the largest image any of them publishes: AArch64's +/// thirty-one general registers, two stack pointers, the program counter +/// and PSTATE make thirty-five, so forty leaves headroom. x86-64 fills +/// eighteen and zero-fills the tail. Frozen with the record layout: a +/// smaller image never shrinks it, a larger one is a wire break. +pub const GUEST_STATE_QWORDS: usize = 40; /// The **semantic roles** the guest protocol cites in the register /// image — the only register vocabulary this agnostic object carries. @@ -66,10 +32,33 @@ pub mod state_index { }; } -/// Bytes of one record slot, both directions — frozen. Exit + full -/// register image + kind/sequence/instr_len fit with headroom; the pad -/// is reserved-zero on write, ignored on read. -pub const GUEST_RECORD_BYTES: usize = 256; +/// Bytes of one record slot, both directions — frozen. Kind, sequence, +/// two payload words, the full register image and the decode-assist tail +/// fit with headroom; the pad is reserved-zero on write, ignored on read. +/// +/// **Why 512, and what would reopen it (#765).** The slot was 256 bytes +/// while the image was x86-64's eighteen quadwords (168 bytes used). The +/// image is now sized for the largest architecture any port will publish +/// — AArch64's thirty-five ([`GUEST_STATE_QWORDS`]) — and the 24-byte +/// head + 320-byte image + 32-byte assist tail does not fit 256, so the +/// slot doubled: one wire break taken now, in place of one on the day an +/// Arm arm lands. The cost is paid per forwarded exit and per +/// `WriteState`, about four more cache lines each way, invisible beside +/// the world switch itself; both rings still fit the one-page window +/// (3456 of 4096 bytes). Weighed and declined: a role-indexed *partial* +/// image (every consumer would then carry a per-arch register map, which +/// is the leak this object exists to prevent) and a per-arch slot size +/// (the window geometry would stop being one frozen vocabulary). Revisit +/// only against a measured exit-path cost on metal, and shrink the +/// *return* copy first — a lighter "advance the instruction pointer by +/// `n`" control kind ([`CTRL_KIND_WRITE_STATE`]'s note) spares most +/// completions the full image write-back — before touching the slot. +pub const GUEST_RECORD_BYTES: usize = 512; + +/// The longest instruction any supported ISA can trap on — x86's fifteen +/// bytes; a fixed 4-byte Arm or RISC-V word fits with room. Sizes the +/// exit record's raw-bytes assist form. +pub const GUEST_INSN_BYTES_MAX: usize = 15; /// Exit-ring capacity — **2 by the #487 §5 proof** (publish-then-park /// bounds ordinary exits to one; the terminal record is the only @@ -116,11 +105,13 @@ pub const GUEST_VCPU_HANDLE_RIGHTS: Rights = Rights::ACCESS // at compile time below, the report-roster discipline. /// Guest touched an address the hypervisor models as a device. -/// **Producer-less today, deliberately** (#481): neither vendor decoder -/// can produce it — device-vs-RAM is knowledge only the hypervisor has -/// (it chose the device windows), so the kernel forwards -/// [`EXIT_KIND_STAGE_FAULT`] and the hypervisor classifies. The number -/// is reserved for a future decode-assist fast path. +/// **Producer-less, deliberately — still, after #765** (#481): device- +/// vs-RAM is knowledge only the hypervisor has (it chose the device +/// windows), so the kernel forwards [`EXIT_KIND_STAGE_FAULT`] *with the +/// decode-assist tail attached* and the hypervisor's device-window table +/// classifies. The number stays reserved for a kernel-classified fast +/// path — a fault the kernel's own stage-2 ledger already places outside +/// guest RAM — should one ever pay for itself. pub const EXIT_KIND_MMIO: u16 = 0x4701; /// A programmed-I/O access trapped (x86-64's port `in`/`out` family; /// an architecture without a port space never raises it): payload `a` @@ -140,7 +131,9 @@ pub const EXIT_KIND_HLT: u16 = 0x4705; /// CPU's own walk of the guest's page tables (an accessed/dirty update /// against a read-only mapping reads as WRITE plus this bit), `4` a /// reserved bit refused the walk. The one fault the kernel forwards -/// (#481). +/// (#481). The record's decode-assist tail (module docs, [`assist`]) +/// carries what the hardware knew about the trapped instruction, so a +/// device-window fault can be completed without a second exit. pub const EXIT_KIND_STAGE_FAULT: u16 = 0x4706; /// Arch-specific exit token the shared taxonomy does not name: `a` = /// the raw token, decoded by the owning backend's vocabulary. @@ -181,6 +174,13 @@ const EXIT_KINDS: [u16; 9] = [ // ── Control-record kinds (hypervisor → kernel) ────────────────────────── /// Install the carried register image before the next entry. +/// +/// Every completion today ships the **full** image back through this +/// record, even when only the instruction pointer and one datum register +/// changed (an MMIO or port completion). A lighter "advance by `n`" +/// control kind is the obvious optimization once exit latency is measured +/// on metal (the rationale under [`GUEST_RECORD_BYTES`]); it takes the +/// next free `CTRL_KIND_*` number and never renumbers this one. pub const CTRL_KIND_WRITE_STATE: u16 = 0x4801; /// Queue an interrupt/exception: `a` packs vector/kind, `b` packs the /// optional error code (see [`GuestCtrlRecord`] docs). @@ -231,6 +231,45 @@ const _: () = { } }; +/// The **decode-assist form** an exit record's tail carries — the +/// [`GuestExitRecord::assist`] discriminant. Plain values, any-bit-valid +/// on read: an unknown form is "no help" to the consumer, never a trap. +/// +/// The three shapes mirror what hardware actually provides (module docs): +/// nothing, the raw instruction bytes, or the access already decoded. A +/// producing architecture fills the richest form it has; a consumer lifts +/// whichever arrives into one decoded access, running an ISA decoder only +/// over [`INSN`](Self::INSN). +pub mod assist { + /// No assist: `insn`, `insn_len`, `access` and `data` are zero and + /// meaningless. The consumer may still refuse the fault fail-closed + /// or fetch and decode by other means. + pub const NONE: u8 = 0; + /// The trapped instruction's raw bytes are in `insn[..insn_len]` + /// (`1..=GUEST_INSN_BYTES_MAX`), fetched by the hardware (AMD decode + /// assists) or by the kernel through the guest's own page tables + /// (Intel). `access` and `data` are zero. + pub const INSN: u8 = 1; + /// The access is already decoded in `access` (and `data`), by the + /// hardware's own syndrome (`ESR_EL2.ISS` on Arm, `htinst` on + /// RISC-V). `insn` is zero. **Reserved — no producer yet**: the + /// packing below is fixed now so the first architecture that + /// decodes in hardware fills a slot rather than reshaping the record. + /// + /// ```text + /// access[3:0] width of the memory access in bytes: 1, 2, 4, 8 + /// access[4] 1 = the guest wrote (data flows guest → device) + /// access[5] 1 = a read sign-extends into the register + /// access[15:8] wire-image index of the datum register; 0xff = none + /// access[23:16] bytes of the register a read overwrites: 8 = all, + /// 4 = the low dword with the upper cleared, 1 or 2 = + /// merged into the low bytes; zero when `[15:8]` is 0xff + /// access[63:24] reserved, zero + /// data the value written when no register carries it + /// ``` + pub const ACCESS: u8 = 2; +} + /// Why a guest retired — the [`EXIT_KIND_RETIRED`] record's `a` payload. /// A plain discriminant (any-bit-valid on read; unknown values are /// "unspecified" to the consumer, never a trap). @@ -346,10 +385,20 @@ pub mod refusal { pub const BAD_PAYLOAD: u64 = 4; } +/// Bytes of an exit record ahead of its reserved pad: the 24-byte head, +/// the register image, then the decode-assist tail (`insn` + `insn_len` +/// = 16, `access` + `data` = 16). +const GUEST_EXIT_RECORD_USED: usize = 24 + 8 * GUEST_STATE_QWORDS + 16 + 16; + +/// Bytes of a control record ahead of its reserved pad: the 24-byte head +/// and the register image. +const GUEST_CTRL_RECORD_USED: usize = 24 + 8 * GUEST_STATE_QWORDS; + /// One **exit record** (kernel → hypervisor): the exit, the guest's full -/// register image, and the producer's sequence — everything the -/// hypervisor needs, shipped once, so the `read_state` round trip #484 -/// measured is dead. Fixed 256-byte slot, `#[repr(C)]`, frozen. +/// register image, the producer's sequence, and the decode-assist tail — +/// everything the hypervisor needs, shipped once, so the `read_state` +/// round trip #484 measured is dead and an MMIO completion needs no +/// second exit. Fixed 512-byte slot, `#[repr(C)]`, frozen. #[repr(C)] #[derive(Clone, Copy, Debug, FromBytes, IntoBytes, Immutable)] pub struct GuestExitRecord { @@ -358,11 +407,13 @@ pub struct GuestExitRecord { pub kind: u16, /// Length in bytes of the instruction that caused the exit, when the /// backend exposes a decoded next-instruction length; - /// `0` = not provided. Frozen now so emulators can stop re-decoding - /// the moment backends fill it (the Intel-diff dowry, #487). + /// `0` = not provided. Filled for the intercepts whose length the + /// hardware reports (port I/O on both x86 vendors); for a stage-2 + /// fault the decoder over the assist bytes is authoritative and this + /// is at most a cross-check. pub instr_len: u8, - /// Reserved, zero. - pub _rsvd: u8, + /// The decode-assist form the tail carries — an [`assist`] value. + pub assist: u8, /// The producer's free-running publish count — lets the consumer name /// a record (e.g. in an `OpRefused` conversation) and detect its own /// bookkeeping drift. @@ -373,10 +424,23 @@ pub struct GuestExitRecord { /// Second payload word — meaning per kind (access bits, write/size /// packing, refusal discriminant). pub b: u64, - /// The full register image, x86-64 wire order (see module docs). + /// The full register image in the architecture's wire order (see + /// module docs); the tail past the architecture's image is zero. pub state: [u64; GUEST_STATE_QWORDS], + /// [`assist::INSN`]: the trapped instruction's raw bytes, the first + /// `insn_len` meaningful, the rest zero. Zero for every other form. + pub insn: [u8; GUEST_INSN_BYTES_MAX], + /// [`assist::INSN`]: how many of `insn` are the instruction + /// (`1..=GUEST_INSN_BYTES_MAX`). Zero for every other form. + pub insn_len: u8, + /// [`assist::ACCESS`]: the decoded access word (packing in [`assist`]). + /// Zero for every other form. + pub access: u64, + /// [`assist::ACCESS`]: the written value when no register carries it. + /// Zero for every other form. + pub data: u64, /// Reserved, zero on write, ignored on read. - pub _pad: [u8; GUEST_RECORD_BYTES - 24 - 8 * GUEST_STATE_QWORDS], + pub _pad: [u8; GUEST_RECORD_BYTES - GUEST_EXIT_RECORD_USED], } /// One **control record** (hypervisor → kernel). Same frozen slot size; @@ -401,7 +465,7 @@ pub struct GuestCtrlRecord { /// The register image for `WriteState`; ignored otherwise. pub state: [u64; GUEST_STATE_QWORDS], /// Reserved, zero on write, ignored on read. - pub _pad: [u8; GUEST_RECORD_BYTES - 24 - 8 * GUEST_STATE_QWORDS], + pub _pad: [u8; GUEST_RECORD_BYTES - GUEST_CTRL_RECORD_USED], } const _: () = { @@ -409,6 +473,17 @@ const _: () = { assert!(size_of::() == GUEST_RECORD_BYTES); assert!(align_of::() <= 64); assert!(align_of::() <= 64); + // The frozen offsets a peer written against this layout relies on: + // the head is 24 bytes on both records, the image follows it, and the + // exit record's assist tail sits right behind the image, its two + // words naturally aligned. + assert!(core::mem::offset_of!(GuestExitRecord, state) == 24); + assert!(core::mem::offset_of!(GuestCtrlRecord, state) == 24); + assert!(core::mem::offset_of!(GuestExitRecord, insn) == 24 + 8 * GUEST_STATE_QWORDS); + assert!(core::mem::offset_of!(GuestExitRecord, insn_len) == 24 + 8 * GUEST_STATE_QWORDS + 15); + assert!(core::mem::offset_of!(GuestExitRecord, access) == 24 + 8 * GUEST_STATE_QWORDS + 16); + assert!(core::mem::offset_of!(GuestExitRecord, data) == 24 + 8 * GUEST_STATE_QWORDS + 24); + assert!(GUEST_INSN_BYTES_MAX + 1 == 16); }; /// Byte offset of the **exit ring** within the per-vCPU window. diff --git a/native/crates/abi/src/guest/x86_64/mod.rs b/native/crates/abi/src/guest/x86_64/mod.rs new file mode 100644 index 0000000..07798c1 --- /dev/null +++ b/native/crates/abi/src/guest/x86_64/mod.rs @@ -0,0 +1,29 @@ +//! Frozen x86-64 register positions shared by the kernel and decoder. +pub const RAX: usize = 0; +pub const RBX: usize = 1; +pub const RCX: usize = 2; +pub const RDX: usize = 3; +pub const RSI: usize = 4; +pub const RDI: usize = 5; +pub const RBP: usize = 6; +pub const R8: usize = 7; +pub const R9: usize = 8; +pub const R10: usize = 9; +pub const R11: usize = 10; +pub const R12: usize = 11; +pub const R13: usize = 12; +pub const R14: usize = 13; +pub const R15: usize = 14; +pub const RIP: usize = 15; +pub const RSP: usize = 16; +pub const RFLAGS: usize = 17; +/// ModRM/REX GPR number to the frozen wire position. +pub const GPR_INDEX: [u8; 16] = [ + RAX as u8, RCX as u8, RDX as u8, RBX as u8, RSP as u8, RBP as u8, RSI as u8, RDI as u8, + R8 as u8, R9 as u8, R10 as u8, R11 as u8, R12 as u8, R13 as u8, R14 as u8, R15 as u8, +]; +const _: () = { + assert!(RAX == super::state_index::IO_DATUM); + assert!(RIP == super::state_index::IP); + assert!(RFLAGS < super::GUEST_STATE_QWORDS); +}; diff --git a/native/crates/abi/src/image.rs b/native/crates/abi/src/image.rs index 955b058..0a4e11e 100644 --- a/native/crates/abi/src/image.rs +++ b/native/crates/abi/src/image.rs @@ -161,6 +161,11 @@ pub const PROFILE_ACCELERATOR: u32 = 1 << 10; /// bit 12 so the two merge without renumbering. pub const PROFILE_GPU: u32 = 1 << 12; +/// Launch with a read-only lease of the archive member `.model`. +/// Only exec resolves this member through its privileged image face; the child +/// receives no archive export authority. Missing data refuses the launch. +pub const PROFILE_MODEL_DATA: u32 = 1 << 13; + pub const PROFILE_KNOWN: u32 = PROFILE_TELEMETRY | PROFILE_FILESYSTEM | PROFILE_FS_VIEW_BINDER @@ -173,7 +178,8 @@ pub const PROFILE_KNOWN: u32 = PROFILE_TELEMETRY | PROFILE_NETWORK | PROFILE_ACCELERATOR | PROFILE_ENGINE - | PROFILE_GPU; + | PROFILE_GPU + | PROFILE_MODEL_DATA; /// Validate the profile vocabulary and its dependencies: both filesystem /// auxiliary endpoint kinds are meaningful only beside an actual FS face. diff --git a/native/crates/abi/src/iommu_status.rs b/native/crates/abi/src/iommu_status.rs new file mode 100644 index 0000000..a6ff2eb --- /dev/null +++ b/native/crates/abi/src/iommu_status.rs @@ -0,0 +1,249 @@ +//! Bounded IOMMU observations. Addresses in fault records are IOVAs, never +//! host physical addresses; these records grant no device authority. +use zerocopy::byteorder::little_endian::{U16, U32, U64}; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Unaligned}; + +/// One controller plus the complete fault history fits the existing observation wire page. +pub const UNITS: usize = 1; +/// One second to collect immutable controller metadata and bounded fault history. +pub const OBSERVATION_BUDGET_NS: u64 = 1_000_000_000; +pub const FAULTS: usize = crate::session::IOMMU_FAULT_RING as usize; +pub const SEGMENT_KNOWN: u8 = 1; +pub const NUMA_KNOWN: u8 = 2; +pub const INCLUDE_ALL: u8 = 4; + +#[repr(C)] +#[derive( + Clone, + Copy, + Debug, + Default, + PartialEq, + Eq, + FromBytes, + IntoBytes, + KnownLayout, + Immutable, + Unaligned, +)] +pub struct Unit { + pub id: U32, + /// 0 unknown, 1 VT-d, 2 AMD-Vi, 3 SMMUv2, 4 SMMUv3, 5 RISC-V. + pub kind: u8, + /// 1 stage 1, 2 stage 2, 3 nested (configured descriptor). + pub stage: u8, + /// log2(maximum lawful leaf bytes); zero means format-limited. + pub leaf_shift: u8, + pub flags: u8, + pub segment: U32, + pub numa: U32, + /// Translation, PASID, ATS, device TLB, interrupt remapping, nested + /// translation, page requests, dirty tracking, respectively in bits 0..7. + /// Hardware capabilities, not proof that each feature is enabled. + pub capabilities: U64, + pub pci_scopes: U16, + pub stream_scopes: U16, + pub reservations: U32, +} + +impl Unit { + pub fn identity(self) -> (u64, u64) { + ( + u64::from(self.id.get()) + | u64::from(self.kind) << 32 + | u64::from(self.stage) << 40 + | u64::from(self.leaf_shift) << 48 + | u64::from(self.flags) << 56, + u64::from(self.segment.get()) | u64::from(self.numa.get()) << 32, + ) + } + pub fn features(self) -> (u64, u64) { + ( + self.capabilities.get(), + u64::from(self.pci_scopes.get()) + | u64::from(self.stream_scopes.get()) << 16 + | u64::from(self.reservations.get()) << 32, + ) + } + pub fn from_observations(identity: (u64, u64), features: (u64, u64)) -> Option { + let row = Self { + id: (identity.0 as u32).into(), + kind: (identity.0 >> 32) as u8, + stage: (identity.0 >> 40) as u8, + leaf_shift: (identity.0 >> 48) as u8, + flags: (identity.0 >> 56) as u8, + segment: (identity.1 as u32).into(), + numa: ((identity.1 >> 32) as u32).into(), + capabilities: features.0.into(), + pci_scopes: (features.1 as u16).into(), + stream_scopes: ((features.1 >> 16) as u16).into(), + reservations: ((features.1 >> 32) as u32).into(), + }; + row.validate().then_some(row) + } + pub fn validate(&self) -> bool { + self.kind <= 5 + && (1..=3).contains(&self.stage) + && (self.leaf_shift == 0 || (12..64).contains(&self.leaf_shift)) + && self.flags & !(SEGMENT_KNOWN | NUMA_KNOWN | INCLUDE_ALL) == 0 + && self.capabilities.get() & !0xff == 0 + && if self.flags & SEGMENT_KNOWN != 0 { + self.segment.get() <= u16::MAX.into() + } else { + self.segment.get() == 0 + } + && (self.flags & NUMA_KNOWN != 0 || self.numa.get() == 0) + } +} + +#[repr(C)] +#[derive( + Clone, + Copy, + Debug, + Default, + PartialEq, + Eq, + FromBytes, + IntoBytes, + KnownLayout, + Immutable, + Unaligned, +)] +pub struct Fault { + pub packed: U64, + /// Existing IommuFaultRecord observation: drain sequence and timestamp. + pub stamp: U64, +} + +#[repr(C)] +#[derive( + Clone, + Copy, + Debug, + Default, + PartialEq, + Eq, + FromBytes, + IntoBytes, + KnownLayout, + Immutable, + Unaligned, +)] +pub struct Snapshot { + pub available: u8, + pub count: u8, + pub fault_count: u8, + /// The census changed while reading history; history is best effort. + pub changed: u8, + pub first: U16, + pub reserved: U16, + pub total: U32, + pub unresponsive: U32, + pub captured_ns: U64, + pub census: U64, + pub units: [Unit; UNITS], + pub faults: [Fault; FAULTS], +} + +impl Snapshot { + /// Alternate tagged view of the existing observation payload. Padding is zero. + pub fn to_wire_page(&self) -> crate::processor_status::ProcessorPage { + let mut page = crate::processor_status::ProcessorPage::default(); + page.as_mut_bytes()[..core::mem::size_of::()].copy_from_slice(self.as_bytes()); + page + } + pub fn from_wire_page(page: &crate::processor_status::ProcessorPage) -> Option { + let (s, tail) = Self::read_from_prefix(page.as_bytes()).ok()?; + (s.validate() && tail.iter().all(|b| *b == 0)).then_some(s) + } + pub fn validate(&self) -> bool { + self.available == 1 + && self.changed <= 1 + && self.reserved.get() == 0 + && u32::from(self.first.get()) <= self.total.get() + && usize::from(self.count) + == (self.total.get() - u32::from(self.first.get())).min(UNITS as u32) as usize + && usize::from(self.fault_count) <= FAULTS + && self.units[..usize::from(self.count)] + .iter() + .all(Unit::validate) + && self.units[usize::from(self.count)..] + .iter() + .all(|u| *u == Unit::default()) + && self.faults[usize::from(self.fault_count)..] + .iter() + .all(|f| *f == Fault::default()) + } +} + +/// Shared by the launcher and renderer; malformed pagination never launches a +/// different observation accidentally. +pub fn requested_first(args: &[u8]) -> Option { + let text = core::str::from_utf8(args).ok()?; + let mut words = text.split_ascii_whitespace(); + if words.next()? != "iommu" { + return None; + } + let first = match words.next() { + Some(s) => s.parse().ok()?, + None => 0, + }; + words.next().is_none().then_some(first) +} + +const _: () = assert!( + core::mem::size_of::() + <= core::mem::size_of::() +); +const _: () = assert!(core::mem::size_of::() <= crate::process::PROCESS_CONTEXT_BYTES); + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn unavailable_is_not_an_empty_machine_and_pages_are_canonical() { + assert!(!Snapshot::default().validate()); + let mut s = Snapshot { + available: 1, + ..Default::default() + }; + assert!(s.validate()); + s.first.set(1); + assert!(!s.validate()); + s.first.set(0); + s.fault_count = (FAULTS + 1) as u8; + assert!(!s.validate()); + } + #[test] + fn unit_words_preserve_capabilities_scopes_and_unknown_locality() { + let u = Unit { + id: 7.into(), + kind: 2, + stage: 2, + leaf_shift: 21, + capabilities: 0xad.into(), + pci_scopes: 12.into(), + stream_scopes: 3.into(), + reservations: 9.into(), + ..Default::default() + }; + assert_eq!(Unit::from_observations(u.identity(), u.features()), Some(u)); + let mut invalid = u; + invalid.segment.set(1); + assert!(!invalid.validate()); + } + #[test] + fn selector_is_exact_and_bounded() { + assert_eq!(requested_first(b"iommu"), Some(0)); + assert_eq!(requested_first(b" iommu 4 "), Some(4)); + for bad in [ + b"iommu 65536".as_slice(), + b"iommu 4 extra", + b"sys", + b"iommu -1", + ] { + assert_eq!(requested_first(bad), None); + } + } +} diff --git a/native/crates/abi/src/lib.rs b/native/crates/abi/src/lib.rs index 255daee..75d503c 100644 --- a/native/crates/abi/src/lib.rs +++ b/native/crates/abi/src/lib.rs @@ -59,6 +59,7 @@ pub mod founding; pub mod guest; pub mod image; pub mod input_status; +pub mod iommu_status; pub mod machine; pub mod mem; pub mod monitor; @@ -67,6 +68,7 @@ pub mod pointer; pub mod process; pub mod processor_status; pub mod session; +pub mod stor; pub mod svc; pub mod task; pub mod uring; diff --git a/native/crates/abi/src/machine.rs b/native/crates/abi/src/machine.rs index 8dfac6a..b65ad15 100644 --- a/native/crates/abi/src/machine.rs +++ b/native/crates/abi/src/machine.rs @@ -1,12 +1,10 @@ //! **The machine-facts block** — the sixth frozen boot-page block, after //! the founding record (`kore-rt::entry::MACHINE_BLOCK_AT`). //! -//! What the runtime needs to size a launch to the machine without an IPC -//! round trip: how many logical CPUs it has, and how many cores of the -//! founding node's best performance class a fitting SPMD region would be -//! granted exclusively (the G6c reservation population — `res/SPMD.md` -//! §7). A dev otherwise hardcodes a width literal and guesses; with -//! this, [`kore_rt::spmd::dispatch`] reads the machine and picks. +//! What the runtime needs to size a launch without an IPC round trip: +//! logical CPU count, the founding node's best-class population, and an +//! optional calibrated same-class cohort fit in the flag word. The factual +//! population remains the fallback when calibration is incomplete. //! //! **Kernel testimony, like the founding block and unlike the clock and //! geometry blocks.** The clock and geometry blocks are relayed by the @@ -39,10 +37,10 @@ pub const MACHINE_ABI: u16 = 1; pub struct MachineBlock { /// Layout version ([`MACHINE_ABI`]); `0` = no record (fail closed). pub abi: U16, - /// Machine capability flags — see [`MACHINE_FLAG_CPU_ENTROPY`]. An - /// additive claim on what was reserved space: an unset bit reads the - /// same as the zero a pre-flags founder wrote, so every flag must be - /// phrased so that "absent" is the safe answer. + /// Machine capability and fit flags — see [`MACHINE_FLAG_CPU_ENTROPY`] + /// and [`pack_spmd_fit_width`]. An additive claim on reserved space: + /// unset bits read as zero from an older founder, so absence must be a + /// safe answer. pub flags: U16, /// Logical CPU count — `ResourceCatalog::cpu_count()` at the stamp /// site. Machine-global (every founding sees the same value). @@ -50,10 +48,10 @@ pub struct MachineBlock { /// The **best-lane population**: how many cores of the founding /// node's best performance class an SPMD region would be granted /// exclusively (the G6c batch reservation count — P-cores on a - /// hybrid box, all cores on a homogeneous one). Node-relative, so + /// hybrid box, all cores on a homogeneous one). Node-relative, so /// stamped per spawn: two foundings on different nodes of a /// heterogeneous machine legitimately see different values. `0` - /// only on an empty catalog. + /// only on an empty catalog. The optional fit in `flags` may be wider. pub best_lane: U32, /// The founding NUMA node id (`current_cpu().numa()`) — the node /// [`best_lane`](Self::best_lane) counts. Context for a reader that @@ -124,6 +122,12 @@ impl MachineBlock { self.validate().then(|| self.best_lane.get()) } + /// A calibrated SPMD cohort width, when the founder had complete samples. + pub fn spmd_fit_width(&self) -> Option { + let width = (self.flags.get() & MACHINE_SPMD_FIT_MASK) >> MACHINE_SPMD_FIT_SHIFT; + (self.validate() && width != 0).then_some(u32::from(width)) + } + /// Whether this CPU has an entropy instruction a task may execute at /// user privilege (`RDRAND` / `RNDR` / a U-mode-readable Zkr `seed`). /// @@ -194,6 +198,20 @@ pub const MACHINE_FLAG_CPU_ENTROPY: u16 = 1 << 0; /// no way to ask the silicon itself. Absent = the boot measurement alone. pub const MACHINE_FLAG_HW_FEEDBACK: u16 = 1 << 1; +/// Flags bits 8..13 carry an optional fit width (1..63). Zero means that +/// calibration was incomplete and the reader should use `best_lane`. This +/// additive claim leaves the 20-byte frozen block and existing flags intact. +const MACHINE_SPMD_FIT_SHIFT: u16 = 8; +const MACHINE_SPMD_FIT_MAX: u16 = (1 << 6) - 1; +const MACHINE_SPMD_FIT_MASK: u16 = MACHINE_SPMD_FIT_MAX << MACHINE_SPMD_FIT_SHIFT; + +/// Encode a measured fit for the machine flags. The runtime's worker ceiling +/// is below 63; saturating keeps the field valid if a larger node appears. +pub const fn pack_spmd_fit_width(width: u32) -> u16 { + let max = MACHINE_SPMD_FIT_MAX as u32; + (if width > max { max } else { width } as u16) << MACHINE_SPMD_FIT_SHIFT +} + const _: () = assert!( core::mem::size_of::() == 20, "the machine block is frozen ABI" @@ -213,6 +231,17 @@ mod tests { assert_eq!(old.low_power_cores(), Some(0)); } + #[test] + fn optional_fit_preserves_legacy_best_lane() { + let old = MachineBlock::new(16, 4, 0, MACHINE_FLAG_HW_FEEDBACK, 0); + assert_eq!(old.spmd_fit_width(), None); + let fit = MachineBlock::new(16, 4, 0, pack_spmd_fit_width(8), 0); + assert_eq!(fit.best_lane(), Some(4)); + assert_eq!(fit.spmd_fit_width(), Some(8)); + let absent = MachineBlock::new(16, 4, 0, pack_spmd_fit_width(0), 0); + assert_eq!(absent.spmd_fit_width(), None); + } + #[test] fn an_empty_block_validates_but_exposes_zero_facts() { let b = MachineBlock::EMPTY; diff --git a/native/crates/abi/src/monitor.rs b/native/crates/abi/src/monitor.rs index ccacbf3..c28e721 100644 --- a/native/crates/abi/src/monitor.rs +++ b/native/crates/abi/src/monitor.rs @@ -34,7 +34,11 @@ pub struct SystemSnapshot { heap_capacity: U64, heap_used: U64, heap_largest: U64, - /// Cumulative IOMMU translation-fault records drained since boot. + /// The IOMMU fault census word ([`SystemMetric::IommuFaults`]'s + /// value): `[31:0]` records drained since boot, `[47:32]` hardware + /// fault-log overflows, `[63:48]` fault-event interrupts landed. + /// + /// [`SystemMetric::IommuFaults`]: crate::session::SystemMetric::IommuFaults iommu_faults: U64, /// The LAST fault record, packed: `[15:0]` source-id, `[23:16]` /// raw reason, `[24]` write, `[63:25]` faulting-IOVA page. @@ -311,8 +315,21 @@ impl SystemSnapshot { pub fn heap_largest(self) -> u64 { self.heap_largest.get() } + /// Translation-fault records drained since boot. pub fn iommu_faults(self) -> u64 { - self.iommu_faults.get() + self.iommu_faults.get() & 0xffff_ffff + } + /// Times a unit's hardware fault log overflowed — records discarded + /// before any drain saw them, so a count of zero here is what makes + /// [`iommu_faults`](Self::iommu_faults) a complete account. + pub fn iommu_fault_overflows(self) -> u64 { + (self.iommu_faults.get() >> 32) & 0xffff + } + /// Fault-event interrupts landed since boot: zero means the fault lane + /// never fired, and every record was found by a polled drain — the + /// record the hardware holds meanwhile is the one that overflows. + pub fn iommu_fault_events(self) -> u64 { + self.iommu_faults.get() >> 48 } pub fn iommu_last(self) -> u64 { self.iommu_last.get() @@ -1402,6 +1419,12 @@ impl ManifoldSnapshot { /// zero the pressure came back `∞` at exactly the load the clamp was meant /// to survive. Clear here means that is happening again. const PRESSURE_FINITE: u32 = 1 << 3; + /// Two 14-bit rates occupy the previously unused upper flag bits, in + /// 100 MB/s units. The fixed process-page ABI remains 40 bytes wide. + const TRAFFIC_SHIFT: u32 = 4; + const TRAFFIC_PEAK_SHIFT: u32 = 18; + const TRAFFIC_MASK: u32 = (1 << 14) - 1; + const TRAFFIC_UNIT_BPS: u64 = 100_000_000; pub const EMPTY: Self = Self { flags: U32::new(0), @@ -1440,6 +1463,32 @@ impl ManifoldSnapshot { } } + /// Attach the independently sampled controller rate to the same snapshot. + pub fn with_traffic(mut self, bps: u64, peak_bps: u64) -> Self { + if peak_bps > 0 { + let rate = (bps / Self::TRAFFIC_UNIT_BPS).min(Self::TRAFFIC_MASK as u64) as u32; + let peak = (peak_bps / Self::TRAFFIC_UNIT_BPS) + .max(1) + .min(Self::TRAFFIC_MASK as u64) as u32; + self.flags = U32::new( + self.flags.get() + | (rate << Self::TRAFFIC_SHIFT) + | (peak << Self::TRAFFIC_PEAK_SHIFT), + ); + } + self + } + + pub fn traffic(self) -> Option<(u64, u64)> { + let flags = self.flags.get(); + let rate = (flags >> Self::TRAFFIC_SHIFT) & Self::TRAFFIC_MASK; + let peak = (flags >> Self::TRAFFIC_PEAK_SHIFT) & Self::TRAFFIC_MASK; + (peak > 0).then_some(( + u64::from(rate) * Self::TRAFFIC_UNIT_BPS, + u64::from(peak) * Self::TRAFFIC_UNIT_BPS, + )) + } + pub fn available(self) -> bool { self.flags.get() & AVAILABLE == AVAILABLE } @@ -1906,6 +1955,16 @@ mod tests { ); } + #[test] + fn manifold_traffic_round_trips_without_growing_the_wire_snapshot() { + let base = ManifoldSnapshot::new(1_000_000, 2_000_000, 0, 0, 0, 0, 0, 0, 0); + assert_eq!(base.traffic(), None); + let observed = base.with_traffic(12_345_678_900, 56_789_123_400); + assert_eq!(observed.traffic(), Some((12_300_000_000, 56_700_000_000))); + assert!(observed.validate()); + assert_eq!(core::mem::size_of::(), 40); + } + #[test] fn filesystem_storage_keeps_namespace_volume_and_allocation_distinct() { let health = FilesystemHealthSnapshot::new(0, 36, 98, 10, 3, 7, 123, 2, 0); diff --git a/native/crates/abi/src/process.rs b/native/crates/abi/src/process.rs index 8d4aece..b4694ba 100644 --- a/native/crates/abi/src/process.rs +++ b/native/crates/abi/src/process.rs @@ -15,6 +15,13 @@ //! the whole-stream terminal: it is legal only with no unanswered //! [`ProcessOutputKind::InputReady`], after which neither direction is used and //! the launcher may block on task lifecycle. +//! +//! The input-side [`ProcessInputKind::Interrupt`] record is the one launcher +//! record that is not input: it is legal at any time, answers no +//! `InputReady`, and asks the child to stop at its next safe point and exit +//! in order (the terminal's Ctrl-C). A launcher sends it once, then allows a +//! bounded grace before it revokes the task; a child that never reads its +//! input simply meets the revoke (`res/LIFECYCLE_HARDENING.md` §3). use zerocopy::byteorder::little_endian::{U16, U32}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Unaligned}; @@ -49,6 +56,27 @@ pub const PROCESS_OUTPUT_WAKE_ROLE: u16 = crate::task::role::USER_BASE; /// Child boot-role instance for the output-wake `ChanSend`. pub const PROCESS_OUTPUT_WAKE_INDEX: u16 = 3; +/// Boot-role used by a **stream host**: a task that holds the *parent* end of +/// another process's stream on its launcher's behalf (an emulator window, +/// `res/SHELL_TERMINAL.md`). The launcher formats the page and keeps the +/// lifecycle pivots (the child's task handle and the channel roots). The host +/// has no output receive: the child's output wake is a send into the host's +/// own executor doorbell, so every host park is one executor park. +pub const PROCESS_STDIO_HOST_ROLE: u16 = crate::task::role::USER_BASE; +/// Host boot-role instance for the stream view. Distinct from every child +/// index above, so one block can never carry both ends' meanings at once. +pub const PROCESS_STDIO_HOST_INDEX: u16 = 4; +/// Host boot-role instance for the **host channel** `ChanRecv`: everything +/// else the launcher hands the host arrives here as capabilities — first the +/// stdin-wake send that completes the parent end, then a delegated +/// `ACCESS | REVOKE` handle for each foreground task the host may end +/// (`res/SHELL_TERMINAL.md` §9). +pub const PROCESS_STDIO_HOST_CHANNEL_INDEX: u16 = 5; +/// Exact rights of a foreground slot: a delegated task handle that may +/// observe and end its task, and nothing else. +pub const FOREGROUND_SLOT_RIGHTS: crate::cap::Rights = + crate::cap::Rights::ACCESS.union(crate::cap::Rights::REVOKE); + /// Exact authority delivered on either child-held stream-wake endpoint. /// /// `ACCESS` is the only right needed to rendezvous. `REVOKE` lets task @@ -125,6 +153,7 @@ pub enum ProcessStreamError { const PROCESS_INPUT_DATA: u16 = 1; const PROCESS_INPUT_CLOSED: u16 = 2; +const PROCESS_INPUT_INTERRUPT: u16 = 3; /// Decoded launcher-to-child input record kind. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -133,11 +162,20 @@ pub enum ProcessInputKind { Data, /// The launcher has closed the child's input stream. Closed, + /// The launcher asks the child to stop at its next safe point: release + /// what it holds and exit, or (a REPL) abandon the current line. Not + /// input: legal at any time, including with no unanswered `InputReady`, + /// and it answers none. A child reading a line when it arrives keeps that + /// request outstanding, so it prompts again without a second + /// `InputReady`, and if it exits instead it exits without `Closed`. The + /// launcher revokes the task if the child leaves the record unread past + /// its grace. + Interrupt, } /// One launcher-to-child foreground-input record. /// -/// Every bit pattern is safe to copy out of a shared ring, but only the two +/// Every bit pattern is safe to copy out of a shared ring, but only the three /// canonical encodings pass [`Self::validate`]. In particular, unused bytes /// must remain zero so malformed peers cannot smuggle ambiguous state through /// padding or an inactive payload suffix. @@ -181,6 +219,16 @@ impl ProcessInputChunk { } } + /// Encode the single canonical interrupt record. + pub const fn interrupt() -> Self { + Self { + tag: U16::new(PROCESS_INPUT_INTERRUPT), + len: U16::new(0), + reserved: U32::new(0), + payload: [0; PROCESS_STDIN_CHUNK_BYTES], + } + } + /// Validate and decode this record's kind. pub fn kind(&self) -> Result { if self.reserved.get() != 0 { @@ -197,18 +245,21 @@ impl ProcessInputChunk { PROCESS_INPUT_CLOSED if len == 0 && self.payload.iter().all(|byte| *byte == 0) => { Ok(ProcessInputKind::Closed) } + PROCESS_INPUT_INTERRUPT if len == 0 && self.payload.iter().all(|byte| *byte == 0) => { + Ok(ProcessInputKind::Interrupt) + } _ => Err(ProcessStreamError::Invalid), } } - /// Whether this is one of the two canonical input encodings. + /// Whether this is one of the three canonical input encodings. pub fn validate(&self) -> bool { self.kind().is_ok() } - /// Validate the record and return its live byte prefix. Closed records - /// return an empty slice; callers that distinguish EOF should inspect - /// [`Self::kind`] first. + /// Validate the record and return its live byte prefix. Closed and + /// interrupt records return an empty slice; callers that distinguish them + /// should inspect [`Self::kind`] first. pub fn bytes(&self) -> Result<&[u8], ProcessStreamError> { self.kind()?; Ok(&self.payload[..usize::from(self.len.get())]) @@ -411,12 +462,11 @@ pub const PROCESS_ARG_BYTES: usize = 272; /// largest context anyone builds, which is the bootstrap command catalogue /// `help` reads — [`MAX_COMMAND_SNAPSHOT_ENTRIES`](crate::command::MAX_COMMAND_SNAPSHOT_ENTRIES) /// rows of name, usage and summary plus four framing bytes each. The -/// catalogue's 35 rows need 2078 bytes at their present verbosity, which -/// 2048 refused — `wasm` could not join without deleting another command's -/// documentation. 2560 carries them with room for several more rows, and -/// still leaves the payload it borrows (3208) most of its width for the -/// output a command writes back. -pub const PROCESS_CONTEXT_BYTES: usize = 2560; +/// catalogue has 40 rows, including the delegated Lua worker. Its complete +/// snapshot no longer fits 2560 bytes. 3072 carries the current catalogue +/// and remains within the existing 3208-byte alternating payload; no page +/// layout or command-output capacity changes. +pub const PROCESS_CONTEXT_BYTES: usize = 3072; const _: () = assert!( PROCESS_CONTEXT_BYTES <= PROCESS_OUTPUT_BYTES, "context borrows the output payload and cannot outgrow it" @@ -430,14 +480,15 @@ const _: () = assert!( /// the link's facts, the device's counters, and the bound radio's) + 16 /// reserved. /// -/// The header's ceiling is the widest bounded reply any status view must -/// fit: the maximal `status sys ` report needs 3099 output bytes +/// The header's historical ceiling came from the widest bounded status reply: +/// the former per-core `status sys ` report needed 3099 output bytes /// (and `status input i2c` 3072), so with the 540-byte header of ABI 11 the /// output had 33 bytes to give. ABI 12 takes 20 of them for the filesystem's /// latency moments; 13 remain, and the next byte past them costs that report /// a row. ABI 13's timer census therefore came out of the argument buffer, /// not the output — and so did the 96 bytes its `status` row needs, which -/// takes the output to 3208 with 19 bytes under the widest report. ABI 13's +/// takes the output to 3208 with 19 bytes under that report. The current +/// compact `status sys` leaves this output allocation ABI-stable. ABI 13's /// frame-class census (16 bytes in the system snapshot) likewise comes out /// of the argument buffer: 312 → 296, still clearing the shell's 256-byte /// line and the files app's 255-byte path word, and the output stays 3208. @@ -843,6 +894,10 @@ mod tests { let input_closed = ProcessInputChunk::closed(); assert_eq!(input_closed.kind(), Ok(ProcessInputKind::Closed)); assert_eq!(input_closed.bytes(), Ok([].as_slice())); + let interrupt = ProcessInputChunk::interrupt(); + assert_eq!(interrupt.kind(), Ok(ProcessInputKind::Interrupt)); + assert_eq!(interrupt.bytes(), Ok([].as_slice())); + assert_ne!(interrupt, input_closed); let stdout = ProcessOutputEvent::stdout(b"42\n").unwrap(); assert!(stdout.validate()); @@ -915,6 +970,15 @@ mod tests { input.tag.set(u16::MAX); assert_eq!(input.kind(), Err(ProcessStreamError::Invalid)); + // An interrupt carries nothing: a length or payload byte is a record + // this launcher did not write. + let mut input = ProcessInputChunk::interrupt(); + input.len.set(1); + assert_eq!(input.kind(), Err(ProcessStreamError::Invalid)); + let mut input = ProcessInputChunk::interrupt(); + input.payload[0] = 3; + assert_eq!(input.kind(), Err(ProcessStreamError::Invalid)); + let mut event = ProcessOutputEvent::stdout(b"x").unwrap(); event.len.set(0); assert_eq!(event.kind(), Err(ProcessStreamError::Invalid)); diff --git a/native/crates/abi/src/processor_status.rs b/native/crates/abi/src/processor_status.rs index 9ba1b9a..48fc6d1 100644 --- a/native/crates/abi/src/processor_status.rs +++ b/native/crates/abi/src/processor_status.rs @@ -2,7 +2,43 @@ use zerocopy::byteorder::little_endian::{U16, U32, U64}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Unaligned}; -pub const ROWS: usize = 8; +/// Keep status pages at 17 rows. More rows would overflow the bounded text +/// reply even though the process context now has room for a larger catalogue. +/// The header occupies 32 bytes; rows include owner-local counter evidence. +pub const ROWS: usize = 17; +/// Minimum spacing between owner-local readings, even for overlapping requests. +pub const SAMPLE_PERIOD_NS: u64 = 250_000_000; +/// One shared half-second capture for all cores: time for an owner timer to +/// take a baseline and a second reading one period later. Late cores report +/// unavailable; requesting a snapshot does not add sampling interrupts. +pub const CAPTURE_WINDOW_NS: u64 = 2 * SAMPLE_PERIOD_NS; +/// Completed observations remain readable for four nominal periods. +pub const MAX_SAMPLE_AGE_NS: u64 = 4 * SAMPLE_PERIOD_NS; + +/// The monitor's processor request. A busy interval never touches the CPU PMU; +/// only an explicit counters selector asks the owner to read it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum ProcessorCapture { + Read = 0, + Busy = 1, + Counters = 2, +} + +impl ProcessorCapture { + pub const fn from_wire(value: u8) -> Option { + match value { + 0 => Some(Self::Read), + 1 => Some(Self::Busy), + 2 => Some(Self::Counters), + _ => None, + } + } + + pub const fn captures(self) -> bool { + !matches!(self, Self::Read) + } +} #[repr(C)] #[derive( @@ -27,7 +63,12 @@ pub struct ProcessorRow { /// Socket, cluster, core (16 bits each), SMT thread (8 bits). /// Bit 63 marks an explicit NUMA proximity domain; bits 56..62 are zero. pub topology: U64, + /// Non-idle time during the owner-published window; zero when unavailable. pub busy_ns: U64, + /// Owner-local publication timestamp, potentially preceding this read. + pub sampled_ns: U64, + /// Zero means no interval measurement is available for this row. + pub window_ns: U64, pub l1d: U32, pub l2: U32, pub l3: U32, @@ -38,31 +79,112 @@ pub struct ProcessorRow { pub feedback: U32, pub nominal_mhz: U32, pub maximum_mhz: U32, - /// This CPU's boot DRAM stream, alone (`SystemMetric::ProcessorStream`), - /// in [`STREAM_UNIT_BPS`] units: fill rate, then copy rate. Zero = - /// unmeasured. + /// This CPU's solo boot DRAM fill and copy rates, in STREAM_UNIT_BPS. pub stream_write: U16, pub stream_copy: U16, + pub counters: CounterSample, } -/// Bytes per second per unit of [`ProcessorRow::stream_write`] / -/// [`ProcessorRow::stream_copy`] (100 MB/s, so 6.5 TB/s per word). +/// Bytes per second per unit of the compact boot stream fields. pub const STREAM_UNIT_BPS: u64 = 100_000_000; impl ProcessorRow { - /// Attach the CPU's solo DRAM stream (bytes per second, saturating). pub fn set_stream(&mut self, write_bps: u64, copy_bps: u64) { let unit = |bps: u64| U16::new((bps / STREAM_UNIT_BPS).min(u64::from(u16::MAX)) as u16); self.stream_write = unit(write_bps); self.stream_copy = unit(copy_bps); } - /// `(write, copy)` bytes per second; `(0, 0)` when unmeasured. + pub fn stream_bps(&self) -> (u64, u64) { ( u64::from(self.stream_write.get()) * STREAM_UNIT_BPS, u64::from(self.stream_copy.get()) * STREAM_UNIT_BPS, ) } + + /// Keep static topology, but discard observations outside this request. + pub fn retain_window(&mut self, start: u64, end: u64) { + let baseline = self.sampled_ns.get().checked_sub(self.window_ns.get()); + if self.window_ns.get() == 0 + || baseline.is_none_or(|t| t < start) + || self.sampled_ns.get() > end + { + self.window_ns.set(0); + self.busy_ns.set(0); + self.counters = CounterSample::default(); + } + } +} + +/// Raw owner publication, or interval deltas when the enclosing row has a window. +/// `at_ns` is the owner timestamp for raw samples, elapsed time for deltas. +/// Profiles: 1 architectural Intel; 2 PTL P; 3 PTL E/LP-E. Zero = unavailable. +#[repr(C)] +#[derive( + Clone, + Copy, + Debug, + Default, + PartialEq, + Eq, + FromBytes, + IntoBytes, + KnownLayout, + Immutable, + Unaligned, +)] +pub struct CounterSample { + pub at_ns: U64, + /// Profile in byte 0, hardware counter width in byte 1; upper bits reserved. + pub config: U64, + /// Instructions, cycles, FP event, L2 misses, extra and traffic (profile-specific). + pub values: [U64; 6], +} +impl CounterSample { + pub fn validate(&self) -> bool { + let config = self.config.get(); + if config == 0 { + return *self == Self::default(); + } + let width = (config >> 8) as u32; + if config >> 16 != 0 + || !(48..=64).contains(&width) + || !(1..=3).contains(&(config & 255)) + || self.at_ns.get() == 0 + { + return false; + } + let mask = u64::MAX >> (64 - width); + self.values.iter().all(|v| v.get() <= mask) + && (config & 255 != 1 || self.values[2..].iter().all(|v| v.get() == 0)) + } + pub fn interval_since(&self, before: &Self) -> Option { + if !self.validate() + || !before.validate() + || self.config.get() == 0 + || self.config != before.config + { + return None; + } + let elapsed = self.at_ns.get().checked_sub(before.at_ns.get())?; + if elapsed == 0 { + return None; + } + let mask = u64::MAX >> (64 - (self.config.get() >> 8)); + let mut out = *self; + out.at_ns.set(elapsed); + for (value, old) in out.values.iter_mut().zip(before.values) { + let delta = value.get().wrapping_sub(old.get()) & mask; + // Half-range ordering rejects backwards/reset-like deltas. This + // assumes less than half a counter range elapses between samples; + // multiple complete wraps cannot be identified from two readings. + if delta > mask / 2 { + return None; + } + value.set(delta); + } + Some(out) + } } #[repr(C)] @@ -101,6 +223,18 @@ impl Default for ProcessorSnapshot { pub type ProcessorPage = ProcessorSnapshot<2>; impl ProcessorSnapshot { + /// A capture acknowledgement carries a bounded deadline, never CPU rows. + pub fn validate_capture(&self) -> bool { + self.available.get() == 1 + && self.reserved.get() == 0 + && self.count.get() == 0 + && u32::from(self.first.get()) <= self.total.get() + && self.total.get() <= 65536 + && self.started_ns.get() != 0 + && self.finished_ns.get().checked_sub(self.started_ns.get()) == Some(CAPTURE_WINDOW_NS) + && self.rows.iter().all(|row| *row == ProcessorRow::default()) + } + pub fn validate(&self) -> bool { if self.available.get() == 0 { return *self == Self::default(); @@ -125,10 +259,12 @@ impl ProcessorSnapshot { return *row == ProcessorRow::default(); } row.cpu.get() as usize == self.first.get() as usize + i + && (row.window_ns.get() == 0 + || (row.busy_ns.get() <= row.window_ns.get() + && row.window_ns.get() <= row.sampled_ns.get())) && row.class <= 3 && matches!(row.vector_bytes, 0 | 16 | 32 | 64) - && row.feedback.get() & !0x1ffff == 0 - && (row.feedback.get() & 0x10000 != 0 || row.feedback.get() == 0) + && row.counters.validate() && row.topology.get() & 0x7f00_0000_0000_0000 == 0 && (row.topology.get() >> 63 != 0 || row.numa.get() == 0) }) @@ -137,14 +273,21 @@ impl ProcessorSnapshot { /// Status selectors that need a processor page. Input and other scopes retain /// their existing context. Decimal page starts are explicit CPU ordinals. -pub fn requested_first(args: &[u8]) -> Option { +pub fn requested_processor(args: &[u8]) -> Option<(u16, ProcessorCapture)> { let mut words = args .split(|b| b.is_ascii_whitespace()) .filter(|w| !w.is_empty()); - match words.next() { - None => Some(0), - Some(b"sys" | b"cpu") => { - let first = match words.next() { + let scope = words.next()?; + match scope { + b"sys" | b"cpu" => { + let next = words.next(); + let (mode, next) = match (scope, next) { + (b"cpu", Some(b"counters")) => (ProcessorCapture::Counters, words.next()), + (b"cpu", Some(b"detail")) => (ProcessorCapture::Read, words.next()), + (b"sys" | b"cpu", next) => (ProcessorCapture::Busy, next), + _ => return None, + }; + let first = match next { None => 0, Some(word) => { if word.is_empty() { @@ -160,22 +303,97 @@ pub fn requested_first(args: &[u8]) -> Option { n } }; - words.next().is_none().then_some(first) + words.next().is_none().then_some((first, mode)) } _ => None, } } +pub fn requested_first(args: &[u8]) -> Option { + requested_processor(args).map(|(first, _)| first) +} + const _: () = assert!(core::mem::size_of::() <= crate::process::PROCESS_CONTEXT_BYTES); #[cfg(test)] mod tests { + #[test] + fn capture_header_and_result_windows_are_distinct() { + let mut page = super::ProcessorPage::default(); + page.available.set(1); + page.total.set(16); + page.started_ns.set(1); + page.finished_ns.set(1 + super::CAPTURE_WINDOW_NS); + assert!(page.validate_capture()); + assert!(!page.validate()); + page.rows[0].cpu.set(1); + assert!(!page.validate_capture()); + } + + #[test] + fn retaining_window_rejects_stale_future_and_underflow_but_keeps_topology() { + for (sample, duration, kept) in [ + (20, 10, true), + (19, 10, false), + (21, 10, false), + (2, 10, false), + (20, 0, false), + ] { + let mut row = super::ProcessorRow::default(); + row.cpu.set(7); + row.l2.set(4096); + row.sampled_ns.set(sample); + row.window_ns.set(duration); + row.busy_ns.set(5); + row.counters.config.set(1); + row.retain_window(10, 20); + assert_eq!(row.window_ns.get() != 0, kept); + assert_eq!(row.busy_ns.get(), if kept { 5 } else { 0 }); + assert_eq!(row.counters.config.get(), if kept { 1 } else { 0 }); + assert_eq!((row.cpu.get(), row.l2.get()), (7, 4096)); + } + } + use super::*; #[test] + fn counter_windows_distinguish_wrap_reset_and_missing() { + let mask = (1u64 << 48) - 1; + let a = CounterSample { + at_ns: 100.into(), + config: (2 | 48 << 8).into(), + values: [ + 0.into(), + 0.into(), + (mask - 3).into(), + 100.into(), + 0.into(), + 0.into(), + ], + }; + let mut b = a; + b.at_ns.set(200); + b.values[2].set(5); + b.values[3].set(104); + let d = b.interval_since(&a).unwrap(); + assert_eq!(d.at_ns.get(), 100); + assert_eq!(d.values[2].get(), 9); + b.values[3].set(0); // reset is not a legitimate small wrapped delta + assert!(b.interval_since(&a).is_none()); + assert!(a.interval_since(&a).is_none()); + assert!(a.interval_since(&CounterSample::default()).is_none()); + b.config.set(3 | 48 << 8); + assert!(b.interval_since(&a).is_none()); + b.config.set(2 | 47 << 8); + assert!(!b.validate()); + b.config.set(2 | 48 << 8); + b.values[2].set(1 << 48); + assert!(!b.validate()); + } + #[test] fn pages_are_complete_ordered_and_canonical() { - assert_eq!(core::mem::size_of::(), 168); - assert_eq!(core::mem::size_of::(), 576); + assert_eq!(core::mem::size_of::(), 328); + assert_eq!(core::mem::size_of::(), 2548); let mut p = ProcessorPage::default(); p.available.set(1); p.first.set(8); @@ -191,7 +409,7 @@ mod tests { p.rows[1].cpu.set(8); assert!(!p.validate()); p = good; - p.rows[0].feedback.set(1); + p.rows[0].counters.config.set(1); assert!(!p.validate()); p = good; p.rows[0].topology.set(1 << 56); @@ -204,15 +422,37 @@ mod tests { p.first.set(11); p.total.set(11); assert!(p.validate()); - p.rows[1].l1d.set(1); + p.rows[1].l2.set(1); assert!(!p.validate()); } #[test] fn selectors_and_unavailable_are_explicit() { - for args in [b"".as_slice(), b"cpu", b"sys"] { + assert_eq!( + requested_processor(b"cpu"), + Some((0, ProcessorCapture::Busy)) + ); + assert_eq!( + requested_processor(b"sys 8"), + Some((8, ProcessorCapture::Busy)) + ); + assert_eq!( + requested_processor(b"cpu counters 8"), + Some((8, ProcessorCapture::Counters)) + ); + assert_eq!( + requested_processor(b"cpu detail"), + Some((0, ProcessorCapture::Read)) + ); + assert_eq!(requested_processor(b"sys counters"), None); + assert_eq!(ProcessorCapture::from_wire(3), None); + for args in [b"cpu".as_slice(), b"sys"] { assert_eq!(requested_first(args), Some(0)); } + assert_eq!(requested_first(b""), None); assert_eq!(requested_first(b"cpu 8"), Some(8)); + assert_eq!(requested_first(b"cpu counters"), Some(0)); + assert_eq!(requested_first(b"cpu counters 8"), Some(8)); + assert_eq!(requested_first(b"cpu counters extra"), None); for args in [b"cpu -1".as_slice(), b"cpu 65536", b"sys 0 extra", b"input"] { assert_eq!(requested_first(args), None); } diff --git a/native/crates/abi/src/session.rs b/native/crates/abi/src/session.rs index 39fe126..70d3933 100644 --- a/native/crates/abi/src/session.rs +++ b/native/crates/abi/src/session.rs @@ -968,6 +968,19 @@ pub enum Status { /// on the frozen vocabulary; older decoders read the raw value as /// [`Malformed`](Self::Malformed), which is equally terminal. Unfounded = 11, + /// A memory-fabric route overlaps an independently admitted exclusive + /// mapping. The caller must retire the alias before opening this route. + FabricRouteConflict = 12, + /// The registration or requested range has no executable route edge. + FabricRouteNoEdge = 13, + /// The requested route access exceeds its registered lending window. + FabricRouteWrongAccess = 14, + /// The live fabric facts reject the requested route or generation. + FabricRouteInfeasible = 15, + /// Route identity or per-supply claim storage is exhausted. + FabricRouteCapacity = 16, + /// The registration capability could not authorize a route child. + FabricRouteCapability = 17, } /// Hard cap on a session's invocation count — heapless, fixed-capacity @@ -1244,7 +1257,7 @@ const _: () = { // Resource-kind derives occupy the low, append-only band. START owns one // value in the global high-flavor band, and must remain distinct from // every defined flavor even where subject kinds currently disambiguate. - assert!((crate::cap::ResourceKind::DeviceMmu as u16) < DERIVE_KIND_TASK_START as u16); + assert!((crate::cap::ResourceKind::MemoryController as u16) < DERIVE_KIND_TASK_START as u16); assert!(DERIVE_KIND_TASK_START != DERIVE_KIND_IOVA_ANON); assert!(DERIVE_KIND_TASK_START != DERIVE_KIND_SET_TP); assert!(DERIVE_KIND_TASK_START != DERIVE_KIND_PROTECT); @@ -1264,7 +1277,7 @@ const _: () = { assert!(DERIVE_KIND_ARENA_EXTEND != DERIVE_KIND_DEVICE_RECORD); assert!(DERIVE_KIND_ARENA_EXTEND != DERIVE_KIND_IOVA_ANON); assert!(DERIVE_KIND_ARENA_EXTEND != DERIVE_KIND_EXEC_SEAL); - assert!((crate::cap::ResourceKind::DeviceMmu as u16) < DERIVE_KIND_ARENA_EXTEND as u16); + assert!((crate::cap::ResourceKind::MemoryController as u16) < DERIVE_KIND_ARENA_EXTEND as u16); // DEVICE_ACTIVATE shares the PciDevice subject with resource derives but // owns the next high-band byte; keep it distinct from every effect flavor. assert!(DERIVE_KIND_DEVICE_ACTIVATE != DERIVE_KIND_ARENA_EXTEND); @@ -1273,13 +1286,13 @@ const _: () = { assert!(DERIVE_KIND_DEVICE_ACTIVATE != DERIVE_KIND_TASK_JOIN); assert!(DERIVE_KIND_DEVICE_ACTIVATE != DERIVE_KIND_DEVICE_RECORD); assert!(DERIVE_KIND_DEVICE_ACTIVATE != DERIVE_KIND_IOVA_ANON); - // DeviceMmu is the current highest resource kind; the whole band must + // MemoryController is the current highest resource kind; the whole band must // stay below the derive-kind flavors. - assert!((crate::cap::ResourceKind::DeviceMmu as u16) < DERIVE_KIND_DEVICE_ACTIVATE as u16); + assert!((crate::cap::ResourceKind::MemoryController as u16) < DERIVE_KIND_DEVICE_ACTIVATE as u16); // The strict MSI-X route shares the PciDevice subject with ordinary route, // BAR, self-grant, and DEVICE_ACTIVATE derives. Its high flavor must remain // outside the resource band and distinct from the other PciDevice effect. - assert!((crate::cap::ResourceKind::DeviceMmu as u16) < DERIVE_KIND_IRQ_ROUTE_MSIX as u16); + assert!((crate::cap::ResourceKind::MemoryController as u16) < DERIVE_KIND_IRQ_ROUTE_MSIX as u16); assert!(DERIVE_KIND_IRQ_ROUTE_MSIX != DERIVE_KIND_DEVICE_ACTIVATE); assert!(DERIVE_KIND_IRQ_ROUTE_MSIX != DERIVE_KIND_SYSTEM_OBSERVE); // The I2C-HID observation shares the PciDevice subject with BAR, route, @@ -1293,7 +1306,7 @@ const _: () = { ); assert!(DERIVE_KIND_I2C_HID_READY_ROUTE != DERIVE_KIND_IRQ_ROUTE_LINE); assert!(DERIVE_KIND_I2C_HID_READY_ROUTE != DERIVE_KIND_I2C_HID_RECORD); - assert!((crate::cap::ResourceKind::DeviceMmu as u16) < DERIVE_KIND_PEER_IOVA as u16); + assert!((crate::cap::ResourceKind::MemoryController as u16) < DERIVE_KIND_PEER_IOVA as u16); assert!(DERIVE_KIND_PEER_IOVA != DERIVE_KIND_IRQ_ROUTE_MSIX); assert!(DERIVE_KIND_PEER_IOVA != DERIVE_KIND_DEVICE_ACTIVATE); // Task-subject flavors must be pairwise distinct for the same reason @@ -1368,7 +1381,7 @@ const DERIVE_FLAVORS: [u8; 35] = [ /// refuses it rather than running it as an unflavored derive (the /// `Revoke` arm's rule, finally applied to `Derive`). pub const fn derive_kind_known(kind: u8) -> bool { - if kind <= crate::cap::ResourceKind::DeviceMmu as u8 { + if kind <= crate::cap::ResourceKind::MemoryController as u8 { return true; } let mut i = 0; @@ -1387,7 +1400,7 @@ const _: () = { // admit would swallow a flavor. let mut i = 0; while i < DERIVE_FLAVORS.len() { - assert!(DERIVE_FLAVORS[i] > crate::cap::ResourceKind::DeviceMmu as u8); + assert!(DERIVE_FLAVORS[i] > crate::cap::ResourceKind::MemoryController as u8); i += 1; } }; @@ -1531,6 +1544,12 @@ impl Committed { 9 => Status::Exited, 10 => Status::AltDelivery, 11 => Status::Unfounded, + 12 => Status::FabricRouteConflict, + 13 => Status::FabricRouteNoEdge, + 14 => Status::FabricRouteWrongAccess, + 15 => Status::FabricRouteInfeasible, + 16 => Status::FabricRouteCapacity, + 17 => Status::FabricRouteCapability, _ => Status::Malformed, } } @@ -1947,14 +1966,28 @@ pub struct ViewMapReq { /// `0` for ordinary cursor placement. For `SURFACES`, selects one of /// [`SurfaceMapLane`]'s two fixed halves. pub lane: u8, - pub reserved: [u8; 5], + /// [`VIEW_MAP_SINK`], or zero. (Formerly the first reserved byte, so a + /// zeroed request keeps its meaning.) + pub flags: u8, + pub reserved: [u8; 4], } +/// [`ViewMapReq::flags`]: when the lessor's lease dies under this mapping, +/// repoint its pages at the receiver space's sink page instead of unmapping +/// them (`res/STORAGE_TRANSPORT.md` §12, option B). For a row that reads a +/// client-funded ring asynchronously: a dead client leaves it bytes that mean +/// nothing, never a fault. The mapping takes base-page grain. `MAP` window +/// only. +pub const VIEW_MAP_SINK: u8 = 1; + impl ViewMapReq { pub const WIRE: usize = 8; pub fn placement(&self) -> Option { - if self.reserved != [0; 5] { + if self.reserved != [0; 4] || self.flags & !VIEW_MAP_SINK != 0 { + return None; + } + if self.sink() && self.window.get() != crate::task::geom::MAP { return None; } match self.window.get() { @@ -1967,6 +2000,12 @@ impl ViewMapReq { } } + /// Whether the mapping sinks rather than unmaps when its lessor dies + /// ([`VIEW_MAP_SINK`]). + pub fn sink(&self) -> bool { + self.flags & VIEW_MAP_SINK != 0 + } + pub fn surface_lane(&self) -> Option { (self.placement() == Some(ViewMapWindow::Surfaces)) .then(|| SurfaceMapLane::from_wire(self.lane)) @@ -2246,6 +2285,28 @@ pub const REVOKE_KIND_UNSHARE_IOVA: u8 = 6; /// only the need to *abandon* an arena to rewrite it. pub const REVOKE_KIND_EXEC_RETURN: u8 = 7; +/// The [`Invocation::kind`] selector for **live** exact IOVA unshare through +/// [`OpCode::Revoke`] — [`REVOKE_KIND_UNSHARE_IOVA`] for a requester that +/// keeps running (`res/STORAGE_TRANSPORT.md` §11). +/// +/// Authentication, argument ([`UnshareReq`]) and share-list effect are those +/// of [`REVOKE_KIND_UNSHARE_IOVA`]. The physical half differs only in the +/// latch: the plain kind closes the requester's DMA latch (Bus Master +/// Enable) before the unmap and leaves it closed, which a driver must then +/// reopen for every surviving window. This kind leaves an active latch +/// open. It removes the window's leaves, completes the page-selective +/// IOTLB and device-TLB invalidation with drain, and only then releases the +/// window's identity, so any access the device still makes through the +/// window is refused by the IOMMU and recorded as a translation fault +/// instead of reaching the backing. Proving that no command still names the +/// window is the caller's own correctness, not the kernel's safety. +/// +/// A requester that is not active retires as under the plain kind. So does +/// an active requester whose latch rests on this window when no other window +/// shares its domain: the latch closes with its last window. When another +/// window survives, the latch re-anchors on it. +pub const REVOKE_KIND_UNSHARE_IOVA_LIVE: u8 = 8; + /// Argument payload for a **page delegation** derive (`Derive`, /// [`kind`](Invocation::kind)` = Region` on a held/staged `Region` /// subject; res/IMAGE.md §9a): re-home the subject region @@ -2608,6 +2669,29 @@ impl PowerReq { pub const WIRE: usize = 8; } +/// `Derive` discriminator on a held `SystemPower` grant for a CPU performance +/// preference effect. Kind zero remains the frozen machine-transition verb; +/// this distinct kind cannot turn a malformed preference into a shutdown. +pub const POWER_PERFORMANCE_KIND: u8 = 0xfe; +/// Kernel-selected per-CPU preference. The 0–100 wire range remains manual. +pub const POWER_PERFORMANCE_AUTO: u8 = 254; + +/// A machine-wide CPU performance preference, from energy saving (0) to +/// maximum performance (100). This is a policy hint, not a frequency in MHz. +/// The kernel applies it on each CPU through that CPU's performance facet. +#[repr(C)] +#[derive(Clone, Copy, Debug, TryFromBytes, KnownLayout, Immutable, Unaligned)] +pub struct PowerPerformanceReq { + /// Inclusive 0–100 manual preference, or [`POWER_PERFORMANCE_AUTO`]. + pub level: u8, + /// Reserved; must be zero. + pub reserved: [u8; 7], +} + +impl PowerPerformanceReq { + pub const WIRE: usize = 8; +} + /// One read-only machine metric exposed through `SystemObserve`. #[repr(u8)] #[derive(Clone, Copy, PartialEq, Eq, Debug, TryFromBytes, KnownLayout, Immutable, Unaligned)] @@ -2622,12 +2706,16 @@ pub enum SystemMetric { KernelHeap = 4, /// Conservative largest single kernel-heap free run. KernelHeapLargest = 5, - /// Cumulative IOMMU translation-fault records drained since boot; - /// auxiliary = the LAST record, packed (`[15:0]` source-id, - /// `[23:16]` raw reason, `[24]` write, `[63:25]` faulting-IOVA - /// page). Sampling DRAINS any records still latched in hardware — - /// the IOMMU's own testimony that a device DMA'd and translation - /// refused it, and a serial-less machine's only view of it. + /// The IOMMU fault census word: `[31:0]` cumulative translation-fault + /// records drained since boot, `[47:32]` hardware fault-log overflows + /// (records the unit discarded before any drain saw them), `[63:48]` + /// fault-event interrupts landed (zero on a machine whose fault lane + /// never fired, so every record here came from a polled drain); each + /// field saturates. Auxiliary = the LAST record, packed (`[15:0]` + /// source-id, `[23:16]` raw reason, `[24]` write, `[63:25]` + /// faulting-IOVA page). Sampling DRAINS any records still latched in + /// hardware — the IOMMU's own testimony that a device DMA'd and + /// translation refused it, and a serial-less machine's only view of it. IommuFaults = 6, /// Live memory-fabric domain registrations (res/MEMORY_FABRIC.md, #697); /// value packs domain count in the low word and registry capacity in the @@ -2713,6 +2801,100 @@ pub enum SystemMetric { /// scrub fill in 100 MB/s units in `[63:40]`; all zero when that CPU /// was not measured (no run to lend, or it never answered). ProcessorStream = 22, + /// Committed CPU preference: value 0–100, or 255 when no preference has + /// been set and the firmware's initial HWP request remains in force. + /// Auxiliary is zero. This reports policy, not delivered clock speed. + CpuPerformance = 23, + /// Latest owner interval duration and busy-time delta. + ProcessorWindow = 24, + /// Publication sequence and owner-local monotonic sample time. + ProcessorCountersStamp = 25, + /// Retired instructions and unhalted core cycles. + ProcessorCountersExecution = 26, + /// Profile-specific FP event and L2 request misses. + ProcessorCountersMemory = 27, + /// Profile-specific extra event and configuration. + ProcessorCountersExtra = 28, + /// Profile-specific cache fill or coherence-stall counter. + ProcessorCountersTraffic = 29, + /// Boot-frozen CPU count; no aggregate runtime scan. + ProcessorCount = 30, + /// Request a kernel-bounded telemetry cache refresh; returns start/deadline. + /// Operand 0 measures scheduler busy time alone; 1 also reads the boot-owned + /// PMU counters. No caller-selected events or register authority move. + ProcessorCapture = 31, + /// Live memory-controller bytes per second; auxiliary is Kore's + /// concurrently measured sustained stream rate. Both zero means no + /// supported counter or no fresh interval, not idle DRAM. + MemoryTraffic = 32, + /// One retained IOMMU fault record: the operand counts back from the + /// newest (0 = the last drained), and the pair is + /// [`IommuFaultRecord::to_observation`] — the packed record as + /// [`IommuFaults`](Self::IommuFaults) carries it, with its drain + /// sequence and the moment it was drained. Reading does not drain. + /// An operand past what the kernel retains + /// ([`IOMMU_FAULT_RING`]) or past the tally refuses, so a reader walks + /// from 0 until refused. + IommuFaultRecord = 33, + /// Catalog controller count; auxiliary = units recorded unresponsive. + IommuCount = 34, + /// Indexed immutable descriptor, Unit::identity() observation. + IommuUnit = 35, + /// Indexed capabilities and scope counts, Unit::features() observation. + IommuUnitFeatures = 36, +} + +/// Records the kernel retains for [`SystemMetric::IommuFaultRecord`]. +pub const IOMMU_FAULT_RING: u16 = 16; + +/// One retained IOMMU fault record, as [`SystemMetric::IommuFaultRecord`] +/// carries it. Testimony only; it grants nothing. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct IommuFaultRecord { + /// The record, packed as the [`IommuFaults`](SystemMetric::IommuFaults) + /// auxiliary: `[15:0]` source-id, `[23:16]` raw reason, `[24]` write, + /// `[63:25]` faulting-IOVA page. + pub packed: u64, + /// Its 1-based place in the drain order (saturating at 24 bits). + pub sequence: u32, + /// Monotonic microseconds since boot at the drain (saturating at 40 + /// bits — thirty-five years). + pub at_us: u64, +} + +impl IommuFaultRecord { + /// `value` = the packed record; `aux` = microseconds in `[39:0]` and + /// the sequence in `[63:40]`. + pub const fn to_observation(self) -> (u64, u64) { + ( + self.packed, + Self::aux_of(self.sequence as u64, self.at_us * 1000), + ) + } + + /// The auxiliary word for a record drained `at_ns` as the `sequence`-th. + pub const fn aux_of(sequence: u64, at_ns: u64) -> u64 { + let us = at_ns / 1000; + let us = if us > (1 << 40) - 1 { + (1 << 40) - 1 + } else { + us + }; + let sequence = if sequence > (1 << 24) - 1 { + (1 << 24) - 1 + } else { + sequence + }; + us | sequence << 40 + } + + pub const fn from_observation(value: u64, aux: u64) -> Self { + Self { + packed: value, + sequence: (aux >> 40) as u32, + at_us: aux & ((1 << 40) - 1), + } + } } /// The three physical frame-size classes, as the census operand names them. @@ -3031,6 +3213,7 @@ impl SystemObserveReq { matches!( metric, SystemMetric::DeviceRow + | SystemMetric::ProcessorCapture | SystemMetric::ProcessorIdentity | SystemMetric::ProcessorTopology | SystemMetric::ProcessorCache @@ -3038,6 +3221,15 @@ impl SystemObserveReq { | SystemMetric::MemoryClass | SystemMetric::MemoryStream | SystemMetric::ProcessorStream + | SystemMetric::IommuFaultRecord + | SystemMetric::IommuUnit + | SystemMetric::IommuUnitFeatures + | SystemMetric::ProcessorWindow + | SystemMetric::ProcessorCountersStamp + | SystemMetric::ProcessorCountersExecution + | SystemMetric::ProcessorCountersMemory + | SystemMetric::ProcessorCountersExtra + | SystemMetric::ProcessorCountersTraffic ) } @@ -3581,6 +3773,8 @@ pub struct FabricRouteInfo { pub access: u8, /// CPU map=0, IOMMU map=1. Other operations have no mapping admission. pub operation: u8, + /// Largest page alignment preserved by the route's exact backing. + pub align_log2: u8, } pub const FABRIC_ROUTE_INFO_KIND: u16 = 0xFF10; pub const FABRIC_ROUTE_INFO_MAGIC: u64 = 0x6B6F_7265_4652_494E; @@ -3588,15 +3782,23 @@ impl FabricRouteInfo { pub fn encode(self) -> crate::cap::CapRecord { let mut rec = inert_frame(FABRIC_ROUTE_INFO_KIND, FABRIC_ROUTE_INFO_MAGIC); rec.scope_rid = self.bytes; - rec.holder = crate::cap::Cid(u64::from(self.access) | (u64::from(self.operation) << 8)); + rec.holder = crate::cap::Cid( + u64::from(self.access) + | (u64::from(self.operation) << 8) + | (u64::from(self.align_log2) << 16), + ); rec } pub fn decode(rec: &crate::cap::CapRecord) -> Option { if !is_inert_report(rec, FABRIC_ROUTE_INFO_KIND, FABRIC_ROUTE_INFO_MAGIC) || rec.scope_rid == 0 || rec.scope_rid % 4096 != 0 - || rec.holder.0 & !0x107 != 0 + || rec.holder.0 & !0x00ff_0107 != 0 || rec.holder.0 & 3 == 0 + || !matches!( + (rec.holder.0 >> 16) as u8, + LEASE_ALIGN_4K | LEASE_ALIGN_2M | LEASE_ALIGN_1G + ) || !rec.rights.is_empty() || rec.epoch != 0 || rec.chain_len != 0 @@ -3609,6 +3811,7 @@ impl FabricRouteInfo { bytes: rec.scope_rid, access: rec.holder.0 as u8, operation: (rec.holder.0 >> 8) as u8, + align_log2: (rec.holder.0 >> 16) as u8, }) } } @@ -4826,6 +5029,13 @@ mod tests { Status::Irq, Status::Exited, Status::AltDelivery, + Status::Unfounded, + Status::FabricRouteConflict, + Status::FabricRouteNoEdge, + Status::FabricRouteWrongAccess, + Status::FabricRouteInfeasible, + Status::FabricRouteCapacity, + Status::FabricRouteCapability, ]; for s in all { for index in [0u16, 1, 7, 63, u16::MAX] { @@ -5109,7 +5319,20 @@ mod tests { req.surface_lane(), (window == ViewMapWindow::Surfaces).then_some(SurfaceMapLane::First) ); - for at in 3..bytes.len() { + // Byte 3 is the flags byte: the sink bit is admitted on MAP only, + // an unknown bit never. Bytes 4.. stay reserved. + let mut sink = bytes; + sink[3] = VIEW_MAP_SINK; + let (req, _) = ViewMapReq::try_ref_from_prefix(&sink).unwrap(); + assert_eq!( + req.placement(), + (window == ViewMapWindow::Map).then_some(ViewMapWindow::Map) + ); + let mut unknown = bytes; + unknown[3] = 2; + let (req, _) = ViewMapReq::try_ref_from_prefix(&unknown).unwrap(); + assert_eq!(req.placement(), None); + for at in 4..bytes.len() { let mut dirty = bytes; dirty[at] = 1; let (req, _) = ViewMapReq::try_ref_from_prefix(&dirty).unwrap(); @@ -5128,7 +5351,8 @@ mod tests { let invalid_map_lane = ViewMapReq { window: ViewMapWindow::Map.role().into(), lane: SurfaceMapLane::Second as u8, - reserved: [0; 5], + flags: 0, + reserved: [0; 4], }; assert_eq!(invalid_map_lane.placement(), None); for role in [ @@ -5140,10 +5364,27 @@ mod tests { let req = ViewMapReq { window: role.into(), lane: 0, - reserved: [0; 5], + flags: 0, + reserved: [0; 4], }; assert_eq!(req.placement(), None); } + // The sink flag is admitted on MAP alone; an unknown flag refuses. + let sink = ViewMapReq { + window: ViewMapWindow::Map.role().into(), + lane: 0, + flags: VIEW_MAP_SINK, + reserved: [0; 4], + }; + assert_eq!(sink.placement(), Some(ViewMapWindow::Map)); + assert!(sink.sink()); + let sink_heap = ViewMapReq { + window: ViewMapWindow::Heap.role().into(), + ..sink + }; + assert_eq!(sink_heap.placement(), None); + let unknown = ViewMapReq { flags: 2, ..sink }; + assert_eq!(unknown.placement(), None); } #[test] @@ -5387,10 +5628,11 @@ mod tests { bytes: 8192, access: 7, operation: 1, + align_log2: LEASE_ALIGN_2M, }; let record = info.encode(); assert_eq!(FabricRouteInfo::decode(&record), Some(info)); - for flags in [0, 4, 8, 0x203, 0x108] { + for flags in [0, 4, 8, 0x203, 0x108, 13 << 16, 31 << 16] { let mut bad = record; bad.holder = crate::cap::Cid(flags); assert!(FabricRouteInfo::decode(&bad).is_none()); @@ -5569,7 +5811,7 @@ mod tests { } if kind == FABRIC_ROUTE_INFO_KIND { rec.scope_rid = 4096; - rec.holder = crate::cap::Cid(0x103); + rec.holder = crate::cap::Cid(0x103 | (u64::from(LEASE_ALIGN_4K) << 16)); } if kind == SYSTEM_OBSERVE_REPORT_KIND { rec.chain_len = SystemMetric::Clock as u8; diff --git a/native/crates/abi/src/stor.rs b/native/crates/abi/src/stor.rs new file mode 100644 index 0000000..3c0e9ba --- /dev/null +++ b/native/crates/abi/src/stor.rs @@ -0,0 +1,309 @@ +//! The **storage transport** wire vocabulary (`res/STORAGE_TRANSPORT.md` §4). +//! +//! A storage client talks to the block row through one client-funded window: +//! a submission channel per [`Class`], one completion channel, and a page of +//! [`FenceRecord`]s. Entries are the io_uring records of [`crate::uring`], +//! read with the field meanings below; nothing on this wire is a capability +//! (a [`GrantId`] is a generation-checked index into grants the ring's +//! authority installed, never authority itself). +//! +//! | `IoUringSqe` field | meaning | +//! |---|---| +//! | `opcode` | [`op::READ_FIXED`], [`op::WRITE_FIXED`], [`op::FSYNC`], [`op::DISCARD`] | +//! | `fd` | [`GrantId`] — slot + generation; a stale id fails closed | +//! | `off` | byte offset within the grant (block-aligned) | +//! | `addr` | byte offset within the target — never a pointer | +//! | `len` | bytes (a block multiple) | +//! | `buf_index` | the target slot | +//! | `op_flags` | [`RWF_DSYNC`] (force unit access), [`RWF_NOWAIT`] | +//! | `personality` | timeline `1..=`[`TIMELINES`]; 0 = completion entry only | +//! | `addr3` | absolute uptime deadline in ns; 0 = the class default | +//! +//! A completion's `res` is the byte count, or a negated [`StorStatus`]. +//! +//! [`op::READ_FIXED`]: crate::uring::op::READ_FIXED +//! [`op::WRITE_FIXED`]: crate::uring::op::WRITE_FIXED +//! [`op::FSYNC`]: crate::uring::op::FSYNC +//! [`op::DISCARD`]: crate::uring::op::DISCARD + +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +/// Submission depth of the Demand and Stream channels. +pub const SQ_DEPTH: u32 = 128; +/// Submission depth of the Commit channel (the filesystem's metadata path). +pub const COMMIT_SQ_DEPTH: u32 = 32; +/// Completion depth. The row pops a submission only while it holds a +/// completion credit, so this bounds what may be in flight plus unreaped — +/// it need not equal the sum of the submission depths. +pub const CQ_DEPTH: u32 = 256; +/// Fence records per window. +pub const TIMELINES: usize = 16; +/// The fence page: its own 4 KiB page so it can be subleased to a device +/// that waits on a monotone word (`res/RESIDENCY.md` §6). +pub const FENCE_PAGE_BYTES: usize = 4096; + +const _: () = assert!(SQ_DEPTH.is_power_of_two()); +const _: () = assert!(COMMIT_SQ_DEPTH.is_power_of_two()); +const _: () = assert!(CQ_DEPTH.is_power_of_two()); +const _: () = assert!(TIMELINES * core::mem::size_of::() <= FENCE_PAGE_BYTES); + +/// `op_flags`: force unit access — the write is durable when it completes. +/// Linux's `RWF_DSYNC`. +pub const RWF_DSYNC: u32 = 0x0000_0002; +/// `op_flags`: refuse with [`StorStatus::Busy`] rather than wait for class +/// budget. Linux's `RWF_NOWAIT`. +pub const RWF_NOWAIT: u32 = 0x0000_0008; +/// Every `op_flags` bit the transport knows. Any other bit is refused as +/// [`StorStatus::Invalid`], never ignored. +pub const RWF_KNOWN: u32 = RWF_DSYNC | RWF_NOWAIT; + +/// The service class a submission channel carries. +/// +/// Each class has its own channel in the window and its own hardware +/// submission queue where the controller offers enough, so no class waits +/// behind another's backlog (`res/STORAGE_TRANSPORT.md` §3). +#[repr(u8)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Class { + /// Interactive reads a task is blocked on. + Demand = 0, + /// Bulk and read-ahead traffic — inference weights, prefetch. + Stream = 1, + /// The filesystem's metadata writes, flushes and force-unit-access. + Commit = 2, +} + +impl Class { + /// Every class, in channel order. + pub const ALL: [Self; 3] = [Self::Demand, Self::Stream, Self::Commit]; + /// Number of classes. + pub const COUNT: usize = 3; + + /// The class a channel index names. + pub const fn from_index(index: usize) -> Option { + match index { + 0 => Some(Self::Demand), + 1 => Some(Self::Stream), + 2 => Some(Self::Commit), + _ => None, + } + } + + /// This class's channel index. + pub const fn index(self) -> usize { + self as usize + } + + /// Submission depth of this class's channel. + pub const fn depth(self) -> u32 { + match self { + Self::Demand | Self::Stream => SQ_DEPTH, + Self::Commit => COMMIT_SQ_DEPTH, + } + } +} + +/// A grant as a submission names it: slot in bits 0..16, generation in +/// bits 16..31. Bit 31 is always clear, so the id is a non-negative `fd`. +/// +/// Generation-checked, never authority: a forged id reaches only grants the +/// ring's authority installed into that same ring, and a revoked slot's +/// successor carries a different generation. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct GrantId(u32); + +impl GrantId { + /// Largest generation the id carries (15 bits). + pub const MAX_GENERATION: u16 = 0x7fff; + + /// Compose from a slot and a generation (masked to 15 bits). + pub const fn new(slot: u16, generation: u16) -> Self { + Self(slot as u32 | (((generation & Self::MAX_GENERATION) as u32) << 16)) + } + + /// Decode a submission's `fd`; `None` for a negative one. + pub const fn from_fd(fd: i32) -> Option { + if fd < 0 { + None + } else { + Some(Self(fd as u32)) + } + } + + /// The `fd` a submission carries. + pub const fn fd(self) -> i32 { + self.0 as i32 + } + + /// The grant slot. + pub const fn slot(self) -> u16 { + self.0 as u16 + } + + /// The slot's generation. + pub const fn generation(self) -> u16 { + (self.0 >> 16) as u16 & Self::MAX_GENERATION + } +} + +/// Why a submission failed. A completion carries it negated in `res`. +#[repr(i32)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum StorStatus { + /// A range fell outside its grant or target. + Range = 1, + /// An offset or length was not a whole number of blocks. + Alignment = 2, + /// The grant does not carry the right the operation needs. + Rights = 3, + /// The grant is being or has been revoked. + Revoked = 4, + /// The grant or target id names nothing live. + Stale = 5, + /// The deadline passed before the device let go of the command. + Deadline = 6, + /// Cancelled before it was issued. + Cancelled = 7, + /// The medium reported an error. + Media = 8, + /// The device was lost; nothing it held completed. + Lost = 9, + /// The operation is not offered on this ring. + Unsupported = 10, + /// [`RWF_NOWAIT`] and the class had no budget. + Busy = 11, + /// The class is above the grant's ceiling or the ring's policy. + Class = 12, + /// A field the transport does not accept (an unknown flag or opcode). + Invalid = 13, +} + +impl StorStatus { + /// The negated form a completion's `res` carries. + pub const fn res(self) -> i32 { + -(self as i32) + } + + /// Decode a completion's `res`; `None` for a success (`res >= 0`) or an + /// unknown code. + pub const fn from_res(res: i32) -> Option { + Some(match res { + -1 => Self::Range, + -2 => Self::Alignment, + -3 => Self::Rights, + -4 => Self::Revoked, + -5 => Self::Stale, + -6 => Self::Deadline, + -7 => Self::Cancelled, + -8 => Self::Media, + -9 => Self::Lost, + -10 => Self::Unsupported, + -11 => Self::Busy, + -12 => Self::Class, + -13 => Self::Invalid, + _ => return None, + }) + } +} + +/// One timeline's monotone completion record, in the window's fence page. +/// +/// Sequences are implicit: the k-th submission naming timeline `t`, counted +/// by both sides from 1, has sequence k. `done` is the highest k such that +/// every sequence at or below it has completed; `fault` is the first failed +/// sequence (0 = none) and is sticky. The row stores `fault` before `done`, +/// both with release ordering, so a client that reads `done` with acquire +/// ordering and then `fault` never misses a fault at or below `done`. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable)] +pub struct FenceRecord { + /// Contiguous completed prefix. + pub done: u64, + /// First failed sequence, or 0. + pub fault: u64, +} + +const _: () = assert!(core::mem::size_of::() == 16); + +/// The level state of a submission channel. Reserved: the client publishes +/// nothing level-shaped today. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable)] +pub struct SqState { + /// Reserved, zero. + pub reserved: u64, +} + +/// The ring is serving. +pub const RING_LIVE: u32 = 0; +/// The ring was detached: nothing more is popped; in-flight work still +/// completes. +pub const RING_DEPARTED: u32 = 1; +/// The device behind the ring was lost; every submission answers +/// [`StorStatus::Lost`]. +pub const RING_LOST: u32 = 2; + +/// The level state of the completion channel, republished by the row. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable)] +pub struct CqState { + /// [`RING_LIVE`], [`RING_DEPARTED`] or [`RING_LOST`]. + pub status: u32, + /// Submissions accepted and not yet completed. + pub in_flight: u32, + /// Completions posted since attach. + pub completed: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn grant_ids_round_trip_through_fd_and_stay_non_negative() { + for (slot, generation) in [(0, 0), (7, 1), (u16::MAX, GrantId::MAX_GENERATION)] { + let id = GrantId::new(slot, generation); + assert!(id.fd() >= 0); + let back = GrantId::from_fd(id.fd()).unwrap(); + assert_eq!((back.slot(), back.generation()), (slot, generation)); + } + // A generation past 15 bits wraps inside its field, never into bit 31. + assert!(GrantId::new(1, u16::MAX).fd() >= 0); + assert_eq!(GrantId::from_fd(-1), None); + } + + #[test] + fn status_codes_round_trip_negated() { + for status in [ + StorStatus::Range, + StorStatus::Alignment, + StorStatus::Rights, + StorStatus::Revoked, + StorStatus::Stale, + StorStatus::Deadline, + StorStatus::Cancelled, + StorStatus::Media, + StorStatus::Lost, + StorStatus::Unsupported, + StorStatus::Busy, + StorStatus::Class, + StorStatus::Invalid, + ] { + assert!(status.res() < 0); + assert_eq!(StorStatus::from_res(status.res()), Some(status)); + } + assert_eq!(StorStatus::from_res(0), None); + assert_eq!(StorStatus::from_res(4096), None); + assert_eq!(StorStatus::from_res(-99), None); + } + + #[test] + fn classes_index_their_channels() { + for (index, class) in Class::ALL.into_iter().enumerate() { + assert_eq!(class.index(), index); + assert_eq!(Class::from_index(index), Some(class)); + } + assert_eq!(Class::from_index(Class::COUNT), None); + assert_eq!(Class::Commit.depth(), COMMIT_SQ_DEPTH); + } +} diff --git a/native/crates/abi/src/svc.rs b/native/crates/abi/src/svc.rs index 372ea84..dc2f661 100644 --- a/native/crates/abi/src/svc.rs +++ b/native/crates/abi/src/svc.rs @@ -195,6 +195,7 @@ pub struct MonitorReqWire { impl MonitorReqWire { pub const OP_SNAPSHOT: u8 = 1; pub const OP_PROCESSOR: u8 = 3; + pub const OP_IOMMU: u8 = 4; /// One page of the PCI function census (`crate::device::census`): /// `reserved[0]` is the page ordinal, `memory` / `exec` are the zero /// records. The monitor assembles the census from its own @@ -214,6 +215,8 @@ pub struct MonitorRepWire { pub grants: u8, pub reserved: [u8; 5], pub snapshot: crate::monitor::TelemetrySnapshot, + /// Tagged observation payload: ProcessorPage for processor replies, or + /// iommu_status::Snapshot encoded by its wire-page methods for IOMMU replies. pub processor: crate::processor_status::ProcessorPage, /// [`TAG_DEVICE_CENSUS`](Self::TAG_DEVICE_CENSUS) only; the zero page /// on every other tag (the codec refuses a stray one). @@ -221,6 +224,7 @@ pub struct MonitorRepWire { } impl MonitorRepWire { + pub const TAG_IOMMU: u8 = 5; pub const TAG_PROCESSOR: u8 = 4; pub const TAG_SNAPSHOT: u8 = 1; pub const TAG_REFUSED: u8 = 2; @@ -234,7 +238,8 @@ impl MonitorRepWire { /// bytes never ride this record (#148) — they ride the **staging /// window**, one shared mapping the client attaches once /// ([`OP_ATTACH`](Self::OP_ATTACH)) and both sides then address by page -/// index. 16 bytes. +/// index. The final word is zero for the original page verbs and carries the +/// checked extent length for the additive extent verbs. 16 bytes. /// /// **Why staging rather than a delivered view per page.** The original /// shape delivered a fresh `LeasedRegion` view with every `Out`/`In`, @@ -251,20 +256,18 @@ impl MonitorRepWire { Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, )] pub struct SwapReqWire { - /// Verb discriminant: [`OP_OUT`](Self::OP_OUT) / - /// [`OP_IN`](Self::OP_IN) / [`OP_DISCARD`](Self::OP_DISCARD) / - /// [`OP_ATTACH`](Self::OP_ATTACH). + /// Verb discriminant; zero is invalid. pub op: u8, /// Reserved; must be zero. pub reserved: [u8; 3], - /// `Out`/`In`: the **staging page index** the bytes occupy. + /// `Out`/`In` and extent store/load: the **first staging page index**. /// `Attach`: the founder-published return-landing correlation value /// naming the transfer this request adopts. `Discard`: zero. pub arg: U32, - /// `In`/`Discard`: the store token a `Stored` reply returned. + /// Restore/load/release: the store token a `Stored` reply returned. pub token: U32, - /// Reserved; must be zero. - pub reserved2: [u8; 4], + /// Extent store/load/release: exact page count. Zero for every other op. + pub pages: U32, } impl SwapReqWire { @@ -275,11 +278,19 @@ impl SwapReqWire { /// `SwapRequest::Discard` — drop a stored page unrestored. pub const OP_DISCARD: u8 = 3; /// `SwapRequest::Attach` — adopt the staging window transferred over - /// the return channel; every later `Out`/`In` addresses it by page. + /// the return channel; later byte-moving verbs address spans within it. pub const OP_ATTACH: u8 = 4; /// `SwapRequest::Status` — read the tier's books (no arguments; the /// answer is [`SwapRepWire::TAG_STATUS`]'s counter block). pub const OP_STATUS: u8 = 5; + /// Store a bounded immutable staging extent; the reply's opaque token + /// names the whole extent. + pub const OP_STORE_EXTENT: u8 = 6; + /// Load an extent without consuming it. Explicit release is a separate + /// operation so prefetch retries and cancellation cannot lose the copy. + pub const OP_LOAD_EXTENT: u8 = 7; + /// Release one extent after all users have stopped reading it. + pub const OP_RELEASE_EXTENT: u8 = 8; } /// The power request wire form (`PowerRequest`'s crossing shape) — #392. @@ -323,6 +334,10 @@ impl PowerReqWire { pub const OP_DISPLAY_EVIDENCE_REPORT: u8 = 15; /// Read one evidence word: index, then zeros. Missing words refuse Unavailable. pub const OP_DISPLAY_EVIDENCE: u8 = 16; + /// Set a 0–100 manual preference or 254 (Manifold auto) in reserved[0]. + pub const OP_PERFORMANCE_SET: u8 = 17; + /// Read the committed CPU preference; all payload bytes are zero. + pub const OP_PERFORMANCE_STATUS: u8 = 18; } /// The power reply wire form. A power service mints no authority, so @@ -355,6 +370,8 @@ impl PowerRepWire { pub const TAG_LID_STATUS: u8 = 8; /// Echoed evidence index and its u32 LE value. pub const TAG_DISPLAY_EVIDENCE: u8 = 9; + /// Requested preference (0–100 manual, 254 auto, 255 unset) in reserved[0]. + pub const TAG_PERFORMANCE_STATUS: u8 = 10; /// The session refused the transition. Authority (`MODIFY` absent) and /// feasibility (no firmware backend realizes it) are **not separable @@ -718,7 +735,8 @@ impl DeviceRepWire { /// token is **data** — the server's private map key, never a capability /// (the SwapBackend contract) — and a swap service grants nothing, ever /// (`grants` is always zero; pinned so the reply layout matches its -/// siblings). 8 bytes. +/// siblings). 104 bytes: an 8-byte outcome header followed by the status +/// counter tail, which is zero on every non-status reply. #[repr(C)] #[derive( Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, @@ -747,13 +765,13 @@ pub struct SwapRepWire { pub stored_pages: U32, /// Attached staging-window pages (`0` = unattached). pub staging_pages: U32, - /// Reserved; must be zero. - pub reserved2: [u8; 4], - /// `Out` ops answered `Stored`. + /// Live stored extents (`TAG_STATUS` only); zero on every other tag. + pub stored_extents: U32, + /// Successful stores (`StoreExtent` and compatibility `Out`). pub outs: U64, - /// `In` ops answered `Restored`. + /// Successful loads (`LoadExtent` and compatibility `In`). pub ins: U64, - /// `Discard` ops answered `Discarded`. + /// Successful releases (`ReleaseExtent` and compatibility `Discard`). pub discards: U64, /// Device commands submitted (disk tier; zram: zero). pub io_submitted: U64, @@ -795,6 +813,10 @@ impl SwapRepWire { pub const TAG_ATTACHED: u8 = 5; /// `SwapResponse::Status` — the books ride the counter block. pub const TAG_STATUS: u8 = 6; + /// `SwapResponse::Loaded` — staging holds a non-destructive extent load. + pub const TAG_LOADED: u8 = 7; + /// `SwapResponse::Released` — the extent identity is retired. + pub const TAG_RELEASED: u8 = 8; /// [`posture`](Self::posture) bit: a backing device is bound (the /// disk tier's controller; always set by the zram tier, whose store @@ -1246,7 +1268,9 @@ impl SpawnWire { /// or retirement metadata only. Ordinary `Invoke` requires it empty; immutable /// generation selection comes exclusively from the admitting capability. #[repr(C)] -#[derive(Clone, Copy, Debug, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] pub struct TaskReqWire { /// Verb discriminant: [`OP_WAIT`](Self::OP_WAIT) / /// [`OP_KILL`](Self::OP_KILL) / [`OP_EXIT`](Self::OP_EXIT) / @@ -1390,6 +1414,7 @@ impl LibraryFailure { impl TaskReqWire { pub const OP_PROCESSOR: u8 = 23; + pub const OP_IOMMU: u8 = 24; // Discriminant 1 is RETIRED: it carried the legacy block-carrying // spawn (`Spawn { entry, stack, flags, block }`), deleted whole by // the 2026-07-28 v2 ruling. It stays refused and must not be @@ -1526,9 +1551,9 @@ impl TaskReqWire { /// [`index`](Self::index) for the selected generation's validated segment /// count. The append-only snapshot tail is populated only on `Spawned`; all /// other tags require -/// [`crate::monitor::TelemetrySnapshot::EMPTY`]. The processor tail is populated -/// only by `ProcessorStatus`; other tags require its canonical all-zero value. -/// 616 bytes; both peers must be rebuilt together. +/// [`crate::monitor::TelemetrySnapshot::EMPTY`]. The observation tail carries +/// `ProcessorStatus` or the tagged IOMMU snapshot; other tags require zero. +/// Both peers must be rebuilt together when adding observation tags. #[repr(C)] #[derive( Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, @@ -1544,7 +1569,8 @@ pub struct TaskRepWire { pub grants: u8, /// `Spawned`: the selected generation's validated segment count (`<= /// crate::image::MAX_SEGMENTS`; zero for an unlinked spawn). - /// Zero on every other tag. + /// On a `LedgerFull` refusal, the bounded diagnostic origin; zero for + /// other refusals and all tags without an index vocabulary. pub index: u8, /// Frozen tag 3: historical service-wait exit status. pub status: I32, @@ -1563,13 +1589,61 @@ pub struct TaskRepWire { /// producer; empty on every other tag and on launches without the /// network profile. pub net: crate::monitor::NetSnapshot, + /// Tagged observation payload: ProcessorPage for processor replies, or + /// iommu_status::Snapshot encoded by its wire-page methods for IOMMU replies. pub processor: crate::processor_status::ProcessorPage, /// [`TAG_DEVICE_CENSUS`](Self::TAG_DEVICE_CENSUS)'s page; the zero /// page on every other tag. pub census: crate::device::DeviceCensusPage, + /// Only `REFUSE_LEDGER_FULL` with `index == 1` may carry this bounded + /// failure snapshot. Every other reply requires the all-zero form. + pub cleanup: CleanupRowDiagnosticWire, +} + +/// One failure-only exec cleanup-row snapshot. No capability names, addresses, +/// or authority cross this wire. Counts are saturated to `u32` at capture. +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct CleanupRowDiagnosticWire { + pub rows: U32, + pub capacity: U32, + pub occupied: U32, + pub rollback: U32, + pub child_live: U32, + pub child_dead: U32, + pub child_retryable: U32, + pub spawned: U32, + pub compensated: U32, + pub reclaimed: U32, + pub refill: u8, + pub mm_idle: u8, + pub truncated: u8, + pub reserved: u8, +} + +impl CleanupRowDiagnosticWire { + pub const EMPTY: Self = Self { + rows: U32::new(0), + capacity: U32::new(0), + occupied: U32::new(0), + rollback: U32::new(0), + child_live: U32::new(0), + child_dead: U32::new(0), + child_retryable: U32::new(0), + spawned: U32::new(0), + compensated: U32::new(0), + reclaimed: U32::new(0), + refill: 0, + mm_idle: 0, + truncated: 0, + reserved: 0, + }; } impl TaskRepWire { + pub const TAG_IOMMU: u8 = 24; pub const TAG_PROCESSOR: u8 = 23; /// `TaskResponse::Spawned` — paired with one parent-handle grant for an /// ordinary owned spawn; the one-shot kernel bootstrap form is data-only. @@ -1738,6 +1812,8 @@ impl TaskRepWire { /// the other is what makes a launch look intermittently slow when it is /// in fact permanently refused. pub const REFUSE_PEER_ADMISSION_WEDGED: u8 = 22; + pub const REFUSE_MODEL_UNAVAILABLE: u8 = 23; + pub const REFUSE_MODEL_MEMORY_UNAVAILABLE: u8 = 24; } // Live task discriminants stay disjoint from the frozen retired band on @@ -2365,6 +2441,14 @@ impl HypervisorReqWire { /// `HypervisorRequest::Status` — one coherent, data-only snapshot of the /// service's single-guest and text-console books. pub const OP_STATUS: u8 = 11; + /// `HypervisorRequest::SpawnMmio` — found the MMIO console witness + /// (`blob::echo_mmio_package`): the echo guest again, but driving its + /// 16550 through a memory-mapped window instead of port I/O, so every + /// console byte is a stage-2 fault the hypervisor must decode and + /// complete (#765). Same gating as [`OP_SPAWN_ECHO`](Self::OP_SPAWN_ECHO). + pub const OP_SPAWN_MMIO: u8 = 12; + /// Two-vCPU SPMD guest witness. + pub const OP_SPAWN_SMP: u8 = 13; } /// Capacity of the hypervisor pair window's **guest-output** console @@ -2521,6 +2605,9 @@ impl HypervisorStatusWire { pub const PAYLOAD_MULTISEG: u8 = 4; /// The MicroPython acceptance payload. pub const PAYLOAD_MICROPYTHON: u8 = 5; + /// The echo guest on the memory-mapped console (#765). + pub const PAYLOAD_MMIO: u8 = 6; + pub const PAYLOAD_SMP: u8 = 7; /// A live guest task currently exists. pub const STATUS_GUEST_EXISTS: u16 = 1 << 0; @@ -2563,6 +2650,103 @@ pub struct AccelBindingWire { pub reserved: [u8; 3], } +/// One member of a staged ordered accelerator submission. The service +/// executes entries in table order and returns one event retaining every +/// named program and buffer. Each member carries the same launch vocabulary +/// as a single `SubmitLaunch` and one ordering edge to the members before +/// it. 232 bytes: the 40-byte head keeps every binding's `U64` words on the +/// record's 8-byte grid, as in [`AccelReqWire`]. +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct AccelSequenceEntryWire { + /// Resident program object id. + pub program: U64, + /// Number of live entries in `bindings` (`1..=MAX_ACCEL_BINDINGS`). + pub binding_count: U32, + /// Member flags: [`FLAG_BARRIER`](Self::FLAG_BARRIER); every other bit + /// is reserved zero and refuses at decode. + pub flags: U32, + /// Explicit workgroup count `[x, y, z]`. With `local` all zero, the + /// six words name the artifact-defined launch; otherwise every axis of + /// both is nonzero, exactly as `SubmitLaunch`'s `target[0..24]`. + pub groups: [U32; 3], + /// Explicit logical workgroup size `[x, y, z]` (see `groups`). + pub local: [U32; 3], + /// Program bindings; entries after `binding_count` must be zero. + pub bindings: [AccelBindingWire; MAX_ACCEL_BINDINGS], +} + +impl AccelSequenceEntryWire { + /// This member starts only after every earlier member's memory writes + /// are complete and visible. Clear, the client declares it independent + /// of the members since the most recent barrier and the provider may + /// overlap them. Ignored on the first member. + pub const FLAG_BARRIER: u32 = 1 << 0; + /// A gate: this member starts only once the word its sync record names + /// holds at least the record's value, and after every earlier member's + /// writes and everything the host and storage wrote while it waited are + /// visible to it (`res/QWEN_EXPERT_STREAMING.md` §3). Takes one record + /// of the table after the entries ([`AccelSequenceSyncWire`]). + pub const FLAG_GATE: u32 = 1 << 1; + /// A signal: once this member retires and its writes are visible to the + /// host, the provider writes the record's value to the word it names. + /// Takes one record, after the member's gate record when it has both. + pub const FLAG_SIGNAL: u32 = 1 << 2; + /// Every flag a member may carry. + pub const FLAGS: u32 = Self::FLAG_BARRIER | Self::FLAG_GATE | Self::FLAG_SIGNAL; + + /// The sync records this member takes from the table. + pub const fn sync_records(&self) -> usize { + let flags = self.flags.get(); + (flags & Self::FLAG_GATE != 0) as usize + (flags & Self::FLAG_SIGNAL != 0) as usize + } +} + +/// One gate's or signal's word: the staged table of these follows the entry +/// table, one record for each [`FLAG_GATE`](AccelSequenceEntryWire::FLAG_GATE) +/// and [`FLAG_SIGNAL`](AccelSequenceEntryWire::FLAG_SIGNAL) in member order +/// (a member's gate before its signal). The word is the 8-byte-aligned +/// quadword at `offset` in a buffer of the queue's context; a gate compares +/// its low dword with `value` (unsigned, at least; `value` fits 32 bits), a +/// signal writes `value`. 32 bytes. +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct AccelSequenceSyncWire { + /// The buffer object id holding the word. + pub buffer: U64, + /// The word's byte offset in the buffer; a multiple of 8. + pub offset: U64, + /// What a gate waits for, or what a signal writes. + pub value: U64, + /// Reserved; must be zero. + pub reserved: [u8; 8], +} + +/// Sync records one staged submission may carry after its entries: what +/// the 16 KiB sequence stage holds past a full entry table (1,536 bytes), +/// more than a frozen window's routed layers need. +pub const MAX_ACCEL_SEQUENCE_SYNCS: usize = 48; + +/// Ordered members one staged submission may contain. +/// +/// Every submission costs the client its round trips to the service +/// (submit, poll, retire), so the ceiling sets how many of them a frozen +/// decode pays per token: Qwen3-32B's 1219 kernels a token were 39 windows +/// at 32 and are 20 at 64. Two bounds hold it at 64: the Xe provider's +/// per-job timing page has one 64-byte timestamp slot per member (4096 / +/// 64), and a 64-entry table (14,848 bytes) still fits the 16 KiB sequence +/// stage the inference client reserves. +pub const MAX_ACCEL_SEQUENCE_ENTRIES: usize = 64; +/// Ping-pong submission capacity: one job can execute while its successor is +/// prepared and queued. Each slot retains distinct commands, completion and +/// operand leases until retirement; increasing this requires matching storage +/// in the provider and client, not merely accepting more event handles. +pub const MAX_ACCEL_PIPELINE_DEPTH: usize = 2; + /// The accel request wire form (`AccelRequest`'s crossing shape) — the /// accelerator-class control plane (`res/DEVICE_SERVICE.md` §7): one /// vocabulary for every backend (NPU / GPU / CPU-SIMD), mirroring the @@ -2717,8 +2901,19 @@ impl AccelReqWire { /// Close the authenticated recyclable arrival. pub const OP_CLOSE_PEER: u8 = 24; /// Owned DRAM import: subject=context, a=bytes, b=alignment, c=usage, - /// d=authenticated return-landing correlation. Domain is Shared. + /// d=authenticated return-landing correlation. Domain is Shared. `c` + /// holds virtio-accel's `BufferUsage` bits and, beside them, + /// [`IMPORT_USAGE_POLLED`](Self::IMPORT_USAGE_POLLED). pub const OP_IMPORT_BUFFER: u8 = 25; + /// `OP_IMPORT_BUFFER`'s usage bit beyond virtio-accel's `BufferUsage`: + /// the importer reads or writes this buffer while the device runs — a + /// sequence's gate and signal words, and what it reads or writes around + /// them (`res/QWEN_EXPERT_STREAMING.md` §4). The provider maps it coherent + /// in both directions mid-job, at a bandwidth cost no streamed payload + /// should pay, or refuses it where it cannot prove that on its platform. + /// Only such an import may hold a gate's or a signal's word. A provider + /// that predates the bit refuses it as an unknown usage. + pub const IMPORT_USAGE_POLLED: u32 = 1 << 31; /// Return an imported allocation after quiescent buffer detachment; subject=buffer. pub const OP_EXPORT_BUFFER: u8 = 26; /// Administration pair: request retirement of the exact recyclable peer @@ -2730,6 +2925,28 @@ impl AccelReqWire { /// `AccelRequest::ReadLog` — copy the tail of the provider's firmware /// log into the staging window: `b` = stage offset, `c` = capacity. pub const OP_READ_LOG: u8 = 29; + /// Submit a dependency-ordered table as one provider event: `subject` is + /// the queue, `a`/`b` name the staged byte extent, `c` is the entry count, + /// and `d` is the relative timeout in ns (zero = infinite). The staged + /// extent is exactly `c * size_of::()` bytes, + /// then one [`AccelSequenceSyncWire`] for each gate and signal the + /// entries' flags name (none when they name none). + pub const OP_SUBMIT_SEQUENCE: u8 = 30; + /// Reserve per-pair buffer/program budgets only when the device owner + /// already holds enough DMA pool capacity. `b` = buffer bytes, `c` = + /// program bytes; no capability is transferred. Other words are zero. + pub const OP_RESERVE_MEMORY: u8 = 31; + /// Read retained failure text without a staging mapping: `a` = byte offset. + pub const OP_READ_FAILURE: u8 = 32; + /// Read the provider's current telemetry text without a staging mapping: + /// `a` = byte offset. Offset zero takes a fresh snapshot; later offsets + /// page through that snapshot. Answered with [`AccelRepWire::TAG_FAILURE`]'s + /// bounded text page. + pub const OP_READ_TELEMETRY: u8 = 33; + /// Configure optional timing for an owned, quiescent context: subject=context, + /// a=enabled (0/1). Enabling starts a fresh interval; disabling preserves it. + /// Unknown providers refuse rather than claim measurements they cannot make. + pub const OP_CONFIGURE_PROFILING: u8 = 34; } /// The accel reply wire form (`AccelResponse`'s crossing shape). Data @@ -2754,7 +2971,8 @@ pub struct AccelRepWire { /// `state` is failed; zero otherwise. pub refusal: u8, /// `TAG_EVENT`: the event's state — 1 pending, 2 complete, 3 failed - /// (see `refusal`), 4 cancelled. Zero for every other tag. + /// (see `refusal`), 4 cancelled. `TAG_FAILURE`: byte count in `info`. + /// Zero for every other tag. pub state: u8, /// Reserved; must be zero. pub reserved: [u8; 4], @@ -2769,7 +2987,9 @@ pub struct AccelRepWire { /// eight limit words (6×4 + 2×8). For an external refusal (including /// failed events and indeterminate submissions), bytes 0..4 hold the /// provider domain (u32 LE), 4..12 the code (i64 LE), and 12..80 are - /// zero. Zero for every other tag/refusal. Old decoders reject the new + /// zero. `TAG_FAILURE` holds up to `state` text bytes, zero-padded. + /// `TAG_PROFILING_CONFIGURED` holds enabled (0/1) in byte 0; its tail is zero. + /// Zero for every other tag/refusal. Old decoders reject the new /// refusal discriminant rather than silently discarding its detail. pub info: [u8; 80], } @@ -2833,6 +3053,15 @@ impl AccelRepWire { /// `AccelResponse::Log` — `id` is how many log bytes now sit at the /// requested stage offset (zero when the provider keeps no log). pub const TAG_LOG: u8 = 24; + /// Retained failure text: `id` echoes offset, `state` is bytes in `info`. + /// The unused info tail is zero; no grants or live device access. + pub const TAG_FAILURE: u8 = 25; + /// Profiling configured: id echoes context and info[0] echoes enabled (0/1). + /// All other payload bytes, state, refusal and grants remain zero. + pub const TAG_PROFILING_CONFIGURED: u8 = 26; + /// One inline diagnostic page occupies the existing info payload. + pub const FAILURE_CHUNK_BYTES: usize = + core::mem::size_of::() - core::mem::offset_of!(Self, info); /// The backend cannot perform the operation (unsupported verb, /// domain, flag set, or cancellation without the capability). @@ -3276,6 +3505,7 @@ const _: () = { + size_of::() + size_of::() + size_of::() + + size_of::() ); assert!(size_of::() == 8 + size_of::() + 16 + 64); // 152 assert!(size_of::() == 16 + size_of::()); // 496 @@ -3328,6 +3558,16 @@ const _: () = { assert!(1152 + 576 <= 0x2000, "fs pair must fit [layout] ring_bytes"); // 48-byte head ‖ 48 target words ‖ 6×32 bindings. assert!(size_of::() == 32); + // 8 program ‖ 4 count ‖ 4 flags ‖ 2×12 geometry ‖ 6×32 bindings. + assert!(size_of::() == 40 + MAX_ACCEL_BINDINGS * 32); // 232 + assert!(size_of::() == 32); + // A full entry table and its sync records fit the client's 16 KiB stage. + assert!( + MAX_ACCEL_SEQUENCE_ENTRIES * size_of::() + + MAX_ACCEL_SEQUENCE_SYNCS * size_of::() + <= 16 * 1024 + ); + assert!(core::mem::offset_of!(AccelSequenceEntryWire, bindings) % 8 == 0); assert!(size_of::() == 48 + 48 + MAX_ACCEL_BINDINGS * 32); // 288 assert!(size_of::() == 288); assert!(size_of::() == 96); @@ -3655,7 +3895,7 @@ mod tests { capacity_pages: U32::new(262144), stored_pages: U32::new(3), staging_pages: U32::new(8), - reserved2: [0; 4], + stored_extents: U32::new(2), outs: U64::new(11), ins: U64::new(9), discards: U64::new(2), @@ -4117,9 +4357,10 @@ pub struct FsReqWire { pub name_len: u8, /// Live bytes of [`name2`](Self::name2) (`0..=FS_NAME_BYTES`). pub name2_len: u8, - /// `Create`: the object subtype to stamp (data, never authority; the - /// values are `kore-dagfs`'s `ObjectSubtype`). `Search`: the freshness - /// policy. Zero elsewhere. + /// `Create`: the object subtype to stamp (data, never authority over the + /// object; the values are `kore-dagfs`'s `ObjectSubtype`, and an + /// attesting one needs its `stamp_rights` on the parent route). + /// `Search`: the freshness policy. Zero elsewhere. pub reserved0: u8, /// `Grant`: the destination session. Zero elsewhere. pub session: U16, @@ -4274,7 +4515,55 @@ impl FsReqWire { pub const OP_WRITE_REPLACEMENT: u8 = 44; pub const OP_COMMIT_REPLACE: u8 = 45; pub const OP_ABORT_REPLACE: u8 = 46; - pub const OP_MAX: u8 = Self::OP_ABORT_REPLACE; + /// **Retired, reserved:** the pack lane's verbs (`PackInfo`, + /// `PackAttach`, `PackRead`, `PackDetach`, `PackPoll`, `PackSubmit`, + /// `PackKick`). The pack region is read through a storage ring now + /// (`OP_STOR_*`, `res/STORAGE_TRANSPORT.md` §4); these codes decode to + /// nothing and are never reassigned. + pub const OP_PACK_INFO: u8 = 47; + pub const OP_PACK_ATTACH: u8 = 48; + pub const OP_PACK_READ: u8 = 49; + pub const OP_PACK_DETACH: u8 = 50; + pub const OP_PACK_POLL: u8 = 51; + pub const OP_PACK_SUBMIT: u8 = 52; + pub const OP_PACK_KICK: u8 = 53; + /// Attach the session's storage ring (`res/STORAGE_TRANSPORT.md` §4): the + /// caller-funded, formatted window rides the retained return leg as a + /// view; `window_offset` carries only the return-post correlation. The + /// reply carries the block size in `count` and the transfer ceiling in + /// `cursor`. + pub const OP_STOR_ATTACH: u8 = 54; + /// Install a read grant over the designated pack region into the + /// session's ring. The reply carries the grant id in `handle` and the + /// region's length in `len`. + pub const OP_STOR_GRANT_PACK: u8 = 55; + /// Adopt one device-mapping fabric route over caller-funded memory as a + /// target of the session's ring; the route rides the retained return leg + /// and `window_offset` carries only the return-post correlation. The + /// reply carries the target slot in `handle` and its length in `len`. + pub const OP_STOR_TARGET: u8 = 56; + /// Release ring target `handle`; refused `BUSY` while a command uses it. + pub const OP_STOR_RELEASE: u8 = 57; + /// The notice a parked row is owed after the caller published into its + /// ring. No operand. + pub const OP_STOR_KICK: u8 = 58; + /// Detach and retire the session's ring; refused `BUSY` until nothing is + /// in flight. No operand. + pub const OP_STOR_DETACH: u8 = 59; + /// Install a read grant over lent version `handle` (a version lease this + /// session holds) into the session's ring: the version's whole blocks, + /// pinned by the lease. The reply carries the grant id in `handle` and + /// the bytes it covers in `len`; the partial last block, if any, is read + /// through the lease as before. Releasing the lease is refused `BUSY` + /// while a read under the grant is in flight. + pub const OP_STOR_GRANT_VERSION: u8 = 60; + /// Read the transport engine's counters — testimony, no authority: the + /// reply carries submissions in `handle`, completions in `len`, device + /// commands in flight in `count`, expired deadlines in `cursor`, and + /// stale completions (low half) and refused timeline completions (high + /// half) in `revision`. No operand. + pub const OP_STOR_INSPECT: u8 = 61; + pub const OP_MAX: u8 = Self::OP_STOR_INSPECT; /// [`kind`](Self::kind): a file. pub const KIND_FILE: u8 = 1; @@ -4309,7 +4598,10 @@ impl FsReqWire { pub struct FsRepWire { /// `STATUS_OK` or a refusal class; zero (unseeded) is invalid. pub status: u8, - /// Object kind on a successful open/create/stat, else zero. + /// Object kind on a successful open/create/stat; on a + /// [`STATUS_PROVIDER_FAILURE`](Self::STATUS_PROVIDER_FAILURE) refusal, + /// the provider's cause (`kore-dagfs`'s frozen `ProviderError` code, zero + /// for none); else zero. pub kind: u8, /// Reply flags — [`FLAG_MORE`](Self::FLAG_MORE) for a paged listing. pub flags: u8, diff --git a/native/crates/abi/src/task.rs b/native/crates/abi/src/task.rs index a5d541b..83eca93 100644 --- a/native/crates/abi/src/task.rs +++ b/native/crates/abi/src/task.rs @@ -373,6 +373,26 @@ pub mod role { /// MM's per-client outstanding-supply policy, in bytes: indices 0 and 1 /// carry the low and high u32 words. Boot data only; grants no authority. pub const SUPPLY_QUOTA: u16 = 43; + /// Read-only model-data lease, delivered by exec for PROFILE_MODEL_DATA. + /// Index 0 is the lease; data indices 1 and 2 carry the exact byte length + /// (low/high u32). The receiver maps it; page padding is not model data. + /// Optional data indices 3..=7 report disk-load phase durations in ms: + /// setup, source validation, allocation/mapping, read/verify/copy, cleanup. + /// u32::MAX means unavailable. These values confer no authority. + pub const MODEL_DATA: u16 = 44; + /// A filesystem launch's storage-ring window (`res/STORAGE_TRANSPORT.md` + /// §4): a `MemLease` of exactly `kore_ring::stor::WINDOW_BYTES`, funded by + /// exec from its own mm face and moved in whole at founding, so the child + /// attaches a ring without an mm face. Exec-synthesized: a caller row + /// naming it is refused. + pub const STOR_RING: u16 = 45; + /// A filesystem launch's ring read target: a fresh `MemLease` exec funds + /// beside [`STOR_RING`] and moves in whole. Nothing maps it before the + /// child registers it on the fabric, which a device route requires — a + /// range of the launcher's own arena could not be one, since the + /// launcher keeps that arena mapped. Exec-synthesized, like + /// [`STOR_RING`]. + pub const STOR_TARGET: u16 = 46; /// First caller-assigned role — roles `>=` this are convention-defined. pub const USER_BASE: u16 = 0x8000; } @@ -426,14 +446,17 @@ pub const INTERACTIVE_CHILD_LOADER_FACES: usize = DESKTOP_BOOTSTRAP_WINDOW_SLOTS + 2 * DESKTOP_CONTROLLER_CAPACITY; /// Live immutable image-generation peers the boot catalogue can hold at once: -/// the 35-row bootstrap catalogue — the desktop's `editor` and `power-ui` -/// beside the two native filesystem verbs `append` and `sync`, and the two -/// WebAssembly rows, `wasm-witness` and `wasm` — plus the shell, the -/// graphical terminal, and three extraction parsers. Named +/// the 40-row bootstrap catalogue — the desktop's `editor` and `power-ui` +/// beside the native filesystem verbs `append`, `sync` and `pack`, the two +/// WebAssembly rows, `wasm-witness` and `wasm`, `inference` / +/// `model-data-witness`, and `shell`, the guest shell `term` opens — plus the +/// founded shell, the graphical terminal, and three extraction parsers. +/// Exec asserts this covers its catalogue (`kore-svc-exec`), the one place +/// that sees both. Named /// separately from the interactive reservation below because the two grow for /// unrelated reasons: this one when a row joins the catalogue, that one when /// the desktop admits another window or controller. -pub const BOOTSTRAP_IMAGE_PEERS: usize = 40; +pub const BOOTSTRAP_IMAGE_PEERS: usize = 45; /// Bootstrap face-supply sizing. Image and runtime grow their face tables and /// rings on demand; this reserves enough initial memory for the boot catalogue diff --git a/native/crates/abi/src/uring.rs b/native/crates/abi/src/uring.rs index 1662edc..07dc0e7 100644 --- a/native/crates/abi/src/uring.rs +++ b/native/crates/abi/src/uring.rs @@ -598,6 +598,11 @@ pub mod op { /// read-write destination necessarily passed the stronger read proof. The /// SQE carries indices only and cannot supply a missing topology proof. pub const PEER_COPY: u8 = KORE_BASE + 2; + /// Storage-transport **discard** (`res/STORAGE_TRANSPORT.md` §4): the + /// grant's range `[off, off + len)` no longer holds data the client + /// needs. Carries no target (`buf_index` ignored). Needs the grant's + /// discard right; a device without deallocate completes it as a no-op. + pub const DISCARD: u8 = KORE_BASE + 3; } #[cfg(test)] diff --git a/native/crates/collections/src/boundedvec.rs b/native/crates/collections/src/boundedvec.rs index 846f0b1..0bc4216 100644 --- a/native/crates/collections/src/boundedvec.rs +++ b/native/crates/collections/src/boundedvec.rs @@ -278,7 +278,10 @@ mod tests { for value in &mut items { *value *= 2; } - assert_eq!((&items).into_iter().copied().collect::>(), [2, 4, 6]); + assert_eq!( + (&items).into_iter().copied().collect::>(), + [2, 4, 6] + ); assert_eq!(items.into_iter().sum::(), 12); } diff --git a/native/crates/dagfs/Cargo.toml b/native/crates/dagfs/Cargo.toml index b395ea7..c063594 100644 --- a/native/crates/dagfs/Cargo.toml +++ b/native/crates/dagfs/Cargo.toml @@ -12,7 +12,7 @@ default = [] # the wire codec, the ring and host clients, and the view capability leaves. # Pure types — names, ids, bounds, errors, paths — never need it, which is # what lets the linker stub and a no-allocator image hold a `Name`. -runtime = ["dep:kore-rt", "dep:kore-svc-extract", "dep:zerocopy"] +runtime = ["dep:kore-rt", "dep:kore-ring", "dep:kore-svc-extract", "dep:zerocopy"] # Plain-text index publication for the version a client just wrote (the # #476 producer seam): the walker's `index` module and its parser. index = ["runtime", "dep:kore-svc-extract-worker-rust"] @@ -22,6 +22,8 @@ heapless = { version = "0.9.3", default-features = false } kore-abi = { path = "../abi" } kore-collections = { path = "../collections", default-features = false } kore-rt = { path = "../rt", default-features = false, optional = true } +# The storage ring window a client submits through (`stor`). +kore-ring = { path = "../ring", optional = true } kore-svc-extract = { path = "../../services/extract", optional = true } kore-svc-extract-worker-rust = { path = "../../services/extract-worker-rust", optional = true } # The little-endian field wrappers the frozen records are built from. No diff --git a/native/crates/dagfs/src/client.rs b/native/crates/dagfs/src/client.rs index 9f324dd..a493d73 100644 --- a/native/crates/dagfs/src/client.rs +++ b/native/crates/dagfs/src/client.rs @@ -101,6 +101,11 @@ impl LentVersion { pub const fn identity(&self) -> InputIdentity { self.identity } + + /// The lease's wire name — for a request that names it, never authority. + pub(crate) const fn lease(&self) -> VersionLease { + self.lease + } } /// Identity readback failed after the filesystem had already installed the @@ -246,6 +251,16 @@ pub struct RingReleasePendingVersionFailure { pub version: PendingVersion, } +/// Rights a ring target route must arrive with. The service maps the route +/// into its controller's DMA domain (deriving the IOVA child) and retires +/// that mapping on detach or peer death, which needs all four; the caller +/// keeps its own CPU child of the same route, so lending these rights to the +/// filesystem never moves the caller's view. +pub const RING_TARGET_RIGHTS: CapRights = CapRights::ACCESS + .union(CapRights::MODIFY) + .union(CapRights::DERIVE) + .union(CapRights::REVOKE); + /// Rights of the detached data-window view received by the filesystem. pub const DATA_WINDOW_RIGHTS: CapRights = CapRights::ACCESS.union(CapRights::MODIFY); /// The launcher-local staged view must additionally carry DERIVE so the @@ -312,6 +327,14 @@ impl<'w, 't, T: rt_transport::Transport> RingFs<'w, 't, T> { self.pair.is_idle() } + /// Drive replies by re-polling the ring rather than the executor's clock + /// — see [`SvcClient::set_direct_polling`]. For a caller that completes + /// its calls under `block_on` from inside a task's poll; restored before + /// the executor drives this client again. + pub fn set_direct_polling(&mut self, direct: bool) { + self.pair.set_direct_polling(direct); + } + /// Resume the one already-published administration request. /// /// Callers retain the operation phase in their affine lifecycle owner; @@ -340,6 +363,20 @@ impl<'w, 't, T: rt_transport::Transport> RingFs<'w, 't, T> { self.finish(reply) } + /// Give one exact administration request a bounded observation slice. + /// Expiry leaves its request correlated on this pair for a later pass. + async fn call_exact_bounded( + &mut self, + request: &FsRequest, + budget_ns: u64, + ) -> Result { + let reply = match self.pair.resume_exact_bounded(request, budget_ns).await { + Err(CallError::NoOutstanding) => self.pair.call_bounded(request, budget_ns).await, + result => result, + }?; + self.finish(reply) + } + async fn checked(&mut self, request: &FsRequest) -> Result { let response = self .call_exact(request) @@ -544,7 +581,10 @@ impl<'w, 't, T: rt_transport::Transport> RingFs<'w, 't, T> { } /// Create an object stamped with `subtype` — data the object carries for - /// its life, never authority. A directory refuses any but `PLAIN`. + /// its life, never authority over it. A directory refuses any but + /// `PLAIN`, and an attesting stamp needs its + /// [`stamp_rights`](crate::ObjectSubtype::stamp_rights) on `parent`'s + /// route (`AccessDenied` otherwise). pub async fn create_typed( &mut self, parent: Handle, @@ -592,6 +632,25 @@ impl<'w, 't, T: rt_transport::Transport> RingFs<'w, 't, T> { } } + /// Observe one status request for at most `budget_ns`. A timeout retains + /// that exact request on the pair; the next call to this method resumes + /// it rather than publishing a second request. + pub async fn system_status_bounded( + &mut self, + window_offset: u32, + budget_ns: u64, + ) -> Result { + match self + .call_exact_bounded(&FsRequest::SystemStatus { window_offset }, budget_ns) + .await + .map_err(RingFsError::Infrastructure)? + { + FsResponse::SystemStatusReported { len } => Ok(len), + FsResponse::Refused(error) => Err(RingFsError::Refused(error)), + _ => Err(RingFsError::Wire), + } + } + pub async fn search( &mut self, handle: Handle, @@ -989,6 +1048,47 @@ impl<'w, 't, T: rt_transport::Transport> RingFs<'w, 't, T> { self.finish(reply).map_err(StagedReturnError::AfterTransfer) } + /// Lend this session's storage-ring window ([`FsRequest::StorAttach`]): + /// a view derived from `source` over its first `bytes`, as the data + /// window is lent. The caller keeps its own mapping of the parent. + pub async fn lend_ring_window( + &mut self, + source: &S, + bytes: u32, + ) -> Result { + let request = FsRequest::StorAttach { + announced: self.return_land, + }; + let reply = self + .pair + .call_with_staged_return(&request, self.return_tx, DATA_WINDOW_RIGHTS, |derive| { + derive.lease_window(source, DATA_WINDOW_STAGED_RIGHTS, 0, u64::from(bytes)) + }) + .await + .map_err(|error| match error { + StagedReturnError::NotTransferred(error) + | StagedReturnError::AfterTransfer(error) => RingFsError::Infrastructure(error), + })?; + self.finish(reply).map_err(RingFsError::Infrastructure) + } + + /// Lend one device-mapping route over caller-funded memory as a target + /// of this session's ring ([`FsRequest::StorTarget`]), with + /// [`RING_TARGET_RIGHTS`]'s custody. + pub async fn lend_ring_target( + &mut self, + route: C, + ) -> Result> { + let request = FsRequest::StorTarget { + announced: self.return_land, + }; + let reply = self + .pair + .call_with_return(&request, self.return_tx, route, RING_TARGET_RIGHTS) + .await?; + self.finish(reply).map_err(ReturnCallError::AfterTransfer) + } + pub async fn bind_view( &mut self, handle: Handle, @@ -1093,6 +1193,14 @@ impl<'w, 't, T: rt_transport::Transport> RingFs<'w, 't, T> { self.call(&FsRequest::TakeAdmission).await } + pub async fn take_admission_bounded( + &mut self, + budget_ns: u64, + ) -> Result { + self.call_exact_bounded(&FsRequest::TakeAdmission, budget_ns) + .await + } + /// Ask the service host to retire one exact dynamic generation. pub async fn retire_peer( &mut self, @@ -1101,6 +1209,15 @@ impl<'w, 't, T: rt_transport::Transport> RingFs<'w, 't, T> { self.call_exact(&FsRequest::RetirePeer { peer }).await } + pub async fn retire_peer_bounded( + &mut self, + peer: kore_rt::svc::PeerId, + budget_ns: u64, + ) -> Result { + self.call_exact_bounded(&FsRequest::RetirePeer { peer }, budget_ns) + .await + } + /// Fence one targeted retirement. Success means the host no longer owns /// this exact generation, so the caller may reclaim the local pair roots /// and backing lease without racing the service's final access. @@ -1110,6 +1227,15 @@ impl<'w, 't, T: rt_transport::Transport> RingFs<'w, 't, T> { ) -> Result { self.call_exact(&FsRequest::ConfirmRetired { peer }).await } + + pub async fn confirm_retired_bounded( + &mut self, + peer: kore_rt::svc::PeerId, + budget_ns: u64, + ) -> Result { + self.call_exact_bounded(&FsRequest::ConfirmRetired { peer }, budget_ns) + .await + } } impl FsClient { @@ -2021,6 +2147,30 @@ mod exact_attach_tests { ); } + #[test] + fn status_request_identity_rejects_a_late_reply_for_another_window() { + let transport = RecordingTransport::refusing(); + let (mut fs, mut serve) = ring_fs(&transport, SlotRef::from_wire(0x41)); + let status = FsRequest::SystemStatus { window_offset: 0 }; + assert_eq!(crate::block_on(fs.call_exact(&status)), Err(CallError::Transport)); + assert_eq!(take_request(&mut serve), status); + let reply = ::encode_rep( + &FsResponse::SystemStatusReported { len: 402 }, 0, + ); + assert!(serve.rep.push(reply).is_ok()); + assert_eq!( + crate::block_on(fs.call_exact(&FsRequest::SystemStatus { window_offset: 1 })), + Err(CallError::Outstanding), + ); + transport.succeed.set(true); + assert_eq!( + crate::block_on(fs.call_exact(&status)), + Ok(FsResponse::SystemStatusReported { len: 402 }), + ); + assert!(fs.is_idle()); + assert!(serve.req.pop().is_none()); + } + #[test] fn attach_retries_only_transport_progress_outcomes() { for error in [ diff --git a/native/crates/dagfs/src/error.rs b/native/crates/dagfs/src/error.rs index 13c123d..041003f 100644 --- a/native/crates/dagfs/src/error.rs +++ b/native/crates/dagfs/src/error.rs @@ -98,6 +98,11 @@ pub enum ProviderError { ReadOnly, /// A bounded queue, tracker, buffer, or placement table is full. Backpressure, + /// A DMA window an earlier client lent has not retired from the device, + /// and the requester-wide retirement that would clear it failed, so the + /// controller admits no new client window until a retry succeeds + /// (`res/LIFECYCLE_HARDENING.md` §6). Existing lanes keep serving. + Retiring, /// A completion exceeded its budget while the backend remained healthy. Timeout, } @@ -119,6 +124,7 @@ impl From for FsError { | ProviderError::Draining | ProviderError::ReadOnly | ProviderError::Backpressure + | ProviderError::Retiring | ProviderError::Timeout => Self::ProviderFailure(error), } } @@ -161,6 +167,7 @@ mod tests { ProviderError::Draining, ProviderError::ReadOnly, ProviderError::Backpressure, + ProviderError::Retiring, ProviderError::Timeout, ] { assert_eq!(FsError::from(cause), FsError::ProviderFailure(cause)); diff --git a/native/crates/dagfs/src/fs.rs b/native/crates/dagfs/src/fs.rs index dc9b276..87b9ac4 100644 --- a/native/crates/dagfs/src/fs.rs +++ b/native/crates/dagfs/src/fs.rs @@ -58,6 +58,16 @@ pub const PEAK_HANDLES: usize = 4; const _: () = assert!(PEAK_HANDLES <= MAX_HANDLES_PER_SESSION); /// This process's private filesystem pair. +impl Fs<'_, '_> { + /// Drive replies by re-polling the ring rather than the executor's clock + /// ([`RingFs::set_direct_polling`]): for a caller that completes its + /// filesystem calls under `block_on` from inside a task's poll, and + /// restores the executor mode before the executor drives them again. + pub fn set_direct_polling(&mut self, direct: bool) { + self.client.set_direct_polling(direct); + } +} + pub struct Fs<'w, 't> { pub(crate) client: RingFs<'w, 't, TrapTransport>, pub(crate) window: &'static mut [u8], @@ -75,6 +85,14 @@ pub struct Fs<'w, 't> { /// endpoint carries REVOKE only, so it cannot be used as a cwd or further /// delegated by the command. pub(crate) grant_target: Option, + /// This launch's storage ring, when its founding delivered one + /// ([`role::STOR_RING`] and [`role::STOR_TARGET`]): set up on the first + /// read that streams through it ([`crate::stor::FacadeRing`]). + pub(crate) ring: crate::stor::FacadeRing, + /// Files read whole through that ring. + pub(crate) ring_files: u64, + /// Why the last whole-file read did or did not take that ring. + pub(crate) ring_note: &'static str, } impl<'w, 't> Fs<'w, 't> { @@ -169,6 +187,14 @@ impl<'w, 't> Fs<'w, 't> { if !attached { return None; } + let ring = crate::stor::FacadeRing::delivered( + rt.bootstrap() + .handle(role::STOR_RING, 0) + .and_then(verbs::take_lease), + rt.bootstrap() + .handle(role::STOR_TARGET, 0) + .and_then(verbs::take_lease), + ); Some(Self { client, window: data, @@ -176,6 +202,9 @@ impl<'w, 't> Fs<'w, 't> { _data_lease: data_lease, view_binding, grant_target, + ring, + ring_files: 0, + ring_note: "unused", }) } @@ -191,6 +220,26 @@ impl<'w, 't> Fs<'w, 't> { self.window } + /// A bounded, validated read of the service's aggregate posture. A + /// timeout leaves the exact status request pending on this pair. + pub async fn system_status_bounded( + &mut self, + budget_ns: u64, + ) -> Result { + use kore_abi::svc::FsSystemStatusWire; + let len = self.client.system_status_bounded(0, budget_ns).await?; + if len as usize != FsSystemStatusWire::BYTES { + return Err(crate::client::RingFsError::Wire); + } + crate::client::read_system_status(self.window, 0) + .map_err(crate::client::RingFsError::Refused) + } + + /// Whether a new filesystem request can safely be published. + pub fn status_pair_idle(&self) -> bool { + self.client.is_idle() + } + /// One round trip. A transport failure is [`Refusal::Pair`] — a caller /// never turns "unreachable" into a filesystem answer. pub async fn call(&mut self, request: &FsRequest) -> Result { diff --git a/native/crates/dagfs/src/graph.rs b/native/crates/dagfs/src/graph.rs index 0e798c3..de24336 100644 --- a/native/crates/dagfs/src/graph.rs +++ b/native/crates/dagfs/src/graph.rs @@ -27,7 +27,10 @@ impl<'w, 't> Fs<'w, 't> { } /// [`Fs::create`], stamping `subtype` — data the object carries for its - /// life, never authority. A directory refuses any but `PLAIN`. + /// life, never authority over it. A directory refuses any but `PLAIN`, + /// and an attesting stamp needs its + /// [`stamp_rights`](ObjectSubtype::stamp_rights) on `parent`'s route + /// (`AccessDenied` otherwise). pub async fn create_typed( &mut self, parent: &View, diff --git a/native/crates/dagfs/src/io.rs b/native/crates/dagfs/src/io.rs index 61b6e71..e9db6b3 100644 --- a/native/crates/dagfs/src/io.rs +++ b/native/crates/dagfs/src/io.rs @@ -62,10 +62,14 @@ impl<'w, 't> Fs<'w, 't> { self.window.len().min(self.client.transfer_limit()) } - /// Read at most one transfer window into caller-owned storage. + /// Read into caller-owned storage: at most one transfer window, or — for + /// a destination of at least 256 KiB, when this launch has a storage ring + /// — the whole destination, streamed through the ring from one pinned + /// version (`res/STORAGE_TRANSPORT.md` §9, S5e). /// /// A short successful read is not necessarily EOF; only zero is EOF. - /// The destination is unchanged on refusal. No allocation or retained borrow. + /// On refusal the destination may hold a prefix of the bytes. No + /// allocation or retained borrow. pub async fn read_at( &mut self, file: &View, @@ -75,6 +79,21 @@ impl<'w, 't> Fs<'w, 't> { if dst.is_empty() { return Ok(0); } + if dst.len() as u64 >= crate::stor::FACADE_MIN_BYTES { + let mut filled = 0usize; + let streamed = self + .read_range_by_ring(file, offset, dst.len() as u64, &mut |bytes: &[u8]| { + let end = filled + bytes.len(); + let slot = dst.get_mut(filled..end).ok_or(Refusal::Pair)?; + slot.copy_from_slice(bytes); + filled = end; + Ok(()) + }) + .await?; + if streamed.is_some() { + return Ok(filled); + } + } let requested = transfer_len(offset, dst.len(), self.chunk())?; let reply = self .checked(&FsRequest::Read { @@ -176,6 +195,15 @@ impl<'w, 't> Fs<'w, 't> { F: FnMut(&[u8]) -> Result<(), Refusal>, { let file = self.open(path, WALK_READ).await?; + // This launch's storage ring, when it has one: one pinned version, + // its whole blocks by DMA (`res/STORAGE_TRANSPORT.md` §9, S5d). + match self.read_range_by_ring(&file, 0, u64::MAX, &mut consume).await { + Ok(None) => {} + done => { + self.close(file).await; + return done.map(|_| ()); + } + } let mut offset = 0u64; let result = loop { let count = match self.read_chunk(&file, offset).await { diff --git a/native/crates/dagfs/src/lib.rs b/native/crates/dagfs/src/lib.rs index a320fb1..bc60e7b 100644 --- a/native/crates/dagfs/src/lib.rs +++ b/native/crates/dagfs/src/lib.rs @@ -27,6 +27,13 @@ #[cfg(test)] extern crate std; +/// The display name of the **library view** every task view carries: the +/// shared `lib/` directory the store installer seeds, granted `READ` alone — +/// no `WRITE`, `MUTATE_NAMESPACE`, `SHARE`, or `BIND_EXECUTABLE`. The +/// service mints the edge under this name and a late loader enters it by +/// this name, so the two halves share one spelling. +pub const LIBS_VIEW: &str = "libs"; + pub mod bounds; #[cfg(feature = "runtime")] mod capability; @@ -52,6 +59,8 @@ pub mod protocol; mod refusal; #[cfg(feature = "runtime")] pub mod search; +#[cfg(feature = "runtime")] +pub mod stor; pub mod text; #[cfg(feature = "runtime")] mod view; @@ -65,10 +74,11 @@ pub use capability::{FS_GRANT_TARGET_DELEGATE_RIGHTS, FS_GRANT_TARGET_RIGHTS, Fs #[cfg(feature = "runtime")] pub use client::{ FilesystemFace as ClientFace, FsClient, GrantEntry, IdentifyVersionFailure, LentVersion, - Opened, PendingVersion, ReleasePendingVersionFailure, ReleaseVersionFailure, RingFs, - RingFsError, RingReleasePendingVersionFailure, RingReleaseVersionFailure, SearchResultPage, - Stat, Transport, VERSION_IDENTITY_BYTES, process_face, read_dir_entry, read_grant_entry, - read_search_hit, read_system_status, read_version_identity, + Opened, RING_TARGET_RIGHTS, PendingVersion, ReleasePendingVersionFailure, + ReleaseVersionFailure, RingFs, RingFsError, RingReleasePendingVersionFailure, + RingReleaseVersionFailure, SearchResultPage, Stat, Transport, VERSION_IDENTITY_BYTES, + process_face, read_dir_entry, read_grant_entry, read_search_hit, read_system_status, + read_version_identity, }; pub use entry::DirEntry; pub use error::{FsError, FsResult, ProviderError}; diff --git a/native/crates/dagfs/src/object.rs b/native/crates/dagfs/src/object.rs index ef82d16..e47d9c5 100644 --- a/native/crates/dagfs/src/object.rs +++ b/native/crates/dagfs/src/object.rs @@ -1,5 +1,7 @@ use kore_collections::SlotId as SlotKey; +use crate::ids::Rights; + /// Volume-stable object identity. Unlike [`ObjectId`], this value survives a /// service restart and is safe to serialize because it conveys no authority. #[derive(Clone, Copy, PartialEq, Eq, Debug, Default, Hash, PartialOrd, Ord)] @@ -76,13 +78,23 @@ pub const ANY_REVISION: Revision = 0; /// /// The class ([`ObjectKind`]) says what may be done to an object; the /// subtype says how its bytes are meant to be read, so a consumer need not -/// infer it from a name suffix or a magic number. It gates nothing: -/// `Rights::BIND_EXECUTABLE`, the executable version lease, and the image -/// service's descriptor validation remain the only executable authority, -/// and an [`EXECUTABLE_IMAGE`](Self::EXECUTABLE_IMAGE) file is opened, -/// read, and written exactly as a plain one is. Directories are always -/// [`PLAIN`](Self::PLAIN). Unknown values are carried, not refused: a newer -/// writer's subtype is data an older reader passes through intact. +/// infer it from a name suffix or a magic number. It grants nothing to +/// whoever holds the object: `Rights::BIND_EXECUTABLE`, the executable +/// version lease, and the image service's descriptor validation remain the +/// only executable authority, and an +/// [`EXECUTABLE_IMAGE`](Self::EXECUTABLE_IMAGE) file is opened and read +/// exactly as a plain one is. Directories are always [`PLAIN`](Self::PLAIN). +/// Unknown values are carried, not refused: a newer writer's subtype is +/// data an older reader passes through intact. +/// +/// **Some subtypes attest.** A stamp a consumer relies on as the store's +/// word about where the bytes came from is worth exactly the authority it +/// takes to write it. So the stamp is not free: +/// [`stamp_rights`](Self::stamp_rights) names the right a route must carry +/// — beyond the operation's own — to stamp the subtype at creation *and* to +/// change the bytes of an object that carries it. Without the second half +/// the first is void: a writer could keep the installer's stamp and swap +/// what it vouches for. #[derive(Clone, Copy, PartialEq, Eq, Debug, Default, Hash, PartialOrd, Ord)] pub struct ObjectSubtype(u8); @@ -90,12 +102,16 @@ impl ObjectSubtype { /// No particular interpretation — every object before subtypes existed. pub const PLAIN: Self = Self(0); /// An executable image the store installer wrote: what exec's seeder - /// re-adopts by digest, and what `inspect` reports. + /// re-adopts by digest, what a late loader demands before it links, and + /// what `inspect` reports. Attesting — only `BIND_EXECUTABLE` stamps or + /// rewrites one. pub const EXECUTABLE_IMAGE: Self = Self(1); /// A package manifest (`kore_abi::pkg::PackageManifest`): the typed /// record at the root of one installed `.pkg` subtree (`res/PKG.md`). /// Data, like every subtype -- an installer reads it to decide which /// Link edges to project into `apps/` and `libs/`; it grants nothing. + /// Attesting for that reason: only `BIND_EXECUTABLE` stamps or rewrites + /// one. pub const PACKAGE_MANIFEST: Self = Self(2); #[inline] @@ -113,6 +129,31 @@ impl ObjectSubtype { pub const fn is_plain(self) -> bool { self.0 == 0 } + + /// The right a route must carry, beyond the operation's own, to stamp + /// this subtype at creation or to change the bytes of an object that + /// carries it. + /// + /// [`PLAIN`](Self::PLAIN) describes nothing and costs nothing. + /// [`EXECUTABLE_IMAGE`](Self::EXECUTABLE_IMAGE) says *the store installer + /// wrote these bytes* — the late loader links nothing else — and + /// [`PACKAGE_MANIFEST`](Self::PACKAGE_MANIFEST) is the record an + /// installer projects `apps/` and `libs/` from: both are installation + /// into the shared executable namespace, which is exactly what + /// [`Rights::BIND_EXECUTABLE`] governs, so a home face (`Rights::HOME`) + /// can neither forge one nor rewrite one it can reach. An unknown value + /// fails closed the same way: this engine cannot vouch for a stamp + /// whose meaning it does not know. A future subtype that only + /// *describes* its bytes joins `PLAIN` here, explicitly. + #[inline] + pub const fn stamp_rights(self) -> Rights { + match self { + Self::PLAIN => Rights::NONE, + Self::EXECUTABLE_IMAGE | Self::PACKAGE_MANIFEST => Rights::BIND_EXECUTABLE, + // Unknown: fail closed until classified. + _ => Rights::BIND_EXECUTABLE, + } + } } #[cfg(test)] @@ -128,4 +169,22 @@ mod subtype_tests { assert!(!ObjectSubtype::EXECUTABLE_IMAGE.is_plain()); assert_eq!(ObjectSubtype::default(), ObjectSubtype::PLAIN); } + + /// Only `PLAIN` is free; every attesting stamp — and every stamp this + /// build cannot interpret — is the install authority's, which the home + /// face withholds. + #[test] + fn attesting_and_unknown_stamps_need_the_install_authority() { + assert_eq!(ObjectSubtype::PLAIN.stamp_rights(), Rights::NONE); + for subtype in [ + ObjectSubtype::EXECUTABLE_IMAGE, + ObjectSubtype::PACKAGE_MANIFEST, + ObjectSubtype::from_bits(0x7f), + ObjectSubtype::from_bits(u8::MAX), + ] { + assert_eq!(subtype.stamp_rights(), Rights::BIND_EXECUTABLE); + assert!(!Rights::HOME.contains(subtype.stamp_rights())); + assert!(Rights::ALL.contains(subtype.stamp_rights())); + } + } } diff --git a/native/crates/dagfs/src/protocol.rs b/native/crates/dagfs/src/protocol.rs index 0bb711c..1138465 100644 --- a/native/crates/dagfs/src/protocol.rs +++ b/native/crates/dagfs/src/protocol.rs @@ -68,7 +68,9 @@ pub enum FsRequest { name: Name, kind: ObjectKind, /// Data stamped at creation; [`ObjectSubtype::PLAIN`] for a directory - /// and for every ordinary file. + /// and for every ordinary file. An attesting stamp is refused + /// unless `parent`'s route carries its + /// [`stamp_rights`](ObjectSubtype::stamp_rights). subtype: ObjectSubtype, }, Read { @@ -314,6 +316,50 @@ pub enum FsRequest { }, /// Retire the exact recyclable pair on which this request arrived. ClosePeer, + /// Attach this session's storage ring (`res/STORAGE_TRANSPORT.md` §4): + /// a window of `kore_ring::stor::WINDOW_BYTES` the caller funded, + /// formatted (`kore_ring::stor::format`) and lent as a view. `announced` + /// is return-post correlation only; the view rides the retained return + /// leg. A session holds at most one ring. From here the caller submits + /// by writing its own window: the pair carries only control. + StorAttach { + announced: kore_abi::session::SlotRef, + }, + /// Install a read grant over the designated pack region into this + /// session's ring: Demand for reads a thread is blocked on, Stream for + /// read-ahead. Answers the grant's id and length. + StorGrantPack, + /// Adopt one device-mapping fabric route over caller-funded memory as a + /// target of this session's ring, with [`RING_TARGET_RIGHTS`]'s custody. + /// Answers the target slot a submission names in `buf_index`. + /// + /// [`RING_TARGET_RIGHTS`]: crate::RING_TARGET_RIGHTS + StorTarget { + announced: kore_abi::session::SlotRef, + }, + /// Hand ring target `slot` back. Refused `Busy` while a command uses it; + /// its device mapping retires once the controller is idle. + StorRelease { + slot: u16, + }, + /// The notice a parked row is owed: the caller published submissions + /// and its publish found the row's park intent. Answered once the row + /// has advanced the ring. + StorKick, + /// Install a read grant over lent version `lease` into this session's + /// ring: the version's whole blocks, pinned by the lease for as long as + /// the lease is held. Answers the grant's id and the bytes it covers; a + /// partial last block is read through the lease. Releasing the lease is + /// refused `Busy` while a read under the grant is in flight. + StorGrantVersion { + lease: crate::VersionLease, + }, + /// Read the transport engine's counters: what a stalled wait reports. + StorInspect, + /// Detach this session's ring, release its targets and retire it. + /// Refused `Busy` while commands are still in flight — the ring stops + /// taking submissions at the first ask; ask again. + StorDetach, } impl FsRequest { @@ -440,6 +486,56 @@ pub enum FsResponse { PeerAdmitted { peer: PeerId, }, + /// The session's ring is attached: the geometry every submission obeys. + StorAttached { + /// Logical block size — every offset and length is a multiple. + block_size: u32, + /// Largest single device command, in bytes; the row splits longer + /// submissions itself. + max_transfer: u32, + }, + /// A grant was installed into the session's ring. `grant` is the + /// `kore_abi::stor::GrantId` bit pattern a submission carries in `fd`; + /// it names the grant only inside that ring. + StorGranted { + grant: u32, + bytes: u64, + }, + /// A target was adopted into the session's ring as `slot`. + StorTargetAdopted { + slot: u16, + bytes: u64, + }, + /// The transport engine's counters, every ring together. + StorInspected { + /// Submissions popped. + submissions: u64, + /// Completions posted. + completions: u64, + /// Device commands in flight now. + in_flight: u32, + /// Ops reported past their deadline. + expired: u32, + /// Device completions naming no live command. + stale: u32, + /// Completions a timeline refused (each stops a fence for good). + timeline_refusals: u32, + /// Device commands the queue set refused to stage. + stage_refusals: u32, + /// Abort commands sent for expired device commands (saturating at + /// 2^24 − 1 on the wire). + aborts: u32, + /// Aborts the controller refused (saturating at 15 on the wire). + aborts_refused: u8, + /// The controller arbitrates by weighted round robin. + weighted: bool, + /// Rings attached: the row's own lanes and every client's (at most 7 + /// on the wire). + rings: u8, + /// The last stage refusal, as `kore_driver::block::BlockError`'s + /// discriminant plus one; zero for none. + last_stage_error: u8, + }, /// The operation was refused. The engine's vocabulary crosses intact. Refused(FsError), } diff --git a/native/crates/dagfs/src/stor.rs b/native/crates/dagfs/src/stor.rs new file mode 100644 index 0000000..e35b191 --- /dev/null +++ b/native/crates/dagfs/src/stor.rs @@ -0,0 +1,1193 @@ +//! Storage rings — the process side of `res/STORAGE_TRANSPORT.md` §4. +//! +//! A session attaches one **ring**: a window it funds from its own mm face, +//! formats and lends as a view ([`RingFs::stor_attach`]). From then on it +//! submits by writing its own window — io_uring records naming a grant, a +//! target and a timeline — and waits on the timeline's fence, which the row +//! stores from its completion path. The pair carries only control: the +//! attach, grants ([`RingFs::stor_grant_pack`]), targets ([`RingFs::stor_target`]), +//! and the one notice a parked row is owed ([`RingFs::stor_publish`]). +//! +//! ```text +//! client window (funded here, mapped by both) +//! Demand SQ ← reads a thread waits on +//! Stream SQ ← read-ahead, bulk +//! CQ → one completion per submission (drained by `StorRing`) +//! fence page → per-timeline {done, fault}, the word a wait sleeps on +//! ``` +//! +//! **Flow control is the client's to keep.** The row pops a submission only +//! while the completion channel has room for its completion, so a ring whose +//! completions are never drained stops. [`StorRing`] drains them on every +//! poll and wait and keeps the first failure, so a caller that works in +//! tickets never has to. + +use core::time::Duration; + +use kore_abi::stor::FenceRecord; +use kore_ring::stor::{FenceCell, StorClient, SubmitError}; +use kore_rt::svc::{ReturnCapability, ReturnLeaseSource}; + +use kore_rt::transport as rt_transport; + +use crate::client::{LentVersion, RingFs, RingFsError}; +use crate::{Fs, FsError, FsRequest, FsResponse, ProviderError}; + +pub use kore_abi::stor::{Class, GrantId, StorStatus}; +/// What every ring call answers when it does not succeed. +pub use crate::Refusal; +pub use kore_ring::stor::{Covering, Op, Request, Ticket, WINDOW_BYTES, covering}; + +/// Pause between kicks once a wait has stopped watching its fence: long +/// enough that a waiting client cannot keep the row's request ring +/// permanently non-empty, short against any command's service time. +const POLL_PAUSE_NS: u64 = 50_000; + +/// How long [`RingFs::stor_detach`] waits out commands still in flight. +pub const DETACH_PATIENCE_NS: u64 = 2_000_000_000; + +/// A filesystem launch's own storage ring (`res/STORAGE_TRANSPORT.md` §9, +/// S5d): the window and read target exec funded and moved in +/// ([`role::STOR_RING`], [`role::STOR_TARGET`]). +/// +/// The ring is attached for one [`read_file`](Fs::read_file) or large +/// [`read_at`](Fs::read_at) and detached before it returns, never left attached: a process can exit at any time, +/// and the kernel tears down a lent window when its lessor dies, under the +/// row's feet (§12). The target is registered once and kept; only its device +/// half is lent to each attachment. A setup that fails, or a read or detach +/// that may leave a command landing late, retires the ring for good, and +/// reads take the pair's window. +/// +/// [`role::STOR_RING`]: kore_abi::task::role::STOR_RING +/// [`role::STOR_TARGET`]: kore_abi::task::role::STOR_TARGET +pub(crate) enum FacadeRing { + /// The founding delivered no ring. + Absent, + /// Delivered, nothing set up yet. + Delivered { + window: kore_rt::cap::LeaseToken, + target: kore_rt::cap::LeaseToken, + }, + /// Window mapped and target registered; no ring attached. + Idle(IdleRing), + /// Unusable. Its custody is kept, never retired. + Broken, +} + +/// A facade ring between reads. Fields drop in declaration order: the +/// target's CPU view before the fabric custody that parents it. +pub(crate) struct IdleRing { + /// This task's mapping of the ring window, `WINDOW_BYTES` long. + window_bytes: core::ptr::NonNull, + /// The target's CPU view, where the ring's reads land. + cpu: core::ptr::NonNull, + target_len: u64, + _cpu_view: kore_rt::cap::MappedLeaseView, + _cpu_scope: kore_rt::cap::FabricRouteToken, + /// Parents each attachment's device child. + route: kore_rt::cap::FabricRouteToken, + _registration: kore_rt::cap::FabricRegistrationToken, + _target_lease: kore_rt::cap::LeaseToken, + window: kore_rt::cap::LeaseToken, +} + +/// A facade ring wait's patience: past the engine's 30 s deadline and +/// abort, so a wedged read reports rather than hangs. +const FACADE_PATIENCE: Duration = Duration::from_secs(40); + +/// How long a facade ring wait sleeps on its fence before also kicking. +const FACADE_WATCH_NS: u64 = 100_000; + +/// Reads smaller than this go through the window: attaching and detaching +/// a ring costs more round trips than the window needs for them. +pub(crate) const FACADE_MIN_BYTES: u64 = 256 * 1024; + +impl FacadeRing { + pub(crate) fn delivered( + window: Option, + target: Option, + ) -> Self { + match (window, target) { + (Some(window), Some(target)) => Self::Delivered { window, target }, + _ => Self::Absent, + } + } + + /// Map the window and register the target, once. `None` leaves nothing + /// to use. + fn set_up( + window: kore_rt::cap::LeaseToken, + target: kore_rt::cap::LeaseToken, + ) -> Result { + use kore_abi::session::{FabricAccess, FabricOp, FabricRouteScopeReq, FabricSync}; + use kore_rt::cap::verbs; + + let mapped = verbs::map_held_rw(window.slot()).ok_or("window map")?; + if mapped.len() < WINDOW_BYTES { + return Err("window size"); + } + let (_, target_len) = verbs::lease_geometry(target.slot()).ok_or("target geometry")?; + // Registered before any alias: one route parents this task's view of + // the target and each attachment's device child. + let registration = + verbs::register_fabric_lease(target.slot()).map_err(|_| "target registration")?; + let route = registration + .open_self_route(target_len, FabricOp::IommuMap, FabricAccess::all(), FabricSync::Fenced) + .map_err(|_| "target route")?; + let cpu_scope = route + .scope(&FabricRouteScopeReq::new(0, target_len, FabricAccess::all())) + .map_err(|_| "target cpu scope")?; + let cpu_view = verbs::map_owned_lease_rw(cpu_scope.slot()).ok_or("target map")?; + Ok(IdleRing { + window_bytes: mapped.cast::(), + cpu: core::ptr::NonNull::new(cpu_view.as_mut_ptr()).ok_or("target map")?, + target_len, + _cpu_view: cpu_view, + _cpu_scope: cpu_scope, + route, + _registration: registration, + _target_lease: target, + window, + }) + } +} + +impl Fs<'_, '_> { + /// Stream bytes `[start, start + limit)` of `file` — clipped at its end — + /// to `consume` through this launch's ring: attach it, pin the file's + /// current version, read the block-aligned middle by DMA into the target a + /// piece at a time, the unaligned head and tail (the partial last block + /// always among them) through the pair's window, and detach. Answers how + /// many bytes were consumed; `Ok(None)` when the ring is not used and + /// nothing was consumed — the caller reads through the window instead. + pub(crate) async fn read_range_by_ring( + &mut self, + file: &crate::View, + start: u64, + limit: u64, + consume: &mut F, + ) -> Result, Refusal> + where + F: FnMut(&[u8]) -> Result<(), Refusal>, + { + use kore_abi::session::{FabricAccess, FabricRouteScopeReq}; + + if limit.min(file.len().saturating_sub(start)) < FACADE_MIN_BYTES { + self.ring_note = "small read"; + return Ok(None); + } + if matches!(self.ring, FacadeRing::Delivered { .. }) { + let FacadeRing::Delivered { window, target } = + core::mem::replace(&mut self.ring, FacadeRing::Broken) + else { + unreachable!("matched above"); + }; + match FacadeRing::set_up(window, target) { + Ok(idle) => self.ring = FacadeRing::Idle(idle), + Err(step) => self.ring_note = step, + } + } + let FacadeRing::Idle(idle) = &self.ring else { + if matches!(self.ring, FacadeRing::Absent) { + self.ring_note = "no ring delivered"; + } + return Ok(None); + }; + // SAFETY: the window's mapping lives in `idle` for the task's life, + // and no other ring borrows it: the last attachment was detached + // (and its `StorRing` consumed) before this one. + let bytes: &'static mut [u8] = + unsafe { core::slice::from_raw_parts_mut(idle.window_bytes.as_ptr(), WINDOW_BYTES) }; + let (cpu, target_len) = (idle.cpu, idle.target_len); + let device = idle + .route + .scope(&FabricRouteScopeReq::new(0, target_len, FabricAccess::all())); + let Ok(device) = device else { + self.ring_note = "target device scope"; + return Ok(None); + }; + let window = &idle.window; + // A session holds one ring: an app that attached its own keeps it. + let mut ring = match self.client.stor_attach(window, bytes).await { + Ok(ring) => ring, + Err(refusal) => { + self.ring_note = refusal_note("attach", &refusal); + return Ok(None); + } + }; + ring.set_patience(Some(FACADE_PATIENCE)); + let target = match self.client.stor_target(device).await { + Ok(target) => target, + Err(refusal) => { + self.ring_note = refusal_note("target", &refusal); + // `Busy`: the row kept a mapping of the target past the + // refusal, so the target may never be lent again. + if matches!(refusal, Refusal::Fs(FsError::Busy)) { + self.break_ring(); + } + if self.client.stor_detach(ring).await.is_err() { + self.break_ring(); + } + return Ok(None); + } + }; + let Ok(version) = self.lend_version(file).await else { + self.ring_note = "lend version"; + if self.client.stor_detach(ring).await.is_err() { + self.break_ring(); + } + return Ok(None); + }; + let end = version.identity().len.min(start.saturating_add(limit)); + let block = u64::from(ring.block_size()).max(1); + // The middle the ring can carry: whole blocks the grant covers, block + // aligned at both ends. Nothing when the range holds no such block. + let middle = match self.client.stor_grant_version(&version).await { + Ok(grant) => { + let from = start.next_multiple_of(block); + let to = (end.min(grant.bytes) / block) * block; + (from < to).then_some((grant, from, to)) + } + // Nothing whole to grant: a sub-block file is all tail. + Err(Refusal::Fs(FsError::Range)) => None, + Err(refusal) => { + self.ring_note = refusal_note("grant", &refusal); + let _ = self.release_version(version).await; + if self.client.stor_detach(ring).await.is_err() { + self.break_ring(); + } + return Ok(None); + } + }; + let (head_end, tail_start) = match middle { + Some((_, from, to)) => (from, to), + None => (end, end), + }; + // Head through the window, then the middle through the ring, then the + // tail through the window: in order, from the same pinned version. + let mut result = self.window_range(&version, start, head_end.max(start).min(end), consume).await; + let mut landed_late = false; + if let Some((grant, from, to)) = middle { + let mut offset = from; + while result.is_ok() && offset < to { + let piece = (to - offset).min(target.bytes); + let read = ReadSpec { + offset, + len: piece, + target, + target_offset: 0, + }; + let landed = match self + .client + .stor_read_batch(&mut ring, Class::Demand, grant, 1, [read], FACADE_WATCH_NS) + .await + { + Ok(Some(ticket)) => self + .client + .stor_wait(&mut ring, ticket, FACADE_WATCH_NS) + .await + .map(|_| ()), + Ok(None) => Ok(()), + Err(refusal) => Err(refusal), + }; + if let Err(refusal) = landed { + // A read that broke off may still land in the target. + landed_late = true; + result = Err(refusal); + break; + } + // SAFETY: the fence covered every command writing these + // bytes, and `piece <= target.bytes`, inside the mapped view. + let bytes = unsafe { core::slice::from_raw_parts(cpu.as_ptr(), piece as usize) }; + result = consume(bytes); + offset += piece; + } + } + if result.is_ok() { + result = self.window_range(&version, tail_start.max(start), end, consume).await; + } + let _ = self.release_version(version).await; + // Detached before returning, so no exit can leave the row reading a + // window whose lessor is gone. A detach that did not settle leaves + // commands that may land late: the ring is never used again. + if self.client.stor_detach(ring).await.is_err() || landed_late { + self.break_ring(); + } + if result.is_ok() { + self.ring_files += 1; + self.ring_note = "streamed"; + } + result.map(|()| Some(end.saturating_sub(start))) + } + + /// Bytes `[from, to)` of `version` to `consume` through the pair's window. + async fn window_range( + &mut self, + version: &LentVersion, + from: u64, + to: u64, + consume: &mut F, + ) -> Result<(), Refusal> + where + F: FnMut(&[u8]) -> Result<(), Refusal>, + { + let mut offset = from; + while offset < to { + let want = (to - offset).min(self.chunk() as u64).min(u64::from(u32::MAX)) as u32; + match self.client.read_version(version, offset, 0, want).await { + Ok(0) => return Err(Refusal::Fs(FsError::Range)), + Ok(count) if count <= want => { + consume(&self.window[..count as usize])?; + offset += u64::from(count); + } + Ok(_) => return Err(Refusal::Pair), + Err(RingFsError::Refused(error)) => return Err(Refusal::Fs(error)), + Err(_) => return Err(Refusal::Pair), + } + } + Ok(()) + } + + /// Why the last whole-file or large read did or did not stream + /// through this launch's ring — a step name, for a witness to print. + pub fn facade_ring_note(&self) -> &'static str { + self.ring_note + } + + /// Retire this launch's facade ring for good, keeping its custody. + fn break_ring(&mut self) { + if let FacadeRing::Idle(idle) = core::mem::replace(&mut self.ring, FacadeRing::Broken) { + core::mem::forget(idle); + } + } +} + +/// A refusal at a ring step, named for [`Fs::facade_ring_note`]. +fn refusal_note(step: &'static str, refusal: &Refusal) -> &'static str { + match (step, refusal) { + ("attach", Refusal::Fs(FsError::AlreadyExists)) => "attach: session has a ring", + ("attach", Refusal::Fs(FsError::OutOfHandles)) => "attach: row out of rings", + ("attach", Refusal::Fs(_)) => "attach refused", + ("attach", _) => "attach: pair failed", + ("target", Refusal::Fs(FsError::Busy)) => "target: mapping outlived refusal", + ("target", Refusal::Fs(_)) => "target refused", + ("target", _) => "target: pair failed", + ("grant", Refusal::Fs(_)) => "grant refused", + _ => "grant: pair failed", + } +} + +/// A grant installed into this session's ring. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RingGrant { + /// The id a submission names. + pub id: GrantId, + /// Bytes the grant covers, from offset zero. + pub bytes: u64, +} + +/// A target adopted into this session's ring. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RingTarget { + /// The slot a submission names. + pub slot: u16, + /// The target's length. + pub bytes: u64, +} + +/// One read of a batch ([`RingFs::stor_read_batch`]): `len` bytes at `offset` +/// of the grant into `target` at `target_offset`. Any length: the batch +/// splits a read longer than [`MAX_SUBMISSION`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ReadSpec { + pub offset: u64, + pub len: u64, + pub target: RingTarget, + pub target_offset: u64, +} + +/// Largest single submission a batch builds (block-aligned for every block +/// size up to 1 GiB); the row splits it further at the device's ceiling. +pub const MAX_SUBMISSION: u64 = 1 << 30; + +/// A timeline's fence, held apart from the ring: all atomics in the +/// window, so any thread may watch it while the ring itself is driven +/// under someone else's lock. +#[derive(Clone, Copy)] +pub struct FenceWatch { + cell: &'static FenceCell, +} + +impl FenceWatch { + /// The fence as it stands. + pub fn load(&self) -> FenceRecord { + self.cell.load() + } + + /// Where the ticket numbered `seq` on this timeline stands. + pub fn state(&self, seq: u64) -> TicketState { + let fence = self.cell.load(); + if fence.fault != 0 && fence.fault <= seq { + TicketState::Faulted + } else if fence.done >= seq { + TicketState::Done + } else { + TicketState::Pending + } + } + + /// Sleep this core on the fence's line until the row stores to it or + /// `cap` passes, when `done` still reads `observed`. The row's store is + /// the wake — `UMWAIT`/`MONITORX` end on a write to the line from any + /// agent — so no wake call crosses the pair. + pub fn sleep_while(&self, observed: u64, cap: Duration) { + let _ = kore_rt::hint::wait_on(self.cell.done_word(), observed as u32, cap); + } +} + +/// What a full channel waits for: the timeline's next sequence to complete +/// — but only one that was issued. Once `done` has caught up with +/// everything issued there is nothing of this timeline's to wait for, and a +/// wait on `done + 1` would never end (the metal hang: a publish's kick let +/// the row drain the channel before the target was chosen). +const fn room_wait_target(done: u64, issued: u64) -> Option { + if done < issued { Some(done + 1) } else { None } +} + +/// Where a ticket stands, as its timeline's fence shows it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TicketState { + /// Not yet complete. + Pending, + /// Complete, and nothing at or before it on its timeline failed. + Done, + /// A submission at or before it on its timeline failed: the bytes it + /// named are unusable. + Faulted, +} + +/// This session's attached ring: the client halves over its own mapping of +/// the window, and the geometry every submission obeys. +pub struct StorRing { + client: StorClient<'static>, + block_size: u32, + max_transfer: u32, + /// The first failed completion drained, with its token. + failure: Option<(u64, StorStatus)>, + /// Completions drained. + completed: u64, + /// How long one wait may last before it gives up and reports. + patience_ns: Option, + /// The last wait that gave up, and what it saw. + stall: Option, +} + +/// The transport engine's counters, every ring together +/// ([`RingFs::stor_inspect`]). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct StorCounters { + pub submissions: u64, + pub completions: u64, + /// Device commands in flight when read. + pub in_flight: u32, + /// Ops reported past their deadline. + pub expired: u32, + /// Device completions naming no live command. + pub stale: u32, + /// Completions a timeline refused — each stops a fence for good. + pub timeline_refusals: u32, + /// Device commands the queue set refused to stage. + pub stage_refusals: u32, + /// The last stage refusal (`BlockError` discriminant plus one; 0 none). + pub last_stage_error: u8, + /// Abort commands sent for expired device commands. + pub aborts: u32, + /// Aborts the controller refused (15 reads "15 or more"). + pub aborts_refused: u8, + /// The controller arbitrates by weighted round robin (`CAP.AMS` offered + /// it); otherwise plain round robin. + pub weighted: bool, + /// Rings attached, the row's own lanes included (7 reads "7 or more"). + pub rings: u8, +} + +/// What a wait saw when its patience ran out ([`StorRing::set_patience`]): +/// enough to tell a submission never popped, from one the device still +/// holds, from one that completed without its fence moving. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StallReport { + /// The ticket waited on. + pub ticket: Ticket, + /// Its timeline's fence. + pub fence: FenceRecord, + /// The row's account of this ring: status, submissions it holds, + /// completions it posted. + pub row: Option, + /// Room left in each class's channel (Demand, Stream, Commit): below + /// the depth means submissions the row has not popped. + pub room: [u32; 3], + /// Completions this client drained. + pub drained: u64, + /// The engine's counters, when the row answered. + pub engine: Option, + /// How long the wait lasted. + pub waited_ns: u64, +} + +impl core::fmt::Display for StallReport { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + f, + "ticket {}#{} after {} ms: fence done {} fault {}; room d/s/c {}/{}/{}; drained {}", + self.ticket.timeline, + self.ticket.seq, + self.waited_ns / 1_000_000, + self.fence.done, + self.fence.fault, + self.room[0], + self.room[1], + self.room[2], + self.drained, + )?; + match self.row { + Some(row) => write!( + f, + "; row status {} holds {} posted {}", + row.status, row.in_flight, row.completed + )?, + None => f.write_str("; row state unreadable")?, + } + match self.engine { + Some(e) => write!( + f, + "; engine popped {} posted {} in flight {} expired {} stale {} refused {} stage-refused {} (last {})", + e.submissions, + e.completions, + e.in_flight, + e.expired, + e.stale, + e.timeline_refusals, + e.stage_refusals, + e.last_stage_error + ), + None => f.write_str("; engine unreadable"), + } + } +} + +impl StorRing { + /// Bound every wait on this ring to `patience`: past it the wait gives + /// up with `Busy` and records a [`StallReport`] ([`take_stall`]). + /// + /// [`take_stall`]: Self::take_stall + pub fn set_patience(&mut self, patience: Option) { + self.patience_ns = patience.map(|patience| patience.as_nanos() as u64); + } + + /// The last wait that gave up, cleared by the read. + pub fn take_stall(&mut self) -> Option { + self.stall.take() + } + + /// Logical block size: every offset and length is a multiple. + pub fn block_size(&self) -> u32 { + self.block_size + } + + /// The row's single-command ceiling. A longer submission is split by + /// the row; this is testimony, not a limit on `len`. + pub fn max_transfer(&self) -> u32 { + self.max_transfer + } + + /// Queue `request` on `class` (visible to the row only after + /// [`RingFs::stor_publish`]). The ticket, when the request names a + /// timeline, is what to wait for. + pub fn submit(&mut self, class: Class, request: &Request) -> Result, SubmitError> { + self.client.submit(class, request) + } + + /// Queue a read of `len` bytes at `offset` of `grant` into `target` at + /// `target_offset`, on `timeline`. + #[allow(clippy::too_many_arguments)] + pub fn read( + &mut self, + class: Class, + grant: RingGrant, + offset: u64, + len: u32, + target: RingTarget, + target_offset: u64, + timeline: u16, + ) -> Result { + let request = Request { + op: Op::Read, + grant: grant.id, + offset, + len, + target: target.slot, + target_offset, + user_data: offset, + timeline, + deadline_ns: 0, + }; + self.client + .submit(class, &request)? + .ok_or(SubmitError::Timeline) + } + + /// Submissions `class`'s channel can still take. + pub fn room(&self, class: Class) -> u32 { + self.client.room(class) + } + + /// Timeline `t`'s fence as it stands. + pub fn fence(&self, t: u16) -> Option { + self.client.fence(t) + } + + /// A handle on timeline `t`'s fence that outlives this borrow. + pub fn watch(&self, t: u16) -> Option { + self.client.fence_cell(t).map(|cell| FenceWatch { cell }) + } + + /// Drain every posted completion, keeping the first failure. Returns + /// how many were drained. + pub fn drain(&mut self) -> u32 { + let mut drained = 0; + while let Some(completion) = self.client.reap() { + drained += 1; + if let Err(status) = completion.result { + self.failure.get_or_insert((completion.user_data, status)); + } + } + self.completed += u64::from(drained); + drained + } + + /// The row's own account of the ring — live or departed, submissions it + /// holds, completions it posted — read without a call. + pub fn row_state(&self) -> Option { + self.client.state() + } + + /// Completions drained since attach. + pub fn completed(&self) -> u64 { + self.completed + } + + /// The first failed completion drained so far — its token (a read's + /// grant offset) and status — cleared by the read. + pub fn take_failure(&mut self) -> Option<(u64, StorStatus)> { + self.failure.take() + } + + /// Where `ticket` stands. Drains completions first, so waiting on + /// tickets alone keeps the ring flowing. + pub fn state(&mut self, ticket: Ticket) -> TicketState { + self.drain(); + match self.client.poll_ticket(ticket) { + Ok(true) => TicketState::Done, + Ok(false) => TicketState::Pending, + Err(_) => TicketState::Faulted, + } + } + + /// Sleep this core on timeline `t`'s fence line until the row stores to + /// it or `cap` passes, when `done` still reads `observed`. The row's + /// store is the wake — `UMWAIT`/`MONITORX` end on a write to the line + /// from any agent — so no wake call crosses the pair. + fn sleep_while(&self, t: u16, observed: u64, cap: Duration) { + if let Some(watch) = self.watch(t) { + watch.sleep_while(observed, cap); + } + } +} + +impl RingFs<'_, '_, T> { + /// One control call on the pair, a service refusal folded into + /// [`Refusal::Fs`] and anything else into [`Refusal::Pair`]. + async fn stor_call(&mut self, request: &FsRequest) -> Result { + match self.call(request).await { + Ok(FsResponse::Refused(error)) => Err(Refusal::Fs(error)), + Ok(response) => Ok(response), + Err(_) => Err(Refusal::Pair), + } + } + + /// Attach `window` — the caller's own mapping of `lease`, page-aligned + /// and at least [`WINDOW_BYTES`] — as this session's ring. The window is + /// formatted here, then lent as a view of `lease`'s first + /// [`WINDOW_BYTES`]. The caller keeps `lease` (and the mapping) for as + /// long as the ring lives: dropping them before + /// [`stor_detach`](RingFs::stor_detach) tears the row's mapping out from + /// under it. + pub async fn stor_attach( + &mut self, + lease: &S, + window: &'static mut [u8], + ) -> Result { + if window.len() < WINDOW_BYTES { + return Err(Refusal::Fs(FsError::Range)); + } + let (window, _) = window.split_at_mut(WINDOW_BYTES); + kore_ring::stor::format(window).map_err(|_| Refusal::Fs(FsError::Range))?; + let (block_size, max_transfer) = + match self.lend_ring_window(lease, WINDOW_BYTES as u32).await { + Ok(FsResponse::StorAttached { + block_size, + max_transfer, + }) => (block_size, max_transfer), + Ok(FsResponse::Refused(error)) | Err(RingFsError::Refused(error)) => { + return Err(Refusal::Fs(error)); + } + _ => return Err(Refusal::Pair), + }; + let ring = kore_ring::stor::client(window).map_err(|_| Refusal::Fs(FsError::Range))?; + Ok(StorRing { + client: StorClient::new(ring), + block_size, + max_transfer, + failure: None, + completed: 0, + patience_ns: None, + stall: None, + }) + } + + /// Install a read grant over the designated pack region into this + /// session's ring. `Unsupported` when the machine designates none. + pub async fn stor_grant_pack(&mut self) -> Result { + match self.stor_call(&FsRequest::StorGrantPack).await? { + FsResponse::StorGranted { grant, bytes } => Ok(RingGrant { + id: GrantId::from_fd(grant as i32).ok_or(Refusal::Pair)?, + bytes, + }), + _ => Err(Refusal::Pair), + } + } + + /// Install a read grant over `version`'s whole blocks into this + /// session's ring. The grant lives as long as the lease: releasing the + /// version is refused `Busy` while a read under it is in flight. The + /// grant covers `bytes`, a whole number of blocks from offset zero; the + /// version's partial last block, if any, is read through the lease. + /// `Range` when the version holds no whole block. + pub async fn stor_grant_version(&mut self, version: &LentVersion) -> Result { + match self + .stor_call(&FsRequest::StorGrantVersion { + lease: version.lease(), + }) + .await? + { + FsResponse::StorGranted { grant, bytes } => Ok(RingGrant { + id: GrantId::from_fd(grant as i32).ok_or(Refusal::Pair)?, + bytes, + }), + _ => Err(Refusal::Pair), + } + } + + /// Lend one device-mapping route over caller-funded memory as a target + /// of this session's ring. The caller keeps its own CPU child of the + /// same route to read what lands. A refusal leaves the row no device + /// mapping of it, except `Busy`: then the route must be held until the + /// ring detaches, as after a release answered `Busy`. + pub async fn stor_target(&mut self, route: C) -> Result { + match self.lend_ring_target(route).await { + Ok(FsResponse::StorTargetAdopted { slot, bytes }) => Ok(RingTarget { slot, bytes }), + Ok(FsResponse::Refused(error)) => Err(Refusal::Fs(error)), + _ => Err(Refusal::Pair), + } + } + + /// Hand `target` back. Success means the row holds no device mapping + /// of it any more: the lent route may retire. `Busy` while a command + /// uses it, or when its mapping outlived the release (then the slot is + /// gone, and the route must be held until the ring detaches). + pub async fn stor_release(&mut self, target: RingTarget) -> Result<(), Refusal> { + self.stor_call(&FsRequest::StorRelease { slot: target.slot }) + .await + .map(|_| ()) + } + + /// Make every queued submission of `class` visible to the row, and send + /// the notice a parked row is owed. While the row is busy the publish + /// costs no call at all. + pub async fn stor_publish(&mut self, ring: &mut StorRing, class: Class) -> Result<(), Refusal> { + if ring.client.publish(class) { + self.stor_call(&FsRequest::StorKick).await?; + } + Ok(()) + } + + /// Queue every read of `reads` on `class` and `timeline`, then publish. + /// Whenever the channel fills, publish and wait out the timeline's next + /// ticket for room — so `class`'s channel must carry only this + /// timeline's submissions while the batch runs. Tickets on one timeline + /// complete in order: the returned last ticket covers the whole batch + /// (`None` for an empty one). + pub async fn stor_read_batch( + &mut self, + ring: &mut StorRing, + class: Class, + grant: RingGrant, + timeline: u16, + reads: impl IntoIterator, + watch_ns: u64, + ) -> Result, Refusal> { + let mut last = None; + for read in reads { + let mut done = 0; + while done < read.len { + let len = (read.len - done).min(MAX_SUBMISSION) as u32; + while ring.room(class) == 0 { + self.stor_publish(ring, class).await?; + // The publish may have kicked the row, and the row may + // have drained the channel before the kick answered: + // look again before choosing what to wait for. + if ring.room(class) != 0 { + break; + } + let done = ring.fence(timeline).map_or(0, |fence| fence.done); + let issued = ring.client.issued(timeline).unwrap_or(0); + match room_wait_target(done, issued) { + Some(seq) => { + self.stor_wait(ring, Ticket { timeline, seq }, watch_ns) + .await?; + } + // Everything this timeline issued has completed, yet + // the channel is full: it holds another timeline's + // submissions. Wake the row and look again, waiting + // on the fence line rather than an executor sleep — + // this runs under `block_on` too. + None => { + self.stor_kick().await?; + ring.sleep_while(timeline, done, Duration::from_nanos(POLL_PAUSE_NS)); + } + } + } + let ticket = ring + .read( + class, + grant, + read.offset + done, + len, + read.target, + read.target_offset + done, + timeline, + ) + .map_err(|_| Refusal::Fs(FsError::Range))?; + last = Some(ticket); + done += u64::from(len); + } + } + self.stor_publish(ring, class).await?; + Ok(last) + } + + /// The transport engine's counters. + pub async fn stor_inspect(&mut self) -> Result { + match self.stor_call(&FsRequest::StorInspect).await? { + FsResponse::StorInspected { + submissions, + completions, + in_flight, + expired, + stale, + timeline_refusals, + stage_refusals, + last_stage_error, + aborts, + aborts_refused, + weighted, + rings, + } => Ok(StorCounters { + submissions, + completions, + in_flight, + expired, + stale, + timeline_refusals, + stage_refusals, + last_stage_error, + aborts, + aborts_refused, + weighted, + rings, + }), + _ => Err(Refusal::Pair), + } + } + + /// Send the notice a parked row is owed, unconditionally — a lost-wake + /// probe: work that completes only after this was never seen. + pub async fn stor_kick(&mut self) -> Result<(), Refusal> { + self.stor_call(&FsRequest::StorKick).await.map(|_| ()) + } + + /// Wait for `ticket`. For the first `watch_ns` this core sleeps on the + /// timeline's fence line, which costs no call; past that each round also + /// kicks the row, so a lost wake costs `watch_ns` once and the wait is + /// never slower than a pair that polls. + /// + /// Answers how many calls the wait made: zero when the fence alone + /// completed it, which is the interrupt path working. + pub async fn stor_wait( + &mut self, + ring: &mut StorRing, + ticket: Ticket, + watch_ns: u64, + ) -> Result { + let mut calls = 0u32; + let started = kore_rt::clock::uptime().map_or(0, |now| now.as_nanos()); + let deadline = kore_rt::clock::uptime() + .ok() + .map(|now| now.as_nanos().saturating_add(watch_ns)); + loop { + if let Some(patience) = ring.patience_ns { + let now = kore_rt::clock::uptime().map_or(0, |now| now.as_nanos()); + if now.saturating_sub(started) >= patience { + let engine = self.stor_inspect().await.ok(); + ring.drain(); + ring.stall = Some(StallReport { + ticket, + fence: ring.fence(ticket.timeline).unwrap_or_default(), + row: ring.row_state(), + room: [ + ring.room(Class::Demand), + ring.room(Class::Stream), + ring.room(Class::Commit), + ], + drained: ring.completed, + engine, + waited_ns: now.saturating_sub(started), + }); + return Err(Refusal::Fs(FsError::Busy)); + } + } + let observed = ring.fence(ticket.timeline).map_or(0, |fence| fence.done); + match ring.state(ticket) { + TicketState::Done => return Ok(calls), + TicketState::Faulted => { + return Err(Refusal::Fs(FsError::ProviderFailure(ProviderError::Lost))); + } + TicketState::Pending => {} + } + let remaining = match (deadline, kore_rt::clock::uptime()) { + (Some(deadline), Ok(now)) => deadline.saturating_sub(now.as_nanos()), + _ => 0, + }; + if remaining != 0 { + ring.sleep_while(ticket.timeline, observed, Duration::from_nanos(remaining)); + continue; + } + calls = calls.saturating_add(1); + if calls > 1 { + // A bounded wait on the fence's own line, not an executor + // sleep: this also runs under `block_on` from a model's read + // callback, where no executor is polling. + ring.sleep_while( + ticket.timeline, + observed, + Duration::from_nanos(POLL_PAUSE_NS), + ); + } + self.stor_call(&FsRequest::StorKick).await?; + } + } + + /// Detach this session's ring: the row stops popping at once, and this + /// returns once nothing it issued is in flight and its targets are + /// released. Only then may the caller unmap the window and retire the + /// targets' routes. + /// + /// Bounded: a device that never completes leaves the ring `Busy` past + /// [`DETACH_PATIENCE_NS`], and the caller must then keep the window + /// mapped and funded — the row drains it when the session closes. + pub async fn stor_detach(&mut self, ring: StorRing) -> Result<(), Refusal> { + drop(ring); + let started = kore_rt::clock::uptime().map_or(0, |now| now.as_nanos()); + loop { + match self.stor_call(&FsRequest::StorDetach).await { + Ok(_) => return Ok(()), + Err(Refusal::Fs(FsError::Busy)) => { + let now = kore_rt::clock::uptime().map_or(u64::MAX, |now| now.as_nanos()); + if now.saturating_sub(started) >= DETACH_PATIENCE_NS { + return Err(Refusal::Fs(FsError::Busy)); + } + kore_rt::sleep_for(POLL_PAUSE_NS).await; + } + Err(other) => return Err(other), + } + } + } +} + +/// The same calls on the facade, over its pair. +impl Fs<'_, '_> { + /// See [`RingFs::stor_attach`]. + pub async fn stor_attach( + &mut self, + lease: &S, + window: &'static mut [u8], + ) -> Result { + self.client.stor_attach(lease, window).await + } + + /// Reads ([`read_file`](Self::read_file), large + /// [`read_at`](Self::read_at)) streamed through this launch's own storage + /// ring — the witness that facade reads take it. + pub fn facade_ring_files(&self) -> u64 { + self.ring_files + } + + /// See [`RingFs::stor_grant_pack`]. + pub async fn stor_grant_pack(&mut self) -> Result { + self.client.stor_grant_pack().await + } + + /// See [`RingFs::stor_grant_version`]. + pub async fn stor_grant_version(&mut self, version: &LentVersion) -> Result { + self.client.stor_grant_version(version).await + } + + /// Pin `file`'s current version and read its identity: the version a + /// ring grant ([`stor_grant_version`](Self::stor_grant_version)) and + /// [`read_version`](Self::read_version) read, unchanged by later writes, + /// until [`release_version`](Self::release_version). + pub async fn lend_version(&mut self, file: &crate::View) -> Result { + let pending = match self.client.lend_version(file.handle(), 0).await { + Ok(pending) => pending, + Err(RingFsError::Refused(error)) => return Err(Refusal::Fs(error)), + Err(_) => return Err(Refusal::Pair), + }; + match pending.identify(self.window, 0) { + Ok(version) => Ok(version), + Err(failure) => { + let _ = self.client.release_pending_version(failure.version).await; + Err(Refusal::Fs(failure.error)) + } + } + } + + /// Read at most one transfer window of `version` at `offset` into `dst` + /// through the pair. `Ok(0)` at or past its end. + pub async fn read_version( + &mut self, + version: &LentVersion, + offset: u64, + dst: &mut [u8], + ) -> Result { + let want = dst.len().min(self.chunk()).min(u32::MAX as usize); + if want == 0 { + return Ok(0); + } + let count = match self.client.read_version(version, offset, 0, want as u32).await { + Ok(count) => count as usize, + Err(RingFsError::Refused(error)) => return Err(Refusal::Fs(error)), + Err(_) => return Err(Refusal::Pair), + }; + if count > want { + return Err(Refusal::Pair); + } + dst[..count].copy_from_slice(&self.window[..count]); + Ok(count) + } + + /// Release a lent version. A version a ring grant still reads through is + /// released once those reads drain; this waits for that, bounded by + /// [`DETACH_PATIENCE_NS`], and hands the version back if it cannot. + pub async fn release_version(&mut self, version: LentVersion) -> Result<(), (LentVersion, Refusal)> { + let started = kore_rt::clock::uptime().map_or(0, |now| now.as_nanos()); + let mut version = version; + loop { + match self.client.release_version(version).await { + Ok(()) => return Ok(()), + Err(failure) => { + version = failure.version; + let now = kore_rt::clock::uptime().map_or(u64::MAX, |now| now.as_nanos()); + match failure.error { + RingFsError::Refused(FsError::Busy) + if now.saturating_sub(started) < DETACH_PATIENCE_NS => + { + kore_rt::sleep_for(POLL_PAUSE_NS).await; + } + RingFsError::Refused(error) => return Err((version, Refusal::Fs(error))), + _ => return Err((version, Refusal::Pair)), + } + } + } + } + } + + /// See [`RingFs::stor_target`]. + pub async fn stor_target(&mut self, route: C) -> Result { + self.client.stor_target(route).await + } + + /// See [`RingFs::stor_release`]. + pub async fn stor_release(&mut self, target: RingTarget) -> Result<(), Refusal> { + self.client.stor_release(target).await + } + + /// See [`RingFs::stor_publish`]. + pub async fn stor_publish(&mut self, ring: &mut StorRing, class: Class) -> Result<(), Refusal> { + self.client.stor_publish(ring, class).await + } + + /// See [`RingFs::stor_read_batch`]. + pub async fn stor_read_batch( + &mut self, + ring: &mut StorRing, + class: Class, + grant: RingGrant, + timeline: u16, + reads: impl IntoIterator, + watch_ns: u64, + ) -> Result, Refusal> { + self.client + .stor_read_batch(ring, class, grant, timeline, reads, watch_ns) + .await + } + + /// See [`RingFs::stor_kick`]. + pub async fn stor_kick(&mut self) -> Result<(), Refusal> { + self.client.stor_kick().await + } + + /// See [`RingFs::stor_wait`]. + pub async fn stor_wait( + &mut self, + ring: &mut StorRing, + ticket: Ticket, + watch_ns: u64, + ) -> Result { + self.client.stor_wait(ring, ticket, watch_ns).await + } + + /// See [`RingFs::stor_detach`]. + pub async fn stor_detach(&mut self, ring: StorRing) -> Result<(), Refusal> { + self.client.stor_detach(ring).await + } + + /// See [`RingFs::stor_inspect`]. + pub async fn stor_inspect(&mut self) -> Result { + self.client.stor_inspect().await + } +} + +#[cfg(test)] +mod tests { + use super::room_wait_target; + + /// A full channel waits for the next issued sequence, never for one + /// past the last issued — the ticket that hung the decode on metal. + #[test] + fn a_full_channel_never_waits_past_the_last_issued_sequence() { + assert_eq!(room_wait_target(997_830, 997_840), Some(997_831)); + assert_eq!(room_wait_target(997_839, 997_840), Some(997_840)); + assert_eq!(room_wait_target(997_840, 997_840), None, "all done: nothing to wait for"); + assert_eq!(room_wait_target(0, 0), None); + } +} diff --git a/native/crates/dagfs/src/walk.rs b/native/crates/dagfs/src/walk.rs index 26d667b..6660d60 100644 --- a/native/crates/dagfs/src/walk.rs +++ b/native/crates/dagfs/src/walk.rs @@ -9,8 +9,9 @@ //! needs a live pair. use crate::{ - Fs, FsError, FsRequest, FsResponse, GrantId, Handle, Name, ObjectKind, ObjectSubtype, Refusal, - Revision, Rights, StableObjectId, Stat, bounds::MAX_PATH_DEPTH, path, read_grant_entry, + Fs, FsError, FsRequest, FsResponse, GrantEntry, GrantId, Handle, Name, ObjectKind, + ObjectSubtype, Refusal, Revision, Rights, StableObjectId, Stat, bounds::MAX_PATH_DEPTH, path, + read_grant_entry, }; /// The three masks a verb walks with. @@ -282,6 +283,14 @@ impl<'w, 't> Fs<'w, 't> { /// Find a named authority edge in this task's view. pub async fn named_grant(&mut self, name: &Name) -> Result, Refusal> { + Ok(self.named_grant_entry(name).await?.map(|entry| entry.grant)) + } + + /// [`named_grant`](Self::named_grant) with the edge's listed rights, for + /// a caller whose decision depends on what the edge permits — not only + /// on its name. Never a containment child of the root: only an edge + /// granted to this session answers. + pub async fn named_grant_entry(&mut self, name: &Name) -> Result, Refusal> { let FsResponse::GrantsListed { entries } = self .checked(&FsRequest::ListGrants { window_offset: 0 }) .await? @@ -291,7 +300,7 @@ impl<'w, 't> Fs<'w, 't> { for index in 0..entries as usize { let entry = read_grant_entry(&self.window[..], 0, index).map_err(Refusal::Fs)?; if !entry.is_private_root && entry.display_name.as_ref() == Some(name) { - return Ok(Some(entry.grant)); + return Ok(Some(entry)); } } Ok(None) diff --git a/native/crates/dagfs/src/wire.rs b/native/crates/dagfs/src/wire.rs index f7966cc..2ba46f8 100644 --- a/native/crates/dagfs/src/wire.rs +++ b/native/crates/dagfs/src/wire.rs @@ -571,6 +571,34 @@ pub fn encode_req(req: &FsRequest) -> FsReqWire { FsRequest::ClosePeer => { w.op = FsReqWire::OP_CLOSE_PEER; } + FsRequest::StorAttach { announced } => { + w.op = FsReqWire::OP_STOR_ATTACH; + w.window_offset = U32::new(announced.index()); + } + FsRequest::StorGrantPack => { + w.op = FsReqWire::OP_STOR_GRANT_PACK; + } + FsRequest::StorTarget { announced } => { + w.op = FsReqWire::OP_STOR_TARGET; + w.window_offset = U32::new(announced.index()); + } + FsRequest::StorRelease { slot } => { + w.op = FsReqWire::OP_STOR_RELEASE; + w.handle = U64::new(u64::from(slot)); + } + FsRequest::StorKick => { + w.op = FsReqWire::OP_STOR_KICK; + } + FsRequest::StorDetach => { + w.op = FsReqWire::OP_STOR_DETACH; + } + FsRequest::StorGrantVersion { lease } => { + w.op = FsReqWire::OP_STOR_GRANT_VERSION; + w.handle = U64::new(lease.to_bits()); + } + FsRequest::StorInspect => { + w.op = FsReqWire::OP_STOR_INSPECT; + } } w } @@ -626,10 +654,11 @@ pub fn decode_req(w: &FsReqWire) -> Option { }; // Likewise `kind` (Create/AdmitPeer/Search and the two boolean controls, - // whose whole payload rides there) and `session` (Grant only). A verb missing - // from this list cannot carry `kind` at all: its own arm never runs, the - // record decodes to nothing, and the serve loop drops it without a reply - // — which the caller cannot distinguish from a service that hangs. + // whose payload rides there) and `session` (Grant + // only). A verb missing from this list cannot carry `kind` at all: its own + // arm never runs, the record decodes to nothing, and the serve loop drops + // it without a reply — which the caller cannot distinguish from a service + // that hangs. if !matches!( w.op, FsReqWire::OP_CREATE @@ -959,6 +988,47 @@ pub fn decode_req(w: &FsReqWire) -> Option { } FsRequest::TakeAdmission } + FsReqWire::OP_STOR_ATTACH | FsReqWire::OP_STOR_TARGET => { + if !control_payload_is_empty(w) { + return None; + } + let announced = SlotRef::from_wire(w.window_offset.get()); + if w.op == FsReqWire::OP_STOR_ATTACH { + FsRequest::StorAttach { announced } + } else { + FsRequest::StorTarget { announced } + } + } + FsReqWire::OP_STOR_GRANT_PACK + | FsReqWire::OP_STOR_KICK + | FsReqWire::OP_STOR_DETACH + | FsReqWire::OP_STOR_INSPECT => { + if w.window_offset.get() != 0 || !control_payload_is_empty(w) { + return None; + } + match w.op { + FsReqWire::OP_STOR_GRANT_PACK => FsRequest::StorGrantPack, + FsReqWire::OP_STOR_KICK => FsRequest::StorKick, + FsReqWire::OP_STOR_INSPECT => FsRequest::StorInspect, + _ => FsRequest::StorDetach, + } + } + FsReqWire::OP_STOR_GRANT_VERSION => { + if w.window_offset.get() != 0 || !control_payload_except_handle_is_empty(w) { + return None; + } + FsRequest::StorGrantVersion { + lease: crate::VersionLease::from_bits(w.handle.get()), + } + } + FsReqWire::OP_STOR_RELEASE => { + if w.window_offset.get() != 0 || !control_payload_except_handle_is_empty(w) { + return None; + } + FsRequest::StorRelease { + slot: u16::try_from(w.handle.get()).ok()?, + } + } FsReqWire::OP_RETIRE_PEER | FsReqWire::OP_CONFIRM_RETIRED => { if w.kind != 0 || w.window_offset.get() != 0 @@ -1027,13 +1097,70 @@ fn status_of(err: FsError) -> u8 { } } -/// The refusal for a wire status. +/// A provider failure's cause, as the refusal's `kind` byte carries it. +/// Frozen: append, never renumber. Zero is no cause, from an encoder that +/// predates the byte, and decodes as [`ProviderError::Media`]. +fn provider_code(error: ProviderError) -> u8 { + match error { + ProviderError::Exhausted => 1, + ProviderError::PlacementsFull => 2, + ProviderError::UnknownKey => 3, + ProviderError::Range => 4, + ProviderError::TooLarge => 5, + ProviderError::Media => 6, + ProviderError::Transport => 7, + ProviderError::Integrity => 8, + ProviderError::Cancelled => 9, + ProviderError::Revoked => 10, + ProviderError::Lost => 11, + ProviderError::Draining => 12, + ProviderError::ReadOnly => 13, + ProviderError::Backpressure => 14, + ProviderError::Timeout => 15, + ProviderError::Retiring => 16, + } +} + +/// [`provider_code`]'s inverse; `None` for a byte no encoder writes. +fn provider_of(code: u8) -> Option { + Some(match code { + 0 | 6 => ProviderError::Media, + 1 => ProviderError::Exhausted, + 2 => ProviderError::PlacementsFull, + 3 => ProviderError::UnknownKey, + 4 => ProviderError::Range, + 5 => ProviderError::TooLarge, + 7 => ProviderError::Transport, + 8 => ProviderError::Integrity, + 9 => ProviderError::Cancelled, + 10 => ProviderError::Revoked, + 11 => ProviderError::Lost, + 12 => ProviderError::Draining, + 13 => ProviderError::ReadOnly, + 14 => ProviderError::Backpressure, + 15 => ProviderError::Timeout, + 16 => ProviderError::Retiring, + _ => return None, + }) +} + +/// The refusal for a wire status and its `kind` byte. /// -/// `PROVIDER_FAILURE` lands on [`ProviderError::Media`]: the provider's own -/// distinction between medium, range, and unknown-key does not survive the -/// boundary, and inventing one back would be a lie. What the client needs to -/// know — the provider failed and the request had no visible effect — does -/// survive. +/// Only `PROVIDER_FAILURE` carries a `kind`: its cause ([`provider_code`]), +/// so a client sees backpressure or a pending retirement as what it is, not +/// as a failed medium. Every other refusal's `kind` is zero. +fn refusal_of(status: u8, kind: u8) -> Option { + if status == FsRepWire::STATUS_PROVIDER_FAILURE { + return provider_of(kind).map(FsError::ProviderFailure); + } + if kind != 0 { + return None; + } + error_of(status) +} + +/// The refusal for a wire status. `PROVIDER_FAILURE` alone lands on +/// [`ProviderError::Media`]; [`refusal_of`] reads its cause. fn error_of(status: u8) -> Option { Some(match status { FsRepWire::STATUS_NOT_FOUND => FsError::NotFound, @@ -1211,8 +1338,60 @@ pub fn encode_rep(rep: &FsResponse, grants: u8) -> FsRepWire { w.status = FsRepWire::STATUS_OK; w.handle = U64::new(u64::from(peer.arrival())); } + FsResponse::StorAttached { + block_size, + max_transfer, + } => { + w.status = FsRepWire::STATUS_OK; + w.count = U32::new(block_size); + w.cursor = U32::new(max_transfer); + } + FsResponse::StorGranted { grant, bytes } => { + w.status = FsRepWire::STATUS_OK; + w.handle = U64::new(u64::from(grant)); + w.len = U64::new(bytes); + } + FsResponse::StorTargetAdopted { slot, bytes } => { + w.status = FsRepWire::STATUS_OK; + w.handle = U64::new(u64::from(slot)); + w.len = U64::new(bytes); + } + FsResponse::StorInspected { + submissions, + completions, + in_flight, + expired, + stale, + timeline_refusals, + stage_refusals, + last_stage_error, + aborts, + aborts_refused, + weighted, + rings, + } => { + w.status = FsRepWire::STATUS_OK; + // `reserved1`: stage refusals, the last stage error, and aborts in + // the top 24 bits. `kind`: weighted round robin, rings, and the + // refused aborts — the only other bytes a non-object reply may + // carry. + w.reserved1 = U64::new( + u64::from(stage_refusals) + | u64::from(last_stage_error) << 32 + | u64::from(aborts.min(0xFF_FFFF)) << 40, + ); + w.kind = u8::from(weighted) | rings.min(7) << 1 | aborts_refused.min(15) << 4; + w.handle = U64::new(submissions); + w.len = U64::new(completions); + w.count = U32::new(in_flight); + w.cursor = U32::new(expired); + w.revision = U64::new(u64::from(stale) | u64::from(timeline_refusals) << 32); + } FsResponse::Refused(err) => { w.status = status_of(err); + if let FsError::ProviderFailure(cause) = err { + w.kind = provider_code(cause); + } } } w @@ -1252,10 +1431,10 @@ pub fn decode_rep_for(request: &FsRequest, w: &FsRepWire) -> Option<(FsResponse, return None; } if w.status != FsRepWire::STATUS_OK { - // A refusal carries a status and nothing else; a refusal with payload - // is a reply this service did not write. - if w.kind != 0 - || w.flags != 0 + // A refusal carries a status, a provider failure its cause in `kind`, + // and nothing else; a refusal with payload is a reply this service + // did not write. + if w.flags != 0 || w.count.get() != 0 || w.handle.get() != 0 || w.len.get() != 0 @@ -1265,7 +1444,7 @@ pub fn decode_rep_for(request: &FsRequest, w: &FsRepWire) -> Option<(FsResponse, { return None; } - return Some((FsResponse::Refused(error_of(w.status)?), 0)); + return Some((FsResponse::Refused(refusal_of(w.status, w.kind)?), 0)); } // Only a listing may set flags, and `FLAG_MORE` is the only defined bit. @@ -1387,6 +1566,32 @@ pub fn decode_rep_for(request: &FsRequest, w: &FsRepWire) -> Option<(FsResponse, FsRequest::AttachWindow { .. } => FsResponse::WindowAttached { max_transfer: w.count.get(), }, + FsRequest::StorAttach { .. } => FsResponse::StorAttached { + block_size: w.count.get(), + max_transfer: w.cursor.get(), + }, + FsRequest::StorGrantPack | FsRequest::StorGrantVersion { .. } => FsResponse::StorGranted { + grant: u32::try_from(w.handle.get()).ok()?, + bytes: w.len.get(), + }, + FsRequest::StorTarget { .. } => FsResponse::StorTargetAdopted { + slot: u16::try_from(w.handle.get()).ok()?, + bytes: w.len.get(), + }, + FsRequest::StorInspect => FsResponse::StorInspected { + submissions: w.handle.get(), + completions: w.len.get(), + in_flight: w.count.get(), + expired: w.cursor.get(), + stale: w.revision.get() as u32, + timeline_refusals: (w.revision.get() >> 32) as u32, + stage_refusals: w.reserved1.get() as u32, + last_stage_error: (w.reserved1.get() >> 32) as u8, + aborts: (w.reserved1.get() >> 40) as u32, + aborts_refused: w.kind >> 4, + weighted: w.kind & 1 != 0, + rings: (w.kind >> 1) & 7, + }, FsRequest::RegisterSystemView { .. } | FsRequest::RegisterPrivateView { .. } | FsRequest::ApplyView { .. } @@ -1408,6 +1613,9 @@ pub fn decode_rep_for(request: &FsRequest, w: &FsRepWire) -> Option<(FsResponse, | FsRequest::AdmitPeer { .. } | FsRequest::RetirePeer { .. } | FsRequest::ConfirmRetired { .. } + | FsRequest::StorRelease { .. } + | FsRequest::StorKick + | FsRequest::StorDetach | FsRequest::ClosePeer => FsResponse::Done, }; if !matches!( @@ -1418,6 +1626,7 @@ pub fn decode_rep_for(request: &FsRequest, w: &FsRepWire) -> Option<(FsResponse, | FsRequest::Create { .. } | FsRequest::Stat { .. } | FsRequest::Search { .. } + | FsRequest::StorInspect ) && w.reserved1.get() != 0 { return None; @@ -1441,8 +1650,7 @@ pub fn decode_rep_unambiguous(w: &FsRepWire) -> Option<(FsResponse, u8)> { return None; } if w.status != FsRepWire::STATUS_OK { - if w.kind != 0 - || w.flags != 0 + if w.flags != 0 || w.count.get() != 0 || w.handle.get() != 0 || w.len.get() != 0 @@ -1451,7 +1659,7 @@ pub fn decode_rep_unambiguous(w: &FsRepWire) -> Option<(FsResponse, u8)> { { return None; } - return Some((FsResponse::Refused(error_of(w.status)?), 0)); + return Some((FsResponse::Refused(refusal_of(w.status, w.kind)?), 0)); } // A payload-free success is `Done` and can be nothing else. if w.kind == 0 @@ -1513,6 +1721,12 @@ pub fn reply_matches(request: &FsRequest, response: &FsResponse) -> bool { FsRequest::Search { .. } => matches!(response, FsResponse::Searched { .. }), FsRequest::TakeAdmission => matches!(response, FsResponse::PeerAdmitted { .. }), FsRequest::AttachWindow { .. } => matches!(response, FsResponse::WindowAttached { .. }), + FsRequest::StorAttach { .. } => matches!(response, FsResponse::StorAttached { .. }), + FsRequest::StorGrantPack | FsRequest::StorGrantVersion { .. } => { + matches!(response, FsResponse::StorGranted { .. }) + } + FsRequest::StorTarget { .. } => matches!(response, FsResponse::StorTargetAdopted { .. }), + FsRequest::StorInspect => matches!(response, FsResponse::StorInspected { .. }), FsRequest::RegisterSystemView { .. } | FsRequest::RegisterPrivateView { .. } | FsRequest::ApplyView { .. } @@ -1534,6 +1748,9 @@ pub fn reply_matches(request: &FsRequest, response: &FsResponse) -> bool { | FsRequest::AdmitPeer { .. } | FsRequest::RetirePeer { .. } | FsRequest::ConfirmRetired { .. } + | FsRequest::StorRelease { .. } + | FsRequest::StorKick + | FsRequest::StorDetach | FsRequest::ClosePeer => matches!(response, FsResponse::Done), } } @@ -1543,16 +1760,112 @@ mod tests { use super::*; use zerocopy::IntoBytes as _; - /// The control verbs that carry no object must survive the round trip. - /// - /// `DeferCommits` is the reason this module exists: the service's own - /// tests call `dispatch` with an `FsRequest` value and never cross the - /// codec, so a request that encodes but does not decode passes every - /// host test and then hangs the caller on the metal — the serve loop - /// drops an unparseable record without replying, which the client - /// cannot tell from a service that simply never answers. - /// `Create` is the one request whose reserved byte carries data, and - /// only there: every other verb still refuses a non-zero byte. + // The control verbs that carry no object must survive the round trip. + // + // `DeferCommits` is the reason this module exists: the service's own + // tests call `dispatch` with an `FsRequest` value and never cross the + // codec, so a request that encodes but does not decode passes every + // host test and then hangs the caller on the metal — the serve loop + // drops an unparseable record without replying, which the client + // cannot tell from a service that simply never answers. + // `Create` is the one request whose reserved byte carries data, and + // only there: every other verb still refuses a non-zero byte. + + /// The ring verbs carry control only: each round-trips, and a stray + /// operand — a handle on a kick, a slot past `u16` — is refused. + #[test] + fn stor_verbs_round_trip_and_refuse_stray_operands() { + let cases = [ + ( + FsRequest::StorAttach { + announced: SlotRef::from_wire(5), + }, + FsResponse::StorAttached { + block_size: 512, + max_transfer: 256 << 10, + }, + ), + ( + FsRequest::StorGrantPack, + FsResponse::StorGranted { + grant: 0x0003_0000, + bytes: 24 << 30, + }, + ), + ( + FsRequest::StorTarget { + announced: SlotRef::from_wire(6), + }, + FsResponse::StorTargetAdopted { + slot: 17, + bytes: 1 << 30, + }, + ), + (FsRequest::StorRelease { slot: 17 }, FsResponse::Done), + (FsRequest::StorKick, FsResponse::Done), + ( + FsRequest::StorGrantVersion { + lease: crate::VersionLease::from_bits(1 << 63 | 9), + }, + FsResponse::StorGranted { + grant: 0x0001_0001, + bytes: 3 << 20, + }, + ), + (FsRequest::StorDetach, FsResponse::Done), + ( + FsRequest::StorInspect, + FsResponse::StorInspected { + submissions: 1 << 40, + completions: (1 << 40) - 3, + in_flight: 3, + expired: 1, + stale: 2, + timeline_refusals: 0, + stage_refusals: 70_000, + last_stage_error: 6, + aborts: 0xAB_CDEF, + aborts_refused: 15, + weighted: true, + rings: 6, + }, + ), + ]; + for (req, reply) in cases { + let wire = encode_req(&req); + assert_eq!(decode_req(&wire), Some(req)); + assert!(reply_matches(&req, &reply)); + assert_eq!( + decode_rep_for(&req, &encode_rep(&reply, 0)), + Some((reply, 0)) + ); + assert!(!reply_matches(&req, &FsResponse::Transferred { count: 1 })); + } + let mut stray = encode_req(&FsRequest::StorKick); + stray.handle = U64::new(1); + assert_eq!(decode_req(&stray), None); + let mut stray = encode_req(&FsRequest::StorRelease { slot: 1 }); + stray.handle = U64::new(1 << 16); + assert_eq!(decode_req(&stray), None); + let mut stray = encode_req(&FsRequest::StorGrantPack); + stray.window_offset = U32::new(1); + assert_eq!(decode_req(&stray), None); + let mut wide = encode_rep(&FsResponse::StorGranted { grant: 1, bytes: 1 }, 0); + wide.handle = U64::new(1 << 32); + assert_eq!(decode_rep_for(&FsRequest::StorGrantPack, &wide), None); + } + + /// The pack lane's retired op codes decode to nothing: a peer still + /// speaking them is refused, never served. + #[test] + fn retired_pack_ops_are_refused() { + for op in FsReqWire::OP_PACK_INFO..=FsReqWire::OP_PACK_KICK { + let mut wire = encode_req(&FsRequest::StorKick); + wire.op = op; + assert_eq!(decode_req(&wire), None, "op {op}"); + } + } + #[test] fn atomic_replacement_wire_round_trips() { let handle = Handle::from_bits(0x1234); @@ -1795,9 +2108,8 @@ mod tests { ); } - // The one deliberately lossy mapping, asserted rather than assumed: - // the provider's own cause does not cross the boundary, and inventing - // one back would be a lie (see `error_of`). + // The status alone cannot name a provider's cause; `kind` does + // (`provider_failures_keep_their_cause_across_the_wire`). let status = status_of(FsError::ProviderFailure(ProviderError::Transport)); assert_eq!(status, FsRepWire::STATUS_PROVIDER_FAILURE); assert_eq!( @@ -1806,6 +2118,80 @@ mod tests { ); } + /// Every provider failure reaches the client with its cause, so + /// backpressure or a pending retirement no longer reads as a failed + /// medium (`res/LIFECYCLE_HARDENING.md` §1). The match is exhaustive: a + /// new cause stops compiling until it is named here and given a code. + #[test] + fn provider_failures_keep_their_cause_across_the_wire() { + let witness = |cause: ProviderError| match cause { + ProviderError::Exhausted + | ProviderError::PlacementsFull + | ProviderError::UnknownKey + | ProviderError::Range + | ProviderError::TooLarge + | ProviderError::Media + | ProviderError::Transport + | ProviderError::Integrity + | ProviderError::Cancelled + | ProviderError::Revoked + | ProviderError::Lost + | ProviderError::Draining + | ProviderError::ReadOnly + | ProviderError::Backpressure + | ProviderError::Timeout + | ProviderError::Retiring => cause, + }; + let target = FsRequest::StorTarget { + announced: SlotRef::from_wire(4), + }; + let mut codes = [false; 256]; + for cause in [ + ProviderError::Exhausted, + ProviderError::PlacementsFull, + ProviderError::UnknownKey, + ProviderError::Range, + ProviderError::TooLarge, + ProviderError::Media, + ProviderError::Transport, + ProviderError::Integrity, + ProviderError::Cancelled, + ProviderError::Revoked, + ProviderError::Lost, + ProviderError::Draining, + ProviderError::ReadOnly, + ProviderError::Backpressure, + ProviderError::Timeout, + ProviderError::Retiring, + ] { + let refused = FsResponse::Refused(FsError::ProviderFailure(witness(cause))); + let wire = encode_rep(&refused, 0); + let code = usize::from(wire.kind); + assert!(code != 0 && !codes[code], "{cause:?} needs its own code"); + codes[code] = true; + assert_eq!(decode_rep_for(&target, &wire), Some((refused, 0))); + assert_eq!(decode_rep_unambiguous(&wire), Some((refused, 0))); + } + // An encoder that predates the byte still decodes, as before. + let mut bare = encode_rep(&FsResponse::Refused(FsError::ProviderFailure(ProviderError::Lost)), 0); + bare.kind = 0; + assert_eq!( + decode_rep_for(&target, &bare), + Some(( + FsResponse::Refused(FsError::ProviderFailure(ProviderError::Media)), + 0 + )) + ); + // A cause no encoder writes, or a `kind` on any other refusal, is not + // a reply this service wrote. + bare.kind = 0xff; + assert_eq!(decode_rep_for(&target, &bare), None); + let mut other = encode_rep(&FsResponse::Refused(FsError::Busy), 0); + other.kind = 14; + assert_eq!(decode_rep_for(&target, &other), None); + assert_eq!(decode_rep_unambiguous(&other), None); + } + /// A full extent table must not reach a client as "the volume is full". #[test] fn a_full_extent_table_keeps_its_own_status() { diff --git a/native/crates/driver/Cargo.toml b/native/crates/driver/Cargo.toml index 64c5940..4935d12 100644 --- a/native/crates/driver/Cargo.toml +++ b/native/crates/driver/Cargo.toml @@ -10,7 +10,7 @@ publish = false # generic over it. Production userland, kernel-internal first-light, and # cfg(test) hardware doubles are distinct compositions over that seam. [dependencies] -virtio-accel-core = { version = "=0.3.10", default-features = false, optional = true } +virtio-accel-core = { version = "=0.4.1", default-features = false, optional = true } kore-abi = { path = "../abi" } # The one memory fill the DMA-pool scrub-once uses (streaming stores, fenced). kore-scrub = { path = "../scrub" } diff --git a/native/crates/driver/src/accel.rs b/native/crates/driver/src/accel.rs index 83c0bcd..7fb2e9d 100644 --- a/native/crates/driver/src/accel.rs +++ b/native/crates/driver/src/accel.rs @@ -5,6 +5,71 @@ pub use virtio_accel_core::*; +/// Opaque, provider-defined evidence for a sealed device epoch. +/// The host forwards both fields through the existing External refusal without +/// interpreting or masking the code. An i64 carries all 64 diagnostic bits; +/// signedness is part of the provider's layout. This is data, not authority. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LossEvidence { + pub domain: u32, + pub code: i64, +} + +/// One member of a provider-ordered execution plan. +/// +/// Programs remain independent lifecycle objects. The event returned for the +/// complete plan retains every program and bound buffer until retirement. +/// Backends that can encode an ordered hardware command list execute the slice +/// without a host completion round trip between members. +/// +/// Each member carries its own [`LaunchShape`], admitted exactly as a single +/// [`NativeAccelerator::submit_with_launch`] of that program would admit it, +/// and one ordering edge to the members before it: +/// +/// * `barrier == true` — the member starts only after every earlier member's +/// memory writes are complete and visible to it (a data dependency); +/// * `barrier == false` — the client declares the member independent of the +/// members since the most recent barrier, so a backend may overlap it with +/// them. It still never starts ahead of a barrier that precedes it. +/// +/// The first member's flag is ignored: nothing in the plan precedes it. A +/// backend that executes every member strictly in order honors both values; +/// the flag grants permission to overlap, never an obligation to. +pub struct OrderedExecution<'a, P, B> { + pub program: &'a P, + pub bindings: &'a [BindingRef<'a, B>], + pub launch: LaunchShape, + pub barrier: bool, + /// Before the member, wait until a host-published word reaches a + /// value (a gate, [`SyncRef`]). A backend with no command-stream wait + /// refuses a list that names one. + pub gate: Option>, + /// Once the member retires and its writes are visible to the host, + /// write a value to a word (a signal, [`SyncRef`]). A backend with no + /// such completion write refuses a list that names one. + pub signal: Option>, +} + +/// A gate's or signal's word: the 8-byte-aligned quadword at `offset` in a +/// buffer the queue's context holds (`res/QWEN_EXPERT_STREAMING.md` §3-4). +/// A gate holds its member until the word's low dword is at least `value` +/// (unsigned); a signal writes `value`, zero-extended, after its member's +/// writes are visible to the host. The host and the device meet on the +/// word alone: nothing else of the buffer is bound. +pub struct SyncRef<'a, B> { + pub buffer: &'a B, + pub offset: u64, + pub value: u64, +} + +impl Clone for SyncRef<'_, B> { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for SyncRef<'_, B> {} + /// One accelerator's DMA demand, stated per frame class /// (`res/ACCEL_MEMORY_SCALE.md` §2.2). /// @@ -220,13 +285,33 @@ pub trait NativeAccelerator: Accelerator { Ok(()) } - /// The packed device-plane witness of a reported `DeviceLost` (the - /// provider's own layout; `0` when none was retained). A sealed epoch's - /// refusals may carry it so a serial-less client can name the fault - /// instead of only the loss. Providers with no device witness keep the - /// default. - fn loss_evidence(&self) -> u64 { - 0 + /// The first retained device-plane witness of a reported DeviceLost. + /// None means no witness; Some with a zero code is valid evidence. + /// Providers own the domain and bit layout; hosts must preserve both. + fn loss_evidence(&self) -> Option { + None + } + + /// Format retained failure metadata before the host hands off the lost + /// epoch. Must not allocate, touch registers, wait, or submit device work. + /// Device-domain addresses are permitted; host physical addresses and + /// buffer contents are not. Truncation is controlled by the host's writer. + fn failure_report(&self, _out: &mut dyn core::fmt::Write) {} + + /// Bounded operational telemetry for a diagnostic reader — clock, device + /// time, completion wakes — formatted from retained counters: no MMIO, + /// waits or DMA access. The default reports nothing. + fn telemetry_report(&self, _out: &mut dyn core::fmt::Write) {} + + /// Configure optional execution timing for one owned, quiescent context. + /// Enabling resets its measurement interval; disabling retains the result. + /// This must not change submission, completion or visibility semantics. + fn configure_profiling( + &self, + _context: &Self::Context, + _enabled: bool, + ) -> Result<(), BackendError> { + Err(BackendError::Unsupported) } /// Copy the newest bytes of the device firmware's own log into `out`, @@ -270,6 +355,25 @@ pub trait NativeAccelerator: Accelerator { LaunchShape::Workgroups(_) => Err(SubmitFailure::Rejected(BackendError::Unsupported)), } } + + /// Submit a non-empty, dependency-ordered program list as one event. + /// + /// The default rejects the native extension. A backend accepting it must + /// preserve slice order, honor every member's + /// [`barrier`](OrderedExecution::barrier), and signal completion only + /// after every member is complete. A member whose launch shape the + /// backend cannot honor rejects the whole list. Failure is atomic with + /// respect to acceptance: a rejected call publishes no member, while an + /// indeterminate call returns the one event retaining the entire list. + fn submit_ordered( + &self, + queue: &Self::Queue, + executions: &[OrderedExecution<'_, Self::Program, Self::Buffer>], + timeout: Timeout, + ) -> Result> { + let _ = (queue, executions, timeout); + Err(SubmitFailure::Rejected(BackendError::Unsupported)) + } } /// Explicitly host an upstream graph provider with artifact-defined launches. diff --git a/native/crates/driver/src/block/memory.rs b/native/crates/driver/src/block/memory.rs index 8f98dfe..7759297 100644 --- a/native/crates/driver/src/block/memory.rs +++ b/native/crates/driver/src/block/memory.rs @@ -39,6 +39,10 @@ pub struct MemoryBlockDevice hold_for: Option, fault: Option, fault_after: usize, + /// Refuse every abort — a controller that will not give a command up. + refuse_aborts: bool, + /// Resets taken. + resets: u32, flushes: u64, submissions: u64, publishes: u64, @@ -83,6 +87,8 @@ impl hold_for: None, fault: None, fault_after: 0, + refuse_aborts: false, + resets: 0, flushes: 0, submissions: 0, publishes: 0, @@ -125,6 +131,16 @@ impl self.fault = Some(error); self.fault_after = successful_submissions; } + /// Refuse (or honour again) every abort. + pub fn refuse_aborts(&mut self, refuse: bool) { + self.refuse_aborts = refuse; + } + + /// Resets this device took. + pub fn resets(&self) -> u32 { + self.resets + } + pub fn hold_completions(&mut self, hold: bool) { self.hold = hold; self.hold_for = None; @@ -288,6 +304,33 @@ impl DirectBlockDevice self.queue(DeviceCompletion { cookie, result }) } + /// A queued (not yet polled) command is abandoned as a controller that + /// honours Abort abandons one: its completion stays queued, now failed + /// `Cancelled`, and is delivered by the ordinary poll. + fn abort(&mut self, cookie: u64) -> bool { + if self.refuse_aborts { + return false; + } + for completion in self.completions.iter_mut().flatten() { + if completion.cookie == cookie { + completion.result = Err(BlockError::Cancelled); + return true; + } + } + false + } + + /// A reset drops every queued completion, as a controller reset drops + /// every command it held: none is ever delivered. + fn reset(&mut self) -> bool { + self.completions = [None; COMPLETIONS]; + self.head = 0; + self.tail = 0; + self.queued = 0; + self.resets += 1; + true + } + fn poll(&mut self) -> Option { if self.hold || self.queued == 0 { return None; @@ -353,6 +396,111 @@ mod tests { assert_eq!(bytes, [0; 1024]); } + /// A streaming target takes several reads in flight at once, each into + /// its own range; refuses writes and channel CPU access while busy; and + /// comes back to its owner intact on release. + #[test] + fn a_streaming_target_admits_many_reads_and_returns_on_release() { + let mut channel = channel(); + // Disk blocks 0..4 hold four distinct patterns. + let source = channel.register_buffer(2048).unwrap(); + for block in 0..4u8 { + channel + .write_buffer(source, block as usize * 512, &[0x10 + block; 512]) + .unwrap(); + } + let write = channel.submit_write(0, 4, source, 0).unwrap(); + assert_eq!(channel.poll().unwrap().request, write); + channel.unregister_buffer(source).unwrap(); + + // The caller-funded buffer (here the double's own allocation stands + // in for an imported lease) becomes one 2 KiB target. + let funded = channel + .control(|device| device.allocate_buffer(2048)) + .unwrap() + .unwrap(); + let target = channel + .adopt_target(funded) + .map_err(|(_, error)| error) + .unwrap(); + assert_eq!( + channel.stats().registered_bytes, + 0, + "not the channel's bytes" + ); + + // A target is a read destination only. + assert_eq!( + channel.submit_write(0, 1, target, 0), + Err(BlockError::ReadOnly) + ); + assert_eq!( + channel.outstanding(), + 0, + "a refused write leaves nothing pending" + ); + + // Four reads in flight against one buffer, landing out of disk order + // (the double queues completions until they are polled). + let mut ids = [RequestId(0); 4]; + for (i, id) in ids.iter_mut().enumerate() { + *id = channel + .submit_read(3 - i as u64, 1, target, i * 512) + .unwrap(); + } + assert_eq!(channel.buffer_inflight(target), Some(4)); + assert_eq!( + channel.submit_read(0, 1, target, 0), + Err(BlockError::Backpressure) + ); + assert_eq!( + channel.read_buffer(target, 0, &mut [0; 1]), + Err(BlockError::BufferBusy) + ); + assert_eq!( + channel.release_buffer(target).err(), + Some(BlockError::BufferBusy) + ); + + let mut done = 0; + while let Some(completion) = channel.wait() { + assert!(ids.contains(&completion.request)); + completion.result.unwrap(); + done += 1; + } + assert_eq!(done, 4); + assert_eq!(channel.buffer_inflight(target), Some(0)); + // The channel never touches a target through the CPU, idle or not: a + // device-only import has no CPU view for it to touch. + let mut landed = [0u8; 2048]; + assert_eq!( + channel.read_buffer(target, 0, &mut landed), + Err(BlockError::Range) + ); + assert_eq!( + channel.write_buffer(target, 0, &[0; 1]), + Err(BlockError::Range) + ); + assert_eq!(channel.fill_buffer(target, 0), Err(BlockError::Range)); + + // Out of range is refused against the target's own length. + assert_eq!( + channel.submit_read(0, 1, target, 2048), + Err(BlockError::Range) + ); + let returned = channel.release_buffer(target).unwrap(); + assert_eq!(returned.len(), 2048); + // Its owner reads what landed. + for (i, chunk) in returned.bytes().chunks(512).enumerate() { + assert!( + chunk.iter().all(|byte| *byte == 0x13 - i as u8), + "range {i}" + ); + } + assert_eq!(channel.buffer_inflight(target), None, "the id is stale"); + assert!(channel.adopt_target(returned).is_ok()); + } + #[test] fn a_batch_publishes_once_and_reaps_a_dense_completion_prefix() { let mut channel = channel(); diff --git a/native/crates/driver/src/block/mod.rs b/native/crates/driver/src/block/mod.rs index c1213ef..4f0a133 100644 --- a/native/crates/driver/src/block/mod.rs +++ b/native/crates/driver/src/block/mod.rs @@ -22,10 +22,8 @@ use kore_abi::monitor::FilesystemLatencySnapshot; pub mod memory; pub mod region; -pub mod shared; pub use region::RegionDevice; -pub use shared::{Lane, SharedBackend, SharedStats}; /// Maximum buffers registered in one direct channel by default. pub const DEFAULT_BUFFERS: usize = 8; @@ -132,6 +130,17 @@ impl BufferId { pub const fn to_bits(self) -> u32 { ((self.generation as u32) << 16) | self.slot as u32 } + + /// The inverse of [`to_bits`](Self::to_bits), for a name that crossed a + /// process boundary as data. It conveys nothing by itself: every channel + /// operation re-checks the slot's live generation, so a forged or stale + /// word answers `UnknownBuffer`. + pub const fn from_bits(bits: u32) -> Self { + Self { + slot: bits as u16, + generation: (bits >> 16) as u16, + } + } } /// Caller-selected correlation identity, unique while outstanding. @@ -466,13 +475,18 @@ pub trait DirectBlockDevice { /// a correlated reply. /// /// The conservative default is `false`, so a backend only claims - /// independence deliberately. [`shared::SharedBackend`] requires it: a - /// parking `progress` future would have to hold the shared borrow across - /// an `await`, so the sharing constructor refuses such a backend rather - /// than leaving that conflict latent. + /// independence deliberately. A composition that shares one backend + /// between several drivers needs it: a parking `progress` future would + /// have to hold the shared borrow across an `await`. const INDEPENDENT_PROGRESS: bool = false; fn geometry(&self) -> BlockGeometry; + /// Where this device's block zero sits in the namespace beneath it — the + /// coordinates a storage-transport grant names (`crate::stor`). Zero for + /// a whole device; a region clamp adds its first block to its backend's. + fn namespace_lba(&self) -> u64 { + 0 + } /// Largest single registered buffer this backend can accept. The default /// is the command-transfer ceiling; bounded test/transport backends may /// advertise a smaller allocation limit. @@ -511,6 +525,27 @@ pub trait DirectBlockDevice { None } + /// Ask the device to abandon the submitted command `cookie` — the + /// escalation for one that outlived its deadline. `true` when the device + /// agreed: the command's own completion still follows, failed, through + /// the ordinary poll, and only then is its buffer free. `false` (the + /// default: a backend with no abort) changes nothing; the command may + /// still complete, late. Control plane only, and it may wait on the + /// device's own answer. + fn abort(&mut self, _cookie: u64) -> bool { + false + } + + /// Reset the device and bring its queues back — the escalation for a + /// command it will not give up (an abort it refused). Every submitted + /// command is lost: none of their completions ever arrives, and their + /// buffers are free the moment this returns `true`. `false` (the + /// default: a backend with no reset, or a reset that failed) changes + /// nothing the caller can rely on. Control plane only. + fn reset(&mut self) -> bool { + false + } + /// Sample physical-device health on the infrequent control plane. /// Backends without a health protocol report `Unsupported`; callers must /// keep geometry/status available rather than fabricate a healthy sample. @@ -608,7 +643,13 @@ struct Pending { struct Registered { generation: u16, - busy: bool, + /// Commands in flight against this buffer. An ordinary registered + /// buffer admits exactly one; a streaming target admits up to + /// `OUTSTANDING`. CPU access through the channel requires zero. + inflight: u16, + /// Adopted through [`BlockChannel::adopt_target`]: caller-funded, read + /// destination only, many disjoint commands in flight at once. + target: bool, buffer: B, } @@ -645,7 +686,13 @@ impl BlockChannel { pub fn new(device: D, mode: CompletionMode) -> Result { - if BUFFERS == 0 || OUTSTANDING == 0 || BUFFERS > u16::MAX as usize { + // Both tables are indexed by `u16` (buffer slots, and a target's + // in-flight count, which the pending table bounds). + if BUFFERS == 0 + || OUTSTANDING == 0 + || BUFFERS > u16::MAX as usize + || OUTSTANDING > u16::MAX as usize + { return Err(BlockError::Geometry); } let geometry = device.geometry().validate()?; @@ -672,6 +719,12 @@ impl self.geometry } + /// Where the device's block zero sits in its namespace + /// ([`DirectBlockDevice::namespace_lba`]). + pub fn namespace_lba(&self) -> u64 { + self.device.namespace_lba() + } + pub fn max_buffer_bytes(&self) -> usize { self.device.max_buffer_bytes() } @@ -720,7 +773,8 @@ impl }; self.buffers[slot] = Some(Registered { generation: id.generation, - busy: false, + inflight: 0, + target: false, buffer, }); self.stats.registered_bytes = self.stats.registered_bytes.saturating_add(bytes); @@ -733,10 +787,12 @@ impl pub fn unregister_buffer(&mut self, id: BufferId) -> Result<(), BlockError> { let slot = self.buffer_slot(id)?; - if self.buffers[slot].as_ref().is_some_and(|b| b.busy) { + if self.buffers[slot].as_ref().is_some_and(|b| b.inflight != 0) { return Err(BlockError::BufferBusy); } - if let Some(registered) = self.buffers[slot].take() { + if let Some(registered) = self.buffers[slot].take() + && !registered.target + { self.stats.registered_bytes = self .stats .registered_bytes @@ -746,6 +802,80 @@ impl Ok(()) } + /// Adopt a backend buffer over memory the **caller** funded — a device + /// mapping of a client's lease, never the backend's own pool — as a + /// **streaming target**. + /// + /// A target differs from a registered buffer in exactly two ways, both + /// following from who owns its bytes: + /// + /// - It admits many commands in flight at once, each into its own + /// block-aligned range: a multi-GiB destination is one buffer, not one + /// buffer per command. The channel does not police overlap between + /// in-flight ranges — the bytes are the caller's, so two reads racing + /// into one range can only hand the caller its own data in an + /// unspecified order. What the channel still guarantees is the + /// exclusion that protects *other* memory: every range is checked + /// against the target's length, and CPU access through the channel is + /// refused while anything is in flight. + /// - It is a read destination only. A write sourcing caller memory is + /// not something a streaming target is for, and refusing it keeps the + /// lane's contract one direction wide. + /// + /// Target bytes are not the channel's allocation, so they are not + /// booked in [`ChannelStats::registered_bytes`]. A refused adoption + /// returns the buffer: it is affine custody of a device mapping, and + /// dropping it here would strand that mapping without its owner's + /// quiescent release. + pub fn adopt_target(&mut self, buffer: D::Buffer) -> Result { + if let Err(error) = self.accepts_control() { + return Err((buffer, error)); + } + if buffer.len() == 0 || buffer.len() % self.geometry.block_size as usize != 0 { + return Err((buffer, BlockError::Alignment)); + } + let Some(slot) = self.buffers.iter().position(Option::is_none) else { + return Err((buffer, BlockError::Backpressure)); + }; + let id = BufferId { + slot: slot as u16, + generation: self.generations[slot], + }; + self.buffers[slot] = Some(Registered { + generation: id.generation, + inflight: 0, + target: true, + buffer, + }); + Ok(id) + } + + /// Remove an idle buffer and return it to its owner — the release twin + /// of [`adopt_target`](Self::adopt_target), for a caller that must + /// retire the backend mapping itself rather than drop it. Works for any + /// registered buffer; refused while a command is in flight against it. + pub fn release_buffer(&mut self, id: BufferId) -> Result { + let slot = self.buffer_slot(id)?; + if self.buffers[slot].as_ref().is_some_and(|b| b.inflight != 0) { + return Err(BlockError::BufferBusy); + } + let registered = self.buffers[slot].take().ok_or(BlockError::UnknownBuffer)?; + if !registered.target { + self.stats.registered_bytes = self + .stats + .registered_bytes + .saturating_sub(registered.buffer.len()); + } + self.generations[slot] = self.generations[slot].wrapping_add(1).max(1); + Ok(registered.buffer) + } + + /// Commands in flight against one buffer, or `None` for a stale id. + pub fn buffer_inflight(&self, id: BufferId) -> Option { + let slot = self.buffer_slot(id).ok()?; + self.buffers[slot].as_ref().map(|b| b.inflight) + } + pub fn write_buffer( &mut self, id: BufferId, @@ -756,9 +886,15 @@ impl let registered = self.buffers[slot] .as_mut() .ok_or(BlockError::UnknownBuffer)?; - if registered.busy { + if registered.inflight != 0 { return Err(BlockError::BufferBusy); } + // A streaming target is the controller's to write and its owner's to + // read: the channel may hold no CPU view of it at all (a device-only + // import), so it never touches one through the CPU. + if registered.target { + return Err(BlockError::Range); + } let end = offset.checked_add(src.len()).ok_or(BlockError::Range)?; let dst = registered .buffer @@ -779,9 +915,15 @@ impl let registered = self.buffers[slot] .as_ref() .ok_or(BlockError::UnknownBuffer)?; - if registered.busy { + if registered.inflight != 0 { return Err(BlockError::BufferBusy); } + // A streaming target is the controller's to write and its owner's to + // read: the channel may hold no CPU view of it at all (a device-only + // import), so it never touches one through the CPU. + if registered.target { + return Err(BlockError::Range); + } let end = offset.checked_add(dst.len()).ok_or(BlockError::Range)?; let src = registered .buffer @@ -798,9 +940,15 @@ impl let registered = self.buffers[slot] .as_mut() .ok_or(BlockError::UnknownBuffer)?; - if registered.busy { + if registered.inflight != 0 { return Err(BlockError::BufferBusy); } + // A streaming target is the controller's to write and its owner's to + // read: the channel may hold no CPU view of it at all (a device-only + // import), so it never touches one through the CPU. + if registered.target { + return Err(BlockError::Range); + } registered.buffer.bytes_mut().fill(value); Ok(()) } @@ -967,7 +1115,7 @@ impl && let Ok(buffer_slot) = self.buffer_slot(buffer) && let Some(registered) = self.buffers[buffer_slot].as_mut() { - registered.busy = false; + registered.inflight = registered.inflight.saturating_sub(1); } Completion { request: pending.id, @@ -1161,10 +1309,16 @@ impl }; if let Some(slot) = buffer_slot { let registered = self.buffers[slot].as_mut().expect("validated buffer"); - if registered.busy { + if registered.target { + // A target is a read destination; its pending table bounds + // its depth, so the count cannot pass `OUTSTANDING`. + if !matches!(command, DeviceCommand::Read { .. }) { + return Err(BlockError::ReadOnly); + } + } else if registered.inflight != 0 { return Err(BlockError::BufferBusy); } - registered.busy = true; + registered.inflight += 1; } let outcome = match buffer_slot { @@ -1186,7 +1340,8 @@ impl }; if let Err(error) = outcome { if let Some(slot) = buffer_slot { - self.buffers[slot].as_mut().expect("validated buffer").busy = false; + let registered = self.buffers[slot].as_mut().expect("validated buffer"); + registered.inflight -= 1; } return Err(error); } @@ -1273,6 +1428,12 @@ impl<'a, D: DirectBlockDevice, const B: usize, const O: usize> BlockBatch<'a, D, self.channel.device.publish(); self.published = true; } + + /// Release a batch that staged nothing without a doorbell: a poll that + /// found no free slot or no queued work should not ring the device. + pub fn skip_publish(mut self) { + self.published = true; + } } impl Drop for BlockBatch<'_, D, B, O> { diff --git a/native/crates/driver/src/block/region.rs b/native/crates/driver/src/block/region.rs index 0931043..58f3cc3 100644 --- a/native/crates/driver/src/block/region.rs +++ b/native/crates/driver/src/block/region.rs @@ -11,6 +11,11 @@ //! The wrapper is deliberately dumb about where designations come from: a //! GPT partition, a fixed platform carve, or a test's arbitrary window all //! narrow the same way. +//! +//! A view can also be sealed [`read_only`](RegionDevice::read_only): a +//! designation that grants read authority only (a pack image) then cannot +//! reach a write or discard through any caller above it — the refusal lives +//! in the clamp's own rebase, not in a channel flag a caller could skip. use super::{ BlockError, BlockGeometry, BlockHealth, DeviceCommand, DeviceCompletion, DirectBlockDevice, @@ -21,6 +26,8 @@ pub struct RegionDevice { device: D, first_lba: u64, blocks: u64, + /// Sealed read-only: writes and discards are refused at rebase. + read_only: bool, } impl RegionDevice { @@ -31,6 +38,7 @@ impl RegionDevice { device, first_lba: 0, blocks: geometry.block_count, + read_only: false, }) } @@ -54,9 +62,18 @@ impl RegionDevice { device: self.device, first_lba, blocks, + read_only: self.read_only, }) } + /// Seal the view read-only. One-way: nothing unseals a view, and + /// [`narrow`](Self::narrow) keeps the seal, so read authority can never + /// widen into write authority by re-deriving the clamp. + pub fn read_only(mut self) -> Self { + self.read_only = true; + self + } + /// The view's extent as `(first_lba, blocks)` in backend coordinates. pub fn region(&self) -> (u64, u64) { (self.first_lba, self.blocks) @@ -96,6 +113,14 @@ impl RegionDevice { } fn rebase_command(&self, command: DeviceCommand) -> Result { + if self.read_only + && matches!( + command, + DeviceCommand::Write { .. } | DeviceCommand::Discard { .. } + ) + { + return Err(BlockError::ReadOnly); + } Ok(match command { DeviceCommand::Read { lba, @@ -130,10 +155,20 @@ impl DirectBlockDevice for RegionDevice { /// A clamp changes addresses, not how the queue advances. const INDEPENDENT_PROGRESS: bool = D::INDEPENDENT_PROGRESS; + fn namespace_lba(&self) -> u64 { + self.device.namespace_lba().saturating_add(self.first_lba) + } + + fn abort(&mut self, cookie: u64) -> bool { + self.device.abort(cookie) + } + fn geometry(&self) -> BlockGeometry { + let inner = self.device.geometry(); BlockGeometry { block_count: self.blocks, - ..self.device.geometry() + read_only: inner.read_only || self.read_only, + ..inner } } @@ -264,6 +299,25 @@ mod tests { assert!(books.snapshot(channel.stats().max_outstanding).measured()); } + /// A clamp names where its block zero sits in the namespace, and a + /// clamp of a clamp adds both offsets — the coordinates a transport + /// grant is written in. + #[test] + fn nested_regions_report_their_namespace_origin() { + let whole = RegionDevice::whole(device()).unwrap(); + assert_eq!(whole.namespace_lba(), 0); + let outer = whole.narrow(2, 6).map_err(|(_, error)| error).unwrap(); + assert_eq!(outer.namespace_lba(), 2); + let inner = RegionDevice::whole(outer) + .unwrap() + .narrow(1, 3) + .map_err(|(_, error)| error) + .unwrap(); + assert_eq!(inner.namespace_lba(), 3); + let channel: BlockChannel<_, 2, 4> = BlockChannel::new(inner, CompletionMode::Poll).unwrap(); + assert_eq!(channel.namespace_lba(), 3); + } + #[test] fn whole_view_is_the_identity() { let region = RegionDevice::whole(device()).unwrap(); @@ -436,4 +490,47 @@ mod tests { assert_eq!(channel.wait().unwrap().request, id); assert!(channel.submit_write(4, 1, buffer, 0).is_err()); } + + /// A sealed view refuses writes and discards in the clamp itself — a + /// device call below a channel that forgot its own read-only state still + /// cannot reach the blocks — while reads rebase exactly as before, and a + /// later narrow keeps the seal. + #[test] + fn a_read_only_view_refuses_writes_below_the_channel() { + let mut region = RegionDevice::whole(device()) + .unwrap() + .read_only() + .narrow(2, 4) + .map_err(|(_, error)| error) + .unwrap(); + assert!(region.geometry().read_only); + let mut buffer = region.allocate_buffer(512).unwrap(); + let write = DeviceCommand::Write { + lba: 0, + blocks: 1, + buffer_offset: 0, + }; + assert_eq!( + region.submit(write, Some(&mut buffer), 1), + Err(BlockError::ReadOnly) + ); + assert_eq!( + region.submit(DeviceCommand::Discard { lba: 0, blocks: 1 }, None, 2), + Err(BlockError::ReadOnly) + ); + let read = DeviceCommand::Read { + lba: 1, + blocks: 1, + buffer_offset: 0, + }; + region.submit(read, Some(&mut buffer), 3).unwrap(); + + let mut channel: BlockChannel<_, 2, 4> = + BlockChannel::new(region, CompletionMode::Poll).unwrap(); + let registered = channel.register_buffer(512).unwrap(); + assert_eq!( + channel.submit_write(0, 1, registered, 0), + Err(BlockError::ReadOnly) + ); + } } diff --git a/native/crates/driver/src/block/shared.rs b/native/crates/driver/src/block/shared.rs deleted file mode 100644 index a704e92..0000000 --- a/native/crates/driver/src/block/shared.rs +++ /dev/null @@ -1,986 +0,0 @@ -//! One controller driving several designated regions — the completion demux. -//! -//! A disk may carry more than one region this system owns under distinct -//! lifecycle policies (a self-healing build-owned store beside conservative -//! user data). Each region wants its own [`BlockChannel`] — its own -//! registered buffers, its own request table, its own [`RegionDevice`] clamp -//! — but there is exactly **one** controller beneath them, and a channel -//! owns its backend by value. -//! -//! [`SharedBackend`] owns that one backend and hands out one [`Lane`] per -//! region. A lane *is* a [`DirectBlockDevice`], so the ordinary composition -//! stacks on top of it unchanged: -//! -//! ```text -//! SharedBackend one affine backend, one submission/completion queue -//! ├ Lane 0 → RegionDevice → BlockChannel → provider (region A) -//! └ Lane 1 → RegionDevice → BlockChannel → provider (region B) -//! ``` -//! -//! ## Why the naive split is wrong -//! -//! [`BlockChannel`] draws its cookies from a private counter that starts at -//! `1`. Two channels over one backend therefore mint the **same** cookie -//! sequence, and a completion carries nothing else to tell them apart. Both -//! failure modes are silent: -//! -//! - **Cross-completion (corruption).** Lane A's cookie `5` completes; lane -//! B polls first, matches `5` against *its own* pending table, and settles -//! B's request — releasing B's registered buffer while B's transfer is -//! still in flight. -//! - **Dropped completion (a wedge).** Lane A polls and receives B's -//! completion. [`BlockChannel::poll`] cannot place it, reports -//! `StaleCompletion`, and it is gone. B waits for a completion that will -//! never come again. -//! -//! Both are answered by structure, not by care at the call sites. -//! -//! ## The demux -//! -//! **Tagged cookies.** A lane stamps its index into the cookie's high -//! [`LANE_BITS`] on the way down and strips it on the way up, so the channel -//! above never sees the tag and the two cookie spaces cannot alias. A -//! payload too wide for the remaining bits is **refused** -//! ([`BlockError::Range`]) rather than truncated: truncation would alias two -//! distinct requests, which is the corruption this exists to prevent. -//! -//! **Per-lane mailboxes.** Whichever lane drains the backend routes *every* -//! completion it sees by tag — its own included — into the owning lane's -//! mailbox, and then serves itself from its own mailbox. A completion is -//! therefore never observed by a lane that cannot place it, so it is never -//! dropped. -//! -//! **The mailbox cannot overflow.** Let `outstanding[L]` be the commands -//! lane `L` has submitted but not yet been *delivered*. A lane's submission -//! is refused once `outstanding[L] == INFLIGHT`; routing moves a completion -//! from the backend into a mailbox without changing the count; delivery pops -//! the mailbox and decrements it. So `mail[L].len() <= outstanding[L] <= -//! INFLIGHT` holds at every step, and a mailbox sized `INFLIGHT` has room by -//! construction. The cap lives here — at the same seam that mints the tag — -//! rather than being inherited from the channel's `OUTSTANDING`, because the -//! bound is what makes the no-drop argument true. -//! -//! A completion that survives all of that (a tag naming a retired lane, a -//! duplicate, a cookie for nothing outstanding) is a backend protocol -//! violation. It is counted as [`SharedStats::stray`] and discarded, because -//! by the invariant above it corresponds to no live request — but it is -//! counted, since on a serial-less machine a number is the only way that -//! fact ever surfaces. -//! -//! ## Ownership and lifetime -//! -//! Lanes borrow the backend (`&'a SharedBackend`), so the borrow checker -//! proves the controller outlives every lane — no reference count, no -//! allocation, and this crate keeps its empty dependency list. A composition -//! that must outlive its own stack frame (a service's mounted filesystem) -//! gives the backend a `'static` home; the lifetime is then a concrete -//! `'static` in the provider's type rather than a self-reference. -//! -//! One worker owns and drives all of a backend's lanes, exactly as it owns -//! one channel today. Interior mutability is [`RefCell`], and no borrow is -//! held across a suspension point — which is why a backend may only be shared -//! when its queue advances on its own -//! ([`DirectBlockDevice::INDEPENDENT_PROGRESS`]); a backend whose `progress` -//! genuinely parks would have to hold the shared borrow across an `await`. -//! [`SharedBackend::new`] refuses such a backend rather than leaving the -//! conflict latent. -//! -//! [`BlockChannel`]: crate::block::BlockChannel -//! [`BlockChannel::poll`]: crate::block::BlockChannel::poll -//! [`RegionDevice`]: crate::block::RegionDevice - -use core::cell::{Cell, RefCell}; - -use super::{ - BlockError, BlockGeometry, BlockHealth, DeviceCommand, DeviceCompletion, DirectBlockDevice, -}; - -/// Width of the lane tag carried in a shared cookie's high bits. -pub const LANE_BITS: u32 = 8; -/// Bit position of the lane tag. -const LANE_SHIFT: u32 = u64::BITS - LANE_BITS; -/// Lanes one shared backend can address with [`LANE_BITS`]. -pub const MAX_LANES: usize = 1usize << LANE_BITS; -/// Largest request cookie a lane may carry beneath its tag. A wider cookie -/// is refused, never truncated — truncation aliases distinct requests. -pub const MAX_LANE_COOKIE: u64 = (1u64 << LANE_SHIFT) - 1; - -/// Counters describing how the demux behaved. Read without borrowing the -/// backend, so a status answer can never contend with the data plane. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct SharedStats { - /// Completions drained from the backend and placed in a mailbox. - pub routed: u64, - /// Completions handed up to their owning lane. - pub delivered: u64, - /// Completions belonging to no live outstanding request — a backend - /// protocol violation, counted rather than merely discarded. - pub stray: u64, - /// Submissions refused because the lane was already at `INFLIGHT`. - pub backpressure: u64, - /// Times the shared borrow was already held. Structurally unreachable - /// (one worker, no borrow across a suspension); non-zero means that - /// invariant broke and the demux degraded instead of panicking. - pub conflicts: u64, -} - -/// A lane's bounded holding queue for completions drained by another lane. -struct Mailbox { - slots: [Option; N], - head: usize, - count: usize, -} - -impl Mailbox { - fn new() -> Self { - Self { - slots: [None; N], - head: 0, - count: 0, - } - } - - fn len(&self) -> usize { - self.count - } - - /// Place one completion. `false` means the invariant documented on this - /// module broke — the caller counts it as stray rather than overwriting a - /// completion that a lane is still owed. - fn push(&mut self, completion: DeviceCompletion) -> bool { - if self.count == N { - return false; - } - let tail = (self.head + self.count) % N; - self.slots[tail] = Some(completion); - self.count += 1; - true - } - - fn pop(&mut self) -> Option { - if self.count == 0 { - return None; - } - let completion = self.slots[self.head].take(); - self.head = (self.head + 1) % N; - self.count -= 1; - completion - } -} - -/// The shared mutable half: the backend and the demux books. -struct Inner { - device: D, - mail: [Mailbox; LANES], - /// Per lane: submitted and not yet delivered. Bounds the mailbox. - outstanding: [usize; LANES], - /// Per lane: handed out once (a lane is affine — never claimed twice). - claimed: [bool; LANES], - /// Per lane: claimed and not yet retired. - live: [bool; LANES], -} - -/// One backend shared by `LANES` cookie-disjoint lanes, each admitting at -/// most `INFLIGHT` commands. -pub struct SharedBackend { - inner: RefCell>, - /// Cached at construction: geometry is immutable for a provisioned - /// backend, so the hot read path never takes the shared borrow. - geometry: BlockGeometry, - max_buffer_bytes: usize, - routed: Cell, - delivered: Cell, - stray: Cell, - backpressure: Cell, - conflicts: Cell, -} - -impl - SharedBackend -{ - /// Take ownership of one backend to share. - /// - /// Refuses a backend whose queue needs an executor-aware - /// [`progress`](DirectBlockDevice::progress) step - /// ([`BlockError::Unsupported`]): a parking `progress` future would have - /// to hold the shared borrow across an `await`, and refusing at - /// construction is the only place that conflict can be named before it - /// becomes a runtime fault. - pub fn new(device: D) -> Result { - if LANES == 0 || LANES > MAX_LANES || INFLIGHT == 0 { - return Err(BlockError::Geometry); - } - if !D::INDEPENDENT_PROGRESS { - return Err(BlockError::Unsupported); - } - let geometry = device.geometry().validate()?; - let max_buffer_bytes = device.max_buffer_bytes(); - Ok(Self { - inner: RefCell::new(Inner { - device, - mail: core::array::from_fn(|_| Mailbox::new()), - outstanding: [0; LANES], - claimed: [false; LANES], - live: [false; LANES], - }), - geometry, - max_buffer_bytes, - routed: Cell::new(0), - delivered: Cell::new(0), - stray: Cell::new(0), - backpressure: Cell::new(0), - conflicts: Cell::new(0), - }) - } - - /// The whole backend's geometry. A lane reports this unchanged; the - /// designated extent is applied by the [`RegionDevice`](crate::block::RegionDevice) - /// clamp above the lane, which is what keeps the clamp beneath the - /// channel where code above it cannot name a block outside the region. - pub fn geometry(&self) -> BlockGeometry { - self.geometry - } - - /// Claim lane `lane`. Refused if it is out of range or already claimed — - /// a lane is affine, so two compositions can never share one cookie tag. - pub fn claim(&self, lane: usize) -> Result, BlockError> { - if lane >= LANES { - return Err(BlockError::Range); - } - let mut inner = self.inner.try_borrow_mut().map_err(|_| self.conflict())?; - if inner.claimed[lane] { - return Err(BlockError::Backpressure); - } - inner.claimed[lane] = true; - inner.live[lane] = true; - Ok(Lane { core: self, lane }) - } - - /// How many commands lane `lane` has submitted and not yet been - /// delivered. `None` when the books are momentarily borrowed. - pub fn outstanding(&self, lane: usize) -> Option { - let inner = self.inner.try_borrow().ok()?; - inner.outstanding.get(lane).copied() - } - - /// Whether lane `lane` is claimed and not yet retired. - pub fn is_live(&self, lane: usize) -> bool { - self.inner - .try_borrow() - .ok() - .and_then(|inner| inner.live.get(lane).copied()) - .unwrap_or(false) - } - - pub fn stats(&self) -> SharedStats { - SharedStats { - routed: self.routed.get(), - delivered: self.delivered.get(), - stray: self.stray.get(), - backpressure: self.backpressure.get(), - conflicts: self.conflicts.get(), - } - } - - /// Run one infrequent control-plane operation against the shared physical - /// backend. No lane may retain the borrow and no request may be in flight; - /// either conflict is a typed transport failure rather than a `RefCell` - /// panic or control/data-plane overlap. - pub fn control(&self, f: impl FnOnce(&mut D) -> T) -> Result { - let mut inner = self.inner.try_borrow_mut().map_err(|_| self.conflict())?; - if inner - .outstanding - .iter() - .any(|outstanding| *outstanding != 0) - { - return Err(BlockError::Draining); - } - Ok(f(&mut inner.device)) - } - - /// Recover the backend. Reachable only once every lane is gone, which - /// the borrow checker enforces: a lane holds `&self`. - pub fn into_device(self) -> D { - self.inner.into_inner().device - } - - fn conflict(&self) -> BlockError { - self.conflicts.set(self.conflicts.get().saturating_add(1)); - BlockError::Transport - } -} - -/// One lane of a shared backend: a [`DirectBlockDevice`] whose cookies carry -/// this lane's tag and whose completions are demuxed to it alone. -pub struct Lane<'a, D: DirectBlockDevice, const LANES: usize, const INFLIGHT: usize> { - core: &'a SharedBackend, - lane: usize, -} - -impl<'a, D: DirectBlockDevice, const LANES: usize, const INFLIGHT: usize> - Lane<'a, D, LANES, INFLIGHT> -{ - /// This lane's index — its cookie tag. - pub fn index(&self) -> usize { - self.lane - } - - /// The backend this lane shares, for stats and sibling inspection. - pub fn backend(&self) -> &'a SharedBackend { - self.core - } - - fn tag(&self, cookie: u64) -> Result { - if cookie > MAX_LANE_COOKIE { - return Err(BlockError::Range); - } - Ok(((self.lane as u64) << LANE_SHIFT) | cookie) - } - - fn submit_inner( - &mut self, - command: DeviceCommand, - buffer: Option<&mut D::Buffer>, - cookie: u64, - deferred: bool, - ) -> Result<(), BlockError> { - let tagged = self.tag(cookie)?; - let core = self.core; - let mut inner = core.inner.try_borrow_mut().map_err(|_| core.conflict())?; - if !inner.live[self.lane] { - return Err(BlockError::Revoked); - } - // The cap that makes the mailbox large enough by construction. - if inner.outstanding[self.lane] >= INFLIGHT { - core.backpressure - .set(core.backpressure.get().saturating_add(1)); - return Err(BlockError::Backpressure); - } - let outcome = if deferred { - inner.device.stage(command, buffer, tagged) - } else { - inner.device.submit(command, buffer, tagged) - }; - outcome?; - inner.outstanding[self.lane] += 1; - Ok(()) - } - - /// File one completion under the lane its tag names. - /// - /// A tag outside the lane set, a retired lane, nothing outstanding, or a - /// full mailbox all mean this completion answers no live request. The - /// invariant documented on this module says that cannot happen; count it - /// so a backend that does it anyway is visible rather than merely - /// swallowed. - fn file( - core: &SharedBackend, - inner: &mut Inner, - completion: DeviceCompletion, - ) { - let lane = (completion.cookie >> LANE_SHIFT) as usize; - let placeable = lane < LANES - && inner.live[lane] - && inner.outstanding[lane] > inner.mail[lane].len() - && inner.mail[lane].push(completion); - if placeable { - core.routed.set(core.routed.get().saturating_add(1)); - } else { - core.stray.set(core.stray.get().saturating_add(1)); - } - } - - /// Drain a bounded batch from the backend into the lane mailboxes — one - /// completion-head doorbell for the whole prefix — and report how many - /// moved, so a caller can drain again rather than answer short. - fn route( - core: &SharedBackend, - inner: &mut Inner, - ) -> usize { - let mut batch = [None; INFLIGHT]; - let count = inner.device.poll_batch(&mut batch); - for completion in batch[..count].iter().flatten().copied() { - Self::file(core, inner, completion); - } - count - } - - /// Drain until this lane holds `want` completions or the backend runs - /// dry. A single batch is capped at `INFLIGHT`, so an interleaved queue - /// can hand back a short prefix while this lane's completions are still - /// sitting behind a sibling's — and a short answer costs the caller a - /// wait budget it did not need to spend. Terminates because every - /// further pass must move at least one completion off a finite queue. - fn route_until( - core: &SharedBackend, - inner: &mut Inner, - lane: usize, - want: usize, - ) { - while inner.mail[lane].len() < want && Self::route(core, inner) > 0 {} - } - - /// Pop this lane's next delivered completion, stripping the tag so the - /// channel above sees the cookie it minted. - fn deliver( - core: &SharedBackend, - inner: &mut Inner, - lane: usize, - ) -> Option { - let completion = inner.mail[lane].pop()?; - inner.outstanding[lane] = inner.outstanding[lane].saturating_sub(1); - core.delivered.set(core.delivered.get().saturating_add(1)); - Some(DeviceCompletion { - cookie: completion.cookie & MAX_LANE_COOKIE, - ..completion - }) - } -} - -impl<'a, D: DirectBlockDevice, const LANES: usize, const INFLIGHT: usize> DirectBlockDevice - for Lane<'a, D, LANES, INFLIGHT> -{ - type Buffer = D::Buffer; - - /// A shared backend is only ever built over an independently-progressing - /// queue ([`SharedBackend::new`]), so a lane advances on its own too. - const INDEPENDENT_PROGRESS: bool = true; - - fn geometry(&self) -> BlockGeometry { - self.core.geometry - } - - fn max_buffer_bytes(&self) -> usize { - self.core.max_buffer_bytes - } - - fn allocate_buffer(&mut self, bytes: usize) -> Result { - let core = self.core; - core.inner - .try_borrow_mut() - .map_err(|_| core.conflict())? - .device - .allocate_buffer(bytes) - } - - fn health(&mut self) -> Result { - let core = self.core; - core.inner - .try_borrow_mut() - .map_err(|_| core.conflict())? - .device - .health() - } - - /// The lanes share one controller, so they share its liveness. A borrow - /// conflict answers `Unknown` rather than vouching for a device it could - /// not read. - fn liveness(&mut self) -> super::Liveness { - let core = self.core; - match core.inner.try_borrow_mut() { - Ok(mut inner) => inner.device.liveness(), - Err(_) => { - core.conflict(); - super::Liveness::Unknown - } - } - } - - /// The shared backend's clock. A momentarily borrowed book reads as - /// "no clock" — that costs one unmeasured sample, never a wrong one. - fn clock_ns(&self) -> Option { - self.core - .inner - .try_borrow() - .ok() - .and_then(|inner| inner.device.clock_ns()) - } - - fn submit( - &mut self, - command: DeviceCommand, - buffer: Option<&mut Self::Buffer>, - cookie: u64, - ) -> Result<(), BlockError> { - self.submit_inner(command, buffer, cookie, false) - } - - fn stage( - &mut self, - command: DeviceCommand, - buffer: Option<&mut Self::Buffer>, - cookie: u64, - ) -> Result<(), BlockError> { - self.submit_inner(command, buffer, cookie, true) - } - - fn publish(&mut self) { - // Publishing a sibling's staged prefix early is lawful — the trait - // forbids relying on staged commands staying invisible — and one - // doorbell for both lanes is the point of sharing a queue. - if let Ok(mut inner) = self.core.inner.try_borrow_mut() { - inner.device.publish(); - } else { - self.core.conflict(); - } - } - - fn finish(&mut self, buffer: Option<&mut Self::Buffer>) -> Result<(), BlockError> { - let core = self.core; - core.inner - .try_borrow_mut() - .map_err(|_| core.conflict())? - .device - .finish(buffer) - } - - fn poll(&mut self) -> Option { - let core = self.core; - let Ok(mut inner) = core.inner.try_borrow_mut() else { - core.conflict(); - return None; - }; - Self::route_until(core, &mut inner, self.lane, 1); - Self::deliver(core, &mut inner, self.lane) - } - - fn poll_batch(&mut self, out: &mut [Option]) -> usize { - let core = self.core; - let Ok(mut inner) = core.inner.try_borrow_mut() else { - core.conflict(); - return 0; - }; - Self::route_until(core, &mut inner, self.lane, out.len()); - let mut count = 0; - for slot in out.iter_mut() { - let Some(completion) = Self::deliver(core, &mut inner, self.lane) else { - break; - }; - *slot = Some(completion); - count += 1; - } - count - } - - /// One bounded wait for *this lane's* next completion. - /// - /// A sibling's completion arriving instead is absorbed into its mailbox - /// and the wait continues — surfacing `None` there would make the caller - /// declare a live request lost. The loop is bounded by the completions - /// that can exist at all: every sibling's `outstanding`, each at most - /// `INFLIGHT`, plus one wait for this lane's own. - fn wait(&mut self) -> Option { - let core = self.core; - let Ok(mut inner) = core.inner.try_borrow_mut() else { - core.conflict(); - return None; - }; - let mut budget = LANES.saturating_mul(INFLIGHT).saturating_add(1); - loop { - Self::route_until(core, &mut inner, self.lane, 1); - if let Some(completion) = Self::deliver(core, &mut inner, self.lane) { - return Some(completion); - } - if inner.outstanding[self.lane] == 0 { - // Nothing of ours can complete; never spend a wait budget - // (or park on a route) on a sibling's behalf. - return None; - } - budget = budget.checked_sub(1)?; - // `wait` is synchronous — a backend parks the thread on its own - // delivery route rather than yielding to the executor — so - // holding the shared borrow across it cannot conflict. - let completion = inner.device.wait()?; - Self::file(core, &mut inner, completion); - } - } - - /// Route whatever the backend has already delivered. - /// - /// A shared backend's queue advances on its own, so there is no - /// correlated step to await here — and a lane must not hold the shared - /// borrow across a suspension point. Draining into the mailboxes is the - /// real progress this seam can make. - async fn progress(&mut self) -> Result<(), BlockError> { - let core = self.core; - let mut inner = core.inner.try_borrow_mut().map_err(|_| core.conflict())?; - while Self::route(core, &mut inner) > 0 {} - Ok(()) - } - - /// Retire this lane. The backend itself is quiesced by the **last** lane - /// to retire: tearing the controller down under a sibling's in-flight - /// transfer would strand its DMA. - fn quiesce(&mut self) -> Result<(), BlockError> { - let core = self.core; - let mut inner = core.inner.try_borrow_mut().map_err(|_| core.conflict())?; - if inner.outstanding[self.lane] != 0 { - return Err(BlockError::Draining); - } - inner.live[self.lane] = false; - if inner.live.iter().any(|live| *live) { - return Ok(()); - } - inner.device.quiesce() - } -} - -#[cfg(test)] -mod tests { - extern crate std; - - use super::*; - use crate::block::memory::MemoryBlockDevice; - use crate::block::{BlockChannel, CompletionMode, RegionDevice}; - - // 4096 disk bytes over 512-byte blocks = an 8-block backend. - type Device = MemoryBlockDevice<4096, 512>; - type Shared = SharedBackend; - - fn shared() -> Shared { - SharedBackend::new(Device::new(512, 8).unwrap()).unwrap() - } - - /// The whole point: the two lanes' cookie spaces cannot alias, so a - /// completion can only ever settle the request that produced it. - #[test] - fn lanes_tag_their_cookies_disjointly() { - let shared = shared(); - let a = shared.claim(0).unwrap(); - let b = shared.claim(1).unwrap(); - assert_eq!(a.tag(1).unwrap(), 1); - assert_eq!(b.tag(1).unwrap(), 1u64 << LANE_SHIFT | 1); - assert_ne!(a.tag(1).unwrap(), b.tag(1).unwrap()); - // A payload that would collide with the tag is refused, never - // truncated into another request's cookie. - assert_eq!(a.tag(MAX_LANE_COOKIE), Ok(MAX_LANE_COOKIE)); - assert_eq!(b.tag(MAX_LANE_COOKIE + 1), Err(BlockError::Range)); - assert_eq!(a.tag(u64::MAX), Err(BlockError::Range)); - } - - #[test] - fn a_lane_is_affine_and_bounded() { - let shared = shared(); - let _a = shared.claim(0).unwrap(); - assert_eq!(shared.claim(0).err(), Some(BlockError::Backpressure)); - assert_eq!(shared.claim(2).err(), Some(BlockError::Range)); - assert!(shared.is_live(0)); - assert!(!shared.is_live(1)); - } - - /// The demux's first duty: a completion drained by the wrong lane still - /// reaches its owner. Lane 1 drains both, keeps its own, and lane 0 - /// finds its completion waiting rather than lost. - #[test] - fn a_completion_drained_by_a_sibling_is_never_dropped() { - let shared = shared(); - let mut a = shared.claim(0).unwrap(); - let mut b = shared.claim(1).unwrap(); - - a.submit(DeviceCommand::Flush, None, 1).unwrap(); - b.submit(DeviceCommand::Flush, None, 1).unwrap(); - assert_eq!(shared.outstanding(0), Some(1)); - assert_eq!(shared.outstanding(1), Some(1)); - - // Lane 1 polls first and drains the queue — including lane 0's. - let completion = b.poll().unwrap(); - assert_eq!(completion.cookie, 1); - assert_eq!(shared.outstanding(1), Some(0)); - assert_eq!(shared.stats().stray, 0); - - // Lane 0's completion survived the sibling's drain. - let completion = a.poll().unwrap(); - assert_eq!(completion.cookie, 1); - assert_eq!(shared.outstanding(0), Some(0)); - assert_eq!(shared.stats().stray, 0); - assert_eq!(shared.stats().delivered, 2); - // Nothing left for either lane. - assert!(a.poll().is_none()); - assert!(b.poll().is_none()); - } - - /// `wait` must not report loss because a *sibling's* completion arrived - /// first: it absorbs the sibling's and keeps waiting for its own. - #[test] - fn wait_absorbs_a_siblings_completion_and_still_returns_its_own() { - let shared = shared(); - let mut a = shared.claim(0).unwrap(); - let mut b = shared.claim(1).unwrap(); - b.submit(DeviceCommand::Flush, None, 9).unwrap(); - a.submit(DeviceCommand::Flush, None, 7).unwrap(); - - let completion = a.wait().expect("its own completion, not a sibling's"); - assert_eq!(completion.cookie, 7); - // The sibling's is still owed, not consumed by the waiter. - assert_eq!(shared.outstanding(1), Some(1)); - assert_eq!(b.poll().unwrap().cookie, 9); - assert_eq!(shared.stats().stray, 0); - } - - /// An idle lane never spends a wait budget on a sibling's behalf. - #[test] - fn wait_on_an_idle_lane_is_not_loss_hunting() { - let shared = shared(); - let mut a = shared.claim(0).unwrap(); - let mut b = shared.claim(1).unwrap(); - b.submit(DeviceCommand::Flush, None, 3).unwrap(); - assert!(a.wait().is_none()); - // The sibling's completion is still there for the sibling. - assert_eq!(b.wait().unwrap().cookie, 3); - } - - /// The mailbox is sized by the same cap that gates submission, so the - /// no-overflow argument is enforced where it is made. - #[test] - fn a_lane_cannot_outrun_its_mailbox() { - let shared = shared(); - let mut a = shared.claim(0).unwrap(); - // INFLIGHT = 4: the fifth submission is refused, not queued into a - // mailbox that has no room for its completion. - for cookie in 1..=4 { - a.submit(DeviceCommand::Flush, None, cookie).unwrap(); - } - assert_eq!( - a.submit(DeviceCommand::Flush, None, 5), - Err(BlockError::Backpressure) - ); - assert_eq!(shared.stats().backpressure, 1); - // Drain one and the lane admits exactly one more. - assert_eq!(a.poll().unwrap().cookie, 1); - a.submit(DeviceCommand::Flush, None, 5).unwrap(); - assert_eq!(shared.outstanding(0), Some(4)); - // Every completion is accounted for; none strayed. - let mut drained = std::vec::Vec::new(); - while let Some(completion) = a.poll() { - drained.push(completion.cookie); - } - assert_eq!(drained, std::vec![2, 3, 4, 5]); - assert_eq!(shared.stats().stray, 0); - } - - #[test] - fn poll_batch_serves_only_its_own_lane() { - let shared = shared(); - let mut a = shared.claim(0).unwrap(); - let mut b = shared.claim(1).unwrap(); - for cookie in 1..=3 { - a.submit(DeviceCommand::Flush, None, cookie).unwrap(); - b.submit(DeviceCommand::Flush, None, cookie + 100).unwrap(); - } - let mut out = [None; 4]; - assert_eq!(a.poll_batch(&mut out), 3); - assert_eq!( - out.iter() - .flatten() - .map(|c| c.cookie) - .collect::>(), - std::vec![1, 2, 3] - ); - let mut out = [None; 4]; - assert_eq!(b.poll_batch(&mut out), 3); - assert_eq!( - out.iter() - .flatten() - .map(|c| c.cookie) - .collect::>(), - std::vec![101, 102, 103] - ); - assert_eq!(shared.stats().stray, 0); - } - - /// The controller stays up until the last lane retires — quiescing it - /// under a sibling's mount would strand that sibling's transfers. - #[test] - fn the_last_lane_out_quiesces_the_backend() { - let shared = shared(); - let mut a = shared.claim(0).unwrap(); - let mut b = shared.claim(1).unwrap(); - a.submit(DeviceCommand::Flush, None, 1).unwrap(); - // A lane with work in flight refuses to retire. - assert_eq!(a.quiesce(), Err(BlockError::Draining)); - assert_eq!(a.poll().unwrap().cookie, 1); - a.quiesce().unwrap(); - assert!(!shared.is_live(0)); - assert!(shared.is_live(1)); - // A retired lane conveys no submission authority. - assert_eq!( - a.submit(DeviceCommand::Flush, None, 2), - Err(BlockError::Revoked) - ); - b.quiesce().unwrap(); - assert!(!shared.is_live(1)); - } - - #[test] - fn control_plane_access_requires_every_lane_to_be_idle() { - let shared = shared(); - let mut lane = shared.claim(0).unwrap(); - lane.submit(DeviceCommand::Flush, None, 1).unwrap(); - assert_eq!( - shared.control(|device| device.geometry()).unwrap_err(), - BlockError::Draining - ); - assert_eq!(lane.poll().unwrap().cookie, 1); - assert_eq!( - shared.control(|device| device.geometry()).unwrap(), - lane.geometry() - ); - } - - /// A backend that needs an executor-aware progress step cannot be - /// shared: its future would hold the shared borrow across an `await`. - #[test] - fn a_parking_backend_is_refused_at_construction() { - struct Parking(Device); - impl DirectBlockDevice for Parking { - type Buffer = ::Buffer; - fn geometry(&self) -> BlockGeometry { - self.0.geometry() - } - fn allocate_buffer(&mut self, bytes: usize) -> Result { - self.0.allocate_buffer(bytes) - } - fn submit( - &mut self, - command: DeviceCommand, - buffer: Option<&mut Self::Buffer>, - cookie: u64, - ) -> Result<(), BlockError> { - self.0.submit(command, buffer, cookie) - } - fn poll(&mut self) -> Option { - self.0.poll() - } - fn quiesce(&mut self) -> Result<(), BlockError> { - self.0.quiesce() - } - } - let device = Parking(Device::new(512, 8).unwrap()); - assert!(!Parking::INDEPENDENT_PROGRESS); - assert_eq!( - SharedBackend::::new(device).err(), - Some(BlockError::Unsupported) - ); - } - - /// The composition slice 4 actually mounts: two disjoint regions, two - /// channels, one controller. Both channels mint cookies from `1`, and - /// each region's bytes must land only in that region. - #[test] - fn two_region_channels_over_one_backend_stay_disjoint() { - let shared = shared(); - // Blocks [0, 4) and [4, 8) of the same 8-block backend. - let system = RegionDevice::whole(shared.claim(0).unwrap()) - .unwrap() - .narrow(0, 4) - .map_err(|(_, error)| error) - .unwrap(); - let user = RegionDevice::whole(shared.claim(1).unwrap()) - .unwrap() - .narrow(4, 4) - .map_err(|(_, error)| error) - .unwrap(); - let mut system: BlockChannel<_, 2, 4> = - BlockChannel::new(system, CompletionMode::Poll).unwrap(); - let mut user: BlockChannel<_, 2, 4> = - BlockChannel::new(user, CompletionMode::Poll).unwrap(); - assert_eq!(system.geometry().block_count, 4); - assert_eq!(user.geometry().block_count, 4); - - let system_buffer = system.register_buffer(512).unwrap(); - let user_buffer = user.register_buffer(512).unwrap(); - system.write_buffer(system_buffer, 0, &[0xa5; 512]).unwrap(); - user.write_buffer(user_buffer, 0, &[0x5a; 512]).unwrap(); - - // Both channels are at cookie 1 — the collision the demux answers. - let system_request = system.submit_write(0, 1, system_buffer, 0).unwrap(); - let user_request = user.submit_write(0, 1, user_buffer, 0).unwrap(); - assert_eq!(system_request.0, user_request.0); - - // Each channel settles its own request and nothing else. - let completion = system.wait().unwrap(); - assert_eq!(completion.request, system_request); - completion.result.unwrap(); - let completion = user.wait().unwrap(); - assert_eq!(completion.request, user_request); - completion.result.unwrap(); - assert_eq!(system.late_completions(), 0); - assert_eq!(user.late_completions(), 0); - assert_eq!(shared.stats().stray, 0); - - // Region 0's write landed at backend block 0, region 1's at block 4: - // read each back through its own clamp. - system.fill_buffer(system_buffer, 0).unwrap(); - let request = system.submit_read(0, 1, system_buffer, 0).unwrap(); - assert_eq!(system.wait().unwrap().request, request); - let mut back = [0u8; 512]; - system.read_buffer(system_buffer, 0, &mut back).unwrap(); - assert_eq!(back, [0xa5; 512]); - - user.fill_buffer(user_buffer, 0).unwrap(); - let request = user.submit_read(0, 1, user_buffer, 0).unwrap(); - assert_eq!(user.wait().unwrap().request, request); - let mut back = [0u8; 512]; - user.read_buffer(user_buffer, 0, &mut back).unwrap(); - assert_eq!(back, [0x5a; 512]); - - // Neither channel can name a block in the other's region. - assert!(system.submit_write(4, 1, system_buffer, 0).is_err()); - assert!(user.submit_write(4, 1, user_buffer, 0).is_err()); - } - - /// A completion that answers no live request is counted, never delivered - /// to a lane that would mis-settle it. By the module's invariant a - /// healthy backend cannot produce one — which is exactly why it has to be - /// visible when a backend does. - #[test] - fn a_completion_answering_no_live_request_is_counted_not_delivered() { - let shared = shared(); - let mut a = shared.claim(0).unwrap(); - // Straight onto the backend, beneath the tagging seam: a cookie for a - // lane that exists but has nothing outstanding, and one whose tag - // names no lane at all. - { - let mut inner = shared.inner.borrow_mut(); - inner.device.submit(DeviceCommand::Flush, None, 1).unwrap(); - inner - .device - .submit(DeviceCommand::Flush, None, 7u64 << LANE_SHIFT | 1) - .unwrap(); - } - assert!(a.poll().is_none()); - let stats = shared.stats(); - assert_eq!(stats.stray, 2); - assert_eq!(stats.delivered, 0); - assert_eq!(stats.routed, 0); - // The lane is still healthy afterwards: its own work still settles. - a.submit(DeviceCommand::Flush, None, 1).unwrap(); - assert_eq!(a.poll().unwrap().cookie, 1); - assert_eq!(shared.stats().stray, 2); - } - - /// With completions held, a sibling's drain cannot manufacture one: - /// nothing is delivered, nothing strays, and both requests stay owed. - #[test] - fn a_held_queue_delivers_to_neither_lane() { - let shared = shared(); - let mut a = shared.claim(0).unwrap(); - let mut b = shared.claim(1).unwrap(); - a.submit(DeviceCommand::Flush, None, 1).unwrap(); - b.submit(DeviceCommand::Flush, None, 1).unwrap(); - { - let mut inner = shared.inner.borrow_mut(); - inner.device.hold_completions(true); - } - assert!(a.poll().is_none()); - assert!(b.poll().is_none()); - assert_eq!(shared.stats().delivered, 0); - assert_eq!(shared.stats().stray, 0); - { - let mut inner = shared.inner.borrow_mut(); - inner.device.hold_completions(false); - } - assert_eq!(a.poll().unwrap().cookie, 1); - assert_eq!(b.poll().unwrap().cookie, 1); - } -} diff --git a/native/crates/driver/src/common/pagetable/alloc.rs b/native/crates/driver/src/common/pagetable/alloc.rs index f9b6f41..fb100d9 100644 --- a/native/crates/driver/src/common/pagetable/alloc.rs +++ b/native/crates/driver/src/common/pagetable/alloc.rs @@ -73,11 +73,19 @@ pub trait TableSource { /// this source allocated (a corrupted tree or a caller bug — the walker /// propagates it as [`MmuError::UnknownTableFrame`]; never a null /// sentinel). A `Some` pointer is valid for [`table_bytes`] bytes; the - /// walker reads/writes it as a `[u64]`. + /// walker reads it as a `[u64]`. Writers use + /// [`resolve_for_write`](Self::resolve_for_write) before each mutation. /// /// [`table_bytes`]: Self::table_bytes fn resolve(&self, addr: Self::Addr) -> Option<*mut u8>; + /// Resolve a frame before writing its entries. Sources with explicit + /// cache publication can retain this write intent; coherent sources use + /// the default. The pointer must not be retained across publication. + fn resolve_for_write(&self, addr: Self::Addr) -> Option<*mut u8> { + self.resolve(addr) + } + /// Release a frame during teardown. Best-effort — a source that cannot free /// individually may no-op and reclaim in bulk. fn release(&mut self, addr: Self::Addr); @@ -99,16 +107,25 @@ pub trait TableSource { #[cfg(feature = "alloc")] mod dma_pool { use alloc::vec::Vec; + use core::cell::Cell; use kore_abi::device::IovaAddr; use super::{MmuError, MmuResult, TableSource}; use crate::seam::{Dma, DmaPool}; + struct TableFrame { + buffer: B, + dirty: Cell, + } + /// [`TableSource`] backed by a [`DmaPool`]. See the module note. pub struct DmaPoolTables { pool: P, - frames: Vec, + // Sorted by the buffer's device address, including after release and + // reuse. Walks resolve on every table read/write: a linear search here + // multiplies their cost by the entire resident tree's frame count. + frames: Vec>, frame_bytes: usize, } @@ -134,6 +151,11 @@ mod dma_pool { &self.pool } + fn frame_index(&self, addr: IovaAddr) -> Result { + self.frames + .binary_search_by_key(&addr.0, |frame| frame.buffer.iova().0) + } + /// Visit every live table frame's CPU window. /// /// This is the cache-maintenance enumeration hook: a device whose @@ -143,8 +165,29 @@ mod dma_pool { /// frame length; it must not retain the pointer beyond the call. pub fn for_each_frame(&self, mut visit: impl FnMut(*mut u8, usize)) { for frame in &self.frames { - visit(frame.cpu(), self.frame_bytes); + visit(frame.buffer.cpu(), self.frame_bytes); + } + } + + /// Publish allocated or written frames, retaining dirtiness if either + /// cache maintenance or its completion barrier fails. Callbacks must + /// not mutate this source or retain its pointers. Read-only resolution + /// never dirties a frame. This is cache publication, not TLB invalidation. + pub fn publish_dirty_frames( + &self, + mut publish: impl FnMut(*mut u8, usize) -> Result<(), E>, + complete: impl FnOnce() -> Result<(), E>, + ) -> Result<(), E> { + for frame in &self.frames { + if frame.dirty.get() { + publish(frame.buffer.cpu(), self.frame_bytes)?; + } + } + complete()?; + for frame in &self.frames { + frame.dirty.set(false); } + Ok(()) } /// Recover the affine DMA supply after every table frame was released. @@ -169,7 +212,17 @@ mod dma_pool { .alloc(self.frame_bytes) .ok_or(MmuError::OutOfTables)?; let iova = buf.iova(); - self.frames.push(buf); + let index = self + .frame_index(iova) + .err() + .ok_or(MmuError::InvalidTableGeometry)?; + self.frames.insert( + index, + TableFrame { + buffer: buf, + dirty: Cell::new(true), + }, + ); Ok(iova) } @@ -177,16 +230,24 @@ mod dma_pool { // A miss (a frame this source never allocated) is a typed `None` // the walker turns into `MmuError::UnknownTableFrame` — in every // build profile, never a null sentinel. - self.frames - .iter() - .find(|frame| frame.iova() == addr) - .map(|frame| frame.cpu()) + self.frame_index(addr) + .ok() + .map(|index| self.frames[index].buffer.cpu()) + } + + fn resolve_for_write(&self, addr: IovaAddr) -> Option<*mut u8> { + self.frame_index(addr).ok().map(|index| { + let frame = &self.frames[index]; + frame.dirty.set(true); + frame.buffer.cpu() + }) } fn release(&mut self, addr: IovaAddr) { - if let Some(pos) = self.frames.iter().position(|f| f.iova() == addr) { - // Drop returns the buffer to the pool. - self.frames.swap_remove(pos); + if let Ok(pos) = self.frame_index(addr) { + // Drop returns the buffer to the pool. Preserve address order + // so the hot read/write lookup needs no separate index. + self.frames.remove(pos); } } @@ -205,6 +266,7 @@ mod dma_pool { struct TestBuf { words: Box<[u64; 512]>, iova: IovaAddr, + address_reads: Cell, } impl Dma for TestBuf { @@ -213,6 +275,7 @@ mod dma_pool { } fn iova(&self) -> IovaAddr { + self.address_reads.set(self.address_reads.get() + 1); self.iova } @@ -237,10 +300,87 @@ mod dma_pool { Some(TestBuf { words: Box::new([0; 512]), iova: IovaAddr(index << 12), + address_reads: Cell::new(0), }) } } + #[test] + fn table_lookup_preserves_identity_across_nonmonotonic_allocation_and_reuse() { + let mut tables = DmaPoolTables::new(TestPool { next: Cell::new(1) }, 0x1000); + // Sparse, descending allocations exercise the pool's freedom to + // recycle addresses; table order is not allocation order. + for page in [9, 5, 1, 7, 3] { + tables.pool().next.set(page); + let addr = tables.alloc_table().unwrap(); + // SAFETY: this test owns the live, zeroed frame; no device + // accesses it, and the one-byte write is within its backing. + unsafe { tables.resolve_for_write(addr).unwrap().write(page as u8) }; + } + tables + .publish_dirty_frames(|_, _| Ok::<_, ()>(()), || Ok(())) + .unwrap(); + tables.release(IovaAddr(5 << 12)); + tables.pool().next.set(5); + let reused = tables.alloc_table().unwrap(); + // SAFETY: resolution returns the owned live test frame's start. + assert_eq!(unsafe { tables.resolve(reused).unwrap().read() }, 0); + for page in [1, 3, 7, 9] { + assert_eq!( + // SAFETY: these frames remain owned and have no device. + unsafe { tables.resolve(IovaAddr(page << 12)).unwrap().read() }, + page as u8 + ); + } + assert!(tables.resolve(IovaAddr(2 << 12)).is_none()); + assert!(tables.resolve(IovaAddr((3 << 12) + 1)).is_none()); + tables.pool().next.set(5); + assert_eq!(tables.alloc_table(), Err(MmuError::InvalidTableGeometry)); + assert_eq!(tables.frame_count(), 5); + tables.release(IovaAddr(2 << 12)); + assert_eq!(tables.frame_count(), 5); + let mut dirty = 0; + tables + .publish_dirty_frames( + |_, _| { + dirty += 1; + Ok::<_, ()>(()) + }, + || Ok(()), + ) + .unwrap(); + assert_eq!(dirty, 1, "only the recycled frame needs publication"); + } + + #[test] + fn table_resolution_cost_grows_logarithmically_with_residency() { + const FRAMES: usize = 1024; + let mut tables = DmaPoolTables::new(TestPool { next: Cell::new(1) }, 0x1000); + for _ in 0..FRAMES { + tables.alloc_table().unwrap(); + } + let probes = |tables: &DmaPoolTables| { + tables + .frames + .iter() + .map(|frame| frame.buffer.address_reads.get()) + .sum::() + }; + // The previous linear scan touched every frame on these lookups. + // Allow two comparisons beyond ceil(log2(n)) for search endpoints. + let ceiling = (usize::BITS - (FRAMES - 1).leading_zeros()) as usize + 2; + let before = probes(&tables); + assert!(tables.resolve(IovaAddr((FRAMES as u64) << 12)).is_some()); + assert!(probes(&tables) - before <= ceiling); + let before = probes(&tables); + assert!( + tables + .resolve_for_write(IovaAddr(((FRAMES + 1) as u64) << 12)) + .is_none() + ); + assert!(probes(&tables) - before <= ceiling); + } + #[test] fn the_table_source_returns_its_affine_supply_only_when_empty() { let pool = TestPool { next: Cell::new(1) }; diff --git a/native/crates/driver/src/common/pagetable/walker.rs b/native/crates/driver/src/common/pagetable/walker.rs index f079dd9..be04de4 100644 --- a/native/crates/driver/src/common/pagetable/walker.rs +++ b/native/crates/driver/src/common/pagetable/walker.rs @@ -154,7 +154,9 @@ where { return Err(MmuError::InvalidTableGeometry); } - let base = src.resolve(table).ok_or(MmuError::UnknownTableFrame)? as *mut u64; + let base = src + .resolve_for_write(table) + .ok_or(MmuError::UnknownTableFrame)? as *mut u64; if !(base as usize).is_multiple_of(align_of::()) { return Err(MmuError::InvalidTableGeometry); } diff --git a/native/crates/driver/src/cookie.rs b/native/crates/driver/src/cookie.rs index fb402db..31abe8a 100644 --- a/native/crates/driver/src/cookie.rs +++ b/native/crates/driver/src/cookie.rs @@ -10,36 +10,145 @@ /// ceiling does not apply. pub const MAX_SLOTS: usize = 256; -/// `id → cookie`, indexed `id % len` (ids are reused round-robin over the -/// table, so the modulus is exact, never lossy, as long as in-flight count -/// stays `< len` — which the producer ring enforces). +/// `id → cookie`, in two allocation forms. /// -/// Under `alloc` the backing is a `Box<[u64]>` sized to the ring's depth; -/// otherwise a fixed `[u64; MAX_SLOTS]`. Both index identically — the -/// house `Neutral` pattern. +/// **Indexed** ([`put`](Self::put)/[`get`](Self::get)): the id is a ring +/// slot the caller already owns, indexed `id % len` — for a device whose +/// slots are reused in ring order (a NIC's descriptor ring), where a slot +/// is free again by the time the ring comes back around to it. +/// +/// **Claimed** ([`claim`](Self::claim)/[`release`](Self::release)): the +/// table hands out an id that no command in flight holds, and takes it +/// back only when that command's completion is reaped — for a device that +/// echoes the id and completes out of order (NVMe, whose CID must be unique +/// among a queue's outstanding commands). Round-robin reuse is NOT safe +/// there: an in-flight count below the depth proves that *some* command +/// completed, not that the one holding the next id did, and a straggling +/// read still in flight when its id is reissued has its cookie overwritten +/// — its completion then answers the newer command before that one's data +/// landed, and the newer command's completion answers nobody, so the +/// straggler's request never completes. +/// +/// Under `alloc` the backing is sized to the ring's depth; otherwise a +/// fixed `MAX_SLOTS`. Both index identically — the house `Neutral` +/// pattern. pub struct CookieTable { #[cfg(feature = "alloc")] inner: alloc::boxed::Box<[u64]>, #[cfg(not(feature = "alloc"))] inner: [u64; MAX_SLOTS], + /// Ids currently claimed and not yet released. + #[cfg(feature = "alloc")] + live: alloc::boxed::Box<[bool]>, + #[cfg(not(feature = "alloc"))] + live: [bool; MAX_SLOTS], + /// The free ids, as a stack: `free[..free_len]`. + #[cfg(feature = "alloc")] + free: alloc::boxed::Box<[u16]>, + #[cfg(not(feature = "alloc"))] + free: [u16; MAX_SLOTS], + free_len: usize, + /// Ids the table hands out (`0..depth`). The heapless posture's arrays + /// are the fixed ceiling, so their length is not the depth — counting + /// live ids from the array length once reported 253 of a 3-deep table + /// as in flight. + depth: usize, } impl CookieTable { /// A zeroed table for `depth` entries, posture-appropriate. Under - /// `alloc`, sized exactly to `depth`; otherwise the fixed ceiling. + /// `alloc`, sized exactly to `depth`; otherwise the fixed ceiling, with + /// only ids below `depth` claimable. #[cfg(feature = "alloc")] pub fn new(depth: u16) -> Self { - Self { - inner: alloc::vec![0u64; depth as usize].into_boxed_slice(), - } + let depth = depth as usize; + let mut table = Self { + inner: alloc::vec![0u64; depth].into_boxed_slice(), + live: alloc::vec![false; depth].into_boxed_slice(), + free: alloc::vec![0u16; depth].into_boxed_slice(), + free_len: 0, + depth: 0, + }; + table.fill_free(depth); + table } /// A zeroed table for `depth` entries, posture-appropriate. #[cfg(not(feature = "alloc"))] - pub fn new(_depth: u16) -> Self { - Self { + pub fn new(depth: u16) -> Self { + let mut table = Self { inner: [0u64; MAX_SLOTS], + live: [false; MAX_SLOTS], + free: [0u16; MAX_SLOTS], + free_len: 0, + depth: 0, + }; + table.fill_free((depth as usize).min(MAX_SLOTS)); + table + } + + /// Stack the ids `0..depth` so the first claim takes 0, the next 1, + /// and so on — the order a fresh queue would have issued them. + fn fill_free(&mut self, depth: usize) { + for (at, id) in (0..depth).rev().enumerate() { + self.free[at] = id as u16; + } + self.free_len = depth; + self.depth = depth; + } + + /// **Claim** an id no command in flight holds, recording `cookie` for + /// it. `None` when every id is claimed: the caller's ring is full. + pub fn claim(&mut self, cookie: u64) -> Option { + if self.free_len == 0 { + return None; } + self.free_len -= 1; + let id = self.free[self.free_len]; + self.live[id as usize] = true; + self.inner[id as usize] = cookie; + Some(id) + } + + /// **Release** id `id` on its completion, returning the cookie it was + /// claimed with. `None` when `id` is not in flight — a completion the + /// device owes nobody, which must not free an id twice. + pub fn release(&mut self, id: u16) -> Option { + let at = id as usize; + if at >= self.live.len() || !self.live[at] { + return None; + } + self.live[at] = false; + self.free[self.free_len] = id; + self.free_len += 1; + Some(self.inner[at]) + } + + /// Release every id at once, as though each command in flight completed + /// — for a queue the device has dropped whole (a controller reset). + /// Answers how many were in flight. + pub fn clear(&mut self) -> u16 { + let lost = self.depth.saturating_sub(self.free_len) as u16; + *self = Self::new(self.depth as u16); + lost + } + + /// Whether `id` is claimed and not yet released. + pub fn is_live(&self, id: u16) -> bool { + self.live.get(id as usize).is_some_and(|live| *live) + } + + /// Ids claimed and not yet released. Derived from `depth`, not the + /// backing length: without `alloc` the live array is the fixed ceiling, + /// so `len - free_len` would report the unclaimable ids as live. + pub fn live_count(&self) -> usize { + self.depth - self.free_len + } + + /// The live id carrying `cookie`, if any — the reverse lookup an abort + /// needs. A scan of the depth: control plane only. + pub fn find(&self, cookie: u64) -> Option { + (0..self.depth as u16).find(|&id| self.is_live(id) && self.get(id) == cookie) } /// Record `cookie` for command id `id`. @@ -65,3 +174,72 @@ impl CookieTable { self.inner.len() == 0 } } + +#[cfg(test)] +mod tests { + use super::*; + + /// A claimed id is never handed out again until its completion releases + /// it, whatever order completions arrive in. + #[test] + fn a_claimed_id_stays_claimed_until_released() { + let mut table = CookieTable::new(4); + let a = table.claim(10).expect("first"); + let b = table.claim(11).expect("second"); + let c = table.claim(12).expect("third"); + let d = table.claim(13).expect("fourth"); + assert_eq!([a, b, c, d], [0, 1, 2, 3]); + assert_eq!(table.claim(14), None, "every id is in flight"); + // The middle one completes first — out of order, as random reads do. + assert_eq!(table.release(b), Some(11)); + let again = table.claim(15).expect("the released id is free again"); + assert_eq!(again, b); + assert!(table.is_live(a) && table.is_live(c) && table.is_live(d)); + assert_eq!(table.claim(16), None, "nothing else was released"); + } + + /// Releasing an id nobody holds frees nothing: a duplicate completion + /// cannot make two commands share an id. + #[test] + fn a_release_of_an_unclaimed_id_is_refused() { + let mut table = CookieTable::new(2); + assert_eq!(table.release(0), None, "never claimed"); + let a = table.claim(1).expect("claim"); + assert_eq!(table.release(a), Some(1)); + assert_eq!(table.release(a), None, "already released"); + assert_eq!(table.release(7), None, "out of range"); + let x = table.claim(2).expect("one"); + let y = table.claim(3).expect("two"); + assert_ne!(x, y); + assert_eq!(table.claim(4), None); + } + + /// The indexed form still answers by slot, untouched by claims. + /// The live count is claims minus releases at every depth — including + /// a table shallower than the heapless ceiling, whose backing arrays are + /// longer than its depth. + #[test] + fn live_count_tracks_claims_at_a_shallow_depth() { + let mut table = CookieTable::new(3); + assert_eq!(table.live_count(), 0); + let a = table.claim(1).unwrap(); + let b = table.claim(2).unwrap(); + assert_eq!(table.live_count(), 2); + assert_eq!(table.release(a), Some(1)); + assert_eq!(table.live_count(), 1); + table.claim(3).unwrap(); + table.claim(4).unwrap(); + assert_eq!(table.live_count(), 3); + assert_eq!(table.claim(5), None, "a 3-deep table is full at 3"); + assert_eq!(table.release(b), Some(2)); + assert_eq!(table.live_count(), 2); + } + + #[test] + fn put_and_get_index_by_slot() { + let mut table = CookieTable::new(3); + table.put(2, 99); + assert_eq!(table.get(2), 99); + assert_eq!(table.live_count(), 0); + } +} diff --git a/native/crates/driver/src/gpt.rs b/native/crates/driver/src/gpt.rs index 7bfd04f..3d6a8e3 100644 --- a/native/crates/driver/src/gpt.rs +++ b/native/crates/driver/src/gpt.rs @@ -98,6 +98,25 @@ pub const KORE_FS_TYPE: Guid = guid!("D673CABE-2DF1-4F42-8B89-B9E56AC86714"); /// ``` pub const KORE_SYSTEM_TYPE: Guid = guid!("D9991ABE-2DF1-409E-8416-E05280489E79"); +/// The partition type GUID that designates **a kore pack region** — an +/// immutable, installer-written image (model weights and similar bulk +/// read-mostly data) that the filesystem row streams at queue depth into +/// client-funded memory without a filesystem in between. +/// +/// The designation grants *read* authority only: the pack lane's +/// `RegionDevice` clamp is read-only by construction, so no kore code path +/// can write, discard, or format these blocks. The layout inside the region +/// belongs to whoever produced the image; kore sees block ranges. Write one +/// from another OS and leave it: +/// +/// ```text +/// sgdisk --new=0:0:+24G \ +/// --typecode=0:55287D31-3555-44E8-A6E8-F9F74A225290 \ +/// --change-name=0:"kore pack" /dev/nvme0n1 +/// dd if=model.pack of=/dev/nvme0n1pN bs=4M oflag=direct +/// ``` +pub const KORE_PACK_TYPE: Guid = guid!("55287D31-3555-44E8-A6E8-F9F74A225290"); + /// The LBA the primary GPT header always occupies (LBA 0 is the /// protective MBR). pub const PRIMARY_HEADER_LBA: u64 = 1; diff --git a/native/crates/driver/src/host.rs b/native/crates/driver/src/host.rs index d3ceded..8ab8359 100644 --- a/native/crates/driver/src/host.rs +++ b/native/crates/driver/src/host.rs @@ -52,7 +52,7 @@ use kore_rt::service::{ }; use crate::seam::{ - Dma, DmaPool, DmaReleaseFailure, Regs, Regs64, RegsNarrow, RegsRegion, RegsWindowError, + Dma, DmaPool, DmaReleaseFailure, Quiet, Regs, Regs64, RegsNarrow, RegsRegion, RegsWindowError, }; /// Page granularity — DMA buffers are page-aligned (rings and list pages @@ -119,15 +119,22 @@ impl HostDeviceToken { supply: DmaImport, iova: u64, device_write: bool, + cpu_view: bool, ) -> Result { match self { Self::Pci(device) => match supply { - DmaImport::Lease(lease) => { + DmaImport::Lease(lease) if cpu_view => { verbs::import_dma_window(lease, device, iova, device_write) } - DmaImport::Route { route, bytes } => { + // A lease import has no device-only form: it is the CPU + // lease itself. Only a route can be lent for the device alone. + DmaImport::Lease(_) => Err(verbs::DmaMapFault::Bounds), + DmaImport::Route { route, bytes, .. } if cpu_view => { verbs::import_route_dma_window(route, device, iova, bytes, device_write) } + DmaImport::Route { route, bytes, .. } => { + verbs::import_route_device_window(route, device, iova, bytes, device_write) + } }, Self::Platform(_) => Err(verbs::DmaMapFault::Bounds), } @@ -256,7 +263,10 @@ pub enum IntakeError { /// The DMA window refused, stage-coded: `code[2:0]` the /// [`DmaMapFault`](kore_rt::cap::verbs::DmaMapFault) stage (1 bounds, /// 2 slot exhaustion, 3 carve, 4 CPU-view map, 5 geometry, 6 device - /// publish), `code >> 3` the kernel status for the commit stages. + /// publish), `code >> 3` the kernel status for legacy commit stages. + /// Fabric-route refusals occupy `0x80..=0x97`: the low two bits give + /// registration/observation/admission stage (1/2/3) and bits 2..4 + /// identify the route status (Conflict through Capability). DmaMap { code: u8 }, /// The class's own bring-up refused the realized resources — the /// controller is present and mappable but did not come up. @@ -1200,7 +1210,15 @@ enum CapBacking { /// service boundary, after all stage-1 cleanup in that operation has returned. struct ImportedAllocation { released: bool, + // Set only by finish_buffer_export after the backend proves exact-buffer + // quiescence and completes stage-1 invalidation. Besides requesting the + // return ticket, this permits live retirement of this outer mapping. return_requested: bool, + // Set only by `retire_quiet` when its live retirement did not complete: + // the engine's `Quiet` proof still covers this window (nothing it popped + // names it, nothing unpopped can reach it), so the retry is the same + // live retirement, with no device idleness needed. + quiet: bool, returned: Option, window: Option, } @@ -1212,12 +1230,14 @@ pub enum DmaImport { Route { route: kore_rt::cap::FabricRouteToken, bytes: u64, + align_log2: u8, }, } impl DmaImport { pub fn claim( owner: kore_rt::cap::ReturnedCapability, + route_access: kore_abi::session::FabricAccess, ) -> Result { let required = kore_abi::cap::Rights::ACCESS | kore_abi::cap::Rights::MODIFY @@ -1228,9 +1248,10 @@ impl DmaImport { } let route = owner.try_into_fabric_route(required)?; match route.describe() { - Ok(info) if info.operation == 1 && info.access & 3 == 3 => Ok(Self::Route { + Ok(info) if route_can_import(&info, route_access) => Ok(Self::Route { route, bytes: info.bytes, + align_log2: info.align_log2, }), _ => Err(route.into_returned_capability()), } @@ -1241,6 +1262,15 @@ impl DmaImport { Self::Route { bytes, .. } => *bytes, } } + /// Authenticated IOVA alignment for this import. Lease callers predate + /// route geometry and retain the established 2 MiB placement; routes + /// carry the backing class end to end. + pub fn align(&self) -> u64 { + match self { + Self::Lease(_) => 2 << 20, + Self::Route { align_log2, .. } => 1 << *align_log2, + } + } pub fn into_returned_capability(self) -> Result { match self { Self::Lease(lease) => lease.into_returned_capability().map_err(Self::Lease), @@ -1249,6 +1279,51 @@ impl DmaImport { } } +/// The outcome of [`DeviceResources::retire_quiet`]. +#[must_use] +pub enum QuietRetire { + /// The window's device mapping is gone and its allocation handed back. + Retired, + /// The live retirement did not complete. The allocation is marked + /// released for the idle-boundary retirement to finish. + Deferred, + /// Not an import of this resource set, or already released: the buffer + /// rides back to its owner. + Refused(CapBuf), +} + +fn route_can_import( + info: &kore_abi::session::FabricRouteInfo, + required: kore_abi::session::FabricAccess, +) -> bool { + use kore_abi::session::{FabricAccess, FabricOp}; + required.contains(FabricAccess::READ) + && FabricAccess::from_bits(info.access) + .is_some_and(|access| access.contains(required)) + && info.operation == FabricOp::IommuMap as u8 +} + +#[cfg(test)] +#[test] +fn route_import_requires_the_requested_device_access() { + use kore_abi::session::{FabricAccess, FabricRouteInfo}; + let mut info = FabricRouteInfo { + bytes: 4096, + access: FabricAccess::READ.bits(), + operation: 1, + align_log2: kore_abi::session::LEASE_ALIGN_4K, + }; + assert!(route_can_import(&info, FabricAccess::READ)); + assert!(!route_can_import(&info, FabricAccess::READ | FabricAccess::WRITE)); + info.access = (FabricAccess::READ | FabricAccess::WRITE).bits(); + assert!(route_can_import(&info, FabricAccess::READ | FabricAccess::WRITE)); + info.operation = 0; + assert!(!route_can_import(&info, FabricAccess::READ)); + info.operation = 1; + info.access = 0x80 | FabricAccess::READ.bits(); + assert!(!route_can_import(&info, FabricAccess::READ)); +} + /// Affine custody of a detached buffer awaiting the native serving boundary. pub enum BufferReturn { Imported(ImportedReturn), @@ -1323,36 +1398,80 @@ fn retire_imports_and_resume( imports: &mut Vec, device_idle: bool, released: impl Fn(&T) -> bool, + live: impl Fn(&T) -> bool, mut retire: impl FnMut(&T) -> bool, activate: impl FnOnce() -> bool, ) -> bool { - // Exact IOVA retirement closes the entire requester's DMA latch. Keep - // every detached owner until the backend proves its admissions quiescent. - if !device_idle { - return true; - } + // An ordinary released import needs requester-wide DMA retirement: it + // waits until every backend admission is quiescent, and a refusal leaves + // the latch unproven, which fails the whole epoch. + // + // A `live` one carries its own quiescence proof (an exact export, a + // storage engine's `Quiet` target). It retires whenever it is reached, + // idle or not, and a refusal touched no latch: its owner stays for the + // next boundary instead of failing the requester + // (`res/LIFECYCLE_HARDENING.md` §6). let mut changed = false; let mut index = 0; while index < imports.len() { - if !released(&imports[index]) { + let entry = &imports[index]; + let proven = live(entry); + if !released(entry) || !(device_idle || proven) { index += 1; continue; } - if !retire(&imports[index]) { + if retire(entry) { + imports.remove(index); + // Activation replays the latch only at the idle boundary; a + // running requester's latch rests on a surviving window. + changed |= device_idle; + continue; + } + if !proven { return false; } - imports.remove(index); - changed = true; + index += 1; } !changed || activate() } impl ImportedAllocation { fn retire(&mut self) -> bool { + self.retire_as(DmaWindow::try_retire_device) + } + + /// [`retire`](Self::retire) with the device still running: the window's + /// device mapping goes through the live unshare. + fn retire_live(&mut self) -> bool { + self.retire_as(DmaWindow::try_retire_device_live) + } + + /// Export's guarded detach already proves this exact mapping quiet. + /// Ordinary release does not carry that proof and retains the requester + /// retirement path. A failed live retirement retains the owner for the + /// caller's containment path; it never grants a return ticket early. + fn retire_released(&mut self) -> bool { + if self.retires_live() { + self.retire_live() + } else { + self.retire() + } + } + + /// Whether this released window's own quiescence proof lets it retire + /// with the requester running. + fn retires_live(&self) -> bool { + self.return_requested || self.quiet + } + + fn retire_as( + &mut self, + retire_device: fn(&mut DmaWindow) -> kore_rt::cap::RetireOutcome, + ) -> bool { // The named IOVA child closes first. Keep registration and CPU-backed // allocation custody until device access is proved retired. if let Some(window) = self.window.as_mut() { - if !window.try_retire_device().is_closed() { + if !retire_device(window).is_closed() { return false; } } @@ -1677,8 +1796,24 @@ struct DmaOwner { /// window owner, so there is no separate parent token to retain. _pool: Option, device: Option, + /// The lowest IOVA the next import may take: past every range any import + /// has ever occupied, not only the live ones. Import IOVAs never move + /// back, for two reasons: + /// + /// - a window retired with the device running (§11) is safe only while + /// its IOVA stays unmapped: a stray access then faults at the IOMMU + /// instead of landing in whatever reused the range; + /// - retirement unmaps leaves but keeps the domain's intermediate tables, + /// so a 2 MiB leaf cannot be installed where a 4 KiB mapping once lived + /// — the kernel's map refuses with `AlreadyMapped` and fail-stops. + import_floor: u64, } +/// Imports stop here, refused typed, rather than reach an IOVA the domain +/// cannot address — where the kernel's map would fail-stop. `1 << 39` is the +/// narrowest second-level address width in use (three-level VT-d / AMD-Vi). +const IMPORT_IOVA_CEILING: u64 = 1 << 39; + impl DmaOwner { fn activate(&mut self) -> bool { let device = self.device.as_ref().expect("retired DMA device owner"); @@ -1980,8 +2115,11 @@ impl DeviceResources { /// serving more work. Only a /// healthy driver epoch may call this; terminal teardown never reactivates /// the requester. `device_idle` must reflect all backend admissions, not - /// only the released buffer. False defers retirement without dropping any - /// owner. A false return requires containment of the entire epoch. + /// only the released buffer. A released import whose own proof lets it + /// retire live (an exact export, a `Quiet` storage target) retires even + /// with the device busy, and one whose live retirement is refused stays + /// owned for the next call ([`Self::deferred_retirements`]) without + /// failing. A false return requires containment of the entire epoch. pub fn resume_retired_dma(&self, device_idle: bool) -> bool { let Some(dma) = self.dma.as_ref() else { return false; @@ -1991,7 +2129,8 @@ impl DeviceResources { &mut imports, device_idle, |owner| owner.borrow().released, - |owner| owner.borrow_mut().retire(), + |owner| owner.borrow().retires_live(), + |owner| owner.borrow_mut().retire_released(), || match (dma.device.as_ref(), dma.window.as_ref()) { (Some(device), Some(window)) => device.activate_window(window), _ => false, @@ -2000,6 +2139,56 @@ impl DeviceResources { healthy && dma.retire_pool_loans() } + /// Retire one imported window while the device keeps running — the + /// per-window counterpart of [`resume_retired_dma`](Self::resume_retired_dma), + /// keyed on the engine's proof instead of device idleness + /// (`res/STORAGE_TRANSPORT.md` §11). Only the buffer's own mapping goes: + /// the requester's latch stays open and every other window stays live, + /// so nothing needs reactivating. + /// + /// [`QuietRetire::Deferred`] keeps the allocation owned, marked released + /// and still covered by its proof: the next + /// [`resume_retired_dma`](Self::resume_retired_dma), idle or not, retries + /// the same live retirement. + pub fn retire_quiet(&self, buffer: Quiet) -> QuietRetire { + let buffer = buffer.into_inner(); + let Some(dma) = self.dma.as_ref() else { + return QuietRetire::Refused(buffer); + }; + let CapBacking::Imported(owner) = &buffer.backing else { + return QuietRetire::Refused(buffer); + }; + let mut imports = dma.imports.borrow_mut(); + let Some(index) = imports.iter().position(|entry| Rc::ptr_eq(entry, owner)) else { + return QuietRetire::Refused(buffer); + }; + if owner.borrow().released { + return QuietRetire::Refused(buffer); + } + if owner.borrow_mut().retire_live() { + imports.remove(index); + QuietRetire::Retired + } else { + let mut owner = owner.borrow_mut(); + owner.released = true; + owner.quiet = true; + QuietRetire::Deferred + } + } + + /// Released imports still owned because their retirement has not + /// completed: live ones [`Self::resume_retired_dma`] retries on every + /// call, the rest at the next idle boundary. + pub fn deferred_retirements(&self) -> usize { + self.dma.as_ref().map_or(0, |dma| { + dma.imports + .borrow() + .iter() + .filter(|owner| owner.borrow().released) + .count() + }) + } + /// Reserve loan custody before the backend removes its stage-1 mapping. /// Call only through the backend's guarded detach preparation callback. pub fn prepare_buffer_export( @@ -2263,6 +2452,30 @@ impl DeviceResources { &mut self, supply: DmaImport, device_write: bool, + ) -> Result { + self.import_dma_with(supply, device_write, true) + } + + /// [`import_dma`](Self::import_dma) for the device alone: a lent route is + /// published into the device's domain and nothing is mapped into this + /// task, so the returned buffer's [`cpu`](CapBuf) pointer is null and the + /// consumer must never read or write it through the CPU. For a streaming + /// target — the device fills it, its owner reads it — the importing + /// service's address space and page tables then stay independent of how + /// large the client's allocation is. + pub fn import_dma_device( + &mut self, + supply: DmaImport, + device_write: bool, + ) -> Result { + self.import_dma_with(supply, device_write, false) + } + + fn import_dma_with( + &mut self, + supply: DmaImport, + device_write: bool, + cpu_view: bool, ) -> Result { if matches!(&supply, DmaImport::Lease(lease) if !lease.is_returned()) { return Err(IntakeError::DmaUnavailable); @@ -2270,6 +2483,7 @@ impl DeviceResources { let dma = self.dma.as_mut().ok_or(IntakeError::DmaUnavailable)?; let device = dma.device.as_ref().ok_or(IntakeError::DmaUnavailable)?; let len = supply.len(); + let align = supply.align(); let bytes = usize::try_from(len).map_err(|_| IntakeError::DmaUnavailable)?; if len == 0 || len % PAGE != 0 { return Err(IntakeError::DmaUnavailable); @@ -2285,6 +2499,7 @@ impl DeviceResources { let owner = Rc::try_new(RefCell::new(ImportedAllocation { released: false, return_requested: false, + quiet: false, returned: None, window: None, })) @@ -2295,7 +2510,8 @@ impl DeviceResources { .chain(dma.extensions.iter().map(|entry| &entry.0)) .map(|window| window.iova() + window.len()) .max() - .ok_or(IntakeError::DmaUnavailable)?; + .ok_or(IntakeError::DmaUnavailable)? + .max(dma.import_floor); for imported in imports.iter() { if let Some(window) = imported.borrow().window.as_ref() { end = end.max( @@ -2307,12 +2523,18 @@ impl DeviceResources { } } let iova = end - .checked_add((2 << 20) - 1) + .checked_add(align - 1) .ok_or(IntakeError::DmaUnavailable)? - & !((2 << 20) - 1); - iova.checked_add(len).ok_or(IntakeError::DmaUnavailable)?; + & !(align - 1); + let import_end = iova.checked_add(len).ok_or(IntakeError::DmaUnavailable)?; + if import_end > IMPORT_IOVA_CEILING { + return Err(IntakeError::DmaUnavailable); + } + // Claimed before the map is attempted: a map that fails part way + // may have touched the range, so it is never offered again either. + dma.import_floor = import_end; let window = device - .import_window(supply, iova, device_write) + .import_window(supply, iova, device_write, cpu_view) .map_err(|fault| IntakeError::DmaMap { code: fault.code() })?; let cpu = window.cpu(); owner.borrow_mut().window = Some(window); @@ -3170,6 +3392,7 @@ fn realize_host_resources( loans: RefCell::new(Vec::new()), _pool: pool, device: Some(device), + import_floor: 0, }), pool: Some(cap_pool), bars, @@ -3296,6 +3519,7 @@ mod tests { &mut owners, false, |released| *released, + |_| false, |_| panic!("unrelated device work excludes outer retirement"), || panic!("deferred retirement must not touch activation") )); @@ -3308,6 +3532,7 @@ mod tests { &mut owners, true, |released| *released, + |_| false, |_| { retired.set(retired.get() + 1); true @@ -3322,6 +3547,7 @@ mod tests { &mut owners, true, |released| *released, + |_| false, |_| panic!("live import must not retire"), || panic!("no retirement means no activation") )); @@ -3331,6 +3557,7 @@ mod tests { true, |released| *released, |_| false, + |_| false, || panic!("refused retirement requires containment") )); assert_eq!( @@ -3343,6 +3570,7 @@ mod tests { &mut owners, true, |released| *released, + |_| false, |_| true, || false ), @@ -3351,6 +3579,56 @@ mod tests { assert_eq!(owners, [false]); } + /// A window carrying its own quiescence proof retires with the device + /// busy, and a refusal of that live retirement keeps its owner without + /// failing the requester (`res/LIFECYCLE_HARDENING.md` §6). + #[test] + fn proven_imports_retire_live_and_a_refusal_never_fails_the_requester() { + let arena = test_arena::TestArena::<4096>::new(); + let mut owners = Vec::new_in(&arena); + // (released, proven live, id) + owners.extend([(true, true, 0u8), (true, false, 1), (false, true, 2), (true, true, 3)]); + let refused = core::cell::Cell::new(3u8); + // A busy device: only released, proven windows are attempted. The + // refused one stays owned, and the latch is left alone. + assert!(super::retire_imports_and_resume( + &mut owners, + false, + |owner| owner.0, + |owner| owner.1, + |owner| owner.2 != refused.get(), + || panic!("a running requester is never reactivated") + )); + assert_eq!(owners, [(true, false, 1), (false, true, 2), (true, true, 3)]); + // The idle boundary retires the ordinary release and reactivates; + // the proven window refused again is still no failure. + let activated = core::cell::Cell::new(false); + assert!(super::retire_imports_and_resume( + &mut owners, + true, + |owner| owner.0, + |owner| owner.1, + |owner| owner.2 != refused.get(), + || { + activated.set(true); + true + } + )); + assert!(activated.get()); + assert_eq!(owners, [(false, true, 2), (true, true, 3)]); + // Its retirement completes on a later pass, busy or not. + refused.set(u8::MAX); + assert!(super::retire_imports_and_resume( + &mut owners, + false, + |owner| owner.0, + |owner| owner.1, + |owner| owner.2 != refused.get(), + || panic!("a live retirement needs no reactivation") + )); + assert_eq!(owners, [(false, true, 2)]); + } + use super::*; #[test] diff --git a/native/crates/driver/src/lib.rs b/native/crates/driver/src/lib.rs index c5cbf85..b4f32de 100644 --- a/native/crates/driver/src/lib.rs +++ b/native/crates/driver/src/lib.rs @@ -56,6 +56,7 @@ pub mod net; pub mod ring; pub mod seam; pub mod stats; +pub mod stor; /// The completion-wait seam — a `Completion` probe plus an explicitly bounded /// spin for bring-up and kernel-internal tests. Userland waits on IRQ routes. pub mod wait; diff --git a/native/crates/driver/src/ring.rs b/native/crates/driver/src/ring.rs index 7991ded..bba83db 100644 --- a/native/crates/driver/src/ring.rs +++ b/native/crates/driver/src/ring.rs @@ -80,6 +80,14 @@ impl SubRing { } } + /// Start over at slot 0 with the producer cycle set — the state a device + /// expects after its queue is created again (an NVMe controller reset). + /// Entries past the tail are never read, so the slots stay as they are. + pub fn reset(&mut self) { + self.tail = 0; + self.cycle = true; + } + /// Usable slots — `capacity` (Modulo) or `capacity - 1` (LinkTrb, the /// last slot is the Link entry). pub fn usable(&self) -> u16 { @@ -207,11 +215,31 @@ impl CompRing { } } + /// Start over at slot 0 expecting phase `true`, with every slot zeroed — + /// the state a device expects after its queue is created again. The + /// zeroing matters: a slot left from before would carry a validity bit + /// that reads as a fresh completion on the next pass. + pub fn reset(&mut self) { + let bytes = self.capacity as usize * size_of::(); + // SAFETY: `buf` is this ring's own device-visible buffer of at least + // `capacity` entries; the device writes it only while its queue + // exists, and the caller resets only once the controller dropped it. + unsafe { core::ptr::write_bytes(self.buf.cpu(), 0, bytes.min(self.buf.len())) }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Release); + self.head = 0; + self.phase = true; + } + /// The consumer dequeue index. pub fn head(&self) -> u16 { self.head } + /// Slots in the ring. + pub fn capacity(&self) -> u16 { + self.capacity + } + /// The expected validity phase for the head slot. pub fn phase(&self) -> bool { self.phase diff --git a/native/crates/driver/src/seam.rs b/native/crates/driver/src/seam.rs index 4068f55..f26a5bb 100644 --- a/native/crates/driver/src/seam.rs +++ b/native/crates/driver/src/seam.rs @@ -667,6 +667,43 @@ pub trait DmaRead: Dma { impl DmaRead for T {} +/// A DMA target its engine handed back with the proof that retiring it needs +/// no device idleness: no command it popped still names the target, the +/// completion of every one that did has been reaped, and no entry not yet +/// popped can reach it any more. +/// +/// Only an engine mints one ([`StorEngine::release_target`]). It is the one +/// input per-window DMA retirement takes, in place of the whole-requester +/// quiescence an exact retirement otherwise waits for. The proof is the +/// driver's own correctness: the kernel's live retirement completes the +/// window's invalidation before the memory can move on, so a proof that +/// were wrong costs a recorded IOMMU fault, not a stray write. +/// +/// [`StorEngine::release_target`]: crate::stor::engine::StorEngine::release_target +#[must_use = "a quiet target is retired, not dropped"] +pub struct Quiet(T); + +impl Quiet { + pub(crate) fn witness(target: T) -> Self { + Self(target) + } + + /// The same memory in another wrapper — a queue's buffer type to the + /// allocation beneath it. + pub fn map(self, f: impl FnOnce(T) -> U) -> Quiet { + Quiet(f(self.0)) + } + + pub fn get(&self) -> &T { + &self.0 + } + + /// Drop the proof, keeping the target. + pub fn into_inner(self) -> T { + self.0 + } +} + /// An allocation returned by a DMA release attempt. These states differ in /// whether ordinary buffer destruction is known to be safe. #[derive(Debug)] diff --git a/native/crates/driver/src/stats.rs b/native/crates/driver/src/stats.rs index ac97ead..9f9a687 100644 --- a/native/crates/driver/src/stats.rs +++ b/native/crates/driver/src/stats.rs @@ -12,4 +12,8 @@ pub struct RingStats { pub completed: u64, /// Completions with a non-success status. pub errors: u64, + /// Completions naming an id with no command in flight — a device + /// answering twice, or answering something never submitted. Dropped, + /// never delivered, and counted so the books show it. + pub stray: u64, } diff --git a/native/crates/driver/src/stor/engine.rs b/native/crates/driver/src/stor/engine.rs new file mode 100644 index 0000000..ee7d64b --- /dev/null +++ b/native/crates/driver/src/stor/engine.rs @@ -0,0 +1,2113 @@ +//! **L1 — the transport engine** (`res/STORAGE_TRANSPORT.md` §2–§5). +//! +//! One engine value is one worker's whole data plane: the rings attached to +//! it, their grants and targets, the submissions in flight, and the queue +//! set it drives. [`StorEngine::advance`] is the only data-path entry. It +//! never waits: it reaps what the device finished, fills each class up to +//! its budget in the policy's order, publishes each touched submission queue +//! once, and commits each touched ring's completion channel once. +//! +//! ## One submission, one completion +//! +//! A client submission becomes an **op**. The engine pops a submission only +//! while the ring holds a completion credit — ops open on the ring are +//! strictly fewer than the free slots in its completion channel — so the +//! completion an op owes can always be posted. An op is issued as one or +//! more device **commands** (split at grant-run boundaries and the device's +//! transfer ceiling), a piece at a time as budget and queue room allow; its +//! completion posts when every issued piece has come back. +//! +//! ## Classes, budgets, fairness +//! +//! The [`ClassPolicy`] maps each class to a hardware submission queue and +//! gives it a command and byte budget. [`advance`](StorEngine::advance) +//! fills classes in the policy's order, and within a class serves rings +//! round robin with a byte quantum, so one ring's stream of large reads +//! cannot keep another ring's small reads waiting. Each ring issues its +//! current op in a class to the end before popping the next — order within +//! a class channel is preserved, and at most one op per ring per class is +//! partially issued. +//! +//! ## Revocation and departure +//! +//! Revoking a grant stops new submissions naming it (they fail +//! [`StorStatus::Revoked`]) and stops issuing the rest of any op already +//! using it; the grant's slot is freed and the authority told +//! ([`StorHost::drained`]) only once nothing that named it is in flight. +//! Detaching a ring stops popping; [`StorHost::departed`] fires once its +//! last op completes, after which its targets can be taken back and the +//! ring retired. + +use kore_abi::stor::{ + Class, CqState, GrantId, RING_DEPARTED, RING_LIVE, RWF_DSYNC, RWF_KNOWN, RWF_NOWAIT, + StorStatus, TIMELINES, +}; +use kore_abi::uring::{IoUringCqe, IoUringSqe, op}; +use kore_ring::stor::ServiceRing; +use zerocopy::FromZeros; + +use super::grant::{ClassSet, Grant, Rights}; +use crate::seam::Quiet; +use super::timeline::Timeline; +use super::{QueueSet, Tag}; +use crate::block::{BlockError, DeviceCommand, DeviceCompletion, LatencyBooks, Liveness}; + +/// A ring as the engine names it: slot plus generation. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct RingId { + slot: u16, + generation: u16, +} + +impl RingId { + /// The ring's slot. + pub const fn slot(self) -> u16 { + self.slot + } +} + +/// A command and byte budget for one class. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Budget { + /// Device commands the class may have in flight. + pub commands: u32, + /// Bytes the class may have in flight. + pub bytes: u64, +} + +/// How classes map onto a queue set and share it. +pub trait ClassPolicy { + /// The order `advance` fills classes in. + const ORDER: [Class; Class::COUNT]; + /// Bytes one ring may issue in a class before the next ring's turn. + const QUANTUM_BYTES: u64; + + /// The hardware submission queue `class` rides. + fn sq(class: Class) -> usize; + /// `class`'s in-flight budget. + fn budget(class: Class) -> Budget; + /// The deadline a submission of `class` gets when it names none. + fn deadline_ns(class: Class) -> u64; +} + +/// Every class on one hardware queue — the policy a [`Single`](super::Single) +/// queue set runs. Commit fills first, then Demand, then Stream; the Stream +/// byte budget keeps bulk reads from occupying the whole queue. +pub struct OneQueue; + +impl ClassPolicy for OneQueue { + const ORDER: [Class; Class::COUNT] = [Class::Commit, Class::Demand, Class::Stream]; + const QUANTUM_BYTES: u64 = 1 << 20; + + fn sq(_class: Class) -> usize { + 0 + } + + fn budget(class: Class) -> Budget { + match class { + Class::Commit => Budget { + commands: 32, + bytes: 4 << 20, + }, + Class::Demand => Budget { + commands: 64, + bytes: 32 << 20, + }, + Class::Stream => Budget { + commands: 128, + bytes: 16 << 20, + }, + } + } + + fn deadline_ns(class: Class) -> u64 { + match class { + // A metadata write under device garbage collection can take + // seconds; the budget matches the filesystem's stall budget. + Class::Commit => 30_000_000_000, + Class::Demand | Class::Stream => 30_000_000_000, + } + } +} + +/// One hardware submission queue per class — Demand on 0, Commit on 1, +/// Stream on 2 — over one completion queue: the policy a queue group runs +/// (`res/STORAGE_TRANSPORT.md` §3). A demand read never sits behind bulk +/// work in the device's own queue, only behind the arbitration burst the +/// controller takes from each non-empty queue in turn; a controller under +/// weighted round robin serves them urgent, high, medium in that order. +/// Budgets and the fill order are [`OneQueue`]'s; only the queue each class +/// rides differs. +pub struct ThreeClass; + +impl ClassPolicy for ThreeClass { + const ORDER: [Class; Class::COUNT] = OneQueue::ORDER; + const QUANTUM_BYTES: u64 = OneQueue::QUANTUM_BYTES; + + fn sq(class: Class) -> usize { + match class { + Class::Demand => 0, + Class::Commit => 1, + Class::Stream => 2, + } + } + + fn budget(class: Class) -> Budget { + OneQueue::budget(class) + } + + fn deadline_ns(class: Class) -> u64 { + OneQueue::deadline_ns(class) + } +} + +/// The side effects a worker's host performs for the engine. +pub trait StorHost { + /// `ring`'s completion channel was committed and its consumer had + /// parked: send the ring's notice. + fn kick(&mut self, ring: RingId); + /// Nothing naming `grant` on `ring` is in flight any more; its slot is + /// free and the blocks it named may be reused. + fn drained(&mut self, ring: RingId, grant: GrantId); + /// Detached `ring` has nothing in flight; its targets can be taken back + /// and the ring retired. + fn departed(&mut self, ring: RingId); + /// An op passed its deadline while the device still held part of it. + /// Reported once per op; recovery (abort, reset) is the host's. + fn expired(&mut self, expiry: Expiry); +} + +/// An op that outlived its deadline. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Expiry { + /// The ring that submitted it. + pub ring: RingId, + /// The submission's token. + pub user_data: u64, + /// Its class. + pub class: Class, + /// Device commands of it still outstanding. + pub outstanding: u16, +} + +/// Which way the device moves bytes through a target. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TargetAccess { + /// Reads land in it: the device writes. + ReadInto, + /// Writes come from it: the device reads. + WriteFrom, + /// Both. + Both, +} + +impl TargetAccess { + const fn admits_read_into(self) -> bool { + matches!(self, Self::ReadInto | Self::Both) + } + + const fn admits_write_from(self) -> bool { + matches!(self, Self::WriteFrom | Self::Both) + } +} + +/// Why a control-plane call was refused. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StorRefusal { + /// The id names nothing live. + Stale, + /// A fixed table is full. + Full, + /// Something still in flight forbids it. + Busy, + /// The ring is detached or not yet idle for this step. + Departed, + /// The request is malformed (a target too large to address, a policy + /// that names a queue the set does not have). + Invalid, +} + +/// Whether a revoke or detach finished at once. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Settle { + /// Nothing was in flight; done now, no host callback follows. + Now, + /// Something is in flight; the host callback fires when it drains. + Later, +} + +/// Engine counters. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct EngineStats { + /// Submissions popped. + pub submissions: u64, + /// Completions posted. + pub completions: u64, + /// Submissions that failed validation. + pub refused: u64, + /// Device commands staged. + pub commands: u64, + /// Device completions naming no live command — never acted on. + pub stale_completions: u64, + /// Ops reported past their deadline. + pub expired: u64, + /// Completions a timeline could not record — each one stops that + /// timeline's fence for good. The pop's window check keeps this zero. + pub timeline_refusals: u64, + /// Device commands the queue set refused to stage. Transient refusals + /// (a full queue, no list page free) retry on the next pass; one that + /// only ever refuses stalls its op, and this count is how that shows. + pub stage_refusals: u64, + /// The last stage refusal's error. + pub last_stage_error: Option, + /// Commands of expired ops the device agreed to abort. + pub aborts: u64, + /// Commands of expired ops the device would not (or could not) abort. + pub aborts_refused: u64, + /// Controller resets: an expired op still held after its abort grace. + pub resets: u64, + /// Resets that did not bring the device back. + pub resets_failed: u64, + /// Completion latency per class, device command granularity. + pub latency: [LatencyBooks; Class::COUNT], +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum GrantState { + Live, + Revoking, +} + +struct GrantSlot { + generation: u16, + grant: Option<(Grant, GrantState)>, + /// Ops naming it that have not completed. + ops: u32, +} + +struct TargetSlot { + target: T, + bytes: u64, + access: TargetAccess, + /// Device commands using it that have not completed. + commands: u32, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum RingState { + Live, + Departed, +} + +struct RingSlot<'w, T, const GRANTS: usize, const TARGETS: usize> { + window: ServiceRing<'w>, + state: RingState, + classes: ClassSet, + grants: [GrantSlot; GRANTS], + targets: [Option>; TARGETS], + timelines: [Timeline; TIMELINES], + /// The op this ring is issuing in each class, if one is partial. + current: [Option; Class::COUNT], + /// Ops popped and not yet completed. + open: u32, + /// Completions posted since attach. + completed: u64, + /// Something was posted this advance; commit the channel. + dirty: bool, + /// `departed` was reported. + departure_reported: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Kind { + Read, + Write, + Flush, + Discard, +} + +#[derive(Clone, Copy)] +struct Op { + ring: u16, + class: Class, + kind: Kind, + user_data: u64, + grant: u16, + target: Option, + timeline: u16, + seq: u64, + /// Block offset inside the grant of the next piece to issue. + next_block: u64, + /// Blocks still to issue. + left_blocks: u64, + /// Byte offset inside the target of the next piece. + next_target: u64, + /// Total bytes the op moves (the success `res`). + bytes: u64, + /// Device commands issued and not completed. + outstanding: u16, + /// Nothing more will be issued. + issued: bool, + error: Option, + deadline_ns: Option, + expiry_reported: bool, + /// Past its deadline and aborted: still held after this, the controller + /// is reset. + abort_deadline_ns: Option, +} + +#[derive(Clone, Copy)] +struct Command { + op: u16, + bytes: u64, + submitted_ns: Option, + depth: u16, +} + +/// The transport engine: one worker's data plane. +/// +/// `RINGS` rings of `GRANTS` grants and `TARGETS` targets each; `OPS` +/// submissions and `COMMANDS` device commands in flight across all of them. +pub struct StorEngine< + 'w, + Q: QueueSet, + P: ClassPolicy, + const RINGS: usize, + const GRANTS: usize, + const TARGETS: usize, + const OPS: usize, + const COMMANDS: usize, +> { + queues: Q, + rings: [Option>; RINGS], + ring_generations: [u16; RINGS], + ops: [Option; OPS], + commands: [Option; COMMANDS], + /// Commands and bytes in flight per class. + in_flight: [(u32, u64); Class::COUNT], + /// Round-robin start per class. + cursor: [usize; Class::COUNT], + /// Submission queues with staged, unpublished commands. + staged: u64, + next_deadline_scan: u64, + stats: EngineStats, + _policy: core::marker::PhantomData

, +} + +/// How often the in-flight table is scanned for expired deadlines. +const DEADLINE_SCAN_NS: u64 = 1_000_000; +/// Completions reaped per device drain. +const REAP_BATCH: usize = 32; + +impl< + 'w, + Q: QueueSet, + P: ClassPolicy, + const RINGS: usize, + const GRANTS: usize, + const TARGETS: usize, + const OPS: usize, + const COMMANDS: usize, +> StorEngine<'w, Q, P, RINGS, GRANTS, TARGETS, OPS, COMMANDS> +{ + /// An engine driving `queues`. Refuses a policy that names a queue the + /// set does not have, tables too large for their index types, or a + /// device with a zero block size. + /// + /// The engine is tens of KiB of fixed tables; a caller whose stack is + /// tight builds it in place with [`init_in`](Self::init_in) instead. + pub fn new(queues: Q) -> Result { + let mut place = core::mem::MaybeUninit::uninit(); + Self::init_in(&mut place, queues)?; + // SAFETY: `init_in` returned `Ok`, so every field is written. + Ok(unsafe { place.assume_init() }) + } + + /// [`new`](Self::new), written straight into `place` — a heap slot, say + /// — so no engine-sized temporary ever exists on the caller's stack. + /// Each table is written element by element for the same reason. On + /// `Err`, `place` is untouched and still uninitialized. + pub fn init_in( + place: &mut core::mem::MaybeUninit, + queues: Q, + ) -> Result<&mut Self, (Q, StorRefusal)> { + let policy_fits = Class::ALL.iter().all(|&class| P::sq(class) < Q::SQS); + let tables_fit = RINGS <= 64 + && GRANTS <= usize::from(u16::MAX) + && TARGETS <= usize::from(u16::MAX) + && OPS <= usize::from(u16::MAX) + && COMMANDS <= u32::MAX as usize + && Q::SQS <= 64; + if !policy_fits || !tables_fit || queues.geometry().block_size == 0 { + return Err((queues, StorRefusal::Invalid)); + } + let this = place.as_mut_ptr(); + // SAFETY: `this` is valid for writes of `Self` (it comes from the + // exclusive `place`); each write targets one field, or one element + // of an array field, by raw place projection — no reference to + // uninitialized memory is formed — and every field is written + // exactly once before `place` is assumed initialized. + unsafe { + use core::ptr::addr_of_mut; + addr_of_mut!((*this).queues).write(queues); + let rings = addr_of_mut!((*this).rings).cast::>>(); + for i in 0..RINGS { + rings.add(i).write(None); + } + addr_of_mut!((*this).ring_generations).write([0; RINGS]); + let ops = addr_of_mut!((*this).ops).cast::>(); + for i in 0..OPS { + ops.add(i).write(None); + } + let commands = addr_of_mut!((*this).commands).cast::>(); + for i in 0..COMMANDS { + commands.add(i).write(None); + } + addr_of_mut!((*this).in_flight).write([(0, 0); Class::COUNT]); + addr_of_mut!((*this).cursor).write([0; Class::COUNT]); + addr_of_mut!((*this).staged).write(0); + addr_of_mut!((*this).next_deadline_scan).write(0); + addr_of_mut!((*this).stats).write(EngineStats::default()); + addr_of_mut!((*this)._policy).write(core::marker::PhantomData); + Ok(place.assume_init_mut()) + } + } + + /// Take the queue set back, once no ring is attached. + pub fn into_queues(self) -> Result { + if self.rings.iter().any(Option::is_some) { + return Err(self); + } + Ok(self.queues) + } + + /// The queue set. + pub fn queues(&self) -> &Q { + &self.queues + } + + /// The queue set, mutably — a control step, a test's knobs. + pub fn queues_mut(&mut self) -> &mut Q { + &mut self.queues + } + + /// Counters. + pub fn stats(&self) -> &EngineStats { + &self.stats + } + + /// The device's out-of-band liveness. + pub fn liveness(&mut self) -> Liveness { + self.queues.liveness() + } + + /// Device commands in flight, all classes. + pub fn commands_in_flight(&self) -> usize { + self.commands.iter().flatten().count() + } + + /// Submissions popped and not yet completed, all rings. + pub fn ops_open(&self) -> usize { + self.ops.iter().flatten().count() + } + + /// Commit to sleeping: the consumer's half of every live ring's + /// submission Dekker pair (`kore_ring::event`). + /// + /// `true` means every submission channel is empty and parked, and the + /// engine holds no op — so nothing will wake its host except a producer + /// that sees `parked` and sends its notice. `false` means work is + /// pending (an op open, or a record queued or published after the + /// channel was read): advance again instead of sleeping. A `false` may + /// leave some channels parked; the cost is one spurious notice. + /// + /// Each channel's wake word is read **before** its queue is checked: + /// a producer that commits between the two moves the word, so + /// [`park`](kore_ring::event::EventSink::park) refuses, and one that + /// commits after the park sees the intent and owes the notice. + pub fn park_idle(&mut self) -> bool { + if self.ops_open() != 0 || self.commands_in_flight() != 0 { + return false; + } + for ring in self.rings.iter().flatten() { + if ring.state != RingState::Live { + continue; + } + for sink in &ring.window.sq { + let seen = sink.seq(); + if sink.outstanding() != 0 || !sink.park(seen) { + return false; + } + } + } + true + } + + // ── control plane ─────────────────────────────────────────────────── + + /// Attach a formatted window's service halves as a new ring admitting + /// `classes`. + pub fn attach( + &mut self, + window: ServiceRing<'w>, + classes: ClassSet, + ) -> Result, StorRefusal)> { + let Some(slot) = self.rings.iter().position(Option::is_none) else { + return Err((window, StorRefusal::Full)); + }; + let mut window = window; + window.cq.replace_state(CqState { + status: RING_LIVE, + in_flight: 0, + completed: 0, + }); + let _ = window.cq.commit(); + self.rings[slot] = Some(RingSlot { + window, + state: RingState::Live, + classes, + grants: core::array::from_fn(|_| GrantSlot { + generation: 0, + grant: None, + ops: 0, + }), + targets: core::array::from_fn(|_| None), + timelines: [Timeline::default(); TIMELINES], + current: [None; Class::COUNT], + open: 0, + completed: 0, + dirty: false, + departure_reported: false, + }); + Ok(RingId { + slot: slot as u16, + generation: self.ring_generations[slot], + }) + } + + fn ring_mut( + &mut self, + id: RingId, + ) -> Result<&mut RingSlot<'w, Q::Target, GRANTS, TARGETS>, StorRefusal> { + let slot = usize::from(id.slot); + if self.ring_generations.get(slot) != Some(&id.generation) { + return Err(StorRefusal::Stale); + } + self.rings[slot].as_mut().ok_or(StorRefusal::Stale) + } + + /// Install `grant` into `ring`. + pub fn install(&mut self, ring: RingId, grant: Grant) -> Result { + let ring = self.ring_mut(ring)?; + if ring.state != RingState::Live { + return Err(StorRefusal::Departed); + } + let slot = ring + .grants + .iter() + .position(|slot| slot.grant.is_none() && slot.ops == 0) + .ok_or(StorRefusal::Full)?; + let entry = &mut ring.grants[slot]; + entry.grant = Some((grant, GrantState::Live)); + Ok(GrantId::new(slot as u16, entry.generation)) + } + + /// Revoke `grant`. [`Settle::Now`] when nothing names it; otherwise + /// [`StorHost::drained`] fires once the last op naming it completes, and + /// the slot is reusable only then. + pub fn revoke(&mut self, ring_id: RingId, grant: GrantId) -> Result { + let ring = self.ring_mut(ring_id)?; + let slot = ring + .grants + .get_mut(usize::from(grant.slot())) + .ok_or(StorRefusal::Stale)?; + if slot.generation & GrantId::MAX_GENERATION != grant.generation() { + return Err(StorRefusal::Stale); + } + match slot.grant.as_mut() { + None => Err(StorRefusal::Stale), + Some((_, state)) if *state == GrantState::Revoking => Ok(Settle::Later), + Some((_, state)) => { + if slot.ops == 0 { + slot.grant = None; + slot.generation = slot.generation.wrapping_add(1); + Ok(Settle::Now) + } else { + *state = GrantState::Revoking; + Ok(Settle::Later) + } + } + } + } + + /// Adopt `target` into `ring` for `access`. The target is the ring's + /// until [`release_target`](Self::release_target) hands it back. + pub fn adopt_target( + &mut self, + ring: RingId, + target: Q::Target, + access: TargetAccess, + ) -> Result { + let ring = match self.ring_mut(ring) { + Ok(ring) => ring, + Err(refusal) => return Err((target, refusal)), + }; + if ring.state != RingState::Live { + return Err((target, StorRefusal::Departed)); + } + let Some(slot) = ring.targets.iter().position(Option::is_none) else { + return Err((target, StorRefusal::Full)); + }; + let bytes = Q::target_bytes(&target); + ring.targets[slot] = Some(TargetSlot { + target, + bytes, + access, + commands: 0, + }); + Ok(slot as u16) + } + + /// Target `slot` of `ring`, mutably — its owner's CPU view, refused while + /// any command uses it. + pub fn target_mut(&mut self, ring: RingId, slot: u16) -> Result<&mut Q::Target, StorRefusal> { + let ring = self.ring_mut(ring)?; + match ring.targets.get_mut(usize::from(slot)) { + Some(Some(entry)) if entry.commands == 0 => Ok(&mut entry.target), + Some(Some(_)) => Err(StorRefusal::Busy), + _ => Err(StorRefusal::Stale), + } + } + + /// Submissions `ring` has popped and not yet completed. + pub fn open(&self, ring: RingId) -> Option { + let slot = usize::from(ring.slot); + if self.ring_generations.get(slot) != Some(&ring.generation) { + return None; + } + self.rings[slot].as_ref().map(|ring| ring.open) + } + + /// Hand target `slot` back, once no command uses it — with the proof + /// that its DMA mapping may retire while the device runs on: a command + /// counts against its target from the pop until its completion is reaped, + /// and the emptied slot is unreachable to any entry not yet popped. + pub fn release_target(&mut self, ring: RingId, slot: u16) -> Result, StorRefusal> { + let ring = self.ring_mut(ring)?; + let entry = ring + .targets + .get_mut(usize::from(slot)) + .ok_or(StorRefusal::Stale)?; + match entry { + None => Err(StorRefusal::Stale), + Some(target) if target.commands != 0 => Err(StorRefusal::Busy), + Some(_) => Ok(Quiet::witness(entry.take().expect("checked occupied").target)), + } + } + + /// Detach `ring`: pop nothing more, stop issuing its partial ops, let + /// what the device holds finish. [`Settle::Now`] when nothing was open. + pub fn detach(&mut self, ring_id: RingId) -> Result { + let ring_slot = usize::from(ring_id.slot); + let ops = &mut self.ops; + let ring = { + if self.ring_generations.get(ring_slot) != Some(&ring_id.generation) { + return Err(StorRefusal::Stale); + } + self.rings[ring_slot].as_mut().ok_or(StorRefusal::Stale)? + }; + if ring.state == RingState::Departed { + return Ok(if ring.open == 0 { Settle::Now } else { Settle::Later }); + } + ring.state = RingState::Departed; + ring.window.cq.state(|state| state.status = RING_DEPARTED); + let _ = ring.window.cq.commit(); + // Partial ops stop issuing; what is out comes back. + for current in ring.current.iter_mut() { + if let Some(index) = current.take() + && let Some(op) = ops[usize::from(index)].as_mut() + { + op.issued = true; + op.error.get_or_insert(StorStatus::Cancelled); + } + } + if ring.open == 0 { + ring.departure_reported = true; + Ok(Settle::Now) + } else { + // Any partial op that just became fully issued with nothing out + // completes on the next advance. + Ok(Settle::Later) + } + } + + /// Retire a detached, idle ring whose targets have all been released, + /// handing its window halves back. + pub fn retire(&mut self, ring_id: RingId) -> Result, StorRefusal> { + let ring = self.ring_mut(ring_id)?; + if ring.state != RingState::Departed || ring.open != 0 { + return Err(StorRefusal::Busy); + } + if ring.targets.iter().any(Option::is_some) { + return Err(StorRefusal::Busy); + } + let slot = usize::from(ring_id.slot); + let ring = self.rings[slot].take().expect("checked occupied"); + self.ring_generations[slot] = self.ring_generations[slot].wrapping_add(1); + Ok(ring.window) + } + + // ── data plane ────────────────────────────────────────────────────── + + /// Drive the data plane once: reap, fill every class in order, publish, + /// reap again, report expiries, commit every touched ring. Never waits. + pub fn advance(&mut self, host: &mut H) { + let now = self.queues.clock_ns(); + self.reap(host); + for class in P::ORDER { + self.fill(class, now); + } + self.publish_staged(); + self.reap(host); + self.finish_idle_ops(host); + if let Some(now) = now + && now >= self.next_deadline_scan + { + self.next_deadline_scan = now.saturating_add(DEADLINE_SCAN_NS); + self.scan_deadlines(now, host); + } + self.commit_rings(host); + } + + fn publish_staged(&mut self) { + let mut staged = core::mem::take(&mut self.staged); + while staged != 0 { + let sq = staged.trailing_zeros() as usize; + staged &= staged - 1; + self.queues.publish(sq); + } + } + + fn commit_rings(&mut self, host: &mut H) { + for (slot, entry) in self.rings.iter_mut().enumerate() { + let Some(ring) = entry.as_mut() else { continue }; + if !ring.dirty { + continue; + } + ring.dirty = false; + let (open, completed) = (ring.open, ring.completed); + ring.window.cq.state(|state| { + state.in_flight = open; + state.completed = completed; + }); + if ring.window.cq.commit() { + host.kick(RingId { + slot: slot as u16, + generation: self.ring_generations[slot], + }); + } + } + } + + /// Fill `class`: rings round robin, each up to one quantum per turn, + /// until the class budget, the device queue, or every ring runs dry. + fn fill(&mut self, class: Class, now: Option) { + let sq = P::sq(class); + let budget = P::budget(class); + loop { + let mut progressed = false; + for turn in 0..RINGS { + let slot = (self.cursor[class.index()] + turn) % RINGS; + let mut quantum = P::QUANTUM_BYTES; + while quantum > 0 { + match self.step(slot, class, sq, budget, now) { + Step::Issued(bytes) => { + progressed = true; + quantum = quantum.saturating_sub(bytes.max(1)); + } + Step::Posted => progressed = true, + Step::Stop => break, + } + } + } + if RINGS > 0 { + self.cursor[class.index()] = (self.cursor[class.index()] + 1) % RINGS; + } + if !progressed { + break; + } + } + } + + /// One unit of progress for `slot` in `class`: issue the next piece of + /// its current op, or pop and plan its next submission. + fn step( + &mut self, + slot: usize, + class: Class, + sq: usize, + budget: Budget, + now: Option, + ) -> Step { + let Some(ring) = self.rings[slot].as_mut() else { + return Step::Stop; + }; + let op_index = match ring.current[class.index()] { + Some(index) => index, + None => { + if ring.state != RingState::Live || !ring.classes.contains(class) { + return Step::Stop; + } + // The completion credit: an op is popped only if the one + // completion it owes can be posted. + if ring.open >= ring.window.cq.capacity_left() { + return Step::Stop; + } + let Some(free_op) = self.ops.iter().position(Option::is_none) else { + return Step::Stop; + }; + // The timeline's window: a submission whose timeline already + // has a window's worth open waits, so every completion it + // owes lands inside the window and the fence keeps moving. + // A batch longer than the window drains through it in order. + let Some(next) = ring.window.sq[class.index()].peek() else { + return Step::Stop; + }; + if let Some(books) = usize::from(next.personality) + .checked_sub(1) + .and_then(|index| ring.timelines.get(index)) + && books.open() >= super::timeline::WINDOW + { + return Step::Stop; + } + let Some(sqe) = ring.window.sq[class.index()].pop() else { + return Step::Stop; + }; + self.stats.submissions += 1; + ring.open += 1; + match plan::( + ring, + slot as u16, + class, + &sqe, + self.queues.geometry(), + now, + budget, + self.in_flight[class.index()], + ) { + Ok(op) => { + self.ops[free_op] = Some(op); + ring.current[class.index()] = Some(free_op as u16); + free_op as u16 + } + Err((status, timeline)) => { + self.stats.refused += 1; + if !post(ring, sqe.user_data, status.res(), timeline, false) { + self.stats.timeline_refusals += 1; + } + ring.open -= 1; + self.stats.completions += 1; + return Step::Posted; + } + } + } + }; + // Issue the next piece of the current op. + let ops = &mut self.ops; + let op = ops[usize::from(op_index)].as_mut().expect("current op is live"); + let grant_state = ring.grants[usize::from(op.grant)].grant.map(|(_, state)| state); + if grant_state != Some(GrantState::Live) { + op.issued = true; + op.error.get_or_insert(StorStatus::Revoked); + } + if op.issued { + ring.current[class.index()] = None; + return Step::Posted; + } + let (in_cmds, in_bytes) = self.in_flight[class.index()]; + if in_cmds >= budget.commands || self.queues.room(sq) == 0 { + return Step::Stop; + } + let Some(tag) = self.commands.iter().position(Option::is_none) else { + return Step::Stop; + }; + let geometry = self.queues.geometry(); + let block = u64::from(geometry.block_size); + let (command, piece_blocks) = match op.kind { + Kind::Flush => (DeviceCommand::Flush, 0), + Kind::Read | Kind::Write | Kind::Discard => { + let (grant, _) = ring.grants[usize::from(op.grant)].grant.expect("grant is live"); + let ceiling = match op.kind { + Kind::Discard => u32::MAX, + _ => geometry.max_transfer_blocks, + }; + let max = op.left_blocks.min(u64::from(ceiling)) as u32; + let Some(piece) = grant.segment(op.next_block, max) else { + // Validated in range; unreachable unless the grant moved. + op.issued = true; + op.error.get_or_insert(StorStatus::Range); + ring.current[class.index()] = None; + return Step::Posted; + }; + let buffer_offset = op.next_target as usize; + let command = match op.kind { + Kind::Read => DeviceCommand::Read { + lba: piece.lba, + blocks: piece.blocks, + buffer_offset, + }, + Kind::Write => DeviceCommand::Write { + lba: piece.lba, + blocks: piece.blocks, + buffer_offset, + }, + _ => DeviceCommand::Discard { + lba: piece.lba, + blocks: piece.blocks, + }, + }; + (command, piece.blocks) + } + }; + let piece_bytes = u64::from(piece_blocks) * block; + let moves_data = matches!(op.kind, Kind::Read | Kind::Write); + // A piece bigger than the whole byte budget still goes alone, or a + // class whose budget is below one transfer could never move. + if moves_data && in_cmds > 0 && in_bytes.saturating_add(piece_bytes) > budget.bytes { + return Step::Stop; + } + let target = match op.target { + Some(t) => ring.targets[usize::from(t)].as_mut().map(|slot| &mut slot.target), + None => None, + }; + let submitted_ns = self.queues.clock_ns(); + if let Err(error) = self.queues.stage(sq, command, target, Tag(tag as u32)) { + self.stats.stage_refusals += 1; + self.stats.last_stage_error = Some(error); + return Step::Stop; + } + self.staged |= 1 << sq; + self.stats.commands += 1; + if let Some(t) = op.target { + if let Some(entry) = ring.targets[usize::from(t)].as_mut() { + entry.commands += 1; + } + } + self.commands[tag] = Some(Command { + op: op_index, + bytes: if moves_data { piece_bytes } else { 0 }, + submitted_ns, + depth: in_cmds.min(u32::from(u16::MAX)) as u16, + }); + self.in_flight[class.index()] = ( + in_cmds + 1, + in_bytes + if moves_data { piece_bytes } else { 0 }, + ); + op.outstanding += 1; + op.next_block += u64::from(piece_blocks); + op.next_target += piece_bytes; + // A flush is one piece of no blocks: it is fully issued once staged. + op.left_blocks -= if op.kind == Kind::Flush { + op.left_blocks + } else { + u64::from(piece_blocks) + }; + if op.left_blocks == 0 { + op.issued = true; + ring.current[class.index()] = None; + } + Step::Issued(piece_bytes) + } + + /// Drain the device's completions. + fn reap(&mut self, host: &mut H) { + let mut batch = [DeviceCompletion { + cookie: 0, + result: Ok(()), + }; REAP_BATCH]; + loop { + let n = self.queues.reap(&mut batch); + for completion in &batch[..n] { + self.complete_command(*completion, host); + } + if n < REAP_BATCH { + break; + } + } + } + + fn complete_command(&mut self, completion: DeviceCompletion, host: &mut H) { + let tag = completion.cookie as usize; + let Some(command) = self.commands.get_mut(tag).and_then(Option::take) else { + self.stats.stale_completions += 1; + return; + }; + let op_index = usize::from(command.op); + let Some(op) = self.ops[op_index].as_mut() else { + self.stats.stale_completions += 1; + return; + }; + let class = op.class; + let ring = self.rings[usize::from(op.ring)] + .as_mut() + .expect("an op's ring outlives it"); + let mut result = completion.result; + if let Some(t) = op.target + && let Some(entry) = ring.targets[usize::from(t)].as_mut() + { + entry.commands -= 1; + if result.is_ok() { + result = self.queues.finish(Some(&mut entry.target)); + } + } + if let Err(error) = result { + op.error.get_or_insert(status_of(error)); + } + op.outstanding -= 1; + let (cmds, bytes) = &mut self.in_flight[class.index()]; + *cmds -= 1; + *bytes -= command.bytes; + if let (Some(start), Some(now)) = (command.submitted_ns, self.queues.clock_ns()) { + self.stats.latency[class.index()].record(now.saturating_sub(start), command.depth); + } + if op.issued && op.outstanding == 0 { + self.finish_op(op_index, host); + } + } + + /// Ops made fully issued with nothing outstanding outside a completion + /// (a revoke or detach stopped them between pieces) complete here. + fn finish_idle_ops(&mut self, host: &mut H) { + for index in 0..OPS { + if let Some(op) = self.ops[index] + && op.issued + && op.outstanding == 0 + { + self.finish_op(index, host); + } + } + // A departed ring with nothing open reports once. + for (slot, entry) in self.rings.iter_mut().enumerate() { + if let Some(ring) = entry.as_mut() + && ring.state == RingState::Departed + && ring.open == 0 + && !ring.departure_reported + { + ring.departure_reported = true; + host.departed(RingId { + slot: slot as u16, + generation: self.ring_generations[slot], + }); + } + } + } + + fn finish_op(&mut self, index: usize, host: &mut H) { + let op = self.ops[index].take().expect("finishing a live op"); + let ring_slot = usize::from(op.ring); + let ring = self.rings[ring_slot] + .as_mut() + .expect("an op's ring outlives it"); + if ring.current[op.class.index()] == Some(index as u16) { + ring.current[op.class.index()] = None; + } + let res = match op.error { + Some(status) => status.res(), + None => i32::try_from(op.bytes).unwrap_or(i32::MAX), + }; + let timeline = (op.timeline != 0).then_some((op.timeline, op.seq)); + if !post(ring, op.user_data, res, timeline, op.error.is_none()) { + self.stats.timeline_refusals += 1; + } + ring.open -= 1; + self.stats.completions += 1; + let ring_id = RingId { + slot: op.ring, + generation: self.ring_generations[ring_slot], + }; + let grant = &mut ring.grants[usize::from(op.grant)]; + grant.ops -= 1; + if grant.ops == 0 + && let Some((_, GrantState::Revoking)) = grant.grant + { + let id = GrantId::new(op.grant, grant.generation); + grant.grant = None; + grant.generation = grant.generation.wrapping_add(1); + host.drained(ring_id, id); + } + if ring.state == RingState::Departed && ring.open == 0 && !ring.departure_reported { + ring.departure_reported = true; + host.departed(ring_id); + } + } + + fn scan_deadlines(&mut self, now: u64, host: &mut H) { + let mut reset = false; + for (index, slot) in self.ops.iter_mut().enumerate() { + let Some(op) = slot.as_mut() else { continue }; + if op.outstanding == 0 { + continue; + } + // The last rung: an op its abort did not release. Aborted or + // refused alike — a refusal may only mean the command was + // completing — it gets the grace, and then the controller. + if op.abort_deadline_ns.is_some_and(|deadline| now > deadline) { + reset = true; + continue; + } + if op.expiry_reported { + continue; + } + if op.deadline_ns.is_some_and(|deadline| now > deadline) { + op.expiry_reported = true; + op.abort_deadline_ns = Some(now.saturating_add(ABORT_GRACE_NS)); + self.stats.expired += 1; + // The escalation: ask the device to abandon every command of + // the op it still holds. An agreed abort fails the command + // through its own completion, so the op fails instead of + // hanging its ticket; one still held past the grace resets + // the controller below. + let sq = P::sq(op.class); + for (tag, command) in self.commands.iter().enumerate() { + if command.is_some_and(|command| usize::from(command.op) == index) { + if self.queues.abort(sq, Tag(tag as u32)) { + self.stats.aborts += 1; + } else { + self.stats.aborts_refused += 1; + } + } + } + host.expired(Expiry { + ring: RingId { + slot: op.ring, + generation: self.ring_generations[usize::from(op.ring)], + }, + user_data: op.user_data, + class: op.class, + outstanding: op.outstanding, + }); + } + } + if reset { + self.reset_device(host); + } + } + + /// Reset the device and fail every command it held + /// (`res/STORAGE_TRANSPORT.md` §7): each completes `Lost` through the + /// ordinary completion path, so its op fails, its fence moves, and its + /// target is free. Rings, grants and targets survive; only the commands + /// are gone. A reset that fails leaves everything as it was, counted. + fn reset_device(&mut self, host: &mut H) { + if !self.queues.reset() { + self.stats.resets_failed += 1; + return; + } + self.stats.resets += 1; + for tag in 0..self.commands.len() { + if self.commands[tag].is_some() { + self.complete_command( + DeviceCompletion { + cookie: tag as u64, + result: Err(BlockError::Lost), + }, + host, + ); + } + } + } +} + +/// How long an aborted command may stay held before the controller is +/// reset. An agreed abort completes the command at once; one still out a +/// second later is one the device will not give up. +const ABORT_GRACE_NS: u64 = 1_000_000_000; + +enum Step { + /// A device command was staged moving this many bytes. + Issued(u64), + /// A completion was posted, or an op stopped issuing. + Posted, + /// Nothing more for this ring in this class this turn. + Stop, +} + +/// Post one completion into `ring`'s channel and, when it names one, +/// advance its timeline and publish the fence. `false` when the timeline +/// refused the completion ([`EngineStats::timeline_refusals`]). +fn post<'w, T, const GRANTS: usize, const TARGETS: usize>( + ring: &mut RingSlot<'w, T, GRANTS, TARGETS>, + user_data: u64, + res: i32, + timeline: Option<(u16, u64)>, + ok: bool, +) -> bool { + let mut cqe = IoUringCqe::new_zeroed(); + cqe.user_data = user_data; + cqe.res = res; + // The completion credit taken at pop guarantees room. + let pushed = ring.window.cq.push(cqe); + debug_assert!(pushed.is_ok(), "completion credit violated"); + ring.completed += 1; + ring.dirty = true; + if let Some((t, seq)) = timeline + && let Some(books) = ring.timelines.get_mut(usize::from(t) - 1) + { + match books.complete(seq, ok) { + Ok(record) => { + if let Some(cell) = ring.window.fences.timeline(t) { + cell.publish(record); + } + } + // Unreachable while the pop holds the window; counted, because + // this is the fence stopping for good. + Err(_) => return false, + } + } + true +} + +/// Validate one popped submission against `ring`'s grants and targets and +/// build its op. A failure returns the status and — when the submission +/// named a valid timeline — the sequence it consumed, so the fence records +/// the failure in order. +#[allow(clippy::too_many_arguments)] +fn plan<'w, T, P: ClassPolicy, const GRANTS: usize, const TARGETS: usize>( + ring: &mut RingSlot<'w, T, GRANTS, TARGETS>, + ring_slot: u16, + class: Class, + sqe: &IoUringSqe, + geometry: crate::block::BlockGeometry, + now: Option, + budget: Budget, + in_flight: (u32, u64), +) -> Result)> { + // The timeline first: a submission that names one consumes a sequence + // even when it fails, because the client counted it. + let timeline = sqe.personality; + if usize::from(timeline) > TIMELINES { + return Err((StorStatus::Invalid, None)); + } + let seq = if timeline == 0 { + 0 + } else { + ring.timelines[usize::from(timeline) - 1].issue() + }; + let named = (timeline != 0).then_some((timeline, seq)); + let fail = |status| Err((status, named)); + + let kind = match sqe.opcode { + op::READ_FIXED => Kind::Read, + op::WRITE_FIXED => Kind::Write, + op::FSYNC => Kind::Flush, + op::DISCARD => Kind::Discard, + _ => return fail(StorStatus::Invalid), + }; + if sqe.flags != 0 || sqe.op_flags & !RWF_KNOWN != 0 { + return fail(StorStatus::Invalid); + } + if sqe.op_flags & RWF_DSYNC != 0 { + // Force-unit-access arrives with the NVMe queue set (S2). + return fail(StorStatus::Unsupported); + } + if sqe.op_flags & RWF_NOWAIT != 0 && in_flight.0 >= budget.commands { + return fail(StorStatus::Busy); + } + let Some(grant_id) = GrantId::from_fd(sqe.fd) else { + return fail(StorStatus::Stale); + }; + let Some(grant_slot) = ring.grants.get_mut(usize::from(grant_id.slot())) else { + return fail(StorStatus::Stale); + }; + if grant_slot.generation & GrantId::MAX_GENERATION != grant_id.generation() { + return fail(StorStatus::Stale); + } + let Some((grant, state)) = grant_slot.grant else { + return fail(StorStatus::Stale); + }; + if state == GrantState::Revoking { + return fail(StorStatus::Revoked); + } + if !grant.classes().contains(class) { + return fail(StorStatus::Class); + } + let needs = match kind { + Kind::Read => Rights::READ, + Kind::Write => Rights::WRITE, + Kind::Flush => Rights::FLUSH, + Kind::Discard => Rights::DISCARD, + }; + if !grant.rights().contains(needs) { + return fail(StorStatus::Rights); + } + let block = u64::from(geometry.block_size); + let (first_block, blocks, target, next_target, bytes) = match kind { + Kind::Flush => (0, 0, None, 0, 0), + Kind::Read | Kind::Write | Kind::Discard => { + let (off, len) = (sqe.off, u64::from(sqe.len)); + if len == 0 || off % block != 0 || len % block != 0 { + return fail(StorStatus::Alignment); + } + let (first, count) = (off / block, len / block); + if first.checked_add(count).is_none_or(|end| end > grant.blocks()) { + return fail(StorStatus::Range); + } + if kind == Kind::Discard { + (first, count, None, 0, 0) + } else { + let Some(entry) = ring + .targets + .get(usize::from(sqe.buf_index)) + .and_then(Option::as_ref) + else { + return fail(StorStatus::Stale); + }; + let admits = match kind { + Kind::Read => entry.access.admits_read_into(), + _ => entry.access.admits_write_from(), + }; + if !admits { + return fail(StorStatus::Rights); + } + let at = sqe.addr; + if at % block != 0 { + return fail(StorStatus::Alignment); + } + if at.checked_add(len).is_none_or(|end| end > entry.bytes) + || usize::try_from(at + len).is_err() + { + return fail(StorStatus::Range); + } + if i32::try_from(len).is_err() { + // `res` carries the byte count; larger submissions split + // on the client side. + return fail(StorStatus::Range); + } + (first, count, Some(sqe.buf_index), at, len) + } + } + }; + grant_slot.ops += 1; + let deadline_ns = now.map(|now| { + now.saturating_add(if sqe.addr3 != 0 { + sqe.addr3.saturating_sub(now) + } else { + P::deadline_ns(class) + }) + }); + Ok(Op { + ring: ring_slot, + class, + kind, + user_data: sqe.user_data, + grant: grant_id.slot(), + target, + timeline, + seq, + next_block: first_block, + left_blocks: blocks, + next_target, + bytes, + outstanding: 0, + issued: false, + error: None, + deadline_ns, + expiry_reported: false, + abort_deadline_ns: None, + }) + .map(|mut op| { + if kind == Kind::Flush { + // One command, no blocks: issue it as a single piece. + op.left_blocks = 1; + } + op + }) +} + +/// A device failure as the wire reports it. +const fn status_of(error: BlockError) -> StorStatus { + match error { + BlockError::Media | BlockError::Integrity => StorStatus::Media, + BlockError::Lost | BlockError::Transport => StorStatus::Lost, + BlockError::Cancelled => StorStatus::Cancelled, + BlockError::Revoked | BlockError::Draining => StorStatus::Revoked, + BlockError::Unsupported => StorStatus::Unsupported, + BlockError::ReadOnly => StorStatus::Rights, + BlockError::Alignment => StorStatus::Alignment, + BlockError::Range | BlockError::TooLarge | BlockError::Geometry => StorStatus::Range, + BlockError::Backpressure + | BlockError::UnknownBuffer + | BlockError::BufferBusy + | BlockError::UnknownRequest + | BlockError::StaleCompletion => StorStatus::Lost, + } +} + +#[cfg(test)] +mod tests { + extern crate std; + + use std::boxed::Box; + use std::vec::Vec; + + use kore_abi::stor::{CQ_DEPTH, FenceRecord, SQ_DEPTH}; + use kore_ring::stor::{ClientRing, WINDOW_BYTES, split}; + + use super::*; + use crate::block::memory::MemoryBlockDevice; + use crate::block::{BlockBuffer, DirectBlockDevice}; + use crate::stor::Single; + use crate::stor::grant::ExtentRun; + + /// 128 blocks of 512 B, a 4 KiB transfer ceiling so splitting shows. + type Dev = MemoryBlockDevice<{ 64 * 1024 }, { 16 * 1024 }>; + type Queues = Single; + type Engine = StorEngine<'static, Queues, Tight, 2, 4, 2, 300, 64>; + + const BLOCK: u64 = 512; + + /// Tiny budgets and a one-block quantum so ordering and fairness show. + struct Tight; + + impl ClassPolicy for Tight { + const ORDER: [Class; Class::COUNT] = [Class::Commit, Class::Demand, Class::Stream]; + const QUANTUM_BYTES: u64 = BLOCK; + + fn sq(_class: Class) -> usize { + 0 + } + + fn budget(class: Class) -> Budget { + match class { + Class::Stream => Budget { + commands: 2, + bytes: 1 << 20, + }, + Class::Commit | Class::Demand => Budget { + commands: 4, + bytes: 1 << 20, + }, + } + } + + fn deadline_ns(_class: Class) -> u64 { + 1_000 + } + } + + #[derive(Default)] + struct Rec { + kicks: Vec, + drained: Vec<(RingId, GrantId)>, + departed: Vec, + expired: Vec, + } + + impl StorHost for Rec { + fn kick(&mut self, ring: RingId) { + self.kicks.push(ring); + } + fn drained(&mut self, ring: RingId, grant: GrantId) { + self.drained.push((ring, grant)); + } + fn departed(&mut self, ring: RingId) { + self.departed.push(ring); + } + fn expired(&mut self, expiry: Expiry) { + self.expired.push(expiry); + } + } + + /// A page-aligned window that lives for the test binary. + fn window() -> &'static mut [u8] { + let layout = std::alloc::Layout::from_size_align(WINDOW_BYTES, 4096).unwrap(); + // SAFETY: non-zero size; leaked on purpose for a 'static window. + let ptr = unsafe { std::alloc::alloc_zeroed(layout) }; + assert!(!ptr.is_null()); + // SAFETY: freshly allocated, exclusively ours. + unsafe { core::slice::from_raw_parts_mut(ptr, WINDOW_BYTES) } + } + + fn engine() -> Box { + let dev = Dev::new(BLOCK as u32, 8).unwrap(); + Box::new(Engine::new(Single::new(dev)).ok().expect("policy fits")) + } + + fn attach(engine: &mut Engine, classes: ClassSet) -> (ClientRing<'static>, RingId) { + let (client, service) = split(window()).unwrap(); + let ring = engine.attach(service, classes).ok().expect("a free ring"); + (client, ring) + } + + fn target(engine: &mut Engine, bytes: usize, fill: u8) -> ::Buffer { + let mut buffer = engine.queues_mut().device_mut().allocate_buffer(bytes).unwrap(); + buffer.bytes_mut().fill(fill); + buffer + } + + #[allow(clippy::too_many_arguments)] + fn sqe( + opcode: u8, + grant: GrantId, + off: u64, + len: u32, + target: u16, + at: u64, + user_data: u64, + timeline: u16, + ) -> IoUringSqe { + let mut entry = IoUringSqe::new_zeroed(); + entry.opcode = opcode; + entry.fd = grant.fd(); + entry.off = off; + entry.len = len; + entry.buf_index = target; + entry.addr = at; + entry.user_data = user_data; + entry.personality = timeline; + entry + } + + fn submit(client: &mut ClientRing<'_>, class: Class, entry: IoUringSqe) { + client.sq[class.index()].push(entry).unwrap(); + let _ = client.sq[class.index()].commit(); + } + + fn completions(client: &mut ClientRing<'_>) -> Vec<(u64, i32)> { + let mut out = Vec::new(); + while let Some(cqe) = client.cq.pop() { + out.push((cqe.user_data, cqe.res)); + } + out + } + + fn rw_grant(runs: &[ExtentRun]) -> Grant { + Grant::new(runs, Rights::READ | Rights::WRITE | Rights::FLUSH, ClassSet::ALL).unwrap() + } + + /// Bytes written through one target come back through another, across + /// two grant runs and three device commands (a 4-block run, then a + /// 12-block run split at the 8-block ceiling) — one completion each. + #[test] + fn a_write_then_read_round_trips_across_runs_and_splits() { + let mut engine = engine(); + let mut host = Rec::default(); + let (mut client, ring) = attach(&mut engine, ClassSet::ALL); + let grant = engine + .install( + ring, + rw_grant(&[ + ExtentRun { lba: 10, blocks: 4 }, + ExtentRun { lba: 40, blocks: 12 }, + ]), + ) + .unwrap(); + let mut source = target(&mut engine, 8192, 0); + for (i, byte) in source.bytes_mut().iter_mut().enumerate() { + *byte = (i * 7 % 251) as u8; + } + let expected: Vec = source.bytes().to_vec(); + let src = engine.adopt_target(ring, source, TargetAccess::WriteFrom).ok().unwrap(); + let sink = target(&mut engine, 8192, 0xee); + let dst = engine.adopt_target(ring, sink, TargetAccess::ReadInto).ok().unwrap(); + + submit(&mut client, Class::Demand, sqe(op::WRITE_FIXED, grant, 0, 8192, src, 0, 1, 1)); + engine.advance(&mut host); + assert_eq!(completions(&mut client), [(1, 8192)]); + assert_eq!(engine.stats().commands, 3, "4 blocks, then 8 + 4 at the ceiling"); + + submit(&mut client, Class::Demand, sqe(op::READ_FIXED, grant, 0, 8192, dst, 0, 2, 1)); + engine.advance(&mut host); + assert_eq!(completions(&mut client), [(2, 8192)]); + assert_eq!( + client.fences.timeline(1).unwrap().load(), + FenceRecord { done: 2, fault: 0 } + ); + let back = engine.release_target(ring, dst).unwrap().into_inner(); + assert_eq!(back.bytes(), &expected[..]); + // The runs landed where the grant said: block 3 of the grant is LBA + // 13, block 4 is LBA 40. + assert!(!host.kicks.is_empty() || client.cq.state().unwrap().completed == 2); + assert_eq!(client.cq.state().unwrap().completed, 2); + } + + /// Every malformed submission fails with its own status and still + /// consumes its timeline sequence, so the fence records the fault in + /// order. + #[test] + fn each_refusal_has_its_status_and_consumes_its_sequence() { + let mut engine = engine(); + let mut host = Rec::default(); + let (mut client, ring) = attach(&mut engine, ClassSet::DATA); + let read_only = engine + .install( + ring, + Grant::new( + &[ExtentRun { lba: 0, blocks: 8 }], + Rights::READ, + ClassSet::only(Class::Demand), + ) + .unwrap(), + ) + .unwrap(); + let buf = target(&mut engine, 4096, 0); + let t = engine.adopt_target(ring, buf, TargetAccess::ReadInto).ok().unwrap(); + let stale = GrantId::new(read_only.slot(), read_only.generation() + 1); + + let cases: [(IoUringSqe, Class, StorStatus); 9] = [ + (sqe(0x77, read_only, 0, 512, t, 0, 10, 2), Class::Demand, StorStatus::Invalid), + (sqe(op::READ_FIXED, stale, 0, 512, t, 0, 11, 2), Class::Demand, StorStatus::Stale), + (sqe(op::WRITE_FIXED, read_only, 0, 512, t, 0, 12, 2), Class::Demand, StorStatus::Rights), + (sqe(op::READ_FIXED, read_only, 100, 512, t, 0, 13, 2), Class::Demand, StorStatus::Alignment), + (sqe(op::READ_FIXED, read_only, 0, 8192, t, 0, 14, 2), Class::Demand, StorStatus::Range), + (sqe(op::READ_FIXED, read_only, 0, 512, t, 4096, 15, 2), Class::Demand, StorStatus::Range), + (sqe(op::READ_FIXED, read_only, 0, 512, 1, 0, 16, 2), Class::Demand, StorStatus::Stale), + (sqe(op::READ_FIXED, read_only, 0, 512, t, 0, 17, 2), Class::Stream, StorStatus::Class), + (sqe(op::READ_FIXED, read_only, 0, 512, t, 0, 18, 99), Class::Demand, StorStatus::Invalid), + ]; + for (entry, class, _) in cases { + submit(&mut client, class, entry); + } + engine.advance(&mut host); + let mut got = completions(&mut client); + got.sort(); + let mut want: Vec<(u64, i32)> = cases + .iter() + .map(|(entry, _, status)| (entry.user_data, status.res())) + .collect(); + want.sort(); + assert_eq!(got, want); + // Eight named timeline 2 and consumed sequences 1..=8, all failed; + // the one naming timeline 99 consumed nothing. + assert_eq!( + client.fences.timeline(2).unwrap().load(), + FenceRecord { done: 8, fault: 1 } + ); + // A good read afterwards takes sequence 9 and leaves the fault. + submit(&mut client, Class::Demand, sqe(op::READ_FIXED, read_only, 0, 512, t, 0, 19, 2)); + engine.advance(&mut host); + assert_eq!(completions(&mut client), [(19, 512)]); + assert_eq!( + client.fences.timeline(2).unwrap().load(), + FenceRecord { done: 9, fault: 1 } + ); + assert_eq!(engine.stats().refused, 9); + } + + /// Revoking a grant with reads in flight: new submissions naming it are + /// refused at once, the in-flight ones complete, and the authority hears + /// `drained` only then — after which the old id is stale and the slot + /// is reusable. + #[test] + fn revoke_drains_before_the_slot_frees() { + let mut engine = engine(); + let mut host = Rec::default(); + let (mut client, ring) = attach(&mut engine, ClassSet::ALL); + let grant = engine.install(ring, rw_grant(&[ExtentRun { lba: 0, blocks: 16 }])).unwrap(); + let buf = target(&mut engine, 8192, 0); + let t = engine.adopt_target(ring, buf, TargetAccess::Both).ok().unwrap(); + + engine.queues_mut().device_mut().hold_completions(true); + submit(&mut client, Class::Demand, sqe(op::READ_FIXED, grant, 0, 4096, t, 0, 1, 0)); + engine.advance(&mut host); + assert!(completions(&mut client).is_empty(), "the device holds it"); + + assert_eq!(engine.revoke(ring, grant), Ok(Settle::Later)); + assert_eq!(engine.release_target(ring, t).err(), Some(StorRefusal::Busy)); + submit(&mut client, Class::Demand, sqe(op::READ_FIXED, grant, 0, 512, t, 0, 2, 0)); + engine.advance(&mut host); + assert_eq!(completions(&mut client), [(2, StorStatus::Revoked.res())]); + assert!(host.drained.is_empty()); + + engine.queues_mut().device_mut().hold_completions(false); + engine.advance(&mut host); + assert_eq!(completions(&mut client), [(1, 4096)]); + assert_eq!(host.drained, [(ring, grant)]); + + submit(&mut client, Class::Demand, sqe(op::READ_FIXED, grant, 0, 512, t, 0, 3, 0)); + engine.advance(&mut host); + assert_eq!(completions(&mut client), [(3, StorStatus::Stale.res())]); + let again = engine.install(ring, rw_grant(&[ExtentRun { lba: 0, blocks: 1 }])).unwrap(); + assert_eq!(again.slot(), grant.slot()); + assert_ne!(again.generation(), grant.generation()); + // An idle grant revokes at once, with no callback. + assert_eq!(engine.revoke(ring, again), Ok(Settle::Now)); + assert_eq!(host.drained.len(), 1); + } + + /// Detach stops popping, lets held commands finish, reports `departed` + /// once; retire waits for the targets to come back. + #[test] + fn detach_departs_after_the_last_completion_and_retire_needs_the_targets() { + let mut engine = engine(); + let mut host = Rec::default(); + let (mut client, ring) = attach(&mut engine, ClassSet::ALL); + let grant = engine.install(ring, rw_grant(&[ExtentRun { lba: 0, blocks: 16 }])).unwrap(); + let buf = target(&mut engine, 4096, 0); + let t = engine.adopt_target(ring, buf, TargetAccess::Both).ok().unwrap(); + + engine.queues_mut().device_mut().hold_completions(true); + submit(&mut client, Class::Demand, sqe(op::READ_FIXED, grant, 0, 512, t, 0, 1, 0)); + engine.advance(&mut host); + assert_eq!(engine.detach(ring), Ok(Settle::Later)); + submit(&mut client, Class::Demand, sqe(op::READ_FIXED, grant, 0, 512, t, 0, 2, 0)); + engine.advance(&mut host); + assert!(host.departed.is_empty()); + assert_eq!(client.cq.state().unwrap().status, RING_DEPARTED); + + engine.queues_mut().device_mut().hold_completions(false); + engine.advance(&mut host); + assert_eq!(completions(&mut client), [(1, 512)], "the popped read completes; the later one is never popped"); + assert_eq!(host.departed, [ring]); + engine.advance(&mut host); + assert_eq!(host.departed.len(), 1, "reported once"); + + assert_eq!(engine.retire(ring).err(), Some(StorRefusal::Busy), "a target is still adopted"); + let _ = engine.release_target(ring, t).unwrap().into_inner(); + assert!(engine.retire(ring).is_ok()); + assert_eq!(engine.install(ring, rw_grant(&[ExtentRun { lba: 0, blocks: 1 }])), Err(StorRefusal::Stale)); + } + + /// The completion credit: with nobody reaping, the engine completes + /// exactly one channel's worth of submissions and pops no further; one + /// reaped completion admits exactly one more. + #[test] + fn a_full_completion_channel_stops_the_popping() { + let mut engine = engine(); + let mut host = Rec::default(); + let (mut client, ring) = attach(&mut engine, ClassSet::ALL); + let grant = engine.install(ring, rw_grant(&[ExtentRun { lba: 0, blocks: 16 }])).unwrap(); + let buf = target(&mut engine, 512, 0); + let t = engine.adopt_target(ring, buf, TargetAccess::ReadInto).ok().unwrap(); + for i in 0..u64::from(SQ_DEPTH) { + submit(&mut client, Class::Demand, sqe(op::READ_FIXED, grant, 0, 512, t, 0, i, 0)); + submit(&mut client, Class::Stream, sqe(op::READ_FIXED, grant, 0, 512, t, 0, 1000 + i, 0)); + } + // The tight budgets admit six commands an advance; run until the + // channel stops moving. + for _ in 0..(2 * SQ_DEPTH) { + engine.advance(&mut host); + } + assert_eq!(client.cq.outstanding(), CQ_DEPTH); + submit(&mut client, Class::Demand, sqe(op::READ_FIXED, grant, 0, 512, t, 0, 9999, 0)); + engine.advance(&mut host); + assert_eq!(client.cq.outstanding(), CQ_DEPTH, "no credit, not popped"); + client.cq.pop().unwrap(); + engine.advance(&mut host); + assert_eq!(client.cq.outstanding(), CQ_DEPTH, "one reaped, one more posted"); + let tail: Vec = core::iter::from_fn(|| client.cq.pop().map(|c| c.user_data)).collect(); + assert_eq!(tail.last(), Some(&9999)); + } + + /// Commit fills before Demand before Stream; each class stays inside its + /// command budget; within Stream two rings take turns a quantum at a + /// time rather than one ring taking the whole budget. + #[test] + fn classes_fill_in_order_within_budget_and_rings_take_turns() { + let mut engine = engine(); + let mut host = Rec::default(); + let (mut a, ring_a) = attach(&mut engine, ClassSet::ALL); + let (mut b, ring_b) = attach(&mut engine, ClassSet::ALL); + let ga = engine.install(ring_a, rw_grant(&[ExtentRun { lba: 0, blocks: 64 }])).unwrap(); + let gb = engine.install(ring_b, rw_grant(&[ExtentRun { lba: 64, blocks: 64 }])).unwrap(); + let ba = target(&mut engine, 512, 0); + let bb = target(&mut engine, 512, 0); + let ta = engine.adopt_target(ring_a, ba, TargetAccess::ReadInto).ok().unwrap(); + let tb = engine.adopt_target(ring_b, bb, TargetAccess::ReadInto).ok().unwrap(); + + engine.queues_mut().device_mut().hold_completions(true); + for i in 0..4 { + submit(&mut a, Class::Stream, sqe(op::READ_FIXED, ga, 0, 512, ta, 0, 100 + i, 0)); + submit(&mut b, Class::Stream, sqe(op::READ_FIXED, gb, 0, 512, tb, 0, 200 + i, 0)); + } + for i in 0..6 { + submit(&mut a, Class::Demand, sqe(op::READ_FIXED, ga, 0, 512, ta, 0, 300 + i, 0)); + } + engine.advance(&mut host); + // Demand: its budget of 4 commands, all from A. Stream: its budget + // of 2 — one from each ring, not both from A. + assert_eq!(engine.commands_in_flight(), 6); + engine.queues_mut().device_mut().hold_completions(false); + engine.advance(&mut host); + let mut first: Vec = completions(&mut a) + .into_iter() + .chain(completions(&mut b)) + .map(|(user, _)| user) + .collect(); + first.sort(); + assert!(first.contains(&100) && first.contains(&200), "each ring got a stream turn: {first:?}"); + assert!(first.iter().filter(|&&u| (300..306).contains(&u)).count() >= 4); + // Everything completes eventually. + for _ in 0..16 { + engine.advance(&mut host); + } + let rest = completions(&mut a).len() + completions(&mut b).len(); + assert_eq!(first.len() + rest, 14); + } + + /// An op the device holds past its deadline is reported once, with its + /// outstanding count, and each command it holds is aborted once; the + /// aborted command's own completion then fails the op. + #[test] + fn an_expired_op_is_reported_once_aborted_and_fails() { + let mut engine = engine(); + let mut host = Rec::default(); + let (mut client, ring) = attach(&mut engine, ClassSet::ALL); + let grant = engine.install(ring, rw_grant(&[ExtentRun { lba: 0, blocks: 16 }])).unwrap(); + let buf = target(&mut engine, 4096, 0); + let t = engine.adopt_target(ring, buf, TargetAccess::ReadInto).ok().unwrap(); + + engine.queues_mut().device_mut().set_clock(10_000, 0); + engine.queues_mut().device_mut().hold_completions(true); + submit(&mut client, Class::Demand, sqe(op::READ_FIXED, grant, 0, 4096, t, 0, 7, 0)); + engine.advance(&mut host); + assert!(host.expired.is_empty()); + + engine.queues_mut().device_mut().set_clock(10_000 + 2_000_000, 0); + engine.advance(&mut host); + assert_eq!(host.expired.len(), 1); + assert_eq!(host.expired[0].user_data, 7); + assert_eq!(host.expired[0].outstanding, 1); + assert_eq!((engine.stats().aborts, engine.stats().aborts_refused), (1, 0)); + engine.queues_mut().device_mut().set_clock(10_000 + 4_000_000, 0); + engine.advance(&mut host); + assert_eq!(host.expired.len(), 1, "once"); + assert_eq!(engine.stats().aborts, 1, "aborted once"); + + // The aborted command's own completion comes back, failed: the op + // fails instead of hanging its ticket. + engine.queues_mut().device_mut().hold_completions(false); + engine.advance(&mut host); + assert_eq!(completions(&mut client), [(7, StorStatus::Cancelled.res())]); + } + + /// The last rung: an op whose abort the device refused is still held + /// past the grace, so the controller is reset. Its command completes + /// `Lost`, its fence moves, its target is free, and the ring, grant and + /// target all keep working. + #[test] + fn a_command_the_device_will_not_give_up_resets_the_controller() { + let mut engine = engine(); + let mut host = Rec::default(); + let (mut client, ring) = attach(&mut engine, ClassSet::ALL); + let grant = engine.install(ring, rw_grant(&[ExtentRun { lba: 0, blocks: 16 }])).unwrap(); + let buf = target(&mut engine, 4096, 0); + let t = engine.adopt_target(ring, buf, TargetAccess::ReadInto).ok().unwrap(); + + let device = engine.queues_mut().device_mut(); + device.set_clock(10_000, 0); + device.hold_completions(true); + device.refuse_aborts(true); + submit(&mut client, Class::Demand, sqe(op::READ_FIXED, grant, 0, 4096, t, 0, 7, 2)); + engine.advance(&mut host); + + // Past its deadline: reported, the abort refused, no reset yet. + engine.queues_mut().device_mut().set_clock(10_000 + 2_000_000, 0); + engine.advance(&mut host); + assert_eq!(host.expired.len(), 1); + assert_eq!((engine.stats().aborts, engine.stats().aborts_refused), (0, 1)); + assert_eq!(engine.stats().resets, 0, "the grace runs first"); + assert!(completions(&mut client).is_empty()); + + // Past the grace: the controller is reset and the command is lost. + engine + .queues_mut() + .device_mut() + .set_clock(10_000 + 2_000_000 + ABORT_GRACE_NS + 1, 0); + engine.advance(&mut host); + assert_eq!(engine.stats().resets, 1); + assert_eq!(engine.queues_mut().device_mut().resets(), 1); + assert_eq!(completions(&mut client), [(7, StorStatus::Lost.res())]); + assert_eq!(engine.commands_in_flight(), 0); + assert_eq!( + client.fences.timeline(2).unwrap().load(), + FenceRecord { done: 1, fault: 1 }, + "the fence moves, faulted" + ); + + // Everything but the command survives: the same ring, grant and + // target serve the next read. + let device = engine.queues_mut().device_mut(); + device.hold_completions(false); + device.refuse_aborts(false); + submit(&mut client, Class::Demand, sqe(op::READ_FIXED, grant, 0, 4096, t, 0, 8, 0)); + engine.advance(&mut host); + assert_eq!(completions(&mut client), [(8, 4096)]); + assert!(engine.release_target(ring, t).is_ok()); + } + + /// A device error on one piece fails the whole op with the mapped + /// status, after every piece has come back. + #[test] + fn a_media_error_on_one_piece_fails_the_op_after_all_pieces_return() { + let mut engine = engine(); + let mut host = Rec::default(); + let (mut client, ring) = attach(&mut engine, ClassSet::ALL); + let grant = engine.install(ring, rw_grant(&[ExtentRun { lba: 0, blocks: 32 }])).unwrap(); + let buf = target(&mut engine, 16384, 0); + let t = engine.adopt_target(ring, buf, TargetAccess::ReadInto).ok().unwrap(); + engine + .queues_mut() + .device_mut() + .inject_after(1, BlockError::Media); + submit(&mut client, Class::Demand, sqe(op::READ_FIXED, grant, 0, 16384, t, 0, 5, 3)); + engine.advance(&mut host); + assert_eq!(completions(&mut client), [(5, StorStatus::Media.res())]); + assert_eq!(engine.stats().commands, 4); + assert_eq!(engine.commands_in_flight(), 0); + assert_eq!( + client.fences.timeline(3).unwrap().load(), + FenceRecord { done: 1, fault: 1 } + ); + // The target is free again once its commands are back. + assert!(engine.release_target(ring, t).is_ok()); + } + + /// The client helper and the engine agree on every sequence: tickets + /// from `StorClient` complete through the engine's fences in order, + /// including across a refused submission on the same timeline. + #[test] + fn client_tickets_complete_through_the_engine() { + use kore_ring::stor::{Op, Request, StorClient}; + + let mut engine = engine(); + let mut host = Rec::default(); + let (client, ring) = attach(&mut engine, ClassSet::ALL); + let mut client = StorClient::new(client); + let grant = engine.install(ring, rw_grant(&[ExtentRun { lba: 0, blocks: 16 }])).unwrap(); + let buf = target(&mut engine, 4096, 0); + let t = engine.adopt_target(ring, buf, TargetAccess::ReadInto).ok().unwrap(); + let read = |len, user_data| Request { + op: Op::Read, + grant, + offset: 0, + len, + target: t, + target_offset: 0, + user_data, + timeline: 4, + deadline_ns: 0, + }; + let first = client.submit(Class::Demand, &read(512, 1)).unwrap().unwrap(); + let refused = client.submit(Class::Demand, &read(100, 2)).unwrap().unwrap(); + let third = client.submit(Class::Stream, &read(4096, 3)).unwrap().unwrap(); + client.publish(Class::Demand); + client.publish(Class::Stream); + engine.advance(&mut host); + assert_eq!(client.poll_ticket(first), Ok(true)); + assert!(client.poll_ticket(refused).is_err(), "the alignment refusal is its fault"); + assert!(client.poll_ticket(third).is_err(), "sticky past the fault"); + assert_eq!(client.fence(4).unwrap(), FenceRecord { done: 3, fault: 2 }); + let mut done = Vec::new(); + while let Some(c) = client.reap() { + done.push((c.user_data, c.result)); + } + done.sort_by_key(|(user, _)| *user); + assert_eq!( + done, + [(1, Ok(512)), (2, Err(StorStatus::Alignment)), (3, Ok(4096))] + ); + } + + /// A submission whose timeline already has a window's worth open waits + /// in its channel instead of being popped — a completion past the + /// window could never be recorded and would stop the fence for good — + /// and is popped once the window has room. + #[test] + fn a_full_timeline_window_holds_the_next_submission() { + let mut engine = engine(); + let mut host = Rec::default(); + let (mut client, ring) = attach(&mut engine, ClassSet::ALL); + let grant = engine.install(ring, rw_grant(&[ExtentRun { lba: 0, blocks: 16 }])).unwrap(); + let buf = target(&mut engine, 4096, 0); + let t = engine.adopt_target(ring, buf, TargetAccess::ReadInto).ok().unwrap(); + // A window's worth of sequences open on timeline 1, as a slow + // `done + 1` with everything behind it complete leaves it. + let slot = usize::from(ring.slot()); + let books = &mut engine.rings[slot].as_mut().unwrap().timelines[0]; + for _ in 0..super::super::timeline::WINDOW { + books.issue(); + } + submit(&mut client, Class::Demand, sqe(op::READ_FIXED, grant, 0, 512, t, 0, 1, 1)); + engine.advance(&mut host); + assert_eq!(client.sq[Class::Demand.index()].capacity_left(), SQ_DEPTH - 1, "held, not popped"); + assert!(completions(&mut client).is_empty()); + assert_eq!(engine.stats().timeline_refusals, 0); + + // The window drains; the held submission goes. + engine.rings[slot].as_mut().unwrap().timelines[0] = Timeline::default(); + engine.advance(&mut host); + assert_eq!(completions(&mut client), [(1, 512)]); + assert_eq!(engine.stats().timeline_refusals, 0); + } + + /// An idle engine parks every submission channel, so the next publish + /// owes a notice; a queued record, or an op still open, refuses the + /// park; and a departed ring is not waited on. + #[test] + fn an_idle_engine_parks_and_the_next_publish_owes_a_notice() { + use kore_ring::stor::{Op, Request, StorClient}; + + let mut engine = engine(); + let mut host = Rec::default(); + let (client, ring) = attach(&mut engine, ClassSet::ALL); + let mut client = StorClient::new(client); + let grant = engine.install(ring, rw_grant(&[ExtentRun { lba: 0, blocks: 16 }])).unwrap(); + let buf = target(&mut engine, 4096, 0); + let t = engine.adopt_target(ring, buf, TargetAccess::ReadInto).ok().unwrap(); + let read = Request { + op: Op::Read, + grant, + offset: 0, + len: 512, + target: t, + target_offset: 0, + user_data: 7, + timeline: 1, + deadline_ns: 0, + }; + + assert!(engine.park_idle(), "nothing queued, nothing open"); + assert!(client.publish(Class::Demand), "a parked consumer is owed its notice"); + assert!(!client.publish(Class::Demand), "once: the publish collected the intent"); + + client.submit(Class::Demand, &read).unwrap(); + assert!(!engine.park_idle(), "a queued record refuses the park"); + let _ = client.publish(Class::Demand); + engine.advance(&mut host); + assert_eq!(client.reap().map(|c| c.result), Some(Ok(512))); + assert!(engine.park_idle(), "drained again"); + + engine.detach(ring).unwrap(); + let _ = client.publish(Class::Demand); + client.submit(Class::Demand, &read).unwrap(); + assert!(engine.park_idle(), "a departed ring pops nothing, so is not waited on"); + } + + /// A flush is one device command and one completion — not a piece + /// that never finishes issuing — and a discard needs the discard right. + #[test] + fn a_flush_issues_once_and_a_discard_needs_its_right() { + let mut engine = engine(); + let mut host = Rec::default(); + let (mut client, ring) = attach(&mut engine, ClassSet::ALL); + let grant = engine.install(ring, rw_grant(&[ExtentRun { lba: 0, blocks: 16 }])).unwrap(); + submit(&mut client, Class::Commit, sqe(op::FSYNC, grant, 0, 0, 0, 0, 1, 0)); + engine.advance(&mut host); + assert_eq!(completions(&mut client), [(1, 0)]); + assert_eq!(engine.stats().commands, 1); + submit(&mut client, Class::Commit, sqe(op::DISCARD, grant, 0, 1024, 0, 0, 2, 0)); + engine.advance(&mut host); + assert_eq!(completions(&mut client), [(2, StorStatus::Rights.res())]); + let discard = engine + .install( + ring, + Grant::new(&[ExtentRun { lba: 0, blocks: 16 }], Rights::DISCARD, ClassSet::ALL).unwrap(), + ) + .unwrap(); + submit(&mut client, Class::Commit, sqe(op::DISCARD, discard, 0, 1024, 0, 0, 3, 0)); + engine.advance(&mut host); + assert_eq!(completions(&mut client), [(3, 0)]); + assert_eq!(engine.commands_in_flight(), 0); + } +} diff --git a/native/crates/driver/src/stor/grant.rs b/native/crates/driver/src/stor/grant.rs new file mode 100644 index 0000000..66e417d --- /dev/null +++ b/native/crates/driver/src/stor/grant.rs @@ -0,0 +1,214 @@ +//! **Grants** — the authority a ring's submissions are checked against +//! (`res/STORAGE_TRANSPORT.md` §5). +//! +//! A grant names up to [`MAX_RUNS`] extents of the device, a set of rights, +//! and the classes a submission against it may ride. Its byte offsets are +//! *logical*: offset 0 is the first byte of the first run, and a range that +//! crosses a run boundary continues at the next run. The filesystem installs +//! one per open file (the file's placement), the pack partition is one READ +//! grant, and the filesystem's own metadata region is one more. +//! +//! [`Grant::segment`] is the pack lane's region clamp and rebase, generalized +//! from one extent to a list: it answers where the next piece of a range +//! lives on the device, never past a run's end or the transfer ceiling. + +use bitflags::bitflags; +use kore_abi::stor::Class; + +/// Extents one grant may name. +pub const MAX_RUNS: usize = 8; + +bitflags! { + /// What a submission against a grant may do. + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] + pub struct Rights: u8 { + /// Read into a target. + const READ = 1 << 0; + /// Write from a target. + const WRITE = 1 << 1; + /// Flush the device's volatile cache. + const FLUSH = 1 << 2; + /// Discard a range. + const DISCARD = 1 << 3; + } +} + +/// A set of [`Class`]es. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct ClassSet(u8); + +impl ClassSet { + /// No class. + pub const NONE: Self = Self(0); + /// Every class. + pub const ALL: Self = Self(0b111); + /// Demand and Stream — what an ordinary client may ride. + pub const DATA: Self = Self(0b011); + + /// The set holding exactly `class`. + pub const fn only(class: Class) -> Self { + Self(1 << class as u8) + } + + /// This set plus `class`. + #[must_use] + pub const fn with(self, class: Class) -> Self { + Self(self.0 | (1 << class as u8)) + } + + /// Whether `class` is in the set. + pub const fn contains(self, class: Class) -> bool { + self.0 & (1 << class as u8) != 0 + } +} + +/// One device extent: `blocks` logical blocks starting at `lba`. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ExtentRun { + /// First logical block. + pub lba: u64, + /// Blocks it spans. + pub blocks: u32, +} + +/// Why a grant could not be built or a range could not resolve. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum GrantError { + /// No runs, more than [`MAX_RUNS`], or an empty run. + Shape, + /// A run's end overflows the block address space. + Overflow, +} + +/// A grant as its authority installs it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Grant { + runs: [ExtentRun; MAX_RUNS], + count: u8, + rights: Rights, + classes: ClassSet, +} + +/// Where the next piece of a range lives. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Segment { + /// First device block. + pub lba: u64, + /// Blocks in this piece. + pub blocks: u32, +} + +impl Grant { + /// A grant over `runs` with `rights`, admitting `classes`. + pub fn new(runs: &[ExtentRun], rights: Rights, classes: ClassSet) -> Result { + if runs.is_empty() || runs.len() > MAX_RUNS || runs.iter().any(|run| run.blocks == 0) { + return Err(GrantError::Shape); + } + if runs + .iter() + .any(|run| run.lba.checked_add(u64::from(run.blocks)).is_none()) + { + return Err(GrantError::Overflow); + } + let mut stored = [ExtentRun::default(); MAX_RUNS]; + stored[..runs.len()].copy_from_slice(runs); + Ok(Self { + runs: stored, + count: runs.len() as u8, + rights, + classes, + }) + } + + /// The grant's runs. + pub fn runs(&self) -> &[ExtentRun] { + &self.runs[..usize::from(self.count)] + } + + /// What the grant permits. + pub const fn rights(&self) -> Rights { + self.rights + } + + /// The classes it admits. + pub const fn classes(&self) -> ClassSet { + self.classes + } + + /// Blocks the grant spans in total. + pub fn blocks(&self) -> u64 { + self.runs().iter().map(|run| u64::from(run.blocks)).sum() + } + + /// The device piece holding logical block `block` of the grant, at most + /// `max_blocks` long and never past its run's end. `None` past the + /// grant's end or for a zero ceiling. + pub fn segment(&self, block: u64, max_blocks: u32) -> Option { + if max_blocks == 0 { + return None; + } + let mut skip = block; + for run in self.runs() { + let span = u64::from(run.blocks); + if skip < span { + let left = span - skip; + let blocks = left.min(u64::from(max_blocks)) as u32; + return Some(Segment { + lba: run.lba + skip, + blocks, + }); + } + skip -= span; + } + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn two_runs() -> Grant { + Grant::new( + &[ + ExtentRun { lba: 100, blocks: 4 }, + ExtentRun { lba: 900, blocks: 6 }, + ], + Rights::READ, + ClassSet::DATA, + ) + .unwrap() + } + + #[test] + fn a_grant_refuses_empty_oversized_and_overflowing_shapes() { + assert_eq!(Grant::new(&[], Rights::READ, ClassSet::ALL), Err(GrantError::Shape)); + let empty_run = [ExtentRun { lba: 1, blocks: 0 }]; + assert_eq!(Grant::new(&empty_run, Rights::READ, ClassSet::ALL), Err(GrantError::Shape)); + let many = [ExtentRun { lba: 0, blocks: 1 }; MAX_RUNS + 1]; + assert_eq!(Grant::new(&many, Rights::READ, ClassSet::ALL), Err(GrantError::Shape)); + let wrap = [ExtentRun { lba: u64::MAX, blocks: 1 }]; + assert_eq!(Grant::new(&wrap, Rights::READ, ClassSet::ALL), Err(GrantError::Overflow)); + } + + #[test] + fn a_logical_block_resolves_across_runs_and_stops_at_run_ends() { + let grant = two_runs(); + assert_eq!(grant.blocks(), 10); + assert_eq!(grant.segment(0, 64), Some(Segment { lba: 100, blocks: 4 })); + assert_eq!(grant.segment(2, 64), Some(Segment { lba: 102, blocks: 2 })); + assert_eq!(grant.segment(4, 64), Some(Segment { lba: 900, blocks: 6 })); + assert_eq!(grant.segment(5, 2), Some(Segment { lba: 901, blocks: 2 })); + assert_eq!(grant.segment(10, 64), None, "past the end"); + assert_eq!(grant.segment(0, 0), None, "no ceiling, no piece"); + } + + #[test] + fn class_sets_hold_what_they_name() { + assert!(ClassSet::DATA.contains(Class::Demand)); + assert!(ClassSet::DATA.contains(Class::Stream)); + assert!(!ClassSet::DATA.contains(Class::Commit)); + assert!(ClassSet::only(Class::Commit).contains(Class::Commit)); + assert!(ClassSet::NONE.with(Class::Stream).contains(Class::Stream)); + } +} diff --git a/native/crates/driver/src/stor/lanes.rs b/native/crates/driver/src/stor/lanes.rs new file mode 100644 index 0000000..9e8ee6d --- /dev/null +++ b/native/crates/driver/src/stor/lanes.rs @@ -0,0 +1,1003 @@ +//! **Lanes over the engine** — the transport underneath today's filesystem +//! composition (`res/STORAGE_TRANSPORT.md` §9, slice S1b). +//! +//! [`EngineBackend`] owns one [`StorEngine`] and gives each region a lane: +//! a ring on the engine carrying a whole-namespace grant, driven through an +//! in-process [`StorClient`]. A lane is a [`DirectBlockDevice`], so the +//! ordinary composition — region clamp, block channel, provider — stacks on +//! it unchanged, as it stacked on the shared-backend lanes the engine +//! replaced: +//! +//! ```text +//! EngineBackend one engine, one queue set +//! ├ lane 0 → RegionDevice → BlockChannel → provider (reads Demand, writes Commit) +//! ├ lane 1 → RegionDevice → BlockChannel → provider +//! └ client rings (after the lanes): windows other tasks lent, grants +//! installed by the filesystem (`attach_client`, `install`) +//! ``` +//! +//! What changes underneath: a lane's commands are ring submissions in a +//! class, so the engine keeps clients' bulk reads inside the Stream budget +//! while filesystem reads fill first; a completion names its own ring, so there is +//! no cookie tag and no demux; and a ring's completion channel can never +//! drop a completion. The lane's region clamp above it is unchanged — the +//! whole-namespace grant is only the transport's ceiling. +//! +//! One worker owns the backend and every lane, as it owned the shared +//! backend: interior mutability is `RefCell`, and no borrow is held across +//! a suspension point. + +use core::cell::{Cell, RefCell}; +use core::ptr::NonNull; + +use kore_abi::stor::{Class, GrantId, StorStatus}; +use kore_ring::stor::{Op, Request, StorClient, split}; + +use super::QueueSet; +use crate::seam::Quiet; +use super::engine::{ + ClassPolicy, EngineStats, Expiry, RingId, Settle, StorEngine, StorHost, StorRefusal, TargetAccess, +}; +use super::grant::{ClassSet, ExtentRun, Grant, MAX_RUNS, Rights}; +use crate::block::{ + BlockBuffer, BlockError, BlockGeometry, BlockHealth, DeviceCommand, DeviceCompletion, + DirectBlockDevice, Liveness, +}; + +/// The class each command kind of a lane rides. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LaneClasses { + /// Reads. + pub read: Class, + /// Writes. + pub write: Class, + /// Flushes. + pub flush: Class, + /// Discards. + pub discard: Class, +} + +impl LaneClasses { + /// A filesystem region: reads on Demand, the durability path on Commit. + pub const FILESYSTEM: Self = Self { + read: Class::Demand, + write: Class::Commit, + flush: Class::Commit, + discard: Class::Commit, + }; + /// A bulk streaming region: everything on Stream. + pub const STREAM: Self = Self { + read: Class::Stream, + write: Class::Stream, + flush: Class::Stream, + discard: Class::Stream, + }; +} + +/// A lane's registered buffer: an engine target the lane adopted, with the +/// CPU view its owner reads and writes between commands. +pub struct LaneBuffer { + slot: u16, + ptr: NonNull, + len: usize, +} + +impl LaneBuffer { + /// The target slot on the lane's ring. + pub const fn slot(&self) -> u16 { + self.slot + } +} + +impl BlockBuffer for LaneBuffer { + fn len(&self) -> usize { + self.len + } + + fn bytes(&self) -> &[u8] { + // SAFETY: the view was taken from the adopted target, which the + // engine keeps at a fixed address for as long as it is adopted, and + // a block channel reaches the bytes only with no command in flight + // against the buffer. + unsafe { core::slice::from_raw_parts(self.ptr.as_ptr(), self.len) } + } + + fn bytes_mut(&mut self) -> &mut [u8] { + // SAFETY: as `bytes`. + unsafe { core::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) } + } +} + +/// The engine's side effects in-process: nothing to kick (the lanes poll), +/// no authority to tell (the lanes never revoke); expiries are counted. +#[derive(Default)] +struct LaneHost { + expired: u64, +} + +impl StorHost for LaneHost { + fn kick(&mut self, _ring: RingId) {} + fn drained(&mut self, _ring: RingId, _grant: GrantId) {} + fn departed(&mut self, _ring: RingId) {} + fn expired(&mut self, _expiry: Expiry) { + self.expired += 1; + } +} + +struct LaneState<'w> { + client: RefCell>, + ring: RingId, + grant: GrantId, + classes: LaneClasses, + claimed: Cell, + retired: Cell, + /// Submissions accepted and not yet reaped. + open: Cell, +} + +/// The whole-namespace grant a lane carries: every block, in as many runs +/// as the run length's width needs. +fn namespace_grant(geometry: BlockGeometry) -> Result { + let mut runs = [ExtentRun::default(); MAX_RUNS]; + let mut count = 0; + let mut lba = 0u64; + while lba < geometry.block_count { + if count == MAX_RUNS { + return Err(BlockError::Geometry); + } + let blocks = (geometry.block_count - lba).min(u64::from(u32::MAX)) as u32; + runs[count] = ExtentRun { lba, blocks }; + count += 1; + lba += u64::from(blocks); + } + Grant::new( + &runs[..count], + Rights::READ | Rights::WRITE | Rights::FLUSH | Rights::DISCARD, + ClassSet::ALL, + ) + .map_err(|_| BlockError::Geometry) +} + +const fn block_error(status: StorStatus) -> BlockError { + match status { + StorStatus::Range => BlockError::Range, + StorStatus::Alignment => BlockError::Alignment, + StorStatus::Rights => BlockError::ReadOnly, + StorStatus::Revoked => BlockError::Revoked, + StorStatus::Stale => BlockError::UnknownBuffer, + StorStatus::Deadline | StorStatus::Lost => BlockError::Lost, + StorStatus::Cancelled => BlockError::Cancelled, + StorStatus::Media => BlockError::Media, + StorStatus::Busy => BlockError::Backpressure, + StorStatus::Unsupported | StorStatus::Class | StorStatus::Invalid => { + BlockError::Unsupported + } + } +} + +/// What one [`EngineBackend::pump`] left behind. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Pump { + /// Device commands are in flight: their completion wakes the host, and + /// that wake pumps again, so a record published meanwhile is seen then. + Busy, + /// Nothing is in flight and every ring is parked: the host may sleep. A + /// client that publishes now sees the park and owes a notice. + Parked, + /// Work stayed pending through every pass (or the engine was borrowed): + /// pump again before sleeping. + Again, +} + +/// Passes one pump makes before giving up on reaching `Busy` or `Parked`. +const PUMP_PASSES: usize = 4; + +const fn refusal_error(refusal: StorRefusal) -> BlockError { + match refusal { + StorRefusal::Stale => BlockError::UnknownBuffer, + StorRefusal::Full => BlockError::Backpressure, + StorRefusal::Busy => BlockError::BufferBusy, + StorRefusal::Departed => BlockError::Revoked, + StorRefusal::Invalid => BlockError::Unsupported, + } +} + +/// One engine shared by `LANES` lanes and `RINGS - LANES` client rings. +pub struct EngineBackend< + 'w, + Q: QueueSet, + P: ClassPolicy, + const LANES: usize, + const RINGS: usize, + const GRANTS: usize, + const TARGETS: usize, + const OPS: usize, + const COMMANDS: usize, +> { + /// Placed by the caller ([`StorEngine::init_in`]), so the tables never + /// pass through a stack frame on their way to the heap. + engine: RefCell<&'w mut StorEngine<'w, Q, P, RINGS, GRANTS, TARGETS, OPS, COMMANDS>>, + host: RefCell, + lanes: [LaneState<'w>; LANES], + geometry: BlockGeometry, + max_buffer_bytes: usize, + conflicts: Cell, +} + +impl< + 'w, + Q: QueueSet, + P: ClassPolicy, + const LANES: usize, + const RINGS: usize, + const GRANTS: usize, + const TARGETS: usize, + const OPS: usize, + const COMMANDS: usize, +> EngineBackend<'w, Q, P, LANES, RINGS, GRANTS, TARGETS, OPS, COMMANDS> +where + Q::Target: BlockBuffer, +{ + /// Lanes over a freshly placed `engine` (no ring attached yet), one lane + /// per window (each exactly [`kore_ring::stor::WINDOW_BYTES`], + /// page-aligned, owned for `'w`), lane `i` riding `classes[i]` and + /// taking ring slot `i`. + pub fn new( + engine: &'w mut StorEngine<'w, Q, P, RINGS, GRANTS, TARGETS, OPS, COMMANDS>, + windows: [&'w mut [u8]; LANES], + classes: [LaneClasses; LANES], + ) -> Result { + // Lanes take the first ring slots (attached here, before any + // client) and the first grant slot of their ring. + if RINGS < LANES || GRANTS == 0 { + return Err(BlockError::Geometry); + } + let geometry = engine.queues().geometry().validate()?; + let max_buffer_bytes = engine.queues().max_target_bytes(); + let grant = namespace_grant(geometry)?; + let mut built: [Option>; LANES] = core::array::from_fn(|_| None); + for (index, window) in windows.into_iter().enumerate() { + let (client, service) = split(window).map_err(|_| BlockError::Geometry)?; + let ring = engine + .attach(service, ClassSet::ALL) + .map_err(|_| BlockError::Backpressure)?; + if usize::from(ring.slot()) != index { + // The engine was not fresh: a lane would share the client + // slot range, and the client calls could reach it. + return Err(BlockError::Geometry); + } + let grant = engine.install(ring, grant).map_err(|_| BlockError::Backpressure)?; + built[index] = Some(LaneState { + client: RefCell::new(StorClient::new(client)), + ring, + grant, + classes: classes[index], + claimed: Cell::new(false), + retired: Cell::new(false), + open: Cell::new(0), + }); + } + Ok(Self { + engine: RefCell::new(engine), + host: RefCell::new(LaneHost::default()), + lanes: built.map(|lane| lane.expect("every window built a lane")), + geometry, + max_buffer_bytes, + conflicts: Cell::new(0), + }) + } + + /// Take the engine back: every lane's ring detached and retired, its + /// targets dropped. For a backend whose device outlives it — a test's + /// reboot of one disk, an orderly teardown. Refused, the backend handed + /// back, while any lane has a submission open. A client ring still + /// attached stays attached; `StorEngine::into_queues` then refuses. + pub fn into_engine( + self, + ) -> Result<&'w mut StorEngine<'w, Q, P, RINGS, GRANTS, TARGETS, OPS, COMMANDS>, Self> { + if self.lanes.iter().any(|lane| lane.open.get() != 0) { + return Err(self); + } + let engine = self.engine.into_inner(); + for lane in &self.lanes { + let _ = engine.detach(lane.ring); + for slot in 0..TARGETS { + drop(engine.release_target(lane.ring, slot as u16)); + } + let _ = engine.retire(lane.ring); + } + Ok(engine) + } + + /// The namespace's geometry. + pub fn geometry(&self) -> BlockGeometry { + self.geometry + } + + /// Claim lane `lane` — once. + pub fn claim(&self, lane: usize) -> Result, BlockError> { + let state = self.lanes.get(lane).ok_or(BlockError::Range)?; + if state.claimed.replace(true) { + return Err(BlockError::Backpressure); + } + Ok(EngineLane { core: self, lane }) + } + + /// Submissions lane `lane` has open. + pub fn outstanding(&self, lane: usize) -> Option { + self.lanes.get(lane).map(|state| state.open.get() as usize) + } + + /// Whether lane `lane` is claimed and not retired. + pub fn is_live(&self, lane: usize) -> bool { + self.lanes + .get(lane) + .is_some_and(|state| state.claimed.get() && !state.retired.get()) + } + + /// Engine counters, with the expiries the host counted. + pub fn stats(&self) -> Option<(EngineStats, u64)> { + let engine = self.engine.try_borrow().ok()?; + let host = self.host.try_borrow().ok()?; + Some((*engine.stats(), host.expired)) + } + + /// Device commands in flight, every ring together. + pub fn commands_in_flight(&self) -> Option { + Some(self.engine.try_borrow().ok()?.commands_in_flight()) + } + + /// Borrow conflicts refused — a re-entrancy bug surfaced as a count. + pub fn conflicts(&self) -> u64 { + self.conflicts.get() + } + + fn engine(&self) -> Result>, BlockError> { + self.engine.try_borrow_mut().map_err(|_| { + self.conflicts.set(self.conflicts.get() + 1); + BlockError::Transport + }) + } + + /// A control step against the queue set, with commands in flight or not. + pub fn control(&self, f: impl FnOnce(&mut Q) -> T) -> Result { + Ok(f(self.engine()?.queues_mut())) + } + + /// A control step that needs the device idle — an IOMMU retirement, + /// teardown. Refused with commands in flight. + pub fn control_idle(&self, f: impl FnOnce(&mut Q) -> T) -> Result { + let mut engine = self.engine()?; + if engine.commands_in_flight() != 0 { + return Err(BlockError::Draining); + } + Ok(f(engine.queues_mut())) + } + + /// Read the queue set without driving it — testimony only. + pub fn inspect(&self, f: impl FnOnce(&Q) -> T) -> Option { + self.engine.try_borrow().ok().map(|engine| f(engine.queues())) + } + + /// Adopt caller memory into `lane` as a streaming target. + pub fn adopt( + &self, + lane: usize, + target: Q::Target, + access: TargetAccess, + ) -> Result { + let Some(state) = self.lanes.get(lane) else { + return Err((target, BlockError::Range)); + }; + let mut engine = match self.engine() { + Ok(engine) => engine, + Err(error) => return Err((target, error)), + }; + let slot = engine + .adopt_target(state.ring, target, access) + .map_err(|(target, _)| (target, BlockError::Backpressure))?; + let view = engine + .target_mut(state.ring, slot) + .expect("a just-adopted target is idle"); + let len = view.len(); + let ptr = NonNull::new(view.bytes_mut().as_mut_ptr()).unwrap_or(NonNull::dangling()); + Ok(LaneBuffer { slot, ptr, len }) + } + + /// Hand `buffer` back from `lane`, once no command uses it. + pub fn release(&self, lane: usize, buffer: LaneBuffer) -> Result, (LaneBuffer, BlockError)> { + let Some(state) = self.lanes.get(lane) else { + return Err((buffer, BlockError::Range)); + }; + let mut engine = match self.engine() { + Ok(engine) => engine, + Err(error) => return Err((buffer, error)), + }; + engine + .release_target(state.ring, buffer.slot) + .map_err(|_| (buffer, BlockError::BufferBusy)) + } + + // ── client rings ──────────────────────────────────────────────────── + // + // A client ring is a window another task funded and lent: its channels, + // indices and records are shared with a party this worker does not + // trust. The engine touches it only through the channel protocols, which + // mask every index and decode any bytes, so a hostile client can wreck + // its own ring and nothing else. Client rings take the slots after the + // lanes, and a lane's ring is never reachable through these calls. + + fn client_ring(ring: RingId) -> Result { + if usize::from(ring.slot()) < LANES { + return Err(BlockError::Range); + } + Ok(ring) + } + + /// Attach a client's formatted window as a ring admitting `classes`. + pub fn attach_client(&self, window: &'w mut [u8], classes: ClassSet) -> Result { + let service = kore_ring::stor::service(window).map_err(|_| BlockError::Geometry)?; + self.engine()? + .attach(service, classes) + .map_err(|(_, _)| BlockError::Backpressure) + } + + /// Install `grant` into client `ring`. + pub fn install(&self, ring: RingId, grant: Grant) -> Result { + let ring = Self::client_ring(ring)?; + self.engine()? + .install(ring, grant) + .map_err(|_| BlockError::Backpressure) + } + + /// Revoke `grant` from client `ring`; [`Settle::Later`] until nothing + /// in flight names it. + pub fn revoke(&self, ring: RingId, grant: GrantId) -> Result { + let ring = Self::client_ring(ring)?; + self.engine()? + .revoke(ring, grant) + .map_err(|_| BlockError::UnknownBuffer) + } + + /// Adopt caller memory into client `ring` as target slot `n`. A refused + /// target comes back [`Quiet`]: the engine never took it, so no command + /// can name it. + pub fn adopt_client( + &self, + ring: RingId, + target: Q::Target, + access: TargetAccess, + ) -> Result, BlockError)> { + if let Err(error) = Self::client_ring(ring) { + return Err((Quiet::witness(target), error)); + } + let mut engine = match self.engine() { + Ok(engine) => engine, + Err(error) => return Err((Quiet::witness(target), error)), + }; + engine + .adopt_target(ring, target, access) + .map_err(|(target, refusal)| (Quiet::witness(target), refusal_error(refusal))) + } + + /// Hand target `slot` of client `ring` back, once no command uses it. + pub fn release_client(&self, ring: RingId, slot: u16) -> Result, BlockError> { + let ring = Self::client_ring(ring)?; + self.engine()? + .release_target(ring, slot) + .map_err(refusal_error) + } + + /// Detach client `ring`: nothing more is popped; what the device holds + /// finishes. [`Settle::Now`] when nothing was open. + pub fn detach_client(&self, ring: RingId) -> Result { + let ring = Self::client_ring(ring)?; + self.engine()?.detach(ring).map_err(refusal_error) + } + + /// Submissions client `ring` has popped and not completed. + pub fn client_open(&self, ring: RingId) -> Option { + let ring = Self::client_ring(ring).ok()?; + self.engine.try_borrow().ok()?.open(ring) + } + + /// Retire a detached, drained client ring whose targets are all + /// released. Its window is no longer referenced once this returns + /// `Ok`, and not before: only then may the lent mapping go. + pub fn retire_client(&self, ring: RingId) -> Result<(), BlockError> { + let ring = Self::client_ring(ring)?; + self.engine()?.retire(ring).map(drop).map_err(refusal_error) + } + + /// One pass on the host's wake: advance, and when the engine is idle + /// commit every ring to parking. See [`Pump`]. + pub fn pump(&self) -> Pump { + let Ok(mut engine) = self.engine() else { + return Pump::Again; + }; + for _ in 0..PUMP_PASSES { + self.advance(&mut engine); + if engine.commands_in_flight() != 0 { + return Pump::Busy; + } + if engine.park_idle() { + return Pump::Parked; + } + } + Pump::Again + } + + fn advance(&self, engine: &mut StorEngine<'w, Q, P, RINGS, GRANTS, TARGETS, OPS, COMMANDS>) { + if let Ok(mut host) = self.host.try_borrow_mut() { + engine.advance(&mut *host); + } + } + + fn stage(&self, lane: usize, command: DeviceCommand, buffer: Option<&LaneBuffer>, cookie: u64) -> Result<(), BlockError> { + let state = &self.lanes[lane]; + if state.retired.get() { + return Err(BlockError::Revoked); + } + let block = u64::from(self.geometry.block_size); + let request = |op, class_lba: Option<(u64, u32)>, target: Option<(u16, usize)>| { + let (offset, len) = class_lba.map_or((0, 0), |(lba, blocks)| { + (lba * block, (u64::from(blocks) * block) as u32) + }); + let (target, target_offset) = target.map_or((0, 0), |(slot, at)| (slot, at as u64)); + Request { + op, + grant: state.grant, + offset, + len, + target, + target_offset, + user_data: cookie, + timeline: 0, + deadline_ns: 0, + } + }; + let (class, request) = match command { + DeviceCommand::Read { + lba, + blocks, + buffer_offset, + } => { + let buffer = buffer.ok_or(BlockError::UnknownBuffer)?; + ( + state.classes.read, + request(Op::Read, Some((lba, blocks)), Some((buffer.slot, buffer_offset))), + ) + } + DeviceCommand::Write { + lba, + blocks, + buffer_offset, + } => { + let buffer = buffer.ok_or(BlockError::UnknownBuffer)?; + ( + state.classes.write, + request(Op::Write, Some((lba, blocks)), Some((buffer.slot, buffer_offset))), + ) + } + DeviceCommand::Flush => (state.classes.flush, request(Op::Flush, None, None)), + DeviceCommand::Discard { lba, blocks } => ( + state.classes.discard, + request(Op::Discard, Some((lba, blocks)), None), + ), + }; + if u64::from(request.len) != request.len as u64 { + return Err(BlockError::TooLarge); + } + state + .client + .try_borrow_mut() + .map_err(|_| BlockError::Transport)? + .submit(class, &request) + .map_err(|_| BlockError::Backpressure)?; + state.open.set(state.open.get() + 1); + Ok(()) + } + + fn publish(&self, lane: usize) { + if let Ok(mut client) = self.lanes[lane].client.try_borrow_mut() { + for class in Class::ALL { + let _ = client.publish(class); + } + } + if let Ok(mut engine) = self.engine() { + self.advance(&mut engine); + } + } + + fn poll(&self, lane: usize) -> Option { + let state = &self.lanes[lane]; + if let Ok(mut engine) = self.engine() { + self.advance(&mut engine); + } + let completion = state.client.try_borrow_mut().ok()?.reap()?; + state.open.set(state.open.get().saturating_sub(1)); + Some(DeviceCompletion { + cookie: completion.user_data, + result: completion.result.map(|_| ()).map_err(block_error), + }) + } + + fn wait(&self, lane: usize) -> Option { + // Bounded: each empty park is the queue set's own wait budget, and + // two in a row with nothing moving is the caller's expiry. + let mut empty = 0; + loop { + if let Some(completion) = self.poll(lane) { + return Some(completion); + } + if self.lanes[lane].open.get() == 0 { + return None; + } + let parked = self.engine().map(|mut engine| engine.queues_mut().park()).unwrap_or(false); + if parked { + empty = 0; + } else { + empty += 1; + if empty >= 2 { + return self.poll(lane); + } + } + } + } + + fn quiesce(&self, lane: usize) -> Result<(), BlockError> { + let state = &self.lanes[lane]; + if state.open.get() != 0 { + return Err(BlockError::Draining); + } + state.retired.set(true); + if self.lanes.iter().all(|lane| lane.retired.get() || !lane.claimed.get()) { + self.engine()?.queues_mut().quiesce() + } else { + Ok(()) + } + } +} + +/// One lane: a [`DirectBlockDevice`] whose commands ride its own ring. +pub struct EngineLane< + 'a, + 'w, + Q: QueueSet, + P: ClassPolicy, + const LANES: usize, + const RINGS: usize, + const GRANTS: usize, + const TARGETS: usize, + const OPS: usize, + const COMMANDS: usize, +> { + core: &'a EngineBackend<'w, Q, P, LANES, RINGS, GRANTS, TARGETS, OPS, COMMANDS>, + lane: usize, +} + +impl< + 'a, + 'w, + Q: QueueSet, + P: ClassPolicy, + const LANES: usize, + const RINGS: usize, + const GRANTS: usize, + const TARGETS: usize, + const OPS: usize, + const COMMANDS: usize, +> EngineLane<'a, 'w, Q, P, LANES, RINGS, GRANTS, TARGETS, OPS, COMMANDS> +where + Q::Target: BlockBuffer, +{ + /// This lane's index. + pub fn index(&self) -> usize { + self.lane + } + + /// The backend it rides. + pub fn backend(&self) -> &'a EngineBackend<'w, Q, P, LANES, RINGS, GRANTS, TARGETS, OPS, COMMANDS> { + self.core + } +} + +impl< + 'a, + 'w, + Q: QueueSet, + P: ClassPolicy, + const LANES: usize, + const RINGS: usize, + const GRANTS: usize, + const TARGETS: usize, + const OPS: usize, + const COMMANDS: usize, +> DirectBlockDevice for EngineLane<'a, 'w, Q, P, LANES, RINGS, GRANTS, TARGETS, OPS, COMMANDS> +where + Q::Target: BlockBuffer, +{ + type Buffer = LaneBuffer; + + /// The engine advances inside every poll; nothing awaits a reply. + const INDEPENDENT_PROGRESS: bool = true; + + fn geometry(&self) -> BlockGeometry { + self.core.geometry + } + + fn max_buffer_bytes(&self) -> usize { + self.core.max_buffer_bytes + } + + fn liveness(&mut self) -> Liveness { + self.core + .engine() + .map(|mut engine| engine.liveness()) + .unwrap_or(Liveness::Unknown) + } + + fn clock_ns(&self) -> Option { + self.core.inspect(|queues| queues.clock_ns()).flatten() + } + + fn health(&mut self) -> Result { + self.core.control(|queues| queues.health())? + } + + fn allocate_buffer(&mut self, bytes: usize) -> Result { + let target = self.core.control(|queues| queues.allocate(bytes))??; + self.core + .adopt(self.lane, target, TargetAccess::Both) + .map_err(|(_, error)| error) + } + + fn submit( + &mut self, + command: DeviceCommand, + buffer: Option<&mut Self::Buffer>, + cookie: u64, + ) -> Result<(), BlockError> { + self.core.stage(self.lane, command, buffer.as_deref(), cookie)?; + self.core.publish(self.lane); + Ok(()) + } + + fn stage( + &mut self, + command: DeviceCommand, + buffer: Option<&mut Self::Buffer>, + cookie: u64, + ) -> Result<(), BlockError> { + self.core.stage(self.lane, command, buffer.as_deref(), cookie) + } + + fn publish(&mut self) { + self.core.publish(self.lane); + } + + fn poll(&mut self) -> Option { + self.core.poll(self.lane) + } + + fn quiesce(&mut self) -> Result<(), BlockError> { + self.core.quiesce(self.lane) + } + + fn wait(&mut self) -> Option { + self.core.wait(self.lane) + } +} + +#[cfg(test)] +mod tests { + extern crate std; + + use std::boxed::Box; + + use super::*; + use crate::block::memory::MemoryBlockDevice; + use crate::block::{BlockChannel, CompletionMode, RegionDevice}; + use crate::stor::Single; + use crate::stor::engine::OneQueue; + use kore_ring::stor::WINDOW_BYTES; + + // Small inline fixtures: the memory double carries its disk and every + // adopted target inline, and a debug build copies the backend on the + // test thread's stack while it is built. + type Dev = MemoryBlockDevice<{ 32 * 1024 }, 4096>; + type Backend = EngineBackend<'static, Single, OneQueue, 2, 3, 2, 4, 32, 32>; + + fn window() -> &'static mut [u8] { + let layout = std::alloc::Layout::from_size_align(WINDOW_BYTES, 4096).unwrap(); + // SAFETY: non-zero size; leaked for a 'static window. + let ptr = unsafe { std::alloc::alloc_zeroed(layout) }; + assert!(!ptr.is_null()); + // SAFETY: freshly allocated, exclusively ours. + unsafe { core::slice::from_raw_parts_mut(ptr, WINDOW_BYTES) } + } + + type Engine = StorEngine<'static, Single, OneQueue, 3, 2, 4, 32, 32>; + + fn backend() -> &'static Backend { + let dev = Dev::new(512, 8).unwrap(); + let place = Box::leak(Box::new(core::mem::MaybeUninit::::uninit())); + let engine = Engine::init_in(place, Single::new(dev)).ok().expect("policy fits"); + Box::leak(Box::new( + Backend::new(engine, [window(), window()], [LaneClasses::FILESYSTEM, LaneClasses::STREAM]) + .unwrap(), + )) + } + + /// The ordinary composition runs unchanged on an engine lane: a block + /// channel over a region clamp writes and reads back, and the two lanes' + /// regions never see each other's bytes. + #[test] + fn a_block_channel_round_trips_over_an_engine_lane() { + let backend = backend(); + let lane0 = backend.claim(0).unwrap(); + let lane1 = backend.claim(1).unwrap(); + assert_eq!(backend.claim(0).err(), Some(BlockError::Backpressure)); + let region0 = RegionDevice::whole(lane0) + .unwrap() + .narrow(0, 32) + .ok() + .expect("region 0"); + let region1 = RegionDevice::whole(lane1) + .unwrap() + .narrow(32, 32) + .ok() + .expect("region 1"); + let mut a: BlockChannel<_, 2, 4> = BlockChannel::new(region0, CompletionMode::Poll).unwrap(); + let mut b: BlockChannel<_, 2, 4> = BlockChannel::new(region1, CompletionMode::Poll).unwrap(); + + let buf_a = a.register_buffer(4096).unwrap(); + let buf_b = b.register_buffer(4096).unwrap(); + a.fill_buffer(buf_a, 0xaa).unwrap(); + b.fill_buffer(buf_b, 0xbb).unwrap(); + let wa = a.submit_write(0, 8, buf_a, 0).unwrap(); + let wb = b.submit_write(0, 8, buf_b, 0).unwrap(); + assert_eq!(a.wait().unwrap().request, wa); + assert_eq!(b.wait().unwrap().request, wb); + + a.fill_buffer(buf_a, 0).unwrap(); + let ra = a.submit_read(0, 8, buf_a, 0).unwrap(); + let done = a.wait().unwrap(); + assert_eq!((done.request, done.result), (ra, Ok(()))); + let mut back = [0u8; 4096]; + a.read_buffer(buf_a, 0, &mut back).unwrap(); + assert!(back.iter().all(|&byte| byte == 0xaa), "lane 0 reads its own bytes"); + // Region 1 (LBA 32..) holds B's bytes, not A's. + b.fill_buffer(buf_b, 0).unwrap(); + let rb = b.submit_read(0, 8, buf_b, 0).unwrap(); + assert_eq!(b.wait().unwrap().request, rb); + b.read_buffer(buf_b, 0, &mut back).unwrap(); + assert!(back.iter().all(|&byte| byte == 0xbb)); + assert_eq!(backend.outstanding(0), Some(0)); + let (stats, expired) = backend.stats().unwrap(); + assert_eq!(stats.stale_completions, 0); + assert_eq!(expired, 0); + } + + /// Quiesce is refused while a lane has anything open and stops the + /// device only when the last claimed lane retires. + #[test] + fn the_last_lane_to_quiesce_stops_the_device() { + let backend = backend(); + let mut lane0 = backend.claim(0).unwrap(); + let mut lane1 = backend.claim(1).unwrap(); + backend.control(|q| q.device_mut().hold_completions(true)).unwrap(); + lane0.submit(DeviceCommand::Flush, None, 1).unwrap(); + assert_eq!(lane0.quiesce(), Err(BlockError::Draining)); + backend.control(|q| q.device_mut().hold_completions(false)).unwrap(); + assert_eq!(lane0.wait().map(|c| c.cookie), Some(1)); + assert_eq!(lane0.quiesce(), Ok(())); + assert!(!backend.is_live(0)); + assert!(backend.is_live(1)); + assert_eq!(lane1.quiesce(), Ok(())); + assert_eq!(lane0.submit(DeviceCommand::Flush, None, 2), Err(BlockError::Revoked)); + } + + /// An adopted caller target serves reads on the stream lane, and comes + /// back only once idle. + #[test] + fn an_adopted_target_serves_the_stream_lane_and_releases_when_idle() { + let backend = backend(); + let mut lane1 = backend.claim(1).unwrap(); + let target = backend.control(|q| q.allocate(4096)).unwrap().unwrap(); + let mut buffer = backend.adopt(1, target, TargetAccess::ReadInto).ok().unwrap(); + backend.control(|q| q.device_mut().hold_completions(true)).unwrap(); + lane1 + .submit( + DeviceCommand::Read { + lba: 0, + blocks: 8, + buffer_offset: 0, + }, + Some(&mut buffer), + 5, + ) + .unwrap(); + let (buffer, error) = backend.release(1, buffer).err().unwrap(); + assert_eq!(error, BlockError::BufferBusy); + backend.control(|q| q.device_mut().hold_completions(false)).unwrap(); + assert_eq!(lane1.wait().map(|c| (c.cookie, c.result)), Some((5, Ok(())))); + assert!(backend.release(1, buffer).is_ok()); + } + + /// A client's own window rides the engine beside the lanes: a read + /// grant over part of the disk streams into a client target, an idle + /// pump parks the ring so the next publish owes a notice, the grant + /// refuses what it does not cover, and the ring retires only once + /// detached and its target released. A lane's ring is never reachable + /// through the client calls. + #[test] + fn a_client_ring_streams_through_its_grant_and_parks_when_idle() { + use kore_ring::stor::{Op, Request, StorClient, client, format}; + + let backend = backend(); + let mut lane0 = backend.claim(0).unwrap(); + // Lay a pattern at LBA 8.. through the filesystem lane. + let mut staging = lane0.allocate_buffer(4096).unwrap(); + for (i, byte) in staging.bytes_mut().iter_mut().enumerate() { + *byte = (i % 251) as u8; + } + lane0 + .submit(DeviceCommand::Write { lba: 8, blocks: 8, buffer_offset: 0 }, Some(&mut staging), 1) + .unwrap(); + assert_eq!(lane0.wait().map(|c| c.result), Some(Ok(()))); + + let shared = window(); + format(shared).unwrap(); + // SAFETY: the test plays both parties over one leaked window; each + // side touches it only through the channel protocols. + let alias = unsafe { core::slice::from_raw_parts_mut(shared.as_mut_ptr(), shared.len()) }; + let mut ring_client = StorClient::new(client(alias).unwrap()); + let ring = backend.attach_client(shared, ClassSet::DATA).unwrap(); + assert_eq!(backend.detach_client(backend.lanes[0].ring), Err(BlockError::Range)); + + let grant = backend + .install( + ring, + Grant::new(&[ExtentRun { lba: 8, blocks: 16 }], Rights::READ, ClassSet::DATA).unwrap(), + ) + .unwrap(); + let target = backend.control(|q| q.allocate(4096)).unwrap().unwrap(); + let slot = backend.adopt_client(ring, target, TargetAccess::ReadInto).ok().unwrap(); + let read = |op, offset, user_data| Request { + op, + grant, + offset, + len: 4096, + target: slot, + target_offset: 0, + user_data, + timeline: 1, + deadline_ns: 0, + }; + let ticket = ring_client.submit(Class::Demand, &read(Op::Read, 0, 10)).unwrap().unwrap(); + let _ = ring_client.publish(Class::Demand); + assert_eq!(backend.pump(), Pump::Parked); + assert_eq!(ring_client.poll_ticket(ticket), Ok(true)); + assert_eq!(ring_client.reap().map(|c| c.result), Some(Ok(4096))); + + // Past the grant, and a write it carries no right for. + ring_client.submit(Class::Demand, &read(Op::Read, 16 * 512, 11)).unwrap(); + ring_client.submit(Class::Demand, &read(Op::Write, 0, 12)).unwrap(); + assert!(ring_client.publish(Class::Demand), "the idle pump parked the ring"); + assert_eq!(backend.pump(), Pump::Parked); + assert_eq!(ring_client.reap().map(|c| c.result), Some(Err(StorStatus::Range))); + assert_eq!(ring_client.reap().map(|c| c.result), Some(Err(StorStatus::Rights))); + + assert_eq!(backend.retire_client(ring), Err(BlockError::BufferBusy)); + assert_eq!(backend.detach_client(ring), Ok(Settle::Now)); + assert_eq!(backend.retire_client(ring), Err(BlockError::BufferBusy), "the target is still adopted"); + let back = backend.release_client(ring, slot).unwrap().into_inner(); + assert!( + back.bytes().iter().enumerate().all(|(i, &byte)| byte == (i % 251) as u8), + "the client's target holds the pattern the lane wrote" + ); + assert_eq!(backend.retire_client(ring), Ok(())); + assert_eq!(backend.client_open(ring), None, "the retired id is stale"); + } +} diff --git a/native/crates/driver/src/stor/mod.rs b/native/crates/driver/src/stor/mod.rs new file mode 100644 index 0000000..5613af0 --- /dev/null +++ b/native/crates/driver/src/stor/mod.rs @@ -0,0 +1,282 @@ +//! The **storage transport** engine (`res/STORAGE_TRANSPORT.md` §2). +//! +//! Three layers of the five live here: +//! +//! - **L0, the queue set** ([`QueueSet`]): one worker's slice of a +//! controller — one completion queue and [`QueueSet::SQS`] submission +//! queues. [`Single`] adapts every existing [`DirectBlockDevice`] as a +//! one-queue set; the NVMe multi-queue set lives in `kore-nvme`. +//! - **L1, the engine** ([`engine::StorEngine`]): pops client submissions +//! from each ring's class channels, validates them against the ring's +//! grants and targets, splits them into device commands, keeps every class +//! inside its budget, and posts exactly one completion per submission. It +//! never waits. +//! - The **authority** half of L3 ([`grant`]): what a grant is and how a +//! byte range inside it resolves to device blocks. +//! +//! Everything is generic and fixed-size; a production row instantiates the +//! engine once with its queue set and class policy. Nothing is `dyn`, and +//! nothing allocates. + +pub mod engine; +pub mod grant; +pub mod lanes; +pub mod timeline; + +use core::sync::atomic::AtomicU32; + +use crate::block::{ + BlockBuffer, BlockError, BlockGeometry, BlockHealth, DeviceCommand, DeviceCompletion, + DirectBlockDevice, Liveness, +}; + +/// An engine-minted command tag: the device echoes it back as the +/// completion's cookie, and the engine indexes its command table by it. +/// Opaque to the queue set. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Tag(pub u32); + +impl Tag { + /// The cookie a device carries for this tag. + pub const fn cookie(self) -> u64 { + self.0 as u64 + } +} + +/// **L0** — one worker's slice of a storage controller: a completion queue +/// and `SQS` submission queues, driven only by the one worker that owns it. +/// +/// Implementations execute only when called; they spawn nothing and take no +/// lock. `stage` never rings a doorbell; `publish` rings one per queue. +pub trait QueueSet { + /// Device-visible caller memory a command reads into or writes from. + type Target; + /// Submission queues in this set. + const SQS: usize; + + /// The namespace's geometry. + fn geometry(&self) -> BlockGeometry; + /// Bytes a target spans. + fn target_bytes(target: &Self::Target) -> u64; + /// Allocate `bytes` of device-visible memory the set's owner funds — + /// its own bounce and metadata buffers. Client memory never comes from + /// here; it is adopted. + fn allocate(&mut self, bytes: usize) -> Result; + /// The largest target [`allocate`](Self::allocate) will make. + fn max_target_bytes(&self) -> usize; + /// Block, bounded, until the device posts a completion, and hold it for + /// the next [`reap`](Self::reap). `false` when nothing arrived within + /// the set's own wait budget (or nothing was in flight). A set with no + /// way to sleep may spin; it must still return. + fn park(&mut self) -> bool { + false + } + /// Commands submission queue `sq` can still accept. + fn room(&self, sq: usize) -> usize; + /// Stage one command on `sq`, to be published by [`publish`](Self::publish). + fn stage( + &mut self, + sq: usize, + command: DeviceCommand, + target: Option<&mut Self::Target>, + tag: Tag, + ) -> Result<(), BlockError>; + /// Make every command staged on `sq` visible to the device. + fn publish(&mut self, sq: usize); + /// Drain up to `out.len()` completions, in the order the device posted + /// them. Each carries the staging [`Tag`]'s cookie. + fn reap(&mut self, out: &mut [DeviceCompletion]) -> usize; + /// Apply completion-side data to `target` — a copied transport finishes + /// a read here; DMA has already landed and uses the default. + fn finish(&mut self, _target: Option<&mut Self::Target>) -> Result<(), BlockError> { + Ok(()) + } + /// Out-of-band liveness, never a command ([`DirectBlockDevice::liveness`]). + fn liveness(&mut self) -> Liveness { + Liveness::Unknown + } + /// Physical health on the infrequent control plane + /// ([`DirectBlockDevice::health`]). + fn health(&mut self) -> Result { + Err(BlockError::Unsupported) + } + /// Stop the device once nothing is in flight. + fn quiesce(&mut self) -> Result<(), BlockError> { + Ok(()) + } + /// Monotonic nanoseconds, when the set's task can read a clock. + fn clock_ns(&self) -> Option { + None + } + /// Ask the device to abandon the command staged on `sq` as `tag` — the + /// deadline's escalation ([`DirectBlockDevice::abort`]'s contract: the + /// command's own completion still follows). `false`, the default, when + /// the set has no abort. + fn abort(&mut self, _sq: usize, _tag: Tag) -> bool { + false + } + /// Reset the device and bring every queue back + /// ([`DirectBlockDevice::reset`]'s contract): `true` means every staged + /// command is gone — none completes — and its target is free. `false`, + /// the default, when the set cannot. + fn reset(&mut self) -> bool { + false + } + /// The word a monitored wait can watch for the next completion. + fn completion_word(&self) -> Option<&AtomicU32> { + None + } +} + +/// Every [`DirectBlockDevice`] as a one-queue [`QueueSet`], at most `DEPTH` +/// commands in flight. +/// +/// This is how the existing devices — the memory double, a clamped region, +/// today's NVMe backend — serve the engine without being rewritten. +pub struct Single { + device: D, + outstanding: usize, + /// A completion `park` took from the device's wait, returned first by + /// the next `reap`. + parked: Option, +} + +impl Single { + /// Wrap `device`. + pub const fn new(device: D) -> Self { + Self { + device, + outstanding: 0, + parked: None, + } + } + + /// The wrapped device. + pub fn device(&self) -> &D { + &self.device + } + + /// The wrapped device, mutably — a test's fault knobs, a control step. + pub fn device_mut(&mut self) -> &mut D { + &mut self.device + } + + /// Commands staged and not yet reaped. + pub const fn outstanding(&self) -> usize { + self.outstanding + } + + /// Unwrap, once nothing is outstanding. + pub fn into_device(self) -> Result { + if self.outstanding == 0 { + Ok(self.device) + } else { + Err(self) + } + } +} + +impl QueueSet for Single { + type Target = D::Buffer; + const SQS: usize = 1; + + fn geometry(&self) -> BlockGeometry { + self.device.geometry() + } + + fn target_bytes(target: &Self::Target) -> u64 { + target.len() as u64 + } + + fn allocate(&mut self, bytes: usize) -> Result { + self.device.allocate_buffer(bytes) + } + + fn max_target_bytes(&self) -> usize { + self.device.max_buffer_bytes() + } + + fn park(&mut self) -> bool { + if self.parked.is_some() { + return true; + } + if self.outstanding == 0 { + return false; + } + self.parked = self.device.wait(); + self.parked.is_some() + } + + fn room(&self, _sq: usize) -> usize { + DEPTH.saturating_sub(self.outstanding) + } + + fn stage( + &mut self, + _sq: usize, + command: DeviceCommand, + target: Option<&mut Self::Target>, + tag: Tag, + ) -> Result<(), BlockError> { + if self.outstanding >= DEPTH { + return Err(BlockError::Backpressure); + } + self.device.stage(command, target, tag.cookie())?; + self.outstanding += 1; + Ok(()) + } + + fn publish(&mut self, _sq: usize) { + self.device.publish(); + } + + fn reap(&mut self, out: &mut [DeviceCompletion]) -> usize { + let mut count = 0; + if let Some(first) = out.first_mut() + && let Some(parked) = self.parked.take() + { + *first = parked; + count = 1; + } + while count < out.len() { + let Some(completion) = self.device.poll() else { + break; + }; + out[count] = completion; + count += 1; + } + self.outstanding = self.outstanding.saturating_sub(count); + count + } + + fn finish(&mut self, target: Option<&mut Self::Target>) -> Result<(), BlockError> { + self.device.finish(target) + } + + fn liveness(&mut self) -> Liveness { + self.device.liveness() + } + + fn health(&mut self) -> Result { + self.device.health() + } + + fn quiesce(&mut self) -> Result<(), BlockError> { + if self.outstanding != 0 || self.parked.is_some() { + return Err(BlockError::Draining); + } + self.device.quiesce() + } + + fn clock_ns(&self) -> Option { + self.device.clock_ns() + } + + fn abort(&mut self, _sq: usize, tag: Tag) -> bool { + self.device.abort(tag.cookie()) + } + + fn reset(&mut self) -> bool { + self.device.reset() + } +} diff --git a/native/crates/driver/src/stor/timeline.rs b/native/crates/driver/src/stor/timeline.rs new file mode 100644 index 0000000..0571047 --- /dev/null +++ b/native/crates/driver/src/stor/timeline.rs @@ -0,0 +1,178 @@ +//! **Timelines** — the monotone completion record a client waits on without +//! a call (`res/STORAGE_TRANSPORT.md` §4). +//! +//! Sequences are implicit: every submission naming a timeline takes the +//! next sequence on both sides. Completions arrive in any order; the record +//! publishes `done` as the highest sequence whose predecessors have all +//! completed, and `fault` as the lowest sequence that failed. The pack lane +//! published exactly this; here it is one fixed-size value with no heap. + +use kore_abi::stor::FenceRecord; + +/// Sequences a timeline can hold completed ahead of `done`. Nothing else +/// bounds the gap — completions run out of order, and once `done + 1` is +/// slow the engine keeps popping behind it — so the engine holds a +/// submission whose timeline already has this many open +/// ([`Timeline::open`]). A completion past the window could never be +/// recorded, and the fence would stop at it for good. +pub const WINDOW: u64 = 256; + +const WORDS: usize = (WINDOW / 64) as usize; + +/// One timeline's books. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Timeline { + issued: u64, + done: u64, + fault: u64, + /// Bit `i` set: sequence `done + 1 + i` has completed. + ahead: [u64; WORDS], +} + +/// A completion the timeline could not accept. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TimelineError { + /// The sequence was never issued, is already done, or already + /// completed. + Unknown, + /// The sequence is further than [`WINDOW`] ahead of `done`. + TooFar, +} + +impl Timeline { + /// Take the next sequence for a new submission. + pub fn issue(&mut self) -> u64 { + self.issued += 1; + self.issued + } + + /// Record `seq`'s completion; `ok` false marks it failed. Returns the + /// record to publish. + pub fn complete(&mut self, seq: u64, ok: bool) -> Result { + if seq <= self.done || seq > self.issued { + return Err(TimelineError::Unknown); + } + let offset = seq - self.done - 1; + if offset >= WINDOW { + return Err(TimelineError::TooFar); + } + let (word, bit) = ((offset / 64) as usize, offset % 64); + if self.ahead[word] & (1 << bit) != 0 { + return Err(TimelineError::Unknown); + } + self.ahead[word] |= 1 << bit; + if !ok && (self.fault == 0 || seq < self.fault) { + self.fault = seq; + } + let run = self.leading_run(); + if run > 0 { + self.shift(run); + self.done += run; + } + Ok(self.record()) + } + + /// The record as it stands. + pub const fn record(&self) -> FenceRecord { + FenceRecord { + done: self.done, + fault: self.fault, + } + } + + /// Sequences issued and not yet within `done`. + pub const fn open(&self) -> u64 { + self.issued - self.done + } + + /// Consecutive completed sequences starting at `done + 1`. + fn leading_run(&self) -> u64 { + let mut run = 0; + for word in self.ahead { + let ones = word.trailing_ones() as u64; + run += ones; + if ones < 64 { + break; + } + } + run + } + + /// Drop the lowest `n` bits of the window. + fn shift(&mut self, n: u64) { + let words = (n / 64) as usize; + let bits = (n % 64) as u32; + for index in 0..WORDS { + let low = self.ahead.get(index + words).copied().unwrap_or(0); + let high = self.ahead.get(index + words + 1).copied().unwrap_or(0); + self.ahead[index] = if bits == 0 { + low + } else { + (low >> bits) | (high << (64 - bits)) + }; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn done_is_the_contiguous_prefix_whatever_the_completion_order() { + let mut t = Timeline::default(); + let seqs: [u64; 5] = core::array::from_fn(|_| t.issue()); + assert_eq!(seqs, [1, 2, 3, 4, 5]); + assert_eq!(t.complete(3, true).unwrap().done, 0); + assert_eq!(t.complete(2, true).unwrap().done, 0); + assert_eq!(t.complete(1, true).unwrap().done, 3, "1..=3 now contiguous"); + assert_eq!(t.complete(5, true).unwrap().done, 3); + assert_eq!(t.complete(4, true).unwrap(), FenceRecord { done: 5, fault: 0 }); + assert_eq!(t.open(), 0); + } + + #[test] + fn the_fault_is_the_lowest_failed_sequence_and_sticks() { + let mut t = Timeline::default(); + for _ in 0..4 { + t.issue(); + } + assert_eq!(t.complete(3, false).unwrap().fault, 3); + assert_eq!(t.complete(2, false).unwrap().fault, 2, "a lower failure wins"); + t.complete(1, true).unwrap(); + let record = t.complete(4, true).unwrap(); + assert_eq!(record, FenceRecord { done: 4, fault: 2 }); + t.issue(); + assert_eq!(t.complete(5, true).unwrap().fault, 2, "sticky past done"); + } + + #[test] + fn unknown_duplicate_and_far_completions_are_refused() { + let mut t = Timeline::default(); + assert_eq!(t.complete(1, true), Err(TimelineError::Unknown), "never issued"); + t.issue(); + t.issue(); + t.complete(2, true).unwrap(); + assert_eq!(t.complete(2, true), Err(TimelineError::Unknown), "twice"); + t.complete(1, true).unwrap(); + assert_eq!(t.complete(1, true), Err(TimelineError::Unknown), "already done"); + for _ in 0..=WINDOW { + t.issue(); + } + assert_eq!(t.complete(2 + WINDOW + 1, true), Err(TimelineError::TooFar)); + } + + /// The window shifts across word boundaries without losing a bit. + #[test] + fn a_long_out_of_order_run_shifts_across_words() { + let mut t = Timeline::default(); + for _ in 0..200 { + t.issue(); + } + for seq in (2..=200).rev() { + assert_eq!(t.complete(seq, true).unwrap().done, 0); + } + assert_eq!(t.complete(1, true).unwrap().done, 200); + assert_eq!(t.ahead, [0; WORDS]); + } +} diff --git a/native/crates/late/src/lib.rs b/native/crates/late/src/lib.rs index e78400f..954b484 100644 --- a/native/crates/late/src/lib.rs +++ b/native/crates/late/src/lib.rs @@ -9,16 +9,33 @@ //! grants; the executable authority is the task's own writable→executable //! finalization (§6); the placement is a window the task owns. //! -//! **The gate.** A native load refuses unless the store object carries -//! [`ObjectSubtype::EXECUTABLE_IMAGE`] (the stamp exec's seeder writes) -//! *and* its bytes pass [`check_image`]. The stamp is a fact the store -//! carries, never authority — a plain file that happens to parse as an ELF -//! is still refused, and the subtype still grants nothing. +//! **Where native code comes from.** A native module runs with the task's +//! whole authority, so it enters only with the provenance a native image +//! needs, and the loader — not each caller — holds that rule on two +//! independent counts: //! -//! **The search list is a capability list.** `require "x"` walks an -//! ordered set of path prefixes the task can already open — its own tree -//! (`""`) and the `libs` view — and "not found" is "no route". Source -//! modules are `.lua`, native ones `.so`. +//! - *The route.* `.so` resolves only in the task's library view +//! ([`LIBS_VIEW`]), entered as the granted authority edge it is — never by +//! walking the path `libs/…`, which a `libs` child of the task's own root +//! would answer first — and [`library_grant`] refuses that edge if it +//! would let the task write or rename beneath it. Never the task's own +//! tree: a tree the task can write is exactly where native code must not +//! come from, and a squatting `.so` there must not shadow the library. +//! - *The stamp.* The object must carry +//! [`ObjectSubtype::EXECUTABLE_IMAGE`] *and* its bytes must pass +//! [`check_image`]. The store takes that stamp — and any later change to +//! the stamped bytes — only through `BIND_EXECUTABLE` +//! (`ObjectSubtype::stamp_rights`), the install authority's right, which +//! no task's home face carries. So the stamp is the store's word that the +//! installer wrote these bytes; it still grants the reader nothing, and a +//! plain file that happens to parse as an ELF is refused. +//! +//! **Source modules search; native modules do not.** A source module +//! `.lua` runs inside the interpreter's VM with only what the VM grants, +//! so `Loader::read_source` walks the caller's [`Search`] — an ordered set +//! of path prefixes the task can already open, typically its own tree +//! (`""`) then `libs` — and "not found" is "no route". `Loader::load_native` +//! takes no search list. //! //! **Placement.** Modules go above everything the startup stub mapped //! ([`ModulePage::high_water`]) inside `geom::LINK`, each at a 64 KiB @@ -40,7 +57,11 @@ extern crate alloc; use kore_abi::image::{ModuleDescriptor, ModulePage, seg}; -use kore_dagfs::{FsError, Name, Refusal, bounds::MAX_NAME_LEN, bounds::MAX_PATH_DEPTH}; +pub use kore_dagfs::LIBS_VIEW; +use kore_dagfs::{ + FsError, GrantEntry, GrantId, Name, Refusal, Rights, bounds::MAX_NAME_LEN, + bounds::MAX_PATH_DEPTH, +}; use kore_ld::apply::ApplyError; use kore_ld::check::CheckError; use kore_ld::late::LateError; @@ -91,8 +112,14 @@ impl ModuleKind { pub enum LoadError { /// The stem does not spell a store name once suffixed. Name, - /// No prefix of the search list holds the object. + /// No prefix of the search list holds the object — for a native module, + /// the task has no [`LIBS_VIEW`] edge or the library holds no + /// `.so`. NotFound, + /// The task's [`LIBS_VIEW`] edge would let it write bytes or names + /// beneath it: a native module never comes from a route its loader can + /// change. + LibsWritable, /// The object is not stamped `EXECUTABLE_IMAGE`. NotExecutableImage, /// The object exceeds [`MAX_MODULE_BYTES`], or the caller's buffer. @@ -147,13 +174,36 @@ impl From for LoadError { } } -/// The ordered prefixes a load may resolve under — each one a directory -/// the task can already open; `b""` is its own tree. +/// The ordered prefixes a *source* load may resolve under — each one a +/// directory the task can already open; `b""` is its own tree. A native +/// load takes none: it resolves only through [`library_grant`]. #[derive(Clone, Copy, Debug)] pub struct Search<'a> { pub prefixes: &'a [&'a [u8]], } +/// Rights a native module's route must not carry: either would let the +/// loading task change what it is about to link. +const LIBRARY_ROUTE_FORBIDS: Rights = Rights::WRITE.union(Rights::MUTATE_NAMESPACE); + +/// Admit the task's library edge — the grant its view lists under +/// [`LIBS_VIEW`], if any — as the one place native modules come from. +/// +/// No edge is no route ([`LoadError::NotFound`]). An edge that would let +/// this task write a byte or a name beneath it is refused +/// ([`LoadError::LibsWritable`]) rather than trusted for its name: the +/// system view is minted `READ` alone, so a writable `libs` is some other +/// edge wearing the name (a self-share where the view's install collided, +/// say). `SHARE` and `BIND_EXECUTABLE` change nothing the loader reads and +/// are not this check's business. +pub fn library_grant(entry: Option) -> Result { + let entry = entry.ok_or(LoadError::NotFound)?; + if entry.is_private_root || !entry.rights.intersection(LIBRARY_ROUTE_FORBIDS).is_empty() { + return Err(LoadError::LibsWritable); + } + Ok(entry.grant) +} + /// `` as a store name, or `None` past [`MAX_NAME_LEN`] or /// for a stem that is not a name (empty, `/`, `.`, `..`, NUL). pub fn object_name(kind: ModuleKind, stem: &[u8]) -> Option { @@ -192,7 +242,6 @@ pub fn compose_path<'b>( Some(&out[..end]) } - /// The host image as the late scope's module 0: its base from the passive /// page's first record (verified against the executable's own name), its /// descriptor from `.kore.ld`. @@ -280,6 +329,58 @@ mod tests { assert!(object_name(ModuleKind::Native, &over).is_none()); } + fn libs_entry(rights: Rights, is_private_root: bool) -> GrantEntry { + GrantEntry { + grant: GrantId::from_bits(7), + rights, + is_private_root, + display_name: Name::from_bytes(LIBS_VIEW.as_bytes()).ok(), + } + } + + /// The system library view is minted `READ` alone, and that is what a + /// native module may come through. Re-sharing or install authority on + /// the edge changes nothing the loader reads. + #[test] + fn the_read_only_library_edge_is_where_native_modules_come_from() { + for rights in [ + Rights::READ_ONLY, + Rights::READ.union(Rights::SHARE), + Rights::READ.union(Rights::BIND_EXECUTABLE), + ] { + assert_eq!( + library_grant(Some(libs_entry(rights, false))), + Ok(GrantId::from_bits(7)) + ); + } + } + + /// A task with no library view has no route to native code — and a + /// `libs` edge the task could write through, or its own tree, is never + /// trusted for its name: that would load code from where the loader + /// itself can put it. + #[test] + fn a_missing_or_writable_library_edge_is_refused() { + assert_eq!(library_grant(None), Err(LoadError::NotFound)); + for rights in [ + Rights::READ.union(Rights::WRITE), + Rights::READ.union(Rights::MUTATE_NAMESPACE), + Rights::HOME, + Rights::ALL, + ] { + assert_eq!( + library_grant(Some(libs_entry(rights, false))), + Err(LoadError::LibsWritable), + "{rights:?}" + ); + } + assert_eq!( + library_grant(Some(libs_entry(Rights::READ_ONLY, true))), + Err(LoadError::LibsWritable), + "the task's own root is never the library" + ); + } + #[test] fn paths_compose_under_a_prefix_or_stand_alone() { let name = object_name(ModuleKind::Native, b"demo").unwrap(); @@ -301,10 +402,16 @@ mod tests { let limit = 0x5800_0000u64; let mut from_unaligned = Placer::new(0x5001_0001, limit - 0x5001_0001); - assert_eq!(from_unaligned.choose(0x3000, LATE_BASE_ALIGN), Ok(0x5002_0000)); + assert_eq!( + from_unaligned.choose(0x3000, LATE_BASE_ALIGN), + Ok(0x5002_0000) + ); let mut from_aligned = Placer::new(0x5002_0000, limit - 0x5002_0000); - assert_eq!(from_aligned.choose(0x3000, LATE_BASE_ALIGN), Ok(0x5002_0000)); + assert_eq!( + from_aligned.choose(0x3000, LATE_BASE_ALIGN), + Ok(0x5002_0000) + ); let mut exact = Placer::new(0x57ff_0000, limit - 0x57ff_0000); assert_eq!(exact.choose(0x1_0000, LATE_BASE_ALIGN), Ok(0x57ff_0000)); @@ -316,7 +423,10 @@ mod tests { ); let mut ceiling = Placer::new(u64::MAX - 1, 1); - assert_eq!(ceiling.choose(1, LATE_BASE_ALIGN), Err(PlaceError::Exhausted)); + assert_eq!( + ceiling.choose(1, LATE_BASE_ALIGN), + Err(PlaceError::Exhausted) + ); } #[test] diff --git a/native/crates/late/src/metal.rs b/native/crates/late/src/metal.rs index 357dcfd..d5aef54 100644 --- a/native/crates/late/src/metal.rs +++ b/native/crates/late/src/metal.rs @@ -20,8 +20,8 @@ use kore_rt::cap::{ExecutableCode, ExecutableMapping, MappedLeaseView, StashedLe use kore_rt::entry::Rt; use crate::{ - HostModule, LoadError, MAX_LOADED_MODULES, MAX_MODULE_BYTES, MAX_SEARCH_PATH, ModuleKind, - Search, compose_path, object_name, read_views, + HostModule, LIBS_VIEW, LoadError, MAX_LOADED_MODULES, MAX_MODULE_BYTES, MAX_SEARCH_PATH, + ModuleKind, Search, compose_path, library_grant, object_name, read_views, }; /// Bytes the TLS layout's tables may carve for a scope of at most @@ -169,11 +169,15 @@ impl Loader { result } - /// Link `.so` from the search list into the task. + /// Link `.so` from the task's library view into the task. + /// + /// No search list: native code resolves only through the READ-only + /// [`LIBS_VIEW`](crate::LIBS_VIEW) edge ([`open_library`]), never the + /// task's own tree — the crate doc's provenance rule, held here rather + /// than by each caller's choice of prefixes. pub async fn load_native( &mut self, fs: &mut Fs<'_, '_>, - search: Search<'_>, stem: &[u8], ) -> Result { if let Some(id) = self.find(stem) { @@ -183,7 +187,7 @@ impl Loader { return Err(LoadError::TooMany); } let name = object_name(ModuleKind::Native, stem).ok_or(LoadError::Name)?; - let view = open_in(fs, search, &name).await?; + let view = open_library(fs, &name).await?; let bytes = match Self::stage(fs, &view).await { Ok(bytes) => { fs.close(view).await; @@ -478,6 +482,22 @@ impl Loader { } } +/// Open `name` in the task's library view. +/// +/// The edge is entered as the grant it is ([`Fs::named_grant_entry`] then +/// [`Fs::open_grant`]), not by walking `libs/`: a path walk asks the +/// task's own root first, so a `libs` directory the task made there (where +/// the view's install collided) would answer instead. [`library_grant`] +/// refuses an edge the task could write beneath. +async fn open_library(fs: &mut Fs<'_, '_>, name: &Name) -> Result { + let view = Name::from_bytes(LIBS_VIEW.as_bytes()).map_err(|_| LoadError::Name)?; + let grant = library_grant(fs.named_grant_entry(&view).await?)?; + let libs = fs.open_grant(grant, WALK_READ).await?; + let member = fs.open_at(&libs, *name, WALK_READ).await; + fs.close(libs).await; + member.map_err(LoadError::from) +} + /// Open `name` under the first prefix that holds it. async fn open_in(fs: &mut Fs<'_, '_>, search: Search<'_>, name: &Name) -> Result { let mut buf = [0u8; MAX_SEARCH_PATH]; diff --git a/native/crates/ld/src/place.rs b/native/crates/ld/src/place.rs index c6053b4..3549f58 100644 --- a/native/crates/ld/src/place.rs +++ b/native/crates/ld/src/place.rs @@ -345,7 +345,12 @@ mod tests { /// this module's doc says that failure is silent rather than loud. #[test] fn shared_and_owned_placers_agree() { - let requests = [(0x1234, 0x1000), (0x1, 0x1_0000), (0x2000, 0x1000), (0x8, 0x8)]; + let requests = [ + (0x1234, 0x1000), + (0x1, 0x1_0000), + (0x2000, 0x1000), + (0x8, 0x8), + ]; let mut owned = Placer::new(0x1000_0000, 0x10_0000); let shared = SharedPlacer::new(0x1000_0000, 0x10_0000); for (len, align) in requests { @@ -396,11 +401,18 @@ mod tests { assert_eq!(spans.len(), THREADS * EACH); spans.sort_unstable(); spans.dedup(); - assert_eq!(spans.len(), THREADS * EACH, "an address was handed out twice"); + assert_eq!( + spans.len(), + THREADS * EACH, + "an address was handed out twice" + ); for pair in spans.windows(2) { assert!(pair[0] + LEN <= pair[1], "spans overlap"); } - assert!(spans.iter().all(|at| at % LEN == 0), "a span was misaligned"); + assert!( + spans.iter().all(|at| at % LEN == 0), + "a span was misaligned" + ); } /// Fail-closed: a `static` placer refuses until its window is known, /// rather than answering an address in an unpublished window. diff --git a/native/crates/ld/tests/built_images.rs b/native/crates/ld/tests/built_images.rs index fdfe086..b3277fb 100644 --- a/native/crates/ld/tests/built_images.rs +++ b/native/crates/ld/tests/built_images.rs @@ -21,8 +21,11 @@ //! `KORE_USER_IMAGE_ARCH`, when set, closes the walk to exactly one of //! `x86_64`, `aarch64`, or `riscv64`. Production packaging uses that //! selector so a partial build for another ISA cannot become a prerequisite -//! merely because its target directory exists. With no selector every ISA -//! present is checked, as in `make user-verify`. +//! merely because its target directory exists. `KORE_USER_IMAGE_PROFILE` +//! further selects the exact baseline/v2/v3/v4 directory produced by that +//! build, so a stale partial sibling cannot poison a production install. With +//! no selectors every ISA and profile present is checked, as in +//! `make user-verify`. //! //! `KORE_USER_IMAGES` unset ⇒ skip: `make test` stays hermetic. An arch whose //! spec dir is absent is skipped (a gated-out ISA), but a set root with *no* @@ -259,56 +262,73 @@ enum ImageKind { }, } -/// Check one arch across its SIMD profile variants: the baseline spec -/// dir and its `-v2` / `-v3` / `-v4` siblings. Today v2 is the Oryon-1 -/// AArch64 profile (`USER_SIMD=v2`), while v3/v4 are the x86-64 psABI -/// classes (`USER_SIMD=v3|v4`). Every variant present is checked — -/// a stale baseline build beside a fresh machine profile both face the -/// regime; an absent variant is skipped (a gated-out ISA / an unselected -/// profile). -fn check_arch(root: &Path, spec: &str) -> usize { - let mut checked = 0; - for variant in [ +/// Verify one exact profile directory. Aggregate callers may treat absence as +/// a skipped profile; production callers require the directory they built. +fn check_variant(root: &Path, variant: &str, required: bool) -> usize { + let dir = root.join(variant).join("release"); + if !dir.is_dir() { + assert!( + !required, + "built_images: required profile {variant} not built" + ); + println!("built_images: {variant} not built — skipped"); + return 0; + } + for (name, kind) in [ + ("libdemo.so", ImageKind::DemoLibrary), + ("libpci_names.so", ImageKind::SharedLibrary), + ("hello", ImageKind::NeedsLibrary("libdemo.so")), + ("time", ImageKind::Executable), + ("sleep", ImageKind::Executable), + ("status", ImageKind::NeedsLibrary("libpci_names.so")), + ("lspci", ImageKind::NeedsLibrary("libpci_names.so")), + ("help", ImageKind::Executable), + ("window", ImageKind::Executable), + ("clear", ImageKind::Executable), + ("inference", ImageKind::Executable), + ("lua", ImageKind::ExportsApi("lua_pushcclosure")), + ( + "liblua_demo.so", + ImageKind::LateModule { + opener: "luaopen_lua_demo", + host_prefix: "lua_", + }, + ), + ("kore-terminal", ImageKind::Executable), + ("kore-editor", ImageKind::Executable), + ("kore-aftershock", ImageKind::Executable), + ("kore-skylilt", ImageKind::Executable), + ] { + println!("built_images: {variant}/{name}"); + verify_image::(&dir.join(name), kind); + } + 17 +} + +fn profile_belongs_to(profile: &str, spec: &str) -> bool { + profile == spec + || ["-v2", "-v3", "-v4"] + .iter() + .any(|suffix| profile == format!("{spec}{suffix}")) +} + +fn check_arch(root: &Path, spec: &str, selected: Option<&str>) -> usize { + if let Some(profile) = selected { + assert!( + profile_belongs_to(profile, spec), + "KORE_USER_IMAGE_PROFILE={profile:?} does not belong to {spec}" + ); + return check_variant::(root, profile, true); + } + [ String::from(spec), format!("{spec}-v2"), format!("{spec}-v3"), format!("{spec}-v4"), - ] { - let dir = root.join(&variant).join("release"); - if !dir.is_dir() { - println!("built_images: {variant} not built — skipped"); - continue; - } - for (name, kind) in [ - ("libdemo.so", ImageKind::DemoLibrary), - ("libpci_names.so", ImageKind::SharedLibrary), - ("hello", ImageKind::NeedsLibrary("libdemo.so")), - ("time", ImageKind::Executable), - ("sleep", ImageKind::Executable), - ("status", ImageKind::NeedsLibrary("libpci_names.so")), - ("lspci", ImageKind::NeedsLibrary("libpci_names.so")), - ("help", ImageKind::Executable), - ("window", ImageKind::Executable), - ("clear", ImageKind::Executable), - ("lua", ImageKind::ExportsApi("lua_pushcclosure")), - ( - "liblua_demo.so", - ImageKind::LateModule { - opener: "luaopen_lua_demo", - host_prefix: "lua_", - }, - ), - ("kore-terminal", ImageKind::Executable), - ("kore-editor", ImageKind::Executable), - ("kore-aftershock", ImageKind::Executable), - ("kore-skylilt", ImageKind::Executable), - ] { - println!("built_images: {variant}/{name}"); - verify_image::(&dir.join(name), kind); - } - checked += 16; - } - checked + ] + .iter() + .map(|variant| check_variant::(root, variant, false)) + .sum() } #[test] @@ -319,15 +339,26 @@ fn built_images_satisfy_regime() { }; let root = PathBuf::from(root); let requested = std::env::var("KORE_USER_IMAGE_ARCH").ok(); + let selected = std::env::var("KORE_USER_IMAGE_PROFILE").ok(); + assert!( + requested.is_some() || selected.is_none(), + "KORE_USER_IMAGE_PROFILE requires KORE_USER_IMAGE_ARCH" + ); let n = match requested.as_deref() { None => { - check_arch::(&root, "x86_64-unknown-kore-user") - + check_arch::(&root, "aarch64-unknown-kore-user") - + check_arch::(&root, "riscv64gc-unknown-kore-user") + check_arch::(&root, "x86_64-unknown-kore-user", None) + + check_arch::(&root, "aarch64-unknown-kore-user", None) + + check_arch::(&root, "riscv64gc-unknown-kore-user", None) + } + Some("x86_64") => { + check_arch::(&root, "x86_64-unknown-kore-user", selected.as_deref()) + } + Some("aarch64") => { + check_arch::(&root, "aarch64-unknown-kore-user", selected.as_deref()) + } + Some("riscv64") => { + check_arch::(&root, "riscv64gc-unknown-kore-user", selected.as_deref()) } - Some("x86_64") => check_arch::(&root, "x86_64-unknown-kore-user"), - Some("aarch64") => check_arch::(&root, "aarch64-unknown-kore-user"), - Some("riscv64") => check_arch::(&root, "riscv64gc-unknown-kore-user"), Some(other) => panic!( "KORE_USER_IMAGE_ARCH={other:?} is unsupported; expected x86_64, aarch64, or riscv64" ), diff --git a/native/crates/ld/tests/link_engine.rs b/native/crates/ld/tests/link_engine.rs index 417cee4..aa9c9df 100644 --- a/native/crates/ld/tests/link_engine.rs +++ b/native/crates/ld/tests/link_engine.rs @@ -1350,7 +1350,11 @@ fn built_executables_link_under_the_stub_caps() { "KORE_USER_IMAGES={} selected no linkable executables", dir.display() ); - println!("linking {} built executables: {}", executables.len(), executables.join(", ")); + println!( + "linking {} built executables: {}", + executables.len(), + executables.join(", ") + ); // A fresh world per link: the engine consumes leases, so a world that // already linked one program is not a clean stand-in for the founding of @@ -1398,7 +1402,10 @@ fn a_reported_placement_cursor_is_stamped_below_the_page() { world.bytes_at(prog.module_page, &mut bytes); let page = ModulePage::read_from_bytes(&bytes).unwrap(); assert_eq!(page.placed.get(), 3); - assert_eq!(page.placed_high_water(prog.module_page), Some(page_va - 3 * PAGE)); + assert_eq!( + page.placed_high_water(prog.module_page), + Some(page_va - 3 * PAGE) + ); } /// A world that does not record placement stamps nothing, and the page @@ -1424,8 +1431,9 @@ fn an_unrepresentable_placement_cursor_refuses_the_link() { diamond(&mut world, &X86_RELOCS); world.page_at = Some(page_va); world.placed_high_water = Some(bad); - let err = link_program::(&mut world, &name("exe"), arena(), LinkCaps::DEFAULT) - .expect_err("a lying placement cursor refuses"); + let err = + link_program::(&mut world, &name("exe"), arena(), LinkCaps::DEFAULT) + .expect_err("a lying placement cursor refuses"); assert!(matches!(err, LinkError::Layout(_)), "{err:?}"); } } diff --git a/native/crates/ring/src/event.rs b/native/crates/ring/src/event.rs index f884384..a6bc839 100644 --- a/native/crates/ring/src/event.rs +++ b/native/crates/ring/src/event.rs @@ -397,6 +397,11 @@ impl EventSink<'_, S, R> { self.records.pop() } + /// The oldest record, left in place. + pub fn peek(&self) -> Option { + self.records.peek() + } + /// Records the producer dropped on a full ring — the counter lives in /// the shared window, so a consumer can see it fell behind without the /// producer having to encode that into state. diff --git a/native/crates/ring/src/lib.rs b/native/crates/ring/src/lib.rs index 05c717b..f9e7f61 100644 --- a/native/crates/ring/src/lib.rs +++ b/native/crates/ring/src/lib.rs @@ -55,6 +55,7 @@ extern crate std; pub mod event; pub mod spsc; +pub mod stor; pub mod transfer; pub mod uring; diff --git a/native/crates/ring/src/stor.rs b/native/crates/ring/src/stor.rs new file mode 100644 index 0000000..e81e0d2 --- /dev/null +++ b/native/crates/ring/src/stor.rs @@ -0,0 +1,686 @@ +//! The **storage transport window** (`res/STORAGE_TRANSPORT.md` §4): the +//! frozen layout one client-funded window holds, and the halves each side +//! attaches. +//! +//! ```text +//! offset region direction +//! DEMAND_AT event SQ_DEPTH client → row +//! STREAM_AT event SQ_DEPTH client → row +//! COMMIT_AT event COMMIT_SQ_DEPTH client → row +//! CQ_AT event CQ_DEPTH row → client +//! FENCE_AT [FenceRecord; TIMELINES] on its own page row → client +//! ``` +//! +//! Every submission and completion channel is an [`event`](crate::event) +//! channel, so a waiter gets the wait ladder and a producer's kick is +//! conditional on the other side having parked. The layout is compiled into +//! both sides — nothing in it is negotiated — and the client formats the +//! window before lending it, as a vif client formats its rings +//! (`res/NETWORK_SERVICE.md` §3). The row attaches the formatted window and +//! refuses one whose channels do not validate. +//! +//! The Commit channel is always laid out; whether the row serves it is the +//! ring's policy, not the layout's. + +use core::marker::PhantomData; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; + +use kore_abi::stor::{ + COMMIT_SQ_DEPTH, CQ_DEPTH, Class, CqState, FENCE_PAGE_BYTES, FenceRecord, GrantId, SQ_DEPTH, + SqState, StorStatus, TIMELINES, +}; +use kore_abi::uring::{IoUringCqe, IoUringSqe, op}; +use zerocopy::FromZeros; + +use crate::event::{self, EventSink, EventSource}; +use crate::spsc::RingError; + +const LINE: usize = 64; + +const fn up(value: usize, to: usize) -> usize { + value.div_ceil(to) * to +} + +const fn sq_bytes(depth: u32) -> usize { + event::region_bytes::(depth) +} + +/// Offset of the Demand submission channel. +pub const DEMAND_AT: usize = 0; +/// Offset of the Stream submission channel. +pub const STREAM_AT: usize = up(DEMAND_AT + sq_bytes(SQ_DEPTH), LINE); +/// Offset of the Commit submission channel. +pub const COMMIT_AT: usize = up(STREAM_AT + sq_bytes(SQ_DEPTH), LINE); +/// Offset of the completion channel. +pub const CQ_AT: usize = up(COMMIT_AT + sq_bytes(COMMIT_SQ_DEPTH), LINE); +/// Offset of the fence page. +pub const FENCE_AT: usize = up( + CQ_AT + event::region_bytes::(CQ_DEPTH), + FENCE_PAGE_BYTES, +); +/// Bytes one window occupies — a whole number of pages. +pub const WINDOW_BYTES: usize = FENCE_AT + FENCE_PAGE_BYTES; + +const _: () = assert!(WINDOW_BYTES % FENCE_PAGE_BYTES == 0); + +/// Offset and depth of `class`'s submission channel. +pub const fn channel(class: Class) -> (usize, u32) { + match class { + Class::Demand => (DEMAND_AT, SQ_DEPTH), + Class::Stream => (STREAM_AT, SQ_DEPTH), + Class::Commit => (COMMIT_AT, COMMIT_SQ_DEPTH), + } +} + +const fn channel_end(class: Class) -> usize { + match class { + Class::Demand => STREAM_AT, + Class::Stream => COMMIT_AT, + Class::Commit => CQ_AT, + } +} + +/// Why a window could not be formatted or attached. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WindowError { + /// The window does not start on a page boundary. + Misaligned, + /// The window is shorter than [`WINDOW_BYTES`]. + TooSmall, + /// A channel did not validate. + Channel(Class, RingError), + /// The completion channel did not validate. + Completion(RingError), +} + +fn check(window: &[u8]) -> Result<(), WindowError> { + if window.as_ptr().align_offset(FENCE_PAGE_BYTES) != 0 { + return Err(WindowError::Misaligned); + } + if window.len() < WINDOW_BYTES { + return Err(WindowError::TooSmall); + } + Ok(()) +} + +/// **Format** a fresh window: every channel empty, every fence zero. The +/// client does this before lending the window; exclusive access, not yet +/// shared. +pub fn format(window: &mut [u8]) -> Result<(), WindowError> { + check(window)?; + for class in Class::ALL { + let (at, depth) = channel(class); + event::init::(&mut window[at..channel_end(class)], depth) + .map_err(|error| WindowError::Channel(class, error))?; + } + event::init::(&mut window[CQ_AT..FENCE_AT], CQ_DEPTH) + .map_err(WindowError::Completion)?; + window[FENCE_AT..WINDOW_BYTES].fill(0); + Ok(()) +} + +/// One timeline's fence, as atomics over a [`FenceRecord`]'s bytes. +#[repr(C)] +pub struct FenceCell { + done: AtomicU64, + fault: AtomicU64, +} + +const _: () = assert!(core::mem::size_of::() == core::mem::size_of::()); + +impl FenceCell { + /// Read the record: `done` with acquire ordering, then `fault` — so a + /// fault at or below the `done` just read is never missed. + pub fn load(&self) -> FenceRecord { + let done = self.done.load(Ordering::Acquire); + let fault = self.fault.load(Ordering::Acquire); + FenceRecord { done, fault } + } + + /// Publish (the row's side): `fault` first, then `done`, both release. + pub fn publish(&self, record: FenceRecord) { + self.fault.store(record.fault, Ordering::Release); + self.done.store(record.done, Ordering::Release); + } + + /// The low 32 bits of `done` — the word a monitored wait watches + /// (kore's targets are little-endian, so the low half is the first + /// word). Any publish that advances `done` by less than 2^32 changes it. + pub fn done_word(&self) -> &AtomicU32 { + // SAFETY: `done` is an 8-byte-aligned AtomicU64; its first four + // bytes are a valid, aligned AtomicU32 on a little-endian target, + // and every access to them is atomic. + unsafe { &*(core::ptr::from_ref(&self.done).cast::()) } + } +} + +const _: () = assert!(cfg!(target_endian = "little")); + +/// The window's fence page. +pub struct Fences<'a> { + base: *const FenceCell, + _window: PhantomData<&'a [u8]>, +} + +// SAFETY: every access is atomic. +unsafe impl Send for Fences<'_> {} + +impl<'a> Fences<'a> { + /// Timeline `t`'s fence (`1..=TIMELINES`); `None` for 0 or past the end. + /// The cell lives as long as the window, not this handle: it is all + /// atomics, so a waiter may hold it while the ring is driven elsewhere. + pub fn timeline(&self, t: u16) -> Option<&'a FenceCell> { + let index = usize::from(t).checked_sub(1)?; + if index >= TIMELINES { + return None; + } + // SAFETY: `base` is the page-aligned fence page of a validated + // window, holding TIMELINES cells. + Some(unsafe { &*self.base.add(index) }) + } +} + +/// # Safety +/// `base..base + WINDOW_BYTES` is a validated window pinned for `'a`. +unsafe fn fences<'a>(base: *mut u8) -> Fences<'a> { + Fences { + // SAFETY: FENCE_AT is inside the window and page-aligned. + base: unsafe { base.add(FENCE_AT) }.cast::(), + _window: PhantomData, + } +} + +/// The client's halves: the submission producers, the completion consumer, +/// the fences. +pub struct ClientRing<'a> { + /// Submission producers, indexed by [`Class::index`]. + pub sq: [EventSource<'a, SqState, IoUringSqe>; Class::COUNT], + /// The completion consumer. + pub cq: EventSink<'a, CqState, IoUringCqe>, + /// The fence page. + pub fences: Fences<'a>, +} + +/// The row's halves: the submission consumers, the completion producer, +/// the fences. +pub struct ServiceRing<'a> { + /// Submission consumers, indexed by [`Class::index`]. + pub sq: [EventSink<'a, SqState, IoUringSqe>; Class::COUNT], + /// The completion producer. + pub cq: EventSource<'a, CqState, IoUringCqe>, + /// The fence page. + pub fences: Fences<'a>, +} + +/// The five disjoint regions, as sub-slices of one window. +fn regions(window: &mut [u8]) -> [&mut [u8]; 5] { + let (demand, rest) = window[..WINDOW_BYTES].split_at_mut(STREAM_AT); + let (stream, rest) = rest.split_at_mut(COMMIT_AT - STREAM_AT); + let (commit, rest) = rest.split_at_mut(CQ_AT - COMMIT_AT); + let (cq, fence) = rest.split_at_mut(FENCE_AT - CQ_AT); + [demand, stream, commit, cq, fence] +} + +fn sq_source( + class: Class, + bytes: &mut [u8], +) -> Result, WindowError> { + event::source::(bytes).map_err(|error| WindowError::Channel(class, error)) +} + +fn sq_sink(class: Class, bytes: &mut [u8]) -> Result, WindowError> { + event::sink::(bytes).map_err(|error| WindowError::Channel(class, error)) +} + +/// Attach the **client** halves of a formatted window (its own mapping). +pub fn client(window: &mut [u8]) -> Result, WindowError> { + check(window)?; + let base = window.as_mut_ptr(); + let [demand, stream, commit, cq, _] = regions(window); + let source = sq_source; + Ok(ClientRing { + sq: [ + source(Class::Demand, demand)?, + source(Class::Stream, stream)?, + source(Class::Commit, commit)?, + ], + cq: event::sink::(cq).map_err(WindowError::Completion)?, + // SAFETY: the window was validated and stays borrowed for 'a. + fences: unsafe { fences(base) }, + }) +} + +/// Attach the **row's** halves of a formatted window (its own mapping). +pub fn service(window: &mut [u8]) -> Result, WindowError> { + check(window)?; + let base = window.as_mut_ptr(); + let [demand, stream, commit, cq, _] = regions(window); + let sink = sq_sink; + Ok(ServiceRing { + sq: [ + sink(Class::Demand, demand)?, + sink(Class::Stream, stream)?, + sink(Class::Commit, commit)?, + ], + cq: event::source::(cq).map_err(WindowError::Completion)?, + // SAFETY: as `client`. + fences: unsafe { fences(base) }, + }) +} + +/// [`format`] + both sides over one exclusively borrowed window — the +/// same-address-space (and host-test) form, as [`event::split`]. +pub fn split(window: &mut [u8]) -> Result<(ClientRing<'_>, ServiceRing<'_>), WindowError> { + format(window)?; + let (base, len) = (window.as_mut_ptr(), window.len()); + // SAFETY: the `&mut` borrow pins the window for the returned lifetime; + // the two sides touch it only through the channel protocols (disjoint + // roles, atomics, seqlocks, spsc hand-off) and the atomic fences. + let (a, b) = unsafe { + ( + core::slice::from_raw_parts_mut(base, len), + core::slice::from_raw_parts_mut(base, len), + ) + }; + Ok((client(a)?, service(b)?)) +} + +/// One submission, as the client builds it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Request { + /// What to do. + pub op: Op, + /// The grant it acts inside. + pub grant: GrantId, + /// Byte offset within the grant (block-aligned). + pub offset: u64, + /// Bytes (a block multiple; 0 for a flush). + pub len: u32, + /// The target slot (ignored by flush and discard). + pub target: u16, + /// Byte offset within the target (block-aligned). + pub target_offset: u64, + /// Echoed back on the completion. + pub user_data: u64, + /// Timeline `1..=TIMELINES`, or 0 for none. + pub timeline: u16, + /// Absolute uptime deadline in ns, or 0 for the class default. + pub deadline_ns: u64, +} + +/// A submission's operation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Op { + /// Read from the grant into the target. + Read, + /// Write from the target into the grant. + Write, + /// Flush the device's volatile cache. + Flush, + /// Discard a range of the grant. + Discard, +} + +impl Request { + /// The wire entry. + pub fn entry(&self) -> IoUringSqe { + let mut sqe = IoUringSqe::new_zeroed(); + sqe.opcode = match self.op { + Op::Read => op::READ_FIXED, + Op::Write => op::WRITE_FIXED, + Op::Flush => op::FSYNC, + Op::Discard => op::DISCARD, + }; + sqe.fd = self.grant.fd(); + sqe.off = self.offset; + sqe.len = self.len; + sqe.buf_index = self.target; + sqe.addr = self.target_offset; + sqe.user_data = self.user_data; + sqe.personality = self.timeline; + sqe.addr3 = self.deadline_ns; + sqe + } +} + +/// Where a submission lands on its timeline: complete when the fence's +/// `done` reaches `seq`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Ticket { + /// The timeline, `1..=TIMELINES`. + pub timeline: u16, + /// The sequence the submission took. + pub seq: u64, +} + +/// A completion, decoded. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Completion { + /// The submission's token. + pub user_data: u64, + /// Bytes moved, or why not. + pub result: Result, +} + +/// Why a submission was not queued. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SubmitError { + /// The class channel is full. + Full, + /// The timeline is outside `0..=TIMELINES`. + Timeline, +} + +/// The client side of one ring: the halves plus each timeline's sequence +/// counter, which must match the row's count exactly — every submission +/// naming a timeline takes a sequence, including one that later fails. +pub struct StorClient<'a> { + ring: ClientRing<'a>, + issued: [u64; TIMELINES], +} + +impl<'a> StorClient<'a> { + /// Drive `ring`. Its timelines must be fresh (a newly formatted window). + pub fn new(ring: ClientRing<'a>) -> Self { + Self { + ring, + issued: [0; TIMELINES], + } + } + + /// Queue `request` on `class`'s channel (not yet visible to the row — + /// see [`publish`](Self::publish)). The ticket, when it names a + /// timeline, is what to wait for. + pub fn submit(&mut self, class: Class, request: &Request) -> Result, SubmitError> { + let t = usize::from(request.timeline); + if t > TIMELINES { + return Err(SubmitError::Timeline); + } + self.ring.sq[class.index()] + .push(request.entry()) + .map_err(|_| SubmitError::Full)?; + Ok((t != 0).then(|| { + self.issued[t - 1] += 1; + Ticket { + timeline: request.timeline, + seq: self.issued[t - 1], + } + })) + } + + /// Make every queued submission on `class` visible to the row. `true` + /// means the row had parked and is owed a notice (ring its doorbell). + pub fn publish(&mut self, class: Class) -> bool { + self.ring.sq[class.index()].commit() + } + + /// Submissions `class`'s channel can still take. + pub fn room(&self, class: Class) -> u32 { + self.ring.sq[class.index()].capacity_left() + } + + /// The last sequence issued on timeline `t` (0 before the first). + pub fn issued(&self, t: u16) -> Option { + let index = usize::from(t).checked_sub(1)?; + self.issued.get(index).copied() + } + + /// The next completion, if one is posted. + pub fn reap(&mut self) -> Option { + let cqe = self.ring.cq.pop()?; + let result = if cqe.res >= 0 { + Ok(cqe.res as u32) + } else { + Err(StorStatus::from_res(cqe.res).unwrap_or(StorStatus::Invalid)) + }; + Some(Completion { + user_data: cqe.user_data, + result, + }) + } + + /// Timeline `t`'s fence as it stands. + pub fn fence(&self, t: u16) -> Option { + self.ring.fences.timeline(t).map(FenceCell::load) + } + + /// Timeline `t`'s fence cell itself, for the window's whole life — what + /// a waiter on another thread watches while this client is driven + /// under someone else's lock. + pub fn fence_cell(&self, t: u16) -> Option<&'a FenceCell> { + self.ring.fences.timeline(t) + } + + /// Whether `ticket` has completed: `Ok(true)` done, `Ok(false)` not yet, + /// `Err` when a submission at or before it on its timeline failed. + pub fn poll_ticket(&self, ticket: Ticket) -> Result { + let Some(record) = self.fence(ticket.timeline) else { + return Err(StorStatus::Invalid); + }; + if record.fault != 0 && record.fault <= ticket.seq { + return Err(StorStatus::Media); + } + Ok(record.done >= ticket.seq) + } + + /// The word a monitored wait watches for `ticket`'s timeline. + pub fn fence_word(&self, t: u16) -> Option<&AtomicU32> { + self.ring.fences.timeline(t).map(FenceCell::done_word) + } + + /// The completion channel's level state (ring status, counts). + pub fn state(&self) -> Option { + self.ring.cq.state() + } + + /// The completion channel, for a wait that parks on it. + pub fn completions(&self) -> &EventSink<'a, CqState, IoUringCqe> { + &self.ring.cq + } +} + +/// A byte range widened to whole blocks: what to submit, and where the +/// caller's bytes sit inside it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Covering { + /// First byte of the first block. + pub offset: u64, + /// Bytes of whole blocks covering the range. + pub len: u64, + /// Where the requested range starts inside the covering one. + pub skew: u64, +} + +/// The whole-block range covering `[offset, offset + len)`; `None` for a +/// zero block size or an overflowing end. +pub const fn covering(offset: u64, len: u64, block: u64) -> Option { + if block == 0 { + return None; + } + let Some(end) = offset.checked_add(len) else { + return None; + }; + let first = offset - offset % block; + let Some(last) = end.checked_next_multiple_of(block) else { + return None; + }; + Some(Covering { + offset: first, + len: last - first, + skew: offset - first, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::alloc::{Layout, alloc_zeroed, dealloc}; + + /// A page-aligned heap window, freed on drop. + struct Page { + ptr: *mut u8, + layout: Layout, + } + + impl Page { + fn new(bytes: usize) -> Self { + let layout = Layout::from_size_align(bytes, FENCE_PAGE_BYTES).unwrap(); + // SAFETY: non-zero size. + let ptr = unsafe { alloc_zeroed(layout) }; + assert!(!ptr.is_null()); + Self { ptr, layout } + } + + fn bytes(&mut self) -> &mut [u8] { + // SAFETY: owned allocation of `layout.size()` bytes. + unsafe { core::slice::from_raw_parts_mut(self.ptr, self.layout.size()) } + } + } + + impl Drop for Page { + fn drop(&mut self) { + // SAFETY: allocated with this layout. + unsafe { dealloc(self.ptr, self.layout) }; + } + } + + fn sqe(user_data: u64) -> IoUringSqe { + let mut entry = IoUringSqe::new_zeroed(); + entry.opcode = op::READ_FIXED; + entry.user_data = user_data; + entry + } + + #[test] + fn the_layout_is_disjoint_line_aligned_and_page_sized() { + assert_eq!(DEMAND_AT, 0); + for class in Class::ALL { + let (at, depth) = channel(class); + assert_eq!(at % LINE, 0); + assert!(at + event::region_bytes::(depth) <= channel_end(class)); + } + assert!(CQ_AT + event::region_bytes::(CQ_DEPTH) <= FENCE_AT); + assert_eq!(FENCE_AT % FENCE_PAGE_BYTES, 0); + assert_eq!(WINDOW_BYTES % FENCE_PAGE_BYTES, 0); + } + + #[test] + fn a_window_refuses_misalignment_and_shortness() { + let mut page = Page::new(WINDOW_BYTES + FENCE_PAGE_BYTES); + let bytes = page.bytes(); + assert_eq!(format(&mut bytes[64..]), Err(WindowError::Misaligned)); + assert_eq!( + format(&mut bytes[..WINDOW_BYTES - FENCE_PAGE_BYTES]), + Err(WindowError::TooSmall) + ); + // Unformatted memory never attaches. + assert!(matches!( + service(&mut bytes[..WINDOW_BYTES]), + Err(WindowError::Channel(Class::Demand, _)) + )); + } + + #[test] + fn each_class_channel_carries_its_own_entries_and_the_cq_comes_back() { + let mut page = Page::new(WINDOW_BYTES); + let (mut client, mut service) = split(page.bytes()).unwrap(); + for class in Class::ALL { + client.sq[class.index()] + .push(sqe(100 + class as u64)) + .unwrap(); + let _ = client.sq[class.index()].commit(); + } + for class in Class::ALL { + let entry = service.sq[class.index()].pop().unwrap(); + assert_eq!(entry.user_data, 100 + class as u64); + assert!(service.sq[class.index()].pop().is_none()); + } + let mut cqe = IoUringCqe::new_zeroed(); + cqe.user_data = 7; + cqe.res = 4096; + service.cq.push(cqe).unwrap(); + service.cq.state(|state| state.completed = 1); + let _ = service.cq.commit(); + let back = client.cq.pop().unwrap(); + assert_eq!((back.user_data, back.res), (7, 4096)); + assert_eq!(client.cq.state().unwrap().completed, 1); + } + + #[test] + fn a_fence_publishes_fault_before_done_and_exposes_its_low_word() { + let mut page = Page::new(WINDOW_BYTES); + let (client, service) = split(page.bytes()).unwrap(); + assert!(service.fences.timeline(0).is_none()); + assert!(service.fences.timeline(TIMELINES as u16 + 1).is_none()); + let cell = service.fences.timeline(3).unwrap(); + cell.publish(FenceRecord { done: 5, fault: 4 }); + let seen = client.fences.timeline(3).unwrap(); + assert_eq!(seen.load(), FenceRecord { done: 5, fault: 4 }); + assert_eq!(seen.done_word().load(Ordering::Acquire), 5); + // Other timelines are untouched. + assert_eq!(client.fences.timeline(2).unwrap().load(), FenceRecord::default()); + } + + #[test] + fn covering_widens_to_whole_blocks_and_records_the_skew() { + assert_eq!(covering(0, 512, 512), Some(Covering { offset: 0, len: 512, skew: 0 })); + assert_eq!(covering(100, 10, 512), Some(Covering { offset: 0, len: 512, skew: 100 })); + assert_eq!(covering(500, 20, 512), Some(Covering { offset: 0, len: 1024, skew: 500 })); + assert_eq!(covering(1024, 0, 512), Some(Covering { offset: 1024, len: 0, skew: 0 })); + assert_eq!(covering(1, 1, 0), None); + assert_eq!(covering(u64::MAX, 1, 512), None); + } + + /// The client counts sequences per timeline exactly as the row will; + /// a ticket polls against the fence, and a fault at or before it is an + /// error while a later one is not. + #[test] + fn tickets_count_per_timeline_and_poll_against_the_fence() { + let mut page = Page::new(WINDOW_BYTES); + let (client, mut service) = split(page.bytes()).unwrap(); + let mut client = StorClient::new(client); + let request = |timeline, user_data| Request { + op: Op::Read, + grant: GrantId::new(0, 0), + offset: 0, + len: 512, + target: 0, + target_offset: 0, + user_data, + timeline, + deadline_ns: 0, + }; + let a = client.submit(Class::Demand, &request(1, 1)).unwrap().unwrap(); + let b = client.submit(Class::Stream, &request(1, 2)).unwrap().unwrap(); + let c = client.submit(Class::Demand, &request(2, 3)).unwrap().unwrap(); + assert_eq!(client.submit(Class::Demand, &request(0, 4)).unwrap(), None); + assert_eq!((a.seq, b.seq, c.seq), (1, 2, 1)); + assert_eq!( + client.submit(Class::Demand, &request(TIMELINES as u16 + 1, 5)), + Err(SubmitError::Timeline) + ); + client.publish(Class::Demand); + client.publish(Class::Stream); + let entry = service.sq[Class::Demand.index()].pop().unwrap(); + assert_eq!((entry.opcode, entry.personality, entry.user_data), (op::READ_FIXED, 1, 1)); + + assert_eq!(client.poll_ticket(a), Ok(false)); + service.fences.timeline(1).unwrap().publish(FenceRecord { done: 1, fault: 0 }); + assert_eq!(client.poll_ticket(a), Ok(true)); + assert_eq!(client.poll_ticket(b), Ok(false)); + service.fences.timeline(1).unwrap().publish(FenceRecord { done: 2, fault: 2 }); + assert_eq!(client.poll_ticket(a), Ok(true), "the fault is after it"); + assert!(client.poll_ticket(b).is_err(), "the fault is it"); + + let mut cqe = IoUringCqe::new_zeroed(); + cqe.user_data = 9; + cqe.res = StorStatus::Revoked.res(); + service.cq.push(cqe).unwrap(); + let _ = service.cq.commit(); + assert_eq!( + client.reap(), + Some(Completion { user_data: 9, result: Err(StorStatus::Revoked) }) + ); + assert_eq!(client.reap(), None); + } +} diff --git a/native/crates/rt/Cargo.toml b/native/crates/rt/Cargo.toml index 05f46db..c359532 100644 --- a/native/crates/rt/Cargo.toml +++ b/native/crates/rt/Cargo.toml @@ -12,6 +12,7 @@ publish = false # mapped extents — it is the thing that *provides* the heap, so it cannot # use one. [dependencies] +heapless = { version = "0.9.3", default-features = false } kore-abi = { path = "../abi" } # The bounded arena-slot planner uses the same dependency-free bitset as the # kernel. Keeping this at the bottom of the userland graph is lawful: diff --git a/native/crates/rt/src/barrier.rs b/native/crates/rt/src/barrier.rs index fd1a6d5..ddbb409 100644 --- a/native/crates/rt/src/barrier.rs +++ b/native/crates/rt/src/barrier.rs @@ -34,7 +34,8 @@ //! every member the intra-chunk kick MESH: the leader's //! [`wake_all`](crate::futex::wake_all) kicks a parked chunk sibling in //! ONE hop, and only a waiter the caller holds no send for (a -//! cross-chunk member of a >MESH_CHUNK region) routes through the hub, +//! cross-chunk member of a region wider than one founding session's +//! [`MESH_CEILING`](crate::star::MESH_CEILING)) routes through the hub, //! leader → hub → member. The two-hop relay was the original release //! path, and the metal terminal priced it: the hub is unpinned and parked, //! so a w≤4 release of a parked member averaged ~½·[`PARK_SLICE`] — diff --git a/native/crates/rt/src/cap/mod.rs b/native/crates/rt/src/cap/mod.rs index fbe0a90..b76a17d 100644 --- a/native/crates/rt/src/cap/mod.rs +++ b/native/crates/rt/src/cap/mod.rs @@ -2651,6 +2651,16 @@ impl ReturnedCapability { Ok(self.into_stashed_lease_after_check()) } + /// Observe the geometry of a received fabric route before accepting its + /// custody. The kernel authenticates the possessed subject; no new route + /// or mapping is created and wrong species return the ordinary refusal. + pub fn describe_fabric_route( + &self, + ) -> Result { + let slot = self.slot().ok_or(verbs::FabricRegistrationFault::Slot)?; + verbs::describe_fabric_route(slot) + } + /// Authenticate a received mapping route without converting it into a /// physical allocation lease or changing its ancestry. pub fn try_into_fabric_route(self, required: Rights) -> Result { @@ -3965,7 +3975,13 @@ impl DmaWindow { return RetireOutcome::Retryable; } let Some(view) = self.cpu_view.as_mut() else { - return RetireOutcome::Retryable; + // A device-only import never had a CPU leaf; an anonymous pool + // window's CPU leaf cannot prove this separate transition. + return if self.cpu.is_null() { + RetireOutcome::AlreadyDead + } else { + RetireOutcome::Retryable + }; }; let parent = view.lease; let outcome = view.owned.retire_via(|slot| retire(slot, parent)); @@ -3984,6 +4000,16 @@ impl DmaWindow { self.retire_device_via(crate::cap::verbs::retire_owned_iova) } + /// [`try_retire_device`](Self::try_retire_device) while the device keeps + /// running: the requester's DMA latch stays open for its other windows. + /// The driver must first prove no command still names this window and + /// reap those that did; the kernel completes the window's invalidation + /// before its identity goes, so an access it missed is a recorded IOMMU + /// fault rather than a write into memory that has moved on. + pub fn try_retire_device_live(&mut self) -> RetireOutcome { + self.retire_device_via(crate::cap::verbs::retire_owned_iova_live) + } + fn retire_device_via( &mut self, retire: impl FnOnce(SlotRef, SlotRef) -> RetireOutcome, diff --git a/native/crates/rt/src/cap/verbs.rs b/native/crates/rt/src/cap/verbs.rs index a70da54..9500670 100644 --- a/native/crates/rt/src/cap/verbs.rs +++ b/native/crates/rt/src/cap/verbs.rs @@ -1238,6 +1238,100 @@ mod metal { Ok(TaskToken::possess_owned(handle, id, rights)) } + /// Found a bounded SMP cohort through one guest Found tail. Every task is + /// dormant on return; the caller retains the shared-space owner until all + /// task handles have completed teardown. No compute barrier or join fold + /// participates in guest execution. + pub fn found_guest_cohort( + space: &GuestSpaceToken, + inputs: heapless::Vec< + (kore_abi::guest::GuestVcpuConfig, GuestVcpuInputs), + { kore_abi::session::MAX_SESSION_FOUNDS }, + >, + ) -> Result< + heapless::Vec, + kore_abi::session::Status, + > { + use kore_abi::session::Status; + #[cfg(not(target_os = "none"))] + { + let _ = (space, inputs); + Err(Status::Denied) + } + #[cfg(target_os = "none")] + { + use crate::session::SessionBuilder; + use crate::transport::{Transport, TrapTransport}; + if inputs.is_empty() { + return Err(Status::Malformed); + } + let mut handles = heapless::Vec::<_, { kore_abi::session::MAX_SESSION_FOUNDS }>::new(); + let mut request = SessionBuilder::new(); + for (cfg, input) in &inputs { + let handle = crate::cap::OwnedSlot::reserve().ok_or(Status::Unfounded)?; + request + .found_guest( + space.slot(), + cfg.entry, + cfg.stack, + cfg.page_table_root, + input.window.slot().ok_or(Status::Malformed)?, + input.resume_recv.slot(), + input.exit_send.slot(), + handle.slot(), + ) + .map_err(|_| Status::Malformed)?; + let _ = handles.push(handle); + } + let committed = match TrapTransport::new().commit_now(&request) { + Ok(committed) => committed, + Err(_) => { + for handle in &mut handles { + handle.quarantine_unknown_publish(); + } + return Err(Status::Malformed); + } + }; + if committed.raw() as u16 > Status::AltDelivery as u16 + || (committed.status() == Status::Ok && committed.failing_index() != 0) + { + for handle in &mut handles { + handle.quarantine_unknown_publish(); + } + return Err(Status::Malformed); + } + if committed.status() != Status::Ok { + return Err(committed.status()); + } + // Authenticate the WHOLE result before surrendering any local + // input owner. Malformed/unknown publication never frees a slot + // whose custody might have changed. + let mut records = heapless::Vec::<_, { kore_abi::session::MAX_SESSION_FOUNDS }>::new(); + for handle in &handles { + match task_record(handle.slot()) { + Some((id, rights)) if rights == crate::cap::GUEST_VCPU_HANDLE_RIGHTS => { + let _ = records.push((id, rights)); + } + _ => { + for handle in &mut handles { + handle.quarantine_unknown_publish(); + } + return Err(Status::Malformed); + } + } + } + let mut tasks = heapless::Vec::new(); + for (((_, input), handle), (id, rights)) in inputs.into_iter().zip(handles).zip(records) + { + input.window.release_after_proven_move(); + input.resume_recv.release_after_proven_move(); + input.exit_send.release_after_proven_move(); + let _ = tasks.push(TaskToken::possess_owned(handle, id, rights)); + } + Ok(tasks) + } + } + #[cfg(not(target_os = "none"))] pub fn found_guest_vcpu( _space: &GuestSpaceToken, @@ -2375,7 +2469,7 @@ mod metal { .map_err(|_| FabricRouteFault::Build)?; match (completion.raw() as u16, completion.failing_index()) { (0, 0) => Ok(crate::cap::FabricRouteToken::new(owned)), - (status @ 1..=6, 0) => Err(FabricRouteFault::Commit(status as u8)), + (status @ (1..=6 | 12..=17), 0) => Err(FabricRouteFault::Commit(status as u8)), _ => { owned.quarantine_unknown_publish(); Err(FabricRouteFault::Indeterminate) @@ -2990,6 +3084,28 @@ mod metal { RetireOutcome::Retryable } + /// [`retire_owned_iova`] for a requester that keeps running: its DMA + /// latch stays open (`REVOKE_KIND_UNSHARE_IOVA_LIVE`). + #[cfg(target_os = "none")] + pub(crate) fn retire_owned_iova_live(view: SlotRef, parent: SlotRef) -> RetireOutcome { + use crate::session::SessionBuilder; + use crate::transport::{Transport, TrapTransport}; + + let mut b = SessionBuilder::new(); + if b.unshare_iova_live(view, parent).is_err() { + return RetireOutcome::Retryable; + } + let Ok(committed) = TrapTransport::new().commit_now(&b) else { + return RetireOutcome::Retryable; + }; + classify_owned_retirement(committed) + } + + #[cfg(not(target_os = "none"))] + pub(crate) fn retire_owned_iova_live(_view: SlotRef, _parent: SlotRef) -> RetireOutcome { + RetireOutcome::Retryable + } + /// Retire a fixed landing-slot capability without an allocator /// reservation. This is the client grant-bank cleanup face: callers keep /// the slot name when the outcome is retryable and may forget it only once @@ -3041,25 +3157,61 @@ mod metal { Publish(u8), /// Fabric registration (1), observation (2), or route admission (3) /// refused. The payload is `stage[1:0] | status << 2`, where - /// `status` is the refusing session's wire status (1..=6) or 7 for + /// `status` is the refusing session's wire status (1..=6 or 12..=17) or 7 for /// a local refusal (no slot, malformed report, busy, quarantined) — - /// see [`fabric`](Self::fabric). Every kernel-side route refusal - /// reaches ring 3 as `Status::Denied`, so the stage is the finer - /// signal until a fault-detail channel exists. + /// see [`fabric`](Self::fabric). Route-specific statuses preserve the + /// kernel's refusal class for a serial-less metal caller. Fabric(u8), } + #[cfg(any(test, target_os = "none"))] + fn route_mapping_allowed( + info: &kore_abi::session::FabricRouteInfo, + len: u64, + device_write: bool, + ) -> bool { + use kore_abi::session::{FabricAccess, FabricOp}; + let Some(access) = FabricAccess::from_bits(info.access) else { + return false; + }; + info.bytes == len + && info.operation == FabricOp::IommuMap as u8 + && access.contains(FabricAccess::READ) + && (!device_write || access.contains(FabricAccess::WRITE)) + } + + #[cfg(test)] + #[test] + fn read_only_route_admits_input_but_never_device_output() { + use kore_abi::session::{FabricAccess, FabricRouteInfo}; + let mut info = FabricRouteInfo { + bytes: 4096, + access: FabricAccess::READ.bits(), + operation: 1, + align_log2: kore_abi::session::LEASE_ALIGN_4K, + }; + assert!(route_mapping_allowed(&info, 4096, false)); + assert!(!route_mapping_allowed(&info, 4096, true)); + info.access = (FabricAccess::READ | FabricAccess::WRITE).bits(); + assert!(route_mapping_allowed(&info, 4096, true)); + assert!(!route_mapping_allowed(&info, 8192, false)); + info.access = FabricAccess::WRITE.bits(); + assert!(!route_mapping_allowed(&info, 4096, false)); + } + impl DmaMapFault { /// The fabric arm with its stage and the fault that stopped it. pub fn fabric(stage: u8, fault: FabricRouteFault) -> Self { let status = match fault { + FabricRouteFault::Commit(status) if matches!(status, 12..=17) => status, FabricRouteFault::Commit(status) => status.min(6), _ => 7, }; Self::Fabric((stage & 3) | (status << 2)) } - /// Pack as `stage[2:0] | status << 3` — the note-detail byte. + /// Pack legacy faults as `stage[2:0] | status << 3`; route-specific + /// statuses use `0x80 | (status - 12) << 2 | fabric_stage`. pub fn code(self) -> u8 { match self { Self::Bounds => 1, @@ -3068,17 +3220,36 @@ mod metal { Self::Map(s) => 4 | (s << 3), Self::Geometry => 5, Self::Publish(s) => 6 | (s << 3), + Self::Fabric(s) if s >> 2 >= 12 => { + // New route-specific outcomes occupy a separate byte + // range; retain the legacy stage/status encoding below. + 0x80 | (((s >> 2) - 12) << 2) | (s & 3) + } Self::Fabric(s) => 7 | (s << 3), } } } + #[cfg(test)] + #[test] + fn route_refusals_keep_the_kernel_reason_in_the_dma_note() { + assert_eq!( + DmaMapFault::fabric(3, FabricRouteFault::Commit(1)).code(), + 63 + ); + for status in 12..=17 { + let code = DmaMapFault::fabric(3, FabricRouteFault::Commit(status)).code(); + assert_eq!(code, 0x80 | ((status - 12) << 2) | 3); + } + assert_eq!(DmaMapFault::fabric(3, FabricRouteFault::Build).code(), 255); + } + /// Map a complete held lease R/W at the kernel-selected address while /// retaining an exact-unshare owner. The parent lease must remain held /// until this view closes; no caller-selected VA or second mapping occurs. #[cfg(target_os = "none")] pub fn map_owned_lease_rw(lease: SlotRef) -> Option { - map_named_cpu_view(lease, crate::entry::geometry::report_slot()?).ok() + try_map_owned_lease_rw(lease).ok() } #[cfg(not(target_os = "none"))] @@ -3086,18 +3257,37 @@ mod metal { None } + /// [`map_owned_lease_rw`], keeping why it refused: no free slot, the map + /// session's status, or an unusable geometry report. + #[cfg(target_os = "none")] + pub fn try_map_owned_lease_rw(lease: SlotRef) -> Result { + let report = crate::entry::geometry::report_slot().ok_or(DmaMapFault::Slot)?; + map_named_cpu_view(lease, report, true) + } + + /// Off the metal there is no session transport: the map cannot be built. + #[cfg(not(target_os = "none"))] + pub fn try_map_owned_lease_rw(_lease: SlotRef) -> Result { + Err(DmaMapFault::Map(0)) + } + #[cfg(target_os = "none")] - fn map_named_cpu_view(lease: SlotRef, report: SlotRef) -> Result { + fn map_named_cpu_view( + lease: SlotRef, + report: SlotRef, + writable: bool, + ) -> Result { use crate::session::SessionBuilder; use crate::transport::{Transport, TrapTransport}; let mut owner = crate::cap::OwnedSlot::reserve().ok_or(DmaMapFault::Slot)?; let mut map = SessionBuilder::new(); + let rights = if writable { + Rights::ACCESS | Rights::MODIFY | Rights::REVOKE + } else { + Rights::ACCESS | Rights::REVOKE + }; let view = map - .map_owned_named( - lease, - Rights::ACCESS | Rights::MODIFY | Rights::REVOKE, - owner.slot(), - ) + .map_owned_named(lease, rights, owner.slot()) .map_err(|_| DmaMapFault::Map(0))?; map.cpu_view_geometry(view, report) .map_err(|_| DmaMapFault::Map(0))?; @@ -3146,17 +3336,19 @@ mod metal { len: u64, contents: DmaContents, device_write: bool, + cpu_writable: bool, ) -> Result<(MappedLeaseView, crate::cap::OwnedSlot), DmaMapFault> { - use crate::session::SessionBuilder; - use crate::transport::{Transport, TrapTransport}; let report = crate::entry::geometry::report_slot().ok_or(DmaMapFault::Slot)?; let mapping_slot = route.slot(); - let cpu_view = map_named_cpu_view(mapping_slot, report)?; + let cpu_view = map_named_cpu_view(mapping_slot, report, cpu_writable)?; if cpu_view.len as u64 != len { return Err(DmaMapFault::Geometry); } if matches!(contents, DmaContents::Zero) { + if !cpu_writable { + return Err(DmaMapFault::Bounds); + } // SAFETY: the successful map installed the pool sub-lease RW for // exactly `len` bytes. No device leaf exists yet. Imports never // enter this branch: their producer already populated the bytes. @@ -3165,6 +3357,23 @@ mod metal { // Both mapping names remain independently retireable before their // shared route, registration and allocation can close. + let device_owner = publish_route_device(route, device, iova, len, device_write)?; + Ok((cpu_view, device_owner)) + } + + /// Publish the named IOMMU leaf of `route` for `device` at `iova`, and + /// nothing else: no CPU view in this task. + #[cfg(target_os = "none")] + fn publish_route_device( + route: &crate::cap::FabricRouteToken, + device: SlotRef, + iova: u64, + len: u64, + device_write: bool, + ) -> Result { + use crate::session::SessionBuilder; + use crate::transport::{Transport, TrapTransport}; + let mapping_slot = route.slot(); let mut device_owner = crate::cap::OwnedSlot::reserve().ok_or(DmaMapFault::Slot)?; let mut publish = SessionBuilder::new(); // Device MODIFY follows device writes, independently of CPU access. @@ -3195,8 +3404,7 @@ mod metal { return Err(DmaMapFault::Publish(committed.status() as u8)); } device_owner.activate(); - - Ok((cpu_view, device_owner)) + Ok(device_owner) } #[cfg(target_os = "none")] @@ -3274,7 +3482,7 @@ mod metal { ) .map_err(|fault| DmaMapFault::fabric(3, fault))?; let (cpu_view, device_owner) = - map_route_dma_views(&route, device, iova, len, contents, device_write)?; + map_route_dma_views(&route, device, iova, len, contents, device_write, true)?; let geom = GeomReport { base: cpu_view.as_ptr() as u64, len, @@ -3388,6 +3596,46 @@ mod metal { /// Map a transferred route using its existing admitted custody chain. /// The caller has checked immutable route info before passing ownership. + /// **Import a lent route for the device alone**: the IOMMU leaf, with no + /// CPU view in this task. For a consumer that only ever points the device + /// at the memory — a block lane streaming into a client's buffer — a CPU + /// view would be pure cost: page tables and address space in the + /// importing service scaled by the client's allocation, which for a + /// multi-GiB model arena is exactly what must not happen. The window's + /// [`cpu`](crate::cap::DmaWindow::cpu) is null; its device leaf and route + /// retire exactly as [`import_route_dma_window`]'s do. + pub fn import_route_device_window( + route: crate::cap::FabricRouteToken, + device: &DeviceToken, + iova: u64, + len: u64, + device_write: bool, + ) -> Result { + #[cfg(target_os = "none")] + { + if len == 0 || len % 4096 != 0 || iova % 4096 != 0 || iova.checked_add(len).is_none() { + return Err(DmaMapFault::Bounds); + } + let info = route + .describe() + .map_err(|_| DmaMapFault::fabric(2, FabricRouteFault::MalformedReport))?; + if !route_mapping_allowed(&info, len, device_write) { + return Err(DmaMapFault::Bounds); + } + let device_owner = + publish_route_device(&route, device.slot(), iova, len, device_write)?; + Ok( + DmaWindow::from_route(core::ptr::null_mut(), iova, len, route) + .with_device_leaf(device_owner), + ) + } + #[cfg(not(target_os = "none"))] + { + let _ = (route, device, iova, len, device_write); + Err(DmaMapFault::Slot) + } + } + pub fn import_route_dma_window( route: crate::cap::FabricRouteToken, device: &DeviceToken, @@ -3403,9 +3651,15 @@ mod metal { let info = route .describe() .map_err(|_| DmaMapFault::fabric(2, FabricRouteFault::MalformedReport))?; - if info.bytes != len || info.operation != 1 || info.access & 3 != 3 { + if !route_mapping_allowed(&info, len, device_write) { return Err(DmaMapFault::Bounds); } + // A READ fabric scope stays read-only in this service's CPU page + // tables as well as in the outer IOMMU. DmaWindow retains the + // existing raw-pointer ABI for cache maintenance; its iVPU + // PROGRAM_INPUT consumer only flushes/reads the view, while + // write_buffer requires TRANSFER_DESTINATION. + let cpu_writable = info.access & kore_abi::session::FabricAccess::WRITE.bits() != 0; let (cpu_view, device_owner) = map_route_dma_views( &route, device.slot(), @@ -3413,6 +3667,7 @@ mod metal { len, DmaContents::Preserve, device_write, + cpu_writable, )?; Ok( DmaWindow::from_route(cpu_view.as_mut_ptr(), iova, len, route) @@ -3820,6 +4075,21 @@ mod metal { /// not remove the PTE or its typed share-list edge. #[cfg(target_os = "none")] pub fn map_view(view: LeasedToken) -> Option> { + map_view_as(view, false) + } + + /// [`map_view`] as a **sink** mapping: if the lessor's lease dies while + /// this task still maps the view, its pages are repointed at this space's + /// sink page instead of unmapped — for a service that reads a + /// client-funded ring asynchronously, where a hole would fault it + /// (`res/STORAGE_TRANSPORT.md` §12, option B). + #[cfg(target_os = "none")] + pub fn map_view_sink(view: LeasedToken) -> Option> { + map_view_as(view, true) + } + + #[cfg(target_os = "none")] + fn map_view_as(view: LeasedToken, sink: bool) -> Option> { use kore_abi::session::GeomReport; use crate::session::SessionBuilder; @@ -3827,8 +4097,12 @@ mod metal { let report_slot = crate::entry::geometry::report_slot()?; let mut b = SessionBuilder::new(); - b.derive_raw(view.slot(), Rights::empty(), 0, report_slot, None) - .ok()?; + if sink { + b.map_view_sink(view.slot(), report_slot).ok()?; + } else { + b.derive_raw(view.slot(), Rights::empty(), 0, report_slot, None) + .ok()?; + } let committed = TrapTransport::new().commit_now(&b).ok()?; if committed.status() != kore_abi::session::Status::Ok { return None; @@ -3959,6 +4233,11 @@ mod metal { /// Off the metal userland targets — the honest typed refusal: there is no trap /// to issue (and no space to install into). + #[cfg(not(target_os = "none"))] + pub fn map_view_sink(_view: LeasedToken) -> Option> { + None + } + #[cfg(not(target_os = "none"))] pub fn map_view(_view: LeasedToken) -> Option> { None @@ -4052,24 +4331,39 @@ mod metal { /// kernel-vouched geometry is returned through the task's bind-inert /// report slot; no capability or map-cursor span is consumed. #[cfg(target_os = "none")] - pub fn lease_geometry(lease: SlotRef) -> Option<(u64, u64)> { + pub fn try_lease_geometry(lease: SlotRef) -> Result<(u64, u64), LeaseWindowFault> { use kore_abi::cap::CapRecord; use kore_abi::session::GeomReport; use crate::session::SessionBuilder; use crate::transport::{Transport, TrapTransport}; - let report_slot = crate::entry::geometry::report_slot()?; + let report_slot = crate::entry::geometry::report_slot().ok_or(LeaseWindowFault::Slot)?; let mut b = SessionBuilder::new(); b.derive_raw(lease, Rights::empty(), 0, report_slot, None) - .ok()?; - let committed = TrapTransport::new().commit_now(&b).ok()?; + .map_err(|_| LeaseWindowFault::Refused)?; + let committed = TrapTransport::new().commit_now(&b).map_err(|_| LeaseWindowFault::Refused)?; if committed.status() != kore_abi::session::Status::Ok { - return None; + return Err(LeaseWindowFault::Geometry(committed.status())); } - let ptr = crate::entry::geometry::slot_record_va(report_slot)? as *const CapRecord; - let geom = GeomReport::decode(&unsafe { core::ptr::read_volatile(ptr) })?; - (geom.len != 0).then_some((geom.base, geom.len)) + let ptr = crate::entry::geometry::slot_record_va(report_slot) + .ok_or(LeaseWindowFault::Refused)? as *const CapRecord; + let geom = GeomReport::decode(&unsafe { core::ptr::read_volatile(ptr) }) + .ok_or(LeaseWindowFault::Refused)?; + (geom.len != 0) + .then_some((geom.base, geom.len)) + .ok_or(LeaseWindowFault::Refused) + } + + /// Lossy geometry query; use `try_lease_geometry` to retain the refusal. + #[cfg(target_os = "none")] + pub fn lease_geometry(lease: SlotRef) -> Option<(u64, u64)> { + try_lease_geometry(lease).ok() + } + + #[cfg(not(target_os = "none"))] + pub fn try_lease_geometry(_lease: SlotRef) -> Result<(u64, u64), LeaseWindowFault> { + Err(LeaseWindowFault::Refused) } #[cfg(not(target_os = "none"))] @@ -4161,13 +4455,13 @@ mod metal { false } - /// Shared whole-lease implementation for the fixed-address view helpers. + /// Shared mapping implementation after exact parent geometry validation. /// /// The arena slot is reserved by the caller and handed in, so an /// exhausted arena is reported as [`LeaseWindowFault::Slot`] instead of - /// being folded into this function's `None` alongside every policy - /// refusal. On any refusal `owned` drops here and releases its - /// reservation, exactly as when the reserve lived inside. + /// being folded into a policy refusal. Kernel mapping refusals retain + /// their exact committed status. On any refusal `owned` drops here and + /// releases its reservation, exactly as when the reserve lived inside. #[cfg(target_os = "none")] fn map_owned_whole_lease_in( owned: crate::cap::OwnedSlot, @@ -4175,16 +4469,15 @@ mod metal { va: u64, len: u64, access: u8, - ) -> Option { + ) -> Result { use crate::session::SessionBuilder; use crate::transport::{Transport, TrapTransport}; - let (_, lease_len) = lease_geometry(lease)?; - if len == 0 || len != lease_len { - return None; - } - let len_usize = usize::try_from(len).ok()?; - let base = NonNull::new(usize::try_from(va).ok()? as *mut u8)?; + let len_usize = usize::try_from(len).map_err(|_| LeaseWindowFault::Refused)?; + let base = NonNull::new( + usize::try_from(va).map_err(|_| LeaseWindowFault::Refused)? as *mut u8, + ) + .ok_or(LeaseWindowFault::Refused)?; let mut b = SessionBuilder::new(); if access & !(kore_abi::session::SHARE_ACCESS_R @@ -4195,19 +4488,19 @@ mod metal { || access & kore_abi::session::SHARE_ACCESS_W != 0 && access & kore_abi::session::SHARE_ACCESS_X != 0 { - return None; + return Err(LeaseWindowFault::Refused); } let mut rights = Rights::ACCESS | Rights::REVOKE; if access & kore_abi::session::SHARE_ACCESS_W != 0 { rights |= Rights::MODIFY; } b.share_cpu_access(lease, rights, va, len, access, owned.slot()) - .ok()?; - let committed = TrapTransport::new().commit_now(&b).ok()?; + .map_err(|_| LeaseWindowFault::Refused)?; + let committed = TrapTransport::new().commit_now(&b).map_err(|_| LeaseWindowFault::Refused)?; if committed.status() != kore_abi::session::Status::Ok { - return None; + return Err(LeaseWindowFault::Commit(committed.status())); } - Some(MappedLeaseView::new(owned, lease, base, len_usize)) + Ok(MappedLeaseView::new(owned, lease, base, len_usize)) } /// Reserve the arena slot, then map — the seam that separates "no slot" @@ -4219,8 +4512,51 @@ mod metal { len: u64, access: u8, ) -> Result { + let (_, lease_len) = try_lease_geometry(lease)?; + if len == 0 || len != lease_len { + return Err(LeaseWindowFault::Refused); + } let owned = crate::cap::OwnedSlot::reserve().ok_or(LeaseWindowFault::Slot)?; - map_owned_whole_lease_in(owned, lease, va, len, access).ok_or(LeaseWindowFault::Refused) + map_owned_whole_lease_in(owned, lease, va, len, access) + } + + /// Map a complete possessed fabric route into this task's CPU space. + /// The route retains allocation custody elsewhere; its exact view must + /// retire before the route closes. This creates no device mapping. + #[cfg(target_os = "none")] + pub fn map_owned_route_at_rw( + route: &crate::cap::FabricRouteToken, + va: u64, + len: u64, + ) -> Option { + use kore_abi::session::{FabricAccess, FabricOp}; + let info = route.describe().ok()?; + let access = FabricAccess::from_bits(info.access)?; + if len == 0 + || len != info.bytes + || !matches!(info.operation, op if op == FabricOp::CpuMap as u8 || op == FabricOp::IommuMap as u8) + || !access.contains(FabricAccess::READ | FabricAccess::WRITE) + { + return None; + } + let owned = crate::cap::OwnedSlot::reserve()?; + map_owned_whole_lease_in( + owned, + route.slot(), + va, + len, + kore_abi::session::SHARE_ACCESS_R | kore_abi::session::SHARE_ACCESS_W, + ) + .ok() + } + + #[cfg(not(target_os = "none"))] + pub fn map_owned_route_at_rw( + _route: &crate::cap::FabricRouteToken, + _va: u64, + _len: u64, + ) -> Option { + None } /// [`map_owned_lease_at_rw`] with the refusal named. @@ -4299,9 +4635,13 @@ mod metal { /// The task's cap arena had no free slot and could not extend. No /// session was built and nothing was asked of the kernel. Slot, - /// Everything else: a malformed ask, a session that failed to build - /// or transport, or one that committed with a non-`Ok` status. A - /// supply lease that has left `LEASE_LIVE` refuses here. + /// The kernel refused the geometry query before mapping admission. + Geometry(kore_abi::session::Status), + /// The kernel refused CPU mapping admission. Fabric conflict/capacity + /// and revoked authority remain distinguishable to the caller. + Commit(kore_abi::session::Status), + /// Local validation, build/transport failure, or a legacy carving + /// refusal for which no committed status was retained. Refused, } @@ -5160,6 +5500,69 @@ mod metal { (report.metric == metric).then_some(report) } + /// Read a bounded group of inert reports in one kernel session. Each + /// result has an allocator-owned slot, released on every exit. Observation + /// produces no live capability, so the slots stay Reserved (no revoke). + #[cfg(target_os = "none")] + pub fn observe_metrics( + token: &ObserveToken, + metrics: [(kore_abi::session::SystemMetric, u16); N], + ) -> Option<[kore_abi::session::SystemObserveReport; N]> { + use crate::{ + session::SessionBuilder, + transport::{Transport, TrapTransport}, + }; + use kore_abi::{ + cap::{CapRecord, Rights}, + session::{DERIVE_KIND_SYSTEM_OBSERVE, Status, SystemObserveReport, SystemObserveReq}, + }; + if N == 0 || N > kore_abi::session::MAX_INVOCATIONS { + return None; + } + let mut slots: [Option; N] = core::array::from_fn(|_| None); + let mut request = SessionBuilder::new(); + for (slot, (metric, operand)) in slots.iter_mut().zip(metrics) { + let owned = super::super::OwnedSlot::reserve()?; + request + .derive_raw( + token.slot(), + Rights::empty(), + DERIVE_KIND_SYSTEM_OBSERVE, + owned.slot(), + Some(&SystemObserveReq::encode(metric, operand)), + ) + .ok()?; + *slot = Some(owned); + } + if TrapTransport::new().commit_now(&request).ok()?.status() != Status::Ok { + return None; + } + let mut reports = [SystemObserveReport { + metric: metrics[0].0, + value: 0, + aux: 0, + }; N]; + for ((dst, owned), (metric, _)) in reports.iter_mut().zip(&slots).zip(metrics) { + let address = crate::entry::geometry::slot_record_va(owned.as_ref()?.slot())?; + // SAFETY: the runtime slot owner pins this thread's read-only + // result alias; commit completed and no await can reuse the slot. + *dst = SystemObserveReport::decode(&unsafe { + core::ptr::read_volatile(address as *const CapRecord) + })?; + if dst.metric != metric { + return None; + } + } + Some(reports) + } + #[cfg(not(target_os = "none"))] + pub fn observe_metrics( + _token: &ObserveToken, + _metrics: [(kore_abi::session::SystemMetric, u16); N], + ) -> Option<[kore_abi::session::SystemObserveReport; N]> { + None + } + /// Consume one kernel-vouched completion for an MM-issued supply rooted /// in `region`, on the `pair` arrival lane. Absence is a normal poll miss. #[cfg(target_os = "none")] @@ -5283,6 +5686,47 @@ mod metal { false } + /// Submit a machine-wide 0–100 CPU performance preference through the + /// same held SystemPower authority as machine transitions. The kernel + /// checks the range and `MODIFY` before publishing it for each CPU. + #[cfg(target_os = "none")] + pub fn request_performance(token: &PowerToken, level: u8) -> bool { + use crate::session::SessionBuilder; + use crate::transport::{Transport, TrapTransport}; + use kore_abi::session::{POWER_PERFORMANCE_KIND, Status}; + + if level > 100 && level != kore_abi::session::POWER_PERFORMANCE_AUTO { + return false; + } + let Some(report_slot) = crate::entry::geometry::report_slot() else { + return false; + }; + let mut arg = [0u8; 8]; + arg[0] = level; + let mut req = SessionBuilder::new(); + if req + .derive_raw( + token.slot(), + Rights::empty(), + POWER_PERFORMANCE_KIND, + report_slot, + Some(&arg), + ) + .is_err() + { + return false; + } + match TrapTransport::new().commit_now(&req) { + Ok(committed) => committed.status() == Status::Ok, + Err(_) => false, + } + } + + #[cfg(not(target_os = "none"))] + pub fn request_performance(_token: &PowerToken, _level: u8) -> bool { + false + } + /// **The metal display map + describe** (A.2) — install the full /// framebuffer aperture in the task's **aperture window** /// (`geom::APERTURE`; **write-combining**, the region's own attrs) diff --git a/native/crates/rt/src/chan.rs b/native/crates/rt/src/chan.rs index 88c0d32..78cd925 100644 --- a/native/crates/rt/src/chan.rs +++ b/native/crates/rt/src/chan.rs @@ -51,6 +51,29 @@ pub enum SendError { Transport, } +/// Why a task-handle delegation ([`ChanTx::delegate_task`]) reports failure. +/// +/// Every variant keeps what the attempt learned, and each says whether the +/// peer can hold a delegated child — which decides whether a retry is safe. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DelegateError { + /// Staging refused; nothing reached the kernel and no child exists. + Build(crate::session::BuildError), + /// The transport could not issue the session; nothing reached the kernel + /// and no child exists. + Transport(E), + /// Proven rollback: no receive post stands on the channel. No child + /// exists; retry once the peer re-posts. + WouldBlock, + /// Proven rollback: the gate refused invocation `failing_index` with + /// `status`. No child exists. + Refused { status: Status, failing_index: u16 }, + /// The completion does not prove rollback: the peer may hold a delegated + /// child. Retrying could delegate a second one; the task's own revocation + /// by its holder remains the backstop. + Unproven { status: Status, failing_index: u16 }, +} + /// Why a recv refused. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum RecvError { @@ -151,6 +174,16 @@ fn classify_rotating_post(committed: kore_abi::session::Committed) -> RotatingPo } } +/// Statuses with which the gate proves a transfer session rolled back +/// entirely: the peer holds nothing from it. Any other refusal leaves custody +/// unproven. +const fn proves_rollback(status: Status) -> bool { + matches!( + status, + Status::Denied | Status::Forged | Status::Revoked | Status::WouldBlock | Status::Conflict + ) +} + fn classify_derived_transfer( committed: kore_abi::session::Committed, invocations: usize, @@ -160,16 +193,7 @@ fn classify_derived_transfer( if status == Status::Ok && failing == 0 && invocations != 0 { return DerivedTransfer::Moved; } - if failing < invocations - && matches!( - status, - Status::Denied - | Status::Forged - | Status::Revoked - | Status::WouldBlock - | Status::Conflict - ) - { + if failing < invocations && proves_rollback(status) { return DerivedTransfer::RolledBack(if status == Status::WouldBlock { SendError::WouldBlock } else { @@ -179,6 +203,25 @@ fn classify_derived_transfer( DerivedTransfer::Indeterminate } +/// Classify a one-invocation delegation's completion without discarding the +/// status or the failing index. +fn delegation_outcome(committed: kore_abi::session::Committed) -> Result<(), DelegateError> { + let status = committed.status(); + let failing_index = committed.failing_index(); + match (status, failing_index) { + (Status::Ok, 0) => Ok(()), + (Status::WouldBlock, 0) => Err(DelegateError::WouldBlock), + (status, 0) if proves_rollback(status) => Err(DelegateError::Refused { + status, + failing_index, + }), + (status, failing_index) => Err(DelegateError::Unproven { + status, + failing_index, + }), + } +} + /// The **send end** witness — a peer's kick-and-transfer authority over /// one channel (`res/USERLAND.md` §4, pinned). pub struct ChanTx<'t, T: Transport> { @@ -240,6 +283,31 @@ impl<'t, T: Transport> ChanTx<'t, T> { } } + /// Delegate a held task handle: the peer's posted slot receives a fresh, + /// separately revocable child handle over the same task, attenuated to + /// `rights`, while `task` stays with the caller unchanged. + /// + /// A delegated handle carrying `REVOKE` is a kill capability for exactly + /// that task: revoking it runs the handle's ordinary teardown (kill, reap, + /// joiners woken) — the terminal controller's foreground slot + /// (`res/SHELL_TERMINAL.md` §9). The caller's handle needs `DERIVE`; the + /// kernel clamps `rights` to a subset of it. + pub async fn delegate_task( + &self, + task: &crate::cap::TaskToken, + rights: Rights, + ) -> Result<(), DelegateError> { + let mut s = SessionBuilder::new(); + s.send(self.send, task.slot(), rights) + .map_err(DelegateError::Build)?; + let committed = self + .transport + .commit(&s) + .await + .map_err(DelegateError::Transport)?; + delegation_outcome(committed) + } + /// Capability transfer (the landed send): `DERIVE`-gated /// re-derivation of `staged`'s capability into the peer's posted /// slot, attenuated to `rights`. Grant-bearing replies use this @@ -619,7 +687,26 @@ impl<'t, T: Transport> NoticeRx<'t, T> { None => return Ok(None), } } + self.stand().map(|()| None) + } + + /// Stand a fresh post now if none stands; never consumes. A consumed + /// notice leaves no post until the next [`try_recv`](Self::try_recv), + /// and a sender that finds none gets `WouldBlock` and may drop its + /// notice. A caller whose handling of a notice spans a round trip + /// re-stands right after consuming it, so the next notice cannot fall + /// into that gap. Only for a channel that carries notices alone: on one + /// that also carries another receiver's posts or capabilities, an + /// early post would take what that receiver expects. + pub fn restand(&mut self) -> Result<(), RecvError> { + if self.standing { + return Ok(()); + } + self.stand() + } + /// Stand a fresh post on the receive endpoint; `Ok` once one stands. + fn stand(&mut self) -> Result<(), RecvError> { // A landing retained by `rearm` is reused: the same slot, scrubbed // by the repost, so a post the kernel already resolved cannot leave // its reservation stranded. @@ -636,7 +723,7 @@ impl<'t, T: Transport> NoticeRx<'t, T> { landing.arm_post(); self.landing = Some(landing); self.standing = true; - Ok(None) + Ok(()) } // A refused repost retains the landing (Reserved, never released): // after `rearm` it may be the target of a post the kernel still @@ -895,6 +982,40 @@ mod tests { use super::*; use kore_abi::session::Committed; + /// A delegation reports exactly what its completion proves: delivered, a + /// retryable missing post, a proven refusal, or custody the peer may hold. + #[test] + fn delegation_outcome_keeps_status_index_and_custody_proof() { + let outcome = |status: Status, index| { + delegation_outcome::<()>(Committed::from_raw(status.encode(index))) + }; + assert_eq!(outcome(Status::Ok, 0), Ok(())); + assert_eq!(outcome(Status::WouldBlock, 0), Err(DelegateError::WouldBlock)); + assert_eq!( + outcome(Status::Revoked, 0), + Err(DelegateError::Refused { + status: Status::Revoked, + failing_index: 0 + }) + ); + // Not a rollback proof: the peer may hold a delegated child. + assert_eq!( + outcome(Status::Malformed, 0), + Err(DelegateError::Unproven { + status: Status::Malformed, + failing_index: 0 + }) + ); + // A one-invocation session cannot fail past its only invocation. + assert_eq!( + outcome(Status::Ok, 1), + Err(DelegateError::Unproven { + status: Status::Ok, + failing_index: 1 + }) + ); + } + #[test] fn derived_transfer_completion_is_fail_closed() { assert_eq!( diff --git a/native/crates/rt/src/entry.rs b/native/crates/rt/src/entry.rs index 21bbd0a..c20fc64 100644 --- a/native/crates/rt/src/entry.rs +++ b/native/crates/rt/src/entry.rs @@ -155,6 +155,8 @@ pub struct Rt { const PROCESS_IO_ATTACHMENT: usize = 1 << 0; #[cfg(any(test, target_os = "none"))] const PROCESS_STDIO_ATTACHMENT: usize = 1 << 1; +#[cfg(any(test, target_os = "none"))] +const PROCESS_STDIO_HOST_ATTACHMENT: usize = 1 << 2; /// Atomically spend one runtime attachment name. /// @@ -303,6 +305,16 @@ impl Rt { geometry::claim_runtime_attachment(PROCESS_STDIO_ATTACHMENT) } + /// Spend this task's sole safe stream-host attachment: the delegated + /// parent end of another process's stream. Same scope and one-shot law as + /// [`claim_process_stdio_attachment`](Self::claim_process_stdio_attachment), + /// and a distinct bit, so a task that is both a stream child and a stream + /// host binds each end exactly once. + #[cfg(target_os = "none")] + pub(crate) fn claim_process_stdio_host_attachment(&self) -> bool { + geometry::claim_runtime_attachment(PROCESS_STDIO_HOST_ATTACHMENT) + } + /// Permanently claim the half-open arena interval `[first, end)` for a /// service's compile-time protocol layout. /// @@ -2266,6 +2278,8 @@ mod runtime_slot_tests { assert!(!claim_attachment_word(&claims, PROCESS_IO_ATTACHMENT)); assert!(claim_attachment_word(&claims, PROCESS_STDIO_ATTACHMENT)); assert!(!claim_attachment_word(&claims, PROCESS_STDIO_ATTACHMENT)); + assert!(claim_attachment_word(&claims, PROCESS_STDIO_HOST_ATTACHMENT)); + assert!(!claim_attachment_word(&claims, PROCESS_STDIO_HOST_ATTACHMENT)); } #[test] diff --git a/native/crates/rt/src/executor/time.rs b/native/crates/rt/src/executor/time.rs index adba06f..79ebaad 100644 --- a/native/crates/rt/src/executor/time.rs +++ b/native/crates/rt/src/executor/time.rs @@ -25,21 +25,23 @@ use crate::transport::ParkPost; /// Concurrent pending deadlines per executor — a design bound (§8's /// posture: refusal on overflow, sized for the declared task set). /// -/// **`MAX_TASKS + 1`, and the `+ 1` is load-bearing.** `register_timer` -/// re-anchors by task, so the registrant set is exactly "one entry per header -/// that can hold a deadline" — and that is the admitted tasks *plus the root -/// future*, whose `TaskHeader` is a frame local in -/// [`Executor::run`](super::Executor::run) rather than a cell in `self.tasks`. -/// At a flat `MAX_TASKS` a service that filled its task set and then slept the -/// root — which is what a serve loop's `next_wake_for` does — overflowed a -/// table that had no room for the seventeenth registrant, and the overflow is -/// a panic: on a boot-founded service, a silent permanent park. -/// -/// Sized to the registrants rather than bounded by a refusal, because the -/// refusal here has no honest caller: a lost wake is not something a task can -/// be told about. +/// `MAX_TASKS + 1` retains room for one wait in every admitted task plus the +/// root. Registrations belong to individual waits, so composed futures in one +/// task share this same bounded table instead of replacing each other's alarm. +/// Callers must keep the total concurrent waits within this bound: inference's +/// NPU/GPU pair and terminal pump require three, not an unbounded task tree. +/// Exhaustion remains an invariant failure; silently losing an alarm is not a +/// valid refusal for these infallible wait APIs. pub const MAX_TIMERS: usize = super::MAX_TASKS + 1; +#[derive(Clone, Copy)] +struct TimerEntry { + deadline_ns: u64, + task: NonNull, + /// Identity and teardown backlink of the pinned registering wait. + owner: NonNull>>>, +} + /// Why the executor's last idle park returned — what [`next_wake`] /// resolves with, so a serve loop can tell a hardware-delivered wake /// ([`Irq`](Self::Irq)) from a peer's kick without any register read. @@ -82,8 +84,8 @@ pub(crate) struct ExecCtx { /// Why the last idle park returned ([`WakeReason`]) — read by /// [`next_wake`]'s ready leg. last_wake: Cell, - /// `(deadline_ns, task)` entries; `None` = free. - timers: [Cell)>>; MAX_TIMERS], + /// Independently owned pending waits; `None` = free. + timers: [Cell>; MAX_TIMERS], /// At most one task may add an alternate standing post to the executor's /// next idle park. The registering future clears it on ready or drop. delivery: Cell< @@ -202,38 +204,48 @@ impl ExecCtx { } } - /// Register (or re-anchor) `task`'s pending deadline — one entry per - /// task (a task has at most one pending `sleep_until` at a time; a - /// re-poll re-registers the same entry). + /// Register or re-anchor one pinned wait, independently of other waits + /// sharing its task. Only `TimerRegistration` supplies the owner pointer. /// /// # Panics /// When the table is full ([`MAX_TIMERS`]) — the §8 refusal made /// loud (a quiet drop would be a lost wake). - pub(crate) fn register_timer(&self, deadline_ns: u64, task: NonNull) { - // Re-anchor an existing entry for this task first… + fn register_timer( + &self, + deadline_ns: u64, + task: NonNull, + owner: NonNull>>>, + ) { + let entry = TimerEntry { + deadline_ns, + task, + owner, + }; + // Re-polling the same wait must not consume another bounded slot. for cell in &self.timers { - if let Some((_, t)) = cell.get() { - if t == task { - cell.set(Some((deadline_ns, task))); - return; - } + if let Some(previous) = cell.get() + && previous.owner == owner + { + assert_eq!(previous.task, task, "timer wait changed owning task"); + cell.set(Some(entry)); + return; } } // …else claim a free one. for cell in &self.timers { if cell.get().is_none() { - cell.set(Some((deadline_ns, task))); + cell.set(Some(entry)); return; } } panic!("executor timer table full (MAX_TIMERS = {MAX_TIMERS})"); } - /// Cancel `task`'s pending timer after another wake source won. A due - /// deadline has already removed its entry, so cancellation is idempotent. - pub(crate) fn clear_timer(&self, task: NonNull) { + /// The owner has already cleared its backlink. Remove only its entry; + /// neither a sibling wait nor a reused slot belongs to this cancellation. + fn clear_timer(&self, owner: NonNull>>>) { for cell in &self.timers { - if cell.get().is_some_and(|(_, registered)| registered == task) { + if cell.get().is_some_and(|entry| entry.owner == owner) { cell.set(None); return; } @@ -245,7 +257,7 @@ impl ExecCtx { pub(crate) fn timers_min(&self) -> Option { self.timers .iter() - .filter_map(|c| c.get().map(|(d, _)| d)) + .filter_map(|c| c.get().map(|entry| entry.deadline_ns)) .min() } @@ -300,13 +312,17 @@ impl ExecCtx { self.now.set(now_ns); } for cell in &self.timers { - if let Some((deadline, task)) = cell.get() { - if deadline <= self.now.get() { + if let Some(entry) = cell.get() { + if entry.deadline_ns <= self.now.get() { cell.set(None); + // SAFETY: the pinned wait removes its registration before + // drop. Invalidate its backlink before this slot can be + // reused, so later cancellation cannot touch a successor. + unsafe { entry.owner.as_ref() }.set(None); // SAFETY: registered headers are live for the run // (static storages; the root header outlives every // entry — `clear` runs before `run` returns). - unsafe { task.as_ref() }.mark_ready(); + unsafe { entry.task.as_ref() }.mark_ready(); } } } @@ -316,7 +332,12 @@ impl ExecCtx { /// header is a stack local; no entry may outlive the frame). pub(crate) fn clear_timers(&self) { for cell in &self.timers { - cell.set(None); + if let Some(entry) = cell.replace(None) { + // SAFETY: as in `advance`: every entry belongs to a pinned, + // still-live wait. Its later Drop must not follow this context + // after `Executor::run_pinned` returns or the executor drops. + unsafe { entry.owner.as_ref() }.set(None); + } } if let Some((_, _, _, mut future_owner)) = self.delivery.replace(None) { // SAFETY: alternate registration accepts this pointer only from a @@ -330,6 +351,54 @@ impl ExecCtx { } } +/// A wait's affine registration in the executor's existing bounded table. +/// The pinned field address identifies it; the context invalidates that field +/// when the timer fires or execution ends, following `AlternateWake`'s teardown +/// pattern. No registration can outlive its wait or follow a destroyed context. +struct TimerRegistration { + ctx: Cell>>, + _pin: PhantomPinned, +} + +impl TimerRegistration { + const fn new() -> Self { + Self { + ctx: Cell::new(None), + _pin: PhantomPinned, + } + } + + fn arm(self: Pin<&mut Self>, ctx: &ExecCtx, task: NonNull, deadline_ns: u64) { + let this = self.as_ref().get_ref(); + let context = NonNull::from(ctx); + if let Some(previous) = this.ctx.get() { + assert_eq!(previous, context, "timer wait changed executor"); + } + ctx.register_timer(deadline_ns, task, NonNull::from(&this.ctx)); + // Store only after successful registration: an exhausted table has + // published nothing for this new owner. + this.ctx.set(Some(context)); + } + + fn clear(self: Pin<&mut Self>) { + self.as_ref().get_ref().clear_registration(); + } + + fn clear_registration(&self) { + let Some(ctx) = self.ctx.take() else { return }; + // SAFETY: expiry and executor teardown invalidate this backlink while + // the context is live. A remaining backlink therefore names the same + // executor that owns this pinned wait, on its poll/drop thread. + unsafe { ctx.as_ref() }.clear_timer(NonNull::from(&self.ctx)); + } +} + +impl Drop for TimerRegistration { + fn drop(&mut self) { + self.clear_registration(); + } +} + /// Sleep until `deadline_ns` (absolute monotonic nanoseconds) — the §4 /// pinned timer verb. Resolves on the executor's clock: the idle path /// folds the earliest pending deadline into its combined park @@ -342,6 +411,7 @@ impl ExecCtx { /// needs its executor's timer table, reached through the cookie waker — /// there is nothing correct to do under another runtime. pub async fn sleep_until(deadline_ns: u64) { + let mut timer = core::pin::pin!(TimerRegistration::new()); poll_fn(move |cx| { let hdr = header_of(cx.waker()).expect("sleep_until requires the kore executor (foreign waker)"); @@ -356,9 +426,10 @@ pub async fn sleep_until(deadline_ns: u64) { // for the duration of the pass. let ctx = unsafe { &*ctx }; if ctx.now() >= deadline_ns { + timer.as_mut().clear(); Poll::Ready(()) } else { - ctx.register_timer(deadline_ns, hdr); + timer.as_mut().arm(ctx, hdr, deadline_ns); Poll::Pending } }) @@ -384,6 +455,7 @@ pub async fn sleep_until(deadline_ns: u64) { /// As [`sleep_until`]: polled outside the kore executor. pub async fn sleep_for(delta_ns: u64) { let mut deadline: Option = None; + let mut timer = core::pin::pin!(TimerRegistration::new()); poll_fn(move |cx| { let hdr = header_of(cx.waker()).expect("sleep_for requires the kore executor (foreign waker)"); @@ -396,9 +468,10 @@ pub async fn sleep_for(delta_ns: u64) { let ctx = unsafe { &*ctx }; let d = *deadline.get_or_insert_with(|| ctx.now().saturating_add(delta_ns)); if ctx.now() >= d { + timer.as_mut().clear(); Poll::Ready(()) } else { - ctx.register_timer(d, hdr); + timer.as_mut().arm(ctx, hdr, d); Poll::Pending } }) @@ -603,6 +676,7 @@ struct AlternateWake { delta_ns: Option, deadline_ns: Option, owner: Option, + timer: TimerRegistration, _pin: PhantomPinned, } @@ -611,7 +685,6 @@ struct AlternateOwner { task: NonNull, ctx: NonNull, delivery_generation: Option, - timer_registered: bool, } impl AlternateWake { @@ -621,11 +694,13 @@ impl AlternateWake { delta_ns, deadline_ns: None, owner: None, + timer: TimerRegistration::new(), _pin: PhantomPinned, } } fn clear_registration(&mut self) { + self.timer.clear_registration(); let Some(owner) = self.owner.take() else { return; }; @@ -635,9 +710,6 @@ impl AlternateWake { // leave `run_pinned`; target panics abort rather than unwind. Normal // executor exit also clears the complete registry as a backstop. let ctx = unsafe { owner.ctx.as_ref() }; - if owner.timer_registered { - ctx.clear_timer(owner.task); - } if let Some(generation) = owner.delivery_generation { ctx.clear_delivery(owner.task, generation); } @@ -685,7 +757,6 @@ impl Future for AlternateWake { task: hdr, ctx: NonNull::from(ctx), delivery_generation: None, - timer_registered: false, }); let future_owner = NonNull::from(&mut this.owner); let generation = ctx.register_delivery(this.post, hdr, future_owner); @@ -695,8 +766,9 @@ impl Future for AlternateWake { .expect("alternate owner disappeared during registration"); owner.delivery_generation = Some(generation); if let Some(deadline) = this.deadline_ns { - ctx.register_timer(deadline, hdr); - owner.timer_registered = true; + // SAFETY: this field stays pinned with AlternateWake and is + // cleared before that parent's Drop can release its storage. + unsafe { Pin::new_unchecked(&mut this.timer) }.arm(ctx, hdr, deadline); } Poll::Pending } @@ -722,6 +794,7 @@ impl Drop for AlternateWake { pub async fn next_wake_for(delta_ns: u64) -> WakeReason { let mut deadline = None; let mut parked = false; + let mut timer = core::pin::pin!(TimerRegistration::new()); poll_fn(move |cx| { let hdr = header_of(cx.waker()) .expect("next_wake_for requires the kore executor (foreign waker)"); @@ -736,14 +809,14 @@ pub async fn next_wake_for(delta_ns: u64) -> WakeReason { // the duration of this pass. let ctx = unsafe { &*ctx }; if parked { - ctx.clear_timer(hdr); + timer.as_mut().clear(); return Poll::Ready(ctx.last_wake()); } let deadline = *deadline.get_or_insert_with(|| ctx.now().saturating_add(delta_ns)); if ctx.now() >= deadline { return Poll::Ready(WakeReason::Deadline); } - ctx.register_timer(deadline, hdr); + timer.as_mut().arm(ctx, hdr, deadline); parked = true; Poll::Pending }) @@ -757,6 +830,7 @@ pub async fn next_wake_for(delta_ns: u64) -> WakeReason { /// new minimum cannot leave the old deadline behind. pub async fn next_wake_until(deadline_ns: u64) -> WakeReason { let mut parked = false; + let mut timer = core::pin::pin!(TimerRegistration::new()); poll_fn(move |cx| { let hdr = header_of(cx.waker()) .expect("next_wake_until requires the kore executor (foreign waker)"); @@ -769,13 +843,13 @@ pub async fn next_wake_until(deadline_ns: u64) -> WakeReason { // SAFETY: the executor installs this pointer for the duration of poll. let ctx = unsafe { &*ctx }; if parked { - ctx.clear_timer(hdr); + timer.as_mut().clear(); return Poll::Ready(ctx.last_wake()); } if ctx.now() >= deadline_ns { return Poll::Ready(WakeReason::Deadline); } - ctx.register_timer(deadline_ns, hdr); + timer.as_mut().arm(ctx, hdr, deadline_ns); parked = true; Poll::Pending }) @@ -788,6 +862,191 @@ mod tests { use super::*; use kore_abi::session::SlotRef; + fn task_waker(ctx: &ExecCtx, task: &TaskHeader) -> core::task::Waker { + task.exec + .store(ctx as *const ExecCtx as *mut ExecCtx, Ordering::Relaxed); + waker_for(NonNull::from(task)) + } + + #[test] + fn two_sleeps_in_one_task_keep_both_deadlines_in_either_poll_order() { + for reverse in [false, true] { + let ctx = ExecCtx::new(); + let task = TaskHeader::new(); + let waker = task_waker(&ctx, &task); + let mut cx = Context::from_waker(&waker); + let mut early = core::pin::pin!(sleep_until(17)); + let mut late = core::pin::pin!(sleep_until(29)); + if reverse { + assert_eq!(late.as_mut().poll(&mut cx), Poll::Pending); + assert_eq!(early.as_mut().poll(&mut cx), Poll::Pending); + } else { + assert_eq!(early.as_mut().poll(&mut cx), Poll::Pending); + assert_eq!(late.as_mut().poll(&mut cx), Poll::Pending); + } + assert_eq!(ctx.timers_min(), Some(17)); + ctx.advance(16); + assert!(!task.take_ready()); + ctx.advance(17); + assert!(task.take_ready()); + assert_eq!(ctx.timers_min(), Some(29)); + // Re-polling the remaining sleep re-anchors its own entry; it + // neither acquires a second slot nor restores the expired one. + assert_eq!(late.as_mut().poll(&mut cx), Poll::Pending); + assert_eq!(early.as_mut().poll(&mut cx), Poll::Ready(())); + assert_eq!( + ctx.timers + .iter() + .filter(|slot| slot.get().is_some()) + .count(), + 1 + ); + ctx.advance(29); + assert!(task.take_ready()); + assert_eq!(late.as_mut().poll(&mut cx), Poll::Ready(())); + assert_eq!(ctx.timers_min(), None); + } + } + + #[test] + fn cancelling_either_deadline_preserves_the_other_wait() { + for cancelled_deadline in [17, 59] { + let ctx = ExecCtx::new(); + let task = TaskHeader::new(); + let waker = task_waker(&ctx, &task); + let mut cx = Context::from_waker(&waker); + let mut retained = core::pin::pin!(sleep_until(41)); + assert_eq!(retained.as_mut().poll(&mut cx), Poll::Pending); + { + let mut cancelled = core::pin::pin!(sleep_until(cancelled_deadline)); + assert_eq!(cancelled.as_mut().poll(&mut cx), Poll::Pending); + } + assert_eq!(ctx.timers_min(), Some(41)); + ctx.advance(40); + assert!(!task.take_ready()); + ctx.advance(41); + assert!(task.take_ready()); + assert_eq!(retained.as_mut().poll(&mut cx), Poll::Ready(())); + } + } + + #[test] + fn expired_wait_cleanup_cannot_remove_a_reused_slot() { + let ctx = ExecCtx::new(); + let task = TaskHeader::new(); + let waker = task_waker(&ctx, &task); + let mut cx = Context::from_waker(&waker); + let mut replacement = core::pin::pin!(sleep_until(29)); + { + let mut expired = core::pin::pin!(sleep_until(17)); + assert_eq!(expired.as_mut().poll(&mut cx), Poll::Pending); + ctx.advance(17); + assert!(task.take_ready()); + assert_eq!(replacement.as_mut().poll(&mut cx), Poll::Pending); + // Drop the expired but not re-polled future after its table slot + // has been reused by another wait in the same task. + } + assert_eq!(ctx.timers_min(), Some(29)); + ctx.advance(29); + assert_eq!(replacement.as_mut().poll(&mut cx), Poll::Ready(())); + } + + #[test] + fn a_deadline_wakes_only_its_owning_task() { + let ctx = ExecCtx::new(); + let first = TaskHeader::new(); + let second = TaskHeader::new(); + let first_waker = task_waker(&ctx, &first); + let second_waker = task_waker(&ctx, &second); + let mut first_cx = Context::from_waker(&first_waker); + let mut second_cx = Context::from_waker(&second_waker); + let mut early = core::pin::pin!(sleep_for(17)); + let mut late = core::pin::pin!(sleep_for(29)); + assert_eq!(early.as_mut().poll(&mut first_cx), Poll::Pending); + assert_eq!(late.as_mut().poll(&mut second_cx), Poll::Pending); + ctx.advance(17); + assert!(first.take_ready()); + assert!(!second.take_ready()); + assert_eq!(early.as_mut().poll(&mut first_cx), Poll::Ready(())); + ctx.advance(29); + assert!(!first.take_ready()); + assert!(second.take_ready()); + assert_eq!(late.as_mut().poll(&mut second_cx), Poll::Ready(())); + } + + #[test] + fn alternate_and_ordinary_waits_do_not_cancel_each_other() { + for reverse in [false, true] { + let ctx = ExecCtx::new(); + let task = TaskHeader::new(); + let waker = task_waker(&ctx, &task); + let mut cx = Context::from_waker(&waker); + let post = ParkPost { + chan: 17, + land: SlotRef::from_wire(19), + recv_tag: 23, + }; + let mut ordinary = core::pin::pin!(next_wake_until(41)); + { + let mut alternate = core::pin::pin!(AlternateWake::new(post, Some(29))); + if reverse { + assert_eq!(ordinary.as_mut().poll(&mut cx), Poll::Pending); + assert_eq!(alternate.as_mut().poll(&mut cx), Poll::Pending); + } else { + assert_eq!(alternate.as_mut().poll(&mut cx), Poll::Pending); + assert_eq!(ordinary.as_mut().poll(&mut cx), Poll::Pending); + } + assert_eq!(ctx.timers_min(), Some(29)); + assert_eq!(ctx.delivery_post(), Some(post)); + // A completion on the alternate post must remove only the + // alternate deadline, regardless of child polling order. + ctx.note_wake(WakeReason::Delivery); + assert_eq!( + alternate.as_mut().poll(&mut cx), + Poll::Ready(WakeReason::Delivery) + ); + } + assert_eq!(ctx.delivery_post(), None); + assert_eq!(ctx.timers_min(), Some(41)); + ctx.advance(41); + ctx.note_wake(WakeReason::Deadline); + assert_eq!( + ordinary.as_mut().poll(&mut cx), + Poll::Ready(WakeReason::Deadline) + ); + + // Conversely, a cancelled ordinary wait cannot remove the + // alternate post's later deadline. + let mut alternate = core::pin::pin!(AlternateWake::new(post, Some(59))); + assert_eq!(alternate.as_mut().poll(&mut cx), Poll::Pending); + { + let mut cancelled = core::pin::pin!(next_wake_for(17)); + assert_eq!(cancelled.as_mut().poll(&mut cx), Poll::Pending); + } + assert_eq!(ctx.timers_min(), Some(100)); + assert_eq!(ctx.delivery_post(), Some(post)); + } + } + + #[test] + fn timer_teardown_allows_future_drop_after_the_context_is_gone() { + let mut sleep = core::pin::pin!(sleep_until(17)); + let mut wake = core::pin::pin!(next_wake_for(29)); + { + let ctx = ExecCtx::new(); + let task = TaskHeader::new(); + let waker = task_waker(&ctx, &task); + let mut cx = Context::from_waker(&waker); + assert_eq!(sleep.as_mut().poll(&mut cx), Poll::Pending); + assert_eq!(wake.as_mut().poll(&mut cx), Poll::Pending); + // The same cleanup Executor::run_pinned and Executor::drop use. + ctx.clear_timers(); + assert_eq!(ctx.timers_min(), None); + } + // Both pinned futures now drop after their context and header. Their + // invalidated registrations must not dereference either dead object. + } + #[test] fn cancelling_an_alternate_wait_releases_its_post_and_timer() { let ctx = ExecCtx::new(); @@ -870,6 +1129,7 @@ mod tests { assert!(wait.as_ref().get_ref().owner.is_some()); ctx.clear_timers(); assert!(wait.as_ref().get_ref().owner.is_none()); + assert!(wait.as_ref().get_ref().timer.ctx.get().is_none()); assert_eq!(ctx.delivery_post(), None); assert_eq!(ctx.timers_min(), None); // `wait` now drops without following the invalidated context backlink. @@ -879,15 +1139,19 @@ mod tests { fn an_early_wake_cancels_its_folded_deadline() { let ctx = ExecCtx::new(); let task = TaskHeader::new(); - let task = NonNull::from(&task); - - ctx.register_timer(17, task); + let waker = task_waker(&ctx, &task); + let mut cx = Context::from_waker(&waker); + let mut wait = core::pin::pin!(next_wake_for(17)); + assert_eq!(wait.as_mut().poll(&mut cx), Poll::Pending); assert_eq!(ctx.timers_min(), Some(17)); - ctx.clear_timer(task); + ctx.note_wake(WakeReason::Doorbell); + assert_eq!( + wait.as_mut().poll(&mut cx), + Poll::Ready(WakeReason::Doorbell) + ); assert_eq!(ctx.timers_min(), None); - // A deadline win removed the same entry before the future's ready - // leg; cancellation must therefore remain harmless. - ctx.clear_timer(task); + ctx.advance(17); + assert!(!task.take_ready()); assert_eq!(ctx.timers_min(), None); } @@ -912,10 +1176,18 @@ mod tests { let task = TaskHeader::new(); let task = NonNull::from(&task); - ctx.register_timer(90, task); - ctx.register_timer(40, task); + let mut registration = core::pin::pin!(TimerRegistration::new()); + registration.as_mut().arm(&ctx, task, 90); + registration.as_mut().arm(&ctx, task, 40); assert_eq!(ctx.timers_min(), Some(40)); - ctx.clear_timer(task); + assert_eq!( + ctx.timers + .iter() + .filter(|slot| slot.get().is_some()) + .count(), + 1 + ); + registration.as_mut().clear(); assert_eq!(ctx.timers_min(), None); } diff --git a/native/crates/rt/src/heap/mod.rs b/native/crates/rt/src/heap/mod.rs index e6ffebe..e58dd39 100644 --- a/native/crates/rt/src/heap/mod.rs +++ b/native/crates/rt/src/heap/mod.rs @@ -56,8 +56,8 @@ mod source; pub use crate::mem_source::MemSource; #[cfg(target_os = "none")] pub use source::carve_and_map; -pub use source::{adopt_supply, reserve_from_top}; pub use source::{HeapSupplySource, StaticMemSource}; +pub use source::{adopt_supply, reserve_from_top}; use core::alloc::{AllocError, Allocator, GlobalAlloc, Layout}; use core::ptr::NonNull; diff --git a/native/crates/rt/src/heap/source.rs b/native/crates/rt/src/heap/source.rs index 53b6468..c3564b9 100644 --- a/native/crates/rt/src/heap/source.rs +++ b/native/crates/rt/src/heap/source.rs @@ -519,7 +519,10 @@ mod window { /// The next granule off the launch supply (`role::HEAP@0`), reserved: /// `(supply, offset, size)`. - fn launch_granule(missing: u64, doubling: u64) -> Option<(kore_abi::session::SlotRef, u64, u64)> { + fn launch_granule( + missing: u64, + doubling: u64, + ) -> Option<(kore_abi::session::SlotRef, u64, u64)> { let supply = crate::entry::geometry::heap_supply_slot()?; let total = super::supply_len(supply)?; let room = total @@ -545,7 +548,10 @@ mod window { /// The next granule off an adopted supply ([`super::adopt_supply`]), /// reserved. Nothing else draws on an adopted lease, so its share is all /// that remains of it. - fn adopted_granule(missing: u64, doubling: u64) -> Option<(kore_abi::session::SlotRef, u64, u64)> { + fn adopted_granule( + missing: u64, + doubling: u64, + ) -> Option<(kore_abi::session::SlotRef, u64, u64)> { super::ADOPTED.iter().find_map(|entry| { let slot = entry.slot.load(Ordering::Acquire).checked_sub(1)?; let room = entry @@ -579,16 +585,21 @@ mod window { Some(kore_abi::session::ViewMapWindow::Heap), )?; GRANULES.fetch_add(1, Ordering::Relaxed); - let start = (granule.as_ptr() as *mut u8 as u64) - .checked_sub(BASE.load(Ordering::Relaxed))? - / PAGE; - let stop = start.checked_add(size / PAGE).filter(|&stop| stop <= u32::MAX as u64)?; + let start = + (granule.as_ptr() as *mut u8 as u64).checked_sub(BASE.load(Ordering::Relaxed))? / PAGE; + let stop = start + .checked_add(size / PAGE) + .filter(|&stop| stop <= u32::MAX as u64)?; // Extend the interval when the granule lands at its end; otherwise // the new granule replaces it and the old tail is left unused. let mut word = FREE.load(Ordering::Acquire); loop { let (at, end) = unpack(word); - let next = if start == end { pack(at, stop) } else { pack(start, stop) }; + let next = if start == end { + pack(at, stop) + } else { + pack(start, stop) + }; match FREE.compare_exchange_weak(word, next, Ordering::AcqRel, Ordering::Acquire) { Ok(_) => return Some(()), Err(actual) => word = actual, diff --git a/native/crates/rt/src/pinned.rs b/native/crates/rt/src/pinned.rs index 7d9ed9a..c347815 100644 --- a/native/crates/rt/src/pinned.rs +++ b/native/crates/rt/src/pinned.rs @@ -189,7 +189,11 @@ mod tests { drop(pinned); }); back.join().unwrap(); - assert_eq!(drops.load(Ordering::SeqCst), 0, "leaked, never dropped off-owner"); + assert_eq!( + drops.load(Ordering::SeqCst), + 0, + "leaked, never dropped off-owner" + ); } #[test] diff --git a/native/crates/rt/src/process.rs b/native/crates/rt/src/process.rs index bdf0a4d..5c07158 100644 --- a/native/crates/rt/src/process.rs +++ b/native/crates/rt/src/process.rs @@ -1,10 +1,11 @@ //! Generic foreground-process launch/result and streaming-I/O pages. +pub mod interrupt; + use core::marker::PhantomData; use core::sync::atomic::{AtomicU32, Ordering}; use core::time::Duration; -#[cfg(any(target_os = "none", test))] use kore_abi::cap::Rights; use kore_abi::monitor::TelemetrySnapshot; #[cfg(target_os = "none")] @@ -154,6 +155,22 @@ pub enum ProcessStdioWaitFor { Incoming, /// Wake for an incoming record or newly available outgoing capacity. IncomingOrCapacity, + /// Wake only for newly available outgoing capacity: the caller holds + /// output that does not fit, and records arriving meanwhile wait for it. + /// Waiting for either would return at once while input sits unread. + Capacity, +} + +impl ProcessStdioWaitFor { + /// Whether a side with `incoming` records unread and `capacity` free + /// outgoing slots has nothing to wait for. + const fn satisfied(self, incoming: bool, capacity: bool) -> bool { + match self { + Self::Incoming => incoming, + Self::IncomingOrCapacity => incoming || capacity, + Self::Capacity => capacity, + } + } } /// Why a stream-side park attempt returned. @@ -175,6 +192,10 @@ pub enum ProcessStdioWaitError { InvalidNotice, /// The kernel resolved the alternate post but its owner found no notice. MissingNotice, + /// A stream host has no dedicated receive: its peer's notices land on the + /// host's own executor doorbell. Arm with `arm_shared_doorbell` and park + /// on the executor instead. + SharedDoorbell, } impl From for ProcessStdioWaitError { @@ -218,12 +239,21 @@ impl Drop for ProcessStdioWaiting<'_> { } } -struct ProcessStdioPeer<'w, 't, T: Transport> { - transport: &'t T, - send: ProcessStdioSend, +/// A dedicated wake receive and the notice machinery bound to its slot. +struct PeerReceive<'t, T: Transport> { /// Retained solely as the affine ownership witness for `notice`'s slot. _recv: ProcessStdioRecv, notice: NoticeRx<'t, T>, +} + +struct ProcessStdioPeer<'w, 't, T: Transport> { + transport: &'t T, + send: ProcessStdioSend, + /// `None` only for a stream host ([`map_stdio_host`]): the child's output + /// wake is a send into the host's own executor doorbell, so the host + /// parks with [`ProcessStdioParent::arm_shared_doorbell`] and has no + /// receive to arm, consume, or disarm. + receive: Option>, /// This side's one-writer signal line: increment after every enqueue or /// dequeue that can make work/capacity visible to the peer. published: &'w AtomicU32, @@ -299,16 +329,13 @@ impl<'w, T: Transport> ProcessStdioPeer<'w, '_, T> { } fn close_after_recv_root_revoked(self) { - let Self { - send, - _recv: recv, - notice, - .. - } = self; - notice.close_after_root_revoked(); - // Root revocation made both endpoint records dead before their affine - // wrappers release local dynamic-arena names. - drop(recv); + let Self { send, receive, .. } = self; + if let Some(PeerReceive { _recv: recv, notice }) = receive { + notice.close_after_root_revoked(); + // Root revocation made both endpoint records dead before their + // affine wrappers release local dynamic-arena names. + drop(recv); + } drop(send); } @@ -372,9 +399,13 @@ impl<'w, T: Transport> ProcessStdioPeer<'w, '_, T> { // Arm the dedicated receive before publishing `waiting`. If a prior // signal already landed, consume it and re-run the caller's drain // without parking. - match self.notice.try_recv() { + let recv_cookie = self.recv_cookie; + let Some(receive) = self.receive.as_mut() else { + return Err(ProcessStdioWaitError::SharedDoorbell); + }; + match receive.notice.try_recv() { Ok(Some(notice)) => { - return if self.notice_is_valid(notice) { + return if notice.cookie == recv_cookie { Ok(ProcessStdioParkStep::Changed) } else { Err(ProcessStdioWaitError::InvalidNotice) @@ -386,7 +417,7 @@ impl<'w, T: Transport> ProcessStdioPeer<'w, '_, T> { } Err(error) => return Err(ProcessStdioWaitError::Receive(error)), } - let Some(post) = self.notice.standing_post() else { + let Some(post) = receive.notice.standing_post() else { return Err(ProcessStdioWaitError::Receive(RecvError::Refused( kore_abi::session::Status::Malformed, ))); @@ -416,10 +447,13 @@ impl<'w, T: Transport> ProcessStdioPeer<'w, '_, T> { reason: WakeReason, ) -> Result { drop(ticket); + let Some(receive) = self.receive.as_mut() else { + return Err(ProcessStdioWaitError::SharedDoorbell); + }; let delivered = if reason == WakeReason::Delivery { - self.notice.settle_delivery()? + receive.notice.settle_delivery()? } else { - self.notice.try_recv()? + receive.notice.try_recv()? }; match (reason, delivered) { (_, Some(notice)) if self.notice_is_valid(notice) => Ok(ProcessStdioPark::StateChanged), @@ -477,7 +511,8 @@ impl ProcessStdioParkTicket<'_> { /// The rings are private so every successful dequeue also publishes newly /// available capacity and every enqueue publishes data before issuing the /// directional kick. Construct this only by consuming -/// [`PreparedProcessStdio::attach_parent`]. +/// [`PreparedProcessStdio::attach_parent`] (the launcher's own end), or, in a +/// stream host, through [`map_stdio_host`] (an end the launcher delegated). #[must_use = "retain the stream endpoint through task quiescence, then close it after both wake roots are revoked"] pub struct ProcessStdioParent<'w, 't, T: Transport> { stdin: Producer<'w, ProcessInputChunk>, @@ -544,6 +579,12 @@ impl ProcessStdioParent<'_, '_, T> { self.stdin.capacity_left() } + /// Advisory input records the child has not consumed yet — how a + /// launcher learns that the child read an interrupt it sent last. + pub fn stdin_unconsumed(&self) -> u32 { + self.stdin.capacity().saturating_sub(self.stdin.capacity_left()) + } + /// Advisory published output events awaiting this parent. pub fn output_outstanding(&self) -> u32 { self.output.outstanding() @@ -562,11 +603,7 @@ impl ProcessStdioParent<'_, '_, T> { self.peer .park_after_drain_inner( drain, - || { - output.outstanding() != 0 - || (wait_for == ProcessStdioWaitFor::IncomingOrCapacity - && stdin.capacity_left() != 0) - }, + || wait_for.satisfied(output.outstanding() != 0, stdin.capacity_left() != 0), None, ) .await @@ -592,11 +629,7 @@ impl ProcessStdioParent<'_, '_, T> { self.peer .park_after_drain_inner( drain, - || { - output.outstanding() != 0 - || (wait_for == ProcessStdioWaitFor::IncomingOrCapacity - && stdin.capacity_left() != 0) - }, + || wait_for.satisfied(output.outstanding() != 0, stdin.capacity_left() != 0), Some(delta_ns), ) .await @@ -657,7 +690,10 @@ impl<'w, T: Transport> ProcessStdioChild<'w, '_, T> { /// [`try_send_output`](Self::try_send_output), with the parent's data /// notice committed synchronously. [`Full`] alone means nothing was /// published. - pub fn send_output_now(&mut self, record: ProcessOutputEvent) -> Result { + pub fn send_output_now( + &mut self, + record: ProcessOutputEvent, + ) -> Result { self.output.push(record)?; Ok(self.peer.signal_now()) } @@ -698,7 +734,11 @@ impl<'w, T: Transport> ProcessStdioChild<'w, '_, T> { ) -> Result { let (stdin, output) = (&self.stdin, &self.output); self.peer - .park_after_drain_inner(drain, || stdio_ready(stdin, output, wait_for), Some(delta_ns)) + .park_after_drain_inner( + drain, + || stdio_ready(stdin, output, wait_for), + Some(delta_ns), + ) .await } @@ -729,15 +769,14 @@ impl<'w, T: Transport> ProcessStdioChild<'w, '_, T> { } } -/// Whether a child's park has nothing to wait for: input arrived, or (when -/// asked) output capacity opened. +/// Whether a child's park has nothing to wait for: input arrived, or output +/// capacity opened, as `wait_for` asks. fn stdio_ready( stdin: &Consumer<'_, ProcessInputChunk>, output: &Producer<'_, ProcessOutputEvent>, wait_for: ProcessStdioWaitFor, ) -> bool { - stdin.outstanding() != 0 - || (wait_for == ProcessStdioWaitFor::IncomingOrCapacity && output.capacity_left() != 0) + wait_for.satisfied(stdin.outstanding() != 0, output.capacity_left() != 0) } /// A freshly formatted parent attachment, before its directional wake @@ -776,7 +815,7 @@ impl<'w> PreparedProcessStdio<'w> { len, transport, ProcessStdioSend::Owned(stdin_send), - ProcessStdioRecv::Owned(output_recv), + Some(ProcessStdioRecv::Owned(output_recv)), ) } } @@ -876,25 +915,28 @@ pub fn init_stdio(window: &mut [u8]) -> Result, RingErr }) } +/// Bind the parent end. `output_recv` is `None` only for a stream host, +/// whose child output wake rings the host's own executor doorbell. unsafe fn bind_parent<'w, 't, T: Transport>( base: *mut u8, len: usize, transport: &'t T, stdin_send: ProcessStdioSend, - output_recv: ProcessStdioRecv, + output_recv: Option, ) -> Result, RingError> { // SAFETY: the caller owns this local mapping attachment for `'w`. let window = unsafe { core::slice::from_raw_parts_mut(base, len) }; let regions = stdio_regions(window)?; - let output_recv_slot = output_recv.slot(); Ok(ProcessStdioParent { stdin: spsc::producer::(regions.stdin)?, output: spsc::consumer::(regions.output)?, peer: ProcessStdioPeer { transport, send: stdin_send, - _recv: output_recv, - notice: NoticeRx::new(transport, output_recv_slot), + receive: output_recv.map(|recv| PeerReceive { + notice: NoticeRx::new(transport, recv.slot()), + _recv: recv, + }), published: regions.to_child, peer_waiting: regions.child_waiting, observed: regions.to_parent, @@ -916,15 +958,16 @@ unsafe fn bind_child<'w, 't, T: Transport>( // SAFETY: the caller owns this local mapping attachment for `'w`. let window = unsafe { core::slice::from_raw_parts_mut(base, len) }; let regions = stdio_regions(window)?; - let stdin_recv_slot = stdin_recv.slot(); Ok(ProcessStdioChild { stdin: spsc::consumer::(regions.stdin)?, output: spsc::producer::(regions.output)?, peer: ProcessStdioPeer { transport, send: output_send, - _recv: stdin_recv, - notice: NoticeRx::new(transport, stdin_recv_slot), + receive: Some(PeerReceive { + notice: NoticeRx::new(transport, stdin_recv.slot()), + _recv: stdin_recv, + }), published: regions.to_parent, peer_waiting: regions.parent_waiting, observed: regions.to_child, @@ -953,7 +996,7 @@ fn split_stdio<'w, 'p, 'c, P: Transport, C: Transport>( len, parent_transport, ProcessStdioSend::Test(SlotRef::from_wire(41)), - ProcessStdioRecv::Test(SlotRef::from_wire(42)), + Some(ProcessStdioRecv::Test(SlotRef::from_wire(42))), )?, bind_child( base, @@ -1066,6 +1109,125 @@ pub fn map_stdio(rt: &Rt) -> Option { } } +/// A stream host's delegated parent end: the stdin producer, the output +/// consumer, and a send-only peer (`res/SHELL_TERMINAL.md` §6). +pub type MappedProcessStdioHost = + ProcessStdioParent<'static, 'static, crate::transport::TrapTransport>; + +/// What a stream host holds at startup: the mapped stream page, not yet a +/// parent end, and the receive of its **host channel**. +/// +/// The launcher formatted the page before either end was founded and keeps +/// every lifecycle pivot: the child's task handle, the channel roots, and the +/// backing the view is carved from. Over the host channel it delivers the +/// stdin-wake send, which [`HostStdioView::bind`] turns the page into a +/// parent end with, and then a delegated handle for each foreground task the +/// host may end (`res/SHELL_TERMINAL.md` §9). +pub struct HostStdio { + pub view: HostStdioView, + /// The host channel's receive: build a [`crate::chan::CapRx`] over its + /// slot. Affine, so this task holds exactly one witness to it. + pub channel: crate::cap::HeldChanRecvToken, +} + +/// The mapped stream page of a stream host, awaiting its stdin-wake send. +pub struct HostStdioView { + base: *mut u8, +} + +impl HostStdioView { + /// Bind the parent end with the stdin-wake send the launcher delivered. + /// + /// The host has no output receive: the child's output wake is a send + /// into this task's own executor doorbell, so the host drains on every + /// executor wake and arms with [`ProcessStdioParent::arm_shared_doorbell`]; + /// `park_after_drain*` answer [`ProcessStdioWaitError::SharedDoorbell`]. + /// Binding validates both ring headers and never formats: re-running + /// [`init_stdio`] here would zero records the child may already have + /// published. + /// Binding refuses — with the ring's own reason, and the view spent — + /// only when the page is not a formatted stream. + pub fn bind(self, stdin_send: ChanSendToken) -> Result { + // SAFETY: the host attachment bit was atomically spent before the + // view was taken, and consuming `self` spends this one binding, so + // safe code cannot construct a second local parent pair. The mapped + // view remains installed for the task's lifetime. + unsafe { + bind_parent( + self.base, + kore_abi::process::PROCESS_STDIO_BYTES, + crate::entry::transport(), + ProcessStdioSend::Owned(stdin_send), + None, + ) + } + } +} + +/// Why a stream host's delegated capabilities did not map. Each variant +/// names the one check that refused. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum HostStdioError { + /// Neither role was staged: an ordinary launch of the same image. + Absent, + /// Only one of the two roles was staged. + Partial, + /// This task already spent its one stream-host attachment. + AlreadyAttached, + /// The view role does not hold a delivered view. + ViewMissing, + /// The view lacks `ACCESS | MODIFY`; it carries these rights. + ViewRights(Rights), + /// The channel role does not hold a receive with `ACCESS`. + ChannelMissing, + /// The kernel refused to map the view. + MapRefused, + /// The mapping is shorter than a stream page, or not cache-line aligned. + Geometry { len: usize, aligned: bool }, +} + +/// Map the stream page and host channel a stream host's launcher delegated. +#[cfg(target_os = "none")] +pub fn map_stdio_host(rt: &Rt) -> Result { + use kore_abi::process::{ + PROCESS_STDIO_HOST_CHANNEL_INDEX, PROCESS_STDIO_HOST_INDEX, PROCESS_STDIO_HOST_ROLE, + }; + let bootstrap = rt.bootstrap(); + let (view_slot, channel_slot) = match ( + bootstrap.handle(PROCESS_STDIO_HOST_ROLE, PROCESS_STDIO_HOST_INDEX), + bootstrap.handle(PROCESS_STDIO_HOST_ROLE, PROCESS_STDIO_HOST_CHANNEL_INDEX), + ) { + (Some(view), Some(channel)) => (view, channel), + // Absent roles are the ordinary launch, and must not spend the claim. + (None, None) => return Err(HostStdioError::Absent), + _ => return Err(HostStdioError::Partial), + }; + if !rt.claim_process_stdio_host_attachment() { + return Err(HostStdioError::AlreadyAttached); + } + let view = crate::cap::verbs::take_view(view_slot).ok_or(HostStdioError::ViewMissing)?; + // As for the child: both ring halves write their local cursors. + if !process_stdio_view_is_writable(view.rights()) { + return Err(HostStdioError::ViewRights(view.rights())); + } + let channel = crate::cap::verbs::take_chan_recv(channel_slot, Rights::ACCESS) + .ok_or(HostStdioError::ChannelMissing)?; + let mapped = crate::cap::verbs::map_view(view).ok_or(HostStdioError::MapRefused)?; + let aligned = (mapped.cast::().as_ptr() as usize).is_multiple_of(64); + if mapped.len() < PROCESS_STDIO_BYTES || !aligned { + return Err(HostStdioError::Geometry { + len: mapped.len(), + aligned, + }); + } + Ok(HostStdio { + view: HostStdioView { + base: mapped.cast::().as_ptr(), + }, + channel, + }) +} + /// Host builds have no mapped capability view. #[cfg(not(target_os = "none"))] pub fn map(_rt: &Rt) -> Option<&'static mut ProcessIo> { @@ -1078,6 +1240,12 @@ pub fn map_stdio(_rt: &Rt) -> Option { None } +/// Host builds have no mapped capability view. +#[cfg(not(target_os = "none"))] +pub fn map_stdio_host(_rt: &Rt) -> Result { + Err(HostStdioError::Absent) +} + #[cfg(test)] mod tests { use super::*; @@ -1236,6 +1404,68 @@ mod tests { assert!(parent.arm_shared_doorbell(parent.begin_drain())); } + /// A stream host binds the parent end with no dedicated receive: it parks + /// only through the shared-doorbell fence, never through a notice post it + /// does not hold, and closes without disarming a landing it never armed. + #[test] + fn a_capacity_wait_ignores_unread_input() { + use ProcessStdioWaitFor::{Capacity, Incoming, IncomingOrCapacity}; + // Output full, input waiting: only a capacity wait still parks. + assert!(Incoming.satisfied(true, false)); + assert!(IncomingOrCapacity.satisfied(true, false)); + assert!(!Capacity.satisfied(true, false)); + assert!(Capacity.satisfied(false, true)); + assert!(!Incoming.satisfied(false, true)); + } + + #[test] + fn a_stream_host_parks_only_through_its_shared_doorbell() { + let transport = OkTransport::new(); + let mut page = Page::zeroed(); + let prepared = init_stdio(&mut page.0).unwrap(); + let (base, len) = prepared.into_raw(); + let (mut host, mut child) = unsafe { + ( + bind_parent( + base, + len, + &transport, + ProcessStdioSend::Test(SlotRef::from_wire(41)), + None, + ) + .unwrap(), + bind_child( + base, + len, + &transport, + ProcessStdioSend::Test(SlotRef::from_wire(43)), + ProcessStdioRecv::Test(SlotRef::from_wire(44)), + ) + .unwrap(), + ) + }; + assert!(host.arm_shared_doorbell(host.begin_drain())); + assert_eq!( + drive(child.try_send_output(ProcessOutputEvent::input_ready())).unwrap(), + ProcessStdioWake::Delivered + ); + assert_eq!( + drive(host.try_recv_output()).unwrap().0.kind(), + Ok(ProcessOutputKind::InputReady) + ); + let drain = host.begin_drain(); + assert_eq!( + drive(host.park_after_drain(drain, ProcessStdioWaitFor::Incoming)), + Err(ProcessStdioWaitError::SharedDoorbell) + ); + drive(host.try_send_stdin(ProcessInputChunk::data(b"ls\n").unwrap())).unwrap(); + assert_eq!( + drive(child.try_recv_stdin()).unwrap().0.bytes(), + Ok(b"ls\n".as_slice()) + ); + host.close_after_wake_roots_revoked(); + } + #[test] fn affine_stream_methods_publish_both_data_and_capacity_edges() { let transport = OkTransport::new(); @@ -1341,6 +1571,28 @@ mod tests { ); } + /// An interrupt is not input: it crosses with no InputReady outstanding, + /// and the launcher learns the child read it from the ring alone. + #[test] + fn an_unsolicited_interrupt_crosses_and_its_read_is_observable() { + let transport = OkTransport::new(); + let mut page = Page::zeroed(); + let (mut parent, mut child) = split_stdio(&mut page.0, &transport, &transport).unwrap(); + + assert_eq!(parent.stdin_unconsumed(), 0); + assert_eq!( + drive(parent.try_send_stdin(ProcessInputChunk::interrupt())).unwrap(), + ProcessStdioWake::PeerNotWaiting + ); + assert_eq!(parent.stdin_unconsumed(), 1); + assert_eq!(child.stdin_outstanding(), 1); + let (record, _) = drive(child.try_recv_stdin()).unwrap(); + assert_eq!(record.kind(), Ok(ProcessInputKind::Interrupt)); + assert_eq!(parent.stdin_unconsumed(), 0); + // The child may still close its stream: nothing was left unanswered. + assert!(drive(child.try_send_output(ProcessOutputEvent::closed())).is_ok()); + } + #[test] fn unreachable_kick_never_rolls_back_published_state() { let transport = RefusedTransport; @@ -1370,7 +1622,7 @@ mod tests { page.0.len(), &transport, ProcessStdioSend::Test(SlotRef::from_wire(41)), - ProcessStdioRecv::Test(SlotRef::from_wire(42)), + Some(ProcessStdioRecv::Test(SlotRef::from_wire(42))), ) }; assert!(matches!(result, Err(RingError::BadHeader))); @@ -1385,7 +1637,7 @@ mod tests { len, &transport, ProcessStdioSend::Test(SlotRef::from_wire(41)), - ProcessStdioRecv::Test(SlotRef::from_wire(42)), + Some(ProcessStdioRecv::Test(SlotRef::from_wire(42))), ) }; assert!(attached.is_ok()); diff --git a/native/crates/rt/src/process/interrupt.rs b/native/crates/rt/src/process/interrupt.rs new file mode 100644 index 0000000..5deb37b --- /dev/null +++ b/native/crates/rt/src/process/interrupt.rs @@ -0,0 +1,142 @@ +//! Ctrl-C for an interactive child (`res/LIFECYCLE_HARDENING.md` §3): +//! notice, bounded grace, revoke. This is the one policy every holder of a +//! process stream's parent end follows — the shell for its foreground child, +//! a `term` window for the program it hosts (`res/SHELL_TERMINAL.md` §7.3). +//! +//! The first press asks the child to stop: the stream's parent sends one +//! [`ProcessInputKind::Interrupt`] record, and a child that reads it tears +//! down what it holds (a device queue, DMA windows another service mapped) +//! before it exits. Revoking the task instead cuts those windows out from +//! under a device that may still be using them, which is what sealed the NPU +//! and GPU after a Ctrl-C. The parent still revokes, but only once the child +//! is plainly not listening: a second press while the record is still unread, +//! or the grace running out with it unread. A child that read the record is +//! handling it (a REPL abandons its line and prompts again; `inference` +//! releases its devices and exits), so reading it ends the grace. +//! +//! The notice is a record the child reads at its own safe points, never a +//! handler run at a distance: no code of the child's runs because of it until +//! the child looks. The authority that ends an unheeding child stays a +//! capability (the parent's task handle, or a delegated foreground slot). +//! +//! [`ProcessInputKind::Interrupt`]: kore_abi::process::ProcessInputKind::Interrupt + +/// How long an unread interrupt waits before the parent revokes the task. Long +/// enough for a child to reach its next safe point from inside a prompt +/// prefill; a second Ctrl-C revokes at once. +pub const INTERRUPT_GRACE_NS: u64 = 60_000_000_000; + +/// What one Ctrl-C press asks of the stream's parent. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Press { + /// Send the child an interrupt record and start the grace. + Forward, + /// The child left an earlier interrupt unread: revoke it now. + Revoke, +} + +/// The one interrupt a stream's parent may have outstanding. +#[derive(Clone, Copy, Debug, Default)] +pub struct Interrupt { + /// When the outstanding record was sent, until the child reads it. + sent_ns: Option, +} + +impl Interrupt { + pub const fn new() -> Self { + Self { sent_ns: None } + } + + /// One Ctrl-C press. `unread` is how many input records the child has + /// not consumed; the interrupt, sent last, is read last. + pub fn press(&mut self, now_ns: u64, unread: u32) -> Press { + self.observe(unread); + if self.sent_ns.is_some() { + return Press::Revoke; + } + self.sent_ns = Some(now_ns); + Press::Forward + } + + /// Whether the grace ran out with the interrupt still unread. Reading it + /// ends the grace. + pub fn expired(&mut self, now_ns: u64, unread: u32) -> bool { + self.observe(unread); + self.sent_ns + .is_some_and(|sent| now_ns.saturating_sub(sent) >= INTERRUPT_GRACE_NS) + } + + /// Whether an interrupt is outstanding, as last observed. + pub fn outstanding(&self) -> bool { + self.sent_ns.is_some() + } + + /// The longest the parent may park before it must look at the grace + /// again, or `None` with no interrupt outstanding. + pub fn park_ns(&self, now_ns: u64) -> Option { + self.sent_ns.map(|sent| { + INTERRUPT_GRACE_NS + .saturating_sub(now_ns.saturating_sub(sent)) + .max(1) + }) + } + + fn observe(&mut self, unread: u32) { + if unread == 0 { + self.sent_ns = None; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_first_press_forwards_and_a_second_unread_one_revokes() { + let mut interrupt = Interrupt::new(); + assert_eq!(interrupt.press(10, 0), Press::Forward); + // The record is in the ring and unread. + assert_eq!(interrupt.press(20, 1), Press::Revoke); + } + + #[test] + fn a_read_interrupt_ends_the_grace_and_the_next_press_forwards_again() { + let mut interrupt = Interrupt::new(); + assert_eq!(interrupt.press(0, 0), Press::Forward); + assert!(!interrupt.expired(1, 1)); + // The child read it (a REPL clearing its line): no kill pending. + assert!(!interrupt.expired(2, 0)); + assert_eq!(interrupt.park_ns(2), None); + assert!(!interrupt.expired(INTERRUPT_GRACE_NS * 2, 0)); + // Another press is a fresh request, not a revoke. + assert_eq!(interrupt.press(3, 0), Press::Forward); + } + + #[test] + fn an_unread_interrupt_expires_after_the_grace() { + let mut interrupt = Interrupt::new(); + assert_eq!(interrupt.press(100, 2), Press::Forward); + assert!(!interrupt.expired(100 + INTERRUPT_GRACE_NS - 1, 3)); + assert!(interrupt.expired(100 + INTERRUPT_GRACE_NS, 3)); + } + + #[test] + fn the_park_bound_counts_down_to_the_grace() { + let mut interrupt = Interrupt::new(); + assert_eq!(interrupt.park_ns(0), None); + let _ = interrupt.press(1_000, 0); + assert_eq!(interrupt.park_ns(1_000), Some(INTERRUPT_GRACE_NS)); + assert_eq!(interrupt.park_ns(1_000 + INTERRUPT_GRACE_NS), Some(1)); + } + + #[test] + fn earlier_unread_input_holds_the_interrupt_behind_it() { + // Two data records the child never read, then the interrupt: all + // three must drain before the grace ends. + let mut interrupt = Interrupt::new(); + assert_eq!(interrupt.press(0, 2), Press::Forward); + assert!(!interrupt.expired(5, 1)); + assert_eq!(interrupt.press(6, 1), Press::Revoke); + } +} diff --git a/native/crates/rt/src/service/mod.rs b/native/crates/rt/src/service/mod.rs index 021deef..17ee8f4 100644 --- a/native/crates/rt/src/service/mod.rs +++ b/native/crates/rt/src/service/mod.rs @@ -499,6 +499,20 @@ pub trait SystemService: crate::svc::SvcProtocol + Sized { true } + /// Advance service-owned device work on **every** executor wake of the + /// generic host, request-bearing or not. + /// + /// A driver row whose parks include its device route + /// ([`enable_irq_parks`](crate::executor::enable_irq_parks)) is re-polled + /// on each interrupt, but the ordinary loop finds no ring work on such a + /// wake and parks again: a completion the row itself must reap and + /// refill (a queue kept at depth for a client that is not calling) + /// would otherwise wait for the next request. This hook is that reap. + /// + /// It must be bounded and must never wait: it runs inside the host's + /// poll, before the request rings are drained. The default is inert. + fn poll_background(&mut self) {} + /// Earliest absolute monotonic deadline owned by this service. Ordinary /// request-only services keep the inert default. fn next_deadline(&self) -> Option { diff --git a/native/crates/rt/src/session/builder.rs b/native/crates/rt/src/session/builder.rs index d0da422..f4da552 100644 --- a/native/crates/rt/src/session/builder.rs +++ b/native/crates/rt/src/session/builder.rs @@ -1038,6 +1038,27 @@ impl SessionBuilder { self.map_view_request(view, window, 0, result) } + /// Map a held view into the `MAP` window as a **sink** mapping + /// ([`VIEW_MAP_SINK`](kore_abi::session::VIEW_MAP_SINK)): should the + /// lessor's lease die under it, its pages are repointed at this space's + /// sink page rather than unmapped. + pub fn map_view_sink( + &mut self, + view: impl Into, + result: SlotRef, + ) -> Result { + let mut req = [0u8; kore_abi::session::ViewMapReq::WIRE]; + req[..2].copy_from_slice(&kore_abi::session::ViewMapWindow::Map.role().to_le_bytes()); + req[3] = kore_abi::session::VIEW_MAP_SINK; + self.derive_raw( + view.into(), + Rights::empty(), + kore_abi::session::DERIVE_KIND_VIEW_MAP, + result, + Some(&req), + ) + } + /// Map one surface into a fixed reusable lane. The lane controls VA /// placement only; authority and geometry still come from the held view. pub fn map_view_surface_lane( @@ -1539,7 +1560,8 @@ impl SessionBuilder { /// the kernel moves into the new vCPU task only after the complete session /// commits. The caller retains the guest-space owner and receives the /// ordinary lifecycle `Task` handle in `handle`. Like process `Found`, - /// this is terminal: no later invocation may be staged in the session. + /// this begins a terminal Found tail: only guest Founds for this same + /// owner may follow, up to `MAX_SESSION_FOUNDS` members. #[allow(clippy::too_many_arguments)] pub fn found_guest( &mut self, @@ -2445,6 +2467,26 @@ impl SessionBuilder { &mut self, view: impl Into, parent: impl Into, + ) -> Result<(), BuildError> { + self.unshare_iova_kind(view, parent, kore_abi::session::REVOKE_KIND_UNSHARE_IOVA) + } + + /// `Revoke × REVOKE_KIND_UNSHARE_IOVA_LIVE` — [`unshare_iova`](Self::unshare_iova) + /// for a requester that keeps running: its DMA latch stays open, and + /// the window's completed invalidation is what cuts the device off. + pub fn unshare_iova_live( + &mut self, + view: impl Into, + parent: impl Into, + ) -> Result<(), BuildError> { + self.unshare_iova_kind(view, parent, kore_abi::session::REVOKE_KIND_UNSHARE_IOVA_LIVE) + } + + fn unshare_iova_kind( + &mut self, + view: impl Into, + parent: impl Into, + kind: u8, ) -> Result<(), BuildError> { let mut req = [0u8; 8]; req[0..4].copy_from_slice(&parent.into().index().to_le_bytes()); @@ -2452,7 +2494,7 @@ impl SessionBuilder { self.push(Inv { op: OpCode::Revoke as u16, requested: 0, - kind: kore_abi::session::REVOKE_KIND_UNSHARE_IOVA, + kind, subject: view.into().index(), result: 0, arg, diff --git a/native/crates/rt/src/session/tests.rs b/native/crates/rt/src/session/tests.rs index da69029..409c73a 100644 --- a/native/crates/rt/src/session/tests.rs +++ b/native/crates/rt/src/session/tests.rs @@ -23,7 +23,11 @@ fn view_placement_requests_only_a_window_not_more_authority() { use kore_abi::session::{DERIVE_KIND_VIEW_MAP, ViewMapReq, ViewMapWindow}; use zerocopy::TryFromBytes; - for window in [ViewMapWindow::Map, ViewMapWindow::Surfaces, ViewMapWindow::Heap] { + for window in [ + ViewMapWindow::Map, + ViewMapWindow::Surfaces, + ViewMapWindow::Heap, + ] { let mut builder = SessionBuilder::new(); builder .map_view_in(SlotRef::from_wire(2), window, SlotRef::from_wire(3)) @@ -42,9 +46,27 @@ fn view_placement_requests_only_a_window_not_more_authority() { assert!(rest.is_empty()); assert_eq!(req.placement(), Some(window)); assert_eq!(req.lane, 0); - assert_eq!(req.reserved, [0; 5]); + assert_eq!(req.flags, 0); + assert_eq!(req.reserved, [0; 4]); + assert!(!req.sink()); } + // A sink request names the MAP window and the one flag, nothing more. + let mut builder = SessionBuilder::new(); + builder + .map_view_sink(SlotRef::from_wire(2), SlotRef::from_wire(3)) + .unwrap(); + let mut out = [0u8; DESCRIPTOR_MAX]; + let len = builder.finish(&mut out).unwrap(); + let (_, invs, args) = parse(&out[..len]); + assert_eq!(invs[0].kind, DERIVE_KIND_VIEW_MAP); + assert_eq!(invs[0].requested, 0); + let (req, _) = ViewMapReq::try_ref_from_prefix(args).unwrap(); + assert_eq!(req.placement(), Some(ViewMapWindow::Map)); + assert!(req.sink()); + assert_eq!(req.flags, kore_abi::session::VIEW_MAP_SINK); + assert_eq!(req.reserved, [0; 4]); + let mut builder = SessionBuilder::new(); builder .map_view_surface_lane( @@ -2285,3 +2307,36 @@ fn task_identity_query_is_standalone_and_has_no_arguments() { assert_eq!(invs[0].arg.get(), u32::MAX); assert!(args.is_empty()); } + +#[test] +fn guest_cohort_encodes_one_owner_and_private_windows_in_one_found_tail() { + use kore_abi::session::{DERIVE_KIND_GUEST_FOUND, GuestFoundReq}; + use zerocopy::TryFromBytes; + let mut builder = SessionBuilder::new(); + for cpu in 0..2u32 { + let input = 10 + cpu * 3; + builder + .found_guest( + SlotRef::from_wire(2), + 0x10000, + 0x1f0000 - u64::from(cpu) * 0x8000, + 0x1000, + SlotRef::from_wire(input), + SlotRef::from_wire(input + 1), + SlotRef::from_wire(input + 2), + SlotRef::from_wire(20 + cpu), + ) + .unwrap(); + } + let mut out = [0u8; DESCRIPTOR_MAX]; + let len = builder.finish(&mut out).unwrap(); + let (_, invs, args) = parse(&out[..len]); + assert_eq!(invs.len(), 2); + for (cpu, inv) in invs.iter().enumerate() { + assert_eq!(inv.kind, DERIVE_KIND_GUEST_FOUND); + assert_eq!(inv.subject.get(), 2); + let (req, _) = GuestFoundReq::try_ref_from_prefix(&args[inv.arg.get() as usize..]).unwrap(); + assert_eq!(req.window.get(), 10 + cpu as u32 * 3); + assert_eq!(req.reserved.get(), 0); + } +} diff --git a/native/crates/rt/src/spmd.rs b/native/crates/rt/src/spmd.rs index a765834..4efe076 100644 --- a/native/crates/rt/src/spmd.rs +++ b/native/crates/rt/src/spmd.rs @@ -91,8 +91,13 @@ pub struct Machine { /// The **best lane**: how many cores of this task's founding node's /// best performance class an SPMD region is granted exclusively — /// the G6c reservation population (P-cores on a hybrid box, all - /// cores on a homogeneous one). The width [`dispatch`] fits to. + /// cores on a homogeneous one). This remains a factual fallback. pub best_lane: u32, + /// A calibrated, node-relative same-class cohort width. `None` means + /// one or more CPUs lacked capacity samples at founding; the fit then + /// uses `best_lane`. This is a throughput starting point, not a measured + /// optimum for a particular workload or dispatch size. + pub calibrated_fit: Option, /// Performance-class cores machine-wide (Intel P / AMD Zen / Arm big). pub performance_cores: u8, /// Efficiency-class cores machine-wide (Intel E / AMD Zen-c / Arm LITTLE). @@ -108,11 +113,29 @@ pub struct Machine { } impl Machine { - /// The width a region should found to run one worker per exclusive - /// best-class core: the best lane, clamped to at least one and to - /// the runtime's ceiling. The fit policy [`dispatch`] defaults to. + /// Default region width from the best reachable calibrated cohort, or + /// the best-class population before calibration. Clamped to the runtime + /// worker ceiling; a caller may impose a smaller resource budget. pub fn fitting_width(&self) -> u32 { - self.best_lane.clamp(1, crate::thread::MAX_THREADS - 1) + self.calibrated_fit + .unwrap_or(self.best_lane) + .clamp(1, crate::thread::MAX_THREADS - 1) + } + + /// For a one-shot grid, use the wider cohort only when the grid can + /// occupy every worker scored at founding. A partial eight-E cohort + /// was never compared against four P cores; for five to seven tiles, + /// keep the measured four-P fallback instead of extrapolating. + pub fn dispatch_fit(&self, tiles: u32) -> u32 { + let width = if self + .calibrated_fit + .is_some_and(|fit| fit > self.best_lane && fit > tiles) + { + self.best_lane + } else { + self.fitting_width() + }; + width.min(tiles).clamp(1, crate::thread::MAX_THREADS - 1) } } @@ -129,6 +152,7 @@ pub fn machine() -> Option { Some(Machine { logical_cpus: block.logical_cpus()?, best_lane: block.best_lane()?, + calibrated_fit: block.spmd_fit_width(), performance_cores, efficient_cores: efficient_cores.saturating_sub(block.low_power_cores()?), low_power_cores: block.low_power_cores()?, @@ -669,37 +693,38 @@ fn launch( // all-or-nothing, degrading to a starless launch rather than // refusing one. Sized to the LAUNCHED width, not the capacity. let star_plan = crate::star::provision(active); - // The gang founding (G5): whole chunks per session — one trap each - // — until the region is founded. A width the descriptor cannot - // carry chunks rather than refuses. A starred chunk is capped at - // [`MESH_CHUNK`](crate::star::MESH_CHUNK): every member's endow - // carries the intra-chunk kick mesh (built inside the spawn, where - // the bands are known), so a barrier leader wakes a parked sibling - // in one hop; starless chunks keep the descriptor's full width. + // The gang founding (G5): one chunk per session — one trap each — + // until the region is founded. A width the descriptor cannot carry + // in one session chunks rather than refuses. A starred session + // carries at most [`MESH_CEILING`](crate::star::MESH_CEILING) + // workers: every member's endow carries the intra-chunk kick mesh + // (built inside the spawn, where the bands are known), quadratic in + // the chunk, so a barrier leader wakes a parked sibling in one hop; + // starless sessions keep the descriptor's full founding width. The + // gang takes as few sessions as that allows, split evenly by this + // width rather than filled to the ceiling: eight meshed workers found + // as 4 + 4, not 7 + 1 with a member meshed to nobody. let mut workers: [Option; N] = [const { None }; N]; - let chunk = if star_plan.is_some() { - crate::star::MESH_CHUNK + let ceiling = if star_plan.is_some() { + crate::star::MESH_CEILING } else { crate::session::MAX_SESSION_FOUNDS }; - let mut at = 0usize; let mut ok = true; - while at < active { - let end = (at + chunk).min(active); + for (at, end) in founding_sessions(active as u32, ceiling as u32) { let founded = crate::thread::spawn_gang_workers( entry, ctx, - at as u32, + at, true, &[], star_plan.as_ref(), - &mut workers[at..end], + &mut workers[at as usize..end as usize], ); if !founded { ok = false; break; } - at = end; } // The hub last — its sends are named by the workers' BAND indices, // known only now. A refused hub is the ruled degrade: the workers @@ -827,9 +852,11 @@ pub fn resident_run(ctx: &'static SpmdCtx, worker: loop { // Park until a new job is published or the region is retired. The // wait is the barrier's two-tier slice: a co-resident submit ends - // the monitored spin the instant it stores `job`; a descheduled - // worker rides `PARK_SLICE` and re-reads. `kick_all` only trims - // the tail — the word is the truth. + // the monitored spin the instant it stores `job`; a parked worker + // is kicked by the submit over the founder's send. A long + // monitored spin here is NOT an option: a spinning worker holds its + // core against the kernel until the tick, and the launcher may be + // sharing it (`res/SPMD.md` §14.2). The word is the truth. crate::barrier::wait_while(ctx.job_word(), ran); // A retire flips the gate off RUN, THEN steps `job` to wake us; the // SeqCst total order makes the DONE visible once the step that woke @@ -853,11 +880,15 @@ pub fn resident_run(ctx: &'static SpmdCtx, worker: // its participation (or reading `env`) when the next job overwrites // `active`/`env` — the generation race the per-job words would // otherwise carry. The one that zeroes `done` publishes the - // completion generation, then posts the star's existing relay send. - // A hubless resident gives its receive end to the submitter. The - // shared word remains authoritative if the notice coalesces/refuses. + // completion generation, then kicks the launcher: the founder is + // peer 0 of the kick mesh, so a launcher parked on `complete` is + // reached in one hop like any registered waiter (`res/SPMD.md` + // §14.2). The star's relay send is posted as well for an async + // observer on the hubless reverse channel. The shared word remains + // authoritative if a notice coalesces or refuses. if ctx.done_word().fetch_sub(1, Ordering::AcqRel) == 1 { ctx.complete_word().store(g, Ordering::SeqCst); + let _ = crate::futex::wake(ctx.complete_word()); let _ = crate::futex::notify_resident_completion(); } } @@ -1111,9 +1142,11 @@ impl ResidentRegion { self.tickets.poll(ticket, complete) } - /// Block until `ticket`'s job completes. Efficient for a co-resident gang - /// through the star's reverse completion channel). A missing/refused - /// channel falls back to the bounded barrier wait and shared-word recheck. + /// Block until `ticket`'s job completes: a short monitored spin on the + /// completion word, then the barrier's park on this thread's doorbell, + /// which the last worker kicks (the founder is peer 0 of the kick mesh, + /// `res/SPMD.md` §14.2). A launcher no worker holds a send for — a + /// region founded off thread 0 — rides the park's slice and re-reads. /// /// Validation runs before any wait, so an invalid ticket can never park on /// a physical generation that this region will not publish. @@ -1121,11 +1154,7 @@ impl ResidentRegion { match self.tickets.position(ticket)? { ResidentTicketPosition::Complete => Ok(()), ResidentTicketPosition::Current(generation) => { - wait_for_resident_generation( - self.region.ctx(), - generation, - self.region.star.as_ref(), - ); + wait_for_resident_generation(self.region.ctx(), generation); Ok(()) } } @@ -1208,11 +1237,7 @@ impl ResidentRegion { // Waiting for the last submitted job first makes the region idle, so // the teardown abandons nothing. (Idle already ⇒ this returns at once.) if self.tickets.last.is_some() { - wait_for_resident_generation( - self.region.ctx(), - self.tickets.generation, - self.region.star.as_ref(), - ); + wait_for_resident_generation(self.region.ctx(), self.tickets.generation); } let ctx = self.region.ctx(); // Tell the gang to exit: flip the gate off RUN, THEN step `job` to @@ -1232,41 +1257,22 @@ impl ResidentRegion { /// Wait for the one compact physical generation already proved current by the /// opaque ticket layer. Keeping this separate lets `retire` drain its own last /// accepted job without manufacturing or re-validating a public token. -fn wait_for_resident_generation( - ctx: &'static SpmdCtx, - generation: u32, - star: Option<&crate::star::Star>, -) { +fn wait_for_resident_generation(ctx: &'static SpmdCtx, generation: u32) { let complete = ctx.complete_word(); let current = complete.load(Ordering::Acquire); if current == generation { return; } + // A job that ends inside the handoff budget costs no park at all. crate::futex::spin_completion(complete, current); - let mut receiver = star.and_then(|star| star.completion_receiver()); + // Then the barrier's park on this thread's own doorbell, registered on + // the completion word so the last worker's `futex::wake` reaches it in + // one hop; the slice bounds a launcher no worker can kick. loop { let current = complete.load(Ordering::Acquire); if current == generation { return; } - if let Some(rx) = receiver.as_mut() { - if rx.try_recv().is_ok() { - if complete.load(Ordering::Acquire) == generation { - return; - } - if let Some(post) = rx.standing_post() { - if crate::futex::wait_standing( - post, - core::time::Duration::from_nanos(RESIDENT_WAKE_BACKSTOP_NS), - ) != crate::futex::WaitOutcome::Unsupported - { - continue; - } - } else { - continue; - } - } - } crate::barrier::wait_while(complete, current); } } @@ -1312,16 +1318,222 @@ pub fn found_resident( #[cfg(target_os = "none")] const DISPATCH_CAP: usize = (crate::thread::MAX_THREADS - 1) as usize; -/// The one re-armable region every [`dispatch`] launches over. A single -/// `static` serves every call: each blocks to join before returning and -/// re-stages the ctx for the next, so a parallel-for in a loop reuses -/// it. A second CONCURRENT dispatch finds the gate un-staged and refuses -/// (`None`) rather than corrupting the region — v1 dispatch is one -/// region at a time per process, the same all-or-nothing gate every -/// launch respects. +/// The one re-armable region a [`dispatch`] launches over when the +/// resident gang below is unavailable. A single `static` serves every +/// call: each blocks to join before returning and re-stages the ctx for +/// the next, so a parallel-for in a loop reuses it. A second CONCURRENT +/// dispatch finds the gate un-staged and refuses (`None`) rather than +/// corrupting the region — v1 dispatch is one region at a time per +/// process, the same all-or-nothing gate every launch respects. #[cfg(target_os = "none")] static DISPATCH_CTX: SpmdCtx = SpmdCtx::new(DISPATCH_CAP as u32); +/// The process-wide **resident dispatch gang** — founded on the first +/// [`dispatch`] and kept for the task's life, so every later dispatch is +/// one [`submit`](ResidentRegion::submit) and one completion wait rather +/// than a founding and a reap. +/// +/// The one-shot [`launch`] path founds fresh worker threads per call and +/// joins them through the thread-wait path, whose refusals back off in +/// 2 ms kernel sleeps. On the Lunar Lake box that priced a dispatch of +/// trivial work at 7 ms for two workers and 10–12 ms for four (`spmd +/// dispatch`), and a model decoder that issues one dispatch per parallel +/// kernel node paid it about 5,000 times per token pass: every parallel +/// node cost 11 ms whatever its size. The resident gang parks on the +/// job word between jobs and is kicked over the star's parent sends on +/// each submit; the same box's barrier bench shows that path at a few +/// hundred microseconds a phase. +/// +/// One job at a time (`busy`): a nested or concurrent dispatch is refused +/// (`None`) exactly as the one-shot gate refuses it, and the caller runs +/// its tiles serially. A founding the machine refuses (a one-lane box, +/// or thread budget spent) is remembered, and every later dispatch takes +/// the one-shot path instead. +#[cfg(target_os = "none")] +static RESIDENT_DISPATCH_CTX: SpmdCtx = SpmdCtx::new(DISPATCH_CAP as u32); + +#[cfg(target_os = "none")] +struct ResidentDispatch { + /// Exclusive access to `region` for the duration of one dispatch. + busy: core::sync::atomic::AtomicBool, + /// [`RESIDENT_UNTRIED`], [`RESIDENT_FOUNDED`] or [`RESIDENT_REFUSED`]. + state: core::sync::atomic::AtomicU8, + /// The founded gang width, stamped once at founding so the books can + /// be read without the dispatcher's claim. + width: core::sync::atomic::AtomicU32, + region: core::cell::UnsafeCell>>, +} + +// SAFETY: `region` is touched only between a successful `busy` claim and +// its release, by the one dispatcher that made the claim. +#[cfg(target_os = "none")] +unsafe impl Sync for ResidentDispatch {} + +#[cfg(target_os = "none")] +const RESIDENT_UNTRIED: u8 = 0; +#[cfg(target_os = "none")] +const RESIDENT_FOUNDED: u8 = 1; +#[cfg(target_os = "none")] +const RESIDENT_REFUSED: u8 = 2; + +#[cfg(target_os = "none")] +static RESIDENT_DISPATCH: ResidentDispatch = ResidentDispatch { + busy: core::sync::atomic::AtomicBool::new(false), + state: core::sync::atomic::AtomicU8::new(RESIDENT_UNTRIED), + width: core::sync::atomic::AtomicU32::new(0), + region: core::cell::UnsafeCell::new(None), +}; + +/// **The resident dispatch gang's books**, read from any thread without +/// the dispatcher's claim — the words a diagnostic reads to name a +/// dispatch that never completed. `job` is the generation last +/// published, `complete` the one last retired by the gang, and `done` +/// how many of the `width` founded workers have yet to account for the +/// current job: a `done` that stays above zero while `job` leads +/// `complete` names a worker that never returned. All zero before the +/// gang is founded, and off metal. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ResidentDispatchBooks { + /// A gang was founded and kept. + pub founded: bool, + /// Workers founded. + pub width: u32, + /// A dispatch currently holds the gang. + pub busy: bool, + /// The generation last published by a submit. + pub job: u32, + /// The generation last completed by the whole gang. + pub complete: u32, + /// Workers yet to account for the current job. + pub done: u32, +} + +/// Read the resident dispatch gang's [`ResidentDispatchBooks`]. +#[cfg(target_os = "none")] +pub fn resident_dispatch_books() -> ResidentDispatchBooks { + use core::sync::atomic::Ordering::{Acquire, Relaxed}; + let gang = &RESIDENT_DISPATCH; + if gang.state.load(Relaxed) != RESIDENT_FOUNDED { + return ResidentDispatchBooks::default(); + } + let ctx = &RESIDENT_DISPATCH_CTX; + ResidentDispatchBooks { + founded: true, + width: gang.width.load(Relaxed), + busy: gang.busy.load(Acquire), + job: ctx.job_word().load(Acquire), + complete: ctx.complete_word().load(Acquire), + done: ctx.done_word().load(Acquire), + } +} + +/// Off metal there is no gang: the empty books. +#[cfg(not(target_os = "none"))] +pub fn resident_dispatch_books() -> ResidentDispatchBooks { + ResidentDispatchBooks::default() +} + +/// The resident gang's worker entry: the resident loop around the same +/// type-erased body [`dispatch_worker`] runs for a one-shot launch. +#[cfg(target_os = "none")] +fn resident_dispatch_entry(ctx: &'static SpmdCtx, worker: u32) { + resident_run(ctx, worker, dispatch_worker); +} + +/// How the resident gang answered a dispatch. +#[cfg(target_os = "none")] +enum ResidentOutcome { + /// Ran and completed. + Done(Dispatched), + /// Another dispatch holds the gang (nested or concurrent): refuse. + Busy, + /// No resident gang on this machine, or it could not take the job: + /// the one-shot launch path decides. + Unavailable, +} + +/// Run one dispatch on the resident gang, founding it on first use. +#[cfg(target_os = "none")] +fn dispatch_resident( + tiles: u32, + width: u32, + body: &(dyn Fn(&'static SpmdCtx, u32) + Sync), +) -> ResidentOutcome { + use core::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release}; + let gang = &RESIDENT_DISPATCH; + if gang + .busy + .compare_exchange(false, true, AcqRel, Acquire) + .is_err() + { + return ResidentOutcome::Busy; + } + let outcome = (|| { + // SAFETY: `busy` is held; see the `Sync` impl. + let slot = unsafe { &mut *gang.region.get() }; + if slot.is_none() { + if gang.state.load(Relaxed) == RESIDENT_REFUSED { + return ResidentOutcome::Unavailable; + } + let fit = machine() + .map(|m| m.fitting_width()) + .unwrap_or(1) + .clamp(1, DISPATCH_CAP as u32); + let founded = (fit >= 2) + .then(|| { + found_resident::( + &RESIDENT_DISPATCH_CTX, + resident_dispatch_entry, + fit as usize, + ) + }) + .flatten(); + match founded { + // A gang without its doorbell star has nothing to park on + // (`res/SPMD.md` §3): its workers would spin between jobs + // on the machine's best cores and starve the founding task. + // Retire it and keep the one-shot path, as the region's + // contract says; an executable takes `PROFILE_CHANNELS` + // to found the star. + Some(region) if !region.star_active() => { + let _ = region.retire(); + gang.state.store(RESIDENT_REFUSED, Relaxed); + return ResidentOutcome::Unavailable; + } + Some(region) => { + gang.width.store(region.width() as u32, Relaxed); + *slot = Some(region); + gang.state.store(RESIDENT_FOUNDED, Relaxed); + } + None => { + gang.state.store(RESIDENT_REFUSED, Relaxed); + return ResidentOutcome::Unavailable; + } + } + } + let Some(region) = slot.as_mut() else { + return ResidentOutcome::Unavailable; + }; + let active = width.clamp(1, region.width() as u32); + let env = &body as *const &(dyn Fn(&'static SpmdCtx, u32) + Sync) as *mut (); + // SAFETY: `body` outlives this call, and the job's completion is + // waited for below before `env` goes out of scope. + let Some(ticket) = (unsafe { region.submit(active, tiles, env) }) else { + return ResidentOutcome::Unavailable; + }; + // A fresh ticket from this region cannot be refused; were the + // region's own books ever inconsistent, the job is published all + // the same, so the wait stands regardless. + let _ = region.wait(ticket); + ResidentOutcome::Done(Dispatched { + width: active, + tiles, + }) + })(); + gang.busy.store(false, Release); + outcome +} + /// What a [`dispatch`] launch chose — the width it fit to and the tile /// span it covered, so a terminal tool can show the auto-decision the caller /// never had to make. @@ -1360,10 +1572,10 @@ fn dispatch_worker(ctx: &'static SpmdCtx, worker: u32) { /// **The width model** — every fact a dispatch width is chosen from, and /// [`choose`](Self::choose) is the only place the choice is made. /// -/// The fit policy (ruling 2026-08-20) reads one fact: the best lane. That -/// is the right default precisely when nothing else is known, and it is -/// what this model returns when nothing else is supplied. What it cannot -/// see is that a gang costs more to *start* the wider it is, so a small +/// The fit input is now the highest-service reachable homogeneous cohort +/// when founding-time calibration is complete, else the best lane. This +/// model returns that input until per-call-site work cost is known. What +/// either static fit cannot see is that a gang costs more to *start*, so a small /// grid can finish sooner on fewer workers — measured on a Lunar Lake box /// 2026-09-03 with `spmd lane ` (median of the per-round minimum, /// 64 rounds, identical body at every width): @@ -1391,13 +1603,14 @@ fn dispatch_worker(ctx: &'static SpmdCtx, worker: u32) { /// (workload-independent, calibrated once), `per_tile_ns` is a workload /// property (per call site). Absent either, `choose` returns the fit /// width — today's answer, exactly — so this model is inert until both are -/// supplied. `res/SPMD.md` §12 carries the open question of where -/// `per_tile_ns` comes from; nothing in the tree feeds it yet. +/// supplied. Crossing classes also changes tile cost; a future calibration +/// must measure that separately rather than treating this one scalar as +/// class-invariant. `res/SPMD.md` §12 carries the open question of where +/// per-call-site cost comes from; nothing in the tree feeds it yet. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct WidthModel<'a> { - /// The fit width: one worker per exclusive best-class core - /// ([`Machine::fitting_width`]), already clamped to the runtime - /// ceiling by the caller. + /// The caller's default cohort width ([`Machine::fitting_width`]), + /// already clamped to the runtime ceiling. pub fit: u32, /// Measured per-batch dispatch overhead in nanoseconds, indexed by /// `width - 1` — the wake-and-drain cost `spmd lane ` prints. @@ -1447,26 +1660,20 @@ impl WidthModel<'_> { } } -/// Pick the launch width for `tiles` under the **fit policy** (ruling -/// 2026-08-20): one worker per exclusive best-class core (`best_lane`), -/// so the region runs co-resident in the monitored tier — never more -/// workers than tiles, never past the runtime ceiling, at least one. A -/// machine with no facts stamped falls back to a single worker (correct, -/// if unparallel — the honest floor rather than a guessed width). +/// Pick a launch width from the calibrated reachable cohort when available, +/// else the factual best-class population. Never more workers than tiles or +/// the runtime ceiling. A machine with no facts falls back to one worker. /// /// (The adaptive fill-past-a-threshold policy the design weighed is a /// future knob on exactly this one function — `res/SPMD.md` §12.) #[cfg(target_os = "none")] fn dispatch_width(tiles: u32) -> u32 { let lane = machine() - .map(|m| m.best_lane) + .map(|m| m.dispatch_fit(tiles)) .filter(|&l| l > 0) .unwrap_or(1); - // No calibration and no per-tile cost: the model returns the fit - // width, which is what this function has always returned. The - // equivalence is host-proven (`the_width_model_reproduces_the_fit_policy`), - // so routing through the model changes no behavior — it only puts the - // policy somewhere a ruling can reach. + // No per-call-site cost or dispatch-overhead table: WidthModel preserves + // this founding-time cohort fit. Its host test proves that fallback. WidthModel { fit: lane.clamp(1, DISPATCH_CAP as u32), overhead_ns: &[], @@ -1474,6 +1681,15 @@ fn dispatch_width(tiles: u32) -> u32 { .choose(tiles, None) } +/// The founding sessions of a gang of `width` workers when one session +/// carries at most `ceiling`: as few as fit, each its even share of the +/// gang ([`block_split`]), as `[start, end)` worker ranges in order. +#[cfg_attr(all(not(target_os = "none"), not(test)), allow(dead_code))] +fn founding_sessions(width: u32, ceiling: u32) -> impl Iterator { + let sessions = width.div_ceil(ceiling.max(1)); + (0..sessions).map(move |session| block_split(width, sessions, session)) +} + /// Even block split of `span` items across `workers`: worker `w` gets /// the contiguous `[start, end)` slice, remainder spread over the first /// workers. Contiguous (not strided) — a range body reads its slice in @@ -1574,6 +1790,11 @@ fn dispatch_erased( width: u32, body: &(dyn Fn(&'static SpmdCtx, u32) + Sync), ) -> Option { + match dispatch_resident(tiles, width, body) { + ResidentOutcome::Done(dispatched) => return Some(dispatched), + ResidentOutcome::Busy => return None, + ResidentOutcome::Unavailable => {} + } let env = &body as *const &(dyn Fn(&'static SpmdCtx, u32) + Sync) as *mut (); let region = launch::( &DISPATCH_CTX, @@ -1888,6 +2109,28 @@ mod tests { } } + #[test] + fn machine_fit_uses_calibrated_cohort_with_best_lane_fallback() { + let mut machine = Machine { + logical_cpus: 16, + best_lane: 4, + calibrated_fit: None, + performance_cores: 4, + efficient_cores: 8, + low_power_cores: 4, + standard_cores: 0, + hardware_feedback: true, + }; + assert_eq!(machine.fitting_width(), 4); + machine.calibrated_fit = Some(8); + assert_eq!(machine.fitting_width(), 8); + assert_eq!(machine.dispatch_fit(3), 3); + assert_eq!(machine.dispatch_fit(5), 4); + assert_eq!(machine.dispatch_fit(8), 8); + machine.calibrated_fit = Some(u32::MAX); + assert_eq!(machine.fitting_width(), crate::thread::MAX_THREADS - 1); + } + /// A partially calibrated table is not a licence to interpolate: if any /// width in range has no measured overhead, the model declines to /// compare and hands back the fit width. @@ -1978,6 +2221,39 @@ mod tests { } } + /// Every gang the runtime can found takes the fewest sessions its + /// width needs, none past the ceiling, sharing the gang evenly: a + /// meshed gang one worker over the ceiling founds as two near-halves, + /// never a full session plus a lone worker. + #[test] + fn a_gang_founds_in_the_fewest_even_sessions() { + let meshed = crate::star::MESH_CEILING as u32; + for ceiling in [meshed, crate::session::MAX_SESSION_FOUNDS as u32] { + for width in 1..crate::thread::MAX_THREADS { + let sessions: std::vec::Vec<(u32, u32)> = founding_sessions(width, ceiling).collect(); + assert_eq!(sessions.len() as u32, width.div_ceil(ceiling), "width {width}"); + let mut next = 0; + for &(start, end) in &sessions { + assert_eq!(start, next, "width {width}: sessions are contiguous"); + assert!((1..=ceiling).contains(&(end - start)), "width {width}"); + next = end; + } + assert_eq!(next, width, "width {width}: every worker founds"); + let sizes = sessions.iter().map(|(start, end)| end - start); + let (min, max) = (sizes.clone().min().unwrap(), sizes.max().unwrap()); + assert!(max - min <= 1, "width {width}: {min}..={max}"); + } + } + let split = |width| -> std::vec::Vec { + founding_sessions(width, meshed) + .map(|(start, end)| end - start) + .collect() + }; + assert_eq!(split(4), [4], "a Lunar Lake gang founds whole"); + assert_eq!(split(meshed + 1), [(meshed + 1) / 2, (meshed + 2) / 2]); + assert_eq!(split(12), [6, 6]); + } + /// A scripted worker: joins yield the scripted outcomes in order. struct MockWorker { outcomes: &'static [Result, crate::cap::TaskWaitError>], diff --git a/native/crates/rt/src/star.rs b/native/crates/rt/src/star.rs index 960502f..56c060a 100644 --- a/native/crates/rt/src/star.rs +++ b/native/crates/rt/src/star.rs @@ -95,6 +95,10 @@ struct Wires { completion: Option< core::cell::RefCell>, >, + /// The founder's own doorbell send — its self-kick — when the founder + /// is thread 0, endowed to every worker as peer 0 of the kick mesh so a + /// resident job's last worker can kick the launcher. + founder_kick: Option, workers: usize, } @@ -155,6 +159,17 @@ pub(crate) fn provision(workers: usize) -> Option { minted(&mut roots, &mut parent_sends); return None; }; + // The founder joins the mesh as peer 0 through its self-kick send + // (`DOORBELL@PEER_BASE`). Only thread 0 holds a send into its OWN + // doorbell — a spawned thread's `DOORBELL@PEER_BASE` names thread 0's — + // so a region founded elsewhere carries no founder kick and its + // launcher rides the park's slice (`res/SPMD.md` §14.2). + let founder_kick = if crate::entry::geometry::shard_index() == Some(0) { + rt.bootstrap() + .handle(kore_abi::task::role::DOORBELL, crate::futex::PEER_BASE) + } else { + None + }; Some(StarPlan { wires: Wires { @@ -165,22 +180,41 @@ pub(crate) fn provision(workers: usize) -> Option { hub_send: hub_pair.send, hub_recv: Some(hub_pair.recv), completion: None, + founder_kick, workers, }, }) } -/// The widest gang chunk a FULLY MESHED session can carry. +/// Descriptor invocations one meshed worker costs its gang session in a +/// chunk of `chunk`: two views, the boot delegate and the Found, plus its +/// endowment of `chunk + 2` caps (own recv, hub send, the founder's send, +/// and a peer send per chunk sibling). +const fn meshed_worker_invocations(chunk: usize) -> usize { + 4 + chunk + 2 +} + +/// The widest chunk one FULLY MESHED gang session can carry — derived +/// from the descriptor's bounds, never chosen: a chunk of `n` costs +/// `n·(n + 6)` invocations against [`MAX_INVOCATIONS`], and no session +/// founds more than [`MAX_SESSION_FOUNDS`] (seven, at today's 107 and 12). +/// A wider region founds in several sessions (see [`spmd::launch`]): the +/// mesh spans a session's chunk, and a cross-chunk waiter is the hub's to +/// reach — the relay hop the mesh exists to keep OFF the common release +/// path. /// -/// Per worker the gang session stages `4 + endow` invocations (two -/// views, the boot delegate, the endow derives, the Found), and a -/// meshed worker's endow is `chunk + 1` caps (own recv, hub send, and -/// a peer send per sibling) — so a chunk of `n` costs `n·(n + 5)` -/// invocations against the descriptor's 107. `8·13 = 104` fits; -/// `9·14 = 126` does not. Wider regions chunk at this width: the mesh -/// spans a chunk, and a cross-chunk waiter is the hub's to reach — -/// the relay hop the mesh exists to keep OFF the common release path. -pub(crate) const MESH_CHUNK: usize = 8; +/// [`MAX_INVOCATIONS`]: kore_abi::session::MAX_INVOCATIONS +/// [`MAX_SESSION_FOUNDS`]: kore_abi::session::MAX_SESSION_FOUNDS +/// [`spmd::launch`]: crate::spmd +pub(crate) const MESH_CEILING: usize = { + let mut chunk = kore_abi::session::MAX_SESSION_FOUNDS; + while chunk > 1 + && chunk * meshed_worker_invocations(chunk) > kore_abi::session::MAX_INVOCATIONS + { + chunk -= 1; + } + chunk +}; #[cfg(target_os = "none")] impl StarPlan { @@ -213,7 +247,7 @@ impl StarPlan { out: &mut [crate::thread::WorkerCap], ) -> Option { let root = self.wires.roots.get(w)?.as_ref()?; - if out.len() < 1 + bands.len() { + if out.len() < 2 + bands.len() { return None; } out[0] = crate::thread::WorkerCap { @@ -231,6 +265,19 @@ impl StarPlan { rights: crate::cap::CHANNEL_ENDPOINT_RIGHTS, }; let mut n = 2; + // The founder as peer 0 (`DOORBELL@PEER_BASE`): a send derived from + // its self-kick, with the notice rights that send carries — the + // same derivation `spawn_with_doorbell` makes for `kick_parent`. + if let Some(kick) = self.wires.founder_kick { + out[n] = crate::thread::WorkerCap { + role: kore_abi::task::role::DOORBELL, + index: crate::futex::PEER_BASE, + subject: kick, + kind: kore_abi::cap::ResourceKind::ChanSend as u16 as u8, + rights: crate::cap::verbs::NOTICE_RIGHTS, + }; + n += 1; + } for &(j, band) in bands { if j == w { continue; @@ -325,12 +372,45 @@ impl Star { /// sends — the gate release, which must never depend on the hub it /// just launched. pub(crate) fn kick_all(&self) { - for s in self.wires.parent_sends.iter().flatten() { - let _ = crate::cap::verbs::post_notice( - s.slot(), - crate::futex::COOKIE_FUTEX, - u64::from(crate::futex::RELAY), - ); + // One session for the whole gang: n notices, one crossing. The + // terminal bench priced the one-post-per-worker form at ~40 µs a + // worker, the workers waking one after another in that rhythm. A + // session is atomic, so a send whose post is not standing (that + // worker is already awake and its post consumed) refuses the whole + // commit; then the posts go one by one, as before, so every parked + // worker still gets its kick. The word is the truth either way. + use crate::session::SessionBuilder; + use crate::transport::{Transport, TrapTransport}; + let sends = self.wires.parent_sends.iter().flatten(); + let mut batch = SessionBuilder::new(); + let mut staged = 0usize; + let mut live = 0usize; + for s in sends.clone() { + live += 1; + if batch + .notice( + s.slot(), + crate::futex::COOKIE_FUTEX, + u64::from(crate::futex::RELAY), + ) + .is_ok() + { + staged += 1; + } + } + let batched = live != 0 + && staged == live + && TrapTransport::new() + .commit_now(&batch) + .is_ok_and(|c| c.status() == kore_abi::session::Status::Ok); + if !batched { + for s in sends { + let _ = crate::cap::verbs::post_notice( + s.slot(), + crate::futex::COOKIE_FUTEX, + u64::from(crate::futex::RELAY), + ); + } } // The hub post reaches a consumer only when a hub thread was spawned // (it holds the sole recv). A hubless resident retains that recv for @@ -389,6 +469,8 @@ impl Star { hub_send, hub_recv, completion, + // A borrowed name in the founder's own table: nothing to retire. + founder_kick: _, workers: _, } = wires; for s in parent_sends.iter_mut() { @@ -492,4 +574,15 @@ mod tests { ); assert_eq!(HUB, u32::MAX, "the hub id can never be a dense worker id"); } + + /// The ceiling is the descriptor's arithmetic: the widest meshed + /// chunk fits one session, and one worker more would not. + #[test] + fn the_mesh_ceiling_is_the_widest_session_the_descriptor_carries() { + use kore_abi::session::{MAX_INVOCATIONS, MAX_SESSION_FOUNDS}; + let cost = |chunk: usize| chunk * meshed_worker_invocations(chunk); + assert!((1..=MAX_SESSION_FOUNDS).contains(&MESH_CEILING)); + assert!(cost(MESH_CEILING) <= MAX_INVOCATIONS); + assert!(MESH_CEILING == MAX_SESSION_FOUNDS || cost(MESH_CEILING + 1) > MAX_INVOCATIONS); + } } diff --git a/native/crates/rt/src/svc/client.rs b/native/crates/rt/src/svc/client.rs index 3816f48..c230d29 100644 --- a/native/crates/rt/src/svc/client.rs +++ b/native/crates/rt/src/svc/client.rs @@ -113,6 +113,45 @@ pub struct ClientEndpoints { pub grants: [Option; MAX_GRANTS], } +/// How this client's reply waits ended, counted since construction: where +/// a slow call's time went. Counters only. A call whose blocking park +/// ([`SvcClient::with_reply_wait`]) already waited counts as `ready`, since +/// its reply is in the ring when it is collected. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ReplyWaitStats { + /// The reply was in the ring at the first look. + pub ready: u64, + /// It arrived after a wake other than the re-kick deadline, normally + /// the service's reply kick. + pub woken: u64, + /// It was first seen only after a re-kick deadline passed: no wake + /// brought it, so the call paid at least one [`REPLY_REKICK_NS`]. + pub ticked: u64, + /// Requests re-kicked on that cadence. + pub rekicks: u64, +} + +impl ReplyWaitStats { + fn record(&mut self, waited: bool, ticked: bool) { + let count = match (waited, ticked) { + (false, _) => &mut self.ready, + (true, false) => &mut self.woken, + (true, true) => &mut self.ticked, + }; + *count = count.saturating_add(1); + } + + /// The waits counted since `before`. + pub fn since(self, before: Self) -> Self { + Self { + ready: self.ready.saturating_sub(before.ready), + woken: self.woken.saturating_sub(before.woken), + ticked: self.ticked.saturating_sub(before.ticked), + rekicks: self.rekicks.saturating_sub(before.rekicks), + } + } +} + /// The re-kick interval inside a reply wait. /// /// Also the granularity of a bounded reply observation when the clock cannot @@ -899,7 +938,7 @@ fn classify_request_park_commit(committed: Committed, deadline_armed: bool) -> R let raw_status = committed.raw() as u16; let status = committed.status(); let failing = committed.failing_index(); - if raw_status > Status::Unfounded as u16 || failing != 0 { + if raw_status > Status::FabricRouteCapability as u16 || failing != 0 { return RequestParkCommit::Terminal(CallError::Refused(Status::Malformed)); } match status { @@ -915,7 +954,13 @@ fn classify_request_park_commit(committed: Committed, deadline_armed: bool) -> R | Status::Irq | Status::Exited | Status::AltDelivery - | Status::Unfounded) => RequestParkCommit::Terminal(CallError::Refused(status)), + | Status::Unfounded + | Status::FabricRouteConflict + | Status::FabricRouteNoEdge + | Status::FabricRouteWrongAccess + | Status::FabricRouteInfeasible + | Status::FabricRouteCapacity + | Status::FabricRouteCapability) => RequestParkCommit::Terminal(CallError::Refused(status)), } } @@ -947,7 +992,7 @@ fn classify_return_commit( } else { failing >= invocations }; - if invocations == 0 || raw_status > Status::Unfounded as u16 || impossible_index { + if invocations == 0 || raw_status > Status::FabricRouteCapability as u16 || impossible_index { // Unknown status vocabulary, an empty transfer, or an impossible // invocation index cannot prove that the affine move did not run. return ReturnCommit::AfterTransfer(CallError::Refused(Status::Malformed)); @@ -978,7 +1023,13 @@ fn classify_return_commit( | Status::Irq | Status::Exited | Status::AltDelivery - | Status::Unfounded) => ReturnCommit::AfterTransfer(CallError::Refused(status)), + | Status::Unfounded + | Status::FabricRouteConflict + | Status::FabricRouteNoEdge + | Status::FabricRouteWrongAccess + | Status::FabricRouteInfeasible + | Status::FabricRouteCapacity + | Status::FabricRouteCapability) => ReturnCommit::AfterTransfer(CallError::Refused(status)), } } @@ -1012,7 +1063,13 @@ fn classify_parked_return_commit(committed: Committed, invocations: usize) -> Re | Status::Irq | Status::Exited | Status::AltDelivery - | Status::Unfounded) => ReturnCommit::AfterTransfer(CallError::Refused(status)), + | Status::Unfounded + | Status::FabricRouteConflict + | Status::FabricRouteNoEdge + | Status::FabricRouteWrongAccess + | Status::FabricRouteInfeasible + | Status::FabricRouteCapacity + | Status::FabricRouteCapability) => ReturnCommit::AfterTransfer(CallError::Refused(status)), } } @@ -1102,6 +1159,8 @@ pub struct SvcClient<'w, 't, T: Transport, S: SvcProtocol> { /// Such a caller has no executor wake registry and must observe the /// level-triggered reply/grant rings directly. direct_polling: bool, + /// How the reply waits ended ([`Self::reply_waits`]). + reply_waits: ReplyWaitStats, _p: PhantomData, } @@ -1174,10 +1233,21 @@ impl<'w, 't, T: Transport, S: SvcProtocol> SvcClient<'w, 't, T, S> { transport, reply_wait: None, direct_polling: false, + reply_waits: ReplyWaitStats { + ready: 0, + woken: 0, + ticked: 0, + rekicks: 0, + }, _p: PhantomData, } } + /// How this client's reply waits have ended so far. + pub fn reply_waits(&self) -> ReplyWaitStats { + self.reply_waits + } + /// The transport already borrowed by this client. Dynamically minted /// sub-clients share it without borrowing the parent client itself. pub fn transport(&self) -> &'t T { @@ -1208,6 +1278,28 @@ impl<'w, 't, T: Transport, S: SvcProtocol> SvcClient<'w, 't, T, S> { self } + /// Switch direct level-triggered polling on or off for a client that + /// the executor ordinarily drives but that must, for a bounded region, + /// be driven synchronously — a model's expert-cache read callback, say, + /// which runs inside a task's poll and completes its filesystem calls + /// under [`block_on`](crate::block::block_on). The executor-clocked + /// reply wait would read a clock only the kore waker can answer; direct + /// polling re-reads the level-triggered ring instead. Only lawful + /// while the pair is idle and no reply wait is installed, and the caller + /// restores the previous mode before the executor polls this client + /// again. + pub fn set_direct_polling(&mut self, direct: bool) { + debug_assert!( + self.reply_wait.is_none(), + "direct polling and a blocking reply wait are distinct owners" + ); + debug_assert!( + self.is_idle(), + "the progress mode changes only between calls" + ); + self.direct_polling = direct; + } + /// Fuse ordinary request notification with a park on the owning /// executor's already-standing reply doorbell. /// @@ -1833,6 +1925,91 @@ impl<'w, 't, T: Transport, S: SvcProtocol> SvcClient<'w, 't, T, S> { third_tx: SlotRef, auxiliaries: [Option; MAX_INVOCATION_AUXILIARIES], auxiliary_tx: [SlotRef; MAX_INVOCATION_AUXILIARIES], + ) -> Result, AffineBundleCallError> { + self.call_with_invocation_bundle_budgeted( + req, + first, + first_rights, + executable, + second, + second_rights, + process_tx, + proof_tx, + third, + third_rights, + third_tx, + auxiliaries, + auxiliary_tx, + None, + ) + .await + } + + /// Observe an atomic invocation for one bounded slice. On timeout the + /// transferred bundle and exact request remain owned by this pair; the + /// caller must resume it, or quarantine the pair and its external backing. + pub async fn call_with_invocation_bundle_bounded< + A: ReturnCapability, + B: ReturnCapability, + C: ReturnCapability, + >( + &mut self, + req: &S::Request, + first: A, + first_rights: Rights, + executable: SlotRef, + second: B, + second_rights: Rights, + process_tx: SlotRef, + proof_tx: SlotRef, + third: C, + third_rights: Rights, + third_tx: SlotRef, + auxiliaries: [Option; MAX_INVOCATION_AUXILIARIES], + auxiliary_tx: [SlotRef; MAX_INVOCATION_AUXILIARIES], + budget_ns: u64, + ) -> Result, AffineBundleCallError> { + self.call_with_invocation_bundle_budgeted( + req, + first, + first_rights, + executable, + second, + second_rights, + process_tx, + proof_tx, + third, + third_rights, + third_tx, + auxiliaries, + auxiliary_tx, + Some(budget_ns), + ) + .await + } + + /// Shared invocation implementation for an optional audited observation + /// slice. A timed-out caller must retain the exact pair and backing. + pub async fn call_with_invocation_bundle_budgeted< + A: ReturnCapability, + B: ReturnCapability, + C: ReturnCapability, + >( + &mut self, + req: &S::Request, + first: A, + first_rights: Rights, + executable: SlotRef, + second: B, + second_rights: Rights, + process_tx: SlotRef, + proof_tx: SlotRef, + third: C, + third_rights: Rights, + third_tx: SlotRef, + auxiliaries: [Option; MAX_INVOCATION_AUXILIARIES], + auxiliary_tx: [SlotRef; MAX_INVOCATION_AUXILIARIES], + budget_ns: Option, ) -> Result, AffineBundleCallError> { macro_rules! not_moved { ($error:expr) => { @@ -1849,6 +2026,9 @@ impl<'w, 't, T: Transport, S: SvcProtocol> SvcClient<'w, 't, T, S> { if let Err(error) = self.require_idle() { return Err(not_moved!(error)); } + if budget_ns.is_some() && self.is_direct_progress() { + return Err(not_moved!(CallError::Wiring)); + } // An absent or locally retired rendezvous destination is a // deterministic local refusal. Check every endpoint that this bundle // will address before publishing its request record; otherwise the @@ -1954,6 +2134,11 @@ impl<'w, 't, T: Transport, S: SvcProtocol> SvcClient<'w, 't, T, S> { return Err(not_moved!(CallError::Refused(Status::Malformed))); } + // Begin before publishing, so the kick and every reply wait share + // the same observation deadline. The grant posts and transfer setup + // above cannot consume a published request. + let observation = ReplyObservation::begin(budget_ns).await; + // Exclusive producer ownership plus the capacity preflight makes // this infallible. Crucially, every deterministic rights, liveness, // duplicate-source, and descriptor-shape check completed before this @@ -1993,7 +2178,7 @@ impl<'w, 't, T: Transport, S: SvcProtocol> SvcClient<'w, 't, T, S> { self.kick_published_request() .await .map_err(AffineBundleCallError::AfterTransfer)?; - self.collect_reply(None) + self.collect_reply(observation) .await .map_err(AffineBundleCallError::AfterTransfer) } @@ -2250,6 +2435,7 @@ impl<'w, 't, T: Transport, S: SvcProtocol> SvcClient<'w, 't, T, S> { // Preserve that immediately-ready shape for executor-free // blocking adapters; only an empty ring may enter the // executor-clocked re-kick loop below. + self.reply_waits.record(false, false); wire } else { // An explicit bounded observation prefers an absolute @@ -2268,8 +2454,12 @@ impl<'w, 't, T: Transport, S: SvcProtocol> SvcClient<'w, 't, T, S> { let absolute_clock = crate::clock::uptime().is_ok(); let mut ticks: u64 = 0; let mut rekick = ReplyRekickCadence::after(crate::executor::clock_ns().await); + // Whether this wait parked at all, and whether a re-kick + // deadline passed before the reply showed. + let (mut waited, mut ticked) = (false, false); loop { if let Some(wire) = self.rings.rep.pop() { + self.reply_waits.record(waited, ticked); break wire; } let executor_now_ns = crate::executor::clock_ns().await; @@ -2281,7 +2471,12 @@ impl<'w, 't, T: Transport, S: SvcProtocol> SvcClient<'w, 't, T, S> { let wait_deadline = observation.map_or(rekick.deadline(), |slice| { slice.next_deadline(rekick.deadline()) }); - crate::executor::next_wake_until(wait_deadline).await; + let reason = crate::executor::next_wake_until(wait_deadline).await; + waited = true; + // A deadline wake is the one no delivery brought. Counting + // every wait that outlasted the cadence instead filed slow + // replies the service's kick did deliver under the re-kick. + ticked |= reason == crate::executor::WakeReason::Deadline; let executor_now_ns = crate::executor::clock_ns().await; let elapsed_tick = rekick.due(executor_now_ns); let expired = observation.is_some_and(|slice| slice.expired(executor_now_ns)) @@ -2298,6 +2493,7 @@ impl<'w, 't, T: Transport, S: SvcProtocol> SvcClient<'w, 't, T, S> { return Err(CallError::Timeout); } if reply_rekick_needed(elapsed_tick, absolute_clock) { + self.reply_waits.rekicks = self.reply_waits.rekicks.saturating_add(1); self.kick_published_request().await?; } if elapsed_tick { @@ -2857,7 +3053,9 @@ impl<'w, 't, T: Transport, S: SvcProtocol> SvcClient<'w, 't, T, S> { }; let committed = commit(self.transport, &session, &spec).map_err(|_| CallError::Transport)?; - if committed.raw() as u16 > Status::Unfounded as u16 || committed.failing_index() != 0 { + if committed.raw() as u16 > Status::FabricRouteCapability as u16 + || committed.failing_index() != 0 + { return Err(CallError::Refused(Status::Malformed)); } match committed.status() { @@ -2896,7 +3094,13 @@ impl<'w, 't, T: Transport, S: SvcProtocol> SvcClient<'w, 't, T, S> { | Status::Irq | Status::Exited | Status::AltDelivery - | Status::Unfounded) => return Err(CallError::Refused(status)), + | Status::Unfounded + | Status::FabricRouteConflict + | Status::FabricRouteNoEdge + | Status::FabricRouteWrongAccess + | Status::FabricRouteInfeasible + | Status::FabricRouteCapacity + | Status::FabricRouteCapability) => return Err(CallError::Refused(status)), } } Err(CallError::Backpressure) diff --git a/native/crates/rt/src/svc/mod.rs b/native/crates/rt/src/svc/mod.rs index b46c53b..8dda50d 100644 --- a/native/crates/rt/src/svc/mod.rs +++ b/native/crates/rt/src/svc/mod.rs @@ -49,7 +49,7 @@ mod serve; pub use client::{ AffineBundleCallError, CallError, CallReply, ClientEndpoints, GrantPin, InvocationAuxiliary, - InvocationAuxiliaryCapability, MAX_INVOCATION_AUXILIARIES, RetainedReturnCallError, + InvocationAuxiliaryCapability, MAX_INVOCATION_AUXILIARIES, ReplyWaitStats, RetainedReturnCallError, ReturnCallError, ReturnCapability, ReturnDeriveError, ReturnDeriver, ReturnLeaseSource, StagedReturn, StagedReturnError, SvcClient, }; @@ -327,6 +327,8 @@ pub const ACCEL_CAPTURE_WORDS: u64 = (32 + 32 + 32 + 32 + 32) / 8; /// Overlap V1: five BLAKE3 identities followed by three rounds, each with /// pending/cancel/terminal flags, two event IDs and two canonical output hashes. pub const ACCEL_OVERLAP_WORDS: u64 = 20 + 3 * 11; +/// DSpark Q/K/V V1: graph, artifact, prepared weight and two output digests. +pub const ACCEL_DSPARK_QKV_WORDS: u64 = 5 * 4; pub const ACCEL_CAPTURE_MAX_WORDS: u64 = ACCEL_OVERLAP_WORDS; /// Versioned capture framing on the existing founded parent channel. @@ -334,18 +336,21 @@ pub const ACCEL_CAPTURE_MAX_WORDS: u64 = ACCEL_OVERLAP_WORDS; pub enum AccelCaptureKind { Regions, Overlap, + DsparkQkv, } impl AccelCaptureKind { pub const fn words(self) -> u64 { match self { Self::Regions => ACCEL_CAPTURE_WORDS, Self::Overlap => ACCEL_OVERLAP_WORDS, + Self::DsparkQkv => ACCEL_DSPARK_QKV_WORDS, } } pub const fn prefix(self) -> &'static str { match self { Self::Regions => "ivpu-capture: V2", Self::Overlap => "ivpu-overlap: V1", + Self::DsparkQkv => "ivpu-dspark-qkv: V1", } } /// Upper 16 bits select framing; lower 48 preserve PCI vendor/device. @@ -353,6 +358,7 @@ impl AccelCaptureKind { let tag = match self { Self::Regions => 0, Self::Overlap => 1, + Self::DsparkQkv => 2, }; (tag << 48) | ((vendor as u64) << 32) | device as u64 } @@ -360,6 +366,7 @@ impl AccelCaptureKind { let kind = match payload >> 48 { 0 => Self::Regions, 1 => Self::Overlap, + 2 => Self::DsparkQkv, _ => return None, }; Some((kind, payload & 0x0000_ffff_ffff_ffff)) @@ -470,6 +477,11 @@ const _: () = { assert!(!kernel_reserved(COOKIE_ACCEL_CAPTURE_SKIPPED)); assert!(!kernel_reserved(COOKIE_ACCEL_GRAPH_ADMITTED)); assert!(!kernel_reserved(COOKIE_INIT_CONFIG_PLAN)); + assert!(!kernel_reserved(COOKIE_EXEC_CLEANUP_A)); + assert!(!kernel_reserved(COOKIE_EXEC_CLEANUP_B)); + assert!(!kernel_reserved(COOKIE_EXEC_CLEANUP_C)); + assert!(COOKIE_EXEC_CLEANUP_A != COOKIE_EXEC_CLEANUP_B); + assert!(COOKIE_EXEC_CLEANUP_B != COOKIE_EXEC_CLEANUP_C); assert!(!kernel_reserved(COOKIE_INIT_CONFIG_POSTED)); assert!(!kernel_reserved(COOKIE_SHELL_CONFIG_POSTED)); assert!(COOKIE_INIT_CONFIG_PLAN != COOKIE_INIT_CONFIG_POSTED); @@ -689,7 +701,8 @@ pub const fn shell_terminal_failure_detail(rung: u64) -> Option<(u64, bool, u64) /// /// Layout within `payload >> 8`: bit 55 detail marker, bits 40..47 exec's /// stable `TaskRepWire::REFUSE_*` code, bits 32..39 init phase, and bits -/// 0..31 the legacy rung. The remaining bits stay zero for future detail. +/// 0..31 the legacy rung. Bits 48..54 carry a bounded `LedgerFull` origin; +/// zero preserves the historical reply shape. Bit 55 remains the marker. pub const fn shell_exec_failure_rung(legacy: u64, phase: u8, refusal: u8) -> u64 { SHELL_FAILURE_DETAIL | ((refusal as u64) << 40) @@ -697,6 +710,25 @@ pub const fn shell_exec_failure_rung(legacy: u64, phase: u8, refusal: u8) -> u64 | (legacy & SHELL_FAILURE_LEGACY_MASK) } +/// Add the exec service's narrow refusal origin without changing the stable +/// refusal code or the low legacy rung. +pub const fn shell_exec_failure_rung_with_site( + legacy: u64, + phase: u8, + refusal: u8, + site: u8, +) -> u64 { + shell_exec_failure_rung(legacy, phase, refusal) | (((site & 0x7f) as u64) << 48) +} + +pub const fn shell_exec_failure_site(rung: u64) -> Option { + if rung & SHELL_FAILURE_DETAIL == 0 { + None + } else { + Some(((rung >> 48) & 0x7f) as u8) + } +} + /// Decode the structured exec-refusal extension, retaining a clean fallback /// for every historical image that posts only a numeric rung. pub const fn shell_exec_failure_detail(rung: u64) -> Option<(u64, u8, u8)> { @@ -747,6 +779,7 @@ mod shell_failure_tests { SHELL_CONFIG_ERROR_TRANSPORT, SHELL_CONFIG_STAGE_EXACT_AFTER_TRANSFER, SHELL_FOUNDING_FAILED, TerminalRendererFailure, shell_config_failure_detail, shell_config_failure_rung, shell_exec_failure_detail, shell_exec_failure_rung, + shell_exec_failure_rung_with_site, shell_exec_failure_site, shell_terminal_failure_detail, shell_terminal_failure_rung, }; @@ -754,7 +787,12 @@ mod shell_failure_tests { fn structured_exec_failure_retains_legacy_phase_and_refusal() { let rung = shell_exec_failure_rung(577, 2, 17); assert_eq!(shell_exec_failure_detail(rung), Some((577, 2, 17))); + assert_eq!(shell_exec_failure_site(rung), Some(0)); + let tagged = shell_exec_failure_rung_with_site(40_564, 5, 3, 2); + assert_eq!(shell_exec_failure_detail(tagged), Some((40_564, 5, 3))); + assert_eq!(shell_exec_failure_site(tagged), Some(2)); assert_eq!(shell_exec_failure_detail(565), None); + assert_eq!(shell_exec_failure_site(565), None); } #[test] @@ -914,6 +952,13 @@ pub const COOKIE_INIT_PHASE: u64 = 0x6B72_5F69_6E69_7450; // "kr_initP" /// or the step, so a stall is one call rather than forty. pub const COOKIE_INIT_STEP: u64 = 0x6B72_5F69_6E69_7453; // "kr_initS" +/// Three ordered, failure-only words of an exec cleanup-row snapshot. Init +/// sends A, B, C on its retained founder channel; C completes one bounded +/// serial record. This is testimony, never a launch or scheduling command. +pub const COOKIE_EXEC_CLEANUP_A: u64 = 0x6B72_5F65_7843_6C41; // "kr_exClA" +pub const COOKIE_EXEC_CLEANUP_B: u64 = 0x6B72_5F65_7843_6C42; // "kr_exClB" +pub const COOKIE_EXEC_CLEANUP_C: u64 = 0x6B72_5F65_7843_6C43; // "kr_exClC" + /// Completion cookie for the boot SPMD validate run (`"kr_spmdW"`), /// posted by INIT after the store seed: it launches `spmd-witness` /// (the one `kore-rt/validate` build) through the ordinary @@ -1113,7 +1158,11 @@ pub fn split_window( #[cfg(test)] #[test] fn accelerator_capture_kind_preserves_identity_and_rejects_unknown_versions() { - for kind in [AccelCaptureKind::Regions, AccelCaptureKind::Overlap] { + for kind in [ + AccelCaptureKind::Regions, + AccelCaptureKind::Overlap, + AccelCaptureKind::DsparkQkv, + ] { let payload = kind.begin(0x8086, 0x643e); assert_eq!( AccelCaptureKind::decode_begin(payload), @@ -1125,6 +1174,6 @@ fn accelerator_capture_kind_preserves_identity_and_rejects_unknown_versions() { AccelCaptureKind::Regions.begin(0x8086, 0x643e), 0x8086_0000_643e ); - assert_eq!(AccelCaptureKind::decode_begin(2 << 48), None); + assert_eq!(AccelCaptureKind::decode_begin(3 << 48), None); assert_eq!(AccelCaptureKind::decode_begin(u64::MAX), None); } diff --git a/native/crates/rt/src/svc/serve.rs b/native/crates/rt/src/svc/serve.rs index 83936ad..d97ec06 100644 --- a/native/crates/rt/src/svc/serve.rs +++ b/native/crates/rt/src/svc/serve.rs @@ -33,6 +33,12 @@ use crate::transport::{ParkPost, Transport, stage_grant}; use super::{COOKIE_REPLY, COOKIE_REPORT, ServeRings}; +/// The bounded idle wait of a host that owes teardown (a peer close the +/// service refused, an admission cleanup, a pending return cleanup): the +/// period at which [`ServeHost::run`] retries without a doorbell. Only +/// while something is owed — a host with nothing owed parks unbounded. +const CLEANUP_RECHECK_NS: u64 = 2_000_000; + use crate::service::SystemService; /// The affine owner of a bound service. @@ -2186,6 +2192,7 @@ impl<'w, 't, T: Transport, S: SystemService> ServeHost<'w, 't, T, S> { let notify = self.svc.drive_deadlines(now); self.notify_pairs(notify).await; + self.svc.poll_background(); while self.serve_next().await {} // A just-served arm may already be due. Settle it before choosing @@ -2224,12 +2231,28 @@ impl<'w, 't, T: Transport, S: SystemService> ServeHost<'w, 't, T, S> { /// The kernel asked this row to quiesce its devices (#809 §L); /// the payload is the remaining budget in nanoseconds. Quiesce(u64), + /// Owed teardown's recheck period elapsed with no ring work: + /// loop once so the retries at its top run again. + Recheck, } loop { self.retry_peer_cleanup(); while !self.retry_admission_cleanup() { crate::sleep_for(1_000).await; } + // A refused peer teardown (a device still finishing the dead + // client's work) is retried only at the top of this loop, and an + // absent client sends no doorbell to get here again. While any + // teardown is owed, the idle wait is therefore bounded; with none + // owed it parks on the doorbell (and a route) alone, as before. + let owed = self.has_pending_cleanup(); + let mut recheck = core::pin::pin!(async move { + if owed { + crate::sleep_for(CLEANUP_RECHECK_NS).await; + } else { + core::future::pending::<()>().await; + } + }); let host_action = { let Self { svc, @@ -2245,6 +2268,9 @@ impl<'w, 't, T: Transport, S: SystemService> ServeHost<'w, 't, T, S> { if let Some(budget_ns) = crate::executor::quiesce_asked(cx) { return core::task::Poll::Ready(Work::Quiesce(budget_ns)); } + // Every wake, including a device route's, reaches the + // service's own device progress before the rings. + svc.poll_background(); for (k, pair) in pairs.iter_mut().enumerate() { if let Some(w) = pair .as_mut() @@ -2270,6 +2296,11 @@ impl<'w, 't, T: Transport, S: SystemService> ServeHost<'w, 't, T, S> { return core::task::Poll::Ready(Work::Return(k)); } } + // Ring work always wins; the recheck only ends an + // otherwise idle wait. + if core::future::Future::poll(recheck.as_mut(), cx).is_ready() { + return core::task::Poll::Ready(Work::Recheck); + } core::task::Poll::Pending }) .await; @@ -2304,6 +2335,7 @@ impl<'w, 't, T: Transport, S: SystemService> ServeHost<'w, 't, T, S> { .await; None } + Work::Recheck => None, } }; if let Some((source, action)) = host_action { diff --git a/native/crates/rt/src/thread.rs b/native/crates/rt/src/thread.rs index 221c546..537d574 100644 --- a/native/crates/rt/src/thread.rs +++ b/native/crates/rt/src/thread.rs @@ -1218,11 +1218,11 @@ pub(crate) fn spawn_gang_workers( // whose per-worker kick-mesh endows name PEER BANDS and therefore // cannot exist yet: they are built below, after the claims loop // fixes every member's band. A meshed chunk is invocation-bounded - // at [`MESH_CHUNK`](crate::star::MESH_CHUNK) (n·(n+5) ≤ 107). + // at [`MESH_CEILING`](crate::star::MESH_CEILING) (n·(n+6) invocations). if !endows.is_empty() && (endows.len() != n || star.is_some()) { return false; } - if star.is_some() && n > crate::star::MESH_CHUNK { + if star.is_some() && n > crate::star::MESH_CEILING { return false; } let Some(self_space) = crate::entry::geometry::self_space_slot() else { @@ -1386,7 +1386,7 @@ pub(crate) fn spawn_gang_workers( kind: 0, rights: kore_abi::cap::Rights::empty(), }; - let mut mesh_store = [[vacant_cap; { crate::star::MESH_CHUNK + 1 }]; MAX_SESSION_FOUNDS]; + let mut mesh_store = [[vacant_cap; { crate::star::MESH_CEILING + 2 }]; MAX_SESSION_FOUNDS]; let mut mesh_lens = [0usize; MAX_SESSION_FOUNDS]; if let Some(plan) = star { let mut bands = [(0usize, 0u32); MAX_SESSION_FOUNDS]; @@ -2283,7 +2283,8 @@ mod tests { fn worker_bootstrap_does_not_inherit_process_attachments() { use kore_abi::process::{ PROCESS_IO_INDEX, PROCESS_IO_ROLE, PROCESS_OUTPUT_WAKE_INDEX, PROCESS_OUTPUT_WAKE_ROLE, - PROCESS_STDIN_WAKE_INDEX, PROCESS_STDIN_WAKE_ROLE, PROCESS_STDIO_INDEX, + PROCESS_STDIO_HOST_CHANNEL_INDEX, PROCESS_STDIN_WAKE_INDEX, PROCESS_STDIN_WAKE_ROLE, + PROCESS_STDIO_HOST_INDEX, PROCESS_STDIO_HOST_ROLE, PROCESS_STDIO_INDEX, PROCESS_STDIO_ROLE, PROCESS_STDIO_WAKE_RIGHTS, }; @@ -2301,6 +2302,8 @@ mod tests { (PROCESS_STDIO_ROLE, PROCESS_STDIO_INDEX), (PROCESS_STDIN_WAKE_ROLE, PROCESS_STDIN_WAKE_INDEX), (PROCESS_OUTPUT_WAKE_ROLE, PROCESS_OUTPUT_WAKE_INDEX), + (PROCESS_STDIO_HOST_ROLE, PROCESS_STDIO_HOST_INDEX), + (PROCESS_STDIO_HOST_ROLE, PROCESS_STDIO_HOST_CHANNEL_INDEX), ] { assert_eq!( bootstrap.handle(role, index), diff --git a/native/crates/rt/src/transport/mod.rs b/native/crates/rt/src/transport/mod.rs index 975abe3..d6570ba 100644 --- a/native/crates/rt/src/transport/mod.rs +++ b/native/crates/rt/src/transport/mod.rs @@ -58,8 +58,9 @@ pub trait Transport { /// The transport-level failure — *could not commit* (e.g. the trap /// gate is not yet issued). Per-invocation outcomes ride the /// [`Committed`] status word instead, exactly as the gate returns - /// them. - type Error; + /// them. `Debug` so that every caller can report the failure it got + /// rather than substitute a guess for it. + type Error: core::fmt::Debug; /// Commit one bounded staged session; resolves when results land. /// `TrapTransport` finishes into the architecture's caller-saved GPR diff --git a/native/crates/window/src/client.rs b/native/crates/window/src/client.rs index ea1c0b0..fcebcd4 100644 --- a/native/crates/window/src/client.rs +++ b/native/crates/window/src/client.rs @@ -386,9 +386,10 @@ impl<'w, 't, T: Transport> WindowClient<'w, 't, T> { /// visibility flip), or `bound` elapses. Level-triggered: drain /// [`pop_event`](Self::pop_event) and re-check state after every /// return. - pub async fn wait_change(&mut self, bound: core::time::Duration) { - let seen = self.channel.seq(); - if self.sync_status().is_err() + /// Work a wait must not sleep past: a failed status sync, unconsumed + /// records, or an appearance the caller has not yet observed. + fn change_pending(&mut self) -> bool { + self.sync_status().is_err() || self.channel.outstanding() != 0 || self.channel.state().is_some_and(|state| { self.observed_scheme @@ -397,13 +398,34 @@ impl<'w, 't, T: Transport> WindowClient<'w, 't, T> { .observed_density .is_some_and(|seen| state.density() != seen) }) - { + } + + pub async fn wait_change(&mut self, bound: core::time::Duration) { + let seen = self.channel.seq(); + if self.change_pending() { return; } kore_rt::event::wait_past(&self.channel, seen, Some(bound)).await; let _ = self.sync_status(); } + /// [`wait_change`](Self::wait_change) for a window whose task also drains + /// other planes woken through the same executor doorbell — a stream host + /// whose child rings it (`res/SHELL_TERMINAL.md` §6). It publishes this + /// channel's park intent, so a compositor batch still wakes it, and then + /// returns on *any* executor wake: a single-plane wait would re-park after + /// the other plane's notice and hide it. The caller must have armed every + /// other plane before calling, drains all of them on return, and keeps + /// its own deadlines. + pub async fn wait_any(&mut self, bound: core::time::Duration) { + let seen = self.channel.seq(); + if self.change_pending() { + return; + } + kore_rt::event::wait_once(&self.channel, seen, Some(bound)).await; + let _ = self.sync_status(); + } + pub const fn configuration(&self) -> Option { self.configuration } diff --git a/native/crates/window/src/lib.rs b/native/crates/window/src/lib.rs index b7285a2..f4518e0 100644 --- a/native/crates/window/src/lib.rs +++ b/native/crates/window/src/lib.rs @@ -29,6 +29,22 @@ pub const BOOTSTRAP_WINDOW_SLOTS: usize = kore_abi::task::DESKTOP_BOOTSTRAP_WIND #[cfg(feature = "runtime")] pub const COMMIT_CAP: u32 = 32; +/// How long an application window may take to leave after its controller's +/// close request was accepted (`Event::CloseRequested` enqueued): its current +/// frame, its own orderly shutdown, and resident-worker and surface +/// retirement. Past it the controller revokes the window task. The +/// controller's clock (`kore-window-session`) and any app that budgets its own +/// close against it (a `term` window asking its program to stop) read this +/// one value, so the app's budget cannot outlast the controller's. +pub const APPLICATION_CLOSE_GRACE_NS: u64 = 15_000_000_000; + +/// The prefix a launcher puts before a hosted program's command line in its +/// terminal window's arguments to ask for the conversation presentation +/// (`term --chat `, `res/SHELL_TERMINAL.md` §6): the same stream +/// host, drawn as a chat with a Stop control instead of a grid. The window +/// strips it from its title. +pub const HOSTED_CHAT_PRESENTATION: &str = "--chat "; + #[cfg(feature = "runtime")] pub use control::{DisplayRequest, DisplayResponse}; #[cfg(feature = "runtime")] diff --git a/native/provenance.json b/native/provenance.json index 5ca6863..c3a8223 100644 --- a/native/provenance.json +++ b/native/provenance.json @@ -1,6 +1,6 @@ { "repository": "https://github.com/MicroPerceptron/kore", - "revision": "9e4172bc22f3560c1352d8e0ad3f0cd36c1412c4", + "revision": "d929d9a3990a9e3f05d38d37b15904ab9551b0a0", "packages": { "crates/abi": "kore-abi", "crates/app": "kore-app", @@ -25,7 +25,7 @@ "files": { "LICENSE": "c9b2b12d3bad9080090798a442b4b2a77c48312ae75089f041281b7c8ad21738", "crates/abi/Cargo.toml": "f9a4811b4b6d06947a47a763354cad3bc739a5ea387194a4dcf22409351f3c7b", - "crates/abi/src/cap.rs": "772ef72eb5f41a9b6b2e45f3ddd0f155a30dd68cf490ae64866f78eb365d46f0", + "crates/abi/src/cap.rs": "72c350a93a23e6c985a806696f27630f0dbbf0132553161ebeb62c55f6ae122d", "crates/abi/src/clock.rs": "cce1c2839cba2377d3cc8696d60812f9195f071f422bd97bb4530ea0368972fc", "crates/abi/src/command.rs": "0670b8b281ac04192cb3fd650013243f2ae038f7c8efa4983302e1472221fed7", "crates/abi/src/crash.rs": "9d2e756b06195d0597250cbc915156c9b97aea3ea722b935db2ebeccc62a4d9c", @@ -34,28 +34,34 @@ "crates/abi/src/device/i2c_admission.rs": "1ea2fc04cdb0b15186d2f1d80bde53187860e2197263e225f330965d96e40876", "crates/abi/src/execution.rs": "106d679a3f993c3f4e7ad4eb625d40f4399f5b9b50947cb0336ba44a1d6b51af", "crates/abi/src/founding.rs": "801b8e9771763ff9223c3537aabb5653713d1883d1df62ff673f43982aa3e802", - "crates/abi/src/guest.rs": "e7a0a04b9335b2e9c2ee9091b2c6fd4e48be2fbccad7697cbd220a1b1978f975", - "crates/abi/src/image.rs": "ab3a4e6984286d6fc708dd2389b224c18f750c518e6744b2926065112c3ed564", + "crates/abi/src/guest/aarch64/mod.rs": "1fd24be3fa6cbc344e9873a0e3e75fabd1b919326e477ec4ea4cf148c01b66f8", + "crates/abi/src/guest/mod.rs": "0c49dab5c0e3d5eddeaedc79494662a7a683fd351f9cac354db7da0ea6f8e2c6", + "crates/abi/src/guest/riscv64/mod.rs": "c3ea09ccbc04959a527e1c75e4122dbf3bb7f30e2ae4f495fd59b6d980aca1a3", + "crates/abi/src/guest/wire.rs": "bec45a73cee701b07d40b37f7c296cb5e96f96d2994cd512fd1d9aad7bc7432f", + "crates/abi/src/guest/x86_64/mod.rs": "58fe4ec322a03c83e30faf3fef4c7ed6ec06b64d42a33a921e977306148be342", + "crates/abi/src/image.rs": "5014cf054a1f602af9876e5efd34708625abaf45c38e78d02e1192e31e4a6010", "crates/abi/src/input_status.rs": "5e48ead8883878c52b7ccdb22ff56d7bf04e4d74d7c689573278e7acc5e5b79c", - "crates/abi/src/lib.rs": "a49ac571688cc9936ef24389258fa943b57af30f0f35b5d9dfbfa83753b55ba1", - "crates/abi/src/machine.rs": "52580c91c35dc9e3044a6247cd3e177b4955f28edcae80ea99856a078cd9628c", + "crates/abi/src/iommu_status.rs": "2a2ef44ec698136ebe213db38b66b08b090b8d2c74a91346a38368ba43c20b02", + "crates/abi/src/lib.rs": "2d0dc5ca2e5ecd87b2b596ceb1c9bf6c6ddd180fd620232dd58918367b8c4940", + "crates/abi/src/machine.rs": "4c9f97ed3e002063430f9bee894676d7d085505984fcf3476a4ebf73498d8f07", "crates/abi/src/mem.rs": "ae502311778be8ef73f7b9edb1c47f157564f96a58db9add4ad74a313eb3a147", - "crates/abi/src/monitor.rs": "8365d4df572ed7cb8060c7c9903cfa5e75846298d1fb57e47f4131f607195068", + "crates/abi/src/monitor.rs": "a875eae2da322aa14c52e00d20f5b0aee5748342f5bd28db16cd2cb12608c045", "crates/abi/src/pkg.rs": "2893ad39761611ddee84d314bebbed45a9c9823ff145c6a8f28d3844cb40155d", "crates/abi/src/pointer.rs": "9971a292c1d00b275d8030e4f11fdd71206eaf12671cda8775d220b106ebcbf8", - "crates/abi/src/process.rs": "46117ea4e465cad1594738f9793130d326773fe447b043766b8ed61688e31930", - "crates/abi/src/processor_status.rs": "ad501715fbc878a4cc70e0f401ccd0598d7200c4ee846702754ca5162aba2b8a", - "crates/abi/src/session.rs": "0db6e96d5dabb38d431a2c4eeae14f4033f9260001fe032e3451e6decfac0b1f", - "crates/abi/src/svc.rs": "657da97b048feb1abc486d765cca3761c26dd9b4b48efd2cf020977747bfb11f", - "crates/abi/src/task.rs": "281fa929fea12e48e482ac9d2583879930bdddd1be76bc6b3fc879b46248450f", - "crates/abi/src/uring.rs": "1e94d4d468c13fc6037e602a0f0b42fdad42d4f86094f5f54178dba060c63e98", + "crates/abi/src/process.rs": "9f08d5cf6b9a213c63580798bd2e2c307070754724577d1c04da9502323c7f66", + "crates/abi/src/processor_status.rs": "c963e48f9b07046ab6c82d4cc2735ee2cc6c94561509a4e6e15927c837c0b0c6", + "crates/abi/src/session.rs": "1bc516ad898112465b68cf171281b4cf1cd6abcf8be9474c4baee89cfa075c99", + "crates/abi/src/stor.rs": "eabad4e5588fd343472660e15cccf32cfcc8d8f4c25eb18dffbd1598b6007db2", + "crates/abi/src/svc.rs": "0706435929cc42e337de6d0aa93fd4ee289f4844da041cc2f2e0984c7c3c088b", + "crates/abi/src/task.rs": "ac51cd35ec0c767b525c47306735811f4550427bd1719b9c1846fb79f905f2c1", + "crates/abi/src/uring.rs": "e98d9db77fbe00d5547b582700a1104d2c5b2546bbe57742dfff8362c32dcc2e", "crates/app/Cargo.toml": "fb793b25eec583da7b5e404b19bd656a6b8bcee2a86a20b0da399be2210a9398", "crates/app/src/lib.rs": "330cea793beb81af147e4f50b1e6fe4ad13a88842937351d493bbc0b5c44e997", "crates/collections/Cargo.toml": "0b570a215ecd3d18b86b0f1b554e95de6636520c513eefd6f02d16cc2f37ebd6", "crates/collections/src/bitmat.rs": "35c1b7e572de0fa4e3dfc8c1c2cb8b7c5a031a812fb9ed7d4c86466b2a8399ec", "crates/collections/src/bitset.rs": "941e147ef204a6f3100c7d06652c6b9c3efb796ca7bdb738660bf6975a7ed815", "crates/collections/src/bitvec.rs": "c9d2e040eeae4b4a4d96601ddf03333190c4046f4f1fd800fc1927891a1b81bb", - "crates/collections/src/boundedvec.rs": "c52d69f65451a96fe8e63a0c87cf0ca9e7581082d448519ef58786df29176540", + "crates/collections/src/boundedvec.rs": "27d9242f80721326b31f5aea7047ef5e5e41c759f88a29386eec0a2fa3bfecb3", "crates/collections/src/genid.rs": "d3b088cc95be2da48ed740d71488987fb764d0aadf346e1766ca71ebcde80b8c", "crates/collections/src/lib.rs": "becfcea6e2d9deeac06e9483a664d063454251ff30c7e83b4367602ff3767399", "crates/collections/src/pages.rs": "1873cce0d35e71bef799ef5ea40cdc782330dedcc1aa6cde8808915ba3aee8be", @@ -64,52 +70,52 @@ "crates/collections/src/slotmap.rs": "891ed1b871293800f377ac7ed81919f564578563390e91f070f8316b829596c5", "crates/collections/src/slotvec.rs": "3b08b9fb5778cd6d8a4b4fa388f008a53f2916244da31eb4ebec2f2462c03188", "crates/collections/src/testing.rs": "ca9f73b2954ee46e002542b464640cb67d8b0e97309b397185dc8001c5e05be3", - "crates/dagfs/Cargo.toml": "7a434f85cf42764f38a5d1ee53fa128acda17337a3a5e67e296bbecf4f5f1ff8", + "crates/dagfs/Cargo.toml": "434ebe9eb980b8b9aa674b3920c8d034606413143a2f2c6f7f0b8db193e72b35", "crates/dagfs/src/bounds.rs": "6a9691cdba604ab12f184379513a4d081117e3e1be36188c40c50c51ca169bd7", "crates/dagfs/src/capability.rs": "820b60b9794947aa8a4afdd5d42cc805e78ce56de7104da333e9a95dbcd32c96", - "crates/dagfs/src/client.rs": "9a66ad9a75c2d05ef944e1c8d65948fce0447fd3bddde9c3d9a4c71484a3c136", + "crates/dagfs/src/client.rs": "745987f54b5c7296d0b09a3d72a8e95f253777b50575dd32f0985c0c2f8fd0c9", "crates/dagfs/src/entry.rs": "ae077f13df5f0949fbc7c867b5f026aa494843ae605dde4fda845c2a531c33a3", - "crates/dagfs/src/error.rs": "c49b69e5b22c92a43bf02d22f35ea18d7b7c1069877fe84a961a0e6568ac08bb", + "crates/dagfs/src/error.rs": "bda2cab29a5b65519888c8be8e571b96b6c291b02b6aa45716e979af78d21c76", "crates/dagfs/src/face.rs": "b22d6f071397395a8ffda7487f5b876a03fc39590e4a1adefea622d5ae737bc9", - "crates/dagfs/src/fs.rs": "143fa8ef4bbc011635d436440e5bf753e3fbe7f52c1245b160ecb5857b83a580", - "crates/dagfs/src/graph.rs": "8e2d953289969d442d5608f057499abaa3aee29c8d9408735b9215dd80da4567", + "crates/dagfs/src/fs.rs": "5814ced189369baa1eafdef83f954e7a45616a02bc800708c3dae89e1dc1fdb3", + "crates/dagfs/src/graph.rs": "ee43a3f6cfda159311b4594ecb0988698388ac3c531316f2e07f690f254d500d", "crates/dagfs/src/ids.rs": "a6cd3127adbbfb3e02c7467f5736e2cabe6b53b9047d98f1cee2ba5de2c78227", "crates/dagfs/src/index.rs": "6c64df4092fd97974db52880460ada1e68390aa5ab993bff1e7355748e0c482e", - "crates/dagfs/src/io.rs": "de17d68185a6ae004e4fb53fce557a16058f0f7c9c366e3d37001483dea24993", - "crates/dagfs/src/lib.rs": "6fcb3f098d1b0b18f4ecde213e072922b3ccacf35b5add60b1b651074c9f185c", + "crates/dagfs/src/io.rs": "e03bdd30b4ce3fedb8f5a0486d8fefede714f1e7dcdbb92d2231e8f179712bce", + "crates/dagfs/src/lib.rs": "1e86372ce6c6e89fddb3404ef57bf19a32d1b3272ae5c8d2a6e50c6666d5ef29", "crates/dagfs/src/name.rs": "a59b8464b96e3a4bb65bd1056e3db5ea80bc3a99da528624ee77757cff4efaf9", - "crates/dagfs/src/object.rs": "634ccd42375e8208d314be598abf94df93253dd550829eab8b6c496c8dd6ddad", + "crates/dagfs/src/object.rs": "7c6821771008a5f83b45d8fa53fd9aa00bba405f7a26abec2274962adc1cc00c", "crates/dagfs/src/path.rs": "e462a2b3a25c8414f6d42315214a89a8435eb53ff7e4bcb0d1ee7d1d25d9cd61", - "crates/dagfs/src/protocol.rs": "13d790f9a287d9fc7c805a4e19834fc7e13a52e6e94962fe7a829eb1960f4cca", + "crates/dagfs/src/protocol.rs": "5c3216ce0c3bb6b4319a27921ba5561fe2b0dc112de937c165f10871571edc2e", "crates/dagfs/src/refusal.rs": "b50bf0b5e52a76be8e3af25261fe459f2d06f7aecea2ea5cdf756452621ad6c0", "crates/dagfs/src/search.rs": "cfc79463a6367d2fc62a9598ca3fe20d6771c3d9aad07709192da93338a9e4ab", + "crates/dagfs/src/stor.rs": "04f8f1832df2282016a52ee839f5ab83e60d58916b95fc8b76c87c37d0964483", "crates/dagfs/src/text.rs": "2fa8ebcf08781c2cce07a5e3af0d27106522f53d249ed4f3f573154f23921512", "crates/dagfs/src/view.rs": "75bb79d8c6258eb28cd67550494cea0334e6738213649bd631e3f5b310cdb106", - "crates/dagfs/src/walk.rs": "4e43b744c57dd2601545f01e255f88d4b370c50605d176753cfbb23294ad90e2", - "crates/dagfs/src/wire.rs": "c75ac9fb8ddd32ef8fce2e735f3675bd3e8acf0d2c51217eb8a299beafa6c75f", - "crates/driver/Cargo.toml": "4de89e5f8b9456354114b6afbe62584e4ebe65fb75987c6fcfae0013436826bd", - "crates/driver/src/accel.rs": "325cb6d3cd3907caf4daf7174316b7c75cfa62fd1023177c6dcf1516443180e6", - "crates/driver/src/block/memory.rs": "13c24290829ab1d2038320bc40e3646b7151685410b20e41f42e498d3e9a07cd", - "crates/driver/src/block/mod.rs": "6a03a6d82da63cb2ea64c1865bb538eeac3ea642ef1345ebc146c03e81f3f5f2", - "crates/driver/src/block/region.rs": "08f2be5e5b35b78a9e0415272147da2d67cf598fd719e99b616a20b312a982aa", - "crates/driver/src/block/shared.rs": "7f0c067c7cb7c71fa25145fb1c04b0b71abfde63241eca82d72689660ea3ed4e", + "crates/dagfs/src/walk.rs": "81e165d867a901ea50c13abd94816719d56efdc217d79943545f3a83c35dcaa0", + "crates/dagfs/src/wire.rs": "b4523d118cdeef548ff839a4329ee1ac7f2c66dc0d3e5b3dab8ff985d67471eb", + "crates/driver/Cargo.toml": "bb386b97586426ae14b1c9abf6290cab7187ab9997ca67b9c8e6808dc09133f8", + "crates/driver/src/accel.rs": "c2355916f1f6647d4bcf8414806c3bb9da45799e458133da70e31190316febfb", + "crates/driver/src/block/memory.rs": "634c09591f0ffbb091fd45fd791cbd306c16755df0f4c92e34868e362a758478", + "crates/driver/src/block/mod.rs": "292dd64d5220adb1bc3161ac405c62ff392cf64e5f7276863a59a7a1fa05eaff", + "crates/driver/src/block/region.rs": "d6099887fc1777f4bdb33c0b44427849a99aff7a18013bae32f9a29c2c0e59c5", "crates/driver/src/common/aperture.rs": "ef7a2ba28666c2381aa6df625c86384cf0769e1afc695ed4ee92c47479df24e7", "crates/driver/src/common/flags.rs": "f7d1c95eba00cdf3aac352b2950485d2f3b51ef6044a258d008f79719af1e5cd", "crates/driver/src/common/mod.rs": "68b6f331e9f5e08e8f9bbae26a153244993b91f86fccb24a29bb95cb65fbcc3e", "crates/driver/src/common/pagetable/addr.rs": "aa18fc1f043a24d2e4a0d33b6d4937da9f09dafa29c8a06a91d2a786880520e1", - "crates/driver/src/common/pagetable/alloc.rs": "9f8a9800f8651ef5afba927421e01c903e6bd8069a30f85bd0261b2d6f6768bd", + "crates/driver/src/common/pagetable/alloc.rs": "2f1d144d3c9dabe9351a66119c4268784ca6cefb7c988e525bd4c5b5f5852101", "crates/driver/src/common/pagetable/entry.rs": "bab69f6d7dc5cd1be2f4da1cf35629f9502207376c437219f95cbc7df4adfe41", "crates/driver/src/common/pagetable/mapping.rs": "983a34a633444d09cc022a3cd7fbb136a826c6eef9ce25804936db18b95b7b6a", "crates/driver/src/common/pagetable/mod.rs": "1ae70fe3e2867f6c4405cf6f587f22aa5bf57c8d58cd06be1c6260e0c86aabf5", - "crates/driver/src/common/pagetable/walker.rs": "7fb6efa823d797346f2a76e2203083c8b573c8e6e28fce745566f914d9c7274f", + "crates/driver/src/common/pagetable/walker.rs": "e61297cb0f341d511aecf9c6632740076d0a261a556c61c2ec05c2020d9961e6", "crates/driver/src/common/smmuv3/descriptor.rs": "d1fc3137384b1daf6a45b426d8222f8761ac0ffea635f9cbcf6b1946111aa66b", "crates/driver/src/common/smmuv3/mod.rs": "b2eec1fe252f286bb30c2e5bc105cec036a19e3621af98698a0e316b94920749", "crates/driver/src/common/smmuv3/stage1.rs": "95b69d9e17c001087e70c8003b1c36bfd702853098247c3ce0a8eb906888e161", - "crates/driver/src/cookie.rs": "91011bfb00e5ce4f326bd28faee0ecbda0ff3930ef7fa064cd04d5c7ccaf07c8", - "crates/driver/src/gpt.rs": "2caecfb6189a21971d6af4e36ef59b7b1960f4e46827f5790ed5ce323e5fe2b2", - "crates/driver/src/host.rs": "f16a9a72678ced95ac496a7ac65796fa7ffbd8ef2ee71ab654ab30ed921b5efa", + "crates/driver/src/cookie.rs": "9c60639fd1646b293eb6c7846c5ce67205e9d2e35e55cc0ad683cfcb329467f8", + "crates/driver/src/gpt.rs": "71f6b9fd9b4f4d78622a48b5a706276fbc1f6aea1ec1005aed6fc8396635f59e", + "crates/driver/src/host.rs": "97cd5a7c4694d36564ec0a59b3c9734de177b1c0853dc47560f61b27c95e69c9", "crates/driver/src/host_double.rs": "e5850fc7d7d39a090a8d30ad5d589e68294669b14edb96d15ff4794a0282a6a4", - "crates/driver/src/lib.rs": "b6ab0b8326e3f7aad232d5c5e6facc9f8570bf67e78e2c16b1252747a5efc2f4", + "crates/driver/src/lib.rs": "8808eb41d003e94815dbfa56b7875d3866bdbada0b968056ccd06e3936318867", "crates/driver/src/net/addr.rs": "c14556ffc4ea3885e2cb94c27994d3cb7deb91991e01a9f3d704243bceebb080", "crates/driver/src/net/backend.rs": "f6e919841e44607403d536330d1a594e9f7aea464bbdf7617fad63266cb1294b", "crates/driver/src/net/filter.rs": "8ed9cacf31874a642a879e9106ea9e1f5e31e8ed44c42376424ec7ef6d5facfa", @@ -118,9 +124,14 @@ "crates/driver/src/net/policy.rs": "87a287ff727c9acd87e9ff6f726b5ec0c5228b1013e3f92fd5c7c158730f1647", "crates/driver/src/net/vif.rs": "3390ef56c41062e24946a8a35c4584b910fce0543a17e0a5c478ed4f7d6efe5f", "crates/driver/src/net/window.rs": "d2f0e2e774ce28a65be10afad5883f1298b292d2a9fe7a0314ae427ec0de6d1d", - "crates/driver/src/ring.rs": "a66d69e4b75d580138b741eee15df7a761414cb530501b5b96f5da96fc0be989", - "crates/driver/src/seam.rs": "3ff051bc27cec25a57fd62591795a590cd58fad692d65506c62cdb9d746d82dc", - "crates/driver/src/stats.rs": "613b544c83211d632726f93a37258a8d70ba62a9005a7daa54cf12519151c853", + "crates/driver/src/ring.rs": "fe02aeb350c852ab28095516e0f5efc31bd787377cb91c95b24523b3bf999817", + "crates/driver/src/seam.rs": "391db6184832923c085395d35a7001d759082b9d557aec2a9837f29ad6bfc5e2", + "crates/driver/src/stats.rs": "ae7a7785429481a709e11407738d92dd8db0e18d423025afbc3e153ee454c07d", + "crates/driver/src/stor/engine.rs": "d4c661785bebd7708a56c46a4225c64342544fe9cfb2bafd17f65a9e0410d950", + "crates/driver/src/stor/grant.rs": "6cb866bd924914f3169c212ef950b4698ec9ae6b174ab4a51c101fd7957babf1", + "crates/driver/src/stor/lanes.rs": "b1a4c5da0cce3d4210c3b161c12220503f50f63d191b71eaf843f69cea116a38", + "crates/driver/src/stor/mod.rs": "faa061c11e3001f83f35c3bb8aaa9ce2c9378854184b862b8a7fd39afff55cbe", + "crates/driver/src/stor/timeline.rs": "5adb419bfc501ca4b679c1c02c705eb28b848eb3ec8055870a79bc39130d1aa9", "crates/driver/src/wait.rs": "f73935286f7a5e6644fbe528f36cb6058529499880b6cd52e996691bfdda4483", "crates/hint/Cargo.toml": "f998b920a2871f80ab15d7bbfb841fa2c2518b245912d2a625959e2c69931a08", "crates/hint/src/aarch64.rs": "f1b4f08acee7c100ad211fbaa8623b897f05b8887adde5d41fe5a8b5f38bfe19", @@ -128,8 +139,8 @@ "crates/hint/src/riscv64.rs": "5af5308b7241a52402a05c638f04a95457f257f6f41d1e766449562934dd4872", "crates/hint/src/x86_64.rs": "9f0e4316f70023277b016b9e6e812a3da0e021f3f8cbbe9ef3c3ccb660e4d85a", "crates/late/Cargo.toml": "3d49205399539aeced9240f2902b3be715ee5baa28d1ecde826fb42515b95e83", - "crates/late/src/lib.rs": "4ede5f17bb0872c57fcec6bb00c5c1aa4efdb0e65074ced38fd97c244ca3b7d9", - "crates/late/src/metal.rs": "0aec71e435af4a5c7c58f2b9efa6689bb6c4f3ec46cebfaa5d54298259f28e69", + "crates/late/src/lib.rs": "f3dc63898c28e521f05824d37fa806647a8a2b6890bf18ebaef0d38bbbc00bca", + "crates/late/src/metal.rs": "29a3784b937edb7723a7d1fcdab2efe3dc773a3b503642c816821f0be24e9611", "crates/ld/Cargo.toml": "bf8a6b091682174695d21d9082243086989b03b79fb38df3cd16a7e8a8d8bd92", "crates/ld/src/apply.rs": "8b6a8217bd38b1bb2d900bb386c7d90845dc2ce4cfb18ded19ff486cf39a327b", "crates/ld/src/check.rs": "c28cddf7980dfb6d6ff3d8ca74d1255f9dd75db5791028bd58ce096f2bb945cf", @@ -138,54 +149,55 @@ "crates/ld/src/lib.rs": "c8b510448ab682116cb1ca3248cff1266c7f1bd24450fefe4f650b21aa6eac49", "crates/ld/src/link.rs": "b193fbd296d0ed84237387682095c9b640c05619a68c74b993aceb62f86d5713", "crates/ld/src/parse.rs": "b1cc85112cf1a6274acd7a9bd14d15380e6dd2c347ddd82063cb7425f76cc75a", - "crates/ld/src/place.rs": "c9260c6b0e1af861aaa7ea82107fa36350e5020fd39f0312469db0326a0ac32e", + "crates/ld/src/place.rs": "072ac0db95d50d0d4042dc87280147cbff4204190ca2c38febf74f06cff2f700", "crates/ld/src/reloc.rs": "d3ef8b6f7deba2bab0f9b89eca554b96cd6d10d0522ccb09fe9cb909d26e6b4a", "crates/ld/src/stage.rs": "569ba1bd846cb5a6fbcedcebd075169523fe176c05e0a5714fbce210c0e18877", "crates/ld/src/store.rs": "ba5a432a2a3e34e64cdd8f02dccc2c69bab5719b6a4223393c72beca0c7a2b18", "crates/ld/src/stub.rs": "b670715026a889f5baaf233a35d3d24c1891223a0e57665782ee326d90e49f5e", "crates/ld/tests/apply_engine.rs": "e01f5f805f34d614717ceb297275d65936c1b48989df90079fcd2f8ec83b6b1c", - "crates/ld/tests/built_images.rs": "9c6a75df826a85cd0e08a2bb4aec7181e3ddf967bc116c8aee562ee3bff31586", + "crates/ld/tests/built_images.rs": "cb635657e4ae99b00b1b504b7949c67dd16c86be81a843d50f098338f1482300", "crates/ld/tests/check_gate.rs": "0ead133777f18f929d5e9b0353a289a5b4fa9e2af929155f525524096ee5578c", "crates/ld/tests/common/mod.rs": "1e4bdfa37511848d9e9458d9a550a1ff2410154411c6d026e9106035044ea438", "crates/ld/tests/late_layout.rs": "9bfc460d8036d69c1acdc52c430ee475e25062519cdc7dd30f235b882414a339", - "crates/ld/tests/link_engine.rs": "1bb5320f1fb4596cc0c2d133290f3a4af0542597a8aa8b805a4f91148674877d", + "crates/ld/tests/link_engine.rs": "a6959aa6f07b4776dadb80d192fa0064fee2823017c9555108fcbfa81a922e6d", "crates/ld/tests/shipping_list.rs": "113bf634f5a8f83103e266149c2c8daf834f3c99b86d7e3e0693f520283a1469", "crates/ld/tests/static_boot_images.rs": "25fb47a0a8e5b11dca5b8c31f64b90e8d6a6edc3c4d81b0fb5e1733afea3ebe3", "crates/ring/Cargo.toml": "b0a6e8d0f919494ef9e2f8c229ff64e24709347ca76c3d7f5a6e413387c20bcb", - "crates/ring/src/event.rs": "c33884f4f8b651bcd28368ea580cf18888e986645d35bc1aeea1957e5f704d14", - "crates/ring/src/lib.rs": "ae9ea82c43efb01c0c3f2c04a90540d64fead78bfef7072499bf97a76b95396c", + "crates/ring/src/event.rs": "72dd838dfe2f163fc240c46ec8160051ba57f91a98dcdbaf35624b7227d2033b", + "crates/ring/src/lib.rs": "847200aad2d16c084b1430a847fc6768e6d9765d0e22af79537c9a81896aa9bb", "crates/ring/src/spsc.rs": "136730d1c645044e0ddce3fc67e9b32410935a522249af5d69039f91e9b5a636", + "crates/ring/src/stor.rs": "e8b08ac46a9e72550d052e88ed82a332c220b6f3a2d5d00aa5a88ac241c4a65e", "crates/ring/src/transfer.rs": "1ae7f7f4f81c7389f893c4b7a1186cf9b942507924d8fbc00cccf8594ea3f153", "crates/ring/src/uring/mod.rs": "34077701d8947e05eb80071a3b78f5a33f1b7f712e3220420212f7605dcd52f5", "crates/ring/src/uring/opcode.rs": "9ee6825d8582fa35e599b0d3b2266421f7af04244e16c6d37b46061f3937a575", "crates/ring/src/uring/types.rs": "ff2dff0f9f55d0ef0181f6c12fb2dd78b726fa294c9b237002ecda639d8f7b19", "crates/rt-macros/Cargo.toml": "c84f262e96a3515dcabb61c96c34d9aec4b5f5a377af0f6da54e2e71b2c7807f", "crates/rt-macros/src/lib.rs": "ae051ab730bd408b6e0449a8871910257b6ccdac6f3324372dff8cec62ede24a", - "crates/rt/Cargo.toml": "6932ff87807bfdb8b853a17d37f69a52658335b1a45d777e97a4bbc819250109", - "crates/rt/src/barrier.rs": "5f6d7edee62156e87aaf18ec57c86cfe907545b75149816dea242da6868531f8", + "crates/rt/Cargo.toml": "0b472a92c92d74738202204ccafe53088ba9d303c3c8d4ac09a9dc29204a4d23", + "crates/rt/src/barrier.rs": "5397d4de097e454bd67f60bdaac0a94198c51e42ca1526eafd86269b85370027", "crates/rt/src/block.rs": "70c2aef8bf149da2f32c7c995cd0f4c63cb492647896b107fdd5dbecd3d3042a", - "crates/rt/src/cap/mod.rs": "25c935de31f6e196255eabc342d3f1e1e049517a10f41fb5283f6f207cc1fb87", + "crates/rt/src/cap/mod.rs": "9e1d428a1f020f8abc2a896ec94e0aa95dc84033852f3b359aaf2f1a8a22b91c", "crates/rt/src/cap/record.rs": "640b0ff60052d4775c259489af5fbff5cd03164370aceb4775186be956817bcd", - "crates/rt/src/cap/verbs.rs": "6fca0b77e941aec2fcc299e9a7a09abb7eeb72e0bf224e57945fafe19896922e", - "crates/rt/src/chan.rs": "a04993d9326b481cb55e61c8f8ef19d94ddc6d83c32ffd5e11e8e50a4264dd7b", + "crates/rt/src/cap/verbs.rs": "6c14391e990375c51dbf7c706d4f8df8e2536671a6bd8bb647c739da47b9bb3c", + "crates/rt/src/chan.rs": "fd50f962b23c2bfa15e72a1800790475d5cf6fcc55503ed39269852f025397e5", "crates/rt/src/clock/aarch64.rs": "47828a573f5fc30a988f012a648a8cb55596b45331cdcdfe1b390cd1d8543339", "crates/rt/src/clock/mod.rs": "d402d793448359338a361563a0fea91ea43d0264a5eb38c75b9abbb87df89af9", "crates/rt/src/clock/riscv64.rs": "17d4ecdb2b4f8152c0cf51f71ce28fe331d929a62405810a1b7eb332bb275c3d", "crates/rt/src/clock/x86_64.rs": "ca499d4dc51b7ed0eb196b831295b5fdecb09a1bd2df628a82e749e27011994a", "crates/rt/src/core_id.rs": "cadde39801a1d0ed05d70d0634a9e52d58043878737d790c4130183751b622cf", "crates/rt/src/crash.rs": "c0c59f8c96c91bdadb4a469f90a4874323f6fd4da72661ad03e1da8d19072531", - "crates/rt/src/entry.rs": "b8012178c3a3c33f0d535bc582df9765eb29958521f37b81e1ab30487ec43357", + "crates/rt/src/entry.rs": "cc534f3474ea2547ada3c7d3008aded316a6b17749bd8a61d621cf579935631f", "crates/rt/src/event.rs": "6d523a6bd1c8a7f956359dab99980731196cb9bc9de7ce076a47a986e758e0af", "crates/rt/src/executor/mod.rs": "4f5ba3aa87dd248aa44f21573b45f05f6226135ea05c271a30331d49fa63cf32", "crates/rt/src/executor/task.rs": "b3017cb97f287b8796c61d245c49e5c1b0ebfabe8c535cc2b3842cdbc854dc29", - "crates/rt/src/executor/time.rs": "e6c79ca042baa52cc68450528b6b47dce6be7e21ea1a5621e4ae7dfc5d28f4d8", + "crates/rt/src/executor/time.rs": "0ba0ca633be749f7e51426de9e1f4478c864a5575b68c34b24e732dfabc1ca12", "crates/rt/src/exit.rs": "3bf7e8ce6dac63b4ce365bdf2e743a9fd171851f1c5e40f8afbb4548877cef1b", "crates/rt/src/fiber_stack.rs": "0d7f634321d3b5d04cf9ce1b0a7dda03e22743083c021ffe81363ffda6754cce", "crates/rt/src/futex.rs": "5161418bf642366b701719dd70838edc00ac1e046dec5a7ef257faeee60ba8b6", - "crates/rt/src/heap/mod.rs": "d3a8ee3efe41b53e17bfaa59430c450621932a0e327ce31758395bddd6987abb", + "crates/rt/src/heap/mod.rs": "d53eb6ba61d18369a537a4cc148800a03ba7ad04c6fba1728c8c7228f5018b34", "crates/rt/src/heap/pool.rs": "789b0e08f01b9d720fa2f7ef933f91aa5843d9f8571b2e12f781ff1715008c3e", "crates/rt/src/heap/shard.rs": "e23e8583fa768c269dc614579956bb95f026e7504ac163988fb0a3a81ae87de8", - "crates/rt/src/heap/source.rs": "0297411534c440c457fe8c67c1d50544293dec7a1036eb6fb61b9b016cdb7c31", + "crates/rt/src/heap/source.rs": "40f1e7aea42c0174a64f6a21790ca4a3883092b3a2791c7904a9c078cf74a7a1", "crates/rt/src/hint.rs": "dec1a305bcda6df6569fc72bf534531a411b2af61b121d90c6bbebeb8e2b1098", "crates/rt/src/lane/fiber.rs": "3755850096e916544cae4b269f301d035848bbad69d284a3b75f049d82a384f9", "crates/rt/src/lane/mod.rs": "6912c495dfbb3bec8e9ddbb6cf0e5458ca5318b6e90882ae3433c2f0649d2313", @@ -194,31 +206,32 @@ "crates/rt/src/lib.rs": "344dd15bb2aa428f751664c41f4db22d2c05eeb006af1c350b3637d37ee60da3", "crates/rt/src/mem_source.rs": "41cbfb3200c222c2d2d0094b19445a9cf7a016e80f20241f8f5cf22872c5ce56", "crates/rt/src/pace.rs": "81217909cee01fa148df9e953e6281d7abf328eed7c6257aac8fbcebc824710c", - "crates/rt/src/pinned.rs": "09f5134b44670786c9404ea343ebc09bc79734cf23e38b26708c9f58bb805e9a", - "crates/rt/src/process.rs": "52c4221118b5ad7e5f5ba158490b7486efc845a1d30fe8731c009e13d8f1ede3", + "crates/rt/src/pinned.rs": "9f86a46572178965c3a111363eb20431fc19916eb9e06c2da50f8f918f3eccf8", + "crates/rt/src/process.rs": "d551775314b6eff02843a039bd82f7c66e2867cb14c34fa45531e0efeee33a62", + "crates/rt/src/process/interrupt.rs": "553bb4aa5678b4efa8d7537c96df0369d8fb810d0b3abb398b71dc7edb3db174", "crates/rt/src/random.rs": "b6d2738f97a9717d3d9481c49003e6c00df545faf52564189ad1cec508d16af6", "crates/rt/src/room/mod.rs": "c6f56dda5064ebe75ee85ef122d82c81a507f3fe498b6f9a8463a977a0566c04", "crates/rt/src/room/x86_64.rs": "122e283074fa754a49c73beb733a92660a5d8c53701b5223297ded797c61fc70", - "crates/rt/src/service/mod.rs": "7943ca3d02bc976be17a4e9adc25f651af861821b54421f14b3de66b9ae0ca86", - "crates/rt/src/session/builder.rs": "48a60383e64dcf4d9c7af3e83832d1a19e26b8e6b5b524f3557199c4e4910ef9", + "crates/rt/src/service/mod.rs": "e64f73db200c075fe3a567e325cb913ea27d1cafaaea0cc974a32a640cbb2879", + "crates/rt/src/session/builder.rs": "b78a7a96e842b6598bf97f89b8fdbe96d47329ed56d608c03c7c47b3e051bb7d", "crates/rt/src/session/flows.rs": "7205afbbe1d859868b313a24dccff8902e6ba3c1c8c45031e9531f6286259cae", "crates/rt/src/session/mod.rs": "466c2fbd8af228b3277075ad90a6d81031c1cb0a4c877f75669fff80e15338ea", "crates/rt/src/session/slots.rs": "82594125aa491429ed8c10b1316c19a82a162dc99f6dc97cdaa76398b92558e9", - "crates/rt/src/session/tests.rs": "047f430de2714b333570093ec2fe8ebc1aa1c1702c70563ced2878260f03b3e8", - "crates/rt/src/spmd.rs": "1b3ce624f4130210ac8e4d911e5d38556d8dce70d4eda0e518206c2e5ae39765", + "crates/rt/src/session/tests.rs": "27e044f75be3585d21e80beba0e667a56ab666e85064b50e178f1641d65959a9", + "crates/rt/src/spmd.rs": "8310d720182b16e5b98681758d4d2483b6905201431568c599247f65e2cd6b9f", "crates/rt/src/stack_arena.rs": "b252d45bfe845038ca9c8dc32941c634e448a2150dd49cf130e537048602f77b", - "crates/rt/src/star.rs": "2870b39a2d25eaddff9fe5637db9457763ff1daf66fd7189d5cb1686e03dbc94", + "crates/rt/src/star.rs": "59d03d7d7b55469a0de19f537e6871b1e91114dc09fec0edbdb6a34655d0a151", "crates/rt/src/startup.rs": "b1106a10e82659b8526ba7c94b7e40849edfdcbc30b077a77242712da8ebbd93", - "crates/rt/src/svc/client.rs": "3ecafdaca48600bf24759f46ca5e5d6f7a1a62c95aa72f43e7a24c93a4488614", - "crates/rt/src/svc/mod.rs": "2ca8f56ef450cde38858149314f779495ea802068ae64cc08edad42a38bd1347", + "crates/rt/src/svc/client.rs": "b3488fea0c25cb5ca307e716ab94c58793f73276fddb397d6f178bbfe5266e0f", + "crates/rt/src/svc/mod.rs": "f518d462a19cc46257c1b2e1ffd0d9e266d90926a8641df7d01d3e2b5d10262f", "crates/rt/src/svc/recyclable.rs": "9e5af8ad316bdd89e001fef7fbb6914b5a936dcdfbab2122740e03c1cf0ee7ec", - "crates/rt/src/svc/serve.rs": "d23cac297392754526b387b29efc2b06f2159411cddd7a0d6d71aacac5b293f6", + "crates/rt/src/svc/serve.rs": "184636ade6f76e803591a2c6553608c9001b5817fb11edef19fe4142cd453d17", "crates/rt/src/task_watch.rs": "96b646f8785e427640519191cbf1f294c6908e5062a8082db271364048aa5b9a", "crates/rt/src/termination.rs": "920685ebb2071db58ae667e57bd02a6127166e8224ed13ec81216465ca315a55", - "crates/rt/src/thread.rs": "d59b9bf732655cf5662b5dcfc1ee18aa3c7319618b9b7876b794bbf6ef63fdb2", + "crates/rt/src/thread.rs": "fd1e690e296483577e0f41eb39b0bb88997b03b319493191a1143087e02ca030", "crates/rt/src/thread_controller.rs": "9bd123d95349780caa4e2d50f281326eae37c207c41fd4171014c5ddbcebae52", "crates/rt/src/tls.rs": "85bd95bafb3de73d249d7a2eb8779b5afc929077e27b4baa5f7c7428063063ab", - "crates/rt/src/transport/mod.rs": "7d45261815f29953e2a49b1dce227e327c55570b2019a44e9addb822ea96b21b", + "crates/rt/src/transport/mod.rs": "9fb930d714458db515d6963f5229ced4e211fa2ae772aa5afe18feea81735e01", "crates/rt/src/validate.rs": "b7a4d3741dffd4a560bdae252128030cc4e6c4117e7274d121c60d8877aef121", "crates/rt/src/wake.rs": "ba92b949d9ba3102b2ebff03f9f31c45c85a31c8ceffd0d823976489947fde87", "crates/rt/src/worker.rs": "97d0e1a6bde5800e4409c1cd7a7b688bc8f06d604a6c86338e1fdc78bed15d50", @@ -237,13 +250,13 @@ "crates/window/Cargo.toml": "96c2d4c89e179c3718d4b4e897bb0a44c55ac7ad66341529da4bfd35e203592e", "crates/window/src/appearance.rs": "b22e65286b94215bc4265a09ef280a5118f368ac45120311b870a56f7cc40a81", "crates/window/src/channel.rs": "256a40e48eaf0a5c96abe4d88edb49767b8bd3da3c74ef647ca5f0f56a94ff46", - "crates/window/src/client.rs": "e433b2d634bffdd1dfd0351df34b939478a3c47bd4d21845d982e5a81baccdc5", + "crates/window/src/client.rs": "ca9b7dfb83e31868d062a3de2a1307181d91fc8a5da8a55b5d54cea87438de7d", "crates/window/src/command.rs": "3ad4867ce62f3a430a68e4d105991229938e131e8cfde8afbbb0f522f99355d3", "crates/window/src/configure.rs": "5731662e919827aa6c9b189dce1d4ae06ebc7f7940c57fafd8662fc33bc46154", "crates/window/src/control.rs": "bf482439516c5d0334acfe764cc3ae4c838743cb2082da5e6476286acdf5d922", "crates/window/src/event.rs": "bbe36a67815ab5190308a11299c078bb232f44ca332f9173a591a81e0a6df683", "crates/window/src/face.rs": "1078d8664e6da529c9d4fd43a7e3df77f287d0061f418ed21859d289e86765a0", - "crates/window/src/lib.rs": "984a1d3271410832ffdca4b48642b3fa057148e8f235602d8f18d6b5c2d38beb", + "crates/window/src/lib.rs": "2045cc4a46d934ae6eb65cada65ef7518c118ad47a7e593bda28ad22de63c04c", "crates/window/src/relay.rs": "c58b879c2f8fc93da59e456f0f49a260caed85fbe223930a81898137ea442c95", "crates/window/src/wire.rs": "efa0657132be5814102d1f688f6220e9772c0e151371ea104a18a6a43a6d50fe", "services/extract-worker-rust/Cargo.toml": "dcd585ea908827f474ad0e2131db6adffebff3fef2ec2ce1945bcb48b44211cd", @@ -254,7 +267,7 @@ "services/extract-worker-rust/src/lib.rs": "83faa6b21a48d4445205eb87d57974e1f605756f382fcdbbc3742f06db745ba7", "services/extract-worker-rust/src/sfnt.rs": "2823398e17336b2cd5049cece64270b774e211d911ee8789e45ffa3ecfe6062f", "services/extract/Cargo.toml": "bc42b58feeefe57a7a85342edced4982e2c951e514cba999225a62b72f35fbd2", - "services/extract/fuzz/Cargo.lock": "2547d3da1e9811fe16a65d94e984acaa7302a8af2f9f237c253316be99b99be2", + "services/extract/fuzz/Cargo.lock": "84299fa3ba87cc0cc17cf0ef8dd39594801996958d2c80d0051d72441ef7d9f5", "services/extract/fuzz/Cargo.toml": "14e61dc1238cd138d65f1e736abd57ba184d6f0c51dc95401cf97c08b8902b0a", "services/extract/fuzz/README.md": "da76177cedd2dde160a6fcdbec3fd5a2cc7ecac8565011fbe8721cf681de6907", "services/extract/fuzz/corpus/adapters/archive-truncated": "833ed3e9b647b19d56c2f29ac4fb3d73218b7b3df3faa5379a04c170adb4bcc1", diff --git a/native/services/extract/fuzz/Cargo.lock b/native/services/extract/fuzz/Cargo.lock index 4f259ae..c3d984b 100644 --- a/native/services/extract/fuzz/Cargo.lock +++ b/native/services/extract/fuzz/Cargo.lock @@ -232,6 +232,7 @@ dependencies = [ name = "kore-rt" version = "0.0.1" dependencies = [ + "heapless", "kore-abi", "kore-collections", "kore-hint", From 995dedbee1c01b6eaf96ba18b705c37903bc1479 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 21:59:27 +0000 Subject: [PATCH 2/2] ffi: carry the provider's Retiring class across the C boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The native resync brought kore's newer `ProviderError::Retiring` (a DMA window an earlier client lent has not retired and the requester-wide retirement failed), and `provider_class` matched it exhaustively, so `kore-ffi` stopped compiling (`targets`, `host`, `miri`). Append `KoreProviderError::Retiring = 16`: additive, like `Timeout = 15` was — every existing discriminant and the completion record's layout are unchanged — map it, and regenerate `kore.h` with cbindgen 0.29.4. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BDbTpiJFNzUJDBdjhHtXDm --- crates/ffi/src/lib.rs | 4 ++++ crates/ffi/src/session.rs | 1 + extern/c/include/kore.h | 6 ++++++ 3 files changed, 11 insertions(+) diff --git a/crates/ffi/src/lib.rs b/crates/ffi/src/lib.rs index 0de1b9a..b3022cb 100644 --- a/crates/ffi/src/lib.rs +++ b/crates/ffi/src/lib.rs @@ -149,6 +149,10 @@ pub enum KoreProviderError { ReadOnly = 13, Backpressure = 14, Timeout = 15, + /// A DMA window an earlier client lent has not retired from the device and + /// the requester-wide retirement failed: no new client window until a retry + /// succeeds. Existing lanes keep serving. + Retiring = 16, } /// Scoped generational name. Copying its bits does not duplicate native ownership. diff --git a/crates/ffi/src/session.rs b/crates/ffi/src/session.rs index b43050b..6176f38 100644 --- a/crates/ffi/src/session.rs +++ b/crates/ffi/src/session.rs @@ -326,6 +326,7 @@ fn provider_class(error: dag::ProviderError) -> KoreProviderError { R::ReadOnly => C::ReadOnly, R::Backpressure => C::Backpressure, R::Timeout => C::Timeout, + R::Retiring => C::Retiring, } } diff --git a/extern/c/include/kore.h b/extern/c/include/kore.h index 815c30c..a9089a0 100644 --- a/extern/c/include/kore.h +++ b/extern/c/include/kore.h @@ -324,6 +324,12 @@ enum KoreProviderError KORE_PROVIDER_ERROR_READ_ONLY = 13, KORE_PROVIDER_ERROR_BACKPRESSURE = 14, KORE_PROVIDER_ERROR_TIMEOUT = 15, + /** + * A DMA window an earlier client lent has not retired from the device and + * the requester-wide retirement failed: no new client window until a retry + * succeeds. Existing lanes keep serving. + */ + KORE_PROVIDER_ERROR_RETIRING = 16, }; #ifndef __cplusplus #if __STDC_VERSION__ >= 202311L