From 9ee816bb913f999f6fe07e25f12228554496530e Mon Sep 17 00:00:00 2001 From: Joaquin Carletti Date: Fri, 11 Sep 2026 16:54:31 -0300 Subject: [PATCH] tooling(recursion): count keccak hashes to verify a proof (excl. grinding) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A host-only diagnostic, behind the `hash-metrics` cargo feature, that counts the keccak-256 hashes done to VERIFY a proof — a fast, deterministic proxy for the recursion guest's dominant cost (Merkle path hashing + Fiat-Shamir). A default build is provably unchanged: PlatformKeccak256 stays = sha3::Keccak256 and every counter compiles to nothing. With the feature on, the host keccak wrapper bumps a global counter on every finalize (inlined, byte-identical digest); the Merkle backends and the grinding PoW check tag their own sub-counters. So test_count_recursion_hashes reports total(excl. grinding), merkle (nodes/leaves), transcript+other, and grinding. Nothing is added to the verifier and there is no enable/disable toggle — no cross-thread race under a parallel verify. Compiled out on the riscv64 guest. RECURSION_DUMP_PRESET= cargo test --release --features hash-metrics \ -p lambda-vm-prover --lib test_count_recursion_hashes -- --ignored --nocapture --- crypto/crypto/Cargo.toml | 5 +- crypto/crypto/src/hash/platform_keccak.rs | 57 ++++++- crypto/crypto/src/hash_metrics.rs | 143 ++++++++++++++++++ crypto/crypto/src/lib.rs | 1 + .../src/merkle_tree/backends/field_element.rs | 8 +- .../backends/field_element_vector.rs | 7 + crypto/stark/src/grinding.rs | 5 + prover/Cargo.toml | 2 + prover/src/tests/recursion_smoke_test.rs | 79 ++++++++++ 9 files changed, 304 insertions(+), 3 deletions(-) create mode 100644 crypto/crypto/src/hash_metrics.rs diff --git a/crypto/crypto/Cargo.toml b/crypto/crypto/Cargo.toml index 532d17e4b..a0bbd5f67 100644 --- a/crypto/crypto/Cargo.toml +++ b/crypto/crypto/Cargo.toml @@ -44,4 +44,7 @@ serde = ["dep:serde"] parallel = ["dep:rayon"] disk-spill = ["std", "dep:memmap2", "dep:tempfile", "dep:libc"] alloc = [] -rkyv = ["dep:rkyv", "math/rkyv"] \ No newline at end of file +rkyv = ["dep:rkyv", "math/rkyv"] +# Host-only diagnostic: count keccak finalizes during verify (see `hash_metrics`). +# Off by default → `PlatformKeccak256 = sha3::Keccak256`, provably unchanged. +hash-metrics = [] \ No newline at end of file diff --git a/crypto/crypto/src/hash/platform_keccak.rs b/crypto/crypto/src/hash/platform_keccak.rs index 3c3cb081e..4ce96a0e2 100644 --- a/crypto/crypto/src/hash/platform_keccak.rs +++ b/crypto/crypto/src/hash/platform_keccak.rs @@ -58,7 +58,62 @@ mod imp { } } -#[cfg(not(target_arch = "riscv64"))] +// Host, `hash-metrics` feature ON: `sha3::Keccak256` plus a finalize counter for +// [`crate::hash_metrics`]. The counter is a PURE SIDE EFFECT — every method +// forwards to the inner hasher (byte-identical digest) and is `#[inline(always)]`, +// so no cross-crate call is added over the bare alias. +#[cfg(all(not(target_arch = "riscv64"), feature = "hash-metrics"))] +mod imp { + use digest::{ + FixedOutput, FixedOutputReset, HashMarker, Output, OutputSizeUser, Reset, Update, + }; + + #[derive(Clone, Default)] + pub struct PlatformKeccak256(sha3::Keccak256); + + impl HashMarker for PlatformKeccak256 {} + + impl OutputSizeUser for PlatformKeccak256 { + type OutputSize = digest::typenum::U32; + } + + impl Update for PlatformKeccak256 { + #[inline(always)] + fn update(&mut self, data: &[u8]) { + // Absorption — the guest's dominant keccak cost (many small + // `stream_bytes` absorbs), which no finalize counter would see. + crate::hash_metrics::count_absorb(data.len()); + Update::update(&mut self.0, data); + } + } + + impl FixedOutput for PlatformKeccak256 { + #[inline(always)] + fn finalize_into(self, out: &mut Output) { + crate::hash_metrics::count_total(); + FixedOutput::finalize_into(self.0, out); + } + } + + impl Reset for PlatformKeccak256 { + #[inline(always)] + fn reset(&mut self) { + Reset::reset(&mut self.0); + } + } + + impl FixedOutputReset for PlatformKeccak256 { + #[inline(always)] + fn finalize_into_reset(&mut self, out: &mut Output) { + crate::hash_metrics::count_total(); + FixedOutputReset::finalize_into_reset(&mut self.0, out); + } + } +} + +// Default host build (no `hash-metrics` feature): the plain alias, provably +// unchanged from upstream. +#[cfg(all(not(target_arch = "riscv64"), not(feature = "hash-metrics")))] mod imp { pub type PlatformKeccak256 = sha3::Keccak256; } diff --git a/crypto/crypto/src/hash_metrics.rs b/crypto/crypto/src/hash_metrics.rs new file mode 100644 index 000000000..cc16e9348 --- /dev/null +++ b/crypto/crypto/src/hash_metrics.rs @@ -0,0 +1,143 @@ +//! Host-only keccak-hash counters for measuring the cost of VERIFYING a proof +//! (a proxy for the recursion guest's dominant work: keccak hashing). +//! +//! Behind the `hash-metrics` cargo feature: a normal build keeps +//! `PlatformKeccak256 = sha3::Keccak256` and every counter call compiles to +//! nothing, so the prover is provably unchanged. With the feature on (host only), +//! the host `PlatformKeccak256` wrapper counts, per keccak op: +//! * `total` — every finalize (leaf / node / transcript squeeze / program-id fold); +//! `merkle`/`merkle_nodes`/`grinding` split it (keccak-only, disjoint subsets); +//! * `absorb_calls` / `absorb_bytes` — every `Update::update` (ABSORPTION). This is +//! the guest's DOMINANT keccak cost — the many 8-byte `stream_bytes` absorbs in +//! opening verification, not the finalize — so it is the dimension a block- +//! absorption optimization moves. A finalize-only number would report such a +//! change as zero improvement; `absorb_*` is what makes it visible. +//! +//! No enable/disable toggle and nothing in the verifier: counting is always on +//! under the feature, and a measuring caller just [`reset`]s before the verify +//! and reads [`snapshot`] after. Grinding is separated by counter, not excluded +//! at a call site, so there is no cross-thread race under a parallel verify. + +/// Snapshot of the verify-hash counters (all zero without the `hash-metrics` +/// feature / on the guest). `total` is finalizes; `absorb_*` is absorption. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Counts { + /// Every keccak-256 finalize. + pub total: u64, + /// Merkle finalizes (keccak-guarded subset of `total`). + pub merkle: u64, + /// Merkle auth-path (parent) compressions (subset of `merkle`). + pub merkle_nodes: u64, + /// Grinding proof-of-work finalizes (subset of `total`). + pub grinding: u64, + /// Keccak absorb (`Update::update`) invocations. + pub absorb_calls: u64, + /// Bytes fed through absorb (`Sum of data.len()`). + pub absorb_bytes: u64, +} + +#[cfg(all(not(target_arch = "riscv64"), feature = "hash-metrics"))] +mod imp { + use super::Counts; + use core::sync::atomic::{AtomicU64, Ordering}; + + static TOTAL: AtomicU64 = AtomicU64::new(0); + static MERKLE: AtomicU64 = AtomicU64::new(0); + static MERKLE_NODES: AtomicU64 = AtomicU64::new(0); + static GRINDING: AtomicU64 = AtomicU64::new(0); + static ABSORB_CALLS: AtomicU64 = AtomicU64::new(0); + static ABSORB_BYTES: AtomicU64 = AtomicU64::new(0); + + /// Every keccak-256 finalize, from any site (host `PlatformKeccak256`). + #[inline(always)] + pub fn count_total() { + TOTAL.fetch_add(1, Ordering::Relaxed); + } + + /// A Merkle finalize (leaf or node), counted ONLY when the backend digest is + /// the platform keccak wrapper — the one whose `finalize` also bumps + /// [`count_total`]. This keeps `merkle` a strict subset of `total` for ANY + /// `D` (a non-keccak backend, as in the crypto tests, does not go through the + /// counted wrapper, so counting it here would let `merkle` exceed `total`). + #[inline(always)] + pub fn count_merkle() { + if core::any::TypeId::of::() + == core::any::TypeId::of::() + { + MERKLE.fetch_add(1, Ordering::Relaxed); + } + } + + /// A Merkle parent (auth-path) compression. Subset of [`count_merkle`]; + /// same keccak-only guard. + #[inline(always)] + pub fn count_merkle_node() { + if core::any::TypeId::of::() + == core::any::TypeId::of::() + { + MERKLE_NODES.fetch_add(1, Ordering::Relaxed); + } + } + + /// A grinding (proof-of-work) finalize. Subset of [`count_total`]; a caller + /// reports `total - grinding` to exclude the PoW check. + #[inline(always)] + pub fn count_grinding() { + GRINDING.fetch_add(1, Ordering::Relaxed); + } + + /// A keccak absorb (`Update::update`) of `nbytes` — the guest's dominant + /// keccak cost, and the dimension a block-absorption optimization moves + /// (finalizes do not change). Bumps the call count and the byte total. + #[inline(always)] + pub fn count_absorb(nbytes: usize) { + ABSORB_CALLS.fetch_add(1, Ordering::Relaxed); + ABSORB_BYTES.fetch_add(nbytes as u64, Ordering::Relaxed); + } + + /// Zero all counters. + pub fn reset() { + TOTAL.store(0, Ordering::Relaxed); + MERKLE.store(0, Ordering::Relaxed); + MERKLE_NODES.store(0, Ordering::Relaxed); + GRINDING.store(0, Ordering::Relaxed); + ABSORB_CALLS.store(0, Ordering::Relaxed); + ABSORB_BYTES.store(0, Ordering::Relaxed); + } + + pub fn snapshot() -> Counts { + Counts { + total: TOTAL.load(Ordering::Relaxed), + merkle: MERKLE.load(Ordering::Relaxed), + merkle_nodes: MERKLE_NODES.load(Ordering::Relaxed), + grinding: GRINDING.load(Ordering::Relaxed), + absorb_calls: ABSORB_CALLS.load(Ordering::Relaxed), + absorb_bytes: ABSORB_BYTES.load(Ordering::Relaxed), + } + } +} + +// Feature off, or the riscv64 guest: every entry compiles to nothing. +#[cfg(any(target_arch = "riscv64", not(feature = "hash-metrics")))] +mod imp { + use super::Counts; + + #[inline(always)] + pub fn count_total() {} + #[inline(always)] + pub fn count_merkle() {} + #[inline(always)] + pub fn count_merkle_node() {} + #[inline(always)] + pub fn count_grinding() {} + #[inline(always)] + pub fn count_absorb(_nbytes: usize) {} + pub fn reset() {} + pub fn snapshot() -> Counts { + Counts::default() + } +} + +pub use imp::{ + count_absorb, count_grinding, count_merkle, count_merkle_node, count_total, reset, snapshot, +}; diff --git a/crypto/crypto/src/lib.rs b/crypto/crypto/src/lib.rs index d7a273d62..a70ac977e 100644 --- a/crypto/crypto/src/lib.rs +++ b/crypto/crypto/src/lib.rs @@ -9,6 +9,7 @@ extern crate alloc; pub mod fiat_shamir; pub mod hash; +pub mod hash_metrics; pub mod merkle_tree; #[cfg(feature = "disk-spill")] pub mod mmap_util; diff --git a/crypto/crypto/src/merkle_tree/backends/field_element.rs b/crypto/crypto/src/merkle_tree/backends/field_element.rs index e8f106f5a..0a61f03f3 100644 --- a/crypto/crypto/src/merkle_tree/backends/field_element.rs +++ b/crypto/crypto/src/merkle_tree/backends/field_element.rs @@ -22,7 +22,7 @@ impl Default for FieldElementBackend IsMerkleTreeBackend +impl IsMerkleTreeBackend for FieldElementBackend where F: IsField, @@ -33,12 +33,18 @@ where type Data = FieldElement; fn hash_data(input: &FieldElement) -> [u8; NUM_BYTES] { + // Merkle leaf finalize (see `crate::hash_metrics`); counts only when `D` + // is the platform keccak (so `merkle ⊆ total`), no-op without the feature. + crate::hash_metrics::count_merkle::(); let mut hasher = D::new(); input.stream_bytes(&mut |b| hasher.update(b)); hasher.finalize().into() } fn hash_new_parent(left: &[u8; NUM_BYTES], right: &[u8; NUM_BYTES]) -> [u8; NUM_BYTES] { + // Merkle auth-path (node) compression; keccak-only guard, no-op without + // the feature. + crate::hash_metrics::count_merkle_node::(); let mut hasher = D::new(); hasher.update(left); hasher.update(right); diff --git a/crypto/crypto/src/merkle_tree/backends/field_element_vector.rs b/crypto/crypto/src/merkle_tree/backends/field_element_vector.rs index 6d0cc6491..ccd5b3f4d 100644 --- a/crypto/crypto/src/merkle_tree/backends/field_element_vector.rs +++ b/crypto/crypto/src/merkle_tree/backends/field_element_vector.rs @@ -43,6 +43,9 @@ use lambda_vm_syscalls::keccak::Keccak256 as SyscallKeccak256; fn hash_streamed( feed: impl Fn(&mut dyn FnMut(&[u8])), ) -> [u8; NUM_BYTES] { + // Metric: a Merkle finalize (leaf or node). Counts only when `D` is the + // platform keccak (so `merkle ⊆ total`); no-op on guest / without the feature. + crate::hash_metrics::count_merkle::(); #[cfg(target_arch = "riscv64")] if NUM_BYTES == 32 && TypeId::of::() == TypeId::of::() { let mut hasher = SyscallKeccak256::new(); @@ -75,6 +78,10 @@ fn hash_new_parent_bytes( left: &[u8; NUM_BYTES], right: &[u8; NUM_BYTES], ) -> [u8; NUM_BYTES] { + // Metric: a Merkle parent (auth-path) compression. On the host this also + // flows through `hash_streamed` (one `count_merkle`), so merkle − nodes = + // leaves. Keccak-only guard; no-op on guest / without the feature. + crate::hash_metrics::count_merkle_node::(); #[cfg(target_arch = "riscv64")] if NUM_BYTES == 32 && TypeId::of::() == TypeId::of::() { let l: &[u8; 32] = left[..].try_into().unwrap(); diff --git a/crypto/stark/src/grinding.rs b/crypto/stark/src/grinding.rs index adb7601b6..b3642c656 100644 --- a/crypto/stark/src/grinding.rs +++ b/crypto/stark/src/grinding.rs @@ -65,6 +65,9 @@ pub fn generate_nonce(seed: &[u8; 32], grinding_factor: u8) -> Option { /// when interpreted as `u64`. #[inline(always)] fn is_valid_nonce_for_inner_hash(inner_hash: &[u8; 32], candidate_nonce: u64, limit: u64) -> bool { + // Tag this finalize as grinding so a verify-hash metric can report it apart + // (see `crypto::hash_metrics`); no-op unless the `hash-metrics` feature is on. + crypto::hash_metrics::count_grinding(); let mut data = [0; 40]; data[..32].copy_from_slice(inner_hash); data[32..].copy_from_slice(&candidate_nonce.to_be_bytes()); @@ -79,6 +82,8 @@ fn is_valid_nonce_for_inner_hash(inner_hash: &[u8; 32], candidate_nonce: u64, li /// Hash(prefix || seed || grinding_factor) /// `prefix` is the bit-string `0x123456789abcded` fn get_inner_hash(seed: &[u8; 32], grinding_factor: u8) -> [u8; 32] { + // Grinding finalize (see `crypto::hash_metrics`); no-op unless enabled. + crypto::hash_metrics::count_grinding(); let mut inner_data = [0u8; 41]; inner_data[0..8].copy_from_slice(&PREFIX); inner_data[8..40].copy_from_slice(seed); diff --git a/prover/Cargo.toml b/prover/Cargo.toml index d4ebdeb0d..7401485cf 100644 --- a/prover/Cargo.toml +++ b/prover/Cargo.toml @@ -15,6 +15,8 @@ instruments = ["stark/instruments"] nvtx = ["cuda", "instruments", "stark/nvtx"] profile-markers = ["stark/profile-markers"] disk-spill = ["stark/disk-spill"] +# Host-only verify-hash counter for `test_count_recursion_hashes` (see `hash_metrics`). +hash-metrics = ["crypto/hash-metrics"] [dependencies] stark = { path = "../crypto/stark" } diff --git a/prover/src/tests/recursion_smoke_test.rs b/prover/src/tests/recursion_smoke_test.rs index 90482a3a4..81de50cf5 100644 --- a/prover/src/tests/recursion_smoke_test.rs +++ b/prover/src/tests/recursion_smoke_test.rs @@ -1052,6 +1052,85 @@ fn test_dump_recursion_input() { } } +/// Count the keccak hashes done to VERIFY the dumped recursion blob — a +/// prover-change metric: fewer hashes ⇒ a cheaper recursion guest. Runs the exact +/// guest verify (`verify_continuation_and_attest`) on `/tmp/recursion_input.bin` +/// (override with `RECURSION_INPUT_PATH`) with `crypto::hash_metrics` counting +/// every keccak-256 finalize, and reports the grinding proof-of-work hashes apart +/// so the headline `total` excludes them. +/// +/// Requires: +/// * the `hash-metrics` cargo feature — without it counting is a no-op (all zeros +/// → this test asserts and tells you to add the feature); +/// * a CONTINUATION dump: `test_dump_recursion_input` with `RECURSION_DUMP_EPOCH_LOG2` +/// set (this path verifies via `verify_continuation_and_attest`); +/// * `RECURSION_DUMP_PRESET` matching the dump (default `min`), else the verify fails. +/// +/// Loop: change the prover → re-run `test_dump_recursion_input` (re-proves + dumps +/// the new blob) → run this (fast, verify-only) → compare `total`. +/// +/// RECURSION_DUMP_PRESET=blowup4 cargo test --release --features hash-metrics \ +/// -p lambda-vm-prover --lib test_count_recursion_hashes -- --ignored --nocapture +#[test] +#[ignore = "diagnostic: counts keccak hashes verifying the dumped recursion blob"] +fn test_count_recursion_hashes() { + let preset_name = std::env::var("RECURSION_DUMP_PRESET").unwrap_or_else(|_| "min".to_string()); + let preset = Preset::ALL + .into_iter() + .find(|p| p.name() == preset_name) + .unwrap_or_else(|| panic!("unknown RECURSION_DUMP_PRESET '{preset_name}'")); + let path = std::env::var("RECURSION_INPUT_PATH") + .unwrap_or_else(|_| "/tmp/recursion_input.bin".to_string()); + let blob = std::fs::read(&path) + .unwrap_or_else(|e| panic!("read {path} (run test_dump_recursion_input first): {e}")); + + // Counting is always-on under the `hash-metrics` feature, so just zero the + // counters, verify, and read — no enable/disable, nothing in the verifier. + crypto::hash_metrics::reset(); + let attestation = recursion::verify_continuation_and_attest(&blob, &preset.options()).expect( + "verify_continuation_and_attest errored — needs a CONTINUATION dump \ + (RECURSION_DUMP_EPOCH_LOG2 set) under a matching RECURSION_DUMP_PRESET", + ); + let c = crypto::hash_metrics::snapshot(); + + assert!( + attestation.is_some(), + "the blob must verify under preset '{}' — does it match the dump's RECURSION_DUMP_PRESET?", + preset.name() + ); + assert!( + c.total > 0, + "hash counters are zero — build with `--features hash-metrics`" + ); + // `merkle` and `grinding` are disjoint subsets of `total`; `nodes` ⊆ `merkle`. + // (Holds for the keccak recursion verify; flags a backend/counter mismatch.) + assert!( + c.merkle_nodes <= c.merkle && c.merkle + c.grinding <= c.total, + "inconsistent counters: {c:?}" + ); + + // Two cost dimensions. FINALIZES: one keccak output per leaf/node/squeeze. + // ABSORPTION (`Update::update`): the guest's DOMINANT keccak cost — the many + // 8-byte `stream_bytes` absorbs in opening verification — which a block- + // absorption optimization moves and no finalize counter would ever see. + println!( + "[hash-count] preset={} blob={}B fri_queries={}\n \ + finalizes: total(excl. grinding)={} merkle={} (nodes={} leaves={}) \ + transcript+other={} grinding={}\n absorb: calls={} bytes={}", + preset.name(), + blob.len(), + preset.options().fri_number_of_queries, + c.total - c.grinding, + c.merkle, + c.merkle_nodes, + c.merkle - c.merkle_nodes, + c.total - c.merkle - c.grinding, + c.grinding, + c.absorb_calls, + c.absorb_bytes, + ); +} + /// Cycle count only of the recursion guest verifying a 1-query inner proof. #[test] #[ignore = "diagnostic: fast; recursion guest cycle count (1 query)"]