From 91c6d79fdcd136bd780accd57cfb671a3f91ed84 Mon Sep 17 00:00:00 2001 From: ael-dev3 <183124839+ael-dev3@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:41:40 +0100 Subject: [PATCH 1/4] fix(dau): verify Hypersnap coverage and expose navigation --- CHANGELOG.md | 2 + crates/allthing-core/src/activity.rs | 929 +++++++++++++++++- crates/allthing-core/src/provenance.rs | 64 ++ .../tests/fixtures/activity-shard-1.json | 113 +++ .../tests/fixtures/activity-shard-2.json | 56 ++ crates/allthing-data/src/activity.rs | 759 +++++++++++--- crates/allthing-data/src/hypersnap.rs | 295 +++++- crates/allthing-web/src/components/clients.rs | 6 + crates/allthing-web/src/components/layout.rs | 1 + crates/allthing-web/src/pages/dau.rs | 117 ++- crates/allthing-worker/src/index.rs | 207 ++-- docs/ARCHITECTURE.md | 50 +- docs/BUDGET.md | 28 +- e2e/allthing.spec.ts | 4 +- e2e/client-activity.fixture.spec.ts | 19 +- e2e/dau.fixture.spec.ts | 118 ++- e2e/presence.fixture.spec.ts | 64 +- e2e/static-navigation.fixture.spec.ts | 23 + migrations/0011_verified_activity.sql | 12 + style/allthing.css | 130 ++- tests/activity-sql.test.mjs | 212 ++++ web/boot.js | 12 +- web/index.html | 4 +- 23 files changed, 2888 insertions(+), 337 deletions(-) create mode 100644 crates/allthing-core/tests/fixtures/activity-shard-1.json create mode 100644 crates/allthing-core/tests/fixtures/activity-shard-2.json create mode 100644 migrations/0011_verified_activity.sql create mode 100644 tests/activity-sql.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fb91c32..3896c454 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to Allthing are recorded here. The format follows [Keep a Ch ### Added +- A persistent DAU navigation link on desktop and phones, with a coverage panel showing the Hypersnap source, snapshot update time, verified start and each shard's block progress. Incomplete totals are marked as lower bounds; responses without coherent verification evidence show a waiting explanation instead of figures. - Private, account-wide Mute and Block controls on profiles and hover cards, with searchable undo controls in Settings and keyboard/touch-accessible help. Preferences sync through SpacetimeDB and suppress posts, quoted/replied-to/recasting accounts, identifiable mentions and account links, notifications and their unread counts. Block also hides existing Allthing conversations locally and rejects new DMs in either direction using verified account ownership; public posts and other Farcaster clients are unaffected. - Durable offline mute/block intent, independent per-choice revisions and retained clear tombstones. Fresh signed-in sessions wait for private filters before drawing content; realtime changes remove already shown content, and newer remote choices supersede stale offline intent. - A private Library for bookmarks and seen history synced across signed-in devices through SpacetimeDB, plus paginated Farcaster likes and recasts. Seen history no longer expires after 30 days or 2,000 posts; fresh Home candidates wait for synced history, while already drawn cards keep their place. Library includes loaded-post filtering, activity times, retry, Back restoration, Mark unread and Clear history controls. @@ -20,6 +21,7 @@ All notable changes to Allthing are recorded here. The format follows [Keep a Ch ### Fixed +- Count active accounts from checked, consecutive Hypersnap events with canonical block times carried across pages, declared event totals and both shards' coverage. The new verification epoch keeps old rows without certifying their totals. Full account and 32-byte signing-key identities prevent prefix collisions; incomplete attribution stays unknown, and a fenced atomic save rejects stale concurrent runs. Action totals are labelled as merge events because the same message can appear on both shards. - Recover one confirmed missing entry-loader request without reloading or changing drafts and sessions; persistent failures show an accessible Reload button even if early Home data arrives later. Healthy startup adds no recovery traffic. - Serve HTML from the same deployment as its asset manifest with no-store caching; missing scripts and WebAssembly return real 404s instead of immutable HTML. A missing startup asset can recover once when a bounded check confirms a different same-origin build; other startup errors retain the manual Reload panel. diff --git a/crates/allthing-core/src/activity.rs b/crates/allthing-core/src/activity.rs index 90b25204..305db8a9 100644 --- a/crates/allthing-core/src/activity.rs +++ b/crates/allthing-core/src/activity.rs @@ -13,8 +13,8 @@ //! failed merges, onchain events and blocks don't; nor do message types //! the list doesn't name. //! - **When:** the time of the block that confirmed it (the block's own -//! `BLOCK_CONFIRMED` event, which comes first), or the message's signed -//! time when the block's event is on an earlier page. +//! `BLOCK_CONFIRMED` event, which comes first), carried across pages. +//! Coverage advances only after all declared block events were read. //! - **Who:** the message's fid, from 1 to 2³² − 1, and the key that signed //! it, which says which app it was sent from (`provenance`). //! @@ -34,9 +34,11 @@ use serde::de::IgnoredAny; use serde::{Deserialize, Serialize}; use crate::emoji::{Emoji, ReactionReply, reaction_text}; -use crate::latest::{Cursor, FARCASTER_EPOCH, base64_to_hex}; +use crate::latest::{Cursor, FARCASTER_EPOCH, SHARDS, base64_to_hex}; use crate::model::UserSummary; -use crate::provenance::{base64_decode, normalize_key, request_fid_from_metadata}; +#[cfg(test)] +use crate::provenance::{base64_decode, request_fid_from_metadata}; +use crate::provenance::{normalize_key, verified_request_fid_from_metadata}; /// What kind of action a counted message is. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -104,7 +106,10 @@ pub struct ActivityPage { pub count: usize, /// Signing keys added onchain on the page (`key_pair`), each with the /// app that requested it, 0 for none. + #[cfg(test)] pub keys: Vec<(u64, u64)>, + /// Canonically observed registrations, matched by every signer byte. + pub exact_keys: Vec<(String, u64)>, /// Emoji reactions merged on the page: replies that are one emoji and /// nothing else (`emoji::reaction`), each with the hash of the cast it /// replies to. @@ -126,6 +131,8 @@ struct Event { kind: Option, #[serde(rename = "blockNumber")] block: Option, + #[serde(rename = "shardIndex")] + shard: Option, #[serde(rename = "blockConfirmedBody")] confirmed: Option, #[serde(rename = "mergeMessageBody")] @@ -148,6 +155,8 @@ struct OnChain { #[derive(Deserialize)] struct ChainEvent { + #[serde(rename = "type")] + kind: Option, fid: Option, #[serde(rename = "signerEventBody")] signer: Option, @@ -155,6 +164,8 @@ struct ChainEvent { #[derive(Deserialize)] struct SignerEvent { + #[serde(rename = "keyType")] + key_type: Option, key: Option, #[serde(rename = "eventType")] kind: Option, @@ -167,7 +178,11 @@ struct SignerEvent { struct Confirmed { #[serde(rename = "blockNumber")] block: Option, + #[serde(rename = "shardIndex")] + shard: Option, timestamp: Option, + #[serde(rename = "totalEvents")] + total: Option, } #[derive(Deserialize)] @@ -190,6 +205,7 @@ struct Data { #[serde(rename = "type")] kind: Option, fid: Option, + #[cfg(test)] timestamp: Option, #[serde(rename = "castAddBody")] add: Option, @@ -274,7 +290,8 @@ fn unix(farcaster: i64) -> Option { /// Read a `/v1/events` page: its counted actions, where it ends, and how far /// in time it reaches. `None` if it isn't an events page. -pub fn activity(json: &[u8]) -> Option { +#[cfg(test)] +fn activity(json: &[u8]) -> Option { let page: Page = serde_json::from_slice(json).ok()?; let mut out = ActivityPage { count: page.events.len(), ..Default::default() }; // Each block's confirmation comes before its messages. @@ -322,7 +339,8 @@ pub fn activity(json: &[u8]) -> Option { // A key added for an account: which app requested it is in its // signed key request (`provenance`), so it needn't be looked up. Some("HUB_EVENT_TYPE_MERGE_ON_CHAIN_EVENT") => { - let Some(ChainEvent { fid: Some(fid), signer: Some(body) }) = event.chain.and_then(|c| c.event) else { + let Some(ChainEvent { fid: Some(fid), signer: Some(body), .. }) = event.chain.and_then(|c| c.event) + else { continue; }; if body.kind.as_deref() != Some("SIGNER_EVENT_TYPE_ADD") { @@ -345,6 +363,293 @@ pub fn activity(json: &[u8]) -> Option { Some(out) } +/// The isolated format of canonical activity counts. Older sets are preserved. +pub const VERIFICATION_VERSION: u8 = 2; +/// Canonical event source, with shard parameters recorded separately. +pub const ACTIVITY_SOURCE: &str = "https://haatz.quilibrium.com/v1/events"; + +/// One canonical block, carried across event pages. `seen` includes its +/// confirmation; only a whole block advances verified coverage. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct BlockProgress { + pub shard: u32, + pub block: u64, + pub at: i64, + pub total: u32, + pub seen: u32, +} + +impl BlockProgress { + pub fn complete(&self) -> bool { + self.seen == self.total + } + + pub fn last_id(&self) -> u64 { + (self.block << 14) | u64::from(self.seen - 1) + } +} + +/// A checked page and its durable per-shard progress. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct VerifiedPage { + pub page: ActivityPage, + pub block: Option, + pub gaps: Vec<(i64, i64)>, +} + +/// A one-event reverse probe establishes only the starting height; it +/// cannot establish temporal coverage or contribute a count. +pub fn activity_head(json: &[u8], shard: u32) -> Option { + let page: Page = serde_json::from_slice(json).ok()?; + if page.events.len() != 1 { + return None; + } + let event = page.events.first()?; + (event.shard == Some(shard) && event.block == Some(event.id >> 14)).then_some(event.id) +} + +/// Read ascending Hypersnap events after `after`. Reject a malformed page +/// as a whole, so its cursor remains retryable. Timestamps come exclusively +/// from canonical block confirmations, including a carried partial block. +pub fn verified_activity( + json: &[u8], + shard: u32, + after: u64, + carried: Option<&BlockProgress>, + now: i64, +) -> Option { + let mut input: Page = serde_json::from_slice(json).ok()?; + let mut previous = after; + let mut block = carried.cloned(); + if block.as_ref().is_some_and(|b| { + b.shard != shard || b.seen == 0 || b.seen > b.total || b.total > 16_384 || b.last_id() != after + }) { + return None; + } + let mut canonical = HashMap::new(); + if let Some(b) = &block { + canonical.insert(b.block, b.at); + } + let mut gaps = Vec::new(); + let mut reached = None; + let mut started = None; + for event in &mut input.events { + if !matches!( + event.kind.as_deref(), + Some( + "HUB_EVENT_TYPE_BLOCK_CONFIRMED" + | "HUB_EVENT_TYPE_MERGE_MESSAGE" + | "HUB_EVENT_TYPE_REVOKE_MESSAGE" + | "HUB_EVENT_TYPE_PRUNE_MESSAGE" + | "HUB_EVENT_TYPE_MERGE_ON_CHAIN_EVENT" + | "HUB_EVENT_TYPE_MERGE_USERNAME_PROOF" + | "HUB_EVENT_TYPE_MERGE_FAILURE" + | "HUB_EVENT_TYPE_CHANNEL_OWNER_CHANGE_HINT" + ) + ) { + return None; + } + let height = event.id >> 14; + let sequence = (event.id & 16_383) as u32; + if event.id <= previous || event.block != Some(height) || event.shard != Some(shard) { + return None; + } + let new_block = block.as_ref().is_none_or(|b| b.block != height); + if new_block { + if sequence != 0 || event.kind.as_deref() != Some("HUB_EVENT_TYPE_BLOCK_CONFIRMED") { + return None; + } + let confirmed = event.confirmed.as_ref()?; + let at = unix(confirmed.timestamp?)?; + let total = confirmed.total.filter(|n| (1..=16_384).contains(n))?; + if confirmed.block != Some(height) || confirmed.shard != Some(shard) || at > now { + return None; + } + if let Some(prior) = &block { + if height <= prior.block || at < prior.at { + return None; + } + if height != prior.block + 1 || !prior.complete() { + gaps.push((prior.at, at.saturating_add(1))); + } + } + canonical.insert(height, at); + started = Some(started.map_or(at, |s: i64| s.min(at))); + block = Some(BlockProgress { shard, block: height, at, total, seen: 1 }); + } else { + let current = block.as_mut()?; + if sequence != current.seen + || current.complete() + || event.kind.as_deref() == Some("HUB_EVENT_TYPE_BLOCK_CONFIRMED") + { + return None; + } + current.seen += 1; + } + let current = block.as_ref()?; + if current.complete() { + reached = Some(current.at - 1); + } + previous = event.id; + } + // Extraction uses the canonical time table, including the carried block; + // a user's signed timestamp never supplies a counting timestamp. + let mut out = ActivityPage { count: input.events.len(), ..ActivityPage::default() }; + for event in input.events { + out.last_id = Some(event.id); + out.first_block = out.first_block.or(event.block); + if event.kind.as_deref() == Some("HUB_EVENT_TYPE_MERGE_MESSAGE") { + let Some(Message { hash, data: Some(data), signer }) = event.merge.and_then(|m| m.message) else { + return None; + }; + let at = *canonical.get(&event.block?)?; + out.reactions.extend(reaction(hash.as_deref(), &data, Some(at))); + out.removed.extend(data.remove.as_ref().and_then(|r| r.target.as_deref()).and_then(hex_hash)); + let kind = data.kind.as_deref()?; + if kind != "MESSAGE_TYPE_NONE" + && kind != "MESSAGE_TYPE_SIGNER_ADD" + && kind != "MESSAGE_TYPE_SIGNER_REMOVE" + && Family::of(kind).is_none() + { + return None; + } + if let Some(family) = Family::of(kind) { + let fid = data.fid.filter(|fid| (1..=u64::from(u32::MAX)).contains(fid))?; + let signer = exact_signer(signer.as_deref()?)?; + out.actions.push(Action { fid, signer, at, family }); + } + } else if matches!( + event.kind.as_deref(), + Some("HUB_EVENT_TYPE_REVOKE_MESSAGE" | "HUB_EVENT_TYPE_PRUNE_MESSAGE") + ) { + if let Some(Message { hash: Some(hash), data: Some(data), .. }) = + event.revoke.or(event.prune).and_then(|m| m.message) + && reaction(Some(&hash), &data, Some(0)).is_some() + { + out.removed.extend(hex_hash(&hash)); + } + } else if event.kind.as_deref() == Some("HUB_EVENT_TYPE_MERGE_ON_CHAIN_EVENT") + && let Some(ChainEvent { kind: Some(kind), fid: Some(fid), signer: Some(body) }) = + event.chain.and_then(|c| c.event) + && kind == "EVENT_TYPE_SIGNER" + && body.key_type == Some(1) + && body.kind.as_deref() == Some("SIGNER_EVENT_TYPE_ADD") + && let Some(identity) = body.key.as_deref().and_then(|key| signer_identity(fid, key)) + { + let app = match (body.metadata_type, body.metadata.as_deref()) { + (Some(0), Some("")) => Some(0), + (Some(1), Some(metadata)) => verified_request_fid_from_metadata(metadata), + _ => None, + }; + if let Some(app) = app { + out.exact_keys.push((identity, app)); + } + } + // Missing or malformed registration metadata remains unknown. + } + out.reached = reached; + out.started = started; + Some(VerifiedPage { page: out, block, gaps }) +} + +/// An Ed25519 key, with every byte retained. +pub fn exact_signer(key: &str) -> Option { + if key.len() == 66 + && key.starts_with("0x") + && key.as_bytes()[2..].iter().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(b)) + { + Some(key.to_owned()) + } else { + normalize_key(key).filter(|key| key.len() == 66) + } +} + +/// A full signer identity. Its fixed-width decimal prefix orders accounts +/// together without truncating either FIDs or keys. +pub fn signer_identity(fid: u64, key: &str) -> Option { + if !(1..=u64::from(u32::MAX)).contains(&fid) { + return None; + } + Some(format!("{fid:010}:{}", exact_signer(key)?)) +} + +pub fn identity_fid(identity: &str) -> Option { + let bytes = identity.as_bytes(); + if bytes.len() != 77 + || &bytes[10..13] != b":0x" + || !bytes[13..].iter().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(b)) + { + return None; + } + let mut fid = 0_u32; + for &digit in &bytes[..10] { + if !digit.is_ascii_digit() { + return None; + } + fid = fid.checked_mul(10)?.checked_add(u32::from(digit - b'0'))?; + } + (fid > 0).then_some(u64::from(fid)) +} + +/// Sorted, complete signer identities, serialized without integer or key +/// prefix loss. Old compact key sets are never read in this namespace. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ExactKeys(Vec); + +impl ExactKeys { + pub fn of(keys: impl IntoIterator) -> Self { + let mut keys: Vec<_> = keys.into_iter().collect(); + keys.sort_unstable(); + keys.dedup(); + Self(keys) + } + pub fn as_slice(&self) -> &[String] { + &self.0 + } + pub fn extend(&mut self, keys: &Self) { + let mut merged = Vec::with_capacity(self.0.len() + keys.0.len()); + let mut own = std::mem::take(&mut self.0).into_iter().peekable(); + let mut other = keys.0.iter().peekable(); + while let (Some(a), Some(b)) = (own.peek(), other.peek()) { + match a.cmp(b) { + std::cmp::Ordering::Less => merged.push(own.next().expect("peeked")), + std::cmp::Ordering::Greater => merged.push(other.next().expect("peeked").clone()), + std::cmp::Ordering::Equal => { + merged.push(own.next().expect("peeked")); + other.next(); + } + } + } + merged.extend(own); + merged.extend(other.cloned()); + self.0 = merged; + } + pub fn encode(&self) -> String { + serde_json::to_string(&self.0).expect("string array") + } + pub fn decode(text: &str) -> Option { + let keys: Vec = serde_json::from_str(text).ok()?; + if !keys.iter().all(|k| identity_fid(k).is_some()) || !keys.windows(2).all(|k| k[0] < k[1]) { + return None; + } + Some(Self(keys)) + } + pub fn chunks(&self, size: usize) -> Vec<&[String]> { + let mut chunks = Vec::new(); + let mut rest = self.0.as_slice(); + while !rest.is_empty() { + let mut end = size.max(1).min(rest.len()); + while end < rest.len() && identity_fid(&rest[end]) == identity_fid(&rest[end - 1]) { + end += 1; + } + let (chunk, after) = rest.split_at(end); + chunks.push(chunk); + rest = after; + } + chunks + } +} + /// The UTC day of Unix time `at`, as days since 1970-01-01. pub fn day(at: i64) -> i64 { at.div_euclid(86_400) @@ -506,19 +811,21 @@ impl Fids { /// From storage; `None` if `text` isn't a stored set of fids. pub fn decode(text: &str) -> Option { - decode_gaps(text)?.into_iter().map(|v| u32::try_from(v).ok()).collect::>().map(Fids) + let values: Vec = decode_gaps(text)?.into_iter().map(|v| u32::try_from(v).ok()).collect::>()?; + let set = Fids(values); + (set.0.first().is_none_or(|fid| *fid > 0) && set.encode() == text).then_some(set) } /// How many distinct fids `sets` hold between them. pub fn union_len<'a>(sets: impl IntoIterator) -> usize { let sets: Vec<&Fids> = sets.into_iter().collect(); - let Some(max) = sets.iter().filter_map(|s| s.0.last()).max() else { return 0 }; - // A bitmap: about 425 KB for today's 3.4 million fids, and linear. - let mut seen = vec![0u64; *max as usize / 64 + 1]; - for fid in sets.iter().flat_map(|s| s.0.iter()) { - seen[*fid as usize / 64] |= 1 << (fid % 64); + let mut values = Vec::with_capacity(sets.iter().map(|set| set.len()).sum()); + for set in sets { + values.extend_from_slice(&set.0); } - seen.iter().map(|w| w.count_ones() as usize).sum() + values.sort_unstable(); + values.dedup(); + values.len() } } @@ -620,13 +927,13 @@ pub const KEEP_DAYS: i64 = 30; /// Days of counts it keeps. pub const HISTORY_DAYS: usize = 60; -/// The name a set is stored under: `h` or `d`. +/// A canonical set name, isolated from preserved legacy sets. pub fn hour_set(hour: i64) -> String { - format!("h{hour}") + format!("v2h{hour}") } pub fn day_set(day: i64) -> String { - format!("d{day}") + format!("v2d{day}") } /// Days of the union the cron keeps once a day for a list to ask who of @@ -637,7 +944,7 @@ pub const WEEK_DAYS: i64 = 7; /// stored under: `w`. The cron writes it only once those days were /// read in full (`Follow::covered`). pub fn week_set(day: i64) -> String { - format!("w{day}") + format!("v2w{day}") } /// Which of some accounts were active in the `WEEK_DAYS` days to the end of @@ -658,7 +965,7 @@ impl ActiveWeek { /// The name an hour's set of active keys is stored under: `k`. pub fn key_set(hour: i64) -> String { - format!("k{hour}") + format!("v2k{hour}") } /// One UTC day's counts. @@ -686,7 +993,17 @@ pub struct Reached { /// The follower's saved state. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Follow { + /// Canonical verification format, isolated from legacy counts. + #[serde(default)] + pub verification_version: u8, + #[serde(default)] + pub revision: u64, + #[serde(default)] + pub starts: Vec, + #[serde(default)] + pub blocks: Vec, /// Where each shard's events have been read to. #[serde(default)] pub cursors: Vec, @@ -715,7 +1032,7 @@ pub struct Follow { /// Active keys (`key_pair`) whose app the last count didn't know, to /// look up. #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub backlog: Vec, + pub backlog: Vec, /// The accounts of the apps with the most active accounts, read once a /// day, on `named`. #[serde(default, skip_serializing_if = "Vec::is_empty")] @@ -740,6 +1057,9 @@ pub struct AppActive { /// counts for each. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct AppsWindow { + /// Only the canonical full-key counter sets this format marker. + #[serde(default)] + pub verification_version: u8, /// The window's end, Unix seconds, on the hour. pub to: i64, /// When it was counted, Unix seconds. @@ -780,6 +1100,20 @@ impl AppsWindow { window } + /// A canonical full-key window with coherent bounds and counts. Old + /// cached Home payloads default to version zero and must not be shown. + pub fn is_verified(&self) -> bool { + self.verification_version == VERIFICATION_VERSION + && self.to >= FARCASTER_EPOCH + && self.to.rem_euclid(3_600) == 0 + && self.counted >= self.to + && self.active <= u64::from(u32::MAX) + && self.unknown <= self.active + && self.none <= self.active + && self.apps.iter().all(|app| app.app > 0 && app.active <= self.active) + && self.apps.iter().enumerate().all(|(i, app)| !self.apps[..i].iter().any(|prior| prior.app == app.app)) + } + /// Accounts active through `app`. pub fn through(&self, app: u64) -> u64 { self.apps.iter().find(|a| a.app == app).map_or(0, |a| a.active) @@ -806,25 +1140,104 @@ pub struct Window { } impl Follow { + /// Structural checks for this isolated state format. A corrupt state + /// must be repaired explicitly, never silently reset or certified. + pub fn valid(&self, now: i64) -> bool { + let unique = |shards: Vec| { + shards.iter().all(|s| SHARDS.contains(s)) + && shards.iter().enumerate().all(|(i, s)| !shards[..i].contains(s)) + }; + self.verification_version == VERIFICATION_VERSION + && unique(self.cursors.iter().map(|s| s.shard).collect()) + && unique(self.reached.iter().map(|s| s.shard).collect()) + && unique(self.starts.iter().map(|s| s.shard).collect()) + && unique(self.blocks.iter().map(|s| s.shard).collect()) + && self.cursors.iter().all(|c| { + self.blocks.iter().any(|b| { + b.shard == c.shard + && b.seen > 0 + && b.seen <= b.total + && b.total <= 16_384 + && b.block == c.event_id >> 14 + && b.last_id() == c.event_id + }) + }) + && self.blocks.iter().all(|b| { + b.at >= FARCASTER_EPOCH + && b.at <= now + && self.cursors.iter().any(|c| c.shard == b.shard) + && self.starts.iter().any(|s| s.shard == b.shard && s.at <= b.at + 1) + && (!b.complete() || self.reached.iter().any(|r| r.shard == b.shard && r.at == b.at - 1)) + }) + && self.reached.iter().all(|r| { + r.at >= FARCASTER_EPOCH && self.blocks.iter().any(|b| b.shard == r.shard && r.at < b.at) && r.at <= now + }) + && self + .starts + .iter() + .all(|s| s.at >= FARCASTER_EPOCH && s.at <= now + 1 && self.blocks.iter().any(|b| b.shard == s.shard)) + && self.since + == SHARDS + .into_iter() + .map(|shard| self.starts.iter().find(|s| s.shard == shard).map(|s| s.at)) + .collect::>>() + .and_then(|starts| starts.into_iter().max()) + && self.keys_since == self.since + && self.gaps.iter().all(|&(from, to)| from >= FARCASTER_EPOCH && from < to && to <= now + 1) + && self.days.windows(2).all(|d| d[0].day < d[1].day) + && self.days.iter().all(|d| { + (day(FARCASTER_EPOCH)..=day(now)).contains(&d.day) + && d.active <= u64::from(u32::MAX) + && (!d.complete || self.covered(d.day * 86_400, (d.day + 1) * 86_400)) + }) + && self.active_30d.is_none_or(|w| { + (day(FARCASTER_EPOCH)..day(now)).contains(&w.day) + && w.active <= u64::from(u32::MAX) + && self.since.is_some_and(|since| since <= w.day * 86_400) + && self.reached().is_some_and(|reached| reached >= (w.day + 1) * 86_400) + && (!w.complete || self.covered((w.day - 29) * 86_400, (w.day + 1) * 86_400)) + }) + && self.backlog.iter().all(|k| identity_fid(k).is_some()) + && self + .apps_24h + .as_ref() + .is_none_or(|w| w.is_verified() && w.counted <= now && self.covered_through(w.to - 86_400, w.to)) + } + /// How far every shard has been read, Unix seconds: the least of them. pub fn reached(&self) -> Option { - self.reached.iter().map(|r| r.at).min() + SHARDS + .into_iter() + .map(|shard| self.reached.iter().find(|r| r.shard == shard).map(|r| r.at)) + .collect::>>()? + .into_iter() + .min() } /// Whether `[from, to)` was read throughout: after counting began, with /// no gap in it, and every shard past its end. pub fn covered(&self, from: i64, to: i64) -> bool { - self.since.is_some_and(|since| since <= from) + self.verification_version == VERIFICATION_VERSION + && SHARDS + .into_iter() + .all(|shard| self.starts.iter().find(|s| s.shard == shard).is_some_and(|s| s.at <= from)) + && self.since.is_some_and(|since| since <= from) && self.reached().is_some_and(|reached| reached >= to) && !self.gaps.iter().any(|&(a, b)| a < to && b > from) } + /// Hourly and rolling windows include their right edge, unlike UTC + /// calendar days. A missing event at `to` invalidates their count. + pub fn covered_through(&self, from: i64, to: i64) -> bool { + self.covered(from, to) && !self.gaps.iter().any(|&(a, b)| a <= to && b > from) + } + /// Whether accounts by app are due to be counted for the window to `to` /// at `now`: its keys were all kept, and it is a window not counted /// yet, or, while more than 1% of accounts' apps aren't known, the last /// count is 10 minutes old (keys are looked up a few at a time). pub fn apps_due(&self, to: i64, now: i64) -> bool { - let kept = self.keys_since.is_some_and(|since| since <= to - 86_400) && self.covered(to - 86_400, to); + let kept = self.keys_since.is_some_and(|since| since <= to - 86_400) && self.covered_through(to - 86_400, to); kept && self.apps_24h.as_ref().is_none_or(|w| w.to < to || (w.known_percent() < 99 && now - w.counted >= 600)) } @@ -859,7 +1272,9 @@ pub struct Batch { pub days: HashMap, pub actions: HashMap>, /// The keys that signed, by hour. + #[cfg(test)] pub keys: HashMap, + pub exact_keys: HashMap, } impl Batch { @@ -867,8 +1282,12 @@ impl Batch { let mut hours: HashMap> = HashMap::new(); let mut days: HashMap> = HashMap::new(); let mut counts: HashMap> = HashMap::new(); + #[cfg(test)] let mut keys: HashMap> = HashMap::new(); + let mut exact_keys: HashMap> = HashMap::new(); for a in actions { + exact_keys.entry(bucket(a.at)).or_default().extend(signer_identity(a.fid, &a.signer)); + #[cfg(test)] keys.entry(bucket(a.at)).or_default().extend(key_pair(a.fid, &a.signer)); hours.entry(bucket(a.at)).or_default().push(a.fid); days.entry(day(a.at)).or_default().push(a.fid); @@ -878,15 +1297,39 @@ impl Batch { hours: hours.into_iter().map(|(h, f)| (h, Fids::of(f))).collect(), days: days.into_iter().map(|(d, f)| (d, Fids::of(f))).collect(), actions: counts, + #[cfg(test)] keys: keys.into_iter().map(|(h, k)| (h, Keys::of(k))).collect(), + exact_keys: exact_keys.into_iter().map(|(h, k)| (h, ExactKeys::of(k))).collect(), } } } +/// The evidence behind one shard's canonical coverage. A partial block is +/// exposed separately from the time through which counting is complete. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ShardVerification { + pub shard: u32, + pub since: Option, + pub reached: Option, + pub cursor: Option, + pub block: Option, + pub block_at: Option, + pub total_events: Option, + pub read_events: Option, +} + /// What `/api/v1/activity` answers: Farcaster's active accounts, as the /// cron last counted them. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct ActivityPayload { + #[serde(default)] + pub verification_version: u8, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub verified_since: Option, + #[serde(default)] + pub source: String, + #[serde(default)] + pub shards: Vec, /// How far the events had been read, Unix seconds. #[serde(default, skip_serializing_if = "Option::is_none")] pub reached: Option, @@ -925,12 +1368,126 @@ pub struct ActivityPayload { pub meta: crate::api::Meta, } +impl ActivityPayload { + /// Validate source-observed canonical evidence, including partial first + /// runs. This does not attest independent consensus or reorg detection. + /// A valid epoch is distinct from any particular window being complete. + pub fn is_verified(&self, now: i64) -> bool { + if self.verification_version != VERIFICATION_VERSION + || self.source != ACTIVITY_SOURCE + || self.meta.backend != crate::provenance::Backend::Hypersnap + || self.meta.fetched_at.div_euclid(1_000) > now + || self.meta.fetched_at < FARCASTER_EPOCH * 1_000 + || self.shards.len() != SHARDS.len() + || !SHARDS.into_iter().all(|shard| self.shards.iter().filter(|s| s.shard == shard).count() == 1) + { + return false; + } + for shard in &self.shards { + match (shard.since, shard.block, shard.block_at, shard.total_events, shard.read_events, shard.cursor) { + (None, None, None, None, None, None) if shard.reached.is_none() => {} + (Some(since), Some(block), Some(at), Some(total), Some(read), Some(cursor)) => { + if !(1..=16_384).contains(&total) + || read == 0 + || read > total + || block != cursor >> 14 + || u64::from(read - 1) != cursor & 16_383 + || at < FARCASTER_EPOCH + || at > self.meta.fetched_at.div_euclid(1_000) + || since < FARCASTER_EPOCH + || since > at + 1 + || shard.reached.is_some_and(|reached| reached < FARCASTER_EPOCH || reached >= at) + || (read == total && shard.reached != Some(at - 1)) + { + return false; + } + } + _ => return false, + } + } + let since = self.shards.iter().map(|s| s.since).collect::>>().and_then(|s| s.into_iter().max()); + let reached = + self.shards.iter().map(|s| s.reached).collect::>>().and_then(|s| s.into_iter().min()); + if self.verified_since != since + || self.since != since + || self.reached != reached + || self.rolling_to != reached.map(rolling_to) + || self.gaps.iter().any(|&(from, to)| from < FARCASTER_EPOCH || from >= to || to > now + 1) + || !self.days.windows(2).all(|d| d[0].day < d[1].day) + || self.days.iter().any(|d| { + d.day < day(FARCASTER_EPOCH) - 1 + || d.day > day(now) + || d.active > u64::from(u32::MAX) + || (d.complete && !self.covers(d.day * 86_400, (d.day + 1) * 86_400)) + }) + { + return false; + } + if self.rolling_24h.is_some() && !self.rolling_to.is_some_and(|to| self.covers_through(to - 86_400, to)) + || self.previous_24h.is_some() + && !self.rolling_to.is_some_and(|to| self.covers_through(to - 2 * 86_400, to - 86_400)) + || self.rolling_24h.is_some_and(|n| n > u64::from(u32::MAX)) + || self.previous_24h.is_some_and(|n| n > u64::from(u32::MAX)) + { + return false; + } + if !self.hourly.is_empty() { + let Some(to) = self.rolling_to else { + return false; + }; + if self.hourly.len() != HOURLY as usize + || self.hourly.iter().enumerate().any(|(i, n)| { + let end = to - (HOURLY - 1 - i as i64) * 3_600; + n.is_some_and(|count| count > u64::from(u32::MAX) || !self.covers_through(end - 3_600, end)) + }) + { + return false; + } + } + if let Some(window) = self.active_30d + && (window.day < day(FARCASTER_EPOCH) + || window.day >= day(now) + || window.active > u64::from(u32::MAX) + || self.verified_since.is_none() + || !self.reached.is_some_and(|reached| reached >= (window.day + 1) * 86_400) + || (window.complete && !self.covers((window.day - 29) * 86_400, (window.day + 1) * 86_400))) + { + return false; + } + if let Some(window) = &self.apps_24h + && (!window.is_verified() + || window.counted > self.meta.fetched_at.div_euclid(1_000) + || !self.covers_through(window.to - 86_400, window.to) + || (self.rolling_to == Some(window.to) && self.rolling_24h != Some(window.active))) + { + return false; + } + true + } + + /// Right-closed coverage for `(from,to]` activity windows. + pub fn covers_through(&self, from: i64, to: i64) -> bool { + self.covers(from, to) && !self.gaps.iter().any(|&(a, b)| a <= to && b > from) + } + + /// Whether the proof includes this complete interval. Call + /// `is_verified` first when consuming untrusted JSON. + pub fn covers(&self, from: i64, to: i64) -> bool { + from <= to + && self.verified_since.is_some_and(|since| since <= from) + && self.reached.is_some_and(|reached| reached >= to) + && !self.gaps.iter().any(|&(a, b)| a < to && b > from) + } +} + /// Accounts active in the 24 hours to `to` (on the hour), `(to − 24 h, to]`, /// if all of them were read. `hours` holds their sets (see `bucket`); an /// hour with no set had no activity. pub fn rolling(follow: &Follow, hours: &HashMap, to: i64) -> Option { let last = hour(to) - 1; - follow.covered(to - 86_400, to).then(|| Fids::union_len(((last - 23)..=last).filter_map(|h| hours.get(&h))) as u64) + follow + .covered_through(to - 86_400, to) + .then(|| Fids::union_len(((last - 23)..=last).filter_map(|h| hours.get(&h))) as u64) } /// The end of the 24-hour windows when events have been read to `reached`: @@ -949,7 +1506,9 @@ pub const HOURLY: i64 = 48; pub fn hourly(follow: &Follow, hours: &HashMap, to: i64) -> Vec> { let last = hour(to) - 1; let counts: Vec> = ((last - HOURLY + 1)..=last) - .map(|h| follow.covered(h * 3_600, (h + 1) * 3_600).then(|| hours.get(&h).map_or(0, |set| set.len() as u64))) + .map(|h| { + follow.covered_through(h * 3_600, (h + 1) * 3_600).then(|| hours.get(&h).map_or(0, |set| set.len() as u64)) + }) .collect(); if counts.iter().any(Option::is_some) { counts } else { Vec::new() } } @@ -995,6 +1554,25 @@ pub fn summary(follow: &Follow, hours: &HashMap, meta: crate::api::Me let reached = follow.reached(); let to = reached.map(rolling_to); ActivityPayload { + verification_version: follow.verification_version, + verified_since: follow.since, + source: ACTIVITY_SOURCE.into(), + shards: SHARDS + .into_iter() + .map(|shard| { + let block = follow.blocks.iter().find(|b| b.shard == shard); + ShardVerification { + shard, + since: follow.starts.iter().find(|s| s.shard == shard).map(|s| s.at), + reached: follow.reached.iter().find(|s| s.shard == shard).map(|s| s.at), + cursor: follow.cursors.iter().find(|s| s.shard == shard).map(|s| s.event_id), + block: block.map(|b| b.block), + block_at: block.map(|b| b.at), + total_events: block.map(|b| b.total), + read_events: block.map(|b| b.seen), + } + }) + .collect(), reached, rolling_to: to, rolling_24h: to.and_then(|t| rolling(follow, hours, t)), @@ -1171,7 +1749,7 @@ mod tests { #[test] fn a_week_says_who_of_those_asked_was_active_in_the_order_asked() { - assert_eq!(week_set(20_385), "w20385"); + assert_eq!(week_set(20_385), "v2w20385"); let week = Fids::of([3, 9, 12]); assert_eq!( ActiveWeek::among(20_385, &week, &[12, 4, 3, 3]), @@ -1210,6 +1788,8 @@ mod tests { fn follow(since: i64, reached: i64) -> Follow { Follow { + verification_version: VERIFICATION_VERSION, + starts: vec![Reached { shard: 1, at: since }, Reached { shard: 2, at: since }], reached: vec![Reached { shard: 1, at: reached }, Reached { shard: 2, at: reached + 30 }], since: Some(since), ..Follow::default() @@ -1461,4 +2041,303 @@ mod tests { assert_eq!(day_name(day(1_792_454_399)), "2026-10-19"); assert_eq!(day_name(day(951_782_400)), "2000-02-29"); } + fn confirmation(shard: u32, height: u64, at: i64, total: u32) -> serde_json::Value { + serde_json::json!({"type":"HUB_EVENT_TYPE_BLOCK_CONFIRMED","id":height << 14,"blockNumber":height,"shardIndex":shard, + "blockConfirmedBody":{"blockNumber":height,"shardIndex":shard,"timestamp":at-FARCASTER_EPOCH,"totalEvents":total}}) + } + + fn canonical_merge(shard: u32, height: u64, seq: u64, signed_at: i64) -> serde_json::Value { + serde_json::json!({"type":"HUB_EVENT_TYPE_MERGE_MESSAGE","id":(height << 14)|seq,"blockNumber":height,"shardIndex":shard, + "mergeMessageBody":{"message":{"signer":format!("0x{:064x}", 1),"data":{"type":"MESSAGE_TYPE_CAST_ADD","fid":7,"timestamp":signed_at-FARCASTER_EPOCH}}}}) + } + + fn canonical_page(events: Vec) -> Vec { + serde_json::to_vec(&serde_json::json!({"events":events})).unwrap() + } + + #[test] + fn canonical_block_time_survives_page_splits_and_equal_timestamp_blocks() { + let at = 1_800_000_000; + let first = verified_activity( + &canonical_page(vec![confirmation(1, 10, at, 3), canonical_merge(1, 10, 1, at - 90_000)]), + 1, + (10 << 14) - 1, + None, + at + 10, + ) + .unwrap(); + assert_eq!(first.page.reached, None, "a partial block is not complete coverage"); + assert_eq!(first.page.actions[0].at, at); + let second = verified_activity( + &canonical_page(vec![canonical_merge(1, 10, 2, at + 90_000)]), + 1, + (10 << 14) | 1, + first.block.as_ref(), + at + 10, + ) + .unwrap(); + assert_eq!(second.page.actions[0].at, at, "the user's signed time cannot move activity"); + assert_eq!(second.page.reached, Some(at - 1)); + let equal = verified_activity( + &canonical_page(vec![confirmation(1, 11, at, 1)]), + 1, + (10 << 14) | 2, + second.block.as_ref(), + at + 10, + ) + .unwrap(); + assert_eq!(equal.page.reached, Some(at - 1), "later blocks can share a second"); + let later = verified_activity( + &canonical_page(vec![confirmation(1, 12, at + 1, 1)]), + 1, + 11 << 14, + equal.block.as_ref(), + at + 10, + ) + .unwrap(); + assert_eq!(later.page.reached, Some(at)); + } + + #[test] + fn canonical_pages_reject_wrong_shards_duplicates_missing_sequences_and_future_blocks() { + let at = 1_800_000_000; + for events in [ + vec![confirmation(2, 10, at, 1)], + vec![confirmation(1, 10, at + 1, 1)], + vec![confirmation(1, 10, at, 3), canonical_merge(1, 10, 2, at)], + vec![confirmation(1, 10, at, 3), canonical_merge(1, 10, 1, at), canonical_merge(1, 10, 1, at)], + ] { + assert!(verified_activity(&canonical_page(events), 1, (10 << 14) - 1, None, at).is_none()); + } + let mut wrong_block = confirmation(1, 10, at, 1); + wrong_block["blockConfirmedBody"]["blockNumber"] = 11.into(); + assert!(verified_activity(&canonical_page(vec![wrong_block]), 1, (10 << 14) - 1, None, at).is_none()); + assert!( + verified_activity(&canonical_page(vec![canonical_merge(1, 10, 1, at)]), 1, 10 << 14, None, at).is_none(), + "no canonical block time means no verified count" + ); + } + + #[test] + fn retention_jumps_leave_a_gap_including_equal_timestamp_edges() { + let at = 1_800_000_000; + let carried = BlockProgress { shard: 1, block: 10, at, total: 2, seen: 1 }; + let next = verified_activity( + &canonical_page(vec![confirmation(1, 12, at + 60, 1)]), + 1, + 10 << 14, + Some(&carried), + at + 60, + ) + .unwrap(); + assert_eq!(next.gaps, vec![(at, at + 61)]); + } + + #[test] + fn coverage_requires_both_shards_and_the_later_shards_start() { + let mut state = follow(1_000, 10_000); + state.starts[1].at = 2_000; + state.since = Some(2_000); + assert!(!state.covered(1_000, 3_000)); + assert!(state.covered(2_000, 3_000)); + state.reached.pop(); + assert_eq!(state.reached(), None); + assert!(!state.covered(2_000, 3_000)); + } + + #[test] + fn exact_signers_preserve_colliding_prefixes_and_all_u32_fids() { + let a = signer_identity(u64::from(u32::MAX), &format!("0x12345678{:056x}", 1)).unwrap(); + let b = signer_identity(u64::from(u32::MAX), &format!("0x12345678{:056x}", 2)).unwrap(); + let keys = ExactKeys::of([a.clone(), b.clone(), a]); + assert_eq!(keys.as_slice().len(), 2); + assert_eq!(ExactKeys::decode(&keys.encode()), Some(keys.clone())); + assert_eq!(identity_fid(&b), Some(u64::from(u32::MAX))); + assert_eq!(keys.chunks(1).len(), 1, "an account is never split between count statements"); + assert!(signer_identity(42, "0x01").is_none()); + assert!(ExactKeys::decode(r#"["0000000042:0x01"]"#).is_none()); + } + #[test] + fn cached_app_windows_need_the_full_key_verification_marker() { + let to = hour(1_800_000_000) * 3_600; + let mut window = AppsWindow::of(to, to + 60, 2, [(Some(9152), 1), (None, 1)]); + assert!(!window.is_verified()); + window.verification_version = VERIFICATION_VERSION; + assert!(window.is_verified()); + window.apps.push(AppActive { app: 9152, active: 1 }); + assert!(!window.is_verified(), "duplicate app rows cannot inflate a count"); + } + #[test] + fn payload_proof_distinguishes_a_partial_epoch_from_invented_complete_counts() { + let at = 1_800_000_000; + let state = Follow { verification_version: VERIFICATION_VERSION, ..Follow::default() }; + let mut payload = summary( + &state, + &HashMap::new(), + crate::api::Meta { fetched_at: at * 1_000, ..crate::api::Meta::default() }, + ); + assert!(payload.is_verified(at), "two explicit waiting shards form a valid incomplete epoch"); + payload.rolling_24h = Some(0); + assert!(!payload.is_verified(at), "missing coverage cannot become a certified zero"); + payload.rolling_24h = None; + payload.shards.pop(); + assert!(!payload.is_verified(at)); + } + + #[test] + fn payload_proof_rejects_unsupported_sources_future_progress_and_false_complete_days() { + let at = 1_800_000_000; + let since = at - 3 * 86_400; + let mut state = follow(since, at - 1); + state.keys_since = state.since; + state.blocks = + SHARDS.into_iter().map(|shard| BlockProgress { shard, block: 10, at, total: 1, seen: 1 }).collect(); + state.cursors = SHARDS.into_iter().map(|shard| Cursor { shard, event_id: 10 << 14 }).collect(); + // The core follow helper's second shard is thirty seconds ahead; + // this proof has both shards at the same canonical checkpoint. + state.reached.iter_mut().for_each(|r| r.at = at - 1); + let payload = summary( + &state, + &HashMap::new(), + crate::api::Meta { fetched_at: at * 1_000, ..crate::api::Meta::default() }, + ); + assert!(payload.is_verified(at)); + for corrupt in 0..5 { + let mut bad = payload.clone(); + match corrupt { + 0 => bad.source = "https://other.example/v1/events".into(), + 1 => bad.shards[1].shard = 1, + 2 => bad.shards[0].block_at = Some(at + 1), + 3 => bad.days.push(DayCount { day: day(since) - 1, active: 1, complete: true, ..DayCount::default() }), + _ => bad.meta.fetched_at = (at + 1) * 1_000, + } + assert!(!bad.is_verified(at)); + } + } + #[test] + fn recorded_hypersnap_blocks_verify_whole_and_split_at_every_page_boundary() { + for (shard, body, height, at, actions) in [ + (1, include_bytes!("../tests/fixtures/activity-shard-1.json").as_slice(), 49_751_747, 1_793_097_607, 2), + (2, include_bytes!("../tests/fixtures/activity-shard-2.json").as_slice(), 49_587_008, 1_793_097_593, 1), + ] { + let whole = verified_activity(body, shard, (height << 14) - 1, None, at + 60).unwrap(); + assert_eq!(whole.page.actions.len(), actions); + assert_eq!(whole.page.reached, Some(at - 1)); + assert!(whole.block.as_ref().unwrap().complete()); + let input: serde_json::Value = serde_json::from_slice(body).unwrap(); + let events = input["events"].as_array().unwrap(); + for cut in 1..events.len() { + let first = verified_activity( + &canonical_page(events[..cut].to_vec()), + shard, + (height << 14) - 1, + None, + at + 60, + ) + .unwrap(); + assert_eq!(first.page.reached, None); + let second = verified_activity( + &canonical_page(events[cut..].to_vec()), + shard, + first.page.last_id.unwrap(), + first.block.as_ref(), + at + 60, + ) + .unwrap(); + let mut split = first.page.actions; + split.extend(second.page.actions); + assert_eq!(split, whole.page.actions, "shard{shard} split after event{cut}"); + assert_eq!(second.page.reached, whole.page.reached); + } + } + } + #[test] + fn a_gap_on_the_right_edge_invalidates_hours_and_rolling_but_not_the_prior_day() { + let to = 20_000 * 86_400; + let mut state = follow(to - 3 * 86_400, to + 60); + state.gaps.push((to, to + 1)); + assert!(state.covered(to - 86_400, to), "the timestamp belongs to the next calendar day"); + assert!(!state.covered_through(to - 86_400, to)); + assert_eq!(rolling(&state, &HashMap::new(), to), None); + assert_eq!(hourly(&state, &HashMap::new(), to).last(), Some(&None)); + state.keys_since = state.since; + assert!(!state.apps_due(to, to + 60)); + } + + #[test] + fn missing_or_unknown_event_types_and_missing_merge_types_are_not_verified_as_zero() { + let at = 1_800_000_000; + for bad_kind in [None, Some("HUB_EVENT_TYPE_UNKNOWN")] { + let mut message = canonical_merge(1, 10, 1, at); + match bad_kind { + None => { + message.as_object_mut().unwrap().remove("type"); + } + Some(kind) => message["type"] = kind.into(), + } + assert!( + verified_activity( + &canonical_page(vec![confirmation(1, 10, at, 2), message]), + 1, + (10 << 14) - 1, + None, + at + ) + .is_none() + ); + } + let mut message = canonical_merge(1, 10, 1, at); + message["mergeMessageBody"]["message"]["data"].as_object_mut().unwrap().remove("type"); + assert!( + verified_activity(&canonical_page(vec![confirmation(1, 10, at, 2), message]), 1, (10 << 14) - 1, None, at) + .is_none() + ); + } + #[test] + fn allocation_free_identity_validation_matches_the_canonical_formatter() { + for fid in [1, 7, 42, 1 << 31, u64::from(u32::MAX)] { + for key in [format!("0x{:064x}", 0), format!("0x12345678{:056x}", 1), format!("0x12345678{:056x}", 2)] { + let identity = signer_identity(fid, &key).unwrap(); + assert_eq!(identity_fid(&identity), Some(fid)); + } + } + let valid = signer_identity(42, &format!("0x{:064x}", 1)).unwrap(); + for invalid in [ + valid.replacen("0000000042", "0000000000", 1), + valid.replacen("0000000042", "4294967296", 1), + valid.replacen("0000000042", "+000000042", 1), + valid.replacen("0000000042", "000000004", 1), + valid.replacen(":0x", ":0X", 1), + format!("{}A", &valid[..76]), + format!("{}é", &valid[..76]), + format!("{} ", &valid[..76]), + ] { + assert_eq!(identity_fid(&invalid), None, "{invalid}"); + } + assert_eq!(exact_signer(&format!("0X{:064X}", 0xABC)), Some(format!("0x{:064x}", 0xabc))); + } + + #[test] + fn exact_key_linear_merges_match_an_independent_set_union() { + use std::collections::BTreeSet; + let a = signer_identity(7, &format!("0x12345678{:056x}", 1)).unwrap(); + let b = signer_identity(7, &format!("0x12345678{:056x}", 2)).unwrap(); + let c = signer_identity(u64::from(u32::MAX), &format!("0x{:064x}", 3)).unwrap(); + for (left, right) in [ + (vec![], vec![a.clone(), b.clone()]), + (vec![a.clone(), b.clone()], vec![]), + (vec![a.clone()], vec![b.clone(), c.clone()]), + (vec![a.clone(), c.clone()], vec![a.clone(), b.clone()]), + (vec![a.clone(), b.clone(), c.clone()], vec![a.clone(), b.clone(), c.clone()]), + ] { + let expected: BTreeSet<_> = left.iter().chain(right.iter()).cloned().collect(); + let mut got = ExactKeys::of(left); + got.extend(&ExactKeys::of(right)); + assert_eq!(got.as_slice(), expected.into_iter().collect::>()); + let chunks = got.chunks(1); + assert!( + chunks.windows(2).all(|w| identity_fid(w[0].last().unwrap()) != identity_fid(w[1].first().unwrap())) + ); + } + } } diff --git a/crates/allthing-core/src/provenance.rs b/crates/allthing-core/src/provenance.rs index c51fff62..b9687aa5 100644 --- a/crates/allthing-core/src/provenance.rs +++ b/crates/allthing-core/src/provenance.rs @@ -130,6 +130,48 @@ pub fn request_fid_from_metadata(metadata: &[u8]) -> Option { (fid > 0).then_some(fid) } +/// Read a positive requesting FID only from complete, canonically encoded +/// signed-key-request metadata. This checks the ABI shape and base64 encoding; +/// it does not independently validate the custody signature. Unknown, malformed +/// and truncated metadata remain unknown instead of becoming a no-app claim. +pub fn verified_request_fid_from_metadata(encoded: &str) -> Option { + fn tuple_fid(fields: &[u8]) -> Option { + let fid = word_as_u64(fields.get(..32)?)?; + if fid == 0 || fields.get(32..44)?.iter().any(|b| *b != 0) { + return None; + } + let signature_offset = usize::try_from(word_as_u64(fields.get(64..96)?)?).ok()?; + if signature_offset != 128 { + return None; + } + word_as_u64(fields.get(96..128)?)?; + let signature_len = usize::try_from(word_as_u64(fields.get(128..160)?)?).ok()?; + if signature_len == 0 { + return None; + } + let signature_end = 160usize.checked_add(signature_len)?; + let padded = signature_len.checked_add(31)? / 32 * 32; + let end = 160usize.checked_add(padded)?; + if fields.len() != end || fields.get(signature_end..end)?.iter().any(|b| *b != 0) { + return None; + } + Some(fid) + } + + let bytes = base64_decode(encoded)?; + if base64_encode(&bytes) != encoded { + return None; + } + // A flat tuple whose request FID is 32 must remain distinguishable from + // abi.encode(struct)'s outer offset: accept only the complete valid layout. + if word_as_u64(bytes.get(..32)?)? == 32 + && let Some(fid) = tuple_fid(bytes.get(32..)?) + { + return Some(fid); + } + tuple_fid(&bytes) +} + #[cfg(test)] mod tests { use super::*; @@ -165,6 +207,28 @@ mod tests { assert_eq!(request_fid_from_metadata(&huge), None); } + #[test] + fn verified_metadata_requires_complete_canonical_abi_in_both_layouts() { + let canonical = |fid, with_offset| { + let mut bytes = metadata(fid, with_offset); + let padding = if with_offset { 257 } else { 225 }; + bytes[padding..].fill(0); + bytes + }; + for (fid, with_offset) in [(9152, true), (309_857, false), (32, false)] { + assert_eq!(verified_request_fid_from_metadata(&base64_encode(&canonical(fid, with_offset))), Some(fid)); + } + assert_eq!(verified_request_fid_from_metadata(&base64_encode(&canonical(0, true))), None); + let mut truncated = canonical(9152, true); + truncated.truncate(160); + assert_eq!(verified_request_fid_from_metadata(&base64_encode(&truncated)), None); + let mut padded = canonical(9152, true); + assert_eq!(verified_request_fid_from_metadata(&base64_encode(&padded)), Some(9152)); + padded[287] = 1; + assert_eq!(verified_request_fid_from_metadata(&base64_encode(&padded)), None); + assert_eq!(verified_request_fid_from_metadata(&(base64_encode(&canonical(9152, true)) + "ignored")), None); + } + #[test] fn decodes_a_real_signer_registration() { // KeyRegistry add event for one of @0xael.eth's signers, as Hypersnap's diff --git a/crates/allthing-core/tests/fixtures/activity-shard-1.json b/crates/allthing-core/tests/fixtures/activity-shard-1.json new file mode 100644 index 00000000..cc727437 --- /dev/null +++ b/crates/allthing-core/tests/fixtures/activity-shard-1.json @@ -0,0 +1,113 @@ +{ + "spdx_license": "AGPL-3.0-or-later", + "source": "https://haatz.quilibrium.com/v1/events?shard_index=1&from_event_id=815132622848&stop_id=815132639232&pageSize=100", + "observed_date": "2026-10-27", + "events": [ + { + "type": "HUB_EVENT_TYPE_BLOCK_CONFIRMED", + "id": 815132622848, + "blockConfirmedBody": { + "blockNumber": 49751747, + "shardIndex": 1, + "timestamp": 183638407, + "blockHash": "0xf4f9cd10f8544096706e13cf549c0abed801a2d2063e96a1c544c4f7aa6aa44e", + "totalEvents": 4 + }, + "blockNumber": 49751747, + "shardIndex": 1 + }, + { + "type": "HUB_EVENT_TYPE_MERGE_MESSAGE", + "id": 815132622849, + "mergeMessageBody": { + "message": { + "data": { + "type": "MESSAGE_TYPE_CAST_ADD", + "fid": 1992198, + "timestamp": 183638406, + "network": "FARCASTER_NETWORK_MAINNET", + "castAddBody": { + "embedsDeprecated": [], + "mentions": [], + "parentCastId": null, + "parentUrl": null, + "text": "\ud83c\udfb0 Just claimed my free daily spin on BASED SLOTS.\nCast about it and you get one too \u2014 no deposit, no catch. Real $BASED payouts on Base.\nSpin it \ud83d\udc47", + "embeds": [ + { + "url": "https://basedslots.com/" + } + ], + "mentionsPositions": [], + "type": "CAST" + } + }, + "hash": "0x0bb0d7a8d040b2438599c4887be088746bf18845", + "hashScheme": "HASH_SCHEME_BLAKE3", + "signature": "Xa25MYkegIpzv7JdTy3oV57AIc/U3XwsUUdcj2WT/VwMaWA46gs5Qlc8oA9bochCCw52kPl2QmpVlZ9pHcNLAQ==", + "signatureScheme": "SIGNATURE_SCHEME_ED25519", + "signer": "0x1df7960edac64770e536f913f87096726116db2451002cbcffa7b7fd37c688be" + }, + "deletedMessages": [] + }, + "blockNumber": 49751747, + "shardIndex": 1 + }, + { + "type": "HUB_EVENT_TYPE_MERGE_MESSAGE", + "id": 815132622850, + "mergeMessageBody": { + "message": { + "data": { + "type": "MESSAGE_TYPE_REACTION_ADD", + "fid": 213222, + "timestamp": 183638407, + "network": "FARCASTER_NETWORK_MAINNET", + "reactionBody": { + "type": "REACTION_TYPE_LIKE", + "targetCastId": { + "fid": 1327602, + "hash": "0x55d780f37caf693e6b63a1f847e7ceb9109208d6" + } + } + }, + "hash": "0x89b1a24c225cabb7b5476991ed0fa1ccc40b3bdf", + "hashScheme": "HASH_SCHEME_BLAKE3", + "signature": "udchjDUV5zguSavFGBuz6i/8zrh11Ujb5npicMQLY7j86vvYoW5n1mLUvvFoqHhx8B1C4Kc7j0ySfyErBr3pBg==", + "signatureScheme": "SIGNATURE_SCHEME_ED25519", + "signer": "0xfdcc7e8e416d6d011f889b71279570a055d46b787b829c2a2df61a9fd1027507" + }, + "deletedMessages": [] + }, + "blockNumber": 49751747, + "shardIndex": 1 + }, + { + "type": "HUB_EVENT_TYPE_PRUNE_MESSAGE", + "id": 815132622851, + "pruneMessageBody": { + "message": { + "data": { + "type": "MESSAGE_TYPE_REACTION_ADD", + "fid": 213222, + "timestamp": 181063611, + "network": "FARCASTER_NETWORK_MAINNET", + "reactionBody": { + "type": "REACTION_TYPE_RECAST", + "targetCastId": { + "fid": 14537, + "hash": "0x860f3504ee0d567e7d47c5e2e93fc3ca6cfee9ef" + } + } + }, + "hash": "0xa662a27e3935d81dabb9bcae0ff061d86e4494c8", + "hashScheme": "HASH_SCHEME_BLAKE3", + "signature": "dvGe9aEMHF8rIMHPBGUSGE6HpXeTPZDYk5hpDKK9CkC/k1AH1MOb5vizmeqhTGdNO310f1ya1OP+Kb40ILhKBQ==", + "signatureScheme": "SIGNATURE_SCHEME_ED25519", + "signer": "0xfdcc7e8e416d6d011f889b71279570a055d46b787b829c2a2df61a9fd1027507" + } + }, + "blockNumber": 49751747, + "shardIndex": 1 + } + ] +} diff --git a/crates/allthing-core/tests/fixtures/activity-shard-2.json b/crates/allthing-core/tests/fixtures/activity-shard-2.json new file mode 100644 index 00000000..fa854322 --- /dev/null +++ b/crates/allthing-core/tests/fixtures/activity-shard-2.json @@ -0,0 +1,56 @@ +{ + "spdx_license": "AGPL-3.0-or-later", + "source": "https://haatz.quilibrium.com/v1/events?shard_index=2&from_event_id=812433539072&stop_id=812433555456&pageSize=100", + "observed_date": "2026-10-27", + "events": [ + { + "type": "HUB_EVENT_TYPE_BLOCK_CONFIRMED", + "id": 812433539072, + "blockConfirmedBody": { + "blockNumber": 49587008, + "shardIndex": 2, + "timestamp": 183638393, + "blockHash": "0x97c330abc6b6544b9f8bee117bd4c35e6f56a5cead6977e40ccc0c6adb82f859", + "totalEvents": 2 + }, + "blockNumber": 49587008, + "shardIndex": 2 + }, + { + "type": "HUB_EVENT_TYPE_MERGE_MESSAGE", + "id": 812433539073, + "mergeMessageBody": { + "message": { + "data": { + "type": "MESSAGE_TYPE_CAST_ADD", + "fid": 2617625, + "timestamp": 183638391, + "network": "FARCASTER_NETWORK_MAINNET", + "castAddBody": { + "embedsDeprecated": [], + "mentions": [], + "parentCastId": null, + "parentUrl": null, + "text": "\ud83c\udfb0 Just claimed my free daily spin on BASED SLOTS.\nCast about it and you get one too \u2014 no deposit, no catch. Real $BASED payouts on Base.\nSpin it \ud83d\udc47", + "embeds": [ + { + "url": "https://basedslots.com/" + } + ], + "mentionsPositions": [], + "type": "CAST" + } + }, + "hash": "0x562eb763e81ff98e33fbed60cae0ad320afc24cf", + "hashScheme": "HASH_SCHEME_BLAKE3", + "signature": "hlOc2qZikNDzA0fzR+li60mutPbGCcBf16SS786CnRuaho/EAPIxwBGFMM+w4E1YF3OaYMdqYbzDZR/gvOt9Cg==", + "signatureScheme": "SIGNATURE_SCHEME_ED25519", + "signer": "0x11e3715d7d2fcc3c837995703ff384f30bec96e83369e9e7e18ac932411a4718" + }, + "deletedMessages": [] + }, + "blockNumber": 49587008, + "shardIndex": 2 + } + ] +} diff --git a/crates/allthing-data/src/activity.rs b/crates/allthing-data/src/activity.rs index 8efd6411..826202ca 100644 --- a/crates/allthing-data/src/activity.rs +++ b/crates/allthing-data/src/activity.rs @@ -11,8 +11,8 @@ use std::collections::{BTreeSet, HashMap}; use allthing_core::activity::{ - self, ActivityPayload, AppsWindow, Batch, Fids, Follow, KEEP_DAYS, KEEP_HOURS, Keys, NAMED_APPS, Reached, - WEEK_DAYS, Window, day, day_set, hour, hour_set, key_pair, key_set, week_set, + self, ActivityPayload, AppsWindow, Batch, ExactKeys, Fids, Follow, KEEP_DAYS, KEEP_HOURS, NAMED_APPS, Reached, + VERIFICATION_VERSION, WEEK_DAYS, Window, day, day_set, hour, hour_set, key_set, week_set, }; use allthing_core::api::Meta; use allthing_core::emoji::ReactionReply; @@ -21,7 +21,6 @@ use allthing_core::normalize; use futures::future::{LocalBoxFuture, join_all}; use crate::Invocation; -use crate::annotate; use crate::hypersnap; use crate::indexer::{IndexError, Snapshot}; @@ -31,7 +30,7 @@ use crate::indexer::{IndexError, Snapshot}; pub const PAGE: u32 = 1000; /// The name the payload is stored and served under. -pub const SUMMARY: &str = "activity"; +pub const SUMMARY: &str = "activity_v2"; /// Reading this close to now (seconds) is keeping up. const CAUGHT_UP: i64 = 600; @@ -48,20 +47,20 @@ const AFTER: u32 = 2; /// Subrequests kept for keys removed since, each a read of the account's /// onchain history. const HISTORY: u32 = 6; -/// Keys counted by app in one statement, about 30 KB of JSON. -const COUNT_CHUNK: usize = 2_000; /// How long emoji reactions are kept for listings' cards, days. pub const REACTIONS_KEPT_DAYS: i64 = 400; /// What the follower writes at the end of a run, in one transaction. pub struct Saving { + /// Exact state this run read. The atomic store must reject a stale writer. + pub expected_state: Option, /// Sets by name. pub sets: Vec<(String, String)>, /// Drop the hourly sets (of fids and of keys) before hour `.0`, and the /// daily ones before day `.1`. pub keep_from: (i64, i64), - /// Keys' apps learned, as `(key_pair, app)`; a key's app never changes. - pub keys: Vec<(u64, u64)>, + /// Keys' apps learned, as `(signer_identity, app)`; a key's app never changes. + pub keys: Vec<(String, u64)>, /// Emoji reactions read, each with the cast it replies to; then the /// casts removed, and reactions revoked or pruned, whose reactions go. pub reactions: Vec<(String, ReactionReply)>, @@ -76,7 +75,7 @@ pub struct Saving { /// One app's count among a run of keys: the app (`None` for keys not looked /// up yet, 0 for keys no app requested), its accounts, and with `None`, the /// keys. -pub type AppCount = (Option, u64, Vec); +pub type AppCount = (Option, u64, Vec); /// The follower's storage (D1 in the Worker). Each call is one round trip. pub trait ActivityStore { @@ -85,15 +84,19 @@ pub trait ActivityStore { /// The stored sets of those `names` there are. fn activity_sets(&self, names: Vec) -> LocalBoxFuture<'static, Result, IndexError>>; - /// The apps of those keys (`key_pair`) that have been looked up. - fn key_apps(&self, pairs: Vec) -> LocalBoxFuture<'static, Result, IndexError>>; - /// Record `keys`' apps (as `save_activity` does, so the count knows - /// them), then count accounts active by app among each run of keys - /// (`Keys::chunks`). + /// The apps of those exact signer identities that have been looked up. + fn key_apps(&self, pairs: Vec) -> LocalBoxFuture<'static, Result, IndexError>>; + /// Record exact immutable attribution facts, then count the named + /// hourly key sets inside the store. Current-run overrides replace + /// persisted sets. Reject malformed identities or a distinct-account + /// total that differs from the independently counted FID window. Return + /// full app counts but at most 400 unresolved identities. fn count_apps( &self, - keys: Vec<(u64, u64)>, - chunks: Vec>, + keys: Vec<(String, u64)>, + names: Vec, + overrides: Vec<(String, String)>, + expected_active: u64, ) -> LocalBoxFuture<'static, Result, IndexError>>; /// Write it all, and store the payload served. fn save_activity(&self, saving: Saving) -> LocalBoxFuture<'static, Result<(), IndexError>>; @@ -121,38 +124,36 @@ pub struct Report { /// Where a first run starts in `shard`: at the start of yesterday (UTC), /// estimated from its newest block at about one block a second, so -/// yesterday is counted whole. Reading starts after the returned event. +/// the actual canonical start is then reported. Reading starts after this event. async fn start(inv: &Invocation, shard: u32, now: i64) -> Result { let body = hypersnap::events(inv, shard, None, 1).await.map_err(|e| IndexError(e.to_string()))?; - let newest = activity::activity(&body).and_then(|p| p.last_id).ok_or_else(|| IndexError("no events".into()))?; + let newest = activity::activity_head(&body, shard).ok_or_else(|| IndexError("invalid activity head".into()))?; let back = (now - (day(now) - 1) * 86_400).max(0) as u64; Ok(((newest >> 14).saturating_sub(back) << 14).saturating_sub(1)) } -/// The app of `pair` among an account's keys, 0 for one no app requested. -fn app_among(keys: &HashMap>, fid: u64, pair: u64) -> Option { - keys.iter().find(|(key, _)| key_pair(fid, key) == Some(pair)).map(|(_, app)| app.unwrap_or(0)) -} - -/// The apps of an account's keys `pairs`, from its active keys (cached in -/// the isolate for an hour, as casts' origins use them), or for a key -/// removed since, its onchain history. Keys whose app couldn't be read are -/// left out, to be looked up again later. -async fn look_up(inv: &Invocation, fid: u64, pairs: Vec, after: u32) -> Option> { - let active = annotate::signer_apps(inv, fid).await?; +/// Match full keys only. A missing, ambiguous, malformed or truncated +/// lookup remains unknown and can be retried; absence never proves no app. +async fn look_up(inv: &Invocation, fid: u64, identities: Vec, after: u32) -> Option> { + let active = hypersnap::verified_signer_apps(inv, fid).await.ok()?; let (mut found, mut removed) = (Vec::new(), Vec::new()); - for pair in pairs { - match app_among(&active, fid, pair) { - Some(app) => found.push((pair, app)), - None => removed.push(pair), + for identity in identities { + let key = identity.split_once(':')?.1; + match active.get(key) { + Some(app) => found.push((identity, *app)), + None => removed.push(identity), } } - // Not read (no subrequest to spare, say): those keys are looked up again. if !removed.is_empty() && inv.remaining_subrequests() > after - && let Ok(history) = hypersnap::all_onchain_signers(inv, fid).await + && let Ok(history) = hypersnap::verified_onchain_signers(inv, fid).await { - found.extend(removed.into_iter().map(|pair| (pair, app_among(&history, fid, pair).unwrap_or(0)))); + for identity in removed { + let key = identity.split_once(':')?.1; + if let Some(app) = history.get(key) { + found.push((identity, *app)); + } + } } Some(found) } @@ -168,9 +169,16 @@ fn d1(inv: &Invocation) -> Result<(), IndexError> { /// once an hour), and store the payload. pub async fn run(inv: &Invocation, store: &dyn ActivityStore) -> Result { d1(inv)?; - let mut follow: Follow = - store.activity_state().await?.and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default(); + let expected_state = store.activity_state().await?; + let mut follow: Follow = match expected_state.as_deref() { + Some(state) => serde_json::from_str(state).map_err(|_| IndexError("invalid verified activity state".into()))?, + None => Follow { verification_version: VERIFICATION_VERSION, ..Follow::default() }, + }; let now = (inv.now() / 1000.0) as i64; + if !follow.valid(now) { + return Err(IndexError("invalid verified activity progress".into())); + } + follow.revision = follow.revision.checked_add(1).ok_or_else(|| IndexError("activity revision exhausted".into()))?; let mut afters = Vec::new(); for shard in SHARDS { let after = match follow.cursors.iter().find(|c| c.shard == shard) { @@ -182,58 +190,86 @@ pub async fn run(inv: &Invocation, store: &dyn ActivityStore) -> Result = HashMap::new(); + let mut learned: HashMap = HashMap::new(); + let (mut reactions, mut removed) = (Vec::new(), Vec::new()); for ((shard, after), page) in afters.into_iter().zip(pages) { - let Some(page) = page.ok().and_then(|body| activity::activity(&body)) else { - // Read again from the same place next run. + let carried = follow.blocks.iter().find(|b| b.shard == shard); + let Some(verified) = page.ok().and_then(|body| activity::verified_activity(&body, shard, after, carried, now)) + else { report.failed.push(shard); continue; }; + let page = verified.page; report.events.push((shard, page.count)); - let previous = follow.reached.iter().find(|r| r.shard == shard).map(|r| r.at); - if let (Some(previous), Some(first), Some(started)) = (previous, page.first_block, page.started) - && first > (after >> 14) + 1 + follow.gaps.extend(verified.gaps); + if let Some(started) = page.started + && !follow.starts.iter().any(|s| s.shard == shard) { - // The node no longer had the blocks between: they are a gap. - follow.gaps.push((previous, started)); - } - if let Some(started) = page.started { - follow.since = Some(follow.since.map_or(started, |s| s.min(started))); - follow.keys_since = Some(follow.keys_since.map_or(started, |s| s.min(started))); + follow.starts.push(Reached { shard, at: started + 1 }); } + follow.since = SHARDS + .into_iter() + .map(|shard| follow.starts.iter().find(|s| s.shard == shard).map(|s| s.at)) + .collect::>>() + .and_then(|starts| starts.into_iter().max()); + follow.keys_since = follow.since; if let Some(last_id) = page.last_id { follow.cursors.retain(|c| c.shard != shard); follow.cursors.push(Cursor { shard, event_id: last_id }); } + if let Some(block) = verified.block { + follow.blocks.retain(|b| b.shard != shard); + follow.blocks.push(block); + } if let Some(at) = page.reached { follow.reached.retain(|r| r.shard != shard); follow.reached.push(Reached { shard, at }); } + learned.extend(page.exact_keys); actions.extend(page.actions); - learned.extend(page.keys); reactions.extend(page.reactions); removed.extend(page.removed); } report.actions = actions.len(); report.learned = learned.len(); + let batch = Batch::of(&actions); - let reached = follow.reached().unwrap_or(now); - report.behind = Some(now - reached); + // Until both shards establish coverage, retain the first shard's + // history from its original start rather than pruning against the wall + // clock (or that shard's advancing cursor). Bound an exceptionally long + // wait and explicitly mark any expired prefix as a gap. + let reached = follow.reached().unwrap_or_else(|| { + let start = follow.starts.iter().map(|s| s.at).min().unwrap_or(now); + let anchor = start.max(now - KEEP_DAYS * 86_400); + let expired_through = (hour(anchor) - KEEP_HOURS) * 3_600; + if expired_through >= start { + let to = expired_through + 1; + if let Some((_, end)) = follow.gaps.iter_mut().find(|(from, _)| *from == start) { + *end = (*end).max(to); + } else { + follow.gaps.push((start, to)); + } + } + anchor + }); + report.behind = follow.reached().map(|at| now - at); // The windows end as the hour in progress began. let (now_hour, today) = (hour(reached), day(reached)); let to = activity::rolling_to(reached); - let keeping_up = report.behind.is_some_and(|b| b < CAUGHT_UP); + let keeping_up = follow.reached().is_some() && report.behind.is_some_and(|b| (0..CAUGHT_UP).contains(&b)); // The apps of the keys that signed, where known. - let signed: Vec = Keys::of(batch.keys.values().flat_map(|k| k.as_slice().iter().copied())).as_slice().to_vec(); + let signed: Vec = + ExactKeys::of(batch.exact_keys.values().flat_map(|k| k.as_slice().iter().cloned())).as_slice().to_vec(); let mut known = if signed.is_empty() { HashMap::new() } else { d1(inv)?; store.key_apps(signed.clone()).await? }; - known.extend(learned.iter().map(|(&pair, &app)| (pair, app))); + + known.extend(learned.iter().map(|(key, &app)| (key.clone(), app))); // The 30 days to the end of yesterday, once a day, when keeping up (the // day sets of a backlog are still filling) and yesterday was read whole. @@ -243,18 +279,32 @@ pub async fn run(inv: &Invocation, store: &dyn ActivityStore) -> Result = batch.hours.keys().map(|&h| hour_set(h)).collect(); names.extend(batch.days.keys().map(|&d| day_set(d))); - names.extend(batch.keys.keys().map(|&h| key_set(h))); + names.extend(batch.exact_keys.keys().map(|&h| key_set(h))); names.extend((now_hour - 48..now_hour).map(hour_set)); if due_30d { names.extend((today - 30..today).map(day_set)); } - if due_apps { - names.extend((now_hour - 24..now_hour).map(key_set)); - } d1(inv)?; let stored = store.activity_sets(names.into_iter().collect()).await?; - let decoded = |name: &str| stored.get(name).and_then(|text| Fids::decode(text)).unwrap_or_default(); - let decoded_keys = |name: &str| stored.get(name).and_then(|text| Keys::decode(text)).unwrap_or_default(); + // A missing row is an empty set; a present but unreadable row is an + // integrity failure. Never overwrite it with an apparently exact zero. + let mut stored_fids = HashMap::new(); + let mut stored_keys = HashMap::new(); + for (name, text) in &stored { + if name.starts_with("v2k") { + stored_keys.insert( + name.clone(), + ExactKeys::decode(text).ok_or_else(|| IndexError(format!("invalid activity key set {name}")))?, + ); + } else { + stored_fids.insert( + name.clone(), + Fids::decode(text).ok_or_else(|| IndexError(format!("invalid activity account set {name}")))?, + ); + } + } + let decoded = |name: &str| stored_fids.get(name).cloned().unwrap_or_default(); + let decoded_keys = |name: &str| stored_keys.get(name).cloned().unwrap_or_default(); // Look up the apps of keys not known yet: this run's first, then those // the last count left, an account a request, keeping subrequests for @@ -262,22 +312,24 @@ pub async fn run(inv: &Invocation, store: &dyn ActivityStore) -> Result)> = Vec::new(); - for &pair in signed.iter().filter(|p| !known.contains_key(p)).chain(follow.backlog.iter()) { - let fid = activity::pair_fid(pair); + let mut accounts: Vec<(u64, Vec)> = Vec::new(); + for pair in signed.iter().filter(|p| !known.contains_key(*p)).chain(follow.backlog.iter()) { + let Some(fid) = activity::identity_fid(pair) else { + return Err(IndexError("invalid signer identity in activity backlog".into())); + }; if let Some((_, pairs)) = accounts.iter_mut().find(|(f, _)| *f == fid) { - if !pairs.contains(&pair) { - pairs.push(pair); + if !pairs.contains(pair) { + pairs.push(pair.clone()); } } else if accounts.len() < limit { - accounts.push((fid, vec![pair])); + accounts.push((fid, vec![pair.clone()])); } } report.looked_up = accounts.len(); let found = join_all(accounts.into_iter().map(|(fid, pairs)| look_up(inv, fid, pairs, after))).await; - let mut keys: Vec<(u64, u64)> = learned.into_iter().collect(); + let mut keys: Vec<(String, u64)> = learned.into_iter().collect(); for (pair, app) in found.into_iter().flatten().flatten() { - known.insert(pair, app); + known.insert(pair.clone(), app); keys.push((pair, app)); } follow.backlog.retain(|pair| !known.contains_key(pair)); @@ -290,17 +342,22 @@ pub async fn run(inv: &Invocation, store: &dyn ActivityStore) -> Result = HashMap::new(); - for (h, pairs) in &batch.keys { + let mut key_overrides = Vec::new(); + for (h, pairs) in &batch.exact_keys { let mut set = decoded_keys(&key_set(*h)); set.extend(pairs); - writes.push((key_set(*h), set.encode())); - key_sets.insert(*h, set); + let row = (key_set(*h), set.encode()); + if (now_hour - 24..now_hour).contains(h) { + key_overrides.push(row.clone()); + } + writes.push(row); } + let mut merged_days = HashMap::new(); for (d, fids) in &batch.days { let mut set = decoded(&day_set(*d)); set.extend(fids); writes.push((day_set(*d), set.encode())); + merged_days.insert(*d, set.clone()); let count = follow.day_mut(*d); count.active = set.len() as u64; for (family, n) in batch.actions.get(d).into_iter().flatten() { @@ -308,7 +365,8 @@ pub async fn run(inv: &Invocation, store: &dyn ActivityStore) -> Result = (today - 30..today).map(|d| decoded(&day_set(d))).collect(); + let days: Vec = + (today - 30..today).map(|d| merged_days.get(&d).cloned().unwrap_or_else(|| decoded(&day_set(d)))).collect(); let complete = follow.covered((today - 30) * 86_400, today * 86_400); follow.active_30d = Some(Window { day: today - 1, active: Fids::union_len(&days) as u64, complete }); report.counted_30d = true; @@ -324,16 +382,13 @@ pub async fn run(inv: &Invocation, store: &dyn ActivityStore) -> Result::to_vec).collect(); + let names = (now_hour - 24..now_hour).map(key_set).collect(); d1(inv)?; - let counts = store.count_apps(keys.clone(), chunks).await?; - follow.backlog = counts.iter().flat_map(|(_, _, pairs)| pairs.iter().copied()).take(BACKLOG).collect(); + let counts = store.count_apps(keys.clone(), names, key_overrides, active).await?; + follow.backlog = counts.iter().flat_map(|(_, _, pairs)| pairs.iter().cloned()).take(BACKLOG).collect(); let counts = counts.into_iter().map(|(app, n, _)| (app, n)); - let window = AppsWindow::of(to, now, active, counts); + let mut window = AppsWindow::of(to, now, active, counts); + window.verification_version = VERIFICATION_VERSION; // The accounts of the apps the page names: those not read yet, and // all of them again each day. Unread, the names read before stay. let named: Vec = window.apps.iter().take(NAMED_APPS).map(|a| a.app).collect(); @@ -356,6 +411,9 @@ pub async fn run(inv: &Invocation, store: &dyn ActivityStore) -> Result Result HttpResponse { status: 200, retry_after: None, body: Some(body.clone().into_bytes()) }, + Some(body) => { + let mut value: serde_json::Value = serde_json::from_str(body).unwrap(); + if url.contains("/events?") + && let Some(events) = value["events"].as_array_mut() + { + let shard: u32 = param(&url, "shard_index").unwrap().parse().unwrap(); + for event in events { + event["shardIndex"] = shard.into(); + if let Some(confirmed) = event.get_mut("blockConfirmedBody") { + confirmed["shardIndex"] = shard.into(); + } + } + } + // Short readable fixture keys stand for complete Ed25519 keys. + fn full_keys(value: &mut serde_json::Value) { + match value { + serde_json::Value::Object(fields) => { + for (name, value) in fields { + if matches!(name.as_str(), "key" | "signer") + && let Some(key) = value.as_str() + { + *value = format!("0x{:0>64}", key.trim_start_matches("0x")).into(); + } else { + full_keys(value); + } + } + } + serde_json::Value::Array(values) => { + for value in values { + full_keys(value); + } + } + _ => {} + } + } + full_keys(&mut value); + if url.contains("/events?") + && let Some(events) = value["events"].as_array_mut() + { + for event in events { + if let Some(body) = event + .get_mut("mergeOnChainEventBody") + .and_then(|b| b.get_mut("onChainEvent")) + .and_then(|e| e.get_mut("signerEventBody")) + { + body["keyType"] = 1.into(); + } + } + } + if url.contains("/signersByFid") + && let Some(signers) = value["signers"].as_array_mut() + { + for signer in signers { + signer["fid"] = fid.parse::().unwrap().into(); + signer["keyType"] = 1.into(); + if signer.get("requestFid").is_some() { + signer["source"] = "SIGNER_SOURCE_OFFCHAIN".into(); + } else { + signer["source"] = "SIGNER_SOURCE_ONCHAIN".into(); + signer["onchainEvent"] = serde_json::json!({"type":"EVENT_TYPE_SIGNER","fid":fid.parse::().unwrap(),"signerEventBody":{"key":signer["key"],"keyType":1,"eventType":"SIGNER_EVENT_TYPE_ADD","metadataType":0,"metadata":""}}); + } + } + } + if url.contains("/onChainEventsByFid") + && let Some(events) = value["events"].as_array_mut() + { + for event in events { + event["type"] = "EVENT_TYPE_SIGNER".into(); + event["fid"] = fid.parse::().unwrap().into(); + event["signerEventBody"]["keyType"] = 1.into(); + event["signerEventBody"]["metadataType"] = 0.into(); + event["signerEventBody"]["metadata"] = "".into(); + } + } + HttpResponse { status: 200, retry_after: None, body: Some(value.to_string().into_bytes()) } + } None => HttpResponse { status: 404, retry_after: None, body: None }, }; async move { Ok(response) }.boxed_local() @@ -445,7 +588,9 @@ mod tests { struct Memory { state: RefCell>, sets: RefCell>, - keys: RefCell>, + read_sets: RefCell>, + count_overrides: RefCell>, + keys: RefCell>, summary: RefCell>, /// Kept emoji reactions by reply: the cast replied to, the reply. reactions: RefCell>, @@ -462,49 +607,67 @@ mod tests { names: Vec, ) -> LocalBoxFuture<'static, Result, IndexError>> { let sets = self.sets.borrow(); + self.read_sets.borrow_mut().extend(names.iter().cloned()); let found = names.into_iter().filter_map(|n| Some((n.clone(), sets.get(&n)?.clone()))).collect(); async move { Ok(found) }.boxed_local() } - fn key_apps(&self, pairs: Vec) -> LocalBoxFuture<'static, Result, IndexError>> { + fn key_apps(&self, pairs: Vec) -> LocalBoxFuture<'static, Result, IndexError>> { let keys = self.keys.borrow(); - let found = pairs.into_iter().filter_map(|p| Some((p, *keys.get(&p)?))).collect(); + let found = pairs.into_iter().filter_map(|p| Some((p.clone(), *keys.get(&p)?))).collect(); async move { Ok(found) }.boxed_local() } - /// As the SQL does: each run's accounts by app, and its keys not - /// looked up. + /// Independent in-memory oracle for the SQL's complete window. fn count_apps( &self, - keys: Vec<(u64, u64)>, - chunks: Vec>, + keys: Vec<(String, u64)>, + names: Vec, + overrides: Vec<(String, String)>, + expected_active: u64, ) -> LocalBoxFuture<'static, Result, IndexError>> { - for (pair, app) in keys { - self.keys.borrow_mut().entry(pair).or_insert(app); + self.count_overrides.borrow_mut().extend(overrides.iter().cloned()); + let overrides: HashMap<_, _> = overrides.into_iter().collect(); + let stored = self.sets.borrow(); + let mut identities = BTreeSet::new(); + for name in names { + if let Some(body) = overrides.get(&name).or_else(|| stored.get(&name)) { + let Some(set) = ExactKeys::decode(body) else { + return async { Err(IndexError("invalid stored key window".into())) }.boxed_local(); + }; + identities.extend(set.as_slice().iter().cloned()); + } } - let keys = self.keys.borrow(); - let mut out = Vec::new(); - for chunk in chunks { - let mut by_app: std::collections::BTreeMap, (BTreeSet, Vec)> = - std::collections::BTreeMap::new(); - for pair in chunk { - let app = keys.get(&pair).copied(); - let (fids, unknown) = by_app.entry(app).or_default(); - fids.insert(activity::pair_fid(pair)); - if app.is_none() { - unknown.push(pair); - } + let accounts: BTreeSet<_> = identities.iter().map(|key| activity::identity_fid(key).unwrap()).collect(); + if accounts.len() as u64 != expected_active { + return async { Err(IndexError("key window account count mismatch".into())) }.boxed_local(); + } + for (key, app) in keys { + self.keys.borrow_mut().entry(key).or_insert(app); + } + let known = self.keys.borrow(); + let mut by_app: std::collections::BTreeMap, (BTreeSet, Vec)> = + std::collections::BTreeMap::new(); + for key in identities { + let app = known.get(&key).copied(); + let (fids, unknown) = by_app.entry(app).or_default(); + fids.insert(activity::identity_fid(&key).unwrap()); + if app.is_none() && unknown.len() < BACKLOG { + unknown.push(key); } - out.extend(by_app.into_iter().map(|(app, (fids, unknown))| (app, fids.len() as u64, unknown))); } + let out = by_app.into_iter().map(|(app, (fids, unknown))| (app, fids.len() as u64, unknown)).collect(); async move { Ok(out) }.boxed_local() } fn save_activity(&self, saving: Saving) -> LocalBoxFuture<'static, Result<(), IndexError>> { + if *self.state.borrow() != saving.expected_state { + return async { Err(IndexError("activity state changed".into())) }.boxed_local(); + } let mut stored = self.sets.borrow_mut(); stored.extend(saving.sets); stored.retain(|name, _| { - let (kind, n) = name.split_at(1); + let (kind, n) = name.strip_prefix("v2").unwrap().split_at(1); let n: i64 = n.parse().unwrap(); if kind == "d" || kind == "w" { n >= saving.keep_from.1 } else { n >= saving.keep_from.0 } }); @@ -539,7 +702,7 @@ mod tests { fn merge(block: u64, seq: u64, fid: u64, kind: &str) -> String { format!( - r#"{{"type":"HUB_EVENT_TYPE_MERGE_MESSAGE","id":{},"blockNumber":{block},"mergeMessageBody":{{"message":{{"data":{{"type":"MESSAGE_TYPE_{kind}","fid":{fid},"timestamp":1}},"signer":"0x0{seq}"}}}}}}"#, + r#"{{"type":"HUB_EVENT_TYPE_MERGE_MESSAGE","id":{},"blockNumber":{block},"mergeMessageBody":{{"message":{{"data":{{"type":"MESSAGE_TYPE_{kind}","fid":{fid},"timestamp":1}},"signer":"0x{seq:064x}"}}}}}}"#, (block << 14) | seq ) } @@ -547,13 +710,48 @@ mod tests { /// A reply by `fid` with `text` to the cast `parent`. fn reply(block: u64, seq: u64, fid: u64, hash: &str, text: &str, parent: &str) -> String { format!( - r#"{{"type":"HUB_EVENT_TYPE_MERGE_MESSAGE","id":{},"blockNumber":{block},"mergeMessageBody":{{"message":{{"hash":"{hash}","data":{{"type":"MESSAGE_TYPE_CAST_ADD","fid":{fid},"timestamp":1,"castAddBody":{{"text":"{text}","parentCastId":{{"fid":3,"hash":"{parent}"}}}}}},"signer":"0x0{seq}"}}}}}}"#, + r#"{{"type":"HUB_EVENT_TYPE_MERGE_MESSAGE","id":{},"blockNumber":{block},"mergeMessageBody":{{"message":{{"hash":"{hash}","data":{{"type":"MESSAGE_TYPE_CAST_ADD","fid":{fid},"timestamp":1,"castAddBody":{{"text":"{text}","parentCastId":{{"fid":3,"hash":"{parent}"}}}}}},"signer":"0x{seq:064x}"}}}}}}"#, (block << 14) | seq ) } fn page(events: &[String]) -> String { - format!(r#"{{"events":[{}]}}"#, events.join(",")) + let mut events: Vec = events.iter().map(|e| serde_json::from_str(e).unwrap()).collect(); + let totals: HashMap = events + .iter() + .map(|e| { + let id = e["id"].as_u64().unwrap(); + (id >> 14, (id & 16_383) + 1) + }) + .fold(HashMap::new(), |mut out, (block, n)| { + out.entry(block).and_modify(|prior| *prior = (*prior).max(n)).or_insert(n); + out + }); + for event in &mut events { + let block = event["id"].as_u64().unwrap() >> 14; + if let Some(body) = event.get_mut("blockConfirmedBody") { + body["totalEvents"] = totals[&block].into(); + } + } + serde_json::json!({ "events": events }).to_string() + } + + fn verified_state(mut follow: Follow) -> String { + follow.verification_version = VERIFICATION_VERSION; + follow.keys_since = follow.since; + follow.starts = SHARDS.into_iter().map(|shard| Reached { shard, at: follow.since.unwrap() }).collect(); + follow.blocks = follow + .cursors + .iter() + .map(|c| activity::BlockProgress { + shard: c.shard, + block: c.event_id >> 14, + at: follow.reached.iter().find(|r| r.shard == c.shard).unwrap().at + 1, + total: ((c.event_id & 16_383) + 1) as u32, + seen: ((c.event_id & 16_383) + 1) as u32, + }) + .collect(); + serde_json::to_string(&follow).unwrap() } fn run_once(hub: &Rc, store: &Memory) -> Result { @@ -653,17 +851,17 @@ mod tests { let store = Memory::default(); let report = run_once(&hub, &store).unwrap(); assert_eq!(report.events, [(1, 4), (2, 4)]); - assert_eq!((report.actions, report.behind, report.failed.len(), report.counted_30d), (5, Some(90), 0, false)); + assert_eq!((report.actions, report.behind, report.failed.len(), report.counted_30d), (5, Some(91), 0, false)); let follow = state(&store); - assert_eq!(follow.since, Some(at)); - assert_eq!(follow.reached(), Some(at)); + assert_eq!(follow.since, Some(at + 31)); + assert_eq!(follow.reached(), Some(at - 1)); let today = follow.days.last().unwrap(); assert_eq!((today.day, today.active), (day(at), 3)); assert_eq!(today.actions[&allthing_core::activity::Family::Cast], 2); assert_eq!(Fids::decode(&store.sets.borrow()[&hour_set(activity::bucket(at))]), Some(Fids::of([7, 9, 42]))); let payload: ActivityPayload = serde_json::from_str(&store.summary.borrow().as_ref().unwrap().body).unwrap(); // Counting began 90 seconds ago: no whole window yet. - assert_eq!((payload.rolling_24h, payload.days.len(), payload.since), (None, 1, Some(at))); + assert_eq!((payload.rolling_24h, payload.days.len(), payload.since), (None, 1, Some(at + 31))); // The next run reads on from each shard's cursor. Shard 2 can't be // read: it keeps its place, and an account seen again isn't @@ -695,7 +893,7 @@ mod tests { }; follow.day_mut(day(at)).active = 0; let store = Memory::default(); - *store.state.borrow_mut() = Some(serde_json::to_string(&follow).unwrap()); + *store.state.borrow_mut() = Some(verified_state(follow.clone())); let mut hub = Hub::default(); let from = ((START << 14) + 1).to_string(); // Shard 1 goes on from the next block; shard 2 jumps 100 blocks. @@ -704,12 +902,12 @@ mod tests { .insert(("2".into(), from), page(&[block(START + 100, at + 100), merge(START + 100, 1, 3, "CAST_ADD")])); run_once(&Rc::new(hub), &store).unwrap(); let follow = state(&store); - assert_eq!(follow.gaps, [(at, at + 100)]); + assert_eq!(follow.gaps, [(at + 1, at + 101)]); assert!(!follow.covered(at - 10, at + 200)); } - fn pair(fid: u64, key: &str) -> u64 { - key_pair(fid, key).unwrap() + fn pair(fid: u64, key: &str) -> String { + activity::signer_identity(fid, &format!("0x{:0>64}", key.trim_start_matches("0x"))).unwrap() } /// Read to a minute ago from 3 days back, keys kept all along, with @@ -723,7 +921,7 @@ mod tests { keys_since: Some(at - 3 * 86_400), ..Follow::default() }; - *store.state.borrow_mut() = Some(serde_json::to_string(&follow).unwrap()); + *store.state.borrow_mut() = Some(verified_state(follow.clone())); let from = ((START << 14) + 1).to_string(); hub.events.insert(("1".into(), from.clone()), page(&[])); hub.events.insert(("2".into(), from), page(&[])); @@ -742,9 +940,9 @@ mod tests { let mut sets = store.sets.borrow_mut(); sets.insert(hour_set(now_hour - 1), Fids::of([7, 42]).encode()); sets.insert(hour_set(now_hour - 20), Fids::of([9, 42]).encode()); - sets.insert(key_set(now_hour - 1), Keys::of([pair(42, "0x01"), pair(7, "0x01")]).encode()); - sets.insert(key_set(now_hour - 20), Keys::of([pair(42, "0x02"), pair(9, "0x03")]).encode()); - sets.insert(key_set(now_hour), Keys::of([pair(5, "0x05")]).encode()); + sets.insert(key_set(now_hour - 1), ExactKeys::of([pair(42, "0x01"), pair(7, "0x01")]).encode()); + sets.insert(key_set(now_hour - 20), ExactKeys::of([pair(42, "0x02"), pair(9, "0x03")]).encode()); + sets.insert(key_set(now_hour), ExactKeys::of([pair(5, "0x05")]).encode()); let mut keys = store.keys.borrow_mut(); keys.insert(pair(42, "0x01"), 9152); keys.insert(pair(7, "0x01"), 0); @@ -818,7 +1016,10 @@ mod tests { let store = Memory::default(); let mut hub = Hub::default(); let now_hour = counting(&store, &mut hub); - store.sets.borrow_mut().insert(key_set(now_hour - 1), Keys::of((1..=40).map(|fid| pair(fid, "0x01"))).encode()); + store + .sets + .borrow_mut() + .insert(key_set(now_hour - 1), ExactKeys::of((1..=40).map(|fid| pair(fid, "0x01"))).encode()); store.sets.borrow_mut().insert(hour_set(now_hour - 1), Fids::of(1..=40).encode()); // Every account has a key removed since, which takes a second read. for fid in 1..=40 { @@ -874,11 +1075,11 @@ mod tests { page(&[ block(START + 1, at), added, - merge(START + 1, 1, 100, "CAST_ADD"), - merge(START + 1, 2, 8, "REACTION_ADD"), + merge(START + 1, 2, 100, "CAST_ADD").replace(&format!("0x{:064x}", 2), &format!("0x{:064x}", 1)), + merge(START + 1, 3, 8, "REACTION_ADD"), ]), ); - hub.signers.insert("8".into(), r#"{"signers":[{"key":"0x02","requestFid":309857}]}"#.into()); + hub.signers.insert("8".into(), r#"{"signers":[{"key":"0x03","requestFid":309857}]}"#.into()); let hub = Rc::new(hub); let report = run_once(&hub, &store).unwrap(); assert_eq!((report.learned, report.looked_up), (1, 1)); @@ -886,10 +1087,10 @@ mod tests { assert!(requests.iter().any(|url| url.contains("/signersByFid?fid=8&"))); assert!(!requests.iter().any(|url| url.contains("fid=100")), "{requests:?}"); let keys = store.keys.borrow(); - assert_eq!((keys[&pair(100, "0x01")], keys[&pair(8, "0x02")]), (9152, 309_857)); + assert_eq!((keys[&pair(100, "0x01")], keys[&pair(8, "0x03")]), (9152, 309_857)); // This run's keys are kept in their hour. - let set = Keys::decode(&store.sets.borrow()[&key_set(activity::bucket(at))]).unwrap(); - assert_eq!(set, Keys::of([pair(8, "0x02"), pair(100, "0x01")])); + let set = ExactKeys::decode(&store.sets.borrow()[&key_set(activity::bucket(at))]).unwrap(); + assert_eq!(set, ExactKeys::of([pair(8, "0x03"), pair(100, "0x01")])); assert_eq!(activity::bucket(at), now_hour); } @@ -908,7 +1109,7 @@ mod tests { since: Some((today - 2) * 86_400), ..Follow::default() }; - *store.state.borrow_mut() = Some(serde_json::to_string(&follow).unwrap()); + *store.state.borrow_mut() = Some(verified_state(follow.clone())); let mut hub = Hub::default(); let from = ((START << 14) + 1).to_string(); hub.events.insert(("1".into(), from.clone()), page(&[])); @@ -922,7 +1123,7 @@ mod tests { // Sets older than the window are dropped. assert!(!store.sets.borrow().contains_key(&day_set(today - 40))); // The week isn't kept until it was read in full. - assert!(!store.sets.borrow().keys().any(|name| name.starts_with('w'))); + assert!(!store.sets.borrow().keys().any(|name| name.starts_with("v2w"))); } #[test] @@ -941,7 +1142,7 @@ mod tests { since: Some((today - 7) * 86_400), ..Follow::default() }; - *store.state.borrow_mut() = Some(serde_json::to_string(&follow).unwrap()); + *store.state.borrow_mut() = Some(verified_state(follow.clone())); let mut hub = Hub::default(); let from = ((START << 14) + 1).to_string(); hub.events.insert(("1".into(), from.clone()), page(&[])); @@ -959,8 +1160,296 @@ mod tests { // A gap in the week: not read in full, so not kept. let store = Memory::default(); let gapped = Follow { gaps: vec![((today - 3) * 86_400, (today - 3) * 86_400 + 60)], ..follow }; - *store.state.borrow_mut() = Some(serde_json::to_string(&gapped).unwrap()); + *store.state.borrow_mut() = Some(verified_state(gapped)); assert!(run_once(&hub, &store).unwrap().counted_30d); - assert!(!store.sets.borrow().keys().any(|name| name.starts_with('w'))); + assert!(!store.sets.borrow().keys().any(|name| name.starts_with("v2w"))); + } + #[test] + fn unreadable_or_legacy_state_is_never_restarted_or_certified() { + for raw in ["broken JSON", r#"{"since":1700000000}"#] { + let store = Memory::default(); + *store.state.borrow_mut() = Some(raw.into()); + let hub = Rc::new(Hub::default()); + assert!(run_once(&hub, &store).is_err()); + assert_eq!(store.state.borrow().as_deref(), Some(raw)); + assert!(store.summary.borrow().is_none()); + assert!(hub.requests.borrow().is_empty()); + } + } + + #[test] + fn a_present_corrupt_set_stops_the_run_without_overwriting_it() { + let store = Memory::default(); + let mut hub = Hub::default(); + let now_hour = counting(&store, &mut hub); + store.sets.borrow_mut().insert(hour_set(now_hour - 1), "not a set".into()); + let before = store.state.borrow().clone(); + assert!(run_once(&Rc::new(hub), &store).is_err()); + assert_eq!(*store.state.borrow(), before); + assert_eq!(store.sets.borrow()[&hour_set(now_hour - 1)], "not a set"); + assert!(store.summary.borrow().is_none()); + } + + #[test] + fn a_missing_first_shard_cannot_produce_a_complete_window() { + let at = NOW - 60; + let store = Memory::default(); + let mut hub = Hub::default(); + for shard in ["1", "2"] { + hub.events.insert((shard.into(), "newest".into()), page(&[block(HEAD, NOW)])); + } + hub.events.insert( + ("1".into(), (START << 14).to_string()), + page(&[block(START, at), merge(START, 1, 42, "CAST_ADD")]), + ); + let report = run_once(&Rc::new(hub), &store).unwrap(); + assert_eq!(report.failed, [2]); + let state = state(&store); + assert_eq!(state.reached(), None); + assert_eq!(state.since, None); + assert!(!state.covered(at - 86_400, at)); + let payload: ActivityPayload = serde_json::from_str(&store.summary.borrow().as_ref().unwrap().body).unwrap(); + assert_eq!(payload.rolling_24h, None); + assert_eq!(payload.verification_version, VERIFICATION_VERSION); + assert!(payload.days.iter().all(|day| !day.complete)); + } + + #[test] + fn the_first_daily_and_weekly_counts_include_the_page_finishing_yesterday() { + let midnight = day(NOW) * 86_400; + let store = Memory::default(); + let follow = Follow { + cursors: SHARDS.into_iter().map(|shard| Cursor { shard, event_id: START << 14 }).collect(), + reached: SHARDS.into_iter().map(|shard| Reached { shard, at: midnight - 3 }).collect(), + since: Some(midnight - 30 * 86_400), + ..Follow::default() + }; + *store.state.borrow_mut() = Some(verified_state(follow)); + store.sets.borrow_mut().insert(day_set(day(midnight) - 1), Fids::of([7]).encode()); + let mut hub = Hub { later: midnight + 60 - NOW, ..Hub::default() }; + let from = ((START << 14) + 1).to_string(); + hub.events.insert( + ("1".into(), from.clone()), + page(&[ + block(START + 1, midnight - 1), + merge(START + 1, 1, 42, "CAST_ADD"), + block(START + 2, midnight + 1), + ]), + ); + hub.events.insert(("2".into(), from), page(&[block(START + 1, midnight + 1)])); + let report = run_once(&Rc::new(hub), &store).unwrap(); + assert!(report.counted_30d); + let state = state(&store); + assert_eq!(state.active_30d.unwrap().active, 2); + assert!(state.active_30d.unwrap().complete); + let week = Fids::decode(&store.sets.borrow()[&week_set(day(midnight) - 1)]).unwrap(); + assert_eq!(week, Fids::of([7, 42]), "the final page must be in both unions before publication"); + } + + #[test] + fn repeated_or_future_pages_leave_each_shards_cursor_and_counts_unchanged() { + let store = Memory::default(); + let mut hub = Hub::default(); + counting(&store, &mut hub); + let before = state(&store); + let from = ((START << 14) + 1).to_string(); + hub.events.insert(("1".into(), from.clone()), page(&[block(START, NOW - 59)])); + hub.events.insert(("2".into(), from), page(&[block(START + 1, NOW + 1)])); + let report = run_once(&Rc::new(hub), &store).unwrap(); + assert_eq!(report.failed, [1, 2]); + let after = state(&store); + assert_eq!(after.cursors, before.cursors); + assert_eq!(after.blocks, before.blocks); + assert_eq!(after.days, before.days); + } + #[test] + fn counting_apps_keeps_old_hourly_key_sets_inside_the_store() { + let store = Memory::default(); + let mut hub = Hub::default(); + let now_hour = counting(&store, &mut hub); + store.sets.borrow_mut().insert(hour_set(now_hour - 1), Fids::of([42]).encode()); + store.sets.borrow_mut().insert(key_set(now_hour - 1), ExactKeys::of([pair(42, "0x01")]).encode()); + assert!(run_once(&Rc::new(hub), &store).unwrap().counted_apps); + assert!( + store.read_sets.borrow().iter().all(|name| !name.starts_with("v2k")), + "the Worker must not read the 24-hour key window" + ); + assert_eq!(state(&store).apps_24h.unwrap().unknown, 1); + } + + #[test] + fn the_store_counts_current_batch_overrides_before_they_are_saved() { + let midnight = day(NOW) * 86_400; + let store = Memory::default(); + let follow = Follow { + cursors: SHARDS.into_iter().map(|shard| Cursor { shard, event_id: START << 14 }).collect(), + reached: SHARDS.into_iter().map(|shard| Reached { shard, at: midnight - 3 }).collect(), + since: Some(midnight - 3 * 86_400), + ..Follow::default() + }; + *store.state.borrow_mut() = Some(verified_state(follow)); + let prior_hour = hour(midnight) - 1; + store.sets.borrow_mut().insert(hour_set(prior_hour), Fids::of([7]).encode()); + store.sets.borrow_mut().insert(key_set(prior_hour), ExactKeys::of([pair(7, "0x01")]).encode()); + let mut hub = Hub { later: midnight + 60 - NOW, ..Hub::default() }; + let from = ((START << 14) + 1).to_string(); + hub.events.insert( + ("1".into(), from.clone()), + page(&[ + block(START + 1, midnight - 1), + merge(START + 1, 1, 42, "CAST_ADD"), + block(START + 2, midnight + 1), + ]), + ); + hub.events.insert(("2".into(), from), page(&[block(START + 1, midnight + 1)])); + assert!(run_once(&Rc::new(hub), &store).unwrap().counted_apps); + let window = state(&store).apps_24h.unwrap(); + assert_eq!((window.active, window.unknown), (2, 2)); + assert_eq!(store.count_overrides.borrow().len(), 1); + assert_eq!( + ExactKeys::decode(&store.count_overrides.borrow()[0].1).unwrap(), + ExactKeys::of([pair(7, "0x01"), pair(42, "0x01")]) + ); + } + + #[test] + fn absent_or_malformed_key_windows_cannot_claim_all_accounts_are_known() { + for corrupt in [None, Some("broken JSON"), Some(r#"["0000000042:0x01"]"#)] { + let store = Memory::default(); + let mut hub = Hub::default(); + let now_hour = counting(&store, &mut hub); + store.sets.borrow_mut().insert(hour_set(now_hour - 1), Fids::of([42]).encode()); + if let Some(body) = corrupt { + store.sets.borrow_mut().insert(key_set(now_hour - 1), body.into()); + } + let before = store.state.borrow().clone(); + assert!(run_once(&Rc::new(hub), &store).is_err()); + assert_eq!(*store.state.borrow(), before); + assert!(store.summary.borrow().is_none()); + assert!(store.read_sets.borrow().iter().all(|name| !name.starts_with("v2k"))); + } + } + #[test] + fn malformed_derived_state_is_rejected_before_any_upstream_read_or_cursor_advance() { + let seed = Memory::default(); + let mut hub = Hub::default(); + counting(&seed, &mut hub); + let base = state(&seed); + let mut cases = Vec::new(); + let mut bad = base.clone(); + bad.days.push(activity::DayCount { day: i64::MIN, ..activity::DayCount::default() }); + cases.push(bad); + let mut bad = base.clone(); + bad.days.push(activity::DayCount { day: day(NOW) + 1, ..activity::DayCount::default() }); + cases.push(bad); + let mut bad = base.clone(); + bad.reached[0].at = 0; + cases.push(bad); + let mut bad = base.clone(); + bad.reached.remove(0); + cases.push(bad); + let mut bad = base.clone(); + bad.reached[0].at -= 1; + cases.push(bad); + let mut bad = base.clone(); + bad.active_30d = Some(Window { day: i64::MAX, active: 1, complete: false }); + cases.push(bad); + let mut bad = base.clone(); + bad.active_30d = Some(Window { day: day(NOW) - 1, active: 1, complete: true }); + cases.push(bad); + let mut bad = base.clone(); + bad.days.push(activity::DayCount { + day: day(NOW) - 4, + active: 1, + complete: true, + ..activity::DayCount::default() + }); + cases.push(bad); + let mut bad = base.clone(); + let mut window = AppsWindow::of(activity::rolling_to(NOW) - 4 * 86_400, NOW, 1, [(Some(9152), 1)]); + window.verification_version = VERIFICATION_VERSION; + bad.apps_24h = Some(window); + cases.push(bad); + let mut bad = base.clone(); + bad.active_30d = Some(Window { day: day(NOW) - 1, active: u64::from(u32::MAX) + 1, complete: false }); + cases.push(bad); + let mut bad = base.clone(); + bad.gaps.push((i64::MIN, 0)); + cases.push(bad); + for (index, bad) in cases.into_iter().enumerate() { + let store = Memory::default(); + let raw = serde_json::to_string(&bad).unwrap(); + *store.state.borrow_mut() = Some(raw.clone()); + let hub = Rc::new(Hub::default()); + assert!(run_once(&hub, &store).is_err(), "corrupt-state case{index}"); + assert!(hub.requests.borrow().is_empty(), "case{index} reached the upstream"); + assert_eq!(store.state.borrow().as_deref(), Some(raw.as_str())); + assert!(store.summary.borrow().is_none()); + assert!(store.sets.borrow().is_empty()); + } + } + fn one_shard_state(first: i64, reached: i64) -> Follow { + Follow { + verification_version: VERIFICATION_VERSION, + starts: vec![Reached { shard: 1, at: first + 1 }], + cursors: vec![Cursor { shard: 1, event_id: START << 14 }], + blocks: vec![activity::BlockProgress { shard: 1, block: START, at: reached + 1, total: 1, seen: 1 }], + reached: vec![Reached { shard: 1, at: reached }], + ..Follow::default() + } + } + + #[test] + fn missing_first_shard_preserves_old_hours_until_common_coverage_can_count_them() { + let first = NOW - 73 * 3_600; + let reached = NOW - 24 * 3_600; + let active_at = NOW - 60 * 3_600; + let store = Memory::default(); + *store.state.borrow_mut() = Some(serde_json::to_string(&one_shard_state(first, reached)).unwrap()); + let old_hour = hour_set(activity::bucket(active_at)); + store.sets.borrow_mut().insert(old_hour.clone(), Fids::of([42]).encode()); + let mut waiting = Hub::default(); + waiting.events.insert(("1".into(), ((START << 14) + 1).to_string()), page(&[])); + waiting.events.insert(("2".into(), "newest".into()), page(&[block(HEAD, NOW)])); + let report = run_once(&Rc::new(waiting), &store).unwrap(); + assert_eq!(report.failed, [2]); + assert!(store.sets.borrow().contains_key(&old_hour), "a 60-hour-old set must survive missing common coverage"); + assert_eq!(state(&store).reached(), None); + + let mut joined = Hub::default(); + joined.events.insert(("1".into(), ((START << 14) + 1).to_string()), page(&[])); + joined.events.insert(("2".into(), "newest".into()), page(&[block(HEAD, NOW)])); + joined.events.insert( + ("2".into(), (START << 14).to_string()), + page(&[block(START, first), block(START + 1, reached + 1)]), + ); + run_once(&Rc::new(joined), &store).unwrap(); + let payload: ActivityPayload = serde_json::from_str(&store.summary.borrow().as_ref().unwrap().body).unwrap(); + assert!(payload.is_verified(NOW)); + assert_eq!(payload.rolling_24h, Some(0)); + assert_eq!( + payload.previous_24h, + Some(1), + "joining the shards must recover the retained account, not a certified zero" + ); + } + + #[test] + fn an_extended_missing_shard_wait_bounds_history_with_an_explicit_expired_gap() { + let first = NOW - 100 * 86_400; + let active_at = NOW - 90 * 86_400; + let store = Memory::default(); + *store.state.borrow_mut() = Some(serde_json::to_string(&one_shard_state(first, NOW - 60)).unwrap()); + let old_hour = hour_set(activity::bucket(active_at)); + store.sets.borrow_mut().insert(old_hour.clone(), Fids::of([42]).encode()); + let mut hub = Hub::default(); + hub.events.insert(("1".into(), ((START << 14) + 1).to_string()), page(&[])); + hub.events.insert(("2".into(), "newest".into()), page(&[block(HEAD, NOW)])); + run_once(&Rc::new(hub), &store).unwrap(); + assert!(!store.sets.borrow().contains_key(&old_hour)); + let retained = state(&store); + assert!(retained.gaps.iter().any(|&(from, to)| from <= active_at && to > active_at)); + assert_eq!(retained.reached(), None); + assert!(retained.valid(NOW)); } } diff --git a/crates/allthing-data/src/hypersnap.rs b/crates/allthing-data/src/hypersnap.rs index 7fed9f7a..ae5d9756 100644 --- a/crates/allthing-data/src/hypersnap.rs +++ b/crates/allthing-data/src/hypersnap.rs @@ -4,7 +4,7 @@ //! The Farcaster app's public API fills in what only the app knows: votes to //! leave it, and channels' hosts, pictures and member counts. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use allthing_core::api::{FollowsDir, LIBRARY_PAGE_SIZE, LibraryEntry, ReactionKind, library_cursor}; use allthing_core::channels::{ @@ -18,7 +18,9 @@ use allthing_core::normalize::{ self, RawAppChannel, RawAppChannelResponse, RawCast, RawChannel, RawFeed, RawHubReactions, RawNotifications, RawReaction, RawReactionList, RawTransfers, RawUser, RawUserData, RawUserList, RawUserSearch, }; -use allthing_core::provenance::{base64_decode, base64_encode, normalize_key, request_fid_from_metadata}; +use allthing_core::provenance::{ + base64_decode, base64_encode, normalize_key, request_fid_from_metadata, verified_request_fid_from_metadata, +}; use allthing_core::ranking::TrendingWindow; use allthing_core::website::Website; use serde::Deserialize; @@ -871,6 +873,149 @@ pub async fn all_onchain_signers(inv: &Invocation, fid: u64) -> Result, + key_type: Option, + event_type: Option, + metadata: Option, + metadata_type: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct VerifiedSignerEvent { + #[serde(rename = "type")] + kind: Option, + fid: Option, + signer_event_body: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct VerifiedSigner { + source: Option, + key: Option, + key_type: Option, + fid: Option, + request_fid: Option, + on_chain_event: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct VerifiedSigners { + // Required: a successful response without the collection is not evidence. + signers: Vec, + next_page_token: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct VerifiedSignerEvents { + events: Vec, + next_page_token: Option, +} + +fn verified_signer_key(value: &str) -> Option { + normalize_key(value).filter(|key| key.len() == 66) +} + +fn verified_onchain_entry(event: &VerifiedSignerEvent, fid: u64) -> Option<(String, u64)> { + if event.kind.as_deref() != Some("EVENT_TYPE_SIGNER") || event.fid != Some(fid) { + return None; + } + let body = event.signer_event_body.as_ref()?; + if body.event_type.as_deref() != Some("SIGNER_EVENT_TYPE_ADD") || body.key_type != Some(1) { + return None; + } + let key = verified_signer_key(body.key.as_deref()?)?; + let app = match (body.metadata_type, body.metadata.as_deref()) { + (Some(0), Some("")) => 0, + (Some(1), Some(metadata)) => verified_request_fid_from_metadata(metadata)?, + _ => return None, + }; + Some((key, app)) +} + +fn insert_verified_app(out: &mut HashMap, conflicts: &mut HashSet, key: String, app: u64) { + if conflicts.contains(&key) { + return; + } + if out.get(&key).is_some_and(|previous| *previous != app) { + out.remove(&key); + conflicts.insert(key); + } else { + out.insert(key, app); + } +} + +fn verified_active_apps(page: VerifiedSigners, fid: u64) -> HashMap { + let partial = page.next_page_token.is_some_and(|token| !token.is_empty()); + let (mut out, mut conflicts) = (HashMap::new(), HashSet::new()); + for signer in page.signers { + if signer.fid != Some(fid) || signer.key_type != Some(1) { + continue; + } + let Some(key) = signer.key.as_deref().and_then(verified_signer_key) else { continue }; + let app = match signer.source.as_deref() { + Some("SIGNER_SOURCE_OFFCHAIN") => signer.request_fid.filter(|app| *app > 0), + Some("SIGNER_SOURCE_ONCHAIN") => signer + .on_chain_event + .as_ref() + .and_then(|event| verified_onchain_entry(event, fid)) + .filter(|(event_key, _)| *event_key == key) + .map(|(_, app)| app), + _ => None, + }; + if let Some(app) = app.filter(|app| !partial || *app > 0) { + insert_verified_app(&mut out, &mut conflicts, key, app); + } + } + out +} + +fn verified_history_apps(page: VerifiedSignerEvents, fid: u64) -> HashMap { + let partial = page.next_page_token.is_some_and(|token| !token.is_empty()); + let (mut out, mut conflicts) = (HashMap::new(), HashSet::new()); + for event in page.events { + if let Some((key, app)) = verified_onchain_entry(&event, fid).filter(|(_, app)| !partial || *app > 0) { + insert_verified_app(&mut out, &mut conflicts, key, app); + } + } + out +} + +/// Exact 32-byte active signer keys with a positively identified requesting +/// app. Zero requires an explicit onchain add with empty type-0 metadata; +/// absent keys and malformed or incomplete evidence remain absent (unknown). +pub async fn verified_signer_apps(inv: &Invocation, fid: u64) -> Result, UpstreamError> { + let page: VerifiedSigners = get_json( + inv, + Upstream::Hypersnap, + &v1("signersByFid", &[("fid", &fid.to_string()), ("pageSize", "1000")]), + false, + ) + .await? + .ok_or_else(|| UpstreamError::Parse("missing signer response".into()))?; + Ok(verified_active_apps(page, fid)) +} + +/// Positive attribution facts for exact keys in one bounded onchain-history +/// page. A missing key is unknown, including when another page exists. +pub async fn verified_onchain_signers(inv: &Invocation, fid: u64) -> Result, UpstreamError> { + let page: VerifiedSignerEvents = get_json( + inv, + Upstream::Hypersnap, + &v1("onChainEventsByFid", &[("fid", &fid.to_string()), ("event_type", "EVENT_TYPE_SIGNER")]), + false, + ) + .await? + .ok_or_else(|| UpstreamError::Parse("missing signer history response".into()))?; + Ok(verified_history_apps(page, fid)) +} + #[cfg(test)] mod tests { use std::cell::RefCell; @@ -991,4 +1136,150 @@ mod tests { assert_eq!(hypersnap_window(TrendingWindow::Day).as_str(), "24h"); assert_eq!(hypersnap_window(TrendingWindow::Week).as_str(), "7d"); } + + fn verified_metadata(fid: u64) -> String { + let word = |n: u64| { + let mut out = vec![0; 24]; + out.extend(n.to_be_bytes()); + out + }; + let mut bytes = word(32); + bytes.extend(word(fid)); + bytes.extend([0; 12]); + bytes.extend([1; 20]); + bytes.extend(word(128)); + bytes.extend(word(200_000_000)); + bytes.extend(word(65)); + bytes.extend([7; 65]); + bytes.extend([0; 31]); + base64_encode(&bytes) + } + + fn verified_add(key: &str, app: Option) -> serde_json::Value { + let (metadata_type, metadata) = match app { + Some(app) => (1, verified_metadata(app)), + None => (0, String::new()), + }; + serde_json::json!({"type":"EVENT_TYPE_SIGNER","fid":42,"signerEventBody":{ + "key":key,"keyType":1,"eventType":"SIGNER_EVENT_TYPE_ADD", + "metadataType":metadata_type,"metadata":metadata + }}) + } + + fn verified_signer(key: &str, app: u64) -> serde_json::Value { + serde_json::json!({"source":"SIGNER_SOURCE_OFFCHAIN","fid":42,"keyType":1,"key":key,"requestFid":app}) + } + + #[test] + fn verified_signer_identity_keeps_keys_with_the_same_prefix_separate() { + let key_a = format!("0x{}", "ab".repeat(32)); + let key_b = format!("0x{}cd", "ab".repeat(31)); + let page = serde_json::from_value(serde_json::json!({"signers":[ + verified_signer(&key_a,9152), verified_signer(&key_b,309857), + verified_signer("0xabababab",9152) + ]})) + .unwrap(); + assert_eq!(verified_active_apps(page, 42), HashMap::from([(key_a, 9152), (key_b, 309857)])); + } + + #[test] + fn verified_metadata_requires_a_complete_positive_request() { + assert_eq!(verified_request_fid_from_metadata(&verified_metadata(9152)), Some(9152)); + assert_eq!(verified_request_fid_from_metadata(&verified_metadata(0)), None); + let mut truncated = base64_decode(&verified_metadata(9152)).unwrap(); + truncated.truncate(160); + assert_eq!(verified_request_fid_from_metadata(&base64_encode(&truncated)), None); + assert_eq!(verified_request_fid_from_metadata(&(verified_metadata(9152) + "ignored")), None); + } + + #[test] + fn verified_onchain_none_requires_explicit_empty_type_zero_metadata() { + let key = format!("0x{}", "ab".repeat(32)); + let good = verified_add(&key, None); + let entry = |value| verified_onchain_entry(&serde_json::from_value(value).unwrap(), 42); + assert_eq!(entry(good.clone()), Some((key.clone(), 0))); + let mut absent = good.clone(); + absent["signerEventBody"].as_object_mut().unwrap().remove("metadata"); + assert_eq!(entry(absent), None); + let mut nonempty = good.clone(); + nonempty["signerEventBody"]["metadata"] = serde_json::json!("AA=="); + assert_eq!(entry(nonempty), None); + let mut unknown = good.clone(); + unknown["signerEventBody"]["metadataType"] = serde_json::json!(99); + assert_eq!(entry(unknown), None); + let mut wrong_fid = good.clone(); + wrong_fid["fid"] = serde_json::json!(7); + assert_eq!(entry(wrong_fid), None); + let mut removed = good; + removed["signerEventBody"]["eventType"] = serde_json::json!("SIGNER_EVENT_TYPE_REMOVE"); + assert_eq!(entry(removed), None); + } + + #[test] + fn verified_active_onchain_keys_must_match_their_add_event() { + let key = format!("0x{}", "ab".repeat(32)); + let other = format!("0x{}", "cd".repeat(32)); + let mut record = serde_json::json!({"source":"SIGNER_SOURCE_ONCHAIN","fid":42, + "keyType":1,"key":key,"onChainEvent":verified_add(&key,Some(9152))}); + let read = |record| { + let page = serde_json::from_value(serde_json::json!({"signers":[record]})).unwrap(); + verified_active_apps(page, 42) + }; + assert_eq!(read(record.clone()), HashMap::from([(key, 9152)])); + record["key"] = serde_json::json!(other); + assert!(read(record).is_empty()); + } + + #[test] + fn verified_pages_leave_missing_and_conflicting_keys_unknown() { + let key_a = format!("0x{}", "ab".repeat(32)); + let key_b = format!("0x{}", "cd".repeat(32)); + let page = serde_json::from_value(serde_json::json!({"events":[ + verified_add(&key_a,Some(9152)), verified_add(&key_b,None) + ],"nextPageToken":"more"})) + .unwrap(); + assert_eq!(verified_history_apps(page, 42), HashMap::from([(key_a.clone(), 9152)])); + let page = serde_json::from_value(serde_json::json!({"signers":[ + verified_signer(&key_a,9152), verified_signer(&key_a,309857), verified_signer(&key_a,9152) + ]})) + .unwrap(); + assert!(verified_active_apps(page, 42).is_empty()); + assert!(serde_json::from_str::(r#"{"nextPageToken":""}"#).is_err()); + assert!(serde_json::from_str::(r#"{"events":null}"#).is_err()); + } + + struct VerifiedSignerHub { + body: String, + requests: RefCell>, + } + + impl Host for VerifiedSignerHub { + fn now(&self) -> f64 { + 0.0 + } + + fn fetch(&self, request: HttpRequest) -> LocalBoxFuture<'static, Result> { + self.requests.borrow_mut().push(request.url); + let response = HttpResponse { status: 200, retry_after: None, body: Some(self.body.clone().into()) }; + async move { Ok(response) }.boxed_local() + } + + fn spawn(&self, _: LocalBoxFuture<'static, ()>) {} + } + + #[test] + fn verified_signer_reads_are_bounded_hypersnap_calls_and_reject_missing_arrays() { + let key = format!("0x{}", "ab".repeat(32)); + let hub = Rc::new(VerifiedSignerHub { + body: serde_json::json!({"signers":[verified_signer(&key,9152)],"nextPageToken":"more"}).to_string(), + requests: RefCell::new(vec![]), + }); + let inv = Invocation::new(hub.clone(), 45); + assert_eq!(verified_signer_apps(&inv, 42).now_or_never().unwrap().unwrap(), HashMap::from([(key, 9152)])); + assert_eq!(hub.requests.take(), [format!("{BASE}/v1/signersByFid?fid=42&pageSize=1000")]); + let hub = Rc::new(VerifiedSignerHub { body: "{}".into(), requests: RefCell::new(vec![]) }); + let inv = Invocation::new(hub.clone(), 45); + assert!(matches!(verified_onchain_signers(&inv, 42).now_or_never().unwrap(), Err(UpstreamError::Parse(_)))); + assert_eq!(hub.requests.take(), [format!("{BASE}/v1/onChainEventsByFid?fid=42&event_type=EVENT_TYPE_SIGNER")]); + } } diff --git a/crates/allthing-web/src/components/clients.rs b/crates/allthing-web/src/components/clients.rs index ec2da21f..4c673d58 100644 --- a/crates/allthing-web/src/components/clients.rs +++ b/crates/allthing-web/src/components/clients.rs @@ -137,6 +137,12 @@ pub fn ClientShareBar( window: Option, #[prop(optional_no_strip)] counted: Option, ) -> impl IntoView { + // Cached Home payloads may predate verified counting. Keep the ordinary + // cast-share bar, but never decorate it with unverified or stale DAU. + let clock = now(); + let counted = counted.filter(|window| { + window.is_verified() && window.counted <= clock && window.to <= clock && clock - window.to <= 7_200 + }); let Some(split) = client_shares(&casts, ¬es, TOP_APPS) else { return ().into_any() }; let known = split.known(); let shares = split.shares; diff --git a/crates/allthing-web/src/components/layout.rs b/crates/allthing-web/src/components/layout.rs index f50d621a..8c4a07d2 100644 --- a/crates/allthing-web/src/components/layout.rs +++ b/crates/allthing-web/src/components/layout.rs @@ -32,6 +32,7 @@ pub fn SiteHeader() -> impl IntoView { {presence()} diff --git a/crates/allthing-web/src/pages/dau.rs b/crates/allthing-web/src/pages/dau.rs index 245d3e96..f1d38f2a 100644 --- a/crates/allthing-web/src/pages/dau.rs +++ b/crates/allthing-web/src/pages/dau.rs @@ -7,7 +7,9 @@ //! 48 hours, 7 or 30 days, what the accounts did, which apps they used, and //! its definitions: how and since when they were counted. -use allthing_core::activity::{ActivityPayload, AppsWindow, DayCount, Family, day, day_name, median_day, weeks}; +use allthing_core::activity::{ + ACTIVITY_SOURCE, ActivityPayload, AppsWindow, DayCount, Family, day, day_name, median_day, weeks, +}; use allthing_core::api::ACTIVITY_PATH; use allthing_core::clients::{client_name, client_profile}; use allthing_core::format::{axis, full_number, plural, tick}; @@ -46,6 +48,11 @@ fn moment(at: i64) -> String { if day(at) == day(now()) { clock(at) } else { format!("{}, {}", date(day(at), true), clock(at)) } } +/// An evidence timestamp always includes its UTC date. +fn checkpoint_time(at: i64) -> String { + format!("{}, {}", date(day(at), true), clock(at)) +} + /// How far behind the clock reading may be before the page says it is /// catching up: a run every 2 minutes is normally a few minutes behind. const BEHIND: i64 = 15 * 60; @@ -134,21 +141,37 @@ pub fn DauPage() -> impl IntoView {

"Daily active accounts"

- "Farcaster accounts with a message merged into the network, each counted once, as " + "Unique Farcaster accounts with a message merged into the network, each counted once, as " "SnapMeter" - " counts them." + " counts them. These are accounts, not people or Allthing readers."

{move || match data.get() { None => loading().into_any(), Some(Err(Failure::Busy { .. })) => view! { }.into_any(), Some(Err(failure)) => view! { }.into_any(), + Some(Ok(fetched)) if !fetched.data.is_verified(now()) => view! { }.into_any(), Some(Ok(fetched)) => crate::split::lazy_page(loading, dashboard(fetched.data.as_ref().clone())).into_any(), }} } } +/// Legacy or inconsistent responses cannot supply dashboard figures. +#[component] +fn AwaitingVerification() -> impl IntoView { + view! { +
+

"Counts await verification"

+

"This response has no coherent verification record for both network shards. Counts and charts appear once Allthing has checked their Hypersnap events. Try again after the next count."

+

+ "Hypersnap event API" + " · ""The data as JSON" +

+
+ } +} + /// Before the cron's first count, and in the static copy, which has no cron. #[component] fn NotCounted() -> impl IntoView { @@ -165,7 +188,7 @@ fn NotCounted() -> impl IntoView { } .into_any() } else { - view! {

"Allthing hasn't counted active accounts yet. The count starts within minutes of a deploy; try again shortly."

}.into_any() + view! {

"Allthing hasn't counted active accounts yet. Counts appear after canonical events from both shards have been read; try again later."

}.into_any() }} } @@ -187,10 +210,11 @@ fn Dashboard(counted: ActivityPayload) -> impl IntoView { let behind = counted.reached.filter(|at| now() - at > BEHIND); let apps = counted.apps_24h.clone().map(|apps| (apps, counted.app_accounts.clone())); view! { + {behind.map(|at| view! {

- "Catching up: Allthing has read the network's messages to "{moment(at)} - " so far, and reads them faster than they come until it is current. The counts below go to then." + "Catching up: complete event coverage reaches "{moment(at)} + ". Later time is still unread; partial totals below are lower bounds."

})} @@ -201,6 +225,59 @@ fn Dashboard(counted: ActivityPayload) -> impl IntoView { } } +/// The source and durable checkpoints behind the figures, before the meter. +#[component] +fn Coverage(counted: ActivityPayload) -> impl IntoView { + let started = counted.verified_since; + let reached = counted.reached; + let has_coverage = started.zip(reached).is_some_and(|(since, to)| since <= to); + let status = if has_coverage { "Coverage checked" } else { "Coverage building" }; + let gap_note = match counted.gaps.len() { + 0 => "No recorded gaps in the time read. Time before verification began and time still unread are excluded.", + _ => "There are recorded gaps. Days touching a gap are partial; complete rolling counts exclude those windows.", + }; + view! { +
+
+

"Hypersnap event coverage"

+ {status} +
+

+ "Hypersnap event API" + " · Snapshot updated "{checkpoint_time(counted.meta.fetched_at.div_euclid(1_000))} +

+
+
"Verified since"
{started.map(checkpoint_time).unwrap_or_else(|| "Waiting for both shards".into())}
+
"Complete through"
{reached.map(checkpoint_time).unwrap_or_else(|| "No common completed coverage yet".into())}
+
+
    + {counted.shards.into_iter().map(|shard| { + let through = shard.reached.map(|at| format!("Complete through {}", checkpoint_time(at))) + .unwrap_or_else(|| "Waiting for a completed block".into()); + let progress = shard.read_events.zip(shard.total_events).map(|(read, total)| { + format!("{} / {} events read · {}", full_number(u64::from(read)), full_number(u64::from(total)), if read == total { "block complete" } else { "partial block" }) + }).unwrap_or_else(|| "Waiting for Hypersnap events".into()); + view! { +
  • +

    {format!("Shard {}", shard.shard)}

    +

    {through}

    +

    {progress}

    + {shard.block.zip(shard.block_at).map(|(block, at)| view! { +
    + "Checkpoint" +

    {format!("Block {} · {}", full_number(block), checkpoint_time(at))}

    +

    "Event cursor "{shard.cursor.unwrap_or_default().to_string()}

    +
    + })} +
    • + } + }).collect_view()} +
+

{gap_note}" Partial day totals and app totals with unresolved keys are lower bounds for their stated windows. This checks the events read from this provider; it does not independently audit the whole network."

+
+ } +} + /// SnapMeter's meter: the past 24 hours against the 24 before, with today /// so far beside them; then yesterday, the past 30 days and the 7-day /// average; and today so far against a typical day. @@ -209,6 +286,8 @@ fn Meter(counted: ActivityPayload, today: i64) -> impl IntoView { let reached = counted.reached; let read = |on: i64| counted.days.iter().find(|d| d.day == on && reached.is_some_and(|at| day(at) >= on)).cloned(); let (so_far, before) = (read(today), read(today - 1)); + let today_lower_bound = reached.is_none_or(|at| !counted.covers(today * 86_400, at)) + || counted.shards.iter().any(|s| s.read_events != s.total_events); let begun_today = counted.since.is_some_and(|since| day(since) >= today); // How far the counts go: the dot is a status, its words say which. let status = reached.map(|at| { @@ -284,22 +363,22 @@ fn Meter(counted: ActivityPayload, today: i64) -> impl IntoView {

})} - + {match (&so_far, reached) { - (Some(_), Some(at)) => format!("00:00 to {}", clock(at)), + (Some(_), Some(at)) => format!("{} to {}", hm(counted.since.unwrap_or(today * 86_400).max(today * 86_400)), clock(at)), _ => "not read yet".to_owned(), }}
- + {match &before { Some(d) => day_note(d, reached), None if begun_today => "before counting began".to_owned(), None => "not read yet".to_owned(), }} - + {match month { Some(w) => { let to = format!("to the end of {}", date(w.day, false)); @@ -325,12 +404,16 @@ fn Tile( label: &'static str, count: Option, #[prop(optional)] kind: &'static str, + #[prop(optional)] lower_bound: bool, #[prop(optional)] children: Option, ) -> impl IntoView { view! {

{label}

-

{count.map_or_else(|| "—".to_owned(), full_number)}

+

+ {(lower_bound && count.is_some()).then(|| view! { "≥ " })} + {count.map_or_else(|| "—".to_owned(), full_number)} +

{children.map(|c| c())}

} @@ -474,7 +557,7 @@ fn by_day(counted: &ActivityPayload, today: i64, span: i64) -> Option { legend, summary, unit: "days", - head: vec!["Day (UTC)", "Active accounts", "Messages"], + head: vec!["Day (UTC)", "Active accounts", "Merge events"], rows, }) } @@ -752,7 +835,7 @@ fn Mix(counted: ActivityPayload) -> impl IntoView { view! {

"What they did"

-

{format!("{} merged {when}, by kind.", plural(total, "message", "messages"))}

+

{format!("{} {when}, by kind.", plural(total, "message merge event", "message merge events"))}

    {families.into_iter().map(|(name, n)| view! {
  1. @@ -826,12 +909,12 @@ fn Apps(apps: AppsWindow, accounts: Vec) -> impl IntoView { #[component] fn Method(counted: ActivityPayload) -> impl IntoView { let gaps = if counted.gaps.is_empty() { - "None: every block since counting began has been read.".to_owned() + "No recorded gaps within the completed coverage shown above.".to_owned() } else { let spans: Vec = counted.gaps.iter().map(|(a, b)| format!("{} to {}", moment(*a), moment(*b))).collect(); format!( - "Not read: {}. Windows and days that touch a gap are left out or marked partly counted.", + "Not fully covered: {}. Missing source history or expired local data leave windows out or mark days partly counted.", spans.join("; ") ) }; @@ -843,7 +926,7 @@ fn Method(counted: ActivityPayload) -> impl IntoView { let mut terms: Vec<(&str, String)> = vec![ ( "Active", - "At least one message merged into Farcaster: a cast, reaction, follow, profile change, verification, username proof, frame action, storage lending, key or channel message, or the removal of one. That is SnapMeter's closed list for Snapchain, message types 1–8 and 11–21. Pruned, revoked and failed messages, and onchain events, don't count. Each account counts once in a window, however many messages it sent, on whichever shard.".to_owned(), + "At least one message merged into Farcaster: a cast, reaction, follow, profile change, verification, username proof, frame action, storage lending, key or channel message, or the removal of one. That is SnapMeter's closed list for Snapchain, message types 1–8 and 11–21. Pruned, revoked and failed messages, and onchain events, don't count. Each account counts once in a window, however many messages it sent, on whichever shard. The action chart counts message merge events: a message present on both shards contributes to both there; account totals still count its account once.".to_owned(), ), ( "Past 24 hours", @@ -863,7 +946,7 @@ fn Method(counted: ActivityPayload) -> impl IntoView { ("Gaps", gaps), ( "Exact", - "Every account is kept, as a compact set per hour and per day, so a window counts each one once. Nothing is sampled or estimated.".to_owned(), + "Every account in the verified events is kept as a compact set per hour and per day. A fully covered window counts each account once, without sampling. Partial counts are lower bounds; events outside verified coverage have not been counted.".to_owned(), ), ]; if counted.hourly.iter().any(Option::is_some) { diff --git a/crates/allthing-worker/src/index.rs b/crates/allthing-worker/src/index.rs index 485b166a..c2de60c0 100644 --- a/crates/allthing-worker/src/index.rs +++ b/crates/allthing-worker/src/index.rs @@ -27,7 +27,7 @@ use worker::wasm_bindgen::JsValue; use worker::{D1Database, Env}; /// The schema, from the migrations that also create it. -const SCHEMA: [&str; 10] = [ +const SCHEMA: [&str; 11] = [ include_str!("../../../migrations/0001_index.sql"), include_str!("../../../migrations/0002_snapshots.sql"), include_str!("../../../migrations/0003_activity.sql"), @@ -38,6 +38,7 @@ const SCHEMA: [&str; 10] = [ include_str!("../../../migrations/0008_poll.sql"), include_str!("../../../migrations/0009_presence.sql"), include_str!("../../../migrations/0010_media_uploads.sql"), + include_str!("../../../migrations/0011_verified_activity.sql"), ]; /// Store casts given as one JSON array of `[hash, ts, points]`, updating the @@ -62,52 +63,90 @@ const STATE: &str = "SELECT value FROM state WHERE key = 'indexer'"; const METAS: &str = "SELECT name, built, total, floor FROM rankings"; const SAVE_STATE: &str = "REPLACE INTO state (key, value) VALUES ('indexer', ?1)"; const SNAPSHOT: &str = "SELECT built, body FROM snapshots WHERE name = ?1"; -const ACTIVITY_STATE: &str = "SELECT value FROM state WHERE key = 'activity'"; +const ACTIVITY_STATE: &str = "SELECT value FROM state WHERE key = 'activity_v2'"; +/// Compare the exact state read by this run, then fence every mutation in +/// the same D1 transaction with a unique runtime-generated token. Identical +/// candidate states from overlapping cron runs cannot impersonate a winner. +const CLAIM_ACTIVITY: &str = "INSERT INTO state (key, value) \ + SELECT 'activity_v2_fence', ?2 WHERE (SELECT value FROM state WHERE key = 'activity_v2') IS ?1 \ + ON CONFLICT(key) DO UPDATE SET value = excluded.value RETURNING value"; const ACTIVITY_SETS: &str = "SELECT name, body FROM activity WHERE name IN (SELECT value FROM json_each(?1))"; /// Store sets given as one JSON array of `[name, body]`. const STORE_SETS: &str = "INSERT INTO activity (name, body) \ - SELECT json_extract(value, '$[0]'), json_extract(value, '$[1]') FROM json_each(?1) WHERE true \ + SELECT json_extract(value, '$[0]'), json_extract(value, '$[1]') FROM json_each(?1) \ + WHERE (SELECT value FROM state WHERE key = 'activity_v2_fence') = ?2 \ ON CONFLICT (name) DO UPDATE SET body = excluded.body"; -/// Drop hourly sets (of fids, `h`, and of keys, `k`) before hour ?1 and -/// daily ones (a day's, `d`, and a week's to a day, `w`) before day ?2. +/// Retain legacy rows; expire only this verified epoch's sets under its fence. const DROP_SETS: &str = "DELETE FROM activity WHERE \ - (substr(name, 1, 1) IN ('h', 'k') AND CAST(substr(name, 2) AS INTEGER) < ?1) OR \ - (substr(name, 1, 1) IN ('d', 'w') AND CAST(substr(name, 2) AS INTEGER) < ?2)"; -/// Keys' apps, given as one JSON array of `[pair, app]`. A key's app never -/// changes, so a key already there is left as it is. -const STORE_KEYS: &str = "INSERT OR IGNORE INTO signer_apps (pair, app) \ - SELECT json_extract(value, '$[0]'), json_extract(value, '$[1]') FROM json_each(?1)"; -/// The apps of the keys given as one JSON array. Pairs reach 2^63, past a -/// JavaScript number: they come back as text. -const KEY_APPS: &str = "SELECT CAST(pair AS TEXT) AS pair, app FROM signer_apps \ - WHERE pair IN (SELECT value FROM json_each(?1))"; -/// Accounts active by app among the keys given as one JSON array, the app -/// NULL for keys not looked up yet, which come back too. -const COUNT_APPS: &str = "SELECT k.app AS app, COUNT(DISTINCT j.value >> 32) AS active, \ - json_group_array(CAST(j.value AS TEXT)) FILTER (WHERE k.pair IS NULL) AS unknown \ - FROM json_each(?1) AS j LEFT JOIN signer_apps AS k ON k.pair = j.value GROUP BY k.app"; -/// The apps whose keys the accounts given as one JSON array of fids have -/// signed with: each account's keys are the pairs from `fid << 32` to -/// `fid << 32 | 0xffffffff` (the next fid's start would pass D1's integers -/// for the last fid kept). Fids and apps only: pairs are past a JavaScript -/// number. + ((substr(name, 1, 3) IN ('v2h', 'v2k') AND CAST(substr(name, 4) AS INTEGER) < ?1) OR \ + (substr(name, 1, 3) IN ('v2d', 'v2w') AND CAST(substr(name, 4) AS INTEGER) < ?2)) \ + AND (SELECT value FROM state WHERE key = 'activity_v2_fence') = ?3"; +/// Full FID/public-key identities, not truncated key prefixes. Positive, +/// immutable registration facts may be shared by concurrent readers. +const STORE_KEYS: &str = "INSERT OR IGNORE INTO verified_signer_apps (identity, fid, app) \ + SELECT json_extract(value, '$[0]'), CAST(substr(json_extract(value, '$[0]'), 1, 10) AS INTEGER), \ + json_extract(value, '$[1]') FROM json_each(?1)"; +const STORE_ACTIVITY_KEYS: &str = "INSERT OR IGNORE INTO verified_signer_apps (identity, fid, app) \ + SELECT json_extract(value, '$[0]'), CAST(substr(json_extract(value, '$[0]'), 1, 10) AS INTEGER), \ + json_extract(value, '$[1]') FROM json_each(?1) \ + WHERE (SELECT value FROM state WHERE key = 'activity_v2_fence') = ?2"; +const KEY_APPS: &str = "SELECT identity, app FROM verified_signer_apps \ + WHERE identity IN (SELECT value FROM json_each(?1))"; +/// Count directly over the 24 stored hourly arrays, replacing rows that +/// this run just changed. Full keys stay in D1; the Worker receives only +/// counts and a bounded lookup backlog. Validate canonical sorted arrays +/// and their global account cardinality before any result may be used. +const COUNT_APPS: &str = "WITH replacements AS (\ + SELECT json_extract(value, '$[0]') AS name, json_extract(value, '$[1]') AS body FROM json_each(?2)), \ + sources AS (SELECT a.name, a.body FROM activity a \ + WHERE a.name IN (SELECT value FROM json_each(?1)) \ + AND NOT EXISTS (SELECT 1 FROM replacements r WHERE r.name = a.name) \ + UNION ALL SELECT name, body FROM replacements WHERE name IN (SELECT value FROM json_each(?1))), \ + entries AS (SELECT s.name, j.value AS identity, j.type AS kind, \ + LAG(j.value) OVER (PARTITION BY s.name ORDER BY CAST(j.key AS INTEGER)) AS prior \ + FROM sources s JOIN json_each(CASE WHEN json_valid(s.body) AND json_type(s.body) = 'array' \ + THEN s.body ELSE json('invalid activity key array') END) j), \ + checked AS (SELECT COALESCE(SUM(CASE WHEN kind = 'text' AND length(identity) = 77 AND length(CAST(identity AS BLOB)) = 77 \ + AND substr(identity, 11, 3) = ':0x' AND substr(identity, 14) NOT GLOB '*[^0-9a-f]*' \ + AND CAST(substr(identity, 1, 10) AS INTEGER) BETWEEN 1 AND 4294967295 \ + AND substr(identity, 1, 10) = printf('%010d', CAST(substr(identity, 1, 10) AS INTEGER)) \ + AND (prior IS NULL OR prior < identity) THEN 0 ELSE 1 END), 0) \ + + (SELECT COUNT(*) - COUNT(DISTINCT name) FROM sources) AS invalid, \ + COUNT(DISTINCT substr(identity, 1, 10)) AS accounts FROM entries), \ + identities AS (SELECT DISTINCT identity FROM entries), \ + known AS (SELECT e.identity, k.app FROM identities e \ + LEFT JOIN verified_signer_apps k ON k.identity = e.identity), \ + counts AS (SELECT app, COUNT(DISTINCT substr(identity, 1, 10)) AS active FROM known GROUP BY app) \ + SELECT c.app, c.active, CASE WHEN c.app IS NULL THEN \ + (SELECT json_group_array(identity) FROM (SELECT identity FROM known WHERE app IS NULL ORDER BY identity LIMIT 400)) \ + ELSE '[]' END AS unknown, checked.invalid, checked.accounts FROM counts c CROSS JOIN checked \ + UNION ALL SELECT NULL, 0, '[]', invalid, accounts FROM checked"; +/// Exact identities also support FIDs above the old signed-integer cutoff. const APPS_OF: &str = "SELECT j.value AS fid, s.app AS app FROM json_each(?1) AS j \ - JOIN signer_apps AS s ON s.pair BETWEEN (j.value << 32) AND ((j.value << 32) | 4294967295) \ + JOIN verified_signer_apps AS s ON s.fid = j.value \ GROUP BY j.value, s.app"; /// The later of the week's sets to the ends of days ?1 and ?2 (`w`), /// if kept: one row. const LATEST_WEEK: &str = "SELECT name, body FROM activity WHERE name IN (?1, ?2) ORDER BY name DESC LIMIT 1"; -const SAVE_ACTIVITY_STATE: &str = "REPLACE INTO state (key, value) VALUES ('activity', ?1)"; +const SAVE_ACTIVITY_STATE: &str = "INSERT INTO state (key, value) \ + SELECT 'activity_v2', ?1 WHERE (SELECT value FROM state WHERE key = 'activity_v2_fence') = ?2 \ + ON CONFLICT(key) DO UPDATE SET value = excluded.value"; +const SAVE_ACTIVITY_SNAPSHOT: &str = "INSERT INTO snapshots (name, built, body) \ + SELECT ?1, ?2, ?3 WHERE (SELECT value FROM state WHERE key = 'activity_v2_fence') = ?4 \ + ON CONFLICT(name) DO UPDATE SET built = excluded.built, body = excluded.body"; /// Emoji reactions, given as one JSON array of `[parent, hash, emoji, fid, /// ts]`; one already kept stays as it is. const STORE_REACTIONS: &str = "INSERT OR IGNORE INTO reactions (hash, parent, emoji, fid, ts) \ SELECT json_extract(value, '$[1]'), json_extract(value, '$[0]'), json_extract(value, '$[2]'), \ - json_extract(value, '$[3]'), json_extract(value, '$[4]') FROM json_each(?1)"; + json_extract(value, '$[3]'), json_extract(value, '$[4]') FROM json_each(?1) \ + WHERE (SELECT value FROM state WHERE key = 'activity_v2_fence') = ?2"; /// Drop the reactions among the casts given as one JSON array: removed, /// revoked or pruned. -const DROP_REACTIONS: &str = "DELETE FROM reactions WHERE hash IN (SELECT value FROM json_each(?1))"; +const DROP_REACTIONS: &str = "DELETE FROM reactions WHERE hash IN (SELECT value FROM json_each(?1)) \ + AND (SELECT value FROM state WHERE key = 'activity_v2_fence') = ?2"; /// Drop reactions from before ?1, Unix seconds. -const OLD_REACTIONS: &str = "DELETE FROM reactions WHERE ts < ?1"; +const OLD_REACTIONS: &str = "DELETE FROM reactions WHERE ts < ?1 \ + AND (SELECT value FROM state WHERE key = 'activity_v2_fence') = ?2"; /// The reactions to the casts given as one JSON array, oldest first. const REACTIONS: &str = "SELECT parent, hash, emoji, fid, ts FROM reactions \ WHERE parent IN (SELECT value FROM json_each(?1)) ORDER BY ts, hash LIMIT 2000"; @@ -247,7 +286,7 @@ struct ReactionRow { #[derive(Deserialize)] struct KeyRow { - pair: String, + identity: String, app: f64, } @@ -256,6 +295,8 @@ struct AppRow { app: Option, active: f64, unknown: String, + invalid: f64, + accounts: f64, } #[derive(Deserialize)] @@ -491,11 +532,10 @@ impl Index for D1Index { .boxed_local() } - /// Fids of 2^31 or more have no keys kept (`activity::key_pair`), and - /// would take the pairs' range past D1's integers: they are left out. + /// Exact signer identities include every supported unsigned 32-bit FID. fn apps_of(&self, fids: &[u64]) -> LocalBoxFuture<'static, Result, IndexError>> { let db = self.db.clone(); - let fids: Vec = fids.iter().copied().filter(|&fid| fid < 1 << 31).collect(); + let fids: Vec = fids.iter().copied().filter(|&fid| (1..=u64::from(u32::MAX)).contains(&fid)).collect(); async move { if fids.is_empty() { return Ok(Vec::new()); @@ -517,7 +557,7 @@ impl Index for D1Index { let names: [JsValue; 2] = [week_set(today - 1).into(), week_set(today - 2).into()]; let statement = db.prepare(LATEST_WEEK).bind(&names).map_err(failed)?; let Some(row) = statement.first::(None).await.map_err(failed)? else { return Ok(None) }; - let day = row.name.strip_prefix('w').and_then(|day| day.parse().ok()); + let day = row.name.strip_prefix("v2w").and_then(|day| day.parse().ok()); let week = Fids::decode(&row.body); match (day, week) { (Some(day), Some(week)) => Ok(Some(ActiveWeek::among(day, &week, &fids))), @@ -613,21 +653,23 @@ impl ActivityStore for D1Index { .boxed_local() } - fn key_apps(&self, pairs: Vec) -> LocalBoxFuture<'static, Result, IndexError>> { + fn key_apps(&self, pairs: Vec) -> LocalBoxFuture<'static, Result, IndexError>> { let db = self.db.clone(); async move { let pairs = serde_json::to_string(&pairs).map_err(|e| IndexError(e.to_string()))?; let statement = db.prepare(KEY_APPS).bind(&[pairs.into()]).map_err(failed)?; let rows = statement.all().await.map_err(failed)?.results::().map_err(failed)?; - Ok(rows.into_iter().filter_map(|r| Some((r.pair.parse().ok()?, r.app as u64))).collect()) + Ok(rows.into_iter().map(|r| (r.identity, r.app as u64)).collect()) } .boxed_local() } fn count_apps( &self, - keys: Vec<(u64, u64)>, - chunks: Vec>, + keys: Vec<(String, u64)>, + names: Vec, + overrides: Vec<(String, String)>, + expected_active: u64, ) -> LocalBoxFuture<'static, Result, IndexError>> { let db = self.db.clone(); async move { @@ -636,20 +678,27 @@ impl ActivityStore for D1Index { statements.push(db.prepare(STORE_KEYS).bind(&[to_json(&keys)?.into()]).map_err(failed)?); } let stored = statements.len(); - for chunk in &chunks { - statements.push(db.prepare(COUNT_APPS).bind(&[to_json(chunk)?.into()]).map_err(failed)?); - } + statements.push( + db.prepare(COUNT_APPS).bind(&[to_json(&names)?.into(), to_json(&overrides)?.into()]).map_err(failed)?, + ); let mut out = Vec::new(); - if statements.is_empty() { - return Ok(out); - } + let mut verified = false; for result in db.batch(statements).await.map_err(failed)?.into_iter().skip(stored) { for row in result.results::().map_err(failed)? { - let unknown: Vec = serde_json::from_str(&row.unknown).unwrap_or_default(); - let unknown = unknown.iter().filter_map(|p| p.parse().ok()).collect(); + if row.invalid != 0.0 || row.accounts != expected_active as f64 { + return Err(IndexError( + "Activity key window failed integrity or account cardinality checks".into(), + )); + } + verified = true; + let unknown: Vec = serde_json::from_str(&row.unknown) + .map_err(|e| IndexError(format!("Invalid unknown signer identities: {e}")))?; out.push((row.app.map(|app| app as u64), row.active as u64, unknown)); } } + if !verified { + return Err(IndexError("Missing activity key-window validation result".into())); + } Ok(out) } .boxed_local() @@ -658,33 +707,65 @@ impl ActivityStore for D1Index { fn save_activity(&self, saving: Saving) -> LocalBoxFuture<'static, Result<(), IndexError>> { let db = self.db.clone(); async move { - let Saving { sets, keep_from, keys, reactions, removed, reactions_from, state, summary } = saving; - let mut statements = Vec::new(); + let Saving { sets, keep_from, keys, reactions, removed, reactions_from, expected_state, state, summary } = + saving; + let token = crate::auth::random_hex(16) + .ok_or_else(|| IndexError("Could not create an activity transaction fence".into()))?; + let expected = expected_state.map_or(JsValue::NULL, Into::into); + let mut statements = + vec![db.prepare(CLAIM_ACTIVITY).bind(&[expected, token.as_str().into()]).map_err(failed)?]; if !sets.is_empty() { - let rows = serde_json::to_string(&sets).map_err(|e| IndexError(e.to_string()))?; - statements.push(db.prepare(STORE_SETS).bind(&[rows.into()]).map_err(failed)?); + statements.push( + db.prepare(STORE_SETS).bind(&[to_json(&sets)?.into(), token.as_str().into()]).map_err(failed)?, + ); } if !keys.is_empty() { - let rows = serde_json::to_string(&keys).map_err(|e| IndexError(e.to_string()))?; - statements.push(db.prepare(STORE_KEYS).bind(&[rows.into()]).map_err(failed)?); + statements.push( + db.prepare(STORE_ACTIVITY_KEYS) + .bind(&[to_json(&keys)?.into(), token.as_str().into()]) + .map_err(failed)?, + ); } - let keep = [number(keep_from.0 as f64), number(keep_from.1 as f64)]; + let keep = [number(keep_from.0 as f64), number(keep_from.1 as f64), token.as_str().into()]; statements.push(db.prepare(DROP_SETS).bind(&keep).map_err(failed)?); if !reactions.is_empty() { let rows: Vec<_> = reactions.into_iter().map(|(cast, r)| (cast, r.hash, r.emoji, r.fid, r.ts)).collect(); - let rows = serde_json::to_string(&rows).map_err(|e| IndexError(e.to_string()))?; - statements.push(db.prepare(STORE_REACTIONS).bind(&[rows.into()]).map_err(failed)?); + statements.push( + db.prepare(STORE_REACTIONS) + .bind(&[to_json(&rows)?.into(), token.as_str().into()]) + .map_err(failed)?, + ); } if !removed.is_empty() { - let hashes = serde_json::to_string(&removed).map_err(|e| IndexError(e.to_string()))?; - statements.push(db.prepare(DROP_REACTIONS).bind(&[hashes.into()]).map_err(failed)?); + statements.push( + db.prepare(DROP_REACTIONS) + .bind(&[to_json(&removed)?.into(), token.as_str().into()]) + .map_err(failed)?, + ); + } + statements.push( + db.prepare(OLD_REACTIONS) + .bind(&[number(reactions_from as f64), token.as_str().into()]) + .map_err(failed)?, + ); + statements + .push(db.prepare(SAVE_ACTIVITY_STATE).bind(&[state.into(), token.as_str().into()]).map_err(failed)?); + let args = [SUMMARY.into(), number(summary.built.floor()), summary.body.into(), token.as_str().into()]; + statements.push(db.prepare(SAVE_ACTIVITY_SNAPSHOT).bind(&args).map_err(failed)?); + // D1 batches are atomic. Even a losing claim executes the batch, + // but its distinct fence makes every following mutation a no-op. + let first = db + .batch(statements) + .await + .map_err(failed)? + .into_iter() + .next() + .ok_or_else(|| IndexError("Missing activity transaction result".into()))?; + let claimed = first.results::().map_err(failed)?; + if claimed.len() != 1 || claimed[0].value != token { + return Err(IndexError("Activity changed during this run; retained the winning checkpoint".into())); } - statements.push(db.prepare(OLD_REACTIONS).bind(&[number(reactions_from as f64)]).map_err(failed)?); - statements.push(db.prepare(SAVE_ACTIVITY_STATE).bind(&[state.into()]).map_err(failed)?); - let args = [SUMMARY.into(), number(summary.built.floor()), summary.body.into()]; - statements.push(db.prepare(SAVE_SNAPSHOT).bind(&args).map_err(failed)?); - db.batch(statements).await.map_err(failed)?; Ok(()) } .boxed_local() diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index cc50ccc7..677439c5 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -205,34 +205,26 @@ Hypersnap ranks trending casts over 7 days at most, and its search ignores dates ## Daily active accounts -Allthing counts Farcaster's daily active accounts as [SnapMeter](https://github.com/ael-dev3/SnapMeter) does (its `docs/metrics.md`): an account is active when one of its messages was merged into the network, and each account counts once in a window, whatever it did. SnapMeter reads every node's events with a collector of its own; Allthing reads the same canonical events from Hypersnap's public node, in the Worker, as its newest casts are read. The logic is in `allthing_core::activity`, the run in `allthing_data::activity`, and the SQL in `crates/allthing-worker/src/index.rs`. - -- **What counts.** A successful `MERGE_MESSAGE` whose type is on SnapMeter's closed list for Snapchain (types 1–8 and 11–21): casts, reactions, follows, verifications, profile changes, username proofs, frame actions, storage lending, keys and channels, and their removals. Pruned, revoked and failed merges, onchain events and blocks don't count, nor do types the list doesn't name. -- **When.** The time of the block that confirmed the message: its `BLOCK_CONFIRMED` event comes first. If that event is on an earlier page, the message's signed time is used instead. -- **The follower** is a cron trigger of its own, every 2 minutes. Each run reads the next page of up to 1,000 events on each shard from where the last run stopped (the cursors are in D1's `state`), and adds each active fid to its UTC hour's and UTC day's set. A first run starts at the start of yesterday, estimated from the newest block at a block a second, so yesterday is counted whole. It catches up about 6 times faster than events come (5.4 to 6.3 times, measured after the first deploy), gaining about 5 hours on the clock an hour: about 5 hours after a deploy just after midnight UTC, up to 10 after one late in the day. A shard whose page can't be read is read again from the same place. If the node no longer has the blocks after a cursor (an outage longer than its retention, about 3 days), the stretch is recorded as a gap. -- **Sets, not rows.** Each hour's and day's active fids are one sorted set (`Fids`), stored as gaps between fids in LEB128 and base64. Days are UTC calendar days. An hour ends on its last second, `(h, h + 1 h]`, as SnapMeter's windows do (`bucket`), so the 24 hours before a time on the hour are exactly SnapMeter's rolling window `(t − 24 h, t]`. A day of 6,000 accounts is about 16 KB, an hour's about 2 KB. So the counts are exact (an account seen on both shards, or in many hours, counts once), yet a run writes 3 or 4 rows. Unions are counted with a bitmap over fids, in linear time. Hourly sets are kept 49 hours and daily ones 30 days, in D1's `activity` table. -- **What it serves.** After each run the follower stores the payload `/api/v1/activity` serves: - - the 24 hours to the start of the hour in progress (`rolling_to`), and the 24 before them; - - each of those 48 hours' own count (`hourly`, the length of its set, which the run reads for the two windows anyway), an account active in several hours counting in each; - - the 30 days to the end of yesterday, counted once a day while keeping up; - - each UTC day's count, with its messages by kind. - - A window that wasn't read throughout (before counting began, or across a gap) is left out, as is an hour (`null`), and a day that wasn't is marked incomplete. -- **By app.** The key that signed a message says which app sent it: the app whose fid requested the key (`provenance`). The follower keeps each hour's signing keys too, as a set (`Keys`) of `key_pair`s, the account's fid above the key's first four bytes, in `activity` as `k`. The app of each key is kept in D1's `signer_apps` and never changes, so it is learned once: - - from the event stream, for a key added onchain while the follower reads (its signed key request names the app); - - otherwise by looking up the account's active keys (`signersByFid`), or its onchain history for a key removed since: up to 24 accounts a run, 6 while catching up. The keys of the run's own messages go first, then those the last count didn't know. - - Once an hour, D1 counts the past 24 hours' keys by app, `COUNT(DISTINCT fid)` per app in runs that never split an account (`Keys::chunks`), so an account that used two apps counts for each, exactly. Keys whose app isn't known yet are counted apart. While they are more than 1% of accounts, the count is redone every 10 minutes as lookups proceed (after a deploy, about 10 hours). The result (`AppsWindow`) is `apps_24h` in `/api/v1/activity`, and the front page's payload carries it for the apps its casts came from: one D1 read, on the first page's build. The first count comes 24 hours after a deploy, once a whole window of keys has been kept. -- **For people lists.** Two reads, a D1 query each, let a Followers or Following page say which apps its people use and who of them was active this week (`Index::apps_of`, `Index::active_week`; no page asks yet). - - **Apps:** the keys the follower learned (`signer_apps`), each account's in the range of its pairs, `fid << 32` to `fid << 32 | 0xffffffff`. They come back as fids and apps, never as pairs, which are past a JavaScript number: the apps whose keys a person has signed with since counting by app began (20 October 2026). - - **This week:** once a day, with the 30-day count, the follower keeps the union of the 7 days to the end of yesterday as one more set (`w`, dropped with the days after 30). It keeps it only once those days were read in full (`Follow::covered`), so from about 27 October 2026. A page reads yesterday's, or before the day's pass the day before's (one row), decodes it once, and looks each person up in it (`Fids::contains`, a binary search). -- **Emoji reactions, for listings.** The same read picks out emoji reactions, replies that are one Pepe and nothing else (`emoji::reaction`, without building any other cast's text), and the casts the page removes, or whose reactions it revokes or prunes (`ActivityPage::reactions`, `removed`). The run keeps the reactions in D1's `reactions` (reply hash, the cast it replies to, emoji, fid, block time) in the batch that saves the rest, drops those taken back, and those older than 400 days. A listing's build (home, new, channel, profile, `annotate_listing`) reads its casts' reactions with one D1 query and carries them in `notes.reactions`, for chips on its cards; a read that can't be made keeps the last build's, and the static copy has none. Reactions before the follower first ran aren't there. -- **The page.** `/dau` (`pages/dau.rs`) lays the payload out after SnapMeter's dashboard, with only what the follower counts. Its one request is `/api/v1/activity`, which the inline script starts. - - **The meter**, as SnapMeter's panel: the past 24 hours, their change on the 24 before as a count and a percent, today so far beside them, and how far reading has got (up to date, or catching up). Under them, readouts of yesterday, the past 30 days, and the 7-day average: the mean of the last 7 days read whole against the 7 before (`activity::weeks`, only once all 14 were). Then today so far against the median of the month's days read whole (`median_day`, once a week of them was). - - **The history**, over SnapMeter's ranges: the 48 hours by the hour (the 24 the meter counts, and the 24 before them lighter), or the 7 or 30 days by UTC day (partly counted days lighter). Each range is drawn once and a button shows it. Bars are thin, on one value axis of round steps (`format::axis`). A bar pointed at, tapped or reached with the arrow keys shows its count and when: above a short bar, beside a tall one's top so it never covers the buttons, with no native tooltip. Each range has a legend, a sentence (the average of the days read whole and the busiest, or the busiest hour), and a table of its days or hours. - - **The rest**: the last whole day's messages by kind, a bar each; accounts by app with the apps' own accounts (the 20 apps with the most, read once a day by the count by app, one `user/bulk` read); and SnapMeter's definitions, term by term: what counts, the windows, the averages, by hour and by app, the source and since when, and any gaps. - - On the client bar's cards, an app's count of active accounts links to the page, and a line under it says how the count is made: any message (a cast, reaction, follow…) signed with a key the app requested counts its account once, from Hypersnap's events. Today and yesterday follow the clock, not how far reading has got: while the cron catches up after a deploy or an outage, the page says how far it has read, marks the day being read as read so far (not as today), keeps today's slot in the chart, empty and unlabelled (the chart's labels, a week apart, count back from the newest day with a bar), and shows what accounts did on the last day read whole, or else on the day being read, said so. The static copy has no count, and says where to find it. +Allthing counts unique Farcaster accounts with at least one successful message merge in a window, using [SnapMeter's definitions](https://github.com/ael-dev3/SnapMeter/blob/main/docs/metrics.md). These are accounts, not people or Allthing readers. Its Worker follows the two data shards of [Hypersnap's public event API](https://haatz.quilibrium.com/v1/events). The logic is in `allthing_core::activity`, the follower in `allthing_data::activity`, and the SQL in `crates/allthing-worker/src/index.rs`. + +- **Source contract and its limits.** The counter checks the ordered events this Hypersnap node supplies. The endpoint and block metadata are described in the pinned [Hypersnap event API](https://github.com/farcasterorg/hypersnap/blob/bdef7b75c1fdf1c84611abfa6e1457ea6261f8ee/site/docs/pages/reference/httpapi/events.md), [event definitions](https://github.com/farcasterorg/hypersnap/blob/bdef7b75c1fdf1c84611abfa6e1457ea6261f8ee/proto/definitions/hub_event.proto) and [event store](https://github.com/farcasterorg/hypersnap/blob/bdef7b75c1fdf1c84611abfa6e1457ea6261f8ee/src/storage/store/account/block_event_store.rs). Read-only samples on 27 October 2026 matched the declared totals of 147 complete blocks across both shards; 30 sampled Ed25519 signatures verified over the hashes supplied by the node. Those samples establish observed endpoint behavior, not a full-window count, the deployed binary's revision, independently recomputed message hashes or signer authorization. The runtime does not independently attest consensus, detect reorgs or audit the whole network. +- **What counts.** A successful `MERGE_MESSAGE` whose type is on SnapMeter's closed Snapchain list, types 1–8 and 11–21: casts, reactions, follows, verifications, profile changes, username proofs, frame actions, storage lending, keys and channels, including removals. Pruned, revoked and failed merges, onchain events, blocks and unknown message types do not count. The action chart and daily action totals count **message merge events**, not unique message hashes: a shared message can appear on both shards and contribute twice there. Account sets deduplicate the account across messages and shards. +- **Canonical time and complete blocks.** Ascending events must match their requested shard, `id >> 14` block number and consecutive within-block sequence. A block starts with sequence-zero `BLOCK_CONFIRMED`; its declared `totalEvents` includes that confirmation. Its timestamp, declared total and progress are carried across page boundaries. A signed message's timestamp never substitutes for a missing confirmation. A malformed or unsupported page contributes nothing and leaves that shard's cursor retryable; a skipped block or unfinished predecessor records a gap rather than certifying the missing interval. Future block times are rejected. Several consecutive blocks can share one timestamp, so completion at second `T` advances the coverage watermark only through `T − 1`; the first observed second is excluded from the verified start for the same reason. +- **The verified epoch.** Format version 2 uses separate `activity_v2` state and snapshot names, `v2h`/`v2d`/`v2k`/`v2w` set names and `verified_signer_apps` (`migrations/0011_verified_activity.sql`). Legacy rows remain stored and are never retrospectively certified or mixed into the new totals. Invalid saved v2 state fails closed instead of silently restarting; a newly built summary must pass the same evidence validator before publication. Each shard keeps its actual start, completed watermark, cursor, current block timestamp and read/declared event counts. The common start is the later shard start and common completion the earlier watermark; both configured shards must be present. A complete interval requires both shards throughout, no intersecting gap and no unread time. Initial, partial and gapped counts are lower bounds; missing complete windows stay absent. +- **The follower** retains its existing cron trigger, every 2 minutes. Each run reads up to 1,000 events on each shard after its saved cursor. An initial reverse probe estimates a starting block near yesterday; only subsequent canonical events establish the actual verified start. It does not promise that yesterday is covered. A failed shard resumes from the same cursor next run. Retention and event rate determine how much history can be recovered; the node exposed roughly three days in the 27 October sample. The earlier follower caught up 5.4–6.3 times faster than events arrived after its first deploy; that historical measurement is not a v2 runtime measurement or an availability deadline. Complete rolling and by-app counts appear only after a whole qualifying window has been observed. +- **Atomic progress.** A run supplies the exact state it read and claims a fresh, random 128-bit fence inside the existing D1 batch. Every mutable count, reaction, checkpoint and snapshot write checks that fence. A stale concurrent writer cannot change the winner's progress or totals, including when two runs proposed the same revision. The extra claim is a SQL statement and fence-row write in that batch, not another network subrequest. +- **Sets and windows.** Active FIDs are sorted sets (`Fids`), gap-encoded with LEB128 and base64. A window unions account identities, so the same account on several hours or shards counts once; unions do not allocate a bitmap through the largest FID. Daily sets use UTC calendar days; hourly buckets preserve SnapMeter's `(t − 24 h, t]` rolling windows. A 6,000-account day is about 16 KB and an hour about 2 KB. Normally hourly sets remain for 49 hours and daily sets for 30 days. While waiting for an initial shard, retention freezes at the earliest observed start; after an exceptional wait of over 30 days, bounded local expiry records a gap so deleted data can never become a certified zero. These compact FID estimates exclude signing keys, their indexes and retained legacy data; see [BUDGET.md §10](BUDGET.md#10-daily-active-accounts-cron-and-d1). +- **What it serves.** `/api/v1/activity` reads the stored summary. It includes the canonical source URL, verification version, actual verified start and per-shard evidence, plus the last completed rolling hour, its 24-hour and previous 24-hour unions, up to 48 individual hourly counts, the latest 30 daily counts and merge-event mix, and the union of the 30 days to yesterday. A complete window that lacks coverage is omitted; an uncovered hour is `null`, and an incomplete day is marked partial. A partial 30-day union is labelled as such. The summary builder and browser validate source/checkpoint consistency, timestamps and complete-window claims; a version marker alone is insufficient. +- **By app.** An app is the account that requested the signing key. `ExactKeys` stores the full FID and all 32 public-key bytes, rather than a four-byte prefix; `verified_signer_apps` stores the full identity, FID and app, with a FID/app index. Observed onchain registrations can supply attribution directly. Otherwise the follower reads the account's active signers and, for a removed key, its onchain history, matching exact account, key and registration metadata. Missing, malformed, truncated or failed evidence stays **unknown** and retryable. App zero requires an explicit supported no-app registration; absence is never cached as zero. These activity lookups use their own checked source path rather than the looser cast-origin cache. + + At most 24 accounts are selected for lookup per run, or 6 while catching up. `HISTORY = 6` reserves subrequests for onchain paging; it is not a hard six-request cap. The invocation's global 45-subrequest guard still applies. Once a complete 24-hour key window is available, D1 expands its 24 hourly JSON arrays with `json_each` and counts `COUNT(DISTINCT fid)` per app directly. Current-run key-hour rows replace their stored versions in that query. D1 validates canonical, sorted full-identity arrays and requires their global distinct-FID count to equal the independently verified rolling account total before accepting any app result. The query follows immutable attribution inserts in the existing D1 batch, with no extra query round trip. + + The Worker reads only the key hours touched by the current run and receives aggregate counts plus at most 400 unresolved identities, about 32 KB of identity JSON. It does not decode, union or serialize the full 24-hour key window. The total number of accounts with unresolved keys is counted across the whole window, independently of that bounded lookup backlog. An account using two apps counts for each; unresolved keys produce app lower bounds and a known-share note. Windows normally recount hourly, or every 10 minutes while over 1% of accounts still have unresolved attribution. No fixed warm-up time is promised. A first-page Home payload carries only verified app counts for the apps it shows, using its existing D1 read. +- **For account lists.** `Index::apps_of` queries full identities through the verified table's FID/app index; results are FIDs and app FIDs, not encoded key pairs. `Index::active_week` reads the latest kept `v2w` union of seven complete days, only once all seven have verified coverage. The existing optional two-read interface remains; no page asks yet. +- **Emoji reactions, for listings.** The same checked event page extracts replies containing one Pepe, plus the casts removed and reactions revoked or pruned. The fenced save updates `reactions` (reply hash, target cast, emoji, FID and block time), removes withdrawals and entries older than 400 days. Listing builds keep their existing single query and `notes.reactions`; the static copy has none. This does not reconstruct reactions from before observation began. +- **Navigation and the page.** DAU is a persistent main-navigation link on desktop and phones, named “DAU, daily active accounts” for assistive technology, built with the route helper and marked as the current page. The inert boot header reserves the same space. A two-row header through 800 px keeps navigation, presence, search and account controls usable; Library retains its existing phone breakpoint. The static `/Allthing` copy links to its own `/dau` route, which points to allthing.social for counts. + + `/dau` (`pages/dau.rs`) makes one `/api/v1/activity` request, started by the inline loader. Its near-top coverage card links Hypersnap and shows when the snapshot was updated, the verified start, common completion and each shard's completed watermark and current block progress, with expandable cursors. Snapshot update time records the count attempt and can advance after a failed source read; common and per-shard completion timestamps establish how far events were verified. Legacy, future or incoherent proof suppresses the meter, charts and app figures and offers the provider and JSON links. Explicit waiting shard records show coverage building. Partial tiles use `≥` and explain lower bounds. The meter, 48-hour/7-day/30-day charts, keyboard/touch readouts and tables retain their existing behavior; today and yesterday follow the clock while catch-up progress is shown separately. Definitions distinguish unique account counts from merge-event totals and explain the provider scope. ## The browser app @@ -528,7 +520,7 @@ The header's small "● 12" says how many people are reading Allthing now. Hover - **Counted, never named.** The answer is `{"online","readers","next"}`: how many were seen in the window; up to 12 of those shown by name, most recent first, with their names and pictures from Hypersnap's `user/bulk`; and the interval. Nothing else is said about anyone, and the popover says "and N more" for the rest. - **The opt-out.** The popover has a "Show me here" switch for a signed-in reader, on unless they turn it off, kept in localStorage `allthing.presence-hidden`. Off, the browser beats at once with `hidden: true`: the Worker stores the reader as one signed out and deletes their row by name, so they are counted and not named. On again, it beats at once too, atomically replacing its anonymous row with its named row, so the count never includes both. Signing out deletes the authenticated named row before the session is cleared. A reader who shows themselves but is missing from an answer counted before their beat leads the list in their own browser without increasing the server's count. - **On the Worker.** A beat must come from Allthing itself (`Origin`, else 403) with a well-formed id (else 400); `API_LIMITER` applies, and every answer is `no-store`. Its work: the answer this Cloudflare location counted, from the Cache API, which is kept 30 s; one D1 batch, which always writes the beat, and only where that answer is missing or 30 s old also deletes the rows past keeping and counts who was seen in the window, with those shown by name, in one more pass; on that recount, `user/bulk` for those named; then the new answer, cached. Ordinary public beats reuse a count for 30 seconds; hiding, an account change (`refresh`) or a newly public reader forces a recount. Every cached response checks its at most twelve named rows in the same D1 batch, so a hide processed by another location cannot leak through an old cached list. Other open clients drop the name on their next beat; there is no push subscription. A beat makes at most four subrequests, plus the schema once per isolate. -- **The chip.** A 6 px green dot and the count in 12.5 px muted text, with no frame and no word: "● 12". Its accessible name is "12 readers online" (`chip_name`), and it has no native tooltip. Home stays as it is: the chip sits after it, out of the flow, so it moves nothing as it comes, and fades in over 600 ms (not under reduced motion), as a new count does. At 920 px and narrower it stays visible as a compact dot and count. Both the boot and app headers reserve its width before the count arrives, keeping Home and the search box clear, including on phones. The count is `format::short_number` (1.2k), and at least 1, the reader being here. Escape closes the popover and puts focus back on the chip, which a key opens and focus alone doesn't. +- **The chip.** A 6 px green dot and the count in 12.5 px muted text, with no frame and no word: "● 12". Its accessible name is "12 readers online" (`chip_name`), and it has no native tooltip. Home and DAU stay visible: the chip sits after the navigation, out of the flow, so it moves nothing as it comes, and fades in over 600 ms (not under reduced motion), as a new count does. At 920 px and narrower it stays visible as a compact dot and count. Both the boot and app headers reserve its width before the count arrives, keeping Home, DAU and the search box clear, including on phones. The count is `format::short_number` (1.2k), and at least 1, the reader being here. Escape closes the popover and puts focus back on the chip, which a key opens and focus alone doesn't. - **Off.** `PRESENCE = "on"` in `wrangler.toml`'s `[vars]` turns it on, and `/api/v1/status` says `presence.ready`. With anything else, the endpoint answers 404, and a browser shows no chip and sends no beat for 6 hours. It is the same where `SESSION_SECRET` isn't set (503, "Presence isn't set up here"). See [DEPLOY.md](DEPLOY.md#presence). - **Tested** by `e2e/presence.fixture.spec.ts` at 1280, 920 and 390: the chip coming after the page's own call and moving nothing (CLS 0), Home kept, and the chip visible and clear of the search box from 360 to 1280 px; the popover by hover, click and keyboard; the opt-out; no beat from a hidden tab or after five idle minutes (a fake clock); one beat for two tabs; the early paint's header; axe. `e2e/presence-worker.fixture.spec.ts` runs a Worker of its own with a local D1 and a stand-in node: counting, naming, hiding, pruning, refusals, the rows' contents, and presence off. diff --git a/docs/BUDGET.md b/docs/BUDGET.md index be36e54f..d12dadb1 100644 --- a/docs/BUDGET.md +++ b/docs/BUDGET.md @@ -302,23 +302,29 @@ Linked HTML uses the same 128 KB body limit in the browser: chunks are copied on ## 10. Daily active accounts: cron and D1 -Farcaster's daily active accounts are counted from Hypersnap's event stream ([ARCHITECTURE.md](ARCHITECTURE.md#daily-active-accounts)). Measured on 20 October 2026 over 24.5 hours: 240,000 events (103 MB of JSON) on the two shards, mostly block confirmations, among them 45,000 counted messages from 6,174 accounts. Against the Free plan: +The verified v2 follower reads the same Hypersnap event API and two data shards ([ARCHITECTURE.md](ARCHITECTURE.md#daily-active-accounts)). The 20 October 2026 observation of 240,000 events / 103 MB over 24.5 hours, including 45,000 counted merge events from 6,174 accounts, is a historical workload sample. The new canonical-page checks, full-key attribution and atomic fence have **not yet been timed in the runtime**. The earlier parse numbers and catch-up rate do not establish their CPU cost or a deadline for the first complete window. | Limit | Free plan | The follower | |---|---|---| -| Worker requests | 100,000/day | A cron trigger of its own every 2 minutes: 720 invocations a day, 0.7% of the allowance | -| Cron triggers | 5 per account | 1 more, 3 in all | -| CPU time | 10 ms per invocation | A run parses one page of up to 1,000 events per shard, about 1 ms per 400 KB page natively, as the newest-casts cron already does every minute, and merges a few small sets. Keeping up, a page holds the 2 minutes since the last run, about 170 events. The payload's 48 hourly counts (since 21 October 2026, for /dau's 48-hour range) are the lengths of sets the run already holds for the 24-hour windows: no more reads, and about 300 bytes more to store and serve | -| Subrequests | 50 per invocation | At most 42, as above; 7 to 11 once every active key's app is known | -| D1 rows written | 100,000/day | Once a day, 1 more: the union of the 7 days to yesterday (`w`), once they were read in full, kept and dropped as the days are. 4 or 5 a run, about 3,600 a day: the hour's and the day's set, the hour's keys, the state and the payload, each one row however many accounts they hold. And one row per key whose app is learned, written once: about 7,000 the first day after a deploy (a key for each active account), then the day's new keys, about 1,500. Emoji reactions (replies that are one Pepe, mostly Allthing readers' own): 3 rows each (the row and its two index entries), written once, and as many again when one is taken back: tens a day | -| D1 rows read | 5,000,000/day | About 50 a run (the state, 48 hourly sets and the day's), 36,000 a day, and 30 day sets once a day for the 30-day count. The run's keys' apps, a row each: 60 to 300 a run, up to 200,000 a day. Counting by app reads each key of the past 24 hours and its row, about 13,000, once an hour (every 10 minutes while more than 1% of accounts' apps aren't known): about 300,000 a day, and up to a million on the first day. Dropping reactions taken back or older than 400 days reads only the rows it deletes, through their indexes; each listing build that reads reactions reads its casts' rows: a few at most | -| D1 storage | 5 GB | A day's set of 6,000 accounts is about 16 KB, an hour's about 2 KB, and an hour's keys about 4 KB; 30 days and 49 hours of them: about a megabyte. Keys' apps take about 30 bytes a key and grow by the day's new keys: under 20 MB a year. Emoji reactions take about 150 bytes each, kept 400 days: kilobytes to a few megabytes | +| Worker requests | 100,000/day | Existing cron every 2 minutes: 720 invocations/day, 0.7% of the allowance; no trigger change | +| Cron triggers | 5 per account | Existing activity trigger; 3 in all | +| CPU time | 10 ms per invocation | Up to 1,000 events per shard, canonical sequence/confirmation/carry checks and compact FID-set unions. Full-key work in the Worker is limited to current-run key hours and a bounded D1 response; D1 expands and aggregates the complete 24-hour key arrays. No measured Worker CPU guarantee is claimed; the limit is unchanged | +| Subrequests | 50 per invocation | Existing global application guard of 45. Selection is at most 24 attribution accounts, or 6 during catch-up, with room reserved for D1 and history paging; the six-request `HISTORY` reservation is not a hard history cap | +| D1 rows written | 100,000/day | Current hour/day account sets, hourly full-key sets, state and snapshot, plus one claim/fence-row write in the existing atomic batch. Crossing buckets writes more sets. New immutable signer identities also write the FID/app index; one weekly union is kept once seven complete days are available. Reaction rows and their two indexes retain their existing costs | +| D1 rows read | 5,000,000/day | State and needed hourly/day FID sets, plus the existing daily 30-day/7-day unions. The Worker reads only current-run key hours; D1 expands the 24 full-key arrays and joins attribution rows for app aggregation. The earlier sample had about 13,000 identities/window; hourly recounts, or ten-minute recounts with unresolved attribution, amplify that D1 work. These are workload estimates, not fixed caps | +| D1 storage | 5 GB | Compact account sets remain about 16 KB/day at 6,000 FIDs and 2 KB/hour. Full signing identities are about 80 JSON bytes each: roughly 12–25 KB/hour at current samples, plus a permanent identity/FID/app row and its index. At 1,500 new identities/day, identity text alone grows about 120 KB/day before table/index overhead. Legacy rows are retained separately; the old “30 bytes/key, under 20 MB/year” estimate no longer applies | -**Load on Hypersnap**: two page reads every 2 minutes, 1,440 a day, through the same token bucket and breaker as page builds; about 100 MB a day, the same events the newest-casts cron reads. A page that can't be read is read again from the same place by the next run. After a deploy, or an outage, the follower reads about 6 times faster than events come until it has caught up (5.4 to 6.3 times, measured after the first deploy on 20 October 2026, as busy and quiet hours go by): from the start of yesterday, about 5 hours after a deploy just after midnight UTC, up to 10 after one late in the day. +**Page reads stay the same:** one API call to the stored `/api/v1/activity` snapshot, with at most three subrequests, and the existing first-page Home read of the by-app snapshot. Source and checkpoint evidence travels in the same payload; the page does not re-fetch Hypersnap to draw the coverage card. The compare-and-swap claim adds a statement and fence-row write inside the existing D1 batch, not another HTTP subrequest. The random 128-bit token prevents a stale run from writing under a newer run's fence. -**Counting by app** looks up the keys whose app isn't known: an account's active keys (`signersByFid`, 2 to 20 KB), cached in the isolate for an hour as casts' origins use them, and its onchain history for a key removed since. At most 24 accounts a run, 6 while catching up, whose full pages take most of a run's CPU: about 0.3 ms of parsing for 24 lookups. Keys added onchain come with their app in the event stream and need no lookup (about 300 a day). After a deploy, every active account's key is looked up once: about 7,000 reads, in about 10 hours, during which the hover cards say what share of accounts' apps is known. Then about 1,500 a day. The first count by app comes 24 hours after a deploy, once a whole window of keys has been kept. +**Load on Hypersnap:** ordinarily two event-page reads every 2 minutes, 1,440/day, through the existing token bucket and breaker. The initial probe and retries after failed reads are additional workload; an unreadable page retains its cursor. The earlier observation of roughly 100 MB/day and 5.4–6.3× catch-up describes the former implementation. The actual verified start, node retention, workload and errors determine v2 coverage and when a complete rolling count is available; deployment alone certifies no historical interval. -**People lists' facts** (`Index::apps_of`, `Index::active_week`; no page asks yet): a Followers or Following page's build will take at most 2 more D1 subrequests. One query reads its people's apps from `signer_apps`, each account's key range by the primary key: a row or a few each, a few hundred for 100 people. One row is its week's union, about 2 bytes an account. Decoding the row and looking up 100 people took 48 µs natively for a set of 6,200 accounts and 254 µs for 50,000 (release builds, 21 October 2026), so a week's union of a few times today's daily accounts stays well under a millisecond of the build's 10. +**Counting by app:** the selected accounts' checked active-signer responses and optional paged onchain history must supply exact FID/key/metadata evidence. Unknown or failed responses are retryable and do not become permanent no-app entries. This path does not reuse the cast-origin cache. A full identity is 77 text bytes before JSON punctuation, about 80 bytes in an hourly key array: a 13,000-key 24-hour window is about **1 MB of JSON**. That full-window JSON remains in D1, where `json_each` expands the 24 hourly arrays and `COUNT(DISTINCT fid)` produces app totals. Current-run key-hour arrays override stored versions; canonical array/identity validation and a global distinct-FID cardinality match against the verified rolling total must pass before results are accepted. + +The Worker reads only key hours changed by the current run and receives aggregate counts plus at most 400 unresolved identities, about **32 KB** of identity JSON. It no longer decodes, unions or serializes the full 13,000-key window. D1 still parses the roughly 1 MB observed window and joins its attribution rows; the storage and database work have not disappeared. The number of accounts with unresolved keys is a full-window distinct-FID count, independent of the 400-identity lookup backlog. Aggregation runs in the existing D1 batch after immutable attribution facts are inserted, without another query round trip. This removes the full-window Worker allocation path; it does not establish a measured 10 ms Worker CPU guarantee. The former 0.3 ms / 24-lookups estimate also does not measure the checked lookups or paged history. App totals remain lower bounds while registration facts are unresolved; explicit supported app-zero records are counted separately. + +**Retention:** v2 hourly account/key sets remain for 49 hours and daily/weekly sets for 30 days. Their storage is in addition to retained legacy activity and prefix-key attribution rows; the migration neither deletes them nor certifies them. Known exact signing identities grow with new keys and include table and FID/app index overhead. Emoji reaction retention remains 400 days. Storage and CPU should be measured with representative full-key windows before making new capacity claims. + +**Account-list facts** (`Index::apps_of`, `Index::active_week`; no page asks yet) retain the optional two-D1-read interface: a verified FID/app-index query and one complete-week union. The older FID-set decode benchmarks do not measure full-key attribution, the new D1 index or the atomic save. ## 11. Scheduled casts diff --git a/e2e/allthing.spec.ts b/e2e/allthing.spec.ts index 6ff3d411..b57eb2ab 100644 --- a/e2e/allthing.spec.ts +++ b/e2e/allthing.spec.ts @@ -146,10 +146,10 @@ test.describe("front page", () => { await expect(page.locator(".cast-card .origin").first()).toContainText("via "); await expect(page.locator(".score")).toHaveCount(0); - // No backend badges, intro, ranking or keyboard boxes; Home and Library links, + // No backend badges, intro, ranking or keyboard boxes; Home, Library and DAU links, // and a one-line footer with the version. await expect(page.locator(".backend")).toHaveCount(0); - await expect(page.locator(".main-nav a")).toHaveText(["Home", "Library"]); + await expect(page.locator(".main-nav a")).toHaveText(["Home", "Library", "DAU"]); // No sidebar: the feed is one centered column. await expect(page.locator(".side")).toHaveCount(0); const feed = await page.locator(".columns .content").boundingBox(); diff --git a/e2e/client-activity.fixture.spec.ts b/e2e/client-activity.fixture.spec.ts index ab93f13c..83088a1b 100644 --- a/e2e/client-activity.fixture.spec.ts +++ b/e2e/client-activity.fixture.spec.ts @@ -35,7 +35,7 @@ const METHOD = "Any message (a cast, reaction, follow…) signed with a key this function counted(unknown = 0) { return { - to, counted: to + 120, active: 6174, unknown, none: 14, + verification_version: 2, to, counted: Math.floor(Date.now() / 1000), active: 6174, unknown, none: 14, apps: [{ app: 9152, active: 4812 }, { app: 309857, active: 1203 }, { app: 5555, active: 120 }, { app: 6666, active: 37 }, { app: 1234, active: 1 }], }; } @@ -225,3 +225,20 @@ test("at 390, how it's counted opens the page of every app's active accounts", a await expect(card).toBeHidden(); await context.close(); }); + +for (const [label, change] of [ + ["legacy", { verification_version: 0 }], + ["future", { counted: Math.floor(Date.now() / 1000) + 3600 }], + ["stale", { to: to - 86400 }], +] as const) { + test(`${label} activity never decorates the ordinary cast-share bar with a verified count`, async ({ page }) => { + await serve(page, { ...counted(), ...change }); + await page.goto("/"); + const legend = page.locator(".client-legend"); + await expect(legend.locator("li")).toHaveCount(5); + await legend.getByRole("link", { name: /Farcaster/ }).hover(); + await page.waitForTimeout(400); + await expect(page.getByRole("dialog")).toHaveCount(0); + await expect(page.locator(".share-bar a")).toHaveCount(0); + }); +} diff --git a/e2e/dau.fixture.spec.ts b/e2e/dau.fixture.spec.ts index 4f8e2a23..4f07bfa0 100644 --- a/e2e/dau.fixture.spec.ts +++ b/e2e/dau.fixture.spec.ts @@ -37,13 +37,28 @@ const yesterday = days[28]; const hourly = Array.from({ length: 48 }, (_, i) => (i === 17 ? 1_300 : 500 + ((i * 37) % 700))); const busiest = `${short(Math.floor((to - 31 * 3_600) / DAY))}, ${hh(to - 31 * 3_600)}–${hh(to - 30 * 3_600)} UTC`; -function payload(extra: object = {}) { +const source = "https://haatz.quilibrium.com/v1/events"; +const stamp = (at: number) => `${short(Math.floor(at / DAY))} ${new Date(at * 1000).getUTCFullYear()}, ${new Date(at * 1000).toISOString().slice(11, 16)} UTC`; +/** Explicit source-observed checkpoints; browser tests do not fetch live data. */ +function verified(body: T) { return { + verification_version: 2, source, verified_since: body.since, + shards: [1, 2].map(shard => { + const block = 1_000_000 + shard; + return { shard, since: body.since, reached: body.reached, block, block_at: body.reached + 1, total_events: 100, read_events: 100, cursor: block * 16_384 + 99 }; + }), + ...body, + }; +} + +function payload(extra: object = {}) { + return verified({ reached, rolling_to: to, rolling_24h: 6_174, previous_24h: 5_982, active_30d: { day: today - 1, active: 41_234, complete: false }, since: (today - 29) * DAY + 5_000, days, hourly, apps_24h: { + verification_version: 2, to, counted: to + 120, active: 6_174, none: 14, apps: [ { app: 9152, active: 4_812 }, { app: 309857, active: 1_203 }, { app: 16823, active: 120 }, { app: 777, active: 3 }, @@ -54,9 +69,9 @@ function payload(extra: object = {}) { { fid: 309857, username: "baseapp.base.eth", display_name: "Base App" }, { fid: 16823, username: "recaster", display_name: "Recaster" }, ], - meta: { backend: "hypersnap", fetched_at: Date.now() }, + meta: { backend: "hypersnap", fetched_at: now * 1000 }, ...extra, - }; + }); } /** Serve `body` (or 503, before the first count); the requests are counted. */ @@ -77,14 +92,14 @@ async function serve(page: Page, body: object | null) { // reading has got to 08:24 yesterday, many hours behind the clock. const behind = (today - 1) * DAY + 8 * 3_600 + 24 * 60 + 7; const actions = (n: number) => ({ reaction: 316 * n, cast: 156 * n, link: 45 * n, user_data: 44 * n, verification: 7 * n, storage_lending: 6 * n }); -const catchingUp = { +const catchingUp = verified({ reached: behind, rolling_to: Math.floor(behind / 3_600) * 3_600, since: (today - 1) * DAY - 1_808, days: [ { day: today - 2, active: 134, complete: false, actions: actions(1) }, { day: today - 1, active: 3_047, complete: false, actions: actions(10) }, ], - meta: { backend: "hypersnap", fetched_at: Date.now() }, -}; + meta: { backend: "hypersnap", fetched_at: now * 1000 }, +}); async function phone(browser: Browser, testInfo: TestInfo) { const context = await browser.newContext({ @@ -120,7 +135,20 @@ test.describe("with the Worker", () => { await page.goto("/dau"); await expect(page).toHaveTitle(/Daily active accounts/); await expect(page.getByRole("heading", { level: 1, name: "Daily active accounts" })).toBeVisible(); + await expect(page.locator(".dau-lede")).toContainText("accounts, not people or Allthing readers"); + const coverage = page.locator(".dau-coverage"); + await expect(coverage.getByRole("heading", { name: "Hypersnap event coverage" })).toBeVisible(); + await expect(coverage.locator(".dau-proof-status")).toHaveText("Coverage checked"); + await expect(coverage.getByRole("link", { name: "Hypersnap event API" })).toHaveAttribute("href", source); + await expect(coverage.locator(".dau-source")).toContainText(`Snapshot updated ${stamp(now)}`); + await expect(coverage.locator(".dau-proof-times dd")).toHaveText([stamp(payload().since), stamp(reached)]); + await expect(coverage.locator(".dau-shards h3")).toHaveText(["Shard 1", "Shard 2"]); + await expect(coverage.locator(".dau-shard-through")).toHaveText([`Complete through ${stamp(reached)}`, `Complete through ${stamp(reached)}`]); + await expect(coverage.locator(".dau-shards li").first()).toContainText("100 / 100 events read · block complete"); + await coverage.getByText("Checkpoint", { exact: true }).first().click(); + await expect(coverage.locator("code").first()).toHaveText(String(payload().shards[0].cursor)); const meter = page.locator(".dau-meter"); + expect((await coverage.boundingBox())!.y).toBeLessThan((await meter.boundingBox())!.y); await expect(meter.getByRole("heading", { name: "Past 24 hours" })).toBeVisible(); await expect(meter.locator(".dau-window")).toHaveText(`Unique accounts active in the 24 hours to ${hour}`); await expect(meter.locator(".dau-big")).toHaveText("6,174"); @@ -135,7 +163,7 @@ test.describe("with the Worker", () => { await expect(meter.locator(".dau-status")).toHaveText("Up to date · 11:59 UTC"); const tiles = meter.locator(".dau-tile"); await expect(tiles.locator(".dau-tile-label")).toHaveText(["Today so far", "Yesterday", "Past 30 days", "7-day average"]); - await expect(tiles.locator(".dau-tile-count")).toHaveText(["2,100", "6,300", "41,234", "6,214"]); + await expect(tiles.locator(".dau-tile-count")).toHaveText(["2,100", "6,300", "≥ 41,234", "6,214"]); await expect(tiles.nth(0).locator(".dau-tile-note")).toHaveText("00:00 to 11:59 UTC"); await expect(tiles.nth(1).locator(".dau-tile-note")).toHaveText(short(yesterday.day)); await expect(tiles.nth(2).locator(".dau-tile-note")).toHaveText(`to the end of ${short(today - 1)}, since counting began`); @@ -262,9 +290,9 @@ test.describe("with the Worker", () => { test("what accounts did, accounts by app, and how they were counted", async ({ page }) => { await serve(page, payload()); await page.goto("/dau"); - // The last whole day's messages, by kind, largest first, a bar each. + // The last whole day's message merge events, by kind, largest first. const mix = page.locator(".dau-mix"); - await expect(mix).toContainText(`45,140 messages merged on ${short(yesterday.day)}, by kind.`); + await expect(mix).toContainText(`45,140 message merge events on ${short(yesterday.day)}, by kind.`); await expect(mix.locator("li .dau-mix-name")).toHaveText(["Reactions", "Casts", "Follows", "Profile changes", "Verifications", "Username proofs"]); await expect(mix.locator("li").first()).toContainText("19,628"); await expect(mix.locator("li").first()).toContainText("43%"); @@ -289,9 +317,10 @@ test.describe("with the Worker", () => { const method = page.locator(".dau-method"); await expect(method.locator("dt")).toHaveText(["Active", "Past 24 hours", "Today, a day", "Past 30 days", "7-day average", "By hour", "By app", "From", "Gaps", "Exact"]); await expect(method).toContainText("SnapMeter's closed list for Snapchain, message types 1–8 and 11–21"); + await expect(method).toContainText("a message present on both shards contributes to both there; account totals still count its account once"); await expect(method).toContainText("(t − 24 h, t]"); await expect(method.locator("dd").nth(7)).toContainText("The event stream of Hypersnap's public node, read every 2 minutes by Allthing's Worker"); - await expect(method.locator("dd").nth(8)).toHaveText("None: every block since counting began has been read."); + await expect(method.locator("dd").nth(8)).toHaveText("No recorded gaps within the completed coverage shown above."); await expect(method.getByRole("link", { name: "SnapMeter's definitions" })).toHaveAttribute("href", "https://github.com/ael-dev3/SnapMeter/blob/main/docs/metrics.md"); await expect(method.getByRole("link", { name: "The data as JSON" })).toHaveAttribute("href", "/api/v1/activity"); }); @@ -300,7 +329,8 @@ test.describe("with the Worker", () => { const body = payload({ previous_24h: 6_500, gaps: [[(today - 3) * DAY + 3_600, (today - 3) * DAY + 7_200]], - apps_24h: { to, counted: to + 120, active: 6_174, unknown: 1_234, apps: [{ app: 9152, active: 3_900 }] }, + days: days.map(d => d.day === today - 3 ? { ...d, complete: false } : d), + apps_24h: { verification_version: 2, to, counted: to + 120, active: 6_174, unknown: 1_234, apps: [{ app: 9152, active: 3_900 }] }, }); await serve(page, body); await page.goto("/dau"); @@ -308,7 +338,7 @@ test.describe("with the Worker", () => { await expect(page.locator(".dau-change")).toHaveClass(/down/); await expect(page.locator(".dau-apps .dau-note")).toHaveText("Apps are known for 80% of active accounts so far: the counts below are floors while the rest are looked up."); await expect(page.locator(".dau-apps-foot")).toHaveText("1,234 accounts with keys whose app isn't known yet."); - await expect(page.locator(".dau-method")).toContainText(`Not read: ${short(today - 3)}`); + await expect(page.locator(".dau-method")).toContainText(`Not fully covered: ${short(today - 3)}`); }); test("before the first count, the page says so", async ({ page }) => { @@ -318,7 +348,10 @@ test.describe("with the Worker", () => { await expect(empty.getByRole("heading", { name: "Not counted yet" })).toBeVisible(); await expect(empty).toContainText("Allthing hasn't counted active accounts yet."); // Before a whole window: no 24-hour figure, and when it comes. - await serve(page, payload({ rolling_24h: undefined, previous_24h: undefined, since: to - 3_600 })); + await serve(page, payload({ + rolling_24h: undefined, previous_24h: undefined, since: to - 3_600, + days: [days[29]], hourly: [], active_30d: undefined, apps_24h: undefined, + })); await page.reload(); const first = to - 3_600 + DAY; const meter = page.locator(".dau-meter"); @@ -328,12 +361,63 @@ test.describe("with the Worker", () => { await expect(page.locator(".dau-behind")).toHaveCount(0); }); + const invalidProofs = [ + { name: "the response predates verified counting", body: () => ({ ...payload(), verification_version: undefined, verified_since: undefined, source: undefined, shards: undefined }) }, + { name: "a configured shard is missing", body: () => ({ ...payload(), shards: payload().shards.slice(0, 1) }) }, + { name: "the checkpoint cursor contradicts its block", body: () => ({ ...payload(), shards: payload().shards.map(s => s.shard === 2 ? { ...s, cursor: s.cursor + 1 } : s) }) }, + { name: "a completion timestamp is in the future", body: () => ({ ...payload(), reached: now + 60, shards: payload().shards.map(s => ({ ...s, reached: now + 60, block_at: now + 61 })) }) }, + { name: "a complete day crosses a coverage gap", body: () => ({ ...payload(), gaps: [[(today - 3) * DAY + 3_600, (today - 3) * DAY + 7_200]] }) }, + { name: "the claimed source is not Hypersnap", body: () => ({ ...payload(), source: "https://example.invalid/events" }) }, + ]; + for (const invalid of invalidProofs) { + test(`unverified counts stay hidden when ${invalid.name}`, async ({ page }) => { + const calls = await serve(page, invalid.body()); + await page.goto("/dau"); + const waiting = page.locator(".dau-unverified"); + await expect(waiting.getByRole("heading", { name: "Counts await verification" })).toBeVisible(); + await expect(waiting).toContainText("both network shards"); + await expect(waiting.getByRole("link", { name: "Hypersnap event API" })).toHaveAttribute("href", source); + await expect(waiting.getByRole("link", { name: "The data as JSON" })).toHaveAttribute("href", "/api/v1/activity"); + await expect(page.locator(".dau-meter, .dau-chart, .dau-mix, .dau-apps, .dau-coverage")).toHaveCount(0); + await expect(page.locator(".dau")).not.toContainText("6,174"); + expect(calls).toEqual(["/api/v1/activity"]); + }); + } + + test("a partial shard checkpoint stays explicit and today's count is a lower bound", async ({ page }) => { + const body = payload(); + body.shards[1] = { ...body.shards[1], block_at: reached + 30, total_events: 100, read_events: 37, cursor: body.shards[1].block * 16_384 + 36 }; + await serve(page, body); + await page.goto("/dau"); + const shard = page.locator(".dau-shards li").nth(1); + await expect(shard).toContainText("37 / 100 events read · partial block"); + await expect(shard.locator(".dau-shard-through")).toHaveText(`Complete through ${stamp(reached)}`); + await expect(page.locator(".dau-coverage-note")).toContainText("lower bounds"); + await expect(page.locator(".dau-today .dau-tile-count")).toHaveText("≥ 2,100"); + await expect(page.locator(".dau-meter .dau-big")).toHaveText("6,174"); + }); + + test("both shards can explicitly wait for their first verified events", async ({ page }) => { + await serve(page, { + verification_version: 2, source, shards: [1, 2].map(shard => ({ shard })), + days: [], meta: { backend: "hypersnap", fetched_at: now * 1000 }, + }); + await page.goto("/dau"); + await expect(page.locator(".dau-proof-status")).toHaveText("Coverage building"); + await expect(page.locator(".dau-proof-times dd")).toHaveText(["Waiting for both shards", "No common completed coverage yet"]); + await expect(page.locator(".dau-shards li")).toHaveCount(2); + await expect(page.locator(".dau-shard-through")).toHaveText(["Waiting for a completed block", "Waiting for a completed block"]); + await expect(page.locator(".dau-meter .dau-big")).toHaveText("—"); + await expect(page.locator(".dau-chart, .dau-mix, .dau-apps")).toHaveCount(0); + await expect(page.locator(".dau-unverified")).toHaveCount(0); + }); + test("while catching up, today and yesterday follow the clock, and the page says how far it has read", async ({ page }) => { await serve(page, catchingUp); await page.goto("/dau"); const year = new Date((today - 1) * DAY * 1000).getUTCFullYear(); await expect(page.locator(".dau-behind")).toHaveText( - `Catching up: Allthing has read the network's messages to ${short(today - 1)} ${year}, 08:24 UTC so far, and reads them faster than they come until it is current. The counts below go to then.`, + `Catching up: complete event coverage reaches ${short(today - 1)} ${year}, 08:24 UTC. Later time is still unread; partial totals below are lower bounds.`, ); await expect(page.getByRole("status").filter({ hasText: "Catching up:" })).toBeVisible(); const meter = page.locator(".dau-meter"); @@ -341,7 +425,7 @@ test.describe("with the Worker", () => { await expect(meter.locator(".dau-what")).toHaveText(`The first 24-hour count comes once the 24 hours to ${short(today)}, 00:00 UTC have been read.`); // Today hasn't been read; yesterday is being read; no whole day yet. const tiles = meter.locator(".dau-tile"); - await expect(tiles.locator(".dau-tile-count")).toHaveText(["—", "3,047", "—", "—"]); + await expect(tiles.locator(".dau-tile-count")).toHaveText(["—", "≥ 3,047", "—", "—"]); await expect(tiles.locator(".dau-tile-note")).toHaveText([ "not read yet", `${short(today - 1)}, read to 08:24 UTC so far`, "once a whole day has been read", "once 14 days have been read whole", ]); @@ -371,7 +455,7 @@ test.describe("with the Worker", () => { await plot.getByText("The days as a table").click(); await expect(plot.locator("tbody tr").first().locator("th")).toContainText("(read so far)"); // What they did: the day being read, said so, not the 31 minutes before it. - await expect(page.locator(".dau-mix")).toContainText(`5,740 messages merged on ${short(today - 1)}, read to 08:24 UTC so far, by kind.`); + await expect(page.locator(".dau-mix")).toContainText(`5,740 message merge events on ${short(today - 1)}, read to 08:24 UTC so far, by kind.`); }); test("a short history keeps a month's room, the newest day on the right", async ({ page }) => { @@ -414,7 +498,7 @@ test.describe("with the Worker", () => { await serve(page, payload()); await page.goto("/dau"); await expect(page.locator(".dau-meter")).toContainText("6,174"); - for (const section of [".dau-meter", ".dau-chart", ".dau-mix", ".dau-apps", ".dau-method"]) { + for (const section of [".dau-coverage", ".dau-meter", ".dau-chart", ".dau-mix", ".dau-apps", ".dau-method"]) { const box = (await page.locator(section).boundingBox())!; expect(box.x, section).toBeGreaterThanOrEqual(0); expect(box.x + box.width, section).toBeLessThanOrEqual(390); diff --git a/e2e/presence.fixture.spec.ts b/e2e/presence.fixture.spec.ts index 8899300d..1e4c9544 100644 --- a/e2e/presence.fixture.spec.ts +++ b/e2e/presence.fixture.spec.ts @@ -7,7 +7,7 @@ import { expect, test, type Browser, type BrowserContext, type Page, type TestIn * (components/presence.rs). The browser beats (POST /api/v1/presence) once * the page's own call is in, never as part of it; only a visible tab in * use beats, and one tab beats for the whole browser. The chip, a dot and - * a count ("● 12", named "12 readers online"), fades in after Home without + * a count ("● 12", named "12 readers online"), fades in after navigation without * moving anything; Home stays as it is, including on a phone. Its popover names the * readers who show themselves and only counts the others. The Worker's * side is presence-worker.fixture.spec.ts. @@ -209,10 +209,9 @@ test.describe("in front of the Worker", () => { await expect(chip).toHaveText("12", { useInnerText: true }); const box = (await chip.boundingBox())!; const [hx, , hw] = before.home!; - const [sx, sy] = before.search!; + const [sx] = before.search!; expect(box.x).toBeGreaterThanOrEqual(hx + hw + 4); - if (width <= 640) expect(box.y + box.height + 8).toBeLessThanOrEqual(sy); - else expect(box.x + box.width + 8).toBeLessThanOrEqual(sx); + expect(box.x + box.width + 8).toBeLessThanOrEqual(sx); expect(box.height).toBeLessThanOrEqual(before.header![3]); // 920 is the exact compact breakpoint; the next pixel is full size. const leftPadding = await chip.evaluate((el) => getComputedStyle(el).paddingLeft); @@ -276,6 +275,8 @@ test.describe("in front of the Worker", () => { await page.keyboard.press("Tab"); await expect(page.getByRole("link", { name: "Library", exact: true })).toBeFocused(); await page.keyboard.press("Tab"); + await expect(page.getByRole("link", { name: "DAU, daily active accounts", exact: true })).toBeFocused(); + await page.keyboard.press("Tab"); await expect(chip).toBeFocused(); // Focus alone doesn't open it: Enter does, as a press would. await page.waitForTimeout(400); @@ -454,7 +455,7 @@ test.describe("in front of the Worker", () => { // A beat a moment ago, from another tab: the chip shows from the first frame. await kept(context, { online: 1234, readers: [], next: 240 }); const tab = await context.newPage(); - for (const width of [1280, 1024, 1000, 960, 921, 920, 901, 900, 841, 700, 641, 640, 390, 360]) { + for (const width of [1280, 1024, 1000, 960, 921, 920, 901, 900, 841, 801, 800, 700, 641, 640, 390, 360, 320]) { await tab.setViewportSize({ width, height: 800 }); if (width === 1280) await tab.goto("/"); const chip = chipOf(tab); @@ -462,6 +463,7 @@ test.describe("in front of the Worker", () => { await settle(tab); const where = `${signedIn ? "signed in" : "signed out"} at ${width}`; await expect(tab.getByRole("link", { name: "Home", exact: true }), where).toBeVisible(); + await expect(tab.getByRole("link", { name: "DAU, daily active accounts", exact: true }), where).toBeVisible(); await expect(chip, where).toBeVisible(); await expect(chip).toHaveText("1.2k", { useInnerText: true }); const laid = await tab.evaluate(() => { @@ -473,15 +475,17 @@ test.describe("in front of the Worker", () => { const searchWithout = r("#app .search-box").width; chip.style.display = ""; return { - chip: r("#app .presence-chip"), home: r("#app .main-nav a"), search: r("#app .search-box"), + chip: r("#app .presence-chip"), home: r("#app .main-nav a"), dau: r("#app .dau-nav"), search: r("#app .search-box"), searchWith, searchWithout, scroll: document.documentElement.scrollWidth, }; }); expect(laid.searchWith, where).toBe(laid.searchWithout); expect(laid.scroll, where).toBeLessThanOrEqual(width); expect(laid.chip.x, where).toBeGreaterThanOrEqual(laid.home.right + 4); - if (width <= 640) expect(laid.chip.bottom + 8, where).toBeLessThanOrEqual(laid.search.y); - else expect(laid.chip.right + 8, where).toBeLessThanOrEqual(laid.search.x); + expect(laid.dau.x, where).toBeGreaterThanOrEqual(laid.home.right + 4); + expect(laid.dau.right, where).toBeLessThanOrEqual(laid.chip.x); + expect(laid.chip.right + 8, where).toBeLessThanOrEqual(laid.search.x); + expect(laid.search.width, where).toBeGreaterThanOrEqual(width === 320 ? 112 : 128); } await context.close(); } @@ -497,19 +501,53 @@ test.describe("in front of the Worker", () => { }); await serve(page, () => ok(9)); await kept(page, { online: 9, readers: [], next: 240 }); - for (const width of [1280, 390]) { + for (const { width, path } of [{ width: 1280, path: "/" }, { width: 390, path: "/" }, { width: 390, path: "/dau" }]) { held = new Promise((done) => (release = done)); await page.setViewportSize({ width, height: 844 }); - await page.goto("/"); - const boot = await page.locator("#boot .site-header").boundingBox(); - expect(boot).not.toBeNull(); + await page.goto(path); + const boot = page.locator("#boot"); + const header = await boot.locator(".site-header").boundingBox(); + expect(header).not.toBeNull(); + await expect(boot.locator(".dau-nav")).toHaveText("DAU"); + await expect(boot.locator(".boot-nav [aria-current='page']")).toHaveText(path === "/" ? "Home" : "DAU"); + await expect(boot.locator(".boot-nav a, .boot-nav button, .boot-nav [tabindex]")).toHaveCount(0); + const dau = await boot.locator(".dau-nav").boundingBox(); + const search = await boot.locator(".boot-search").boundingBox(); release(); await expect(page.locator("#boot")).toHaveCount(0); await expect(chipOf(page)).toBeVisible(); - expect(await page.locator("#app .site-header").boundingBox()).toEqual(boot); + expect(await page.locator("#app .site-header").boundingBox()).toEqual(header); + expect(await page.locator("#app .dau-nav").boundingBox()).toEqual(dau); + expect(await page.locator("#app .search-box").boundingBox()).toEqual(search); } }); + for (const width of [1280, 390]) { + test(`DAU stays in the main navigation and marks the current page at width ${width}`, async ({ browser, page: desktop }, info) => { + const { page, close } = await opened(browser, desktop, info, width); + await serve(page, () => ok(1)); + await page.goto("/"); + const nav = page.getByRole("navigation", { name: "Main", exact: true }); + const dau = nav.getByRole("link", { name: "DAU, daily active accounts", exact: true }); + await expect(dau).toBeVisible(); + await expect(dau).toHaveText("DAU"); + await expect(dau).toHaveAttribute("href", "/dau"); + expect(await dau.getAttribute("title")).toBeNull(); + await expect(nav.locator("[aria-current='page']")).toHaveText("Home"); + await dau.click(); + await expect(page).toHaveURL(/\/dau$/); + await expect(nav.locator("[aria-current='page']")).toHaveText("DAU"); + await expect(dau).toHaveAttribute("aria-current", "page"); + if (width === 390) { + expect(await dau.evaluate(el => getComputedStyle(el, "::after").height)).toBe("44px"); + } + await nav.getByRole("link", { name: "Home", exact: true }).click(); + await expect(page).toHaveURL(/\/$/); + await expect(nav.locator("[aria-current='page']")).toHaveText("Home"); + await close(); + }); + } + test("axe finds nothing in the header with the popover open", async ({ page }) => { await signIn(page, me); await serve(page, () => ok(14, [ann, bob, cat])); diff --git a/e2e/static-navigation.fixture.spec.ts b/e2e/static-navigation.fixture.spec.ts index 134ee474..75a16c00 100644 --- a/e2e/static-navigation.fixture.spec.ts +++ b/e2e/static-navigation.fixture.spec.ts @@ -42,6 +42,29 @@ async function at(page: Page, path: RegExp) { } for (const width of [1280, 390]) { + test(`at ${width}, DAU is visible and opens under /Allthing`, async ({ browser }, testInfo) => { + const phone = width === 390; + const context = await browser.newContext({ + baseURL: testInfo.project.use.baseURL, + viewport: { width, height: 844 }, + ...(phone ? { isMobile: true, hasTouch: true } : {}), + }); + const page = await context.newPage(); + await serve(page); + const nav = page.getByRole("navigation", { name: "Main", exact: true }); + const dau = nav.getByRole("link", { name: "DAU, daily active accounts", exact: true }); + await expect(dau).toBeVisible(); + await expect(dau).toHaveAttribute("href", "/Allthing/dau"); + if (phone) await dau.tap(); + else await dau.click(); + await at(page, /\/Allthing\/dau$/); + await expect(dau).toHaveAttribute("aria-current", "page"); + await expect(nav.locator("[aria-current='page']")).toHaveText("DAU"); + await expect(page.locator('main a[href="https://allthing.social/dau"]')).toBeVisible(); + expect(await page.evaluate(() => document.documentElement.scrollWidth)).toBeLessThanOrEqual(width); + await context.close(); + }); + test(`at ${width}, a click on a cast's card opens its post under /Allthing`, async ({ browser }, testInfo) => { const phone = width === 390; const context = await browser.newContext({ diff --git a/migrations/0011_verified_activity.sql b/migrations/0011_verified_activity.sql new file mode 100644 index 00000000..c8772fa9 --- /dev/null +++ b/migrations/0011_verified_activity.sql @@ -0,0 +1,12 @@ +-- SPDX-License-Identifier: AGPL-3.0-or-later +-- Exact public signing identities for the verified activity epoch. Legacy +-- prefix-key attribution and activity rows remain intact but are not certified. +CREATE TABLE IF NOT EXISTS verified_signer_apps ( + identity TEXT PRIMARY KEY, + fid INTEGER NOT NULL CHECK (fid BETWEEN 1 AND 4294967295), + app INTEGER NOT NULL CHECK (app >= 0), + CHECK (length(identity) = 77 AND length(CAST(identity AS BLOB)) = 77 AND substr(identity, 1, 10) = printf('%010d', fid) + AND substr(identity, 11, 3) = ':0x' + AND substr(identity, 14) NOT GLOB '*[^0-9a-f]*') +) WITHOUT ROWID; +CREATE INDEX IF NOT EXISTS verified_signer_apps_fid ON verified_signer_apps (fid, app); diff --git a/style/allthing.css b/style/allthing.css index 2ececaf5..361f2faf 100644 --- a/style/allthing.css +++ b/style/allthing.css @@ -291,7 +291,7 @@ code { } /* Who is reading Allthing now (components/presence.rs): a dot and a count - after Home, out of the flow, so nothing moves as it comes, and Home stays + after the main navigation, out of the flow, so nothing moves as it comes, as it is. Both headers reserve its space before the answer arrives, including the desktop side of the 920px compact breakpoint. */ .header-nav, @@ -358,6 +358,10 @@ code { /* Keep Home and the compact dot/count on phones too. The early header reserves the same width, so hydration and the first count move nothing. */ @media (max-width: 920px) { + .header-inner { + gap: 12px; + } + .header-nav, .boot-nav { padding-right: 58px; @@ -8166,38 +8170,51 @@ p.jumbo .emoji img { } } -@media (max-width: 640px) { - body { - font-size: 14.5px; - } - - /* Home, the online count and the account's controls keep their room. - Search has its own row rather than shrinking below its placeholder. - The early header uses the same rows before the app is ready. */ +/* Navigation and search share the second row where a single row would + squeeze the search box. The early header uses the same grid. */ +@media (max-width: 800px) { .header-inner { - flex-wrap: wrap; - gap: 8px; - padding: 10px 14px 8px; + display: grid; + grid-template-columns: max-content minmax(0, 1fr); + grid-template-rows: 36px auto; + gap: 8px 12px; } .header-nav, .boot-nav { + grid-column: 1; + grid-row: 2; min-height: 36px; align-items: center; } .header-actions, - .boot-actions { + .boot-actions, + .boot-readonly { + grid-column: 2; + grid-row: 1; + justify-self: end; flex: none; } .search-box, .boot-search { - order: 1; - flex-basis: 100%; + grid-column: 2; + grid-row: 2; max-width: none; margin-left: 0; } +} + +@media (max-width: 640px) { + body { + font-size: 14.5px; + } + + .header-inner { + gap: 8px; + padding: 10px 14px 8px; + } .wordmark { font-size: 27px; @@ -8567,6 +8584,7 @@ p.jumbo .emoji img { } .dau-lede a, +.dau-coverage a, .dau-method a, .dau-empty a { color: var(--accent); @@ -8600,6 +8618,84 @@ p.jumbo .emoji img { line-height: 1.45; } +/* Source-observed event coverage stays beside the figures it qualifies. */ +.dau-coverage-head { + display: flex; + align-items: baseline; + justify-content: space-between; + flex-wrap: wrap; + gap: 6px 12px; +} + +.dau-coverage-head h2 { + margin-bottom: 6px; +} + +.dau-proof-status { + color: var(--accent); + font: 600 12px/1.4 var(--font-ui); +} + +.dau-source, +.dau-card .dau-coverage-note { + color: var(--ink-2); + font-size: 12.5px; + line-height: 1.5; +} + +.dau-proof-times, +.dau-shards { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 12px 18px; + margin: 12px 0; + padding: 0; +} + +.dau-proof-times dt, +.dau-shards h3 { + margin: 0 0 3px; + font: 600 12px/1.4 var(--font-ui); + color: var(--ink); +} + +.dau-proof-times dd, +.dau-shards p, +.dau-shards details { + margin: 0; + font-size: 12px; + line-height: 1.5; + color: var(--ink-2); +} + +.dau-shards { + list-style: none; + border-block: 1px solid var(--line); + padding: 12px 0; +} + +.dau-shards summary { + padding: 5px 0; + cursor: pointer; + color: var(--accent); +} + +.dau-shards code { + font: 11px var(--font-mono); + overflow-wrap: anywhere; +} + +@media (pointer: coarse) { + .dau-shards summary { + min-height: 44px; + padding: 12px 0; + } +} + +.dau-lower-bound { + font-size: 0.8em; +} + /* The meter: the past 24 hours, today so far beside them, readouts, and today against a typical day. */ .dau-meter { @@ -9219,6 +9315,10 @@ p.jumbo .emoji img { } @media (max-width: 640px) { + .dau-shards { + grid-template-columns: minmax(0, 1fr); + } + .dau-meter { padding: 18px 16px 16px; } diff --git a/tests/activity-sql.test.mjs b/tests/activity-sql.test.mjs new file mode 100644 index 00000000..d3446bb4 --- /dev/null +++ b/tests/activity-sql.test.mjs @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import test from "node:test"; + +const source = readFileSync(new URL("../crates/allthing-worker/src/index.rs", import.meta.url), "utf8"); +const names = [ + "ACTIVITY_STATE", "CLAIM_ACTIVITY", "ACTIVITY_SETS", "STORE_SETS", "DROP_SETS", + "STORE_KEYS", "STORE_ACTIVITY_KEYS", "KEY_APPS", "COUNT_APPS", "APPS_OF", + "SAVE_ACTIVITY_STATE", "SAVE_ACTIVITY_SNAPSHOT", "STORE_REACTIONS", "DROP_REACTIONS", "OLD_REACTIONS", +]; +const sql = Object.fromEntries(names.map(name => { + const statement = source.match(new RegExp(`const ${name}: &str = "([^"]+)";`))?.[1].replace(/\\\n\s*/g, ""); + assert(statement, `missing Worker SQL constant ${name}`); + return [name, statement]; +})); +const schema = [ + "0001_index.sql", "0002_snapshots.sql", "0003_activity.sql", "0004_signer_apps.sql", + "0006_reactions.sql", "0011_verified_activity.sql", +].map(name => readFileSync(new URL(`../migrations/${name}`, import.meta.url), "utf8")).join("\n"); + +function scenario(body) { + // Match the supported Node22.12 setup, where SQLite needs the explicit flag. + const run = spawnSync(process.execPath, ["--experimental-sqlite", "--input-type=module", "-e", ` + import assert from 'node:assert/strict'; + import { DatabaseSync } from 'node:sqlite'; + const db = new DatabaseSync(':memory:'); + db.exec(${JSON.stringify(schema)}); + const sql = ${JSON.stringify(sql)}; + const json = JSON.stringify; + const rows = (table, order) => db.prepare('SELECT * FROM '+table+' ORDER BY '+order).all().map(row => ({...row})); + const dump = () => ({ + state: rows('state','key'), snapshots: rows('snapshots','name'), activity: rows('activity','name'), + keys: rows('verified_signer_apps','identity'), reactions: rows('reactions','hash'), legacyKeys: rows('signer_apps','pair'), + }); + const identity = (fid, suffix) => String(fid).padStart(10,'0')+':0x12345678'+suffix.toString(16).padStart(56,'0'); + const seed = () => { + db.prepare('INSERT INTO state VALUES (?,?)').run('activity_v2','before'); + db.prepare('INSERT INTO state VALUES (?,?)').run('activity','legacy-state'); + db.prepare('INSERT INTO snapshots VALUES (?,?,?)').run('activity_v2',10,'before-snapshot'); + db.prepare('INSERT INTO snapshots VALUES (?,?,?)').run('activity',5,'legacy-snapshot'); + for (const name of ['h1','k1','d1','w1','v2h1','v2k1','v2d1','v2w1','v2h20','v2d20']) { + db.prepare('INSERT INTO activity VALUES (?,?)').run(name,'original-'+name); + } + db.prepare('INSERT INTO signer_apps VALUES (?,?)').run(123,9152); + db.prepare('INSERT INTO reactions VALUES (?,?,?,?,?)').run('remove-me','parent','pepe',42,200); + db.prepare('INSERT INTO reactions VALUES (?,?,?,?,?)').run('expire-me','parent','pepe',42,50); + db.prepare('INSERT INTO reactions VALUES (?,?,?,?,?)').run('keep-me','parent','pepe',42,500); + }; + // D1 batch is transactional. Execute every statement after a failed claim + // too: each write and delete must enforce its own winning token. + const save = ({expected, token, state='after', sets=[['v2h20','winner-hour'],['v2d20','winner-day']], + keys=[[identity(4294967295,1),9152]], reactions=[['parent','winner-reaction','pepe',4294967295,300]], + removed=['remove-me'], keepHour=10, keepDay=10, reactionsFrom=100, built=20, body='winner-snapshot', fail=false}) => { + db.exec('BEGIN'); + try { + const claim = db.prepare(sql.CLAIM_ACTIVITY).get(expected,token); + db.prepare(sql.STORE_SETS).run(json(sets),token); + db.prepare(sql.STORE_ACTIVITY_KEYS).run(json(keys),token); + db.prepare(sql.DROP_SETS).run(keepHour,keepDay,token); + db.prepare(sql.STORE_REACTIONS).run(json(reactions),token); + db.prepare(sql.DROP_REACTIONS).run(json(removed),token); + db.prepare(sql.OLD_REACTIONS).run(reactionsFrom,token); + db.prepare(sql.SAVE_ACTIVITY_STATE).run(state,token); + db.prepare(sql.SAVE_ACTIVITY_SNAPSHOT).run('activity_v2',built,body,token); + if (fail) db.prepare(sql.SAVE_ACTIVITY_SNAPSHOT).run('activity_v2',built+1,null,token); + db.exec('COMMIT'); + return claim?.value === token; + } catch (error) { + db.exec('ROLLBACK'); + throw error; + } + }; + ${body} + db.close(); + `], { encoding: "utf8" }); + assert.equal(run.status, 0, run.stderr || run.stdout); +} + +test("an activity winner writes its batch and stale tokens cannot write or delete anything", () => scenario(` + seed(); + assert.equal(save({expected:'before',token:'winner-token'}),true); + assert.equal(db.prepare(sql.ACTIVITY_STATE).get().value,'after'); + assert.deepEqual(rows('snapshots','name'),[ + {name:'activity',built:5,body:'legacy-snapshot'}, + {name:'activity_v2',built:20,body:'winner-snapshot'}, + ]); + assert.deepEqual(db.prepare(sql.ACTIVITY_SETS).all(json(['v2h20','v2d20'])).map(r=>({...r})).sort((a,b)=>a.name.localeCompare(b.name)),[ + {name:'v2d20',body:'winner-day'},{name:'v2h20',body:'winner-hour'}, + ]); + assert.deepEqual(rows('verified_signer_apps','identity'),[{identity:identity(4294967295,1),fid:4294967295,app:9152}]); + assert.deepEqual(rows('reactions','hash').map(r=>r.hash),['keep-me','winner-reaction']); + assert(!rows('activity','name').some(r=>['v2h1','v2k1','v2d1','v2w1'].includes(r.name))); + const won=dump(); + for (const [token,candidate] of [['stale-different','stale-state'],['stale-identical','after']]) { + assert.equal(save({expected:'before',token,state:candidate, + sets:[['v2h20','corrupt-existing'],['v2h21','stale-only']], + keys:[[identity(4294967295,2),309857]],reactions:[['parent','stale-only','pepe',7,400]], + removed:['keep-me','winner-reaction'],keepHour:999,keepDay:999,reactionsFrom:999, + built:999,body:'stale-snapshot'}),false); + assert.deepEqual(dump(),won,'a stale batch changed persisted data'); + } +`)); + +test("the first activity batch claims a null state and rejects another null-state reader", () => scenario(` + assert.equal(db.prepare(sql.ACTIVITY_STATE).get(),undefined); + assert.equal(save({expected:null,token:'first-token'}),true); + const first=dump(); + assert.equal(save({expected:null,token:'other-token',state:'after',sets:[['v2h20','loser']], + removed:['winner-reaction'],reactions:[],keepHour:999,keepDay:999,reactionsFrom:999}),false); + assert.deepEqual(dump(),first); +`)); + +test("a late failing activity statement rolls back its claim and every earlier mutation", () => scenario(` + seed(); + const before=dump(); + assert.throws(()=>save({expected:'before',token:'failing-token',fail:true}),/NOT NULL/); + assert.deepEqual(dump(),before); + assert.equal(save({expected:'before',token:'retry-after-rollback'}),true); +`)); + +test("verified activity retention expires only v2 rows and preserves legacy sets", () => scenario(` + const names=['h1','k1','d1','w1','v3h1','v2h9','v2k9','v2d4','v2w4','v2h10','v2k10','v2d5','v2w5']; + for(const name of names) db.prepare('INSERT INTO activity VALUES (?,?)').run(name,'body-'+name); + assert.equal(db.prepare(sql.CLAIM_ACTIVITY).get(null,'retention-token').value,'retention-token'); + db.prepare(sql.DROP_SETS).run(10,5,'wrong-token'); + assert.equal(rows('activity','name').length,names.length); + db.prepare(sql.DROP_SETS).run(10,5,'retention-token'); + assert.deepEqual(rows('activity','name').map(r=>r.name),['d1','h1','k1','v2d5','v2h10','v2k10','v2w5','v3h1','w1']); +`)); + +test("full signer keys sharing a prefix stay distinct and app queries count high FIDs exactly", () => scenario(` + const high=4294967295, other=2147483648; + const a=identity(high,1), b=identity(high,2), c=identity(other,3), extra=identity(high,4), missing=identity(high,5); + assert.equal(a.slice(0,21),b.slice(0,21)); + db.prepare(sql.STORE_KEYS).run(json([[a,9152],[b,309857],[c,9152],[extra,9152]])); + db.prepare(sql.STORE_KEYS).run(json([[a,999]])); + assert.equal(rows('verified_signer_apps','identity').length,4); + assert.deepEqual(db.prepare(sql.KEY_APPS).all(json([a,b,missing])).map(r=>({...r})).sort((x,y)=>x.identity.localeCompare(y.identity)),[ + {identity:a,app:9152},{identity:b,app:309857}, + ]); + const hours=Array.from({length:24},(_,i)=>'v2k'+(100+i)); + for(const name of hours) db.prepare('INSERT INTO activity VALUES (?,?)').run(name,json([a,b,c,extra,missing].sort())); + const raw=db.prepare(sql.COUNT_APPS).all(json(hours),json([])); + assert(raw.every(r=>r.invalid===0&&r.accounts===2),'global cardinality must deduplicate every account across all24 arrays'); + assert.deepEqual(raw.filter(r=>r.active===0).map(r=>({...r})),[{app:null,active:0,unknown:'[]',invalid:0,accounts:2}]); + const counts=raw.filter(r=>r.active>0).map(r=>({app:r.app,active:r.active,unknown:JSON.parse(r.unknown)})); + assert.deepEqual(counts.sort((x,y)=>(x.app??-1)-(y.app??-1)),[ + {app:null,active:1,unknown:[missing]}, {app:9152,active:2,unknown:[]}, {app:309857,active:1,unknown:[]}, + ]); + assert.deepEqual(db.prepare(sql.APPS_OF).all(json([high,other])).map(r=>({...r})).sort((x,y)=>x.fid-y.fid||x.app-y.app),[ + {fid:other,app:9152},{fid:high,app:9152},{fid:high,app:309857}, + ]); + assert.throws(()=>db.prepare('INSERT INTO verified_signer_apps VALUES (?,?,?)').run(identity(high,99),42,9152),/CHECK constraint/i); +`)); + +test("stored activity key windows validate canonical arrays, replace the current batch and bound only the unknown backlog", () => scenario(` + const hours=Array.from({length:24},(_,i)=>'v2k'+(100+i)); + const count=(overrides=[])=>db.prepare(sql.COUNT_APPS).all(json(hours),json(overrides)).map(r=>({...r})); + const put=(name,body)=>db.prepare('INSERT INTO activity VALUES (?,?) ON CONFLICT(name) DO UPDATE SET body=excluded.body').run(name,body); + const clear=()=>db.exec('DELETE FROM activity'); + const old=identity(42,1),fresh=identity(7,2); + assert.deepEqual(count(),[{app:null,active:0,unknown:'[]',invalid:0,accounts:0}], + 'an empty window still returns a validation/cardinality sentinel'); + db.prepare(sql.STORE_KEYS).run(json([[old,9152],[fresh,309857]])); + put(hours[0],json([old])); + put('v2k99',json([identity(99,1)])); // outside the requested24-hour window + let answer=count([[hours[0],json([fresh])],[hours[1],json([fresh])]]); + assert(answer.every(r=>r.invalid===0&&r.accounts===1)); + assert.deepEqual(answer.filter(r=>r.active>0).map(({app,active,unknown})=>({app,active,unknown})), + [{app:309857,active:1,unknown:'[]'}],'an override replaces old keys; it never appends them'); + put(hours[0],'not JSON'); + assert(count([[hours[0],json([fresh])]]).every(r=>r.invalid===0&&r.accounts===1), + 'a valid current-run replacement must exclude the old corrupt body entirely'); + + const malformed=[ + ['invalid JSON','not JSON'],['JSON object',json({})],['JSON null','null'],['JSON string',json('text')], + ['JSON number','42'],['null entry',json([null])],['number entry',json([7])],['boolean entry',json([true])], + ['object entry',json([{}])],['nested array',json([[fresh]])], + ['zero FID',json([identity(0,1)])],['FID aboveu32',json([identity(4294967296,1)])], + ['noncanonical FID',json(['000000000x'+fresh.slice(10)])], + ['short key',json([fresh.slice(0,-2)])],['long key',json([fresh+'ab'])], + ['nonhex key',json([fresh.slice(0,-1)+'g'])],['uppercase key',json([fresh.slice(0,-1)+'A'])], + ['NUL suffix',json([fresh+String.fromCharCode(0)+'hidden'])], + ['unsorted identities',json([old,fresh])],['duplicate identity',json([fresh,fresh])], + ]; + for(const [label,body] of malformed){ + clear();put(hours[0],body); + let rejected=false; + try { const rows=count();rejected=rows.length>0&&rows.every(r=>r.invalid>0); } + catch { rejected=true; } + assert(rejected,'corrupt key window was accepted: '+label); + } + clear(); + assert(count([[hours[0],json([fresh])],[hours[0],json([fresh])]]).every(r=>r.invalid>0), + 'duplicate replacement rows cannot masquerade as canonical hourly sets'); + + const unknown=[]; + for(let fid=1000;fid<1450;fid++) unknown.push(identity(fid,1)); + for(let fid=1000;fid<1100;fid++) unknown.push(identity(fid,2)); + unknown.sort(); + for(let i=0;ij%24===i||j%17===0))); + answer=count(); + assert(answer.every(r=>r.invalid===0&&r.accounts===450)); + const unknownRow=answer.find(r=>r.app===null&&r.active>0); + assert.equal(unknownRow.active,450,'unknown account cardinality is not capped by the lookup backlog'); + const backlog=JSON.parse(unknownRow.unknown); + assert.equal(backlog.length,400); + assert.deepEqual(backlog,unknown.slice(0,400)); + assert.equal(new Set(backlog).size,400,'the bounded backlog still contains distinct complete keys'); +`)); diff --git a/web/boot.js b/web/boot.js index c8be936d..e4a92744 100644 --- a/web/boot.js +++ b/web/boot.js @@ -60,17 +60,19 @@ if (location.hostname === "www.allthing.social") { } } -// The boot screen's header as the app's will be: Home marked on the front -// page, and in the account's place what the app will show there, Sign in or +// The boot screen's header as the app's will be: its current page marked, +// and in the account's place what the app will show there, Sign in or // the signed-in reader's picture (the user cookie says which), or the static // copy's read-only note. index.html has the rest. function bootHeader() { const account = document.getElementById("boot-account"); if (!account) return; const base = "{{BASE}}"; - if (location.pathname === base + "/" || location.pathname === base) { - account.closest(".header-inner")?.querySelector(".boot-nav span")?.setAttribute("aria-current", "page"); - } + const path = location.pathname.slice(base.length) || "/"; + account.closest(".header-inner")?.querySelectorAll(".boot-nav [data-route]").forEach((item) => { + const route = item.getAttribute("data-route"); + if (route === "/" ? path === "/" : path.startsWith(route)) item.setAttribute("aria-current", "page"); + }); if (base) { account.outerHTML = 'read-only'; } else if (/(?:^|;\s*)__Host-allthing-user=[^;]/.test(document.cookie)) { diff --git a/web/index.html b/web/index.html index a64bb10f..61736dd2 100644 --- a/web/index.html +++ b/web/index.html @@ -54,8 +54,8 @@ From c46879e15966110cbeb62ad7b6162f402fc26d44 Mon Sep 17 00:00:00 2001 From: ael-dev3 <183124839+ael-dev3@users.noreply.github.com> Date: Tue, 27 Oct 2026 13:12:16 +0100 Subject: [PATCH 2/4] fix(feed): make Show less substantial and stabilize headers --- AGENTS.md | 2 +- CHANGELOG.md | 2 + crates/allthing-core/src/ranking.rs | 232 +++++++++++------- .../src/components/profile_card.rs | 4 +- .../allthing-web/src/components/show_less.rs | 8 +- crates/allthing-web/src/leanings.rs | 4 +- crates/allthing-web/src/pages/home.rs | 11 +- docs/ARCHITECTURE.md | 6 +- e2e/early-paint.fixture.spec.ts | 25 +- e2e/feed-session-ranges.fixture.spec.ts | 4 +- e2e/following-feed.fixture.spec.ts | 10 +- e2e/follows-pages.fixture.spec.ts | 14 ++ e2e/hover-card-leanings.fixture.spec.ts | 2 +- e2e/leaning-hints.fixture.spec.ts | 2 +- e2e/leaning-strength.fixture.spec.ts | 62 ++--- e2e/search-dismissal.fixture.spec.ts | 7 + e2e/show-less.fixture.spec.ts | 40 ++- style/allthing.css | 44 +++- web/boot.js | 11 +- web/paint.js | 4 + 20 files changed, 304 insertions(+), 190 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f130e414..50017dda 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ Allthing is a Farcaster client for reading, written in Rust: a Leptos WebAssembly app plus a workers-rs API Worker, deployed to Cloudflare Workers (Free plan) at https://allthing.social. Read `docs/ARCHITECTURE.md` first. - **One QR flow: sign in, and write, with a key the reader approves once.** By the owner's decision, where the Worker endorses keys there is one approval and one dialog: the reader approves Allthing's key request in the Farcaster app (a QR code on a computer, a link to the app on a phone), which signs them in (`POST /api/v1/auth/key`) and lets them cast, reply, quote, like, recast, follow and delete from here, with no second scan or separate grant step. A reader signed in without a key in this browser (signed in before keys, after Revoke permissions, or with its storage cleared) meets the same dialog, in its approve form, on their first write (`Session::grant`), never by itself on a page load. Never add a second approval dialog or QR flow. Sign In With Farcaster by QR code is only the fallback where the Worker endorses no keys, and makes no relay request where it does (`allthing-worker/src/auth.rs`, `allthing-core/src/siwf.rs`, `allthing-web/src/components/signin.rs`; see `docs/ARCHITECTURE.md#sign-in`). Once a reader has approved Allthing in the Farcaster app, casts, replies, quotes, likes, recasts, follows and deletes go from Allthing: the browser makes and keeps the key and signs (`allthing-core/src/app_key.rs` and `message.rs`, `allthing-web/src/keys.rs` and `components/writer.rs`; see `docs/ARCHITECTURE.md#liking-and-recasting`), and the Worker keeps scheduled casts the browser signed until their time (`allthing-worker/src/scheduled_casts.rs`, `allthing-data/src/scheduled.rs`). Allthing asks for those keys as its own Farcaster account, @allthingapp (FID 3356318, `ALLTHING_FID` in `wrangler.toml`), which its owner registered for it, and shows the casts "via Allthing" (`allthing-core/src/clients.rs`). By the owner's decision that is Allthing's only account: never add another, a seed phrase import or a wallet connection. The Worker endorses readers' keys with that account's custody key, the `ALLTHING_CUSTODY_KEY` secret, set in Cloudflare only (`allthing-worker/src/app_key.rs`; see `docs/DEPLOY.md#secrets`), and uses it for nothing else; it never holds a reader's key or signs as a reader. Until the secret is set (`writes.ready` in `/api/v1/status`), and for a reader signed out, a write opens the cast, the profile or a draft ready to post in the Farcaster app instead (`allthing-web/src/components/handoff.rs`, `allthing-core/src/compose.rs`; see `docs/ARCHITECTURE.md#handing-writes-to-the-farcaster-app`). Never add other login methods, wallet connections or write paths unless asked to. Messages (`/messages`) is Allthing's own, on SpacetimeDB (`allthing-web/src/pages/messages`, `spacetime/`; see `docs/ARCHITECTURE.md#messages-on-spacetimedb`): never add Farcaster Direct Cast API calls or an API-key paste step to it unless asked. -- **No automatic account-based exclusions.** Allthing shows no account labels; spam is judged by a cast's content (see Mini apps and spam below). The owner's explicit exception is reader-selected private Mute and Block: hide content involving those accounts throughout that reader's Allthing view, including identifiable mentions, quoted/replied-to/recasting accounts and notifications. Keep these preferences caller-private and synced through SpacetimeDB; Block also rejects new Allthing DMs in either direction using verified FIDs. This does not delete public posts or establish a block in another Farcaster client. Preserve the recognition limits and undo/sync explanations in `docs/ARCHITECTURE.md#private-mute-and-block`; never turn these choices into a shared author blacklist. +- **No automatic account-based exclusions.** Allthing shows no account labels; spam is judged by a cast's content (see Mini apps and spam below). The owner's explicit exception is reader-selected private Mute and Block: hide content involving those accounts throughout that reader's Allthing view, including identifiable mentions, quoted/replied-to/recasting accounts and notifications. Keep these preferences caller-private and synced through SpacetimeDB; Block also rejects new Allthing DMs in either direction using verified FIDs. This does not delete public posts or establish a block in another Farcaster client. The owner also explicitly chose stronger reader-selected Show less: omit a stable approximately 90% of that author's Home, Following and channel candidates and downrank the remainder; keep New chronological and direct profiles/search available. It remains a private browser preference, not an automatic shared account blacklist. Preserve the recognition limits and undo/sync explanations in `docs/ARCHITECTURE.md#private-mute-and-block`; never turn these choices into a shared author blacklist. - **Rust first.** The owner prefers Rust for application behavior and native tooling. Put pure shared rules in `allthing-core`, interactive UI in `allthing-web`, and native build/check policy in `xtask`. Keep `web/paint.js`'s inert first-card renderer: the owner chose it after a paired cold-load comparison beat the Rust prototype (see `docs/PERFORMANCE.md#measured-exception-the-javascript-first-card-renderer`). Retain the existing Node static test server (`e2e/pages-server.mjs`) after an unresolved pre-mount stall in migration CI; the server's role in that stall was not established (see `docs/PERFORMANCE.md#retained-node-static-test-server`). Keep the small JavaScript loader and inline preload/data hints, the SpacetimeDB TypeScript SDK adapter/generated bindings, and official Playwright browser tests at their platform boundaries. CSS, SQL, HTML and config remain platform data. Preserve measured performance, license rules and independent test oracles; do not mask handwritten files with Linguist attributes. - **Where code goes.** - Pure logic and data types, with unit tests, go in `crates/allthing-core`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 3896c454..53c19652 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ All notable changes to Allthing are recorded here. The format follows [Keep a Ch ### Changed +- Show less now omits roughly 90% of that account's feed posts across Home, Following and channels, with stable selection and a stronger penalty for remaining Hot/Top posts. New keeps chronological order; direct profiles remain complete and the same button undoes the choice. + - Following opens with New selected; For you still opens with Hot. Explicit sort choices remain selected when changing the time range. ### Fixed diff --git a/crates/allthing-core/src/ranking.rs b/crates/allthing-core/src/ranking.rs index 1b55dc97..d22cd44a 100644 --- a/crates/allthing-core/src/ranking.rs +++ b/crates/allthing-core/src/ranking.rs @@ -263,11 +263,10 @@ pub fn sorted(casts: &[Cast], sort: Sort) -> Vec { out } -/// A reader's own leanings about accounts, kept in their browser and never -/// shared: their favorites' casts rise far up their hot and top, and the -/// casts of those they see less of drop far down, and lower in new too. It -/// isn't a label on an account, and it hides nothing: every cast stays in -/// the list, only in another place. +/// A reader's private preferences. Favorites rise in hot and top; +/// Show less keeps roughly one in ten authored casts in feeds and strongly +/// reduces the remaining casts' rank. Direct profiles and search use the +/// ordinary ranking helpers and remain available. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct Leanings { pub favorites: BTreeSet, @@ -292,25 +291,29 @@ pub struct Leanings { /// 60th to about 4th in hot and in top (`leanings_move_a_typical_account_far`). pub const FAVORITE_WEIGHT: f64 = 25.0; -/// The cast of an account the reader sees less of weighs a 25th of its -/// points in hot and top: [`FAVORITE_WEIGHT`] the other way, as strong. In -/// top, 270 points, the most on the front page on 21 October 2026, then -/// weigh less than its quietest cast's 22; in hot, a cast is placed as if -/// posted 17½ hours earlier for each day the list spans, past the oldest of -/// its casts unless it is far busier than they are. In 120 casts over a day -/// or a week, a typical account's median place goes from about 60th to -/// about 117th in hot and in top. Its casts are all still there, at the -/// end. -pub const SHOW_LESS_WEIGHT: f64 = 1.0 / FAVORITE_WEIGHT; - -/// In new, the casts of an account the reader sees less of are placed as if -/// posted 6 hours before they were for each day the list spans (the time -/// they show stays true): about a quarter of the list further down, whether -/// it spans a day, as a busy channel's does, or a week, as a quiet one's -/// (a typical account's median place goes from about 60th to about 87th of -/// 120). In the front page's new, the past hour or so of casts, it is the -/// end of the list. A favorite's casts stay in time order. -pub const SHOW_LESS_NEW_SECS: i64 = 6 * 3600; +/// Retained Show less casts carry one hundredth of their usual engagement +/// weight in hot and top. This also overrides favorite and followed lifts. +pub const SHOW_LESS_WEIGHT: f64 = 0.01; + +/// Stable approximate retention rate for explicit Show less preferences. +/// It is a reduction, not a mute: a large set retains roughly one in ten. +pub const SHOW_LESS_KEEP_DENOMINATOR: u64 = 10; + +/// Whether a feed keeps this authored cast for the reader. Sampling depends +/// only on author FID and cast hash, so pagination, sorting, refreshed counts +/// and timestamps cannot make a suppressed post reappear. FNV-1a is fixed +/// here rather than using a process-seeded standard-library hasher. +pub fn keep_for_feed(cast: &Cast, leanings: &Leanings) -> bool { + if !leanings.less.contains(&cast.author.fid) { + return true; + } + let mut bucket = 0xcbf2_9ce4_8422_2325_u64; + for byte in cast.author.fid.to_le_bytes().into_iter().chain(cast.hash.bytes().map(|byte| byte.to_ascii_lowercase())) + { + bucket = (bucket ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3); + } + bucket.is_multiple_of(SHOW_LESS_KEEP_DENOMINATOR) +} /// In For you's hot, a cast by an account the reader follows weighs as if /// it had 20 times its points. @@ -335,7 +338,7 @@ pub const FOLLOWED_HOT_WEIGHT: f64 = 20.0; pub const FOLLOWED_TOP_WEIGHT: f64 = 6.0; /// How many days `casts` span, from the oldest to the newest, and one at -/// least: in hot and new a leaning moves a cast by its move for a day as +/// least: in hot a leaning moves a cast by its move for a day as /// many times, so as far down, or up, a list of a week as of a day. pub fn span_days(casts: &[Cast]) -> f64 { let oldest = casts.iter().map(|c| c.ts).min().unwrap_or_default(); @@ -361,51 +364,48 @@ impl Leanings { Sort::New => 1.0, }; match sort { - Sort::New => base.min(1.0), + Sort::New => 1.0, Sort::Hot | Sort::Top => base * lift, } } /// How much more, or less, `cast` weighs for this reader in hot and top. - /// A favorite wins over seeing less, though the app never keeps an - /// account as both. + /// Show less wins over a stale overlapping favorite preference. pub fn weight(&self, cast: &Cast) -> f64 { let fid = cast.author.fid; - if self.favorites.contains(&fid) { - FAVORITE_WEIGHT - } else if self.less.contains(&fid) { + if self.less.contains(&fid) { SHOW_LESS_WEIGHT + } else if self.favorites.contains(&fid) { + FAVORITE_WEIGHT } else { 1.0 } } } -/// [`sorted`], with the reader's leanings ([`Leanings::weight`]). In top a -/// cast weighs its points plus one, times its weight: the one is so that a -/// cast nobody has reacted to yet moves too, and the others keep their -/// order either way. In hot the log of its points moves by the weight's -/// once for each day the list spans ([`span_days`]), as if it were 17½ -/// hours newer or older for each. In new only seeing less moves a cast, as -/// if it were [`SHOW_LESS_NEW_SECS`] older for each day. On a tie, a -/// favorite's cast goes first and one seen less of last. Every cast is -/// kept. +/// Feed ranking with the reader's leanings and stable Show less reduction. +/// Hot and top retain the existing favorite/followed boosts, while remaining +/// Show less posts receive the smaller weight. New keeps actual timestamps +/// and strict chronological order after thinning. pub fn sorted_for(casts: &[Cast], sort: Sort, leanings: &Leanings) -> Vec { if leanings.is_empty() { return sorted(casts, sort); } - // In new, a favorite's and a followed account's casts are in time order - // like anyone's. + if sort == Sort::New { + let mut kept: Vec = casts.iter().filter(|cast| keep_for_feed(cast, leanings)).cloned().collect(); + sort_casts(&mut kept, sort); + return kept; + } let weight = |cast: &Cast| leanings.weight_in(cast, sort); let days = span_days(casts); let key = |cast: &Cast, weight: f64| match sort { Sort::Top => (points(cast) + 1) as f64 * weight, Sort::Hot => hot_score(points(cast), cast.ts) + weight.log10() * days, - Sort::New if weight < 1.0 => cast.ts as f64 - SHOW_LESS_NEW_SECS as f64 * days, Sort::New => cast.ts as f64, }; let mut keyed: Vec<(f64, f64, &Cast)> = casts .iter() + .filter(|cast| keep_for_feed(cast, leanings)) .map(|cast| { let weight = weight(cast); (key(cast, weight), weight, cast) @@ -633,38 +633,91 @@ mod tests { } #[test] - fn showing_less_drops_far_in_hot_and_top_and_lower_in_new() { - // The busiest cast is by an account the reader sees less of; the - // quietest is a stranger's, 5 hours older. + fn showing_less_removes_most_posts_and_strongly_lowers_the_remaining_hot_and_top() { + let retained = format!("0x{:040x}", 0); + let suppressed = format!("0x{:040x}", 1); let casts = vec![ - by(1, cast("less", 100, 0, 0, "2026-10-17T12:00:00Z")), - by(2, cast("twenty", 20, 0, 0, "2026-10-17T12:00:00Z")), - by(3, cast("five", 5, 0, 0, "2026-10-17T12:00:00Z")), + by(1, cast(&retained, 20, 0, 0, "2026-10-17T12:00:00Z")), + by(1, cast(&suppressed, 1_000_000, 0, 0, "2026-10-17T14:00:00Z")), + by(2, cast("twenty", 20, 0, 0, "2026-10-17T11:00:00Z")), + by(3, cast("five", 5, 0, 0, "2026-10-17T13:00:00Z")), by(4, cast("quiet", 0, 0, 0, "2026-10-17T07:00:00Z")), ]; - assert_eq!(order(sorted(&casts, Sort::Top)), ["less", "twenty", "five", "quiet"]); - assert_eq!(order(sorted(&casts, Sort::Hot)), ["less", "twenty", "five", "quiet"]); - assert_eq!(order(sorted(&casts, Sort::New)), ["five", "less", "twenty", "quiet"]); - let leanings = Leanings { less: BTreeSet::from([1]), ..Leanings::default() }; - assert!(!leanings.is_empty()); - // 100 points weigh as (100 + 1) / 25, about 4: under 5, over none. In - // hot, as if 17½ hours older, as the casts span less than a day. - // Nothing is left out. - assert_eq!(order(sorted_for(&casts, Sort::Top, &leanings)), ["twenty", "five", "less", "quiet"]); - assert_eq!(order(sorted_for(&casts, Sort::Hot, &leanings)), ["twenty", "five", "less", "quiet"]); - // In new, as if 6 hours older, for the same reason: below the cast 5 - // hours older. - assert_eq!(order(sorted_for(&casts, Sort::New, &leanings)), ["five", "twenty", "quiet", "less"]); - // One nobody has liked yet goes below a stranger's with no likes - // either, however much newer: (0 + 1) / 25 is less than 0 + 1. - let unliked = vec![by(1, cast("less", 0, 0, 0, "2026-10-17T12:00:00Z")), casts[3].clone()]; - assert_eq!(order(sorted_for(&unliked, Sort::Top, &leanings)), ["quiet", "less"]); - assert_eq!(leanings.weight(&casts[0]), SHOW_LESS_WEIGHT); - assert_eq!(leanings.weight(&casts[1]), 1.0); - // An account kept as both counts as a favorite, in new too. - let both = Leanings { favorites: BTreeSet::from([1]), less: BTreeSet::from([1]), ..Leanings::default() }; - assert_eq!(both.weight(&casts[0]), FAVORITE_WEIGHT); - assert_eq!(sorted_for(&casts, Sort::New, &both), sorted(&casts, Sort::New)); + let less = Leanings { less: BTreeSet::from([1]), ..Leanings::default() }; + assert!(keep_for_feed(&casts[0], &less)); + assert!(!keep_for_feed(&casts[1], &less)); + for sort in [Sort::Hot, Sort::Top] { + assert_eq!( + order(sorted_for(&casts, sort, &less)), + vec!["twenty".to_owned(), "five".to_owned(), "quiet".to_owned(), retained.clone()] + ); + } + assert_eq!( + order(sorted_for(&casts, Sort::New, &less)), + vec!["five".to_owned(), retained.clone(), "twenty".to_owned(), "quiet".to_owned()] + ); + let both = Leanings { favorites: BTreeSet::from([1]), followed: BTreeSet::from([1]), ..less.clone() }; + for sort in Sort::ALL { + assert_eq!(sorted_for(&casts, sort, &both), sorted_for(&casts, sort, &less)); + } + assert_eq!(both.weight(&casts[0]), SHOW_LESS_WEIGHT); + assert_eq!(both.weight_in(&casts[0], Sort::Hot), SHOW_LESS_WEIGHT); + assert_eq!(both.weight_in(&casts[0], Sort::New), 1.0); + assert_eq!(sorted(&casts, Sort::New).len(), casts.len(), "ordinary/direct ranking does not thin posts"); + } + + #[test] + fn show_less_selection_is_stable_across_pages_permutations_and_count_refreshes() { + let leaning = Leanings { less: BTreeSet::from([42]), ..Leanings::default() }; + let casts: Vec<_> = + (0..1_000).map(|i| by(42, cast(&format!("0x{i:040x}"), i, 0, 0, "2026-10-17T12:00:00Z"))).collect(); + let selected = |casts: &[Cast]| { + casts + .iter() + .filter(|cast| keep_for_feed(cast, &leaning)) + .map(|cast| cast.hash.clone()) + .collect::>() + }; + let kept = selected(&casts); + assert!((80..=120).contains(&kept.len()), "About 90% should disappear: kept {} of 1,000", kept.len()); + let paged: BTreeSet<_> = casts.chunks(37).flat_map(selected).collect(); + assert_eq!(paged, kept); + let mut reversed = casts.clone(); + reversed.reverse(); + assert_eq!(selected(&reversed), kept); + let mut refreshed = casts.clone(); + for cast in &mut refreshed { + cast.ts += 86_400; + cast.likes += 999; + cast.recasts = 500; + cast.replies = 100; + cast.hash = cast.hash.to_ascii_uppercase(); + } + assert_eq!( + selected(&refreshed).into_iter().map(|hash| hash.to_ascii_lowercase()).collect::>(), + kept + ); + for cast in &mut refreshed { + cast.author.fid = 7; + } + assert!(refreshed.iter().all(|cast| keep_for_feed(cast, &leaning)), "unselected authors keep every post"); + // Fixed vectors lock selection across browser sessions and builds. + let typical = Leanings { less: BTreeSet::from([100]), ..Leanings::default() }; + let selected: Vec<_> = (200..260) + .filter(|id| { + keep_for_feed(&by(100, cast(&format!("0x{id:040x}"), 1, 0, 0, "2026-10-17T12:00:00Z")), &typical) + }) + .collect(); + assert_eq!(selected, [200, 215, 222, 224, 239, 247], "The 60-post browser fixture keeps 6 posts"); + assert_eq!( + casts + .iter() + .take(40) + .filter(|cast| keep_for_feed(cast, &leaning)) + .map(|cast| cast.hash.clone()) + .collect::>(), + vec![format!("0x{:040x}", 1), format!("0x{:040x}", 10)] + ); } /// The account the measured test leans on. @@ -724,11 +777,14 @@ mod tests { for sort in Sort::ALL { let anyone = sorted(&list, sort); let (lowered, raised) = (sorted_for(&list, sort, &less), sorted_for(&list, sort, &favorite)); - // Every cast is there, once. - for leaned in [&lowered, &raised] { - assert_eq!(leaned.len(), list.len()); - assert_eq!(hashes(leaned), hashes(&list)); - } + assert_eq!(raised.len(), list.len()); + assert_eq!(hashes(&raised), hashes(&list)); + assert_eq!(lowered.iter().filter(|c| c.author.fid == TYPICAL).count(), 1); + assert_eq!(lowered.len(), 115); + assert_eq!( + hashes(&lowered.iter().filter(|c| c.author.fid != TYPICAL).cloned().collect::>()), + hashes(&list.iter().filter(|c| c.author.fid != TYPICAL).cloned().collect::>()) + ); let before = median_place(&anyone, TYPICAL); let (down, up) = (median_place(&lowered, TYPICAL), median_place(&raised, TYPICAL)); println!( @@ -738,11 +794,10 @@ mod tests { // A typical account: in the middle of the list to start with. assert!((0.4 * n..=0.6 * n).contains(&before), "{days}d {sort:?} starts at {before}"); if sort == Sort::New { - // 6 hours for each day the list spans is about a quarter - // of it, so a cast moves about a quarter of the list down, - // and those near the end only as far as the end: a fifth - // at least. - assert!(down - before >= 0.2 * n, "{days}d new: {before} to {down}"); + assert!( + lowered.windows(2).all(|pair| pair[0].ts >= pair[1].ts), + "new remains chronological after thinning" + ); // A favorite's casts stay in time order. assert_eq!(raised, anyone); } else { @@ -811,7 +866,7 @@ mod tests { } #[test] - fn hot_and_new_moves_grow_with_the_days_a_list_spans() { + fn hot_moves_grow_with_span_while_new_stays_chronological_after_thinning() { // Two days of casts, from the oldest to the newest. let newest = by(1, cast("newest", 10, 0, 0, "2026-10-17T00:00:00Z")); let oldest = by(3, cast("oldest", 1, 0, 0, "2026-10-15T00:00:00Z")); @@ -826,16 +881,19 @@ mod tests { let hot = |list: Vec| order(sorted_for(&list, Sort::Hot, &favorites)); assert_eq!(hot(with_favorite("2026-10-15T14:00:00Z")), ["favorite", "newest", "oldest"]); assert_eq!(hot(with_favorite("2026-10-15T13:00:00Z")), ["newest", "favorite", "oldest"]); - // In new, a cast seen less of is as if twice 6 hours older: between - // casts 11 and 13 hours older. let less = Leanings { less: BTreeSet::from([1]), ..Leanings::default() }; + let mut retained = newest.clone(); + retained.hash = format!("0x{:040x}", 0); let casts = vec![ - newest.clone(), + retained.clone(), by(2, cast("eleven", 1, 0, 0, "2026-10-16T13:00:00Z")), by(4, cast("thirteen", 1, 0, 0, "2026-10-16T11:00:00Z")), oldest.clone(), ]; - assert_eq!(order(sorted_for(&casts, Sort::New, &less)), ["eleven", "newest", "thirteen", "oldest"]); + assert_eq!( + order(sorted_for(&casts, Sort::New, &less)), + vec![retained.hash, "eleven".into(), "thirteen".into(), "oldest".into()] + ); } #[test] diff --git a/crates/allthing-web/src/components/profile_card.rs b/crates/allthing-web/src/components/profile_card.rs index cfc4ca90..2e4ed589 100644 --- a/crates/allthing-web/src/components/profile_card.rs +++ b/crates/allthing-web/src/components/profile_card.rs @@ -181,7 +181,9 @@ fn leanings(fid: u64, username: String) -> impl IntoView { let (favorite, less) = untrack(|| (leaning.is_favorite(fid), leaning.shows_less(fid))); said.set(match (favorite, less) { (true, _) => format!("@{handle}'s casts rise in your feeds."), - (_, true) => format!("@{handle}'s casts sink in your feeds."), + (_, true) => { + format!("About 90% fewer posts from @{handle} after the next feed load. Press Show less again to undo.") + } _ => format!("@{handle}'s casts are back in place in your feeds."), }); }); diff --git a/crates/allthing-web/src/components/show_less.rs b/crates/allthing-web/src/components/show_less.rs index 47a1fd3a..09ad36d4 100644 --- a/crates/allthing-web/src/components/show_less.rs +++ b/crates/allthing-web/src/components/show_less.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: AGPL-3.0-or-later //! Show less (`crate::leanings`): on a profile, the reader sees much less of -//! the account, in this browser. Its casts drop well down the reader's hot -//! and top feeds, and lower in new (`allthing_core::ranking::sorted_for`); -//! they carry no mark, and nothing is hidden. +//! the account, in this browser. Stable sampling omits about 90% of their +//! feed posts; remaining hot/top candidates rank lower, while New retains +//! time order (`allthing_core::ranking::sorted_for`). Profiles stay available. use leptos::prelude::*; @@ -48,7 +48,7 @@ pub fn ShowLessButton( {on_press.is_none().then(|| { hint.view( "show-less-what", - "Considerably reduces their casts in your feeds: well down in hot and top, lower in new. Nothing is hidden. Kept in this browser only.", + "Omits about 90% of their posts from Home, Following and channel feeds. The rest rank much lower in hot and top; new stays chronological. Profiles stay available. Toggle again to undo. Kept in this browser only.", ) })} diff --git a/crates/allthing-web/src/leanings.rs b/crates/allthing-web/src/leanings.rs index ae02f1eb..43c43d6c 100644 --- a/crates/allthing-web/src/leanings.rs +++ b/crates/allthing-web/src/leanings.rs @@ -3,8 +3,8 @@ //! their favorites and those they see less of, kept in this browser's //! localStorage (`allthing.favorites`, `allthing.show-less`) and sent nowhere. //! A favorite's casts rise well up the reader's hot and top feeds, and carry -//! a star there; the casts of an account they see less of drop well down hot -//! and top, and lower in new, with no mark, and nothing is hidden +//! a star there; Show less omits about 90% of that account's feed posts and +//! lowers the rest in hot/top. New retains chronological order //! (`allthing_core::ranking::sorted_for`). An account is one or the other, //! or neither. //! The reader's other tabs follow a change (the `storage` event). The static diff --git a/crates/allthing-web/src/pages/home.rs b/crates/allthing-web/src/pages/home.rs index 0fc05278..3e90abdd 100644 --- a/crates/allthing-web/src/pages/home.rs +++ b/crates/allthing-web/src/pages/home.rs @@ -8,7 +8,7 @@ use allthing_core::following::{FEED_PARAM, Feed, FollowingPayload, following_pat use allthing_core::format::full_number; use allthing_core::listing::top_level; use allthing_core::model::Cast; -use allthing_core::ranking::{Sort, TimeRange, TrendingWindow, sorted, sorted_for}; +use allthing_core::ranking::{Sort, TimeRange, TrendingWindow, sorted_for}; use allthing_core::time::format_date; use leptos::prelude::*; use leptos_router::hooks::{use_location, use_query_map}; @@ -157,9 +157,10 @@ fn after_a_paint(sort: Memo) -> ReadSignal { /// The newest casts on Farcaster, read from Snapchain's event stream and /// rebuilt at the edge every 30 seconds: strictly newest first within the -/// selected range, irrespective of favorites or Show less. +/// selected range. Show less reduces candidates without changing their time order. #[component] fn Latest(sort: Memo, range: Memo) -> impl IntoView { + let leaning = crate::leanings::use_leaning(); let data = LocalResource::new(|| async { api::get::(&latest_path()).await }); // Only another client filter redraws the list: a tap on a sort tab // changes the query too, and mustn't rebuild the cards it's leaving. @@ -182,10 +183,12 @@ fn Latest(sort: Memo, range: Memo) -> impl IntoView { Some(fid) => from_client(pool, ¬es, fid), None => pool, }; + let sorted_by = StoredValue::new(None); let casts = Memo::new(move |_| { let window = range.get().window(); let within: Vec = pool.iter().filter(|c| window.holds(c.ts, now_secs())).cloned().collect(); - sorted(&within, Sort::New) + let leanings = leaning.for_list(sorted_by, (window, client)); + sorted_for(&within, Sort::New, &leanings) }); view! { @@ -547,7 +550,7 @@ fn Following(sort: Memo, listed: ReadSignal, range: Memo) // Hot scales a leaning by the days its input spans: // casts outside this range must not inflate that lift. let within: Vec = casts.iter().filter(|c| window.holds(c.ts, now)).cloned().collect(); - if order == Sort::New { sorted(&within, order) } else { sorted_for(&within, order, &leanings) } + sorted_for(&within, order, &leanings) }; let mut all = rank(&pool); more.with(|pages| { diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 677439c5..a41e9b46 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -241,9 +241,9 @@ Allthing counts unique Farcaster accounts with at least one successful message m - What each does shows under the two buttons, in Allthing's style (`favorite::Hint`): when the pointer rests on it, when keyboard focus is on it, and for 3 seconds after a press, as a phone can't hover. It's also the button's description. No native tooltip: one shows late, in the system's style, and never on a phone. Their hover styles apply only where a pointer hovers, so on a phone a button just turned off doesn't keep the look of one that's on. - They are kept in this browser's localStorage (`allthing.favorites`, `allthing.show-less`) and sent nowhere, so they need no sign-in and the static copy has them too. The reader's other tabs follow a change (the `storage` event). - Profile cards have them too (`components/profile_card.rs`): one row of two small buttons, 28 px tall, between the card's counts and its facts, on anyone's card but the reader's own. They are the profile's buttons and its state: pressed in a card, they show pressed on the profile and in the reader's other tabs, and the other way round. In a card a press says what it did to screen readers ("@alice's casts rise in your feeds") instead of showing the hint, so the card keeps its size. - - In the reader's hot and top feeds, on the front page and in channels, a favorite's cast weighs as if it had 25 times its points, and carries a small star (not on the account's own profile, where Favorite is). The cast of an account they see less of weighs a 25th, and carries no mark. In top a cast counts its points plus one, so one nobody has reacted to yet moves too; in hot the log of the points moves by the weight's once for each day the list spans, from its oldest cast to its newest (`span_days`, a day at least), as if the cast were 17½ hours newer or older for each. Over many days hot is nearly time order, so a move of fixed hours would barely move a cast there; this way a cast moves as far through a quiet channel's week of casts, or the past week's front page, as through a day's (`allthing_core::ranking::sorted_for`, `FAVORITE_WEIGHT`, `SHOW_LESS_WEIGHT`). On a list of a day or of a week, a typical account's casts go from its middle to its first or last few places: the unit tests and `e2e/leaning-strength.fixture.spec.ts` measure it. The front page ranks each page of 100 casts as it comes, so that cards already shown stay where they are, and a cast goes as far as the end or the top of its own page. A feed on screen keeps its order when either changes in its own tab, so the cast under a profile card stays under it: the change applies when the feed is next loaded, sorted or filtered (`Leaning::for_list`), and a favorite's star shows at once. A change made in another tab re-sorts it in place. - - On the front page, New is strictly newest first within its selected time range, in both tabs: favorites, followed accounts and Show less do not change that order. The time selector stays available in every sort. In channels, Show less still places casts as if posted 6 hours earlier for each day the list spans (`SHOW_LESS_NEW_SECS`; the time shown stays true); favorites stay in time order. - - Nothing is hidden: every cast stays in the list, and leanings only move the reader's own feeds, and only theirs. + - In the reader's hot and top feeds, on the front page and in channels, a favorite's cast weighs as if it had 25 times its points, and carries a small star (not on the account's own profile, where Favorite is). Show less first omits a stable approximately 90% of that author's feed candidates, then weights the remainder at 1/100 in Hot and Top, with no mark. Selection hashes the FID and full post hash, so it does not reshuffle across pages, refreshes or sorts; it overrides Favorite and Followed boosts. A percentage describes larger samples: a small batch may retain no posts from that account. In top a cast counts its points plus one, so one nobody has reacted to yet moves too; in hot the log of the points moves by the weight's once for each day the list spans, from its oldest cast to its newest (`span_days`, a day at least), as if the cast were 17½ hours newer or older for each. Over many days hot is nearly time order, so a move of fixed hours would barely move a cast there; this way a cast moves as far through a quiet channel's week of casts, or the past week's front page, as through a day's (`allthing_core::ranking::sorted_for`, `FAVORITE_WEIGHT`, `SHOW_LESS_WEIGHT`). On a list of a day or of a week, a typical account's casts go from its middle to its first or last few places: the unit tests and `e2e/leaning-strength.fixture.spec.ts` measure it. The front page ranks each page of 100 casts as it comes, so that cards already shown stay where they are, and a cast goes as far as the end or the top of its own page. A feed on screen keeps its order when either changes in its own tab, so the cast under a profile card stays under it: the change applies when the feed is next loaded, sorted or filtered (`Leaning::for_list`), and a favorite's star shows at once. A change made in another tab re-sorts it in place. + - On Home, Following and channels, New stays strictly chronological within its selected time range after Show less sampling; Favorite and Followed boosts never change that order. The time selector stays available in every sort. Direct profiles, search and post pages remain available in full. + - Show less changes only this reader's feed candidates and can be undone with the same button. It does not mute notifications, hide mentions, remove public posts or block communication; private Mute and Block provide those separate controls. Preferences remain in this browser and sync with its other tabs. Anonymous early cards wait for the Rust filter when Show less preferences exist, avoiding a flash of candidates the app will omit; ordinary early paint retains the measured renderer. - **Speed** ([PERFORMANCE.md](PERFORMANCE.md)): - posts open instantly from the feed's own data; - cards prefetch on hover and in idle time, within caps, and the cards in view have who liked and recasted them, and their replies, preloaded in batches; a link to a post or a channel fetches that page's data as it's pressed or touched, or once a mouse has rested on it for 100 ms (`split.rs`; not a profile's link, whose press opens the account's card); diff --git a/e2e/early-paint.fixture.spec.ts b/e2e/early-paint.fixture.spec.ts index 4387dd19..c62c0245 100644 --- a/e2e/early-paint.fixture.spec.ts +++ b/e2e/early-paint.fixture.spec.ts @@ -481,24 +481,27 @@ test("a signed-in reader's Following tab isn't drawn early: the app draws it", a expect(await observedSignedInBoot(page)).toEqual([]); }); -test("an account seen less of is lower in the first cards too, as in the app", async ({ page }) => { +test("Show less waits for the Rust feed filter without flashing suppressed early cards", async ({ page }) => { const held = await hold(page); await serve(page); - await page.addInitScript(() => localStorage.setItem("allthing.show-less", JSON.stringify([{ fid: 1, username: "ann" }]))); + await page.addInitScript(() => { + localStorage.setItem("allthing.show-less", JSON.stringify([{ fid: 1, username: "ann" }])); + (window as any).__lessEarlyCards = false; + new MutationObserver(() => { + if (document.querySelector("#boot .cast-card")) (window as any).__lessEarlyCards = true; + }).observe(document, { childList: true, subtree: true }); + }); + const response = page.waitForResponse(r => r.url().includes("/api/v1/home")); await page.goto("/"); + await (await response).finished(); + await page.waitForTimeout(100); const boot = page.locator("#boot"); - // Hot, seeing less of ann: her cast, second without it, is as if hours - // older and goes below dee's quiet ones, 5th, as in the app. - await expect(boot.locator(".cast-card")).toHaveCount(6); - expect(await boot.locator(".cast-card").evaluateAll((cards) => cards.map((c) => (c as HTMLElement).dataset.hash))).toEqual( - [busy, favorite, pair, later, rich, rest[0]].map((c) => c.hash), - ); + await expect(boot.locator(".cast-card")).toHaveCount(0); await moduleIn(page, held); - await settled(page, "#boot"); - const early = await shown(page, "#boot"); held.js("continue"); await expect(boot).toHaveCount(0); - expect((await shown(page, "#app")).slice(0, early.length)).toEqual(early); + await expect(page.locator("#app .cast-card").first()).toBeVisible(); + expect(await page.evaluate(() => (window as any).__lessEarlyCards)).toBe(false); }); test("a card with Pepe reactions keeps its box: the early card has the row their chips come into", async ({ page }) => { diff --git a/e2e/feed-session-ranges.fixture.spec.ts b/e2e/feed-session-ranges.fixture.spec.ts index 00ec5edf..80254f51 100644 --- a/e2e/feed-session-ranges.fixture.spec.ts +++ b/e2e/feed-session-ranges.fixture.spec.ts @@ -43,7 +43,7 @@ for (const width of [1280, 390]) { await page.setViewportSize({ width, height: 844 }); await signIn(page, me); await serve(page); - // Neither favorites nor Show less may disturb New's time order. + // Favorites cannot move New; Show less preserves the time order of its subset. await page.addInitScript(() => { localStorage.setItem("allthing.favorites", JSON.stringify([{ fid: 8, username: "author8" }])); localStorage.setItem("allthing.show-less", JSON.stringify([{ fid: 1, username: "author1" }])); @@ -57,7 +57,7 @@ for (const width of [1280, 390]) { const selector = page.getByRole("combobox", { name: "Time range" }); await expect(selector).toBeVisible(); await expect(selector).toHaveValue(range); - const expected = pool.filter((c) => c.ts <= now && now - c.ts < span).map((c) => c.text); + const expected = pool.filter((c) => c.author.fid !== 1 && c.ts <= now && now - c.ts < span).map((c) => c.text); await expect.poll(async () => { // Earlier ranges can genuinely be read while their page loads. // Use this generation's control before checking exact membership. diff --git a/e2e/following-feed.fixture.spec.ts b/e2e/following-feed.fixture.spec.ts index cf7c8b08..3386d988 100644 --- a/e2e/following-feed.fixture.spec.ts +++ b/e2e/following-feed.fixture.spec.ts @@ -103,14 +103,14 @@ test("a signed-in reader's Following tab ranks only whom they follow, at 390", a await context.close(); }); -test("Show less demotes a followed account, and a favorite rises", async ({ page }) => { +test("Show less reduces a followed account in every sort", async ({ page }) => { await signIn(page, me); await page.addInitScript(() => localStorage.setItem("allthing.show-less", JSON.stringify([{ fid: 1, username: "ann" }]))); await serve(page); - await page.goto("/?sort=top&feed=following"); - await expect.poll(() => order(page)).toEqual(["Followed quiet", "Followed busy"]); - await page.goto("/?sort=hot&feed=following"); - await expect.poll(() => order(page)).toEqual(["Followed quiet", "Followed busy"]); + for (const sort of ["top", "hot", "new"]) { + await page.goto(`/?sort=${sort}&feed=following`); + await expect.poll(() => order(page)).toEqual(["Followed quiet"]); + } }); test("a favorite among whom they follow rises in their feed", async ({ page }) => { diff --git a/e2e/follows-pages.fixture.spec.ts b/e2e/follows-pages.fixture.spec.ts index 54f55006..22a12c99 100644 --- a/e2e/follows-pages.fixture.spec.ts +++ b/e2e/follows-pages.fixture.spec.ts @@ -240,6 +240,20 @@ for (const phone of [false, true]) { await settled(page); expect((await page.locator(".follows-toolbar").boundingBox())!.y).toBe(toolbar!.y); expect(await shift(page)).toBeLessThan(0.01); + // The sticky toolbar must remain below the actual header, including + // the phone/tablet grid, rather than disappearing behind its second row. + for (const layoutWidth of phone ? [390, 700, 801] : [1280]) { + await page.setViewportSize({ width: layoutWidth, height: phone ? 844 : 900 }); + await page.evaluate(() => window.scrollTo(0, 600)); + await expect.poll(() => page.evaluate(() => { + const header = document.querySelector(".site-header")!.getBoundingClientRect(); + const toolbar = document.querySelector(".follows-toolbar")!.getBoundingClientRect(); + return Math.abs(toolbar.top - header.bottom); + })).toBeLessThanOrEqual(1); + } + await page.setViewportSize({ width, height: phone ? 844 : 900 }); + await page.evaluate(() => window.scrollTo(0, 0)); + await settled(page); await page.screenshot({ path: testInfo.outputPath(`follows-${width}.png`) }); await page.evaluate(axe.source); const violations = await page.evaluate(async () => { diff --git a/e2e/hover-card-leanings.fixture.spec.ts b/e2e/hover-card-leanings.fixture.spec.ts index 44a2c892..98e1695f 100644 --- a/e2e/hover-card-leanings.fixture.spec.ts +++ b/e2e/hover-card-leanings.fixture.spec.ts @@ -177,7 +177,7 @@ for (const phone of [false, true]) { await press(panel.getByRole("button", { name: "Show less" }), phone); await pressed(panel, false, true); - await expect(live).toHaveText("@alice's casts sink in your feeds."); + await expect(live).toHaveText("About 90% fewer posts from @alice after the next feed load. Press Show less again to undo."); expect(await kept(page)).toEqual({ favorites: [], less: [42] }); await page.screenshot({ path: testInfo.outputPath(`hover-card-leanings-${width}.png`) }); diff --git a/e2e/leaning-hints.fixture.spec.ts b/e2e/leaning-hints.fixture.spec.ts index 782534d6..fd5c3fc5 100644 --- a/e2e/leaning-hints.fixture.spec.ts +++ b/e2e/leaning-hints.fixture.spec.ts @@ -25,7 +25,7 @@ async function phone(browser: Browser, testInfo: TestInfo) { } const FAVORITE = "Considerably boosts their casts in your hot and top feeds. Kept in this browser only."; -const LESS = "Considerably reduces their casts in your feeds: well down in hot and top, lower in new. Nothing is hidden. Kept in this browser only."; +const LESS = "Omits about 90% of their posts from Home, Following and channel feeds. The rest rank much lower in hot and top; new stays chronological. Profiles stay available. Toggle again to undo. Kept in this browser only."; /** The hint is on screen, inside the profile's card, under its button. */ async function under(page: Page, hint: Locator, button: Locator, width: number) { diff --git a/e2e/leaning-strength.fixture.spec.ts b/e2e/leaning-strength.fixture.spec.ts index 174237f2..01431f3c 100644 --- a/e2e/leaning-strength.fixture.spec.ts +++ b/e2e/leaning-strength.fixture.spec.ts @@ -1,16 +1,9 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import { expect, test, type Page } from "@playwright/test"; -/* - * How far Show less and Favorite move an account's casts, measured on the - * front page: a day of 120 casts, 6 of them by a typical account whose - * median place starts in the middle of hot, top and new. Seen less of, its - * casts drop by at least 40% of the list in hot and top, into the last - * quarter, while New stays chronological; as a favorite, they rise as far - * in hot and top, into the first quarter, and new stays in time order. - * Every cast is still listed. The casts are made as allthing_core::ranking's - * `a_day_of_casts` makes them for `leanings_move_a_typical_account_far`. - */ +/* Show less removes a stable nine tenths of an account's candidates across + * all feed sorts. Remaining Hot/Top posts sink; New is a chronological subset. + * Favorite retains its independent lift on the original 120-post fixture. */ test.skip(Boolean(process.env.E2E_PAGES), "uses the Worker API contract"); test.use({ viewport: { width: 1280, height: 900 } }); @@ -54,15 +47,15 @@ const day = aDayOfCasts(); const n = day.length; const hashes = day.map((c) => c.hash).sort(); -async function serve(page: Page) { +async function serve(page: Page, pool = day) { await page.route("**/api/v1/**", (route) => route.fulfill({ status: 404, json: { error: "Fixture endpoint" } })); await page.route("**/api/v1/home**", (route) => { const later = new URL(route.request().url()).searchParams.has("offset"); - return route.fulfill({ json: { pool: later ? [] : day, notes: {}, meta } }); + return route.fulfill({ json: { pool: later ? [] : pool, notes: {}, meta } }); }); - await page.route("**/api/v1/latest", (route) => route.fulfill({ json: { pool: day, notes: {}, cursors: [], meta } })); + await page.route("**/api/v1/latest", (route) => route.fulfill({ json: { pool, notes: {}, cursors: [], meta } })); await page.route("**/api/v1/user?*", (route) => - route.fulfill({ json: { user: typical, pool: day.filter((c) => c.author.fid === TYPICAL), notes: {}, meta } }), + route.fulfill({ json: { user: typical, pool: pool.filter((c) => c.author.fid === TYPICAL), notes: {}, meta } }), ); } @@ -72,7 +65,7 @@ const SORTS = Object.keys(FEEDS) as Sort[]; /** The front page's list in `sort`, every card drawn (it grows by 25 as its * end comes near): each card's hash and account. */ -async function listed(page: Page, sort: Sort) { +async function listed(page: Page, sort: Sort, expectedHashes = hashes) { await page.goto(FEEDS[sort]); await expect(page.locator("#app .cast-list .cast-card:not(.skeleton), #app .seen-notice").first()).toBeVisible(); // Compare ranking with the complete pool through the reader's real control. @@ -88,12 +81,12 @@ async function listed(page: Page, sort: Sort) { }, { timeout: 30_000 }, ) - .toBe(n); + .toBe(expectedHashes.length); const list = await cards.evaluateAll((all) => all.map((card) => ({ hash: (card as HTMLElement).dataset.hash ?? "", fid: Number((card as HTMLElement).dataset.fid) })), ); // Every cast is listed, once. - expect(list.map((card) => card.hash).sort()).toEqual(hashes); + expect(list.map((card) => card.hash).sort()).toEqual(expectedHashes); return list; } @@ -104,11 +97,11 @@ function medianPlace(list: { fid: number }[]) { } /** The typical account's median place in each sort, and the lists. */ -async function measure(page: Page) { +async function measure(page: Page, expectedHashes = hashes) { const lists = {} as Record; const places = {} as Record; for (const sort of SORTS) { - lists[sort] = await listed(page, sort); + lists[sort] = await listed(page, sort, expectedHashes); places[sort] = medianPlace(lists[sort]); } return { lists, places }; @@ -121,21 +114,32 @@ async function lean(page: Page, button: "Show less" | "Favorite") { await expect(pressed).toHaveAttribute("aria-pressed", "true"); } -test("Show less drops a typical account's casts from the middle of hot and top into the last quarter, while New stays chronological", async ({ page }) => { - await serve(page); - const before = await measure(page); +test("Show less omits nine in ten posts consistently across sorts and keeps New chronological", async ({ page }) => { + const ordinary = day.filter(c => c.author.fid !== TYPICAL); + const template = day.find(c => c.author.fid === TYPICAL)!; + const targets = Array.from({ length: 60 }, (_, i) => ({ + ...template, hash: hex(200 + i), text: `Typical cast ${200 + i}`, + ts: now - (2 + 4 * (i % 6)) * 3600, likes: 20 + 10 * (i % 6), + })); + const pool = [...ordinary, ...targets]; + // Golden selections are shared protocol fixtures, not a second ranking + // implementation: the native suite locks these exact six hash identities. + const retained = [200, 215, 222, 224, 239, 247].map(hex); + const expected = [...ordinary.map(c => c.hash), ...retained].sort(); + await serve(page, pool); + const before = await measure(page, pool.map(c => c.hash).sort()); await lean(page, "Show less"); - const after = await measure(page); - console.log(`Show less: the typical account's median place of ${n}, before and after:`, JSON.stringify({ before: before.places, after: after.places })); + await expect(page.locator(`.cast-card[data-hash="${hex(201)}"]`)).toBeVisible(); + const after = await measure(page, expected); for (const sort of SORTS) { - expect(before.places[sort], `${sort} starts in the middle`).toBeGreaterThanOrEqual(0.4 * n); - expect(before.places[sort], `${sort} starts in the middle`).toBeLessThanOrEqual(0.6 * n); + expect(before.lists[sort].filter(c => c.fid === TYPICAL)).toHaveLength(60); + expect(after.lists[sort].filter(c => c.fid === TYPICAL).map(c => c.hash).sort()).toEqual([...retained].sort()); + expect(after.lists[sort].filter(c => c.fid !== TYPICAL)).toEqual(before.lists[sort].filter(c => c.fid !== TYPICAL)); } for (const sort of ["hot", "top"] as const) { - expect(after.places[sort] - before.places[sort], `${sort}: down by 40% of the list`).toBeGreaterThanOrEqual(0.4 * n); - expect(after.places[sort], `${sort}: in the last quarter`).toBeGreaterThan(0.75 * n); + expect(after.places[sort], `${sort}: remaining posts rank in the last quarter`).toBeGreaterThan(0.75 * expected.length); } - expect(after.lists.new, "new: in time order, as before").toEqual(before.lists.new); + expect(after.lists.new).toEqual(before.lists.new.filter(c => expected.includes(c.hash))); }); test("Favorite raises a typical account's casts from the middle of hot and top into the first quarter, and new stays in time order", async ({ page }) => { diff --git a/e2e/search-dismissal.fixture.spec.ts b/e2e/search-dismissal.fixture.spec.ts index 4562fe3f..801d4013 100644 --- a/e2e/search-dismissal.fixture.spec.ts +++ b/e2e/search-dismissal.fixture.spec.ts @@ -24,6 +24,13 @@ async function searches(page: Page) { await box.fill("alpha"); await expect(list).toBeVisible(); await expect(list.getByRole("option").first()).toContainText("alpha person"); + const inputBounds = (await box.boundingBox())!; + const resultsBounds = (await list.boundingBox())!; + expect(resultsBounds.y).toBeGreaterThanOrEqual(inputBounds.y + inputBounds.height + 4); + if (page.viewportSize()!.width <= 640) { + const header = (await page.locator(".site-header").boundingBox())!; + expect(resultsBounds.y).toBeGreaterThanOrEqual(header.y + header.height + 4); + } await box.fill("beta"); await expect.poll(() => requests.includes("beta")).toBe(true); return { box, list, release }; diff --git a/e2e/show-less.fixture.spec.ts b/e2e/show-less.fixture.spec.ts index 53a804eb..2e7e48d4 100644 --- a/e2e/show-less.fixture.spec.ts +++ b/e2e/show-less.fixture.spec.ts @@ -4,15 +4,9 @@ import { signIn } from "./helpers/farcaster"; import { showSeenPosts } from "./helpers/app"; import { NOW } from "./fixture-time"; -/* - * Show less: on a profile, the reader sees much less of the account, in this - * browser. Its casts drop well down the reader's hot and top feeds, on the - * front page and in channels (a 25th of their points), and lower in new (as - * if 6 hours older for each day the list spans; these casts span less than - * a day), with no mark; nothing is hidden. An account is a - * favorite or one the reader sees less of, not both. Nothing is sent - * anywhere. How far, measured: leaning-strength.fixture.spec.ts. - */ +/* Show less keeps a stable ~10% of the account's feed posts across every + * sort, while direct profiles remain available and the choice is reversible. + * The reduction and retained-post demotion are measured in leaning-strength. */ test.skip(Boolean(process.env.E2E_PAGES), "uses the Worker API contract"); @@ -64,7 +58,7 @@ const kept = (page: Page) => less: JSON.parse(localStorage.getItem("allthing.show-less") ?? "[]"), })); -test("show less drops an account's casts to the end of hot and top, with no mark or hidden casts; home New remains chronological", async ({ page }) => { +test("show less suppresses stable candidates across all feeds and can be undone", async ({ page }) => { await serve(page); await feed(page, "/?sort=top", ["Busy", "Middling", "Newest"]); await feed(page, "/?sort=new", ["Newest", "Busy", "Middling"]); @@ -73,23 +67,21 @@ test("show less drops an account's casts to the end of hot and top, with no mark const less = page.getByRole("button", { name: "Show less" }); await expect(less).toHaveAttribute("aria-pressed", "false"); await expect(less).toHaveAccessibleDescription( - "Considerably reduces their casts in your feeds: well down in hot and top, lower in new. Nothing is hidden. Kept in this browser only.", + "Omits about 90% of their posts from Home, Following and channel feeds. The rest rank much lower in hot and top; new stays chronological. Profiles stay available. Toggle again to undo. Kept in this browser only.", ); await less.click(); await expect(less).toHaveAttribute("aria-pressed", "true"); expect(await kept(page)).toEqual({ favorites: [], less: [{ fid: 1, username: "ann" }] }); - // Ann's 30 points weigh as (30 + 1) / 25, about 1.2: under bob's 20 and - // even cat's 1 (2, with the one). In hot, as if 17½ hours older: under - // cat's cast too. Still listed. - await feed(page, "/?sort=top", ["Middling", "Newest", "Busy"]); - await expect(page.locator(".cast-card", { hasText: "Busy" })).toBeVisible(); - await expect(page.locator(".cast-card", { hasText: "Busy" }).locator(".favorite-mark")).toHaveCount(0); - await feed(page, "/?sort=hot", ["Middling", "Newest", "Busy"]); - await feed(page, "/c/books?sort=top", ["Middling", "Newest", "Busy"]); - // In new, as if 6 hours older: under bob's cast from the same moment. - await feed(page, "/c/books?sort=new", ["Newest", "Middling", "Busy"]); - await feed(page, "/?sort=new", ["Newest", "Busy", "Middling"]); + // This fixture's hash is outside Ann's retained tenth. Her direct + // profile stays complete; Home and channel feeds omit that candidate. + await expect(page.locator('.cast-card[data-hash="' + busy.hash + '"]')).toBeVisible(); + await feed(page, "/?sort=top", ["Middling", "Newest"]); + await expect(page.locator('.cast-card[data-hash="' + busy.hash + '"]')).toHaveCount(0); + await feed(page, "/?sort=hot", ["Middling", "Newest"]); + await feed(page, "/c/books?sort=top", ["Middling", "Newest"]); + await feed(page, "/c/books?sort=new", ["Newest", "Middling"]); + await feed(page, "/?sort=new", ["Newest", "Middling"]); // Undone, everything is as it was. await page.goto("/u/ann"); @@ -112,8 +104,8 @@ test("an account is a favorite or one you see less of, not both, and another tab await expect(less).toHaveAttribute("aria-pressed", "true"); await expect(favorite).toHaveAttribute("aria-pressed", "false"); expect(await kept(other)).toEqual({ favorites: [], less: [{ fid: 1, username: "ann" }] }); - // The first tab's feed re-sorts in place. - await expect.poll(() => order(page)).toEqual(["Middling", "Newest", "Busy"]); + // Another tab applies the same suppression immediately. + await expect.poll(() => order(page)).toEqual(["Middling", "Newest"]); await favorite.click(); await expect(less).toHaveAttribute("aria-pressed", "false"); expect(await kept(other)).toEqual({ favorites: [{ fid: 1, username: "ann" }], less: [] }); diff --git a/style/allthing.css b/style/allthing.css index 361f2faf..81bcff08 100644 --- a/style/allthing.css +++ b/style/allthing.css @@ -4607,7 +4607,7 @@ p.jumbo .emoji img { @media (max-width: 640px) { .search-suggest { position: fixed; - top: 62px; + top: 106px; left: 12px; right: 12px; width: auto; @@ -4828,10 +4828,9 @@ p.jumbo .emoji img { margin: auto 8px; } -/* Searching and sorting the list, under the site's header as it scrolls: - the header is 60.5 px tall, 56.5 at 640 px and narrower (measured on 21 - October 2026), and the toolbar tucks half a pixel under it, so no row - shows between them whichever way a font rounds. */ +/* Keep the list toolbar under the header: 60.5 px on desktop, 104.5 px + in the tablet grid and 100.5 px on phones. Tuck half a pixel under its + border so rows cannot show through a fractional rounding gap. */ .follows-page { --site-header-height: 60px; } @@ -5056,9 +5055,15 @@ p.jumbo .emoji img { text-decoration: underline; } +@media (max-width: 800px) { + html { scroll-padding-top: 128px; } + .follows-page { --site-header-height: 104px; } +} + @media (max-width: 640px) { + html { scroll-padding-top: 124px; } .follows-page { - --site-header-height: 56px; + --site-header-height: 100px; } .follows-head { @@ -5699,6 +5704,14 @@ p.jumbo .emoji img { gap: 10px; } +/* Inert boot slots reserve the mounted Settings and signed-in bell. */ +.boot-control-space { + width: 36px; + height: 36px; + flex: none; + visibility: hidden; +} + .reply-button { display: inline-flex; align-items: center; @@ -5730,25 +5743,29 @@ p.jumbo .emoji img { /* The header's Cast: as tall as the search box, so the header is as tall with it as without. On a page without Cast its place is kept, unseen and out of reach, so the search box doesn't move from page to page. */ -.header-cast { +.header-cast, +.boot-header-cast { display: flex; flex: none; } -.header-cast:not(.here) { +.header-cast:not(.here), +.boot-header-cast { visibility: hidden; } /* Exactly the search box's height, whatever the display face's metrics: its line box rounds a pixel taller on some systems, which made the header a pixel taller with Cast than without. */ -.header-cast .cast-pill { +.header-cast .cast-pill, +.boot-cast-pill { height: 36px; padding-top: 0; padding-bottom: 0; } -.cast-pill { +.cast-pill, +.boot-cast-pill { display: inline-flex; flex: none; gap: 8px; @@ -5785,6 +5802,7 @@ p.jumbo .emoji img { } .cast-pill .icon, +.boot-cast-pill .icon, .cast-fab .icon, .reply-button .icon, .compose-post .icon { @@ -7497,7 +7515,8 @@ p.jumbo .emoji img { /* Cast floats at the bottom right, in reach of a thumb, and notices sit above it. */ - .header-cast { + .header-cast, + .boot-header-cast { display: none; } @@ -10156,7 +10175,8 @@ p.jumbo .emoji img { border-radius: 50%; } - .header-actions { + .header-actions, + .boot-actions { gap: 12px; } } diff --git a/web/boot.js b/web/boot.js index e4a92744..6bb182f5 100644 --- a/web/boot.js +++ b/web/boot.js @@ -73,13 +73,18 @@ function bootHeader() { const route = item.getAttribute("data-route"); if (route === "/" ? path === "/" : path.startsWith(route)) item.setAttribute("aria-current", "page"); }); + // The mounted header reserves Settings and Cast even before a reader can + // use them. Match their actual boxes, including the display-font width, + // rather than letting the boot search share their space until handover. + const settings = ''; + const cast = 'Cast'; if (base) { - account.outerHTML = 'read-only'; + account.outerHTML = '' + settings + 'read-only'; } else if (/(?:^|;\s*)__Host-allthing-user=[^;]/.test(document.cookie)) { - account.outerHTML = ''; + account.outerHTML = '' + settings + cast + ''; } else { account.outerHTML = - ''; + '' + settings + cast + ''; } } diff --git a/web/paint.js b/web/paint.js index 4967f4fd..28c54179 100644 --- a/web/paint.js +++ b/web/paint.js @@ -126,6 +126,10 @@ async function paint(payload, { sort, range, reader }) { reading = reader !== null; const notes = payload.notes || {}; const lean = leanings(reader); + // The Rust app applies stable Show less sampling. Wait for that guard + // rather than briefly drawing candidates it will omit; ordinary early + // cards keep the existing renderer and request path. + if (lean.less.size) return; // The same inclusive present and exclusive lower edge as TrendingWindow::holds. const now = Math.floor(Date.now() / 1000); const inWindow = (cast) => cast.ts <= now && cast.ts > now - SPANS[range]; From 8918b3ddc1cbebcf815294cc9de3e93ee4aa224d Mon Sep 17 00:00:00 2001 From: ael-dev3 <183124839+ael-dev3@users.noreply.github.com> Date: Tue, 27 Oct 2026 13:22:46 +0100 Subject: [PATCH 3/4] fix(dau): keep early app statistics behind verification --- docs/ARCHITECTURE.md | 2 +- e2e/early-paint.fixture.spec.ts | 30 ++++++++++++++++++++++-- tests/activity-paint.test.mjs | 41 +++++++++++++++++++++++++++++++++ web/paint.js | 20 +++++++++++++++- 4 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 tests/activity-paint.test.mjs diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index a41e9b46..4b852c8d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -241,7 +241,7 @@ Allthing counts unique Farcaster accounts with at least one successful message m - What each does shows under the two buttons, in Allthing's style (`favorite::Hint`): when the pointer rests on it, when keyboard focus is on it, and for 3 seconds after a press, as a phone can't hover. It's also the button's description. No native tooltip: one shows late, in the system's style, and never on a phone. Their hover styles apply only where a pointer hovers, so on a phone a button just turned off doesn't keep the look of one that's on. - They are kept in this browser's localStorage (`allthing.favorites`, `allthing.show-less`) and sent nowhere, so they need no sign-in and the static copy has them too. The reader's other tabs follow a change (the `storage` event). - Profile cards have them too (`components/profile_card.rs`): one row of two small buttons, 28 px tall, between the card's counts and its facts, on anyone's card but the reader's own. They are the profile's buttons and its state: pressed in a card, they show pressed on the profile and in the reader's other tabs, and the other way round. In a card a press says what it did to screen readers ("@alice's casts rise in your feeds") instead of showing the hint, so the card keeps its size. - - In the reader's hot and top feeds, on the front page and in channels, a favorite's cast weighs as if it had 25 times its points, and carries a small star (not on the account's own profile, where Favorite is). Show less first omits a stable approximately 90% of that author's feed candidates, then weights the remainder at 1/100 in Hot and Top, with no mark. Selection hashes the FID and full post hash, so it does not reshuffle across pages, refreshes or sorts; it overrides Favorite and Followed boosts. A percentage describes larger samples: a small batch may retain no posts from that account. In top a cast counts its points plus one, so one nobody has reacted to yet moves too; in hot the log of the points moves by the weight's once for each day the list spans, from its oldest cast to its newest (`span_days`, a day at least), as if the cast were 17½ hours newer or older for each. Over many days hot is nearly time order, so a move of fixed hours would barely move a cast there; this way a cast moves as far through a quiet channel's week of casts, or the past week's front page, as through a day's (`allthing_core::ranking::sorted_for`, `FAVORITE_WEIGHT`, `SHOW_LESS_WEIGHT`). On a list of a day or of a week, a typical account's casts go from its middle to its first or last few places: the unit tests and `e2e/leaning-strength.fixture.spec.ts` measure it. The front page ranks each page of 100 casts as it comes, so that cards already shown stay where they are, and a cast goes as far as the end or the top of its own page. A feed on screen keeps its order when either changes in its own tab, so the cast under a profile card stays under it: the change applies when the feed is next loaded, sorted or filtered (`Leaning::for_list`), and a favorite's star shows at once. A change made in another tab re-sorts it in place. + - In the reader's hot and top feeds, on the front page and in channels, a favorite's cast weighs as if it had 25 times its points, and carries a small star (not on the account's own profile, where Favorite is). Show less first omits a stable approximately 90% of that author's feed candidates, then weights the remainder at 1/100 in Hot and Top, with no mark. Selection hashes the FID and full post hash, so it does not reshuffle across pages, refreshes or sorts; it overrides Favorite and Followed boosts. A percentage describes larger samples: a small batch may retain no posts from that account. In top a cast counts its points plus one, so one nobody has reacted to yet moves too; in hot the log of the points moves by the weight's once for each day the list spans, from its oldest cast to its newest (`span_days`, a day at least), with a logarithmic freshness shift scaled to that span. Over many days hot is nearly time order, so a move of fixed hours would barely move a cast there; this way a cast moves as far through a quiet channel's week of casts, or the past week's front page, as through a day's (`allthing_core::ranking::sorted_for`, `FAVORITE_WEIGHT`, `SHOW_LESS_WEIGHT`). On a list of a day or of a week, a typical account's casts go from its middle to its first or last few places: the unit tests and `e2e/leaning-strength.fixture.spec.ts` measure it. The front page ranks each page of 100 casts as it comes, so that cards already shown stay where they are, and a cast goes as far as the end or the top of its own page. A feed on screen keeps its order when either changes in its own tab, so the cast under a profile card stays under it: the change applies when the feed is next loaded, sorted or filtered (`Leaning::for_list`), and a favorite's star shows at once. A change made in another tab re-sorts it in place. - On Home, Following and channels, New stays strictly chronological within its selected time range after Show less sampling; Favorite and Followed boosts never change that order. The time selector stays available in every sort. Direct profiles, search and post pages remain available in full. - Show less changes only this reader's feed candidates and can be undone with the same button. It does not mute notifications, hide mentions, remove public posts or block communication; private Mute and Block provide those separate controls. Preferences remain in this browser and sync with its other tabs. Anonymous early cards wait for the Rust filter when Show less preferences exist, avoiding a flash of candidates the app will omit; ordinary early paint retains the measured renderer. - **Speed** ([PERFORMANCE.md](PERFORMANCE.md)): diff --git a/e2e/early-paint.fixture.spec.ts b/e2e/early-paint.fixture.spec.ts index c62c0245..1782fcaf 100644 --- a/e2e/early-paint.fixture.spec.ts +++ b/e2e/early-paint.fixture.spec.ts @@ -102,7 +102,7 @@ const apps = { 9152: { fid: 9152, username: "farcaster", display_name: "Farcaster" }, 2904: { fid: 2904, username: "wake", display_name: "wake" }, }; -const apps24h = { to: now - 600, counted: now - 590, active: 120, apps: [{ app: 9152, active: 110 }, { app: 2904, active: 9 }], none: 1 }; +const apps24h = { verification_version: 2, to: Math.floor(now / HOUR) * HOUR, counted: now, active: 120, apps: [{ app: 9152, active: 110 }, { app: 2904, active: 9 }], none: 1 }; const quotes = { [quoted.hash]: quoted, [hiddenQuote.hash]: hiddenQuote }; const payload = { pool, notes: { origins, apps, quotes }, meta, apps_24h: apps24h }; @@ -296,7 +296,7 @@ test("with most of the casts' apps still being read, the early usage bar is the const held = await hold(page); // The first five casts' apps are known; the other 21 are being read. const known = Object.fromEntries(pool.slice(0, 5).map((c) => [c.hash, origins[c.hash]])); - await serve(page, { ...payload, notes: { ...payload.notes, origins: known, pending: pool.length - 5 } }); + await serve(page, { ...payload, apps_24h: { ...apps24h, apps: undefined, unknown: 120, none: 0 }, notes: { ...payload.notes, origins: known, pending: pool.length - 5 } }); await page.goto("/"); const boot = page.locator("#boot"); await expect(boot.locator(".cast-card")).toHaveCount(painted.length); @@ -596,3 +596,29 @@ test("if the app can't load, its panel shows instead of the early cards", async await expect(page.locator("#boot .cast-card")).toHaveCount(0); await expect(page.locator("#boot")).not.toHaveAttribute("data-painted", ""); }); + + +test("legacy or incoherent app counts never decorate early cards", async ({ page }) => { + for (const change of [ + { verification_version: 0 }, + { to: Math.floor(now / HOUR) * HOUR - 3 * HOUR }, + { counted: now + HOUR }, + { apps: [{ app: 9152, active: 121 }] }, + { apps: [{ app: 9152, active: 110 }, { app: 9152, active: 9 }] }, + ]) { + const held = await hold(page); + await serve(page, { ...payload, apps_24h: { ...apps24h, ...change } }); + await page.goto("/"); + const boot = page.locator("#boot"); + await expect(boot.locator(".cast-card")).toHaveCount(painted.length); + await expect(boot.locator(".share-bar")).not.toHaveClass(/\bcounted\b/); + await expect(boot.locator(".share-bar .seg-slot a")).toHaveCount(0); + await moduleIn(page, held); + await settled(page, "#boot"); + const early = await shown(page, "#boot"); + held.js("continue"); + await expect(boot).toHaveCount(0); + expect((await shown(page, "#app")).slice(0, early.length)).toEqual(early); + await expect(page.locator("#app .share-bar")).not.toHaveClass(/\bcounted\b/); + } +}); diff --git a/tests/activity-paint.test.mjs b/tests/activity-paint.test.mjs new file mode 100644 index 00000000..d70545ab --- /dev/null +++ b/tests/activity-paint.test.mjs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { runInNewContext } from "node:vm"; + +test("early app statistics accept coherent windows and reject legacy, stale or malformed evidence", () => { + const source = readFileSync(new URL("../web/paint.js", import.meta.url), "utf8"); + const start = source.indexOf("function verifiedAppCount("); + const end = source.indexOf("\n// Markup.", start); + assert(start >= 0 && end > start); + const accepts = runInNewContext(`(${source.slice(start, end)})`, { + Date: { now: () => 1_800_000_060_000 }, HOUR: 3600, + }); + const valid = { + verification_version: 2, to: 1_800_000_000, counted: 1_800_000_030, + active: 120, unknown: 1, none: 0, apps: [{ app: 9152, active: 110 }], + }; + assert.equal(accepts(valid), true); + assert.equal(accepts({ ...valid, active: 4_294_967_295 }), true); + const unnamed = { ...valid, unknown: 120 }; + delete unnamed.apps; + assert.equal(accepts(unnamed), true, "an empty named-app list is valid evidence too"); + const defaults = { ...valid }; + delete defaults.unknown; + delete defaults.none; + assert.equal(accepts(defaults), true); + for (const bad of [ + undefined, null, {}, { ...valid, verification_version: 0 }, + { ...valid, to: valid.to + 1 }, { ...valid, to: valid.to - 10800 }, + { ...valid, counted: valid.to - 1 }, { ...valid, counted: 1_800_000_061 }, + { ...valid, active: 4_294_967_296 }, { ...valid, active: -1 }, { ...valid, unknown: null }, + { ...valid, unknown: 121 }, { ...valid, none: 121 }, + { ...valid, apps: null }, { ...valid, apps: [null] }, + { ...valid, apps: [{ app: 0, active: 1 }] }, + { ...valid, apps: [{ app: 9152, active: 121 }] }, + { ...valid, apps: [{ app: 9152, active: 110 }, { app: 9152, active: 1 }] }, + ]) { + assert.equal(accepts(bad), false, `unverified window: ${JSON.stringify(bad)}`); + } +}); diff --git a/web/paint.js b/web/paint.js index 28c54179..bd916d4d 100644 --- a/web/paint.js +++ b/web/paint.js @@ -228,6 +228,24 @@ function leanings(reader) { return { favorites: fids("allthing.favorites"), less: fids("allthing.show-less"), followed }; } +// The inert bar follows AppsWindow::is_verified and ClientShareBar's time +// guard too: a cached legacy or inconsistent total must not decorate it. +function verifiedAppCount(window) { + if (!window || window.verification_version !== 2) return false; + const now = Math.floor(Date.now() / 1000); + const whole = value => Number.isSafeInteger(value) && value >= 0; + const unknown = window.unknown === undefined ? 0 : window.unknown; + const none = window.none === undefined ? 0 : window.none; + const apps = window.apps === undefined ? [] : window.apps; + return whole(window.to) && window.to >= 1609459200 && window.to % HOUR === 0 && + whole(window.counted) && window.counted >= window.to && window.counted <= now && + window.to <= now && now - window.to <= 2 * HOUR && whole(window.active) && window.active <= 4294967295 && + whole(unknown) && unknown <= window.active && whole(none) && none <= window.active && + Array.isArray(apps) && apps.every((app, i, all) => + app && typeof app === "object" && whole(app.app) && app.app > 0 && whole(app.active) && app.active <= window.active && + !all.slice(0, i).some(prior => prior.app === app.app)); +} + // Markup. function el(tag, attrs, ...children) { @@ -344,7 +362,7 @@ function shareBar(payload, notes, range) { const unknownPercent = split.pop(); split.forEach((p, i) => (shares[i].percent = p)); const unknownLabel = unknown > 0 ? `${unknown} not known${pending > 0 ? " yet" : ""}` : null; - const counted = Boolean(payload.apps_24h); + const counted = verifiedAppCount(payload.apps_24h); const period = PERIODS[range]; const pct = (p) => (p === 0 ? "<1%" : p + "%"); // The feed with the reader's sort and range kept, and the app filter set (clients.rs, `client_href`). From bd8f2a96e923e921151c2115584a980bc87fc5b6 Mon Sep 17 00:00:00 2001 From: ael-dev3 <183124839+ael-dev3@users.noreply.github.com> Date: Tue, 27 Oct 2026 14:48:09 +0100 Subject: [PATCH 4/4] fix(ui): preserve phone search and verify activity fixtures --- README.md | 8 +-- e2e/follows-facts.fixture.spec.ts | 114 ++++++++++++++++++++---------- e2e/presence.fixture.spec.ts | 25 +++++-- e2e/split-parts.fixture.spec.ts | 11 +++ e2e/tap-bands.fixture.spec.ts | 2 +- style/allthing.css | 69 ++++++++++++++++-- 6 files changed, 176 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 0fb4ea89..178e16dd 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Allthing is an independent Farcaster client, written in Rust, that shows trendin

    A profile: bio, website, followers and following, join date, NS vote and wallet activity by chain - The daily active accounts page, counted as SnapMeter counts them + The daily active accounts page

    ## Features @@ -37,11 +37,11 @@ Allthing is an independent Farcaster client, written in Rust, that shows trendin - **Posts and threads.** A post with its parent, quoted casts (inline up to three levels) and threaded replies that collapse and expand. - **Honest engagement.** Likes, recasts and replies, the only counts Farcaster has. Hovering or tapping a count lists who liked, recasted or replied. - **Hover cards.** A name or picture opens a profile card (bio, followers and following, join date, NS vote, wallet activity by chain), and names inside those lists open nested cards of their own. -- **Profiles.** Bio, website, followers and following pages (`/u//followers`, `/following`), the [NS vote](docs/VOTE.md) and wallet activity by chain for verified addresses. Favorite and Show less, kept in your browser, raise or lower an account's casts in your feeds without hiding anything. +- **Profiles.** Bio, website, followers and following pages (`/u//followers`, `/following`), the [NS vote](docs/VOTE.md) and wallet activity by chain for verified addresses. Favorite and Show less are private preferences kept in your browser. Favorite raises an account's casts in Hot and Top; Show less omits about 90% of their posts from Home, Following and channel feeds and ranks the rest much lower in Hot and Top. New stays chronological, direct profiles remain available, and pressing Show less again undoes it. - **Media in place.** Images open in an accessible viewer; Farcaster's HLS videos play inline (through a small HLS player written in Rust where the browser has none), muted until you turn sound on. Media loads straight from its hosts. - **Pepe emoji and reactions.** `:pepe_name:` shows one of 219 Pepe emoji, mostly animated. Pepe reactions (one-emoji replies) gather under casts as chips. - **Provenance.** Each cast names the app that registered its signing key ([docs/PROVENANCE.md](docs/PROVENANCE.md)). -- **Daily active accounts.** [/dau](https://allthing.social/dau) counts Farcaster's active accounts from the network's event stream, exactly as [SnapMeter](https://github.com/ael-dev3/SnapMeter) does. +- **Daily active accounts.** [/dau](https://allthing.social/dau) counts unique Farcaster accounts from checked Hypersnap events, using [SnapMeter](https://github.com/ael-dev3/SnapMeter)'s activity-type definitions. It shows each shard's checkpoints and gaps; partial totals are lower bounds and complete windows count each account once. It checks this provider's events, not the whole network independently. - **Allthing's poll.** One question on the front page: would you like an Allthing mobile client? - **Less noise, judged by content.** Spam caught by a few fixed content rules is left out by the Worker. Casts that embed or link to mini apps or frames, which Allthing doesn't run, are hidden too unless you turn on Show mini app casts in Settings; then each links to open in Farcaster. There are no automatic account-based exclusions; signed-in readers can choose their own private Mute and Block preferences. @@ -59,7 +59,7 @@ Allthing is an independent Farcaster client, written in Rust, that shows trendin ### Fast and frugal - **One API call per page**, cached at the edge, so the whole site runs on the Cloudflare Workers Free plan. -- **Early paint.** For anonymous readers, a small JavaScript renderer draws the front page's first cards from its existing response while the Rust app downloads. Signed-in Home waits for private history membership and account filters to avoid flashing previously read or muted/blocked posts. It stays in JavaScript because it beat the Rust prototype in the [paired cold-load comparison](docs/PERFORMANCE.md#measured-exception-the-javascript-first-card-renderer). +- **Early paint.** For anonymous readers without Show less choices, a small JavaScript renderer draws the front page's first cards from its existing response while the Rust app downloads. Feeds with Show less choices wait for the Rust filter, so omitted posts do not flash first. Signed-in Home waits for private history membership and account filters to avoid flashing previously read or muted/blocked posts. The early renderer stays in JavaScript because it beat the Rust prototype in the [paired cold-load comparison](docs/PERFORMANCE.md#measured-exception-the-javascript-first-card-renderer). - **Split WebAssembly.** Pages, the composer, writing, sign-in and the DAU dashboard are code of their own, fetched when used, so every visit downloads less. - **Instant navigation.** Posts open from the feed's own data, cards are prefetched within a capped budget, and Back returns to the same place. CI holds smoothness to budgets ([docs/PERFORMANCE.md](docs/PERFORMANCE.md)). - **Accessible.** Keyboard throughout, alt text, focus kept in dialogs, reduced motion respected, and it fits a phone. diff --git a/e2e/follows-facts.fixture.spec.ts b/e2e/follows-facts.fixture.spec.ts index a1db120c..0495a067 100644 --- a/e2e/follows-facts.fixture.spec.ts +++ b/e2e/follows-facts.fixture.spec.ts @@ -8,10 +8,10 @@ import { expect, test } from "@playwright/test"; /* * What a Followers or Following page will read of the active-accounts * cron's D1 for its people (no page asks yet): the apps each signed with, - * from the keys the cron learned (`signer_apps`), and whether each was - * active in the past week, from the union of its days the cron keeps once - * a day (`w` in `activity`), kept as the days are. This runs the - * Worker's own statements, read from crates/allthing-worker/src/index.rs, + * from the full keys the verified cron learned (`verified_signer_apps`), + * and whether each was active in the past week, from the union of its days + * (`v2w` in `activity`). Legacy rows remain outside this verified epoch. + * This runs the Worker's own statements, read from index.rs, * on a local D1 made from the migrations, as wrangler runs D1. */ @@ -29,9 +29,15 @@ function statement(name: string): string { return found[1].replace(/\\\n\s*/g, "").replace(/\\"/g, '"'); } -/** `sql` with its parameters (?1, ?2) as literals, as D1 binds them. */ -const bind = (sql: string, ...values: (string | bigint)[]) => - values.reduce((bound, value, i) => bound.replaceAll(`?${i + 1}`, typeof value === "bigint" ? String(value) : `'${value.replace(/'/g, "''")}'`), sql); +/** `sql` with all numbered parameters as literals, including a SQL NULL. */ +function bind(sql: string, ...values: (string | bigint | null)[]): string { + return sql.replace(/\?(\d+)/g, (_, parameter: string) => { + const index = Number(parameter) - 1; + if (index < 0 || index >= values.length) throw new Error(`unbound SQL parameter ?${parameter}`); + const value = values[index]; + return value === null ? "NULL" : typeof value === "bigint" ? String(value) : `'${value.replace(/'/g, "''")}'`; + }); +} let state = ""; @@ -46,13 +52,14 @@ function d1(sql: string): Record[] { return results[results.length - 1].results; } -/** A key as the cron keeps it: its account's fid in the high 32 bits, the key's first 4 bytes in the low. */ -const pair = (fid: number, head: number) => (BigInt(fid) << 32n) | BigInt(head); +/** Every FID and public-key byte, in the verified counter's canonical format. */ +const identity = (fid: number, head: number, tail = 0) => + `${String(fid).padStart(10, "0")}:0x${head.toString(16).padStart(8, "0")}${tail.toString(16).padStart(56, "0")}`; test.beforeAll(() => { test.setTimeout(120_000); state = mkdtempSync(path.join(tmpdir(), "allthing-follows-facts-")); - for (const migration of ["0003_activity.sql", "0004_signer_apps.sql"]) { + for (const migration of ["0001_index.sql", "0003_activity.sql", "0004_signer_apps.sql", "0011_verified_activity.sql"]) { execFileSync("npx", ["wrangler", "d1", "execute", "allthing-index", "--local", "--persist-to", state, "--file", path.join(ROOT, "migrations", migration)], { cwd: ROOT, stdio: "ignore", @@ -64,46 +71,77 @@ test.afterAll(() => { if (state) rmSync(state, { recursive: true, force: true }); }); -test("each account's apps come from its keys' range, by fid, never as a key", () => { - const last = 2 ** 31 - 1; - const keys: [bigint, number][] = [ - // Two keys from the Farcaster app and one from Allthing. - [pair(1, 0x0a0b0c0d), 9152], - [pair(1, 0x00000001), 9152], - [pair(1, 0xffffffff), 3356318], - // A key no app requested, its account's first. - [pair(2, 0), 0], - // The next account's first key isn't the one before's. - [pair(3, 0), 309857], - // The highest fid a key is kept for. - [pair(last, 5), 9152], +test("each account's apps come from its full signer keys, by fid, never as a key", () => { + const aboveOldLimit = 2 ** 31; + const last = 2 ** 32 - 1; + const keys: [string, number][] = [ + // Full keys sharing the same first four bytes remain distinct; two + // belong to Farcaster and the third to Allthing for this account. + [identity(1, 0x0a0b0c0d, 1), 9152], + [identity(1, 0x0a0b0c0d, 2), 9152], + [identity(1, 0x0a0b0c0d, 3), 3356318], + // Explicitly verified as no requesting app. + [identity(2, 0, 1), 0], + // The same public key under another FID is a separate identity. + [identity(3, 0, 1), 309857], + [identity(aboveOldLimit, 5), 309857], + [identity(last, 5), 9152], ]; - d1(`INSERT INTO signer_apps (pair, app) VALUES ${keys.map(([p, app]) => `(${p}, ${app})`).join(", ")}`); - const rows = d1(bind(statement("APPS_OF"), JSON.stringify([1, 2, 4, last]))); - // Fids and apps only, each pair of them once. + d1(bind(statement("STORE_KEYS"), JSON.stringify(keys))); + expect(d1("SELECT COUNT(*) AS total FROM verified_signer_apps")).toEqual([{ total: keys.length }]); + // Legacy prefix attribution is preserved but cannot enter verified facts. + const legacyPair = (1n << 32n) | 0x0a0b0c0dn; + d1(`INSERT INTO signer_apps (pair, app) VALUES (${legacyPair}, 9999)`); + const rows = d1(bind(statement("APPS_OF"), JSON.stringify([1, 2, 4, aboveOldLimit, last]))); + // Fids and apps only, each pair of them once; never expose signing keys. for (const row of rows) expect(Object.keys(row).sort()).toEqual(["app", "fid"]); const found = rows.map((row) => [Number(row.fid), Number(row.app)]).sort((a, b) => a[0] - b[0] || a[1] - b[1]); expect(found).toEqual([ [1, 9152], [1, 3356318], [2, 0], + [aboveOldLimit, 309857], [last, 9152], ]); expect(d1(bind(statement("APPS_OF"), "[]"))).toEqual([]); + expect(d1(`SELECT pair, app FROM signer_apps WHERE pair = ${legacyPair}`)).toEqual([{ pair: Number(legacyPair), app: 9999 }]); }); -test("a page reads the latest week kept, yesterday's or the day before's, in one row; old weeks go as old days do", () => { +test("a page reads the latest verified week; fenced expiry removes old verified sets and preserves legacy history", () => { const today = 20_386; - d1(`INSERT INTO activity (name, body) VALUES ('w${today - 3}', 'AQEB'), ('w${today - 2}', 'AQID'), ('d${today - 1}', 'AQ')`); - const latest = () => d1(bind(statement("LATEST_WEEK"), `w${today - 1}`, `w${today - 2}`)); - // Before the cron's daily pass: the day before's. - expect(latest()).toEqual([{ name: `w${today - 2}`, body: "AQID" }]); - // After it: yesterday's. - d1(`INSERT INTO activity (name, body) VALUES ('w${today - 1}', 'AgIC')`); - expect(latest()).toEqual([{ name: `w${today - 1}`, body: "AgIC" }]); - // Weeks are dropped with the days, before the 30 kept. - d1(`INSERT INTO activity (name, body) VALUES ('w${today - 40}', 'AQ'), ('d${today - 40}', 'AQ'), ('h489000', 'AQ')`); - d1(bind(statement("DROP_SETS"), BigInt(today * 24 - 48), BigInt(today - 30))); + const hourCutoff = today * 24 - 49; + const dayCutoff = today - 30; + d1(`INSERT INTO activity (name, body) VALUES ('v2w${today - 3}', 'AQEB'), ('v2w${today - 2}', 'AQID'), ('v2d${today - 1}', 'AQ'), ('w${today - 1}', 'AQ')`); + const latest = () => d1(bind(statement("LATEST_WEEK"), `v2w${today - 1}`, `v2w${today - 2}`)); + // Before the daily pass, the verified day-before row wins even though a + // newer legacy week exists. Afterwards the verified yesterday row wins. + expect(latest()).toEqual([{ name: `v2w${today - 2}`, body: "AQID" }]); + d1(`INSERT INTO activity (name, body) VALUES ('v2w${today - 1}', 'AgIC')`); + expect(latest()).toEqual([{ name: `v2w${today - 1}`, body: "AgIC" }]); + d1(`INSERT INTO activity (name, body) VALUES + ('v2w${today - 40}', 'AQ'), ('v2d${today - 40}', 'AQ'), + ('v2h${hourCutoff - 1}', 'AQ'), ('v2k${hourCutoff - 1}', '[]'), + ('v2w${dayCutoff}', 'AQ'), ('v2d${dayCutoff}', 'AQ'), + ('v2h${hourCutoff}', 'AQ'), ('v2k${hourCutoff}', '[]'), + ('w${today - 40}', 'AQ'), ('d${today - 40}', 'AQ'), ('h489000', 'AQ'), ('k489000', 'AQ')`); + const winning = "follows-facts-winning-fence"; + const losing = "follows-facts-losing-fence"; + expect(d1(bind(statement("CLAIM_ACTIVITY"), null, winning))).toEqual([{ value: winning }]); + const checkpoint = JSON.stringify({ verification_version: 2, revision: 1, cursors: [], reached: [], starts: [], blocks: [], days: [] }); + d1(bind(statement("SAVE_ACTIVITY_STATE"), checkpoint, winning)); + const before = d1("SELECT name, body FROM activity ORDER BY name"); + // A stale run cannot acquire the fence or expire any rows. + expect(d1(bind(statement("CLAIM_ACTIVITY"), null, losing))).toEqual([]); + d1(bind(statement("DROP_SETS"), BigInt(hourCutoff), BigInt(dayCutoff), losing)); + expect(d1("SELECT name, body FROM activity ORDER BY name")).toEqual(before); + expect(d1("SELECT value FROM state WHERE key = 'activity_v2_fence'")).toEqual([{ value: winning }]); + // The winning run expires only older verified sets, keeping both exact + // cutoff rows and every legacy row. + d1(bind(statement("DROP_SETS"), BigInt(hourCutoff), BigInt(dayCutoff), winning)); const names = d1("SELECT name FROM activity ORDER BY name").map((row) => row.name); - expect(names).toEqual([`d${today - 1}`, `w${today - 3}`, `w${today - 2}`, `w${today - 1}`]); + expect(names).toEqual([ + `v2d${today - 1}`, `v2d${dayCutoff}`, `v2w${today - 3}`, `v2w${today - 2}`, `v2w${today - 1}`, `v2w${dayCutoff}`, + `v2h${hourCutoff}`, `v2k${hourCutoff}`, `w${today - 1}`, `w${today - 40}`, `d${today - 40}`, "h489000", "k489000", + ].sort()); + expect(latest()).toEqual([{ name: `v2w${today - 1}`, body: "AgIC" }]); }); diff --git a/e2e/presence.fixture.spec.ts b/e2e/presence.fixture.spec.ts index 1e4c9544..d5091837 100644 --- a/e2e/presence.fixture.spec.ts +++ b/e2e/presence.fixture.spec.ts @@ -209,9 +209,17 @@ test.describe("in front of the Worker", () => { await expect(chip).toHaveText("12", { useInnerText: true }); const box = (await chip.boundingBox())!; const [hx, , hw] = before.home!; - const [sx] = before.search!; expect(box.x).toBeGreaterThanOrEqual(hx + hw + 4); - expect(box.x + box.width + 8).toBeLessThanOrEqual(sx); + if (width <= 640) { + const [sx, sy, sw] = before.search!; + expect(sy).toBeGreaterThanOrEqual(box.y + box.height + 8); + expect(sx).toBeGreaterThanOrEqual(0); + expect(sx + sw).toBeLessThanOrEqual(width); + expect(sw).toBeGreaterThanOrEqual(width - 16); + } else { + const [sx] = before.search!; + expect(box.x + box.width + 8).toBeLessThanOrEqual(sx); + } expect(box.height).toBeLessThanOrEqual(before.header![3]); // 920 is the exact compact breakpoint; the next pixel is full size. const leftPadding = await chip.evaluate((el) => getComputedStyle(el).paddingLeft); @@ -476,6 +484,7 @@ test.describe("in front of the Worker", () => { chip.style.display = ""; return { chip: r("#app .presence-chip"), home: r("#app .main-nav a"), dau: r("#app .dau-nav"), search: r("#app .search-box"), + controls: r("#app .header-actions"), searchWith, searchWithout, scroll: document.documentElement.scrollWidth, }; }); @@ -484,8 +493,16 @@ test.describe("in front of the Worker", () => { expect(laid.chip.x, where).toBeGreaterThanOrEqual(laid.home.right + 4); expect(laid.dau.x, where).toBeGreaterThanOrEqual(laid.home.right + 4); expect(laid.dau.right, where).toBeLessThanOrEqual(laid.chip.x); - expect(laid.chip.right + 8, where).toBeLessThanOrEqual(laid.search.x); - expect(laid.search.width, where).toBeGreaterThanOrEqual(width === 320 ? 112 : 128); + if (width <= 640) { + // Phones reserve the whole second row for search. Navigation, + // count and account controls must remain separate above it. + expect(laid.chip.right + 4, where).toBeLessThanOrEqual(laid.controls.x); + expect(laid.search.y, where).toBeGreaterThanOrEqual(Math.max(laid.home.bottom, laid.dau.bottom, laid.chip.bottom, laid.controls.bottom) + 8); + expect(laid.search.width, where).toBeGreaterThanOrEqual(width - 16); + } else { + expect(laid.chip.right + 8, where).toBeLessThanOrEqual(laid.search.x); + expect(laid.search.width, where).toBeGreaterThanOrEqual(128); + } } await context.close(); } diff --git a/e2e/split-parts.fixture.spec.ts b/e2e/split-parts.fixture.spec.ts index 0b44b942..9f61cf48 100644 --- a/e2e/split-parts.fixture.spec.ts +++ b/e2e/split-parts.fixture.spec.ts @@ -19,6 +19,17 @@ const now = Math.floor(Date.now() / 1000); const today = Math.floor(now / DAY); const meta = { backend: "hypersnap", fetched_at: Date.now() }; const activity = { + verification_version: 2, + source: "https://haatz.quilibrium.com/v1/events", + verified_since: (today - 29) * DAY, + shards: [1, 2].map(shard => { + const block = 1_000_000 + shard; + return { + shard, since: (today - 29) * DAY, reached: now - 60, + block, block_at: now - 59, total_events: 100, read_events: 100, + cursor: block * 16_384 + 99, + }; + }), reached: now - 60, rolling_to: Math.floor((now - 60) / 3_600) * 3_600, rolling_24h: 6_174, previous_24h: 5_982, since: (today - 29) * DAY, days: Array.from({ length: 30 }, (_, i) => ({ day: today - 29 + i, active: 6_000 + i, complete: i < 29, actions: { cast: 100 } })), diff --git a/e2e/tap-bands.fixture.spec.ts b/e2e/tap-bands.fixture.spec.ts index 4b3b231d..27ee4ced 100644 --- a/e2e/tap-bands.fixture.spec.ts +++ b/e2e/tap-bands.fixture.spec.ts @@ -47,7 +47,7 @@ const apps = { }; const to = Math.floor(Date.now() / 3_600_000) * 3_600; const apps24h = { - to, counted: to + 120, active: 6174, unknown: 0, none: 14, + verification_version: 2, to, counted: Math.floor(Date.now() / 1000), active: 6174, unknown: 0, none: 14, apps: [{ app: 9152, active: 4812 }, { app: 309857, active: 1203 }, { app: 5555, active: 120 }, { app: 6666, active: 37 }, { app: 1234, active: 1 }], }; const address = "0x76d0e7a13248945ee9f808b4a472262b28778942"; diff --git a/style/allthing.css b/style/allthing.css index 81bcff08..b7a04a30 100644 --- a/style/allthing.css +++ b/style/allthing.css @@ -8189,8 +8189,9 @@ p.jumbo .emoji img { } } -/* Navigation and search share the second row where a single row would - squeeze the search box. The early header uses the same grid. */ +/* On tablets, navigation and search share a second row rather than + squeezing a single row. Phones below use a full-width search row. + The early header uses the same grids. */ @media (max-width: 800px) { .header-inner { display: grid; @@ -8231,17 +8232,50 @@ p.jumbo .emoji img { } .header-inner { - gap: 8px; - padding: 10px 14px 8px; + grid-template-columns: max-content minmax(0, 1fr) max-content; + gap: 8px 4px; + padding: 10px 8px 8px; } .wordmark { - font-size: 27px; + font-size: 24px; + } + + /* Keep navigation and the account controls together above a full-width + search field. Compact labels leave the 44 px account tap bands intact + at 360 px; the inert header reserves the same columns and count. */ + .header-nav, + .boot-nav { + grid-column: 2; + grid-row: 1; + padding-right: 58px; + min-width: 0; + } + + .main-nav, + .boot-nav { + gap: 4px; + font-size: 12px; } .main-nav a, .boot-nav span { - padding: 5px 8px; + padding: 5px 3px; + } + + .header-nav > .presence { + width: 52px; + } + + .header-actions, + .boot-actions, + .boot-readonly { + grid-column: 3; + } + + .search-box, + .boot-search { + grid-column: 1 / -1; } .readonly-pill, @@ -8359,6 +8393,29 @@ p.jumbo .emoji img { } } +/* Small phones keep the same two rows and account tap bands; only the + wordmark and passive reader count compact further. */ +@media (max-width: 360px) { + .wordmark { + font-size: 20px; + } + + .header-nav, + .boot-nav { + padding-right: 40px; + } + + .header-nav > .presence { + width: 40px; + } + + .popover-trigger.presence-chip { + padding: 0 2px; + gap: 2px; + font-size: 11px; + } +} + @media (prefers-reduced-motion: reduce) { *, *::before,