diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a95334f..f45aa009 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,6 +87,7 @@ jobs: node-version: 22 cache: npm - run: npm ci + - run: cargo xtask browser-mirror - run: npx playwright install --with-deps chromium - name: Wait for the intended deployment run: cargo xtask wait-for-deploy https://allthing.social "$(git rev-parse HEAD)" @@ -115,11 +116,15 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@v6 + - name: Install the pinned Rust toolchain + run: rustup toolchain install + - uses: Swatinem/rust-cache@v2 - uses: actions/setup-node@v6 with: node-version: 22 cache: npm - run: npm ci + - run: cargo xtask browser-mirror - run: npx playwright install --with-deps chromium - run: npx playwright test e2e/speed-report.spec.ts env: @@ -152,6 +157,7 @@ jobs: - run: npm ci - run: cargo xtask npm-audit - run: npm run test:tools + - run: cargo xtask browser-mirror - run: npx playwright install --with-deps chromium - run: bash scripts/build.sh - name: Test the local build without live data diff --git a/CHANGELOG.md b/CHANGELOG.md index c50e352c..f0912878 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ All notable changes to Allthing are recorded here. The format follows [Keep a Ch ### Fixed +- Let a cold location serve a validated shared Home copy inside the existing stale period while one background refresh runs. Preserve source age, time bounds and request limits; joined readers cannot duplicate the refresh. - Explain that complete 24-hour activity totals require a gap-free window instead of promising a first count from its start time alone. - Count active accounts from checked, consecutive Hypersnap events with canonical block times carried across pages, declared event totals and both shards' coverage. The new verification epoch keeps old rows without certifying their totals. Full account and 32-byte signing-key identities prevent prefix collisions; incomplete attribution stays unknown, and a fenced atomic save rejects stale concurrent runs. Action totals are labelled as merge events because the same message can appear on both shards. - Recover one confirmed missing entry-loader request without reloading or changing drafts and sessions; persistent failures show an accessible Reload button even if early Home data arrives later. Healthy startup adds no recovery traffic. @@ -34,6 +35,8 @@ All notable changes to Allthing are recorded here. The format follows [Keep a Ch ### Development +- Select Canonical’s Ubuntu archive before browser dependency installation on GitHub Linux runners, preserving all packages, test phases, performance limits and job caps. The mirror configuration rewrite is native Rust and has local format and safety checks. + - Make attachment handoff fixtures verify file-selection order under deliberately reversed upload completion, and wait for the current finite animations when a transition is canceled or replaced. - Pause the search fixture's clock before navigation so an old real-time timer cannot move it backward; preserve the exact timer-cancellation and 250 ms debounce checks. diff --git a/crates/allthing-data/src/lib.rs b/crates/allthing-data/src/lib.rs index ef118355..4bc88deb 100644 --- a/crates/allthing-data/src/lib.rs +++ b/crates/allthing-data/src/lib.rs @@ -163,19 +163,22 @@ pub struct Built { /// When its data was read (Unix milliseconds), for a stored copy built /// earlier; `None` means now. pub fetched_at: Option, + /// A validated stale shared Day Home copy. The edge may answer it now + /// and start one normal refresh, retaining its original source age. + pub revalidate_home: bool, } impl Built { pub fn json(value: &T, policy: CachePolicy) -> Result { serde_json::to_vec(value) - .map(|body| Built { body, status: 200, policy, fetched_at: None }) + .map(|body| Built { body, status: 200, policy, fetched_at: None, revalidate_home: false }) .map_err(|e| Failure::internal(&e.to_string())) } /// A cacheable "no such thing" answer, so bad links don't reach upstream. pub fn not_found(message: &str) -> Built { let body = serde_json::to_vec(&ApiError { error: message.to_owned(), retry_after: None }).unwrap_or_default(); - Built { body, status: 404, policy: allthing_core::api::MISS_CACHE, fetched_at: None } + Built { body, status: 404, policy: allthing_core::api::MISS_CACHE, fetched_at: None, revalidate_home: false } } } diff --git a/crates/allthing-data/src/pages.rs b/crates/allthing-data/src/pages.rs index 91f99c12..392b5021 100644 --- a/crates/allthing-data/src/pages.rs +++ b/crates/allthing-data/src/pages.rs @@ -330,11 +330,11 @@ const HOME_SNAPSHOT_MAX_BYTES: usize = 1024 * 1024; #[cfg(test)] mod home_snapshot_tests; -fn fresh_home_snapshot(stored: Snapshot, now_ms: f64) -> Option { +fn shared_home_snapshot(stored: Snapshot, now_ms: f64) -> Option { let age = now_ms - stored.built; if !stored.built.is_finite() || stored.built <= 0.0 - || !(0.0..f64::from(api::HOME_CACHE.fresh) * 1000.0).contains(&age) + || !(0.0..f64::from(api::HOME_CACHE.stale) * 1000.0).contains(&age) || stored.body.len() > HOME_SNAPSHOT_MAX_BYTES { return None; @@ -358,7 +358,13 @@ fn fresh_home_snapshot(stored: Snapshot, now_ms: f64) -> Option { payload.notes.keep_users(BTreeMap::new(), named); serde_json::to_vec(&payload).ok()? }; - Some(Built { body, status: 200, policy: api::HOME_CACHE, fetched_at: Some(stored.built) }) + Some(Built { + body, + status: 200, + policy: api::HOME_CACHE, + fetched_at: Some(stored.built), + revalidate_home: age >= f64::from(api::HOME_CACHE.fresh) * 1000.0, + }) } fn shared_home_may_replace(previous: Option<&[u8]>, now_ms: f64) -> bool { @@ -400,7 +406,7 @@ pub async fn home( && inv.remaining_subrequests() > 2 && shared_home_may_replace(previous.as_deref().map(Vec::as_slice), inv.now()) && let Some(copy) = stored(&inv, HOME_SNAPSHOT).await - && let Some(built) = fresh_home_snapshot(copy, inv.now()) + && let Some(built) = shared_home_snapshot(copy, inv.now()) { return Ok(built); } @@ -884,7 +890,13 @@ pub async fn latest(inv: Rc, previous: Option>>) -> Resul } else { serde_json::to_vec(&payload).map_err(|e| Failure::internal(&e.to_string()))? }; - return Ok(Built { body, status: 200, policy: api::LATEST_CACHE, fetched_at: Some(stored.built) }); + return Ok(Built { + body, + status: 200, + policy: api::LATEST_CACHE, + fetched_at: Some(stored.built), + revalidate_home: false, + }); } warm = serde_json::from_str(&stored.body).ok(); } @@ -922,7 +934,7 @@ pub async fn activity(inv: Rc) -> Result { }); }; let body = stored.body.into_bytes(); - Ok(Built { body, status: 200, policy: api::ACTIVITY_CACHE, fetched_at: Some(stored.built) }) + Ok(Built { body, status: 200, policy: api::ACTIVITY_CACHE, fetched_at: Some(stored.built), revalidate_home: false }) } /// The copy of `name` the cron stores, if there is an index and it has one. diff --git a/crates/allthing-data/src/pages/home_snapshot_tests.rs b/crates/allthing-data/src/pages/home_snapshot_tests.rs index 86cb55df..60212192 100644 --- a/crates/allthing-data/src/pages/home_snapshot_tests.rs +++ b/crates/allthing-data/src/pages/home_snapshot_tests.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later use std::cell::{Cell, RefCell}; +use futures::channel::oneshot; use futures::future::{FutureExt, LocalBoxFuture}; use serde_json::json; @@ -21,6 +22,7 @@ struct State { background: RefCell>>, now: Cell, read_delay: Cell, + source_gate: RefCell>>, } impl Index for State { @@ -85,6 +87,7 @@ impl Host for Source { } fn fetch(&self, request: HttpRequest) -> LocalBoxFuture<'static, Result> { self.requests.borrow_mut().push(request.url.clone()); + let gate = self.state.source_gate.borrow_mut().take(); let body = if request.url.contains("/feed/trending?") { json!({"casts": [{"hash": "0xfresh", "author": {"fid": 0, "username": "fresh"}, "text": if self.large { "x".repeat(HOME_SNAPSHOT_MAX_BYTES) } else { "fresh source".into() }, @@ -100,7 +103,13 @@ impl Host for Source { retry_after: None, body: (!self.fail).then(|| serde_json::to_vec(&body).unwrap()), }; - async move { Ok(response) }.boxed_local() + async move { + if let Some(gate) = gate { + let _ = gate.await; + } + Ok(response) + } + .boxed_local() } } @@ -163,6 +172,7 @@ fn cold_copy_uses_one_read_no_upstream_and_preserves_original_bytes_and_age() { let built = built.unwrap(); assert_eq!(built.body, copy.body.as_bytes()); assert_eq!((built.fetched_at, built.status, built.policy), (Some(copy.built), 200, api::HOME_CACHE)); + assert!(!built.revalidate_home); assert_eq!(inv.remaining_subrequests(), 2, "the edge-cache put still fits"); } assert_eq!(*host.state.reads.borrow(), [HOME_SNAPSHOT, HOME_SNAPSHOT]); @@ -205,7 +215,7 @@ fn invalid_or_expired_shared_copies_fall_back_to_normal_builds() { copy }; let cases = [ - snapshot(60_000.0, &[]), + snapshot(600_000.0, &[]), snapshot(-1.0, &[]), Snapshot { built: f64::NAN, ..valid.clone() }, Snapshot { built: f64::INFINITY, ..valid.clone() }, @@ -229,7 +239,7 @@ fn invalid_or_expired_shared_copies_fall_back_to_normal_builds() { assert_eq!(writes[0].body.as_bytes(), built.body); assert_eq!(writes[0].built, payload.meta.fetched_at as f64); } - assert!(fresh_home_snapshot(snapshot(59_999.0, &[]), NOW).is_some()); + assert!(!shared_home_snapshot(snapshot(59_999.0, &[]), NOW).unwrap().revalidate_home); } #[test] @@ -244,7 +254,7 @@ fn shared_read_failure_does_not_fail_a_valid_home_build() { #[test] fn failed_write_or_failed_full_build_keeps_the_previous_shared_copy() { - let old = snapshot(60_000.0, &[]); + let old = snapshot(600_000.0, &[]); let host = source(Some(old.clone())); host.state.write_fails.set(true); assert!(build(&host, 45, 0, TrendingWindow::Day, None).1.is_ok()); @@ -296,7 +306,7 @@ fn one_through_four_slots_preserve_normal_source_availability_without_an_optiona #[test] fn pending_background_persistence_cannot_delay_the_response_or_reset_its_age() { - let old = snapshot(60_000.0, &[]); + let old = snapshot(600_000.0, &[]); let host = source(Some(old.clone())); host.state.write_waits.set(true); let (inv, result) = build(&host, 8, 0, TrendingWindow::Day, None); @@ -372,12 +382,18 @@ fn refreshed_edges_other_windows_offsets_and_following_never_read_the_shared_hom (0, TrendingWindow::All), ] { let host = source(None); - assert!(build(&host, 45, offset, window, None).1.is_ok()); + assert!(!build(&host, 45, offset, window, None).1.unwrap().revalidate_home); assert!(!host.state.reads.borrow().contains(&HOME_SNAPSHOT)); assert!(host.state.writes.borrow().is_empty()); } let host = source(None); - following(Rc::new(Invocation::new(host.clone(), 45)), 123, None, None).now_or_never().unwrap().unwrap(); + assert!( + !following(Rc::new(Invocation::new(host.clone(), 45)), 123, None, None) + .now_or_never() + .unwrap() + .unwrap() + .revalidate_home + ); assert!(!host.state.reads.borrow().contains(&HOME_SNAPSHOT)); assert!(host.state.writes.borrow().is_empty()); } @@ -390,7 +406,7 @@ fn without_storage_or_with_an_oversized_build_the_response_still_serves() { assert!(result.is_ok()); assert!(host.state.reads.borrow().is_empty() && host.state.writes.borrow().is_empty()); assert_eq!(inv.remaining_subrequests(), 1); - let old = snapshot(60_000.0, &[]); + let old = snapshot(600_000.0, &[]); let mut host = source(Some(old.clone())); Rc::get_mut(&mut host).unwrap().large = true; let (_, result) = build(&host, 45, 0, TrendingWindow::Day, None); @@ -398,3 +414,83 @@ fn without_storage_or_with_an_oversized_build_the_response_still_serves() { assert!(host.state.writes.borrow().is_empty()); assert_eq!(*host.state.copy.borrow(), Some(old)); } + +#[test] +fn stale_shared_copies_request_one_refresh_only_within_the_existing_stale_interval() { + for age in [60_000.0, 120_000.0, 599_999.0] { + let copy = snapshot(age, &[(NOW / 1000.0) as i64 - 10]); + let host = source(Some(copy.clone())); + let (inv, result) = build(&host, 3, 0, TrendingWindow::Day, None); + let built = result.unwrap(); + assert!(built.revalidate_home); + assert_eq!(built.body, copy.body.as_bytes()); + assert_eq!(built.fetched_at, Some(copy.built)); + assert_eq!(inv.remaining_subrequests(), 2); + assert!(host.requests.borrow().is_empty() && host.state.background.borrow().is_empty()); + } + let host = source(Some(snapshot(599_999.0, &[]))); + host.state.read_delay.set(2.0); + let (_, result) = build(&host, 45, 0, TrendingWindow::Day, None); + let built = result.unwrap(); + assert!(!built.revalidate_home, "a copy expiring during the D1 read cannot seed an edge"); + assert_eq!(host.requests.borrow().len(), 1); + assert_eq!(built.fetched_at, Some(NOW + 2.0)); +} + +#[test] +fn stale_response_precedes_delayed_real_source_revalidation_without_resetting_its_age() { + let old = snapshot(120_000.0, &[(NOW / 1000.0) as i64 - 10]); + let host = source(Some(old.clone())); + let (release, gate) = oneshot::channel(); + *host.state.source_gate.borrow_mut() = Some(gate); + let (inv, result) = build(&host, 45, 0, TrendingWindow::Day, None); + let seeded = result.unwrap(); + assert!(seeded.revalidate_home); + assert_eq!(seeded.fetched_at, Some(old.built)); + assert_eq!(seeded.body, old.body.as_bytes()); + assert!(host.requests.borrow().is_empty(), "the cold response does not await the source"); + assert_eq!(inv.remaining_subrequests(), 44); + assert!(inv.take_subrequests(1), "charge the initial edge seed put to the same invocation"); + + let previous = Rc::new(seeded.body.clone()); + let mut refresh = Box::pin(home(inv.clone(), 0, TrendingWindow::Day, Some(previous))); + assert!(refresh.as_mut().now_or_never().is_none(), "the actual source request remains held"); + assert_eq!(host.requests.borrow().len(), 1); + assert_eq!(host.state.reads.borrow().iter().filter(|&&name| name == HOME_SNAPSHOT).count(), 1); + assert!(host.state.writes.borrow().is_empty()); + assert_eq!(*host.state.copy.borrow(), Some(old.clone())); + + host.state.now.set(NOW + 20_000.0); + release.send(()).unwrap(); + let built = refresh.now_or_never().unwrap().unwrap(); + assert!(!built.revalidate_home); + let payload: HomePayload = serde_json::from_slice(&built.body).unwrap(); + assert_eq!(payload.pool[0].hash, "0xfresh"); + assert_eq!(built.fetched_at, Some(NOW + 20_000.0)); + assert_eq!(payload.meta.fetched_at as f64, NOW + 20_000.0); + assert_eq!(seeded.fetched_at, Some(old.built), "refreshing cannot make the returned source younger"); + assert_eq!(inv.remaining_subrequests(), 37); + assert!(inv.take_subrequests(1), "the refresh's edge put also fits the original budget"); + finish_save(&host); + assert_eq!(host.state.writes.borrow().len(), 1); + assert_eq!(host.state.copy.borrow().as_ref().unwrap().body.as_bytes(), built.body); +} + +#[test] +fn failed_delayed_revalidation_cannot_replace_the_shared_source() { + let old = snapshot(120_000.0, &[(NOW / 1000.0) as i64 - 10]); + let mut host = source(Some(old.clone())); + Rc::get_mut(&mut host).unwrap().fail = true; + let (release, gate) = oneshot::channel(); + *host.state.source_gate.borrow_mut() = Some(gate); + let (inv, result) = build(&host, 45, 0, TrendingWindow::Day, None); + let seeded = result.unwrap(); + assert!(seeded.revalidate_home); + let mut refresh = Box::pin(home(inv, 0, TrendingWindow::Day, Some(Rc::new(seeded.body)))); + assert!(refresh.as_mut().now_or_never().is_none()); + release.send(()).unwrap(); + assert!(refresh.now_or_never().unwrap().is_err()); + assert_eq!(*host.state.copy.borrow(), Some(old)); + assert!(host.state.writes.borrow().is_empty()); + assert_eq!(host.state.reads.borrow().iter().filter(|&&name| name == HOME_SNAPSHOT).count(), 1); +} diff --git a/crates/allthing-worker/src/cache.rs b/crates/allthing-worker/src/cache.rs index e011ee22..5a21af1e 100644 --- a/crates/allthing-worker/src/cache.rs +++ b/crates/allthing-worker/src/cache.rs @@ -18,7 +18,7 @@ //! The Cache API is per data center and is a no-op on `*.workers.dev`, so it //! only takes effect on the custom domain. -use std::cell::RefCell; +use std::cell::{Cell, RefCell}; use std::collections::HashMap; use std::rc::Rc; use std::time::Duration; @@ -35,6 +35,16 @@ pub struct Entry { /// Unix milliseconds. pub fetched_at: f64, pub policy: CachePolicy, + /// Only a newly seeded stale shared Home copy requests another build. + /// Neither this signal nor its consume-once claim is persisted. + revalidate_home: bool, + refresh_claimed: Cell, +} + +impl Entry { + pub(crate) fn new(body: Rc>, status: u16, fetched_at: f64, policy: CachePolicy) -> Self { + Self { body, status, fetched_at, policy, revalidate_home: false, refresh_claimed: Cell::new(false) } + } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -105,13 +115,20 @@ async fn lookup(inv: &Invocation, key: &str) -> Option { let fetched_at = header_f64(headers, "x-allthing-fetched")?; let status = header_f64(headers, "x-allthing-status")? as u16; let body = response.bytes().await.ok()?; - Some(Entry { body: Rc::new(body), status, fetched_at, policy: entry_policy }) + Some(Entry::new(Rc::new(body), status, fetched_at, entry_policy)) } async fn store(inv: &Invocation, key: &str, built: Built) -> Rc { // A stored copy is as old as when it was built, so it is refreshed soon. let fetched_at = built.fetched_at.unwrap_or_else(|| inv.now()); - let entry = Rc::new(Entry { body: Rc::new(built.body), status: built.status, fetched_at, policy: built.policy }); + let entry = Rc::new(Entry { + body: Rc::new(built.body), + status: built.status, + fetched_at, + policy: built.policy, + revalidate_home: built.revalidate_home, + refresh_claimed: Cell::new(false), + }); put(inv, key, &entry, built.policy.keep).await; entry } @@ -153,13 +170,22 @@ pub async fn expire(inv: &Invocation, key: &str, ours: impl Fn(&Entry) -> bool) /// Start the build of `key` in this request. It also runs under /// `waitUntil`, so it completes and is cached even if the client goes away. -fn start_build(inv: &Rc, key: &str, build: &Builder, previous: Option>>) -> SharedBuild { +fn start_build( + inv: &Rc, + key: &str, + build: &Builder, + previous: Option>>, + allow_shared_home: bool, +) -> SharedBuild { let slot: Slot = Rc::new(RefCell::new(None)); let running = Running { started: inv.now(), slot: slot.clone() }; INFLIGHT.with(|m| m.borrow_mut().insert(key.to_owned(), running)); let (inv2, key2, build2) = (inv.clone(), key.to_owned(), build.clone()); let future = async move { let outcome = match build2(inv2.clone(), previous).await { + Ok(built) if built.revalidate_home && (!allow_shared_home || !valid_home_refresh(&built, inv2.now())) => { + Err(Failure::internal("invalid shared Home refresh")) + } Ok(built) => Ok(store(&inv2, &key2, built).await), Err(failure) => Err(failure), }; @@ -182,6 +208,50 @@ fn start_build(inv: &Rc, key: &str, build: &Builder, previous: Optio future } +fn valid_home_refresh(built: &Built, now: f64) -> bool { + built.fetched_at.is_some_and(|at| valid_home_age(built.status, built.policy, at, now)) +} + +fn valid_home_age(status: u16, policy: CachePolicy, at: f64, now: f64) -> bool { + let age = now - at; + status == 200 + && policy == allthing_core::api::HOME_CACHE + && at.is_finite() + && at > 0.0 + && (f64::from(policy.fresh) * 1000.0..f64::from(policy.stale) * 1000.0).contains(&age) +} + +/// One old-slot result may reach several waiting requests. Consume its +/// claim before checking the current build, including if that build fails +/// before a later waiter resumes. The request claiming the refresh owns +/// its I/O; no request awaits another request's platform future. +fn start_home_refresh(inv: &Rc, key: &str, build: &Builder, entry: &Entry) -> bool { + if !entry.revalidate_home || entry.refresh_claimed.replace(true) { + return false; + } + let running = INFLIGHT.with(|m| m.borrow().get(key).is_some_and(|r| inv.now() - r.started < ABANDONED_MS)); + if running { + return false; + } + drop(start_build(inv, key, build, Some(entry.body.clone()), false)); + true +} + +fn serve_shared_home( + inv: &Rc, + key: &str, + build: &Builder, + entry: Rc, +) -> Result<(Rc, Served), Failure> { + start_home_refresh(inv, key, build, &entry); + // A slow seed put or late old-slot waiter can cross the existing stale + // deadline. Keep refreshing, but do not serve that now-expired copy. + if !valid_home_age(entry.status, entry.policy, entry.fetched_at, inv.now()) { + return Err(still_building()); + } + Ok((entry, Served::Revalidating)) +} + fn still_building() -> Failure { Failure { status: 503, message: "Still gathering this page; try again shortly".into(), retry_after: Some(5) } } @@ -221,7 +291,7 @@ pub async fn serve(inv: &Rc, key: &str, build: Builder) -> Result<(R Freshness::Stale => { // Answer now; refresh in the background unless already running. if running.is_none() { - drop(start_build(inv, key, &build, previous)); + drop(start_build(inv, key, &build, previous, false)); } return Ok((entry.clone(), Served::Revalidating)); } @@ -230,9 +300,10 @@ pub async fn serve(inv: &Rc, key: &str, build: Builder) -> Result<(R } let outcome = match running { Some(slot) => join_build(slot).await, - None => start_build(inv, key, &build, previous).await, + None => start_build(inv, key, &build, previous, true).await, }; match outcome { + Ok(entry) if entry.revalidate_home => serve_shared_home(inv, key, &build, entry), Ok(entry) => Ok((entry, Served::Built)), Err(failure) => match cached { Some(entry) if entry.status == 200 => Ok((entry, Served::StaleIfError)), @@ -240,3 +311,147 @@ pub async fn serve(inv: &Rc, key: &str, build: Builder) -> Result<(R }, } } + +#[cfg(test)] +mod tests { + use super::*; + use allthing_data::{FetchError, Host, HttpRequest, HttpResponse}; + use futures::channel::oneshot; + + const NOW: f64 = 1_800_000_000_000.0; + + #[derive(Default)] + struct Runtime { + now: Cell, + background: RefCell>>, + } + + impl Host for Runtime { + fn now(&self) -> f64 { + self.now.get() + } + fn fetch(&self, _: HttpRequest) -> LocalBoxFuture<'static, Result> { + panic!("the test builder must not reach the network") + } + fn spawn(&self, future: LocalBoxFuture<'static, ()>) { + self.background.borrow_mut().push(future); + } + } + + fn runtime() -> (Rc, Rc) { + let runtime = Rc::new(Runtime { now: Cell::new(NOW), ..Runtime::default() }); + let invocation = Rc::new(Invocation::new(runtime.clone(), 45)); + (runtime, invocation) + } + + fn seed(age: f64) -> Rc { + Rc::new(Entry { + body: Rc::new(br#"{"source":"shared"}"#.to_vec()), + status: 200, + fetched_at: NOW - age, + policy: allthing_core::api::HOME_CACHE, + revalidate_home: true, + refresh_claimed: Cell::new(false), + }) + } + + fn finish_background(runtime: &Runtime) { + for task in runtime.background.take() { + assert!(task.now_or_never().is_some()); + } + } + + #[test] + fn shared_home_refresh_claim_survives_fast_failure_and_a_late_old_slot_waiter() { + let (runtime, inv) = runtime(); + let entry = seed(120_000.0); + let old_slot = Rc::new(RefCell::new(Some(Ok(entry.clone())))); + let calls = Rc::new(Cell::new(0)); + let observed = calls.clone(); + let build: Builder = Rc::new(move |_, previous| { + observed.set(observed.get() + 1); + assert_eq!(previous.as_deref().map(Vec::as_slice), Some(br#"{"source":"shared"}"#.as_slice())); + async { Err(Failure::internal("failed refresh")) }.boxed_local() + }); + let (_, served) = serve_shared_home(&inv, "fast-failure", &build, entry.clone()).unwrap(); + assert_eq!(served, Served::Revalidating); + finish_background(&runtime); + assert_eq!(calls.get(), 1); + assert!(!INFLIGHT.with(|m| m.borrow().contains_key("fast-failure"))); + + let late = join_build(old_slot).now_or_never().unwrap().unwrap(); + assert!(Rc::ptr_eq(&entry, &late)); + assert_eq!(serve_shared_home(&inv, "fast-failure", &build, late).unwrap().1, Served::Revalidating); + assert!(runtime.background.borrow().is_empty()); + assert_eq!(calls.get(), 1, "a completed failed refresh must not reopen the old result's claim"); + } + + #[test] + fn another_inflight_refresh_consumes_the_old_result_claim_without_duplicating_work() { + let (runtime, inv) = runtime(); + let (release, gate) = oneshot::channel(); + let gate = Rc::new(RefCell::new(Some(gate))); + let calls = Rc::new(Cell::new(0)); + let observed = calls.clone(); + let build: Builder = Rc::new(move |_, _| { + observed.set(observed.get() + 1); + let gate = gate.borrow_mut().take().unwrap(); + async move { + let _ = gate.await; + Err(Failure::internal("failed existing refresh")) + } + .boxed_local() + }); + let mut running = Box::pin(start_build(&inv, "already-running", &build, None, true)); + assert!(running.as_mut().now_or_never().is_none()); + let old = seed(120_000.0); + assert!(!start_home_refresh(&inv, "already-running", &build, &old)); + release.send(()).unwrap(); + assert!(running.now_or_never().unwrap().is_err()); + finish_background(&runtime); + assert!(!start_home_refresh(&inv, "already-running", &build, &old)); + assert_eq!(calls.get(), 1); + } + + #[test] + fn a_home_seed_expiring_during_store_or_before_a_late_waiter_is_not_served() { + let (runtime, inv) = runtime(); + let old = seed(599_999.0); + let build: Builder = Rc::new(|_, _| async { Err(Failure::internal("held source")) }.boxed_local()); + assert_eq!(serve_shared_home(&inv, "expires", &build, old.clone()).unwrap().1, Served::Revalidating); + finish_background(&runtime); + runtime.now.set(NOW + 1.0); + let failure = serve_shared_home(&inv, "expires", &build, old.clone()).err().unwrap(); + assert_eq!((failure.status, failure.retry_after), (503, Some(5))); + assert_eq!(old.fetched_at, NOW - 599_999.0); + assert!(runtime.background.borrow().is_empty()); + let delayed_put = seed(599_999.0); + let failure = serve_shared_home(&inv, "delayed-put", &build, delayed_put).err().unwrap(); + assert_eq!((failure.status, failure.retry_after), (503, Some(5))); + finish_background(&runtime); + } + + #[test] + fn shared_home_revalidation_is_one_hop_and_never_implicit_for_other_snapshots() { + let (runtime, inv) = runtime(); + let build: Builder = Rc::new(|_, _| { + let mut built = Built::json(&"shared", allthing_core::api::HOME_CACHE).unwrap(); + built.fetched_at = Some(NOW - 120_000.0); + built.revalidate_home = true; + async move { Ok(built) }.boxed_local() + }); + assert!(start_build(&inv, "one-hop", &build, None, false).now_or_never().unwrap().is_err()); + finish_background(&runtime); + for policy in [allthing_core::api::LATEST_CACHE, allthing_core::api::ACTIVITY_CACHE] { + let built = Built::json(&"snapshot", policy).unwrap(); + assert!(!built.revalidate_home); + assert!(!valid_home_refresh(&built, NOW)); + } + for (age, valid) in [(59_999.0, false), (60_000.0, true), (599_999.0, true), (600_000.0, false)] { + assert_eq!(valid_home_age(200, allthing_core::api::HOME_CACHE, NOW - age, NOW), valid); + } + for at in [f64::NAN, f64::INFINITY, 0.0, NOW + 1.0] { + assert!(!valid_home_age(200, allthing_core::api::HOME_CACHE, at, NOW)); + } + } +} diff --git a/crates/allthing-worker/src/presence.rs b/crates/allthing-worker/src/presence.rs index b7f1ca1d..e50c32e1 100644 --- a/crates/allthing-worker/src/presence.rs +++ b/crates/allthing-worker/src/presence.rs @@ -129,7 +129,7 @@ pub async fn handle(mut req: Request, env: &Env, ctx: Context, now_ms: f64) -> R fids.iter().filter_map(|fid| users.iter().find(|user| user.fid == *fid).map(normalize::user_summary)).collect(); let body = serde_json::to_vec(&Answer { online, readers, next: next_interval(online) })?; let policy = CachePolicy { fresh: 30, stale: 30, keep: 60, browser: 0 }; - let entry = Entry { body: Rc::new(body), status: 200, fetched_at: now_ms, policy }; + let entry = Entry::new(Rc::new(body), 200, now_ms, policy); cache::put(&inv, &key, &entry, policy.keep).await; answer_with(entry.body.to_vec()) } diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 4b852c8d..e7122ac3 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -42,7 +42,7 @@ Allthing is a Farcaster client for reading: it shows Farcaster, and the Worker p | `allthing-data` | native + wasm32 | The page builders that assemble each payload, and the Hypersnap and RPC clients behind guards. The runtime is abstracted as a `Host` (clock, HTTP with a timeout, background work), so the same code runs in the Worker and in the browser. | | `allthing-worker` | wasm32 (Worker) | The `/api/v1` handler: routing, the per-IP rate limit, the edge cache with coalescing, and a `Host` over Workers `fetch`, `AbortController`, `waitUntil` and D1. Its cron handler runs the indexer. `media.rs` keeps readers' [attachments](#attachments) in R2 and serves them. `auth.rs` is [sign-in](#sign-in) and `app_key.rs` [endorses readers' keys](#liking-and-recasting); between them they hold the Worker's cryptography (secp256k1 signer recovery and signing, Keccak-256, HMAC-SHA256) and the QR encoder, none of which ships to browsers. A reader's own key is their browser's (WebCrypto's Ed25519). | | `allthing-web` | wasm32 (browser) | The Leptos app (client-side rendering): routes, pages, components, the media viewer, the HLS player, prefetching and scroll restoration, and liking and recasting with the reader's key (`keys.rs`, `components/writer.rs`). With the `static-host` feature it builds payloads in-process through a browser `Host`. | -| `xtask` | native | Build orchestration, Brotli-11 compression, esbuild CLI/SDK-version checks, metadata and credit audits, attestation key generation, optimizer discovery, fixture phases, npm audit, isolated SpacetimeDB validation and deployment waiting. `dist` builds `dist/`; `pages` builds `dist-pages/`. | +| `xtask` | native | Build orchestration, Brotli-11 compression, esbuild CLI/SDK-version checks, metadata and credit audits, Ubuntu runner mirror selection, attestation key generation, optimizer discovery, fixture phases, npm audit, isolated SpacetimeDB validation and deployment waiting. `dist` builds `dist/`; `pages` builds `dist-pages/`. | `scripts/build.sh` only bootstraps rustup and the pinned toolchain on images without Rust, then runs `cargo xtask build`. The native build selects the locked tools, builds the Worker, strips its relocations and builds the browser assets. `cargo xtask dist` runs `cargo build`, wasm-split for the interactive app, `wasm-bindgen --target web` and `wasm-opt -Oz`. It writes fingerprinted Wasm, glue, loaders, CSS and HTML, and compresses the optional `.wasm.br` copies in Rust at quality 11, window 24. The shared SpacetimeDB adapter is bundled through the installed esbuild CLI after checking its SDK version. WebAssembly release builds use `opt-level = "z"`, LTO, one codegen unit and `panic = "abort"`. @@ -153,7 +153,7 @@ Payloads are cached with the Cache API under `https:///__cache/api/v1/...` Unknown users and casts are cached as 404s for 5 minutes. -A missing or expired first-page 24-hour Home edge copy can start from the public D1 snapshot `home-day-v1` (`pages::home`). A bounded metadata-only scan identifies retained previous data at least 600 seconds old: an edge `miss` can mean an absent entry or a foreground rebuild of an expired one still kept for a day. The normal 60–600-second stale-while-revalidate background refresh still builds from upstream. Malformed, future or oversized previous metadata cannot prove expiry and takes the normal build. The shared copy is used only while younger than the existing 60-second Home freshness limit, with a finite, positive, nonfuture build time matching its payload metadata, no stale marker or indexed-range metadata, and at most 1 MiB of JSON. Its pool is filtered against the current clock's day and future bound. An unchanged pool keeps the exact bytes; a filtered pool is serialized again, without changing or saving the source copy. Either keeps its original age when admitted to the edge cache. A missing, invalid or expired shared copy takes the normal full build. Other ranges, later offsets and private Following never read or save it. +A missing or expired first-page 24-hour Home edge copy can start from the public D1 snapshot `home-day-v1` (`pages::home`). A bounded metadata-only scan identifies retained previous data at least 600 seconds old: an edge `miss` can mean an absent entry or a foreground rebuild of an expired one still kept for a day. The normal 60–600-second stale-while-revalidate background refresh still builds from upstream. Malformed, future or oversized previous metadata cannot prove expiry and takes the normal build. The shared copy is used only inside the existing 600-second Home stale period, with a finite, positive, nonfuture build time matching its payload metadata, no stale marker or indexed-range metadata, and at most 1 MiB of JSON. Its pool is filtered against the current clock's day and future bound. An unchanged pool keeps the exact bytes; a filtered pool is serialized again, without changing or saving the source copy. Either keeps its original age when admitted to the edge cache. A copy younger than 60 seconds needs no immediate refresh. A copy aged 60–600 seconds answers as revalidating and starts one coalesced background build from upstream; a transient claim shared by joined requests prevents a second refresh even if the first one fails quickly. This signal is specific to shared Home copies and is never stored in cache headers. The refresh carries that body forward and cannot start another shared-copy refresh. If a slow cache write or delayed joined request crosses the 600-second boundary, the refresh continues but that response returns the existing retryable 503 instead of serving the now-expired seed. A missing, invalid or expired shared copy takes the normal full build. Other ranges, later offsets and private Following never read or save it. Successful normal first-page Day builds reserve one charged D1 write before annotations and save best-effort under their own invocation's `waitUntil`, so neither a slow nor a failed optional save delays the response. The source timestamp is retained through background scheduling. A conditional upsert keeps a concurrent older build from replacing a newer present snapshot; an impossible future-built stored row can be repaired, judged against SQLite's UTC clock at statement execution rather than before the D1 await. This uses the existing migrated table, never runs schema creation on the request path, and adds no cron or warm-up. Scarce budgets skip optional lookups or saves before taking the normal source's only slot. diff --git a/docs/BUDGET.md b/docs/BUDGET.md index d12dadb1..ceeeb403 100644 --- a/docs/BUDGET.md +++ b/docs/BUDGET.md @@ -104,7 +104,7 @@ Public payloads add compact identity provenance before excerpts and actor trunca Library content hydration is one public API request per page. `/api/v1/library?fid=&kind=likes|recasts&cursor=` makes one Hypersnap reaction-page read and, for nonempty pages, one bulk cast read (up to 25 hashes). `/api/v1/library/casts?hashes=` hydrates at most 25 hashes with one bulk read. Existing bounded annotations share the invocation's 45-subrequest ceiling, including cache calls; cursors and ordered canonical hash batches form public cache keys. These endpoints contain public casts/reactions only, never private membership or bookmark data. A user can explicitly load more pages beyond the in-memory Back cache without truncating server history. -**A cold data center's default Home:** a missing first-page 24-hour edge copy, or a retained one at least 600 seconds old, can read the public `home-day-v1` snapshot (one charged D1 read). The previous body's bounded metadata scan distinguishes expired retained entries from normal 60–600-second stale-while-revalidate refreshes, which still build normally without the shared read. A shared copy younger than 60 seconds answers without upstream reads; its unchanged JSON bytes and original build time are preserved, and casts outside the current day or in the future are removed before serving. Bodies are capped at 1 MiB before snapshot parsing. Missing, malformed, future-built or expired shared copies use the normal full build. Untrustworthy previous metadata also takes the normal build. Other windows, offsets and private Following never use it. Edge hits and normal refresh reads are unchanged. +**A cold data center's default Home:** a missing first-page 24-hour edge copy, or a retained one at least 600 seconds old, can read the public `home-day-v1` snapshot (one charged D1 read). The previous body's bounded metadata scan distinguishes expired retained entries from normal 60–600-second stale-while-revalidate refreshes, which still build normally without the shared read. A shared copy younger than 60 seconds answers without upstream reads. A validated copy aged 60–600 seconds answers immediately as revalidating, then starts one coalesced upstream build in the request that claims its refresh. Joined requests share a consume-once refresh claim; the signal is not persisted. Its unchanged JSON bytes and original build time are preserved, and casts outside the current day or in the future are removed before serving. Bodies are capped at 1 MiB before snapshot parsing. Missing, malformed, future-built or expired shared copies use the normal full build. Untrustworthy previous metadata also takes the normal build. Other windows, offsets and private Following never use it. Edge hits and normal refresh reads are unchanged. Each successful normal first-page Day build reserves at most one D1 write before annotations, alongside their existing reserve for the Cache API put, and saves best-effort under its own `waitUntil`; the response does not wait for this optional write. Saving it is one conditional statement against the migrated table, with no schema or cron call. A slower older build cannot overwrite a newer present one, and an invalid future-built row can be repaired using the database's execution-time clock. Scarce budgets skip optional work instead of taking the normal source's only slot. Failed or oversized saves keep the previous copy and do not fail the response. @@ -153,6 +153,7 @@ Each invocation gets a budget of 45 subrequests, under the cap of 50. Every `fet |---|---| | Any cache hit | 1 (cache read) | | home, first-page Day, absent or expired edge with a fresh shared copy | cache read 1 + shared public snapshot 1 (D1) + cache write 1; no upstream calls (3 total) | +| home, first-page Day, absent or expired edge with a shared copy aged 60–600 seconds | the same 3 calls before answering, plus one normal background Home build and its cache write; the refresh normally skips a second shared snapshot read. If the carried source expires before background work starts, the normal bounded shared lookup may run again. Seeding adds one cache write compared with a full foreground build. The refresh uses the claiming request’s existing 45-call guard. When seeding and refresh share an invocation, the extra cache write leaves fewer optional annotations; failure keeps the original dated copy | | home, full build | cache read 1 + shared Home snapshot ≤1 (D1, only first-page Day with an absent or provably expired edge) + shared Home save ≤1 (D1, only first-page Day) + stored newest casts 1 (D1, first non-indexed page only) + trending casts 1 (for 12 hours, Hypersnap's 24-hour list, cut to the window: the same one call) + link pages ≤6 + quoted casts ≤3 (one bulk read per level of quotes, up to three levels) + emoji reactions ≤1 (a D1 read, when a listed cast has replies, taken before origins) + origins ≤36 (12 authors × at most 3, reduced to the remaining budget) + app names ≤1 + the apps' active accounts 1 (the follower's stored payload, a D1 read alongside the trending read, on the first page only) + the poll's counts 1 (a D1 read alongside the trending read, on the first page only) + cache write 1; all within the same 45 | | home, 30 days and All time | as home, with the trending read replaced by the index's ranking row 1 (a D1 query, counted as a subrequest to be safe) and the page's casts in bulk 1 | | the indexer (cron) | D1 3 (the state with the rankings' metadata, in one batch; the casts, ranking and state written in one batch; the schema once per isolate) + one Hypersnap page 1; first, [scheduled casts](#11-scheduled-casts) due: D1 1 (takes what is due, and reads it) + submits ≤5 + D1 1 (what became of them, only when some were due) (≤ 11) | diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index a52788ba..22b686e5 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -23,6 +23,8 @@ The post-opening check also retains `post-open-diagnostics.json` for each attemp CI runs functional fixture files with two workers, then runs `smoothness.fixture.spec.ts` alone with one worker (`cargo xtask browser-fixtures`). Both phases run even if the first fails, and either failure fails the existing browser job. Their HTML reports and test results are kept separately, including from the later static checks; the production performance job, test retries, timing assertions and 45-minute job cap are unchanged. +Before installing Playwright’s dependencies, GitHub Linux runners use `cargo xtask browser-mirror` to replace active Azure Ubuntu archive URLs with Canonical’s HTTPS archive in the supported Ubuntu APT mirror configuration. The native tool preserves the remaining configuration, including package suites, components, security URLs and signing settings; the existing `playwright install --with-deps chromium` still installs all dependencies and fonts. A recorded runner fetched 25.9 MB through Azure’s mirror in 10 minutes 11 seconds, versus one second for the same package set on an earlier run, leaving too little time for the complete suite. Mirror selection addresses that acquisition path without increasing the job cap or changing test limits. Future mirror throughput remains external, so a complete normal CI run is still required. + How the app answers a reader's taps, clicks and scrolls: `e2e/smoothness.fixture.spec.ts`, which CI's browser job runs on every pull request and push to `main`, on the local build with the Worker's API, the node and the images as stand-ins (the API answers after 60 to 180 ms). A signed-in reader with an approved key, on a phone (390 × 844, touch, the CPU slowed 2×) and on a computer (1280 × 900, a mouse): - taps the sort tabs: new, top, hot; diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 4232f174..066c2d85 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -33,11 +33,12 @@ fn main() -> ExitCode { Some("check-credits") if args.len() == 1 => tooling::metadata::check_emoji_sources(&root()), Some("find-wasm-opt") => tooling::find_wasm_opt::run(&args[1..]), Some("browser-fixtures") => tooling::commands::browser_fixtures(&root(), &args[1..]), + Some("browser-mirror") => tooling::browser_mirror::run(&args[1..]), Some("module-check") => tooling::commands::module_check(&root(), &args[1..]), Some("npm-audit") => tooling::commands::npm_audit(&root(), &args[1..]), Some("attest-key") => tooling::attest::run(&root(), &args[1..]), Some("wait-for-deploy") => tooling::deploy::run(&root(), &args[1..]), - _ => Err("usage: cargo xtask dist | pages [base] | build | check | check-repo | check-credits | find-wasm-opt [cache...] | browser-fixtures | module-check | npm-audit | attest-key | wait-for-deploy | strip-worker [path]".into()), + _ => Err("usage: cargo xtask dist | pages [base] | build | check | check-repo | check-credits | find-wasm-opt [cache...] | browser-fixtures | browser-mirror | module-check | npm-audit | attest-key | wait-for-deploy | strip-worker [path]".into()), }; match result { Ok(()) => ExitCode::SUCCESS, diff --git a/xtask/src/tooling/browser_mirror.rs b/xtask/src/tooling/browser_mirror.rs new file mode 100644 index 00000000..39f249f7 --- /dev/null +++ b/xtask/src/tooling/browser_mirror.rs @@ -0,0 +1,318 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +//! Select Canonical's Ubuntu archive before Playwright installs its unchanged +//! dependencies. Only the GitHub Linux runner's regular Ubuntu configs apply. + +use std::fs::{self, File}; +use std::io::{Read, Write}; +use std::process::{Command, Stdio}; + +use crate::Result; + +const MAX_BYTES: usize = 64 * 1024; +const ARCHIVE: &str = "https://archive.ubuntu.com/ubuntu"; + +#[derive(Clone, Copy)] +enum Format { + Mirrors, + Sources, + Deb822, +} + +const CONFIGS: [(&str, Format); 3] = [ + ("/etc/apt/apt-mirrors.txt", Format::Mirrors), + ("/etc/apt/sources.list", Format::Sources), + ("/etc/apt/sources.list.d/ubuntu.sources", Format::Deb822), +]; + +fn guard(args: &[String], os: &str, github_actions: Option<&str>) -> Result<()> { + if !args.is_empty() { + return Err("usage: cargo xtask browser-mirror".into()); + } + if os != "linux" || github_actions != Some("true") { + return Err("browser-mirror requires Linux GitHub Actions; no APT files were accessed".into()); + } + Ok(()) +} + +fn azure(token: &str) -> bool { + matches!( + token, + "http://azure.archive.ubuntu.com/ubuntu" + | "http://azure.archive.ubuntu.com/ubuntu/" + | "https://azure.archive.ubuntu.com/ubuntu" + | "https://azure.archive.ubuntu.com/ubuntu/" + ) +} + +fn security_suite(suite: &str) -> bool { + suite.trim_end_matches('/').ends_with("-security") +} + +/// Active ASCII-whitespace tokens, keeping their original byte ranges. A +/// comment starts at its own token; a URL suffix or fragment is never exact. +fn tokens(line: &str, offset: usize) -> Vec<(usize, usize)> { + let bytes = line.as_bytes(); + let mut found = Vec::new(); + let mut at = 0; + while at < bytes.len() { + if bytes[at].is_ascii_whitespace() { + at += 1; + continue; + } + let start = at; + while at < bytes.len() && !bytes[at].is_ascii_whitespace() { + at += 1; + } + if bytes[start] == b'#' { + break; + } + found.push((offset + start, offset + at)); + } + found +} + +fn line_ranges(input: &str, format: Format, offset: usize, line: &str, ranges: &mut Vec<(usize, usize)>) { + let parts = tokens(line, offset); + let Some(&(first, end)) = parts.first() else { return }; + let uri = match format { + Format::Mirrors => { + let security = parts.iter().skip(1).any(|&(a, b)| { + input[a..b].split_once(':').is_some_and(|(name, suites)| { + name.eq_ignore_ascii_case("suite") && suites.split(',').any(security_suite) + }) + }); + if security { None } else { Some((first, end)) } + } + Format::Sources if matches!(&input[first..end], "deb" | "deb-src") => { + let mut at = 1; + if parts.get(at).is_some_and(|&(a, _)| input[a..].starts_with('[')) { + while parts.get(at).is_some_and(|&(_, b)| !input[..b].ends_with(']')) { + at += 1; + } + at += 1; + } + // Security repositories retain their existing mirror, including + // a source list whose URI happens to be Azure's archive. + let security = parts.get(at + 1).is_some_and(|&(a, b)| security_suite(&input[a..b])); + if security { None } else { parts.get(at).copied() } + } + _ => None, + }; + if let Some((a, b)) = uri + && azure(&input[a..b]) + { + ranges.push((a, b)); + } +} + +fn deb822_ranges(input: &str, offset: usize, stanza: &str, ranges: &mut Vec<(usize, usize)>) { + let mut field = ""; + let mut security = false; + let mut disabled = false; + let mut uris = Vec::new(); + let mut at = offset; + for line in stanza.split_inclusive('\n') { + let leading = line.bytes().next().is_some_and(|b| b.is_ascii_whitespace()); + if line.trim_start().starts_with('#') { + at += line.len(); + continue; + } + let (start, value) = if leading { + (at, line) + } else if let Some((name, value)) = line.split_once(':') { + field = name; + (at + name.len() + 1, value) + } else { + field = ""; + at += line.len(); + continue; + }; + let parts = tokens(value, start); + if field.eq_ignore_ascii_case("Suites") { + security |= parts.iter().any(|&(a, b)| security_suite(&input[a..b])); + } else if field.eq_ignore_ascii_case("Enabled") { + disabled |= parts.iter().any(|&(a, b)| input[a..b].eq_ignore_ascii_case("no")); + } else if field.eq_ignore_ascii_case("URIs") { + uris.extend(parts.into_iter().filter(|&(a, b)| azure(&input[a..b]))); + } + at += line.len(); + } + if !security && !disabled { + ranges.extend(uris); + } +} + +fn rewrite(input: &str, format: Format) -> Result<(String, usize)> { + if input.len() > MAX_BYTES || input.contains('\0') { + return Err("APT configuration exceeds the bound or contains a NUL byte".into()); + } + let mut ranges = Vec::new(); + let mut at = 0; + let mut stanza = 0; + for line in input.split_inclusive('\n') { + if matches!(format, Format::Deb822) { + if line.trim().is_empty() { + deb822_ranges(input, stanza, &input[stanza..at], &mut ranges); + stanza = at + line.len(); + } + } else { + line_ranges(input, format, at, line, &mut ranges); + } + at += line.len(); + } + if matches!(format, Format::Deb822) { + deb822_ranges(input, stanza, &input[stanza..], &mut ranges); + } + let mut output = String::with_capacity(input.len()); + let mut previous = 0; + for &(a, b) in &ranges { + output.push_str(&input[previous..a]); + output.push_str(ARCHIVE); + previous = b; + } + output.push_str(&input[previous..]); + Ok((output, ranges.len())) +} + +fn read_regular(path: &str) -> Result> { + let metadata = match fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(_) => return Err(format!("could not inspect Ubuntu APT configuration {path}")), + }; + if !metadata.file_type().is_file() { + return Err(format!("Ubuntu APT configuration is not a regular file: {path}")); + } + let file = File::open(path).map_err(|_| format!("could not read Ubuntu APT configuration {path}"))?; + let mut bytes = Vec::new(); + file.take((MAX_BYTES + 1) as u64) + .read_to_end(&mut bytes) + .map_err(|_| format!("could not read Ubuntu APT configuration {path}"))?; + if bytes.len() > MAX_BYTES { + return Err(format!("Ubuntu APT configuration exceeds the size bound: {path}")); + } + String::from_utf8(bytes).map(Some).map_err(|_| format!("Ubuntu APT configuration is not UTF-8: {path}")) +} + +fn write_regular(path: &str, body: &str) -> Result<()> { + let mut child = Command::new("/usr/bin/sudo") + .args(["-n", "/usr/bin/tee", "--", path]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(|_| format!("could not write Ubuntu APT configuration {path}"))?; + let written = child + .stdin + .take() + .ok_or_else(|| format!("could not open APT writer for {path}")) + .and_then(|mut stdin| stdin.write_all(body.as_bytes()).map_err(|_| format!("could not write {path}"))); + let status = child.wait().map_err(|_| format!("could not finish APT writer for {path}"))?; + written?; + if status.success() { Ok(()) } else { Err(format!("Ubuntu APT writer failed for {path}")) } +} + +pub fn run(args: &[String]) -> Result<()> { + guard(args, std::env::consts::OS, std::env::var("GITHUB_ACTIONS").ok().as_deref())?; + // Validate every bounded, fixed-path input before any privileged write. + let mut changes = Vec::new(); + for (path, format) in CONFIGS { + if let Some(input) = read_regular(path)? { + let (body, count) = rewrite(&input, format)?; + if count > 0 { + changes.push((path, body, count)); + } + } + } + for (path, body, count) in &changes { + write_regular(path, body)?; + println!("browser-mirror: updated {count} Azure Ubuntu URI token(s) in {path}"); + } + if changes.is_empty() { + println!("browser-mirror: no Azure Ubuntu archive URI changes needed"); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mirror_list_preserves_comments_priority_whitespace_and_other_uris() { + let input = "# http://azure.archive.ubuntu.com/ubuntu stays a comment\r\n\t http://azure.archive.ubuntu.com/ubuntu/\tpriority:1 # keep\r\nhttps://security.ubuntu.com/ubuntu priority:2\nhttps://archive.ubuntu.com/ubuntu\nhttp://azure.archive.ubuntu.com/ubuntu\tsuite:noble,noble-security/\n"; + let expected = "# http://azure.archive.ubuntu.com/ubuntu stays a comment\r\n\t https://archive.ubuntu.com/ubuntu\tpriority:1 # keep\r\nhttps://security.ubuntu.com/ubuntu priority:2\nhttps://archive.ubuntu.com/ubuntu\nhttp://azure.archive.ubuntu.com/ubuntu\tsuite:noble,noble-security/\n"; + assert_eq!(rewrite(input, Format::Mirrors).unwrap(), (expected.into(), 1)); + assert_eq!(rewrite(expected, Format::Mirrors).unwrap(), (expected.into(), 0)); + } + + #[test] + fn exact_uri_tokens_only_accept_all_four_azure_spellings() { + for uri in [ + "http://azure.archive.ubuntu.com/ubuntu", + "http://azure.archive.ubuntu.com/ubuntu/", + "https://azure.archive.ubuntu.com/ubuntu", + "https://azure.archive.ubuntu.com/ubuntu/", + ] { + assert_eq!(rewrite(uri, Format::Mirrors).unwrap(), (ARCHIVE.into(), 1)); + } + for suffix in ["//", "/pool", "?query=1", "#fragment", ".evil", ":80", "\u{a0}priority:1"] { + let input = format!("http://azure.archive.ubuntu.com/ubuntu{suffix}"); + assert_eq!(rewrite(&input, Format::Mirrors).unwrap(), (input, 0)); + } + for input in ["http://azure.archive.ubuntu.com.evil/ubuntu", "https://user@azure.archive.ubuntu.com/ubuntu"] { + assert_eq!(rewrite(input, Format::Mirrors).unwrap(), (input.into(), 0)); + } + } + + #[test] + fn legacy_sources_preserve_signing_suites_components_and_security() { + let input = "deb [arch=amd64 signed-by=/usr/share/keyrings/ubuntu.gpg] http://azure.archive.ubuntu.com/ubuntu noble main universe # Ubuntu\ndeb-src https://azure.archive.ubuntu.com/ubuntu/ noble-updates main\ndeb http://azure.archive.ubuntu.com/ubuntu noble-security main\ndeb http://azure.archive.ubuntu.com/ubuntu noble-security/\n# deb http://azure.archive.ubuntu.com/ubuntu noble main\n"; + let expected = "deb [arch=amd64 signed-by=/usr/share/keyrings/ubuntu.gpg] https://archive.ubuntu.com/ubuntu noble main universe # Ubuntu\ndeb-src https://archive.ubuntu.com/ubuntu noble-updates main\ndeb http://azure.archive.ubuntu.com/ubuntu noble-security main\ndeb http://azure.archive.ubuntu.com/ubuntu noble-security/\n# deb http://azure.archive.ubuntu.com/ubuntu noble main\n"; + assert_eq!(rewrite(input, Format::Sources).unwrap(), (expected.into(), 2)); + } + + #[test] + fn deb822_rewrites_only_uri_fields_and_continuations_of_regular_enabled_stanzas() { + let input = "Types: deb deb-src\r\nURIs: http://azure.archive.ubuntu.com/ubuntu/\r\n https://azure.archive.ubuntu.com/ubuntu https://other.example/ubuntu\r\nSuites: noble noble-updates noble-backports\r\nComponents: main restricted universe multiverse\r\nSigned-By: http://azure.archive.ubuntu.com/ubuntu\r\n\r\nTypes: deb\nURIs: http://azure.archive.ubuntu.com/ubuntu\nSuites: noble\n noble-security/\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg\n\nTypes: deb\nURIs: http://azure.archive.ubuntu.com/ubuntu\nSuites: noble\nEnabled: no\n"; + let expected = input + .replacen("URIs: http://azure.archive.ubuntu.com/ubuntu/", &format!("URIs: {ARCHIVE}"), 1) + .replacen(" https://azure.archive.ubuntu.com/ubuntu ", &format!(" {ARCHIVE} "), 1); + assert_eq!(rewrite(input, Format::Deb822).unwrap(), (expected.clone(), 2)); + assert_eq!(rewrite(&expected, Format::Deb822).unwrap(), (expected, 0)); + } + + #[test] + fn configuration_bounds_and_nul_are_rejected_without_rewriting() { + assert!(rewrite(&"x".repeat(MAX_BYTES + 1), Format::Mirrors).is_err()); + assert!(rewrite("http://azure.archive.ubuntu.com/ubuntu\0", Format::Mirrors).is_err()); + let exact = "#".repeat(MAX_BYTES); + assert_eq!(rewrite(&exact, Format::Deb822).unwrap(), (exact, 0)); + } + + #[test] + fn platform_and_environment_guard_is_exact_and_precedes_all_io() { + assert!(guard(&[], "linux", Some("true")).is_ok()); + for (os, github) in [ + ("macos", Some("true")), + ("windows", Some("true")), + ("linux", None), + ("linux", Some("TRUE")), + ("linux", Some("1")), + ] { + assert!(guard(&[], os, github).unwrap_err().contains("no APT files were accessed")); + } + assert_eq!( + CONFIGS.map(|(path, _)| path), + ["/etc/apt/apt-mirrors.txt", "/etc/apt/sources.list", "/etc/apt/sources.list.d/ubuntu.sources"] + ); + } + + #[test] + fn command_rejects_all_arguments_before_platform_or_file_access() { + for args in [vec!["--help".into()], vec!["/tmp/config".into()], vec!["--path".into(), "/etc/apt/other".into()]] + { + assert_eq!(run(&args).unwrap_err(), "usage: cargo xtask browser-mirror"); + } + } +} diff --git a/xtask/src/tooling/mod.rs b/xtask/src/tooling/mod.rs index d5c6b17c..8df41ab6 100644 --- a/xtask/src/tooling/mod.rs +++ b/xtask/src/tooling/mod.rs @@ -2,6 +2,7 @@ //! Native repository tooling. Browser-only integrations stay at their platform boundary. pub mod attest; +pub mod browser_mirror; pub mod build; pub mod commands; pub mod deploy;