diff --git a/Cargo.lock b/Cargo.lock index 54d394296..4b3453699 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -684,20 +684,24 @@ dependencies = [ "flate2", "futures", "futures-util", + "getrandom 0.4.2", "http", "http-body", "hyper", "hyper-util", + "ipnet", "log", "mime", "redb", "reqwest", + "rustls", "serde", "serde_json", "simple_logger", "tempfile", "thiserror 2.0.20", "tokio", + "tokio-rustls", "toml", "tower", "tracing", diff --git a/Cargo.toml b/Cargo.toml index 659c493b6..4e56d6629 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,12 +62,14 @@ fern = "0.7" flate2 = { version = "=1.1.9", default-features = false, features = ["rust_backend"] } futures = { version = "0.3", features = ["std", "executor"] } futures-util = { version = "0.3", features = ["alloc", "io"] } +getrandom = "=0.4.2" handlebars = "6" http = "1" http-body = "1" http-body-util = "0.1" hyper = { version = "1", features = ["http1", "server"] } hyper-util = { version = "0.1", features = ["tokio"] } +ipnet = "=2.12.0" log = "0.4" miniz_oxide = { version = "=0.8.9", default-features = false, features = ["simd", "with-alloc"] } log-fastly = "=0.13.1" @@ -83,6 +85,7 @@ simd-adler32 = { version = "=0.3.9", default-features = false } # file (`/.spin/sqlite_key_value.db`) using the schema # vendored from spinframework/spin's `crates/key-value-spin/src/store.rs`. rusqlite = { version = "0.32", default-features = false, features = ["bundled"] } +rustls = { version = "=0.23.40", default-features = false, features = ["std", "tls12", "aws_lc_rs"] } ryu = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" @@ -102,6 +105,7 @@ tempfile = "3" toml_edit = "0.25" thiserror = "2" tokio = { version = "1", features = ["io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time"] } +tokio-rustls = { version = "=0.26.4", default-features = false, features = ["tls12", "aws_lc_rs"] } trybuild = "1" toml = { version = "1.1" } tower = { version = "0.5", features = ["util"] } diff --git a/crates/edgezero-adapter-axum/Cargo.toml b/crates/edgezero-adapter-axum/Cargo.toml index d4fe4084f..b732f7320 100644 --- a/crates/edgezero-adapter-axum/Cargo.toml +++ b/crates/edgezero-adapter-axum/Cargo.toml @@ -21,6 +21,8 @@ axum = [ "dep:http-body", "dep:hyper", "dep:hyper-util", + "dep:ipnet", + "dep:getrandom", "dep:mime", "dep:reqwest", "dep:redb", @@ -47,10 +49,12 @@ bytes = { workspace = true } ctor = { workspace = true, optional = true } futures = { workspace = true } futures-util = { workspace = true, optional = true } +getrandom = { workspace = true, optional = true } http = { workspace = true } http-body = { workspace = true, optional = true } hyper = { workspace = true, optional = true } hyper-util = { workspace = true, optional = true } +ipnet = { workspace = true, optional = true } log = { workspace = true } mime = { workspace = true, optional = true } redb = { workspace = true, optional = true } @@ -80,6 +84,8 @@ flate2 = { workspace = true } http-body = { workspace = true } hyper = { workspace = true } hyper-util = { workspace = true } +rustls = { workspace = true } serde = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true, features = ["io-util", "macros", "net", "rt", "rt-multi-thread"] } +tokio-rustls = { workspace = true } diff --git a/crates/edgezero-adapter-axum/src/dev_server.rs b/crates/edgezero-adapter-axum/src/dev_server.rs index 4fabce192..b6b61d653 100644 --- a/crates/edgezero-adapter-axum/src/dev_server.rs +++ b/crates/edgezero-adapter-axum/src/dev_server.rs @@ -15,13 +15,16 @@ use std::future::Future; #[cfg(test)] use std::iter; use std::net::{SocketAddr, TcpListener as StdTcpListener}; +use std::panic::catch_unwind; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::str::FromStr as _; use std::sync::Arc; +use crate::diagnostics::{NativeDiagnosticsHandle, NativeSession}; +use crate::proxy::TrustedProxyPolicy; pub use crate::run_options::AxumRunOptions; -use crate::run_options::{JsonBodyLimit, failure, shutdown_grace}; +use crate::run_options::{JsonBodyLimit, failure, native_settings, shutdown_grace}; use anyhow::Context as _; use edgezero_core::probe::LifecyclePhase; use std::time::Duration; @@ -45,7 +48,7 @@ use edgezero_core::secret_store::SecretHandle; use edgezero_core::store_registry::{ BoundSecretStore, ConfigRegistry, ConfigStoreBinding, KvRegistry, SecretRegistry, StoreRegistry, }; -use log::LevelFilter; +use log::{Level, LevelFilter}; use simple_logger::SimpleLogger; use std::collections::BTreeMap; @@ -136,6 +139,7 @@ struct Stores { pub struct AxumDevServer { config: AxumDevServerConfig, json_body_limit: JsonBodyLimit, + native: NativeHosting, router: RouterService, stores: Stores, } @@ -147,6 +151,7 @@ impl AxumDevServer { Self { config: AxumDevServerConfig::default(), json_body_limit: JsonBodyLimit::DEFAULT, + native: NativeHosting::default(), router, stores: Stores::default(), } @@ -161,16 +166,41 @@ impl AxumDevServer { config, stores, json_body_limit, + native, } = self; run_local( ListenerSource::Bind(config.addr), config.enable_ctrl_c, Duration::from_secs(10), future::pending::<()>(), - async move { prepare_transport(App::new(router), stores.into(), json_body_limit) }, + async move { prepare_transport(App::new(router), stores.into(), json_body_limit, native) }, ) } + /// Uses an explicit incoming-proxy policy, ignoring ambient proxy settings. + #[must_use] + #[inline] + pub fn with_trusted_proxy_policy(mut self, policy: TrustedProxyPolicy) -> Self { + self.native.policy = policy; + self + } + + /// Controls only default request records, not observers or counters. + #[must_use] + #[inline] + pub fn with_request_records(mut self, enabled: bool) -> Self { + self.native.request_records = enabled; + self + } + + /// Shares managed counters and the optional native observer with the runner. + #[must_use] + #[inline] + pub fn with_diagnostics(mut self, diagnostics: NativeDiagnosticsHandle) -> Self { + self.native.diagnostics = diagnostics; + self + } + #[cfg(test)] async fn run_with_listener(self, listener: TokioTcpListener) -> anyhow::Result<()> { let AxumDevServer { @@ -178,6 +208,7 @@ impl AxumDevServer { config, stores, json_body_limit, + native, } = self; serve_with_stores( App::new(router), @@ -185,6 +216,7 @@ impl AxumDevServer { config.enable_ctrl_c, stores, json_body_limit, + native, ) .await } @@ -195,6 +227,7 @@ impl AxumDevServer { Self { config, json_body_limit: JsonBodyLimit::DEFAULT, + native: NativeHosting::default(), router, stores: Stores::default(), } @@ -379,6 +412,7 @@ async fn serve_with_stores( enable_ctrl_c: bool, stores: Stores, json_body_limit: JsonBodyLimit, + native: NativeHosting, ) -> anyhow::Result<()> { struct ShutdownOnDrop(Option>); @@ -402,6 +436,7 @@ async fn serve_with_stores( enable_ctrl_c, stores, json_body_limit, + native, shutdown_receiver, ) }); @@ -417,6 +452,7 @@ fn serve_local( enable_ctrl_c: bool, stores: Stores, json_body_limit: JsonBodyLimit, + native: NativeHosting, shutdown_receiver: ShutdownReceiver<()>, ) -> anyhow::Result<()> { run_local( @@ -426,11 +462,34 @@ fn serve_local( async move { let _closed = shutdown_receiver.await; }, - async move { prepare_transport(app, stores.into(), json_body_limit) }, + async move { prepare_transport(app, stores.into(), json_body_limit, native) }, ) } -type PreparedApp = (App, PreparedStores, Arc, JsonBodyLimit); +#[derive(Clone, Debug)] +struct NativeHosting { + diagnostics: NativeDiagnosticsHandle, + policy: TrustedProxyPolicy, + request_records: bool, +} + +impl Default for NativeHosting { + fn default() -> Self { + Self { + diagnostics: NativeDiagnosticsHandle::new(), + policy: TrustedProxyPolicy::no_trust(), + request_records: true, + } + } +} + +type PreparedApp = ( + App, + PreparedStores, + Arc, + JsonBodyLimit, + NativeHosting, +); enum ListenerSource { Bind(SocketAddr), @@ -463,10 +522,11 @@ fn prepare_transport( app: App, stores: PreparedStores, json_body_limit: JsonBodyLimit, + native: NativeHosting, ) -> anyhow::Result { let transport = AxumOutboundClient::try_transport() .map_err(|_error| failure("transport", "HTTP client", "initialization failed"))?; - Ok((app, stores, transport, json_body_limit)) + Ok((app, stores, transport, json_body_limit, native)) } struct Signals { @@ -545,13 +605,39 @@ where let runtime = RuntimeBuilder::new_current_thread() .enable_all() .build() - .map_err(|_error| failure("runtime", "local runner", "creation failed"))?; + .map_err(|_error| { + log_native_lifecycle( + LifecyclePhase::Starting, + "runtime_creation_failed", + grace, + true, + ); + failure("runtime", "local runner", "creation failed") + })?; let local = LocalSet::new(); runtime.block_on(local.run_until(async move { - let mut signals = Signals::register(enable_signals)?; + let mut signals = Signals::register(enable_signals).inspect_err(|_error| { + log_native_lifecycle( + LifecyclePhase::Starting, + "signal_registration_failed", + grace, + true, + ); + })?; let (phase, reader) = watch::channel(LifecyclePhase::Starting); + log_native_lifecycle(LifecyclePhase::Starting, "starting", grace, false); let result = run_owned(listener, grace, stop, prepare, &mut signals, &phase, reader).await; phase.send_replace(LifecyclePhase::Stopped); + log_native_lifecycle( + LifecyclePhase::Stopped, + if result.is_ok() { + "stopped" + } else { + "runner_failed" + }, + grace, + result.is_err(), + ); result })) } @@ -580,17 +666,21 @@ where signal = next_signal(signals) => { signal?; phase.send_replace(LifecyclePhase::Draining); + log_native_lifecycle(LifecyclePhase::Draining, "signal_stop", grace, false); return Ok(()); } () = &mut stop => { phase.send_replace(LifecyclePhase::Draining); + log_native_lifecycle(LifecyclePhase::Draining, "stop_requested", grace, false); return Ok(()); } - prepared = &mut prepare => prepared?, + prepared = &mut prepare => prepared.inspect_err(|_error| log_native_lifecycle(LifecyclePhase::Starting, "startup_failed", grace, true))?, }; - let (app, stores, transport, json_body_limit) = prepared; - let mut service = AxumServiceState::with_transport(app, transport, json_body_limit) - .with_phase(reader.clone()); + let (app, stores, transport, json_body_limit, native) = prepared; + let session = NativeSession::new(native.diagnostics, native.request_records)?; + let mut service = + AxumServiceState::with_transport(app, transport, json_body_limit, session, native.policy) + .with_phase(reader.clone()); if let Some(registry) = stores.config { service = service.with_config_registry(registry); } @@ -603,12 +693,12 @@ where // Give stop observation another poll before bind/adoption and readiness. let listener = tokio::select! { biased; - signal = next_signal(signals) => { signal?; phase.send_replace(LifecyclePhase::Draining); return Ok(()); } - () = &mut stop => { phase.send_replace(LifecyclePhase::Draining); return Ok(()); } + signal = next_signal(signals) => { signal?; phase.send_replace(LifecyclePhase::Draining); log_native_lifecycle(LifecyclePhase::Draining, "signal_stop", grace, false); return Ok(()); } + () = &mut stop => { phase.send_replace(LifecyclePhase::Draining); log_native_lifecycle(LifecyclePhase::Draining, "stop_requested", grace, false); return Ok(()); } result = async { yield_now().await; adopt_listener(listener_source) } => result?, }; phase.send_replace(LifecyclePhase::Ready); - log::info!("[edgezero] native server ready"); + log_native_lifecycle(LifecyclePhase::Ready, "ready", grace, false); let mut tasks = JoinSet::new(); let mut outcome = Ok(()); loop { @@ -626,9 +716,12 @@ where match accepted { Ok((stream, remote_addr)) => { let responses = EgressConnection::default(); - let connection_service = service.for_connection(remote_addr, responses.clone()); + // Own the guard before submission, including unpolled cancellation. + let (activity, connection_guard) = service.connection_activity(); + let connection_service = service.for_connection(remote_addr, responses.clone()).with_activity(activity); let connection_phase = reader.clone(); tasks.spawn_local(async move { + let _connection_guard = connection_guard; if serve_http1_with_shutdown(stream, connection_service, responses, Some(connection_phase)).await.is_err() { log::debug!("axum HTTP/1 connection closed"); } @@ -643,6 +736,16 @@ where } } phase.send_replace(LifecyclePhase::Draining); + log_native_lifecycle( + LifecyclePhase::Draining, + if outcome.is_ok() { + "drain_requested" + } else { + "runner_failed" + }, + grace, + outcome.is_err(), + ); drop(listener); let result = drain_tasks(&mut tasks, grace, signals, outcome).await; // Service/registry clones are dropped only after every owned connection settles. @@ -661,6 +764,12 @@ async fn drain_tasks( mut outcome: anyhow::Result<()>, ) -> anyhow::Result<()> { let Some(deadline) = NativeInstant::now().checked_add(grace) else { + log_native_lifecycle( + LifecyclePhase::Draining, + "unusable_grace_deadline", + grace, + true, + ); tasks.abort_all(); while tasks.join_next().await.is_some() {} return Err(failure("shutdown", "budget", "unusable deadline")); @@ -669,10 +778,12 @@ async fn drain_tasks( tokio::select! { biased; signal = next_signal(signals) => { + log_native_lifecycle(LifecyclePhase::Draining, "forced_stop", grace, true); outcome = Err(signal.err().unwrap_or_else(|| failure("shutdown", "signal", "forced stop"))); break; } () = sleep_until(deadline) => { + log_native_lifecycle(LifecyclePhase::Draining, "grace_exhausted", grace, true); outcome = Err(failure("shutdown", "budget", "grace exhausted")); break; } @@ -688,6 +799,20 @@ async fn drain_tasks( outcome } +fn log_native_lifecycle(phase: LifecyclePhase, reason: &'static str, grace: Duration, fault: bool) { + // Callers are private owner branches and pass fixed categories, never returned errors. + let phase_name = match phase { + LifecyclePhase::Starting => "starting", + LifecyclePhase::Ready => "ready", + LifecyclePhase::Draining => "draining", + LifecyclePhase::Stopped => "stopped", + }; + let level = if fault { Level::Error } else { Level::Info }; + let _logged = catch_unwind( + || log::log!(target: "edgezero::native", level, "event=lifecycle phase={phase_name} reason={reason} grace_ms={}", grace.as_millis()), + ); +} + /// Entry point for an Axum dev-server application. /// /// Portable store config is baked into `A` by the `app!` macro; adapter-specific @@ -709,7 +834,7 @@ fn build_app_for_dispatch() -> anyhow::Result { /// binding, or connection serving fails. #[inline] pub fn run_app() -> anyhow::Result<()> { - let (env, json_body_limit) = capture_dev_environment(vars_os())?; + let (env, json_body_limit, native) = capture_dev_environment(vars_os())?; let grace = shutdown_grace(&env)?; let resolution = resolve_addr(&env); run_local( @@ -733,12 +858,12 @@ pub fn run_app() -> anyhow::Result<()> { config: build_config_registry(metadata.config, &env), secrets: build_secret_registry(metadata.secrets, &env), }; - prepare_transport(app, stores, json_body_limit) + prepare_transport(app, stores, json_body_limit, native) }, ) } -fn capture_dev_environment(vars: I) -> anyhow::Result<(EnvConfig, JsonBodyLimit)> +fn capture_dev_environment(vars: I) -> anyhow::Result<(EnvConfig, JsonBodyLimit, NativeHosting)> where I: IntoIterator, { @@ -747,24 +872,38 @@ where let Some(key) = os_key.to_str() else { continue; }; - let recognized = key - .strip_prefix("EDGEZERO__") - .is_some_and(|path| path.eq_ignore_ascii_case("ADAPTER__JSON_BODY_LIMIT_BYTES")); - match os_value.to_str() { - Some(value) => pairs.push((key.to_owned(), value.to_owned())), - None if recognized => { - return Err(failure( - "settings", - "JSON_BODY_LIMIT_BYTES", - "non-Unicode value", - )); - } - None => {} + let recognized = key.strip_prefix("EDGEZERO__").and_then(|path| { + [ + ("ADAPTER__JSON_BODY_LIMIT_BYTES", "JSON_BODY_LIMIT_BYTES"), + ("ADAPTER__TRUSTED_PROXY_CIDRS", "TRUSTED_PROXY_CIDRS"), + ( + "ADAPTER__FORWARDING_HEADER_FAMILY", + "FORWARDING_HEADER_FAMILY", + ), + ("LOGGING__REQUEST_RECORDS", "LOGGING__REQUEST_RECORDS"), + ] + .into_iter() + .find(|(candidate, _)| path.eq_ignore_ascii_case(candidate)) + .map(|(_, setting)| setting) + }); + match (os_value.to_str(), recognized) { + (Some(value), _) => pairs.push((key.to_owned(), value.to_owned())), + (None, Some(setting)) => return Err(failure("settings", setting, "non-Unicode value")), + (None, None) => {} } } let env = EnvConfig::from_vars(pairs); let limit = JsonBodyLimit::from_config(&env)?; - Ok((env, limit)) + let (policy, request_records) = native_settings(&env)?; + Ok(( + env, + limit, + NativeHosting { + policy, + request_records, + diagnostics: NativeDiagnosticsHandle::new(), + }, + )) } fn init_logging(level: LevelFilter) { @@ -890,7 +1029,17 @@ where ); failure("initializer", "application", &category) })?; - Ok((app, stores, transport, options.json_body_limit())) + Ok(( + app, + stores, + transport, + options.json_body_limit(), + NativeHosting { + diagnostics: options.diagnostics(), + policy: options.trusted_proxy_policy(), + request_records: options.request_records(), + }, + )) }, ) } @@ -1242,7 +1391,7 @@ mod tests { #[test] fn json_limit_development_capture_and_explicit_router_builder() { - let (env, selected) = capture_dev_environment([ + let (env, selected, _native) = capture_dev_environment([ ( OsString::from("EDGEZERO__ADAPTER__HOST"), OsString::from("invalid-host"), @@ -1265,6 +1414,7 @@ mod tests { App::new(explicit.router), explicit.stores.into(), explicit.json_body_limit, + explicit.native, ) .expect("prepared transport"); assert_eq!(prepared.3.get().get(), 64); @@ -1280,7 +1430,7 @@ mod tests { fn development_capture_skips_unrelated_binary_settings_but_rejects_binary_json_limit() { use std::os::unix::ffi::OsStringExt as _; let binary = OsString::from_vec(vec![0xff]); - let (env, selected) = capture_dev_environment([ + let (env, selected, _native) = capture_dev_environment([ (binary.clone(), OsString::from("ignored")), (OsString::from("UNRELATED"), binary.clone()), (OsString::from("EDGEZERO__ADAPTER__HOST"), binary.clone()), diff --git a/crates/edgezero-adapter-axum/src/diagnostics.rs b/crates/edgezero-adapter-axum/src/diagnostics.rs new file mode 100644 index 000000000..c07f5a0a7 --- /dev/null +++ b/crates/edgezero-adapter-axum/src/diagnostics.rs @@ -0,0 +1,1521 @@ +//! Safe native request observations and managed-lifetime accounting. + +#![expect( + clippy::arbitrary_source_item_ordering, + reason = "Public record fields follow their diagnostic meaning; native identity and lifetime owners remain grouped with their transitions rather than alphabetized" +)] + +use std::fmt; +use std::net::{IpAddr, SocketAddr}; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::sync::atomic::{AtomicU16, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard, OnceLock, PoisonError}; +use std::time::Duration; + +use edgezero_core::http::{Method, StatusCode}; +use edgezero_core::ingress::{ + CheckedEffectiveHost, CheckedHostSource, NormalizedIngressHeadSummary, +}; +use edgezero_core::response_egress::{ + ResponseEgressBodyKind, ResponseEgressCompletion, ResponseEgressFailureClassification, + ResponseEgressFailureKind, ResponseEgressFallbackDisposition, ResponseEgressOutcome, + ResponseEgressReport, +}; +use edgezero_core::{MonotonicClock, MonotonicInstant}; + +use crate::proxy::{CheckedProxyMetadata, ForwardingResult, Scheme}; + +/// Framework-generated correlation value, not a credential or visitor-supplied ID. +#[derive(Clone, Copy, Eq, Hash, PartialEq)] +pub struct RequestId { + nonce: [u8; 16], + host: u64, + request: u64, +} + +impl fmt::Display for RequestId { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for byte in self.nonce { + write!(f, "{byte:02x}")?; + } + write!(f, "{:016x}{:016x}", self.host, self.request) + } +} + +impl fmt::Debug for RequestId { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +/// Immutable checked facts captured before managed admission. +#[derive(Clone)] +pub struct NativeIngressMetadata { + request_id: RequestId, + proxy: CheckedProxyMetadata, + received: NormalizedIngressHeadSummary, +} + +impl NativeIngressMetadata { + pub(crate) fn new( + request_id: RequestId, + proxy: CheckedProxyMetadata, + received: NormalizedIngressHeadSummary, + ) -> Self { + Self { + request_id, + proxy, + received, + } + } + + #[must_use] + #[inline] + pub fn request_id(&self) -> RequestId { + self.request_id + } + + #[must_use] + #[inline] + pub fn direct_peer(&self) -> Option { + self.proxy.direct_peer() + } + + #[must_use] + #[inline] + pub fn effective_client(&self) -> Option { + self.proxy.effective_client() + } + + #[must_use] + #[inline] + pub fn effective_host(&self) -> &CheckedEffectiveHost { + self.proxy.effective_host() + } + + #[must_use] + #[inline] + pub fn effective_scheme(&self) -> Scheme { + self.proxy.effective_scheme() + } + + #[must_use] + #[inline] + pub fn client_source(&self) -> CheckedHostSource { + self.proxy.client_source() + } + + #[must_use] + #[inline] + pub fn host_source(&self) -> CheckedHostSource { + self.proxy.effective_host().source() + } + + #[must_use] + #[inline] + pub fn scheme_source(&self) -> CheckedHostSource { + self.proxy.scheme_source() + } + + #[must_use] + #[inline] + pub fn forwarding_result(&self) -> ForwardingResult { + self.proxy.forwarding_result() + } + + #[must_use] + #[inline] + pub fn received_normalized_head(&self) -> &NormalizedIngressHeadSummary { + &self.received + } +} + +/// One coherent, momentary observation of managed guards, not kernel or detached work. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct NativeDiagnosticsSnapshot { + pub active_requests: u64, + /// Includes idle keep-alive connection tasks. + pub active_connections: u64, + pub idle_connections: u64, +} + +pub trait NativeRequestObserver: Send + Sync + 'static { + fn observe(&self, record: &NativeRequestRecord); +} + +/// Sharing this handle explicitly aggregates counters across managed runners. +#[derive(Clone, Default)] +pub struct NativeDiagnosticsHandle { + counters: Arc>, + observer: Option>, +} + +impl NativeDiagnosticsHandle { + #[must_use] + #[inline] + pub fn new() -> Self { + Self::default() + } + + #[must_use] + #[inline] + pub fn with_request_observer(mut self, observer: O) -> Self + where + O: NativeRequestObserver, + { + self.observer = Some(Arc::new(observer)); + self + } + + #[must_use] + #[inline] + pub fn snapshot(&self) -> NativeDiagnosticsSnapshot { + *lock(&self.counters) + } +} + +impl fmt::Debug for NativeDiagnosticsHandle { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("NativeDiagnosticsHandle") + .field("snapshot", &self.snapshot()) + .finish_non_exhaustive() + } +} + +/// A no-response disposition, deliberately distinct from response-egress outcomes. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum IngressDisposition { + Aborted, + Draining, + Abandoned, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum NativeRequestOutcome { + Egress(ResponseEgressOutcome), + Ingress(IngressDisposition), +} + +/// Categories captured by owning framework boundaries, never error messages or sources. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum NativeRequestFailure { + FrameworkRouting { kind: &'static str }, + PropagatedError { kind: &'static str }, + FallbackBodyExceeded, + FallbackReadTimedOut, + AdmissionRefused, +} + +/// Allowlisted observation. Bytes and selected status do not establish client receipt. +#[derive(Clone, Debug)] +pub struct NativeRequestRecord { + pub event: &'static str, + pub request_id: RequestId, + pub method: &'static str, + /// Registered template only, or the fixed `unknown` value. + pub route: String, + pub status: Option, + /// Absent when the owning monotonic clock moved backwards. + pub duration: Option, + pub outcome: NativeRequestOutcome, + pub failure: Option, + pub forwarding_result: ForwardingResult, + pub bytes_written: Option, + pub body_kind: Option, + pub fallback_disposition: Option, +} + +impl NativeRequestRecord { + /// Severity follows typed ownership, never status alone. + #[must_use] + #[inline] + pub fn level(&self) -> log::Level { + if self.duration.is_none() { + return log::Level::Error; + } + let ordinary_failure = matches!( + self.failure, + None | Some(NativeRequestFailure::FrameworkRouting { .. }) + ); + let ordinary_outcome = matches!( + self.outcome, + NativeRequestOutcome::Egress( + ResponseEgressOutcome::Completed | ResponseEgressOutcome::HostHandoff + ) | NativeRequestOutcome::Ingress(IngressDisposition::Draining) + ); + if ordinary_failure + && ordinary_outcome + && self.forwarding_result != ForwardingResult::Malformed + { + log::Level::Info + } else { + log::Level::Warn + } + } +} + +#[derive(Clone)] +pub(crate) struct NativeSession { + identity: Arc, + handle: NativeDiagnosticsHandle, + request_records: bool, +} + +struct SessionIdentity { + nonce: [u8; 16], + host: u64, + requests: AtomicU64, +} + +static PROCESS_NONCE: OnceLock> = OnceLock::new(); +static HOST_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone, Copy, Debug, thiserror::Error)] +enum IdentityFault { + #[error("native_identity_entropy_failed")] + Entropy, + #[error("native_identity_host_sequence_exhausted")] + HostSequence, + #[error("native_identity_request_sequence_exhausted")] + RequestSequence, + #[error("native_diagnostics_connection_scope_mismatch")] + ConnectionScope, +} + +impl NativeSession { + pub(crate) fn new( + handle: NativeDiagnosticsHandle, + request_records: bool, + ) -> anyhow::Result { + let nonce = *PROCESS_NONCE.get_or_init(|| nonce_from(getrandom::fill)); + Self::from_sources(handle, request_records, nonce, &HOST_SEQUENCE).map_err(Into::into) + } + + fn from_sources( + handle: NativeDiagnosticsHandle, + request_records: bool, + nonce: Result<[u8; 16], IdentityFault>, + hosts: &AtomicU64, + ) -> Result { + Ok(Self { + identity: Arc::new(SessionIdentity { + nonce: nonce?, + host: next_sequence(hosts).ok_or(IdentityFault::HostSequence)?, + requests: AtomicU64::new(0), + }), + handle, + request_records, + }) + } + + /// Construct before submitting the owned task, including tasks never polled. + pub(crate) fn connection(&self) -> (ConnectionActivity, ConnectionGuard) { + let activity = ConnectionActivity { + state: Arc::new(Mutex::new(ConnectionState { + requests: 0, + closed: false, + })), + counters: Arc::clone(&self.handle.counters), + }; + let mut counts = lock(&activity.counters); + // Metrics must not introduce a panic in managed guard ownership. + counts.active_connections = counts.active_connections.saturating_add(1); + counts.idle_connections = counts.idle_connections.saturating_add(1); + drop(counts); + let guard = ConnectionGuard { + activity: activity.clone(), + }; + (activity, guard) + } + + pub(crate) fn request( + &self, + method: &Method, + start: MonotonicInstant, + clock: MonotonicClock, + activity: Option, + ) -> anyhow::Result { + if activity + .as_ref() + .is_some_and(|link| !Arc::ptr_eq(&link.counters, &self.handle.counters)) + { + return Err(IdentityFault::ConnectionScope.into()); + } + let Some(sequence) = next_sequence(&self.identity.requests) else { + isolated( + || log::error!(target: "edgezero::native", "lifecycle reason=native_identity_request_sequence_exhausted"), + ); + return Err(IdentityFault::RequestSequence.into()); + }; + if let Some(link) = &activity { + let mut connection = lock(&link.state); + let mut counts = lock(&link.counters); + if !connection.closed { + if connection.requests == 0 { + counts.idle_connections = counts.idle_connections.saturating_sub(1); + } + connection.requests = connection.requests.saturating_add(1); + } + counts.active_requests = counts.active_requests.saturating_add(1); + } else { + let mut counts = lock(&self.handle.counters); + counts.active_requests = counts.active_requests.saturating_add(1); + } + Ok(RequestGuard { + session: self.clone(), + id: RequestId { + nonce: self.identity.nonce, + host: self.identity.host, + request: sequence, + }, + method: method_category(method), + start, + clock, + activity, + forwarding_result: ForwardingResult::Absent, + failure: None, + released: false, + }) + } +} + +fn nonce_from(fill: impl FnOnce(&mut [u8]) -> Result<(), E>) -> Result<[u8; 16], IdentityFault> { + let mut nonce = [0; 16]; + fill(&mut nonce).map_err(|_entropy_error| IdentityFault::Entropy)?; + Ok(nonce) +} + +fn next_sequence(sequence: &AtomicU64) -> Option { + sequence + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| { + value.checked_add(1) + }) + .ok() +} + +struct ConnectionState { + requests: u64, + closed: bool, +} + +#[derive(Clone)] +pub(crate) struct ConnectionActivity { + state: Arc>, + counters: Arc>, +} + +pub(crate) struct ConnectionGuard { + activity: ConnectionActivity, +} + +impl Drop for ConnectionGuard { + fn drop(&mut self) { + let mut connection = lock(&self.activity.state); + let mut counts = lock(&self.activity.counters); + connection.closed = true; + counts.active_connections = counts.active_connections.saturating_sub(1); + if connection.requests == 0 { + counts.idle_connections = counts.idle_connections.saturating_sub(1); + } + } +} + +#[derive(Clone, Default)] +pub(crate) struct SelectedStatus(Arc); + +impl SelectedStatus { + pub(crate) fn select(&self, status: StatusCode) { + self.0.store(status.as_u16(), Ordering::Relaxed); + } + + fn get(&self) -> Option { + StatusCode::from_u16(self.0.load(Ordering::Relaxed)).ok() + } +} + +pub(crate) struct RequestGuard { + session: NativeSession, + id: RequestId, + method: &'static str, + start: MonotonicInstant, + clock: MonotonicClock, + activity: Option, + forwarding_result: ForwardingResult, + failure: Option, + released: bool, +} + +impl RequestGuard { + pub(crate) fn request_id(&self) -> RequestId { + self.id + } + + pub(crate) fn set_forwarding_result(&mut self, result: ForwardingResult) { + self.forwarding_result = result; + } + + pub(crate) fn set_failure(&mut self, failure: Option) { + self.failure = failure.map(|classification| match classification.kind() { + ResponseEgressFailureKind::PropagatedError => { + let kind = classification.error_kind().unwrap_or("unknown"); + if classification.is_framework_route_error() { + NativeRequestFailure::FrameworkRouting { kind } + } else { + NativeRequestFailure::PropagatedError { kind } + } + } + ResponseEgressFailureKind::FallbackBodyExceeded => { + NativeRequestFailure::FallbackBodyExceeded + } + ResponseEgressFailureKind::FallbackReadTimedOut => { + NativeRequestFailure::FallbackReadTimedOut + } + _ => NativeRequestFailure::PropagatedError { kind: "unknown" }, + }); + } + + pub(crate) fn set_ingress_error_kind(&mut self, kind: &'static str) { + self.failure = Some(NativeRequestFailure::PropagatedError { kind }); + } + + pub(crate) fn refused(&mut self) { + self.failure = Some(NativeRequestFailure::AdmissionRefused); + } + + pub(crate) fn completion(mut self) -> (ResponseEgressCompletion, SelectedStatus) { + let selected = SelectedStatus::default(); + let captured = selected.clone(); + let completion = + ResponseEgressCompletion::new_with_terminal_time(move |report, terminal| { + self.release(); + let record = self.record( + NativeRequestOutcome::Egress(report.outcome), + terminal, + captured.get(), + Some(report), + ); + self.emit(&record); + }); + (completion, selected) + } + + pub(crate) fn finish_ingress(mut self, disposition: IngressDisposition) { + self.finish_ingress_at(disposition, self.clock.now()); + } + + fn finish_ingress_at(&mut self, disposition: IngressDisposition, terminal: MonotonicInstant) { + self.release(); + let record = self.record( + NativeRequestOutcome::Ingress(disposition), + terminal, + None, + None, + ); + self.emit(&record); + } + + fn record( + &self, + outcome: NativeRequestOutcome, + terminal: MonotonicInstant, + status: Option, + report: Option<&ResponseEgressReport>, + ) -> NativeRequestRecord { + NativeRequestRecord { + event: if report.is_some() { + "request_terminal" + } else { + "request_ingress" + }, + request_id: self.id, + method: self.method, + route: report + .and_then(|terminal_report| terminal_report.route.as_ref()) + .map_or_else(|| "unknown".into(), |route| route.pattern().to_owned()), + status, + duration: terminal.checked_duration_since(self.start), + outcome, + failure: self.failure, + forwarding_result: self.forwarding_result, + bytes_written: report.map(|terminal_report| terminal_report.bytes_written), + body_kind: report.map(|terminal_report| terminal_report.body_kind), + fallback_disposition: report + .and_then(|terminal_report| terminal_report.fallback_disposition), + } + } + + fn emit(&self, record: &NativeRequestRecord) { + emit_with( + record, + self.session.request_records, + |safe_record| log::log!(target: "edgezero::native", safe_record.level(), "{}", SafeRecord(safe_record)), + self.session.handle.observer.as_deref(), + ); + } + + fn release(&mut self) { + if self.released { + return; + } + self.released = true; + if let Some(activity) = &self.activity { + let mut connection = lock(&activity.state); + let mut counts = lock(&activity.counters); + counts.active_requests = counts.active_requests.saturating_sub(1); + if !connection.closed { + connection.requests = connection.requests.saturating_sub(1); + if connection.requests == 0 { + counts.idle_connections = counts.idle_connections.saturating_add(1); + } + } + } else { + let mut counts = lock(&self.session.handle.counters); + counts.active_requests = counts.active_requests.saturating_sub(1); + } + } +} + +impl Drop for RequestGuard { + fn drop(&mut self) { + if !self.released { + // No report/status for cancellation or an unbegun envelope's abandonment. + self.release(); + isolated(|| self.finish_ingress_at(IngressDisposition::Abandoned, self.clock.now())); + } + } +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) +} + +fn isolated(action: impl FnOnce()) { + drop(catch_unwind(AssertUnwindSafe(action))); +} + +fn emit_with( + record: &NativeRequestRecord, + enabled: bool, + log_sink: impl FnOnce(&NativeRequestRecord), + observer: Option<&dyn NativeRequestObserver>, +) { + if enabled { + isolated(|| log_sink(record)); + } + if let Some(sink) = observer { + isolated(|| sink.observe(record)); + } +} + +fn method_category(method: &Method) -> &'static str { + match *method { + Method::GET => "GET", + Method::HEAD => "HEAD", + Method::POST => "POST", + Method::PUT => "PUT", + Method::DELETE => "DELETE", + Method::CONNECT => "CONNECT", + Method::OPTIONS => "OPTIONS", + Method::TRACE => "TRACE", + Method::PATCH => "PATCH", + _ => "other", + } +} + +struct SafeRecord<'record>(&'record NativeRequestRecord); + +impl fmt::Display for SafeRecord<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let record = self.0; + write!( + f, + "{} request_id={} method={} route=", + record.event, record.request_id, record.method + )?; + write!(f, "\"{}\"", record.route.escape_debug())?; + match record.status { + Some(status) => write!(f, " status={}", status.as_u16())?, + None => f.write_str(" status=none")?, + } + match record.duration { + Some(duration) => write!(f, " duration_us={}", duration.as_micros())?, + None => f.write_str(" duration_us=none clock_fault=backwards")?, + } + write!( + f, + " outcome={} failure={} forwarding_result={}", + outcome_name(record.outcome), + failure_name(record.failure), + record.forwarding_result.as_str() + )?; + if let Some(bytes) = record.bytes_written { + write!(f, " bytes_written={bytes}")?; + } + if let Some(kind) = record.body_kind { + f.write_str(match kind { + ResponseEgressBodyKind::Application => " body_kind=application", + ResponseEgressBodyKind::Fallback => " body_kind=fallback", + _ => " body_kind=unknown", + })?; + } + if let Some(disposition) = record.fallback_disposition { + f.write_str(match disposition { + ResponseEgressFallbackDisposition::Aborted => " fallback_disposition=aborted", + ResponseEgressFallbackDisposition::Completed => " fallback_disposition=completed", + _ => " fallback_disposition=unknown", + })?; + } + Ok(()) + } +} + +fn outcome_name(outcome: NativeRequestOutcome) -> &'static str { + match outcome { + NativeRequestOutcome::Ingress(IngressDisposition::Aborted) => "ingress_aborted", + NativeRequestOutcome::Ingress(IngressDisposition::Draining) => "ingress_draining", + NativeRequestOutcome::Ingress(IngressDisposition::Abandoned) => "ingress_abandoned", + NativeRequestOutcome::Egress(egress_outcome) => match egress_outcome { + ResponseEgressOutcome::ClientDisconnected => "client_disconnected", + ResponseEgressOutcome::Completed => "completed", + ResponseEgressOutcome::ConversionError => "conversion_error", + ResponseEgressOutcome::DeadlineExceeded => "deadline_exceeded", + ResponseEgressOutcome::HostHandoff => "host_handoff", + ResponseEgressOutcome::RequestCancelled => "request_cancelled", + ResponseEgressOutcome::SourceError => "source_error", + ResponseEgressOutcome::TransportError => "transport_error", + ResponseEgressOutcome::Unspecified => "unspecified", + _ => "unknown", + }, + } +} + +fn failure_name(failure: Option) -> &'static str { + match failure { + None => "none", + Some( + NativeRequestFailure::FrameworkRouting { kind } + | NativeRequestFailure::PropagatedError { kind }, + ) => kind, + Some(NativeRequestFailure::FallbackBodyExceeded) => "fallback_body_exceeded", + Some(NativeRequestFailure::FallbackReadTimedOut) => "fallback_read_timed_out", + Some(NativeRequestFailure::AdmissionRefused) => "admission_refused", + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + use std::iter::repeat_with; + use std::sync::atomic::AtomicUsize; + use std::thread; + + use edgezero_core::http::{HeaderMap, Version}; + use edgezero_core::response_egress::{ + ResponseEgressAttempt, ResponseEgressHead, ResponseEgressObserverHandle, + }; + use edgezero_core::router::RouteMetadata; + + use super::*; + + #[derive(Clone, Default)] + struct RecordingObserver { + records: Arc>>, + snapshots: Arc>>, + handle: NativeDiagnosticsHandle, + panic: bool, + } + + impl NativeRequestObserver for RecordingObserver { + fn observe(&self, record: &NativeRequestRecord) { + lock(&self.records).push(record.clone()); + lock(&self.snapshots).push(self.handle.snapshot()); + assert!(!self.panic, "observer fixture panic"); + } + } + + fn session(handle: NativeDiagnosticsHandle, enabled: bool) -> NativeSession { + NativeSession::from_sources(handle, enabled, Ok([0x19; 16]), &AtomicU64::new(0)) + .expect("test identity") + } + + fn observed_session(enabled: bool) -> (NativeSession, RecordingObserver) { + let observer = RecordingObserver::default(); + let handle = observer + .handle + .clone() + .with_request_observer(observer.clone()); + (session(handle, enabled), observer) + } + + fn request(session: &NativeSession, activity: Option) -> RequestGuard { + session + .request( + &Method::GET, + MonotonicInstant::now(), + MonotonicClock::default(), + activity, + ) + .expect("request") + } + + fn attempt( + completion: ResponseEgressCompletion, + start: MonotonicInstant, + egress_start: MonotonicInstant, + clock: MonotonicClock, + route: Option<&RouteMetadata>, + ) -> ResponseEgressAttempt { + let headers = HeaderMap::new(); + let head = ResponseEgressHead::new( + StatusCode::OK, + Version::HTTP_11, + &headers, + None, + start, + route, + ); + ResponseEgressAttempt::new( + &head, + egress_start, + completion, + ResponseEgressObserverHandle::default(), + clock, + ) + } + + #[test] + fn identity_is_fixed_hex_and_sequences_are_shared_only_with_session_clones() { + let hosts = AtomicU64::new(0); + let first = NativeSession::from_sources( + NativeDiagnosticsHandle::new(), + false, + Ok([0xab; 16]), + &hosts, + ) + .unwrap(); + let second = NativeSession::from_sources( + NativeDiagnosticsHandle::new(), + false, + Ok([0xab; 16]), + &hosts, + ) + .unwrap(); + let ids = [ + request(&first, None).request_id(), + request(&first.clone(), None).request_id(), + request(&second, None).request_id(), + ]; + assert_eq!(ids.into_iter().collect::>().len(), 3); + assert_eq!( + ids[0].to_string(), + format!("{}{:016x}{:016x}", "ab".repeat(16), 0_u64, 0_u64) + ); + assert_eq!(ids[1].request, 1); + assert_eq!(ids[2].host, 1); + assert_eq!(ids[2].request, 0); + for id in ids { + let rendered = id.to_string(); + assert_eq!(rendered.len(), 64); + assert!(rendered.bytes().all(|byte| byte.is_ascii_hexdigit())); + assert_eq!(format!("{id:?}"), rendered); + } + } + + #[test] + fn entropy_and_host_exhaustion_fail_without_fallback_or_wrap() { + let nonce = nonce_from(|bytes| { + bytes.fill(0x12); + Ok::<_, ()>(()) + }) + .unwrap(); + assert_eq!(nonce, [0x12; 16]); + let failure = nonce_from(|_| Err("secret entropy source")); + assert_eq!( + failure.unwrap_err().to_string(), + "native_identity_entropy_failed" + ); + let hosts = AtomicU64::new(u64::MAX - 1); + let first = + NativeSession::from_sources(NativeDiagnosticsHandle::new(), false, Ok(nonce), &hosts) + .unwrap(); + assert_eq!(first.identity.host, u64::MAX - 1); + let error = + NativeSession::from_sources(NativeDiagnosticsHandle::new(), false, Ok(nonce), &hosts) + .err() + .unwrap(); + assert_eq!(error.to_string(), "native_identity_host_sequence_exhausted"); + assert_eq!(hosts.load(Ordering::Relaxed), u64::MAX); + let unused_hosts = AtomicU64::new(0); + assert!( + NativeSession::from_sources( + NativeDiagnosticsHandle::new(), + false, + failure, + &unused_hosts + ) + .is_err() + ); + assert_eq!(unused_hosts.load(Ordering::Relaxed), 0); + let other = NativeSession::from_sources( + NativeDiagnosticsHandle::new(), + false, + Ok([0x13; 16]), + &AtomicU64::new(0), + ) + .unwrap(); + assert_ne!( + request(&first, None).request_id(), + request(&other, None).request_id() + ); + } + + #[test] + fn request_exhaustion_preserves_counters_and_never_reuses_an_id() { + let session = session(NativeDiagnosticsHandle::new(), false); + session + .identity + .requests + .store(u64::MAX - 1, Ordering::Relaxed); + let guard = request(&session, None); + assert_eq!(guard.request_id().request, u64::MAX - 1); + drop(guard); + assert!( + session + .request( + &Method::GET, + MonotonicInstant::now(), + MonotonicClock::default(), + None + ) + .is_err() + ); + assert_eq!(session.identity.requests.load(Ordering::Relaxed), u64::MAX); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + + #[test] + fn concurrent_requests_share_checked_sequence_without_collisions() { + let session = session(NativeDiagnosticsHandle::new(), false); + let workers: Vec<_> = repeat_with(|| { + let worker_session = session.clone(); + thread::spawn(move || { + repeat_with(|| request(&worker_session, None).request_id()) + .take(64) + .collect::>() + }) + }) + .take(8) + .collect(); + let ids: HashSet<_> = workers + .into_iter() + .flat_map(|worker| worker.join().unwrap()) + .collect(); + assert_eq!(ids.len(), 512); + assert_eq!(session.handle.snapshot().active_requests, 0); + } + + #[test] + fn connection_counts_exist_before_polling_and_track_two_request_lifetimes() { + let session = session(NativeDiagnosticsHandle::new(), false); + let (activity, connection) = session.connection(); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot { + active_requests: 0, + active_connections: 1, + idle_connections: 1 + } + ); + let first = request(&session, Some(activity.clone())); + let second = request(&session, Some(activity)); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot { + active_requests: 2, + active_connections: 1, + idle_connections: 0 + } + ); + drop(first); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot { + active_requests: 1, + active_connections: 1, + idle_connections: 0 + } + ); + drop(second); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot { + active_requests: 0, + active_connections: 1, + idle_connections: 1 + } + ); + // No future needs to have been polled for task cancellation to release this guard. + drop(connection); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + + #[test] + fn closed_connection_late_release_cannot_recreate_idle_connections() { + let session = session(NativeDiagnosticsHandle::new(), false); + let (activity, connection) = session.connection(); + let first = request(&session, Some(activity.clone())); + drop(connection); + let late = request(&session, Some(activity)); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot { + active_requests: 2, + active_connections: 0, + idle_connections: 0 + } + ); + drop(first); + drop(late); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + + #[test] + fn explicit_handle_sharing_aggregates_connections_and_requests() { + let handle = NativeDiagnosticsHandle::new(); + let first = session(handle.clone(), false); + let second = session(handle.clone(), false); + let (_, connection) = first.connection(); + let guard = request(&second, None); + assert_eq!( + handle.snapshot(), + NativeDiagnosticsSnapshot { + active_requests: 1, + active_connections: 1, + idle_connections: 1 + } + ); + drop((connection, guard)); + assert_eq!(handle.snapshot(), NativeDiagnosticsSnapshot::default()); + } + + #[test] + fn activity_from_a_different_handle_fails_without_incrementing_either_scope() { + let first = session(NativeDiagnosticsHandle::new(), false); + let second = session(NativeDiagnosticsHandle::new(), false); + let (activity, connection) = first.connection(); + assert!( + second + .request( + &Method::GET, + MonotonicInstant::now(), + MonotonicClock::default(), + Some(activity) + ) + .is_err() + ); + assert_eq!(first.handle.snapshot().active_requests, 0); + assert_eq!( + second.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + drop(connection); + } + + #[test] + fn ingress_and_unbegun_completion_abandonment_have_no_status_or_report_fields() { + let (session, observer) = observed_session(false); + request(&session, None).finish_ingress(IngressDisposition::Aborted); + request(&session, None).finish_ingress(IngressDisposition::Draining); + let (completion, status) = request(&session, None).completion(); + status.select(StatusCode::OK); + drop(completion); + let records = lock(&observer.records); + assert_eq!(records.len(), 3); + assert_eq!( + records[0].outcome, + NativeRequestOutcome::Ingress(IngressDisposition::Aborted) + ); + assert_eq!( + records[1].outcome, + NativeRequestOutcome::Ingress(IngressDisposition::Draining) + ); + assert_eq!( + records[2].outcome, + NativeRequestOutcome::Ingress(IngressDisposition::Abandoned) + ); + for record in records.iter() { + assert_eq!(record.event, "request_ingress"); + assert_eq!(record.status, None); + assert_eq!(record.bytes_written, None); + assert_eq!(record.body_kind, None); + assert_eq!(record.fallback_disposition, None); + } + assert!( + lock(&observer.snapshots) + .iter() + .all(|snapshot| snapshot.active_requests == 0) + ); + } + + #[test] + fn authoritative_terminal_time_excludes_prior_app_callback_and_sink_delays() { + let (session, observer) = observed_session(false); + let start = MonotonicInstant::now(); + let egress_start = start + Duration::from_secs(7); + let terminal = start + Duration::from_secs(11); + let now = Arc::new(Mutex::new(terminal)); + let source = Arc::clone(&now); + let clock = MonotonicClock::new(move || *lock(&source)); + let guard = session + .request(&Method::GET, start, clock.clone(), None) + .unwrap(); + let private_id = guard.request_id(); + let (native, status) = guard.completion(); + status.select(StatusCode::OK); + let app_source = Arc::clone(&now); + let app_reports = Arc::new(Mutex::new(Vec::new())); + let captured_reports = Arc::clone(&app_reports); + let app = ResponseEgressCompletion::new(move |report| { + lock(&captured_reports).push(report.clone()); + *lock(&app_source) = terminal + Duration::from_secs(100); + }); + let route = RouteMetadata::new(Method::GET, "/orders/{id}"); + let mut attempt = attempt(app.join(native), start, egress_start, clock, Some(&route)); + assert!(attempt.begin_writing()); + assert!(attempt.account_bytes(9, terminal)); + assert!(attempt.complete(terminal)); + assert!(!attempt.complete(terminal)); + drop(attempt); + let records = lock(&observer.records); + assert_eq!(records.len(), 1); + let record = &records[0]; + assert_eq!(record.request_id, private_id); + assert_eq!(record.duration, Some(Duration::from_secs(11))); + assert_eq!(lock(&app_reports)[0].elapsed, Duration::from_secs(4)); + assert_eq!(record.status, Some(StatusCode::OK)); + assert_eq!(record.route, "/orders/{id}"); + assert_eq!(record.bytes_written, Some(9)); + assert_eq!( + record.outcome, + NativeRequestOutcome::Egress(ResponseEgressOutcome::Completed) + ); + assert_eq!(lock(&observer.snapshots)[0].active_requests, 0); + emit_with( + record, + true, + |_| *lock(&now) = terminal + Duration::from_secs(200), + None, + ); + assert_eq!(record.duration, Some(Duration::from_secs(11))); + } + + #[test] + fn failing_attempt_retains_selected_status_and_fallback_updates_use_same_slot() { + for fallback in [false, true] { + let (session, observer) = observed_session(false); + let start = MonotonicInstant::now(); + let (completion, status) = session + .request(&Method::GET, start, MonotonicClock::default(), None) + .unwrap() + .completion(); + status.select(StatusCode::OK); + let mut attempt = attempt(completion, start, start, MonotonicClock::default(), None); + if fallback { + assert!(attempt.begin_fallback(ResponseEgressOutcome::ConversionError)); + status.select(StatusCode::INTERNAL_SERVER_ERROR); + assert!(attempt.begin_writing()); + assert!( + attempt.finish_fallback(ResponseEgressFallbackDisposition::Completed, start) + ); + } else { + assert!(attempt.begin_writing()); + assert!(attempt.account_bytes(3, start)); + assert!(attempt.terminate(ResponseEgressOutcome::SourceError, start)); + } + drop(attempt); + let records = lock(&observer.records); + assert_eq!(records.len(), 1); + assert_eq!( + records[0].status, + Some(if fallback { + StatusCode::INTERNAL_SERVER_ERROR + } else { + StatusCode::OK + }) + ); + assert_eq!( + records[0].outcome, + NativeRequestOutcome::Egress(if fallback { + ResponseEgressOutcome::ConversionError + } else { + ResponseEgressOutcome::SourceError + }) + ); + assert_eq!(records[0].level(), log::Level::Warn); + assert_eq!(session.handle.snapshot().active_requests, 0); + } + } + + #[test] + fn drop_and_completion_release_before_panicking_observer() { + let observer = RecordingObserver { + panic: true, + ..RecordingObserver::default() + }; + let session = session( + observer + .handle + .clone() + .with_request_observer(observer.clone()), + false, + ); + catch_unwind(AssertUnwindSafe(|| drop(request(&session, None)))) + .expect("observer panic must not escape request Drop"); + let start = MonotonicInstant::now(); + let (completion, status) = request(&session, None).completion(); + status.select(StatusCode::OK); + let mut attempt = attempt(completion, start, start, MonotonicClock::default(), None); + attempt.begin_writing(); + assert!(attempt.complete(start + Duration::from_secs(1))); + drop(attempt); + assert_eq!(lock(&observer.records).len(), 2); + assert!( + lock(&observer.snapshots) + .iter() + .all(|snapshot| snapshot.active_requests == 0) + ); + assert_eq!(session.handle.snapshot().active_requests, 0); + } + + #[test] + fn sink_panics_are_independent_and_never_retry_logging() { + let (session, observer) = observed_session(false); + drop(request(&session, None)); + let record = lock(&observer.records)[0].clone(); + let logs = AtomicUsize::new(0); + emit_with( + &record, + true, + |_| { + logs.fetch_add(1, Ordering::Relaxed); + panic!("logger fixture panic"); + }, + Some(&observer), + ); + assert_eq!(logs.load(Ordering::Relaxed), 1); + assert_eq!(lock(&observer.records).len(), 2); + let panic_observer = RecordingObserver { + panic: true, + ..RecordingObserver::default() + }; + catch_unwind(AssertUnwindSafe(|| { + emit_with( + &record, + true, + |_| { + logs.fetch_add(1, Ordering::Relaxed); + panic!("logger fixture panic"); + }, + Some(&panic_observer), + ); + })) + .expect("logger and observer panics must be contained independently"); + assert_eq!(logs.load(Ordering::Relaxed), 2); + assert_eq!(lock(&panic_observer.records).len(), 1); + } + + #[test] + fn opt_out_suppresses_only_log_sink_and_keeps_observers_and_accounting() { + let (session, observer) = observed_session(false); + drop(request(&session, None)); + let record = lock(&observer.records)[0].clone(); + emit_with( + &record, + false, + |_| panic!("disabled log sink must not run"), + Some(&observer), + ); + assert_eq!(lock(&observer.records).len(), 2); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + + #[test] + fn severity_is_typed_not_status_and_extension_methods_never_leak() { + let (session, observer) = observed_session(false); + let start = MonotonicInstant::now(); + let extension = Method::from_bytes(b"SECRET-METHOD").unwrap(); + let (completion, status) = session + .request(&extension, start, MonotonicClock::default(), None) + .unwrap() + .completion(); + status.select(StatusCode::PAYLOAD_TOO_LARGE); + let mut attempt = attempt(completion, start, start, MonotonicClock::default(), None); + attempt.begin_writing(); + attempt.complete(start); + let ordinary = lock(&observer.records)[0].clone(); + assert_eq!(ordinary.method, "other"); + assert_eq!(ordinary.level(), log::Level::Info); + assert!(!SafeRecord(&ordinary).to_string().contains("SECRET")); + let mut classified = ordinary.clone(); + classified.failure = Some(NativeRequestFailure::AdmissionRefused); + assert_eq!(classified.level(), log::Level::Warn); + classified.failure = Some(NativeRequestFailure::FrameworkRouting { kind: "not_found" }); + assert_eq!(classified.level(), log::Level::Info); + classified.failure = Some(NativeRequestFailure::PropagatedError { kind: "not_found" }); + assert_eq!(classified.level(), log::Level::Warn); + classified.failure = Some(NativeRequestFailure::FrameworkRouting { + kind: "method_not_allowed", + }); + assert_eq!(classified.level(), log::Level::Info); + classified.failure = Some(NativeRequestFailure::PropagatedError { + kind: "method_not_allowed", + }); + assert_eq!(classified.level(), log::Level::Warn); + classified.failure = Some(NativeRequestFailure::PropagatedError { kind: "internal" }); + assert_eq!(classified.level(), log::Level::Warn); + classified.failure = Some(NativeRequestFailure::FallbackBodyExceeded); + assert_eq!(classified.level(), log::Level::Warn); + classified.failure = Some(NativeRequestFailure::FallbackReadTimedOut); + assert_eq!(classified.level(), log::Level::Warn); + classified.failure = None; + classified.forwarding_result = ForwardingResult::Malformed; + assert_eq!(classified.level(), log::Level::Warn); + classified.duration = None; + assert_eq!(classified.level(), log::Level::Error); + } + + #[test] + fn backwards_clock_is_an_explicit_safe_fault_not_wall_time_or_zero_duration() { + let (session, observer) = observed_session(false); + let start = MonotonicInstant::now(); + let clock = MonotonicClock::new(move || start); + let guard = session + .request(&Method::GET, start + Duration::from_secs(1), clock, None) + .unwrap(); + guard.finish_ingress(IngressDisposition::Aborted); + let record = lock(&observer.records)[0].clone(); + assert_eq!(record.duration, None); + assert_eq!(record.level(), log::Level::Error); + assert!( + SafeRecord(&record) + .to_string() + .contains("clock_fault=backwards") + ); + assert_eq!(session.handle.snapshot().active_requests, 0); + } + + #[test] + fn metadata_preserves_original_summary_and_public_replacement_cannot_change_private_identity() { + use crate::proxy::{TrustedProxyPolicy, normalize, strip_forwarding_headers}; + use edgezero_core::http::{HeaderValue, Request}; + use edgezero_core::ingress::{ + IngressHeadLimits, validate_normalized_ingress_parts_with_summary, + }; + + let (session, observer) = observed_session(false); + let guard = request(&session, None); + let authoritative = guard.request_id(); + let other_guard = request(&session, None); + let mut received = Request::new(edgezero_core::Body::empty()); + *received.uri_mut() = "/orders/SECRET-PATH?token=SECRET-QUERY".parse().unwrap(); + received + .headers_mut() + .insert("host", HeaderValue::from_static("direct.example:8080")); + received.headers_mut().insert( + "forwarded", + HeaderValue::from_static("for=192.0.2.99;host=spoof.example;proto=https"), + ); + received.headers_mut().insert( + "x-request-id", + HeaderValue::from_static("SECRET-VISITOR-ID"), + ); + let (mut parts, _) = received.into_parts(); + let summary = + validate_normalized_ingress_parts_with_summary(&parts, IngressHeadLimits::default()) + .unwrap(); + let peer: SocketAddr = "192.0.2.1:4567".parse().unwrap(); + let checked = normalize(&TrustedProxyPolicy::no_trust(), Some(peer), &parts); + let metadata = NativeIngressMetadata::new(authoritative, checked.clone(), summary); + assert_eq!(metadata.request_id(), authoritative); + assert_eq!(metadata.direct_peer(), Some(peer)); + assert_eq!(metadata.effective_client(), Some(peer.ip())); + assert_eq!( + metadata.effective_host().authority(), + Some("direct.example:8080") + ); + assert_eq!(metadata.effective_scheme(), Scheme::Http); + assert_eq!(metadata.client_source(), CheckedHostSource::Direct); + assert_eq!(metadata.host_source(), CheckedHostSource::Direct); + assert_eq!(metadata.scheme_source(), CheckedHostSource::Direct); + assert_eq!( + metadata.forwarding_result(), + ForwardingResult::UntrustedPeer + ); + assert_eq!(*metadata.received_normalized_head(), summary); + strip_forwarding_headers(&mut parts.headers); + let stripped = + validate_normalized_ingress_parts_with_summary(&parts, IngressHeadLimits::default()) + .unwrap(); + assert!(summary.header_bytes() > stripped.header_bytes()); + assert_eq!(summary.header_count(), stripped.header_count() + 1); + assert_eq!(*metadata.received_normalized_head(), summary); + parts.extensions.insert(metadata); + // A caller can replace its public extension with another request's copy. + parts.extensions.insert(NativeIngressMetadata::new( + other_guard.request_id(), + checked, + summary, + )); + assert_ne!( + parts + .extensions + .get::() + .unwrap() + .request_id(), + authoritative + ); + guard.finish_ingress(IngressDisposition::Aborted); + let records = lock(&observer.records); + assert_eq!(records.len(), 1); + assert_eq!(records[0].request_id, authoritative); + let safe = SafeRecord(&records[0]).to_string(); + assert!(!safe.contains("SECRET")); + assert!(!safe.contains("example")); + drop(records); + drop(other_guard); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + + #[test] + fn cancellation_unwind_releases_counts_even_if_observer_also_panics() { + let observer = RecordingObserver { + panic: true, + ..RecordingObserver::default() + }; + let session = session( + observer + .handle + .clone() + .with_request_observer(observer.clone()), + false, + ); + let (activity, connection) = session.connection(); + let unwind = catch_unwind(AssertUnwindSafe(|| { + let _guard = request(&session, Some(activity)); + panic!("cancel managed request fixture"); + })); + assert!(unwind.is_err()); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot { + active_requests: 0, + active_connections: 1, + idle_connections: 1 + } + ); + assert_eq!(lock(&observer.records).len(), 1); + drop(connection); + assert_eq!( + session.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + + #[test] + fn snapshots_remain_coherent_during_connection_request_updates() { + let handle = NativeDiagnosticsHandle::new(); + let session = session(handle.clone(), false); + let finished = Arc::new(AtomicUsize::new(0)); + let workers: Vec<_> = repeat_with(|| { + let worker_session = session.clone(); + let worker_finished = Arc::clone(&finished); + thread::spawn(move || { + for _ in 0_usize..128 { + let (activity, connection) = worker_session.connection(); + let first = request(&worker_session, Some(activity.clone())); + let second = request(&worker_session, Some(activity)); + drop(first); + drop(connection); + drop(second); + } + worker_finished.fetch_add(1, Ordering::Release); + }) + }) + .take(4) + .collect(); + while finished.load(Ordering::Acquire) < 4 { + let snapshot = handle.snapshot(); + assert!(snapshot.idle_connections <= snapshot.active_connections); + if snapshot.active_requests == 0 { + assert_eq!(snapshot.idle_connections, snapshot.active_connections); + } + thread::yield_now(); + } + for worker in workers { + worker.join().unwrap(); + } + assert_eq!(handle.snapshot(), NativeDiagnosticsSnapshot::default()); + } + + #[test] + fn metric_saturation_and_underflow_cannot_panic_during_guard_ownership() { + let saturated = session(NativeDiagnosticsHandle::new(), false); + // Inject impossible-scale telemetry to exercise arithmetic boundaries, + // without changing the checked identity sequence or real guard count. + *lock(&saturated.handle.counters) = NativeDiagnosticsSnapshot { + active_requests: u64::MAX, + active_connections: u64::MAX, + idle_connections: u64::MAX, + }; + let (_, saturated_connection) = saturated.connection(); + let saturated_request = request(&saturated, None); + assert_eq!(saturated.handle.snapshot().active_requests, u64::MAX); + assert_eq!(saturated.handle.snapshot().active_connections, u64::MAX); + assert_eq!(saturated.handle.snapshot().idle_connections, u64::MAX); + drop(saturated_request); + drop(saturated_connection); + assert_eq!(saturated.handle.snapshot().active_requests, u64::MAX - 1); + assert_eq!(saturated.handle.snapshot().active_connections, u64::MAX - 1); + assert_eq!(saturated.handle.snapshot().idle_connections, u64::MAX - 1); + + let underflow = session(NativeDiagnosticsHandle::new(), false); + let (activity, connection) = underflow.connection(); + let guard = request(&underflow, Some(activity)); + *lock(&underflow.handle.counters) = NativeDiagnosticsSnapshot::default(); + drop(guard); + drop(connection); + assert_eq!( + underflow.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + + #[test] + fn route_template_output_is_escaped_for_one_line() { + let (session, observer) = observed_session(false); + drop(request(&session, None)); + let mut record = lock(&observer.records)[0].clone(); + record.route = "/orders/{id}\n\r\t\"\\".into(); + let rendered = SafeRecord(&record).to_string(); + assert!(!rendered.contains(['\n', '\r', '\t'])); + assert!(rendered.contains("/orders/{id}\\n\\r\\t\\\"\\\\")); + } +} diff --git a/crates/edgezero-adapter-axum/src/lib.rs b/crates/edgezero-adapter-axum/src/lib.rs index 1cde330e3..0bad5b12b 100644 --- a/crates/edgezero-adapter-axum/src/lib.rs +++ b/crates/edgezero-adapter-axum/src/lib.rs @@ -9,10 +9,14 @@ pub mod context; #[cfg(feature = "axum")] pub mod dev_server; #[cfg(feature = "axum")] +pub mod diagnostics; +#[cfg(feature = "axum")] pub mod key_value_store; #[cfg(feature = "axum")] pub mod outbound; #[cfg(feature = "axum")] +pub mod proxy; +#[cfg(feature = "axum")] pub mod request; #[cfg(feature = "axum")] mod response; diff --git a/crates/edgezero-adapter-axum/src/proxy.rs b/crates/edgezero-adapter-axum/src/proxy.rs new file mode 100644 index 000000000..c77ae013f --- /dev/null +++ b/crates/edgezero-adapter-axum/src/proxy.rs @@ -0,0 +1,1894 @@ +//! Checked incoming-proxy assertions for the managed native socket boundary. + +use std::error::Error; +use std::fmt; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; +use std::str::from_utf8; + +use edgezero_core::http::{Authority, HeaderMap, HeaderName, RequestParts}; +use edgezero_core::ingress::{CheckedEffectiveHost, CheckedHostSource}; +use ipnet::{IpNet, Ipv4Net}; + +const MAX_BYTES: usize = 0x4000; +const MAX_CHAIN_SLOTS: usize = 64; +const MAX_NETWORKS: usize = 256; +const MAX_PARAMETER_SLOTS: usize = 16; +const X_HEADERS: [&str; 3] = ["x-forwarded-for", "x-forwarded-host", "x-forwarded-proto"]; + +/// The sole forwarding assertion family selected by an operator. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ForwardingHeaderFamily { + Forwarded, + XForwarded, +} + +/// Invalid trust configuration. Formatting never includes supplied values. +#[derive(Clone, Copy, Eq, PartialEq)] +pub struct ProxyTrustConfigError; + +impl fmt::Display for ProxyTrustConfigError { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("invalid_trusted_proxy_configuration") + } +} + +impl fmt::Debug for ProxyTrustConfigError { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +#[expect( + clippy::missing_trait_methods, + reason = "the source-free safe category uses Error defaults; remaining methods are deprecated or unstable" +)] +impl Error for ProxyTrustConfigError {} + +/// Immutable, bounded literal-network policy. Empty policies never enable trust. +#[derive(Clone, Eq, PartialEq)] +pub struct TrustedProxyPolicy { + family: Option, + networks: Vec, +} + +impl fmt::Debug for TrustedProxyPolicy { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TrustedProxyPolicy") + .field("family", &self.family) + .field("network_count", &self.networks.len()) + .finish() + } +} + +impl Default for TrustedProxyPolicy { + #[inline] + fn default() -> Self { + Self::no_trust() + } +} + +impl TrustedProxyPolicy { + /// Parses captured standalone settings. Unknown supplied families always fail. + /// + /// # Errors + /// Rejects invalid settings and nonempty networks without a selected family. + #[inline] + pub fn from_values( + cidrs: Option<&str>, + family: Option<&str>, + ) -> Result { + let selected_family = family + .map(|value| match value.trim_matches([' ', '\t']) { + "forwarded" => Ok(ForwardingHeaderFamily::Forwarded), + "x-forwarded" => Ok(ForwardingHeaderFamily::XForwarded), + _ => Err(ProxyTrustConfigError), + }) + .transpose()?; + let raw = cidrs.unwrap_or(""); + if raw.len() > MAX_BYTES + || raw + .bytes() + .any(|byte| byte.is_ascii_control() && byte != b'\t') + { + return Err(ProxyTrustConfigError); + } + if raw.trim_matches([' ', '\t']).is_empty() { + return Ok(Self { + family: selected_family, + networks: Vec::new(), + }); + } + Self::new( + selected_family.ok_or(ProxyTrustConfigError)?, + raw.split(','), + ) + } + + /// Validates literal IPs/CIDRs without exposing the network-library type. + /// + /// # Errors + /// Rejects invalid literals, parser-budget excesses and whole-family trust. + #[inline] + pub fn new( + family: ForwardingHeaderFamily, + networks: I, + ) -> Result + where + I: IntoIterator, + S: AsRef, + { + let mut normalized = Vec::new(); + let mut bytes = 0_usize; + for (index, entry) in networks.into_iter().enumerate() { + if index >= MAX_NETWORKS { + return Err(ProxyTrustConfigError); + } + let raw = entry.as_ref(); + bytes = bytes + .checked_add(raw.len()) + .and_then(|size| size.checked_add(usize::from(index != 0))) + .filter(|size| *size <= MAX_BYTES) + .ok_or(ProxyTrustConfigError)?; + if raw + .bytes() + .any(|byte| byte.is_ascii_control() && byte != b'\t') + { + return Err(ProxyTrustConfigError); + } + let literal = raw.trim_matches([' ', '\t']); + let parsed = literal + .parse::() + .or_else(|_error| literal.parse::().map(IpNet::from)) + .map_err(|_error| ProxyTrustConfigError)?; + let network = normalize_network(parsed)?; + if !normalized.contains(&network) { + normalized.push(network); + } + } + if covers_family(&normalized, false) || covers_family(&normalized, true) { + return Err(ProxyTrustConfigError); + } + Ok(Self { + family: Some(family), + networks: normalized, + }) + } + + #[must_use] + #[inline] + pub fn no_trust() -> Self { + Self { + family: None, + networks: Vec::new(), + } + } + + fn trusts(&self, address: IpAddr) -> bool { + let canonical = canonical_ip(address); + self.networks + .iter() + .any(|network| network.contains(&canonical)) + } +} + +/// A bounded assertion outcome, never a raw header or source error. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ForwardingResult { + Absent, + Accepted, + Malformed, + Partial, + UnresolvedChain, + UntrustedPeer, +} + +impl ForwardingResult { + #[must_use] + #[inline] + pub const fn as_str(self) -> &'static str { + match self { + Self::Absent => "absent", + Self::Accepted => "accepted", + Self::Partial => "partial", + Self::UntrustedPeer => "untrusted_peer", + Self::Malformed => "malformed", + Self::UnresolvedChain => "unresolved_chain", + } + } +} + +/// The checked public scheme, independent of the client-address chain. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Scheme { + Http, + Https, +} + +impl Scheme { + #[must_use] + #[inline] + pub const fn as_str(self) -> &'static str { + match self { + Self::Http => "http", + Self::Https => "https", + } + } +} + +/// Checked ingress facts, not host authorization or authenticated visitor identity. +#[derive(Clone)] +pub struct CheckedProxyMetadata { + client_source: CheckedHostSource, + direct_peer: Option, + effective_client: Option, + effective_host: CheckedEffectiveHost, + effective_scheme: Scheme, + forwarding_result: ForwardingResult, + scheme_source: CheckedHostSource, +} + +impl fmt::Debug for CheckedProxyMetadata { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CheckedProxyMetadata") + .field("forwarding_result", &self.forwarding_result) + .finish_non_exhaustive() + } +} + +impl CheckedProxyMetadata { + #[must_use] + #[inline] + pub fn client_source(&self) -> CheckedHostSource { + self.client_source + } + + #[must_use] + #[inline] + pub fn direct_peer(&self) -> Option { + self.direct_peer + } + + #[must_use] + #[inline] + pub fn effective_client(&self) -> Option { + self.effective_client + } + + #[must_use] + #[inline] + pub fn effective_host(&self) -> &CheckedEffectiveHost { + &self.effective_host + } + + #[must_use] + #[inline] + pub fn effective_scheme(&self) -> Scheme { + self.effective_scheme + } + + #[must_use] + #[inline] + pub fn forwarding_result(&self) -> ForwardingResult { + self.forwarding_result + } + + #[must_use] + #[inline] + pub fn scheme_source(&self) -> CheckedHostSource { + self.scheme_source + } +} + +#[derive(Default)] +struct ParsedAssertion { + host: Option, + nodes: Vec>, + scheme: Option, +} + +fn canonical_ip(address: IpAddr) -> IpAddr { + match address { + IpAddr::V6(v6) => v6.to_ipv4_mapped().map_or(IpAddr::V6(v6), IpAddr::V4), + IpAddr::V4(v4) => IpAddr::V4(v4), + } +} + +fn normalize_network(network: IpNet) -> Result { + let truncated = network.trunc(); + if truncated.prefix_len() == 0 { + return Err(ProxyTrustConfigError); + } + if let IpNet::V6(v6) = truncated { + let mapped_start = Ipv6Addr::from(0xffff_u128 << 32_u32); + if v6.prefix_len() < 96 && v6.contains(&mapped_start) { + // A shorter overlapping prefix contains every mapped IPv4 address. + return Err(ProxyTrustConfigError); + } + if let Some(v4) = v6.network().to_ipv4_mapped() { + let prefix = v6 + .prefix_len() + .checked_sub(96) + .ok_or(ProxyTrustConfigError)?; + if prefix == 0 { + return Err(ProxyTrustConfigError); + } + return Ipv4Net::new(v4, prefix) + .map(IpNet::V4) + .map_err(|_error| ProxyTrustConfigError); + } + } + Ok(truncated) +} + +#[expect( + clippy::arithmetic_side_effects, + reason = "the maximum endpoint returns before end + 1; IPv4 endpoints fit u128 and the IPv6 maximum is handled explicitly" +)] +fn covers_family(networks: &[IpNet], ipv6: bool) -> bool { + let mut intervals: Vec<(u128, u128)> = networks + .iter() + .filter_map(|network| match network { + IpNet::V4(v4) if !ipv6 => Some(( + u128::from(u32::from(v4.network())), + u128::from(u32::from(v4.broadcast())), + )), + IpNet::V6(v6) if ipv6 => Some((u128::from(v6.network()), u128::from(v6.broadcast()))), + IpNet::V4(_) | IpNet::V6(_) => None, + }) + .collect(); + intervals.sort_unstable(); + let maximum = if ipv6 { + u128::MAX + } else { + u128::from(u32::MAX) + }; + let mut next = 0_u128; + for (start, end) in intervals { + if start > next { + return false; + } + if end == maximum { + return true; + } + next = next.max(end + 1); + } + false +} + +pub(crate) fn normalize( + policy: &TrustedProxyPolicy, + peer: Option, + parts: &RequestParts, +) -> CheckedProxyMetadata { + let direct_authority = match parts.uri.authority() { + Some(authority) => parse_authority(authority.as_str().as_bytes()).ok(), + None => parts + .headers + .get("host") + .and_then(|value| parse_authority(value.as_bytes()).ok()), + }; + let host_source = if direct_authority.is_some() { + CheckedHostSource::Direct + } else { + CheckedHostSource::Unavailable + }; + let mut metadata = CheckedProxyMetadata { + direct_peer: peer, + effective_client: peer.map(|socket| canonical_ip(socket.ip())), + effective_host: CheckedEffectiveHost::from_ingress(direct_authority, host_source), + effective_scheme: Scheme::Http, + client_source: if peer.is_some() { + CheckedHostSource::Direct + } else { + CheckedHostSource::Unavailable + }, + scheme_source: CheckedHostSource::Direct, + forwarding_result: ForwardingResult::Absent, + }; + let present = match policy.family { + Some(ForwardingHeaderFamily::Forwarded) => parts.headers.contains_key("forwarded"), + Some(ForwardingHeaderFamily::XForwarded) => X_HEADERS + .iter() + .any(|name| parts.headers.contains_key(*name)), + None => parts.headers.keys().any(is_forwarding_header), + }; + if !present { + return metadata; + } + if !peer.is_some_and(|socket| policy.trusts(socket.ip())) { + metadata.forwarding_result = ForwardingResult::UntrustedPeer; + return metadata; + } + let (parse_result, source) = match policy.family { + Some(ForwardingHeaderFamily::Forwarded) => ( + parse_forwarded(&parts.headers), + CheckedHostSource::TrustedForwarded, + ), + Some(ForwardingHeaderFamily::XForwarded) => ( + parse_x_forwarded(&parts.headers), + CheckedHostSource::TrustedXForwarded, + ), + None => return metadata, + }; + let Ok(parsed) = parse_result else { + metadata.forwarding_result = ForwardingResult::Malformed; + return metadata; + }; + let mut accepted_client = false; + let mut unresolved = false; + if !parsed.nodes.is_empty() { + unresolved = true; + for predecessor in parsed.nodes.iter().rev() { + let Some(address) = predecessor else { + break; + }; + if !policy.trusts(*address) { + metadata.effective_client = Some(*address); + metadata.client_source = source; + accepted_client = true; + unresolved = false; + break; + } + } + } + let accepted_host = parsed.host.is_some(); + let accepted_scheme = parsed.scheme.is_some(); + if let Some(host) = parsed.host { + metadata.effective_host = CheckedEffectiveHost::from_ingress(Some(host), source); + } + if let Some(scheme) = parsed.scheme { + metadata.effective_scheme = scheme; + metadata.scheme_source = source; + } + metadata.forwarding_result = if unresolved { + ForwardingResult::UnresolvedChain + } else if accepted_client && accepted_host && accepted_scheme { + ForwardingResult::Accepted + } else if accepted_client || accepted_host || accepted_scheme { + ForwardingResult::Partial + } else { + ForwardingResult::Absent + }; + metadata +} + +fn is_forwarding_header(name: &HeaderName) -> bool { + name.as_str() == "forwarded" || name.as_str().starts_with("x-forwarded-") +} + +/// Removes all raw forwarding assertions, including unselected extensions. +pub(crate) fn strip_forwarding_headers(headers: &mut HeaderMap) { + while let Some(name) = headers + .keys() + .find(|name| is_forwarding_header(name)) + .cloned() + { + headers.remove(name); + } +} + +// Input is preflighted before any parsing scratch or retained nodes are allocated. +fn preflight(headers: &HeaderMap, names: &[&str]) -> Result<(), ()> { + let mut bytes = 0_usize; + for name in names { + for (index, value) in headers.get_all(*name).iter().enumerate() { + bytes = bytes + .checked_add(value.as_bytes().len()) + .and_then(|size| size.checked_add(usize::from(index != 0))) + .filter(|size| *size <= MAX_BYTES) + .ok_or(())?; + } + } + Ok(()) +} + +#[expect( + clippy::indexing_slicing, + reason = "both endpoints are positions found in this slice, and first non-OWS cannot follow last non-OWS" +)] +fn trim_ows(value: &[u8]) -> &[u8] { + let Some(start) = value.iter().position(|byte| !matches!(byte, b' ' | b'\t')) else { + return &[]; + }; + let Some(end) = value.iter().rposition(|byte| !matches!(byte, b' ' | b'\t')) else { + return &[]; + }; + &value[start..=end] +} + +// Slots include empty members; repeated-line boundaries join rather than add slots. +#[expect( + clippy::arithmetic_side_effects, + clippy::indexing_slicing, + reason = "private callers cap slots at 64 or 16; delimiter indices come from the same preflighted slice and start advances at most to its length" +)] +fn split_slots<'input>( + value: &'input [u8], + delimiter: u8, + slots: &mut usize, + limit: usize, + quoting: bool, +) -> Result, ()> { + let mut result = Vec::new(); + let mut start = 0; + let mut quoted = false; + let mut escaped = false; + *slots += 1; + if *slots > limit { + return Err(()); + } + for (index, byte) in value.iter().copied().enumerate() { + if quoting && escaped { + escaped = false; + } else if quoting && quoted && byte == b'\\' { + escaped = true; + } else if quoting && byte == b'"' { + quoted = !quoted; + } else if !quoted && byte == delimiter { + *slots += 1; + if *slots > limit { + return Err(()); + } + result.push(&value[start..index]); + start = index + 1; + } else { + // An ordinary byte does not change delimiter/quoting state. + } + } + if quoted || escaped { + return Err(()); + } + result.push(&value[start..]); + Ok(result) +} + +fn token_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || b"!#$%&'*+-.^_`|~".contains(&byte) +} + +// Unknown extensions are validated without allocating or retaining their payload. +#[expect( + clippy::arithmetic_side_effects, + clippy::indexing_slicing, + reason = "the quoted branch checks length >= 2 before excluding the two quote bytes" +)] +fn validate_value(value: &[u8]) -> Result { + if value.first() != Some(&b'"') { + return if !value.is_empty() && value.iter().copied().all(token_byte) { + Ok(false) + } else { + Err(()) + }; + } + if value.len() < 2 || value.last() != Some(&b'"') { + return Err(()); + } + let mut escaped = false; + for byte in value[1..value.len() - 1].iter().copied() { + if escaped { + if !(byte == b'\t' || byte == b' ' || (0x21..=0x7e).contains(&byte) || byte >= 0x80) { + return Err(()); + } + escaped = false; + } else if byte == b'\\' { + escaped = true; + } else if !(byte == b'\t' + || byte == b' ' + || byte == 0x21 + || (0x23..=0x5b).contains(&byte) + || (0x5d..=0x7e).contains(&byte) + || byte >= 0x80) + { + return Err(()); + } else { + // RFC qdtext, including obs-text, requires no state change. + } + } + if escaped { Err(()) } else { Ok(true) } +} + +#[expect( + clippy::arithmetic_side_effects, + clippy::indexing_slicing, + reason = "only called after validate_value; quoted input has two validated delimiters and fits the assertion byte budget" +)] +fn decode_value(value: &[u8], quoted: bool) -> Vec { + if !quoted { + return value.to_vec(); + } + let mut decoded = Vec::with_capacity(value.len() - 2); + let mut escaped = false; + for byte in value[1..value.len() - 1].iter().copied() { + if !escaped && byte == b'\\' { + escaped = true; + } else { + decoded.push(byte); + escaped = false; + } + } + decoded +} + +#[expect( + clippy::arithmetic_side_effects, + clippy::indexing_slicing, + reason = "equals is a position found within the same nonempty parameter slice, so both name/value ranges and equals + 1 are bounded" +)] +fn parse_forwarded(headers: &HeaderMap) -> Result { + preflight(headers, &["forwarded"])?; + let mut parsed = ParsedAssertion::default(); + let mut slots = 0; + for field in headers.get_all("forwarded") { + for raw_element in split_slots(field.as_bytes(), b',', &mut slots, MAX_CHAIN_SLOTS, true)? { + let element = trim_ows(raw_element); + if element.is_empty() { + continue; + } + let mut parameter_slots = 0; + let mut names: Vec<&[u8]> = Vec::new(); + let mut node = None; + let mut host = None; + let mut scheme = None; + for raw_parameter in split_slots( + element, + b';', + &mut parameter_slots, + MAX_PARAMETER_SLOTS, + true, + )? { + let parameter = trim_ows(raw_parameter); + if parameter.is_empty() { + continue; + } + let equals = parameter.iter().position(|byte| *byte == b'=').ok_or(())?; + let name = ¶meter[..equals]; + if name.is_empty() + || !name.iter().copied().all(token_byte) + || names + .iter() + .any(|previous| previous.eq_ignore_ascii_case(name)) + { + return Err(()); + } + names.push(name); + let value = ¶meter[equals + 1..]; + let quoted = validate_value(value)?; + if name.eq_ignore_ascii_case(b"for") || name.eq_ignore_ascii_case(b"by") { + let decoded = decode_value(value, quoted); + let address = parse_node(&decoded, false)?; + if name.eq_ignore_ascii_case(b"for") { + node = address; + } + } else if name.eq_ignore_ascii_case(b"host") { + host = Some(parse_authority(&decode_value(value, quoted))?); + } else if name.eq_ignore_ascii_case(b"proto") { + scheme = Some(parse_scheme(&decode_value(value, quoted))?); + } else { + // Unknown payloads are syntactically checked, never retained. + } + } + if names.is_empty() { + continue; + } + parsed.nodes.push(node); + // A new substantive element replaces, rather than borrows, final-hop facts. + parsed.host = host; + parsed.scheme = scheme; + } + } + Ok(parsed) +} + +fn singleton<'headers>( + headers: &'headers HeaderMap, + name: &str, +) -> Result, ()> { + let mut values = headers.get_all(name).iter(); + let value = values.next().map(|field| trim_ows(field.as_bytes())); + if values.next().is_some() || value.is_some_and(|bytes| bytes.contains(&b',')) { + return Err(()); + } + Ok(value) +} + +fn parse_x_forwarded(headers: &HeaderMap) -> Result { + preflight(headers, &X_HEADERS)?; + let mut parsed = ParsedAssertion::default(); + let mut slots = 0; + for value in headers.get_all("x-forwarded-for") { + for node in split_slots(value.as_bytes(), b',', &mut slots, MAX_CHAIN_SLOTS, false)? { + // XFF has no RFC empty-list tolerance: never skip a predecessor slot. + parsed.nodes.push(parse_node(trim_ows(node), true)?); + } + } + parsed.host = singleton(headers, "x-forwarded-host")? + .map(parse_authority) + .transpose()?; + parsed.scheme = singleton(headers, "x-forwarded-proto")? + .map(parse_scheme) + .transpose()?; + Ok(parsed) +} + +fn parse_scheme(value: &[u8]) -> Result { + if value.eq_ignore_ascii_case(b"http") { + Ok(Scheme::Http) + } else if value.eq_ignore_ascii_case(b"https") { + Ok(Scheme::Https) + } else { + Err(()) + } +} + +fn valid_obfuscated(value: &str) -> bool { + value.starts_with('_') + && value.len() > 1 + && value + .as_bytes() + .iter() + .skip(1) + .all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(byte)) +} + +fn validate_node_port(port: &str) -> Result<(), ()> { + if valid_obfuscated(port) + || (!port.is_empty() + && port.bytes().all(|byte| byte.is_ascii_digit()) + && port.parse::().is_ok()) + { + Ok(()) + } else { + Err(()) + } +} + +#[expect( + clippy::arithmetic_side_effects, + clippy::string_slice, + reason = "input is checked ASCII before slicing; closing is the bracket position found in that same string and its successor is at most the string length" +)] +fn parse_node(bytes: &[u8], allow_bare_ipv6: bool) -> Result, ()> { + let value = from_utf8(bytes).map_err(|_error| ())?; + if value.is_empty() || !value.is_ascii() { + return Err(()); + } + if let Some(bracketed) = value.strip_prefix('[') { + let closing = bracketed.find(']').ok_or(())?; + let address = bracketed[..closing] + .parse::() + .map_err(|_error| ())?; + let remainder = &bracketed[closing + 1..]; + if !remainder.is_empty() { + validate_node_port(remainder.strip_prefix(':').ok_or(())?)?; + } + return Ok(Some(canonical_ip(IpAddr::V6(address)))); + } + if allow_bare_ipv6 && let Ok(address) = value.parse::() { + return Ok(Some(canonical_ip(IpAddr::V6(address)))); + } + let (node, optional_port) = value + .split_once(':') + .map_or((value, None), |(name, number)| (name, Some(number))); + if let Some(port) = optional_port { + validate_node_port(port)?; + } + if node.eq_ignore_ascii_case("unknown") || valid_obfuscated(node) { + return Ok(None); + } + node.parse::() + .map(|address| Some(IpAddr::V4(address))) + .map_err(|_error| ()) +} + +#[expect( + clippy::arithmetic_side_effects, + clippy::string_slice, + reason = "input is checked ASCII before slicing; closing is found in the same bracketed authority and closing + 1 cannot exceed its length" +)] +fn parse_authority(bytes: &[u8]) -> Result { + let value = from_utf8(bytes).map_err(|_error| ())?; + if value.is_empty() + || !value.is_ascii() + || value.bytes().any(|byte| { + byte.is_ascii_whitespace() || byte.is_ascii_control() || b"@/?#\\,".contains(&byte) + }) + { + return Err(()); + } + let authority = value.parse::().map_err(|_error| ())?; + if authority.host().is_empty() { + return Err(()); + } + let optional_port = if let Some(bracketed) = value.strip_prefix('[') { + let closing = bracketed.find(']').ok_or(())?; + bracketed[..closing] + .parse::() + .map_err(|_error| ())?; + let suffix = &bracketed[closing + 1..]; + if suffix.is_empty() { + None + } else { + Some(suffix.strip_prefix(':').ok_or(())?) + } + } else { + let mut components = value.split(':'); + let host = components.next().ok_or(())?; + if host.is_empty() || host.contains(['[', ']']) { + return Err(()); + } + let port = components.next(); + if components.next().is_some() { + return Err(()); + } + port + }; + if let Some(port) = optional_port + && (port.is_empty() + || !port.bytes().all(|byte| byte.is_ascii_digit()) + || port.parse::().is_err()) + { + return Err(()); + } + Ok(authority) +} + +#[cfg(test)] +mod tests { + use std::fmt::Write as _; + use std::iter::{empty, repeat, repeat_n}; + + use super::*; + use edgezero_core::body::Body; + use edgezero_core::http::{HeaderName, HeaderValue, request_builder}; + + const PEER: &str = "10.0.0.2:1234"; + const CLIENT: &str = "198.51.100.7"; + + fn policy(family: ForwardingHeaderFamily) -> TrustedProxyPolicy { + TrustedProxyPolicy::new(family, ["10.0.0.0/24"]).unwrap() + } + + fn parts(headers: &[(&str, &str)]) -> RequestParts { + let mut request = request_builder().uri("/").body(Body::empty()).unwrap(); + request + .headers_mut() + .insert("host", HeaderValue::from_static("direct.example:8080")); + for (name, value) in headers { + request.headers_mut().append( + name.parse::().unwrap(), + HeaderValue::from_str(value).unwrap(), + ); + } + request.into_parts().0 + } + + fn checked(family: ForwardingHeaderFamily, headers: &[(&str, &str)]) -> CheckedProxyMetadata { + normalize( + &policy(family), + Some(PEER.parse().unwrap()), + &parts(headers), + ) + } + + fn direct_fallback(metadata: &CheckedProxyMetadata, result: ForwardingResult) { + assert_eq!(metadata.forwarding_result(), result); + assert_eq!(metadata.direct_peer(), Some(PEER.parse().unwrap())); + assert_eq!( + metadata.effective_client(), + Some("10.0.0.2".parse().unwrap()) + ); + assert_eq!(metadata.client_source(), CheckedHostSource::Direct); + assert_eq!( + metadata.effective_host().authority(), + Some("direct.example:8080") + ); + assert_eq!( + metadata.effective_host().source(), + CheckedHostSource::Direct + ); + assert_eq!(metadata.effective_scheme(), Scheme::Http); + assert_eq!(metadata.scheme_source(), CheckedHostSource::Direct); + } + + fn assert_client(metadata: &CheckedProxyMetadata, client: &str) { + assert_eq!(metadata.effective_client(), Some(client.parse().unwrap())); + } + + #[test] + fn defaults_and_strict_configuration() { + let default_policy = TrustedProxyPolicy::default(); + assert_eq!(default_policy, TrustedProxyPolicy::no_trust()); + assert!(!default_policy.trusts("127.0.0.1".parse().unwrap())); + assert_eq!( + TrustedProxyPolicy::from_values(None, None).unwrap(), + default_policy + ); + for raw in ["", " ", "\t "] { + assert!( + TrustedProxyPolicy::from_values(Some(raw), None) + .unwrap() + .networks + .is_empty() + ); + } + for family in ["forwarded", "x-forwarded"] { + assert!( + TrustedProxyPolicy::from_values(None, Some(family)) + .unwrap() + .networks + .is_empty() + ); + } + assert!( + TrustedProxyPolicy::new(ForwardingHeaderFamily::Forwarded, empty::<&str>()) + .unwrap() + .networks + .is_empty() + ); + TrustedProxyPolicy::from_values(Some("10.0.0.1"), None).unwrap_err(); + for family in ["", "auto", "Forwarded", "forwarded\n", "sentinel-secret"] { + TrustedProxyPolicy::from_values(None, Some(family)).unwrap_err(); + } + for cidrs in [ + "all", + "private", + "localhost", + "example.com", + "10.0.0.1,", + ",10.0.0.1", + "10.0.0.1,,10.0.0.2", + "10.0.0.1/33", + "\n", + "10.0.0.1\r", + "10.0.0.1\0", + ] { + TrustedProxyPolicy::from_values(Some(cidrs), Some("forwarded")).expect_err(cidrs); + } + TrustedProxyPolicy::from_values(Some(" \t10.0.0.1 , 10.0.0.2/32\t"), Some("forwarded")) + .unwrap(); + TrustedProxyPolicy::new(ForwardingHeaderFamily::Forwarded, [""]).unwrap_err(); + } + + #[test] + fn configuration_errors_are_safe() { + let error = TrustedProxyPolicy::from_values(Some("sentinel-secret"), Some("forwarded")) + .unwrap_err(); + assert_eq!(error.to_string(), "invalid_trusted_proxy_configuration"); + assert_eq!(format!("{error:?}"), error.to_string()); + let policy = policy(ForwardingHeaderFamily::Forwarded); + assert!(!format!("{policy:?}").contains("10.0.0")); + } + + #[test] + fn configuration_entry_and_byte_bounds_precede_deduplication() { + let family = ForwardingHeaderFamily::Forwarded; + assert_eq!( + TrustedProxyPolicy::new(family, repeat_n("10.0.0.1", MAX_NETWORKS)) + .unwrap() + .networks + .len(), + 1 + ); + TrustedProxyPolicy::new(family, repeat("10.0.0.1")).unwrap_err(); + let exact = format!("{}127.0.0.1", " ".repeat(MAX_BYTES - "127.0.0.1".len())); + TrustedProxyPolicy::new(family, [&exact]).unwrap(); + TrustedProxyPolicy::from_values(Some(&exact), Some("forwarded")).unwrap(); + let over = format!(" {exact}"); + TrustedProxyPolicy::new(family, [&over]).unwrap_err(); + TrustedProxyPolicy::from_values(Some(&over), Some("forwarded")).unwrap_err(); + let second = "10.0.0.2"; + let first = format!( + "{}10.0.0.1", + " ".repeat(MAX_BYTES - 1 - second.len() - "10.0.0.1".len()) + ); + TrustedProxyPolicy::new(family, [first.as_str(), second]).unwrap(); + TrustedProxyPolicy::new(family, [format!(" {first}"), second.to_owned()]).unwrap_err(); + } + + #[test] + fn cidrs_truncate_deduplicate_and_project_mapped_ipv6() { + let policy = TrustedProxyPolicy::new( + ForwardingHeaderFamily::Forwarded, + [ + "192.0.2.19/24", + "192.0.2.0/24", + "::ffff:192.0.2.99/120", + "2001:db8::9/32", + ], + ) + .unwrap(); + assert_eq!(policy.networks.len(), 2); + for address in [ + "192.0.2.0", + "192.0.2.255", + "::ffff:192.0.2.7", + "2001:db8::ffff", + ] { + assert!(policy.trusts(address.parse().unwrap()), "{address}"); + } + for address in [ + "192.0.3.0", + "::ffff:192.0.3.1", + "2001:db9::", + "127.0.0.1", + "10.0.0.1", + ] { + assert!(!policy.trusts(address.parse().unwrap()), "{address}"); + } + let mapped = + TrustedProxyPolicy::new(ForwardingHeaderFamily::Forwarded, ["::ffff:192.0.2.7"]) + .unwrap(); + assert!(mapped.trusts("192.0.2.7".parse().unwrap())); + assert!(!mapped.trusts("192.0.2.8".parse().unwrap())); + } + + #[test] + fn universal_and_mapped_crossing_policies_are_rejected() { + for entries in [ + vec!["0.0.0.0/0"], + vec!["::/0"], + vec!["0.0.0.0/1", "128.0.0.0/1"], + vec!["0.0.0.0/2", "64.0.0.0/2", "128.0.0.0/1"], + vec!["::/1", "8000::/1"], + vec!["::ffff:0.0.0.0/96"], + vec!["::fffe:0:0/95"], + vec!["::/80"], + vec!["0.0.0.0/1", "::ffff:128.0.0.0/97"], + ] { + TrustedProxyPolicy::new(ForwardingHeaderFamily::Forwarded, entries).unwrap_err(); + } + // Exercise interval merging's maximum endpoint without enumerating addresses. + let v6 = ["::/1".parse().unwrap(), "8000::/1".parse().unwrap()]; + assert!(covers_family(&v6, true)); + assert!(!covers_family(&["ffff::/16".parse().unwrap()], true)); + assert!(!covers_family( + &["0.0.0.0/1".parse().unwrap(), "192.0.0.0/2".parse().unwrap()], + false + )); + TrustedProxyPolicy::new( + ForwardingHeaderFamily::Forwarded, + ["::fffd:0:0/96", "ffff::/16"], + ) + .unwrap(); + } + + #[test] + #[expect( + clippy::shadow_unrelated, + reason = "the same checked variable describes separate missing-peer and nonmatching-peer fixture observations" + )] + fn absent_disabled_untrusted_and_missing_peer_never_parse_assertions() { + let assertion = parts(&[ + ("forwarded", "invalid sentinel-secret"), + ("x-forwarded-host", "evil.example"), + ]); + for policy in [ + TrustedProxyPolicy::no_trust(), + policy(ForwardingHeaderFamily::Forwarded), + ] { + let peer: SocketAddr = "127.0.0.1:10".parse().unwrap(); + let checked = normalize(&policy, Some(peer), &assertion); + assert_eq!(checked.forwarding_result(), ForwardingResult::UntrustedPeer); + assert_eq!(checked.effective_client(), Some(peer.ip())); + assert_eq!( + checked.effective_host().authority(), + Some("direct.example:8080") + ); + let checked = normalize(&policy, None, &assertion); + assert_eq!(checked.effective_client(), None); + assert_eq!(checked.client_source(), CheckedHostSource::Unavailable); + assert_eq!(checked.forwarding_result(), ForwardingResult::UntrustedPeer); + } + direct_fallback( + &checked(ForwardingHeaderFamily::Forwarded, &[]), + ForwardingResult::Absent, + ); + let empty = + TrustedProxyPolicy::new(ForwardingHeaderFamily::Forwarded, empty::<&str>()).unwrap(); + assert_eq!( + normalize(&empty, Some(PEER.parse().unwrap()), &assertion).forwarding_result(), + ForwardingResult::UntrustedPeer + ); + } + + #[test] + #[expect( + clippy::shadow_unrelated, + reason = "paired metadata observations compare Forwarded and X-Forwarded family selection with the same assertions" + )] + fn family_selection_never_borrows_or_validates_other_family() { + let metadata = checked( + ForwardingHeaderFamily::Forwarded, + &[ + ("forwarded", "for=198.51.100.7"), + ("x-forwarded-host", "evil.example"), + ("x-forwarded-proto", "garbage"), + ], + ); + assert_client(&metadata, CLIENT); + assert_eq!( + metadata.effective_host().authority(), + Some("direct.example:8080") + ); + assert_eq!(metadata.effective_scheme(), Scheme::Http); + assert_eq!(metadata.forwarding_result(), ForwardingResult::Partial); + let metadata = checked( + ForwardingHeaderFamily::XForwarded, + &[ + ("forwarded", "malformed"), + ("x-forwarded-host", "public.example"), + ("x-forwarded-proto", "https"), + ], + ); + assert_eq!(metadata.forwarding_result(), ForwardingResult::Partial); + assert_client(&metadata, "10.0.0.2"); + assert_eq!( + metadata.effective_host().authority(), + Some("public.example") + ); + assert_eq!(metadata.effective_scheme(), Scheme::Https); + direct_fallback( + &checked( + ForwardingHeaderFamily::Forwarded, + &[("x-forwarded-for", CLIENT)], + ), + ForwardingResult::Absent, + ); + } + + #[test] + #[expect( + clippy::shadow_unrelated, + reason = "paired chain fixtures compare a final asserted host/scheme against final-hop omission without changing assertion names" + )] + fn repeated_forwarded_order_and_final_proxy_host_scheme() { + let metadata = checked( + ForwardingHeaderFamily::Forwarded, + &[ + ( + "forwarded", + "for=203.0.113.99;host=attacker.example;proto=http, for=198.51.100.7", + ), + ("forwarded", "for=10.0.0.1;HOST=public.example;PROTO=HTTPS"), + ], + ); + assert_client(&metadata, CLIENT); + assert_eq!(metadata.forwarding_result(), ForwardingResult::Accepted); + assert_eq!( + metadata.effective_host().authority(), + Some("public.example") + ); + assert_eq!( + metadata.effective_host().source(), + CheckedHostSource::TrustedForwarded + ); + assert_eq!(metadata.effective_scheme(), Scheme::Https); + assert_eq!( + metadata.scheme_source(), + CheckedHostSource::TrustedForwarded + ); + assert_eq!( + metadata.client_source(), + CheckedHostSource::TrustedForwarded + ); + let metadata = checked( + ForwardingHeaderFamily::Forwarded, + &[( + "forwarded", + "for=198.51.100.7;host=earlier.example;proto=https, for=10.0.0.1", + )], + ); + assert_eq!( + metadata.effective_host().authority(), + Some("direct.example:8080") + ); + assert_eq!(metadata.effective_scheme(), Scheme::Http); + } + + #[test] + fn forwarded_rfc_empty_slots_and_unknown_extensions() { + for value in [",,", "", " ; ; ", " , ; ; , "] { + direct_fallback( + &checked(ForwardingHeaderFamily::Forwarded, &[("forwarded", value)]), + ForwardingResult::Absent, + ); + } + let metadata = checked( + ForwardingHeaderFamily::Forwarded, + &[( + "forwarded", + " , ;for=198.51.100.7;;host=public.example;proto=https;ext=\"a,b;c\\\"d\\\\e\"; , ", + )], + ); + assert_eq!(metadata.forwarding_result(), ForwardingResult::Accepted); + assert_client(&metadata, CLIENT); + let mut parts = parts(&[]); + parts.headers.append( + "forwarded", + HeaderValue::from_bytes( + b"for=198.51.100.7;host=public.example;proto=https;ext=\"\x80\xff\\\xfe\"", + ) + .unwrap(), + ); + assert_eq!( + normalize( + &policy(ForwardingHeaderFamily::Forwarded), + Some(PEER.parse().unwrap()), + &parts + ) + .forwarding_result(), + ForwardingResult::Accepted + ); + } + + #[test] + fn quoted_values_unescape_before_recognized_validation() { + let metadata = checked( + ForwardingHeaderFamily::Forwarded, + &[( + "forwarded", + "for=\"198.51.100.7\";host=\"public\\.example:443\";proto=\"h\\ttps\"", + )], + ); + assert_eq!(metadata.forwarding_result(), ForwardingResult::Accepted); + assert_eq!( + metadata.effective_host().authority(), + Some("public.example:443") + ); + for value in [ + "for=\"198.51.100.7", + "for=198.51.100.7;ext=\"a\\\"", + "for=198.51.100.7;ext=\"a\"x", + "for=198.51.100.7;ext=a b", + "for =198.51.100.7", + "for= 198.51.100.7", + "for=198.51.100.7;ext=", + ] { + direct_fallback( + &checked(ForwardingHeaderFamily::Forwarded, &[("forwarded", value)]), + ForwardingResult::Malformed, + ); + } + } + + #[test] + #[expect( + clippy::shadow_unrelated, + reason = "each node table row runs identical metadata assertions against both family-specific results" + )] + fn recognized_nodes_and_ports_follow_family_grammar() { + for (value, expected) in [ + ("198.51.100.7", Some(CLIENT)), + ("198.51.100.7:65535", Some(CLIENT)), + ("198.51.100.7:0", Some(CLIENT)), + ("198.51.100.7:_masked", Some(CLIENT)), + ("[2001:db8::7]", Some("2001:db8::7")), + ("[2001:db8::7]:443", Some("2001:db8::7")), + ("[2001:db8::7]:_masked", Some("2001:db8::7")), + ("[::ffff:198.51.100.7]:80", Some(CLIENT)), + ("unknown", None), + ("UNKNOWN:80", None), + ("_hidden:_port", None), + ] { + let metadata = checked( + ForwardingHeaderFamily::Forwarded, + &[( + "forwarded", + &format!("for=\"{value}\";host=public.example;proto=https"), + )], + ); + let metadata_x = checked( + ForwardingHeaderFamily::XForwarded, + &[ + ("x-forwarded-for", value), + ("x-forwarded-host", "public.example"), + ("x-forwarded-proto", "https"), + ], + ); + for metadata in [&metadata, &metadata_x] { + if let Some(address) = expected { + assert_client(metadata, address); + assert_eq!( + metadata.forwarding_result(), + ForwardingResult::Accepted, + "{value}" + ); + } else { + assert_client(metadata, "10.0.0.2"); + assert_eq!( + metadata.forwarding_result(), + ForwardingResult::UnresolvedChain, + "{value}" + ); + assert_eq!(metadata.effective_scheme(), Scheme::Https); + } + } + } + let bare_v6 = checked( + ForwardingHeaderFamily::XForwarded, + &[("x-forwarded-for", "2001:db8::7")], + ); + assert_client(&bare_v6, "2001:db8::7"); + direct_fallback( + &checked( + ForwardingHeaderFamily::Forwarded, + &[("forwarded", "for=\"2001:db8::7\";proto=https")], + ), + ForwardingResult::Malformed, + ); + for value in [ + "", + "example.com", + "_", + "_bad!", + "unknown:", + "198.51.100.7:65536", + "198.51.100.7:-1", + "198.51.100.7:+1", + "198.51.100.7:_", + "[2001:db8::7]:65536", + "[fe80::1%eth0]", + "[2001:db8::7]extra", + "[2001:db8::7", + "[198.51.100.7]", + "010.0.0.1", + "[::1]::80", + ] { + parse_node(value.as_bytes(), true).expect_err(value); + parse_node(value.as_bytes(), false).expect_err(value); + } + } + + #[test] + fn whole_family_validation_includes_unselected_prefix_and_by_nodes() { + for value in [ + "for=garbage,for=198.51.100.7;host=public.example;proto=https", + "for=198.51.100.7;by=garbage;host=public.example;proto=https", + "for=198.51.100.7;FOR=198.51.100.8;host=public.example;proto=https", + "for=198.51.100.7;ext=a;EXT=b;host=public.example;proto=https", + "for=203.0.113.9;proto=ftp,for=198.51.100.7;host=public.example;proto=https", + "for=203.0.113.9;host=\"bad.example:65536\",for=198.51.100.7;host=public.example;proto=https", + "for=\"198.51.100.7:65536\";host=public.example;proto=https", + ] { + direct_fallback( + &checked(ForwardingHeaderFamily::Forwarded, &[("forwarded", value)]), + ForwardingResult::Malformed, + ); + } + direct_fallback( + &checked( + ForwardingHeaderFamily::XForwarded, + &[ + ("x-forwarded-for", "garbage,198.51.100.7"), + ("x-forwarded-host", "public.example"), + ("x-forwarded-proto", "https"), + ], + ), + ForwardingResult::Malformed, + ); + let valid = checked( + ForwardingHeaderFamily::Forwarded, + &[("forwarded", "for=198.51.100.7;by=\"[2001:db8::1]:_port\"")], + ); + assert_client(&valid, CLIENT); + } + + #[test] + fn xff_empty_members_invalidate_even_good_host_and_proto() { + for value in [ + "", + " ", + ",", + ",198.51.100.7", + "198.51.100.7,", + "198.51.100.7,,10.0.0.1", + "198.51.100.7, \t,10.0.0.1", + ] { + direct_fallback( + &checked( + ForwardingHeaderFamily::XForwarded, + &[ + ("x-forwarded-for", value), + ("x-forwarded-host", "public.example"), + ("x-forwarded-proto", "https"), + ], + ), + ForwardingResult::Malformed, + ); + } + direct_fallback( + &checked( + ForwardingHeaderFamily::XForwarded, + &[ + ("x-forwarded-for", CLIENT), + ("x-forwarded-for", ""), + ("x-forwarded-host", "public.example"), + ("x-forwarded-proto", "https"), + ], + ), + ForwardingResult::Malformed, + ); + let no_xff = checked( + ForwardingHeaderFamily::XForwarded, + &[ + ("x-forwarded-host", "public.example"), + ("x-forwarded-proto", "https"), + ], + ); + assert_eq!(no_xff.forwarding_result(), ForwardingResult::Partial); + assert_eq!(no_xff.effective_host().authority(), Some("public.example")); + assert_eq!(no_xff.effective_scheme(), Scheme::Https); + assert_client(&no_xff, "10.0.0.2"); + } + + #[test] + fn x_forwarded_host_and_proto_are_strict_singletons() { + for headers in [ + vec![ + ("x-forwarded-for", CLIENT), + ("x-forwarded-host", "public.example"), + ("x-forwarded-host", "public.example"), + ], + vec![ + ("x-forwarded-for", CLIENT), + ("x-forwarded-proto", "https"), + ("x-forwarded-proto", "https"), + ], + vec![ + ("x-forwarded-for", CLIENT), + ("x-forwarded-host", "public.example,evil.example"), + ], + vec![ + ("x-forwarded-for", CLIENT), + ("x-forwarded-proto", "https,http"), + ], + vec![("x-forwarded-for", CLIENT), ("x-forwarded-host", "")], + vec![("x-forwarded-for", CLIENT), ("x-forwarded-proto", "")], + ] { + direct_fallback( + &checked(ForwardingHeaderFamily::XForwarded, &headers), + ForwardingResult::Malformed, + ); + } + let accepted = checked( + ForwardingHeaderFamily::XForwarded, + &[ + ("x-forwarded-for", "198.51.100.7, 10.0.0.1"), + ("x-forwarded-host", " \tpublic.example:443 "), + ("x-forwarded-proto", " HTTPS "), + ], + ); + assert_eq!(accepted.forwarding_result(), ForwardingResult::Accepted); + assert_eq!( + accepted.effective_host().source(), + CheckedHostSource::TrustedXForwarded + ); + assert_eq!( + accepted.client_source(), + CheckedHostSource::TrustedXForwarded + ); + assert_eq!( + accepted.scheme_source(), + CheckedHostSource::TrustedXForwarded + ); + } + + #[test] + fn authorities_and_schemes_are_checked_before_acceptance() { + for host in [ + "public.example", + "public.example:0", + "public.example:65535", + "[2001:db8::1]", + "[::1]:443", + "192.0.2.1:80", + ] { + parse_authority(host.as_bytes()).expect(host); + } + for host in [ + "", + "user@public.example", + "public.example/path", + "public.example?q=secret", + "public.example#secret", + "public.example\\path", + " public.example", + "public.example ", + "public.example:", + "public.example:65536", + "public.example:-1", + "public.example:_port", + "public.example:443:80", + "[fe80::1%eth0]", + "[::1]:", + "[::1]extra", + ":443", + "a,b", + ] { + parse_authority(host.as_bytes()).expect_err(host); + let encoded_host = host.replace('\\', "\\\\").replace('"', "\\\""); + direct_fallback( + &checked( + ForwardingHeaderFamily::Forwarded, + &[( + "forwarded", + &format!("for=198.51.100.7;host=\"{encoded_host}\";proto=https"), + )], + ), + ForwardingResult::Malformed, + ); + } + for scheme in ["ftp", "https:", "wss", "", "https http"] { + direct_fallback( + &checked( + ForwardingHeaderFamily::Forwarded, + &[( + "forwarded", + &format!("for=198.51.100.7;host=public.example;proto=\"{scheme}\""), + )], + ), + ForwardingResult::Malformed, + ); + } + } + + #[test] + fn walk_stops_at_first_untrusted_boundary_and_never_skips_gaps() { + for (chain, expected, result) in [ + ( + "203.0.113.99,198.51.100.7,10.0.0.1", + CLIENT, + ForwardingResult::Partial, + ), + ( + "203.0.113.99,198.51.100.7", + CLIENT, + ForwardingResult::Partial, + ), + ( + "203.0.113.99,10.1.0.1", + "10.1.0.1", + ForwardingResult::Partial, + ), + ( + "198.51.100.7,unknown,10.0.0.1", + "10.0.0.2", + ForwardingResult::UnresolvedChain, + ), + ( + "198.51.100.7,_gap,10.0.0.1", + "10.0.0.2", + ForwardingResult::UnresolvedChain, + ), + ("10.0.0.1", "10.0.0.2", ForwardingResult::UnresolvedChain), + ( + "10.0.0.1,10.0.0.3", + "10.0.0.2", + ForwardingResult::UnresolvedChain, + ), + ("unknown,198.51.100.7", CLIENT, ForwardingResult::Partial), + ] { + let metadata = checked( + ForwardingHeaderFamily::XForwarded, + &[("x-forwarded-for", chain)], + ); + assert_client(&metadata, expected); + assert_eq!(metadata.forwarding_result(), result, "{chain}"); + } + let gap = checked( + ForwardingHeaderFamily::Forwarded, + &[( + "forwarded", + "for=198.51.100.7,host=public.example;proto=https", + )], + ); + assert_eq!(gap.forwarding_result(), ForwardingResult::UnresolvedChain); + assert_client(&gap, "10.0.0.2"); + assert_eq!(gap.effective_host().authority(), Some("public.example")); + assert_eq!(gap.effective_scheme(), Scheme::Https); + } + + #[test] + fn mapped_peer_and_asserted_client_match_equivalent_ipv4_trust() { + let peer: SocketAddr = "[::ffff:10.0.0.2]:1234".parse().unwrap(); + let parts = parts(&[( + "forwarded", + "for=\"[::ffff:198.51.100.7]:_p\",for=\"[::ffff:10.0.0.1]\";host=public.example;proto=https", + )]); + let metadata = normalize( + &policy(ForwardingHeaderFamily::Forwarded), + Some(peer), + &parts, + ); + assert_eq!(metadata.direct_peer(), Some(peer)); + assert_client(&metadata, CLIENT); + assert_eq!(metadata.forwarding_result(), ForwardingResult::Accepted); + } + + #[test] + fn forwarded_list_slot_bounds_are_exact_across_repeated_fields_and_empty_members() { + let for_node = "for=198.51.100.7"; + let exact = repeat_n(for_node, MAX_CHAIN_SLOTS) + .collect::>() + .join(","); + assert_eq!( + checked(ForwardingHeaderFamily::Forwarded, &[("forwarded", &exact)]) + .forwarding_result(), + ForwardingResult::Partial + ); + direct_fallback( + &checked( + ForwardingHeaderFamily::Forwarded, + &[("forwarded", &format!("{exact},{for_node}"))], + ), + ForwardingResult::Malformed, + ); + let two_nodes = format!("{for_node},{for_node}"); + let mut repeated = parts(&[]); + for _ in 0_usize..32 { + repeated + .headers + .append("forwarded", HeaderValue::from_str(&two_nodes).unwrap()); + } + assert_eq!( + normalize( + &policy(ForwardingHeaderFamily::Forwarded), + Some(PEER.parse().unwrap()), + &repeated + ) + .forwarding_result(), + ForwardingResult::Partial + ); + repeated + .headers + .append("forwarded", HeaderValue::from_static("")); + direct_fallback( + &normalize( + &policy(ForwardingHeaderFamily::Forwarded), + Some(PEER.parse().unwrap()), + &repeated, + ), + ForwardingResult::Malformed, + ); + let exact_empty_slots = format!("{}{for_node}", ",".repeat(63)); + assert_eq!( + checked( + ForwardingHeaderFamily::Forwarded, + &[("forwarded", &exact_empty_slots)] + ) + .forwarding_result(), + ForwardingResult::Partial + ); + direct_fallback( + &checked( + ForwardingHeaderFamily::Forwarded, + &[("forwarded", &format!(",{exact_empty_slots}"))], + ), + ForwardingResult::Malformed, + ); + } + + #[test] + fn xff_list_slot_bounds_are_exact_across_repeated_fields() { + let exact = repeat_n(CLIENT, MAX_CHAIN_SLOTS) + .collect::>() + .join(","); + assert_eq!( + checked( + ForwardingHeaderFamily::XForwarded, + &[("x-forwarded-for", &exact)] + ) + .forwarding_result(), + ForwardingResult::Partial + ); + direct_fallback( + &checked( + ForwardingHeaderFamily::XForwarded, + &[("x-forwarded-for", &format!("{exact},{CLIENT}"))], + ), + ForwardingResult::Malformed, + ); + let mut repeated = parts(&[]); + for _ in 0_usize..32 { + repeated.headers.append( + "x-forwarded-for", + HeaderValue::from_str(&format!("{CLIENT},{CLIENT}")).unwrap(), + ); + } + assert_eq!( + normalize( + &policy(ForwardingHeaderFamily::XForwarded), + Some(PEER.parse().unwrap()), + &repeated + ) + .forwarding_result(), + ForwardingResult::Partial + ); + repeated + .headers + .append("x-forwarded-for", HeaderValue::from_str(CLIENT).unwrap()); + direct_fallback( + &normalize( + &policy(ForwardingHeaderFamily::XForwarded), + Some(PEER.parse().unwrap()), + &repeated, + ), + ForwardingResult::Malformed, + ); + } + + #[test] + fn parameter_slot_bounds_include_empty_parameters() { + let exact = format!("for=198.51.100.7{}", ";".repeat(15)); + assert_eq!( + checked(ForwardingHeaderFamily::Forwarded, &[("forwarded", &exact)]) + .forwarding_result(), + ForwardingResult::Partial + ); + direct_fallback( + &checked( + ForwardingHeaderFamily::Forwarded, + &[("forwarded", &format!("{exact};"))], + ), + ForwardingResult::Malformed, + ); + let mut named = String::from("for=198.51.100.7"); + for index in 0_usize..15 { + write!(named, ";e{index}=value").unwrap(); + } + assert_eq!( + checked(ForwardingHeaderFamily::Forwarded, &[("forwarded", &named)]) + .forwarding_result(), + ForwardingResult::Partial + ); + direct_fallback( + &checked( + ForwardingHeaderFamily::Forwarded, + &[("forwarded", &format!("{named};e15=value"))], + ), + ForwardingResult::Malformed, + ); + } + + #[test] + fn assertion_byte_budget_is_exact_and_counts_repeated_delimiters() { + let prefix = "for=198.51.100.7;host=public.example;proto=https;ext=\""; + let exact = format!("{prefix}{}\"", "a".repeat(MAX_BYTES - prefix.len() - 1)); + assert_eq!( + checked(ForwardingHeaderFamily::Forwarded, &[("forwarded", &exact)]) + .forwarding_result(), + ForwardingResult::Accepted + ); + let over = exact.replacen("ext=\"", "ext=\"a", 1); + direct_fallback( + &checked(ForwardingHeaderFamily::Forwarded, &[("forwarded", &over)]), + ForwardingResult::Malformed, + ); + let second = "for=198.51.100.7;host=public.example;proto=https"; + let first = format!("ext=\"{}\"", "a".repeat(MAX_BYTES - second.len() - 1 - 6)); + assert_eq!(first.len() + second.len() + 1, MAX_BYTES); + assert_eq!( + checked( + ForwardingHeaderFamily::Forwarded, + &[("forwarded", &first), ("forwarded", second)] + ) + .forwarding_result(), + ForwardingResult::Accepted + ); + direct_fallback( + &checked( + ForwardingHeaderFamily::Forwarded, + &[("forwarded", &format!(" {first}")), ("forwarded", second)], + ), + ForwardingResult::Malformed, + ); + let padding = " ".repeat(MAX_BYTES - CLIENT.len()); + assert_eq!( + checked( + ForwardingHeaderFamily::XForwarded, + &[("x-forwarded-for", &format!("{padding}{CLIENT}"))] + ) + .forwarding_result(), + ForwardingResult::Partial + ); + direct_fallback( + &checked( + ForwardingHeaderFamily::XForwarded, + &[ + ("x-forwarded-for", &format!(" {padding}{CLIENT}")), + ("x-forwarded-host", "public.example"), + ], + ), + ForwardingResult::Malformed, + ); + } + + #[test] + fn stripping_removes_every_forwarding_extension_without_changing_direct_head() { + let mut parts = parts(&[ + ("Forwarded", "for=198.51.100.7"), + ("X-Forwarded-For", CLIENT), + ("x-forwarded-host", "public.example"), + ("x-forwarded-proto", "https"), + ("X-Forwarded-Secret", "sentinel-secret"), + ("x-request-id", "visitor-id"), + ]); + let original_uri = parts.uri.clone(); + strip_forwarding_headers(&mut parts.headers); + assert!(!parts.headers.keys().any(is_forwarding_header)); + assert_eq!(parts.headers["host"], "direct.example:8080"); + assert_eq!(parts.headers["x-request-id"], "visitor-id"); + assert_eq!(parts.uri, original_uri); + } + + #[test] + #[expect( + clippy::shadow_unrelated, + reason = "sequential direct-head fixtures assert metadata with authority present and then unavailable" + )] + fn direct_authority_uses_http_authority_without_inferring_transport_scheme() { + let mut parts = parts(&[]); + parts.uri = "https://uri.example:443/path?secret=value".parse().unwrap(); + let metadata = normalize(&TrustedProxyPolicy::no_trust(), None, &parts); + assert_eq!( + metadata.effective_host().authority(), + Some("uri.example:443") + ); + assert_eq!(metadata.effective_scheme(), Scheme::Http); + assert_eq!(metadata.scheme_source(), CheckedHostSource::Direct); + parts.uri = "/".parse().unwrap(); + parts.headers.remove("host"); + let metadata = normalize(&TrustedProxyPolicy::no_trust(), None, &parts); + assert_eq!(metadata.effective_host().authority(), None); + assert_eq!( + metadata.effective_host().source(), + CheckedHostSource::Unavailable + ); + parts + .headers + .insert("host", HeaderValue::from_static("bad.example:65536")); + assert_eq!( + normalize(&TrustedProxyPolicy::no_trust(), None, &parts) + .effective_host() + .authority(), + None + ); + } + + #[test] + fn debug_metadata_does_not_dump_private_ingress_values() { + let metadata = checked( + ForwardingHeaderFamily::XForwarded, + &[ + ("x-forwarded-for", CLIENT), + ("x-forwarded-host", "sentinel-secret.example"), + ("x-forwarded-proto", "https"), + ], + ); + let debug = format!("{metadata:?}"); + for private in [CLIENT, "sentinel-secret", "10.0.0.2"] { + assert!(!debug.contains(private)); + } + for result in [ + ForwardingResult::Absent, + ForwardingResult::Accepted, + ForwardingResult::Partial, + ForwardingResult::UntrustedPeer, + ForwardingResult::Malformed, + ForwardingResult::UnresolvedChain, + ] { + assert!( + result + .as_str() + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte == b'_') + ); + } + assert_eq!(Scheme::Http.as_str(), "http"); + assert_eq!(Scheme::Https.as_str(), "https"); + } +} diff --git a/crates/edgezero-adapter-axum/src/response.rs b/crates/edgezero-adapter-axum/src/response.rs index 722d82c83..3612b83dd 100644 --- a/crates/edgezero-adapter-axum/src/response.rs +++ b/crates/edgezero-adapter-axum/src/response.rs @@ -15,14 +15,14 @@ use edgezero_core::response_egress::{ RESPONSE_EGRESS_FALLBACK_SAFETY_BUDGET, ResponseEgressAttempt, ResponseEgressEnvelope, ResponseEgressFallbackDisposition, ResponseEgressOutcome, }; -use edgezero_core::response_egress_framing::{ - PreparedResponseEgress, prepare_response_egress, response_egress_source_error_category, -}; +use edgezero_core::response_egress_framing::{PreparedResponseEgress, prepare_response_egress}; use edgezero_core::time::{Deadline, MonotonicClock}; use http_body::{Body as HttpBody, Frame, SizeHint}; use tokio::sync::Notify; use tokio::time::Instant as TokioInstant; +use crate::diagnostics::SelectedStatus; + const INTERNAL_FALLBACK_BODY: &[u8] = b"internal server error"; const TIMEOUT_FALLBACK_BODY: &[u8] = b"response write deadline exceeded"; @@ -444,11 +444,8 @@ impl HttpBody for AxumEgressBody { } Poll::Ready(Some(Ok(Frame::data(bytes)))) } - Poll::Ready(Some(Err(error))) => { - log::warn!( - "response-egress Axum body source failed after commit: {}", - response_egress_source_error_category(&error) - ); + Poll::Ready(Some(Err(_error))) => { + // The native terminal owner records this once, including request-log opt-out. this.source = ResponseSource::Done; this.fail(ResponseEgressOutcome::SourceError); Poll::Ready(Some(Err(AxumBodyError(ResponseEgressOutcome::SourceError)))) @@ -472,9 +469,13 @@ impl HttpBody for AxumEgressBody { /// Converts one owned core envelope into the private body used by `EdgeZero`'s Hyper server. pub(crate) fn prepare_egress_response( - egress: ResponseEgressEnvelope, + mut egress: ResponseEgressEnvelope, connection: &EgressConnection, + selected: Option<&SelectedStatus>, ) -> Response { + if let Some(slot) = selected { + slot.select(egress.response_mut().status()); + } let request_method = egress.request_method().clone(); match egress.begin() { Ok((prepared, policy, attempt, clock)) => { @@ -485,6 +486,7 @@ pub(crate) fn prepare_egress_response( attempt, &clock, connection, + selected, ); } match start_prepared( @@ -494,6 +496,7 @@ pub(crate) fn prepare_egress_response( clock.clone(), EgressKind::Application, connection, + selected, ) { Ok(response) => response, Err(returned_attempt) => prepare_fallback( @@ -502,12 +505,20 @@ pub(crate) fn prepare_egress_response( *returned_attempt, &clock, connection, + selected, ), } } Err(failure) => { let (attempt, clock, cause) = failure.into_parts(); - prepare_fallback(&request_method, cause, attempt, &clock, connection) + prepare_fallback( + &request_method, + cause, + attempt, + &clock, + connection, + selected, + ) } } } @@ -519,10 +530,14 @@ fn start_prepared( clock: MonotonicClock, kind: EgressKind, connection: &EgressConnection, + selected: Option<&SelectedStatus>, ) -> Result, Box> { let declared_length = prepared.declared_length(); let transmits_body = prepared.transmits_body(); let (parts, core_body) = prepared.into_response().into_parts(); + if let Some(slot) = selected { + slot.select(parts.status); + } match AxumEgressBody::owned( core_body, declared_length, @@ -544,11 +559,18 @@ fn prepare_fallback( mut attempt: ResponseEgressAttempt, clock: &MonotonicClock, connection: &EgressConnection, + selected: Option<&SelectedStatus>, ) -> Response { if !attempt.begin_fallback(cause) { + if let Some(slot) = selected { + slot.select(StatusCode::INTERNAL_SERVER_ERROR); + } attempt.terminate(ResponseEgressOutcome::ConversionError, clock.now()); return detached_fallback(ResponseEgressOutcome::ConversionError); } + if let Some(slot) = selected { + slot.select(fallback_parts(cause).0); + } let started_at = clock.now(); let Some(fallback_deadline) = started_at.checked_add(RESPONSE_EGRESS_FALLBACK_SAFETY_BUDGET) else { @@ -567,6 +589,7 @@ fn prepare_fallback( clock.clone(), EgressKind::Fallback, connection, + selected, ) { Ok(response) => response, Err(mut returned_attempt) => { @@ -611,7 +634,8 @@ mod tests { use edgezero_core::body::Body; use edgezero_core::http::{HeaderMap, Method, StatusCode, Version}; use edgezero_core::response_egress::{ - ResponseEgressAttempt, ResponseEgressCompletion, ResponseEgressHead, + RESPONSE_EGRESS_FALLBACK_SAFETY_BUDGET, ResponseEgressAttempt, ResponseEgressBodyKind, + ResponseEgressCompletion, ResponseEgressFallbackDisposition, ResponseEgressHead, ResponseEgressObserver, ResponseEgressObserverHandle, ResponseEgressOutcome, ResponseEgressReport, }; @@ -621,11 +645,88 @@ mod tests { use futures_util::stream::{pending, poll_fn as poll_stream}; use http_body::Body as _; - use super::{AxumEgressBody, EgressConnection}; + use super::{AxumEgressBody, EgressConnection, SelectedStatus, prepare_fallback}; + use crate::diagnostics::{ + NativeDiagnosticsHandle, NativeDiagnosticsSnapshot, NativeRequestObserver, + NativeRequestOutcome, NativeRequestRecord, NativeSession, + }; + + #[derive(Default)] + struct NativeFallbackProbe { + completion_calls: Arc, + diagnostics: NativeDiagnosticsHandle, + native: RecordingNativeObserver, + observer: RecordingObserver, + } + + #[derive(Clone, Default)] + struct RecordingNativeObserver(Arc>>); #[derive(Clone, Default)] struct RecordingObserver(Arc>>); + impl NativeFallbackProbe { + fn attempt( + &self, + now: MonotonicInstant, + clock: MonotonicClock, + ) -> (ResponseEgressAttempt, SelectedStatus) { + let session = NativeSession::new( + self.diagnostics + .clone() + .with_request_observer(self.native.clone()), + false, + ) + .expect("native session"); + let guard = session + .request(&Method::GET, now, clock.clone(), None) + .expect("native request"); + let (native_completion, selected) = guard.completion(); + selected.select(StatusCode::OK); + let calls = Arc::clone(&self.completion_calls); + let app_completion = ResponseEgressCompletion::new(move |_report| { + calls.fetch_add(1, Ordering::SeqCst); + }); + let headers = HeaderMap::new(); + let head = ResponseEgressHead::new( + StatusCode::OK, + Version::HTTP_11, + &headers, + None, + now, + None, + ); + ( + ResponseEgressAttempt::new( + &head, + now, + app_completion.join(native_completion), + ResponseEgressObserverHandle::new(self.observer.clone()), + clock, + ), + selected, + ) + } + + fn record(&self) -> NativeRequestRecord { + assert_eq!(self.completion_calls.load(Ordering::SeqCst), 1); + assert_eq!(self.observer.0.lock().expect("app reports").len(), 1); + assert_eq!( + self.diagnostics.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + let records = self.native.0.lock().expect("native records"); + assert_eq!(records.len(), 1); + records.first().expect("one native record").clone() + } + } + + impl NativeRequestObserver for RecordingNativeObserver { + fn observe(&self, record: &NativeRequestRecord) { + self.0.lock().expect("native records").push(record.clone()); + } + } + impl ResponseEgressObserver for RecordingObserver { fn complete(&self, report: &ResponseEgressReport) { self.0.lock().expect("reports lock").push(report.clone()); @@ -790,6 +891,130 @@ mod tests { assert!(connection.is_empty()); } + #[test] + fn native_duplicate_live_fallback_selects_detached_500_before_terminal_record() { + let probe = NativeFallbackProbe::default(); + let now = MonotonicInstant::now(); + let clock = MonotonicClock::new(move || now); + let (mut attempt, selected) = probe.attempt(now, clock.clone()); + assert!(attempt.begin_fallback(ResponseEgressOutcome::DeadlineExceeded)); + selected.select(StatusCode::GATEWAY_TIMEOUT); + assert_eq!(probe.diagnostics.snapshot().active_requests, 1); + let connection = EgressConnection::default(); + let response = prepare_fallback( + &Method::GET, + ResponseEgressOutcome::ConversionError, + attempt, + &clock, + &connection, + Some(&selected), + ); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + let record = probe.record(); + assert_eq!(record.status, Some(StatusCode::INTERNAL_SERVER_ERROR)); + assert_eq!( + record.outcome, + NativeRequestOutcome::Egress(ResponseEgressOutcome::DeadlineExceeded) + ); + assert_eq!(record.body_kind, Some(ResponseEgressBodyKind::Fallback)); + assert_eq!( + record.fallback_disposition, + Some(ResponseEgressFallbackDisposition::Aborted) + ); + assert_eq!(record.bytes_written, Some(0)); + drop(response); + assert_eq!(probe.record().request_id, record.request_id); + assert!(connection.is_empty()); + } + + #[test] + fn native_already_terminal_fallback_cannot_rewrite_the_prior_200_record() { + let probe = NativeFallbackProbe::default(); + let now = MonotonicInstant::now(); + let clock = MonotonicClock::new(move || now); + let (mut attempt, selected) = probe.attempt(now, clock.clone()); + assert!(attempt.begin_writing()); + assert!(attempt.complete(now)); + let original = probe.record(); + assert_eq!(original.status, Some(StatusCode::OK)); + let connection = EgressConnection::default(); + let response = prepare_fallback( + &Method::GET, + ResponseEgressOutcome::DeadlineExceeded, + attempt, + &clock, + &connection, + Some(&selected), + ); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + drop(response); + let retained = probe.record(); + assert_eq!(retained.request_id, original.request_id); + assert_eq!(retained.status, original.status); + assert_eq!(retained.duration, original.duration); + assert_eq!( + retained.outcome, + NativeRequestOutcome::Egress(ResponseEgressOutcome::Completed) + ); + assert_eq!( + retained.body_kind, + Some(ResponseEgressBodyKind::Application) + ); + assert_eq!(retained.fallback_disposition, None); + assert_eq!(retained.bytes_written, Some(0)); + assert!(connection.is_empty()); + } + + #[test] + fn native_failed_fallback_registration_retains_500_or_504_before_aborted_record() { + for (cause, status) in [ + ( + ResponseEgressOutcome::ConversionError, + StatusCode::INTERNAL_SERVER_ERROR, + ), + ( + ResponseEgressOutcome::DeadlineExceeded, + StatusCode::GATEWAY_TIMEOUT, + ), + ] { + let probe = NativeFallbackProbe::default(); + let now = MonotonicInstant::now(); + let expired = now + .checked_add(RESPONSE_EGRESS_FALLBACK_SAFETY_BUDGET) + .expect("fallback deadline"); + // Preparation samples now, registration sees expiry, then settlement keeps that instant. + let clock = scripted_clock(vec![now, expired, expired]); + let (attempt, selected) = probe.attempt(now, clock.clone()); + assert_eq!(probe.diagnostics.snapshot().active_requests, 1); + let connection = EgressConnection::default(); + let response = prepare_fallback( + &Method::GET, + cause, + attempt, + &clock, + &connection, + Some(&selected), + ); + assert_eq!(response.status(), status); + let record = probe.record(); + assert_eq!(record.status, Some(status)); + assert_eq!(record.outcome, NativeRequestOutcome::Egress(cause)); + assert_eq!( + record.duration, + Some(RESPONSE_EGRESS_FALLBACK_SAFETY_BUDGET) + ); + assert_eq!(record.body_kind, Some(ResponseEgressBodyKind::Fallback)); + assert_eq!( + record.fallback_disposition, + Some(ResponseEgressFallbackDisposition::Aborted) + ); + assert_eq!(record.bytes_written, Some(0)); + drop(response); + assert_eq!(probe.record().request_id, record.request_id); + assert!(connection.is_empty()); + } + } + #[tokio::test(flavor = "current_thread")] async fn response_egress_post_ready_deadline_precedence() { let observer = RecordingObserver::default(); diff --git a/crates/edgezero-adapter-axum/src/run_options.rs b/crates/edgezero-adapter-axum/src/run_options.rs index 971043cf3..1c6506b4c 100644 --- a/crates/edgezero-adapter-axum/src/run_options.rs +++ b/crates/edgezero-adapter-axum/src/run_options.rs @@ -10,6 +10,9 @@ use edgezero_core::app::StoresMetadata; use edgezero_core::env_config::EnvConfig; use log::LevelFilter; +use crate::diagnostics::NativeDiagnosticsHandle; +use crate::proxy::TrustedProxyPolicy; + /// Validated native JSON buffering ceiling. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) struct JsonBodyLimit(NonZeroUsize); @@ -55,11 +58,14 @@ pub struct AxumRunOptions { addr: SocketAddr, config_dir: Option, data_dir: Option, + diagnostics: NativeDiagnosticsHandle, env: EnvConfig, grace: Duration, json_body_limit: JsonBodyLimit, level: LevelFilter, non_unicode_overrides: Vec>, + request_records: bool, + trusted_proxy_policy: TrustedProxyPolicy, } impl AxumRunOptions { @@ -75,6 +81,10 @@ impl AxumRunOptions { self.data_dir.as_deref() } + pub(crate) fn diagnostics(&self) -> NativeDiagnosticsHandle { + self.diagnostics.clone() + } + pub(crate) fn env_config(&self) -> &EnvConfig { &self.env } @@ -143,6 +153,7 @@ impl AxumRunOptions { } options.grace = shutdown_grace(&env)?; options.json_body_limit = JsonBodyLimit::from_config(&env)?; + (options.trusted_proxy_policy, options.request_records) = native_settings(&env)?; if let Some(root) = env.get(&["adapter", "config_dir"]) { options = options.with_config_dir(root)?; } @@ -174,18 +185,29 @@ impl AxumRunOptions { addr, config_dir: None, data_dir: None, + diagnostics: NativeDiagnosticsHandle::new(), env: EnvConfig::default(), grace: Duration::from_secs(10), json_body_limit: JsonBodyLimit::DEFAULT, level: LevelFilter::Info, non_unicode_overrides: Vec::new(), + request_records: true, + trusted_proxy_policy: TrustedProxyPolicy::no_trust(), }) } + pub(crate) fn request_records(&self) -> bool { + self.request_records + } + pub(crate) fn shutdown_grace(&self) -> Duration { self.grace } + pub(crate) fn trusted_proxy_policy(&self) -> TrustedProxyPolicy { + self.trusted_proxy_policy.clone() + } + pub(crate) fn validate_stores(&self, metadata: StoresMetadata) -> anyhow::Result<()> { for (kind, declaration) in [ ("config", metadata.config), @@ -258,6 +280,14 @@ impl AxumRunOptions { Ok(self) } + /// Shares native counters and the optional observer with this runner. + #[inline] + #[must_use] + pub fn with_diagnostics(mut self, diagnostics: NativeDiagnosticsHandle) -> Self { + self.diagnostics = diagnostics; + self + } + /// Selects the ceiling for framework-managed JSON buffering, in bytes. /// /// # Errors @@ -275,6 +305,14 @@ impl AxumRunOptions { self } + /// Enables only default native request logs; observers and accounting remain active. + #[inline] + #[must_use] + pub fn with_request_records(mut self, enabled: bool) -> Self { + self.request_records = enabled; + self + } + /// Selects a positive, representable integer-second drain budget. /// /// # Errors @@ -285,6 +323,40 @@ impl AxumRunOptions { self.grace = grace; Ok(self) } + + /// Installs an explicit immutable incoming-proxy policy without reading the environment. + #[inline] + #[must_use] + pub fn with_trusted_proxy_policy(mut self, policy: TrustedProxyPolicy) -> Self { + self.trusted_proxy_policy = policy; + self + } +} + +pub(crate) fn native_settings(env: &EnvConfig) -> anyhow::Result<(TrustedProxyPolicy, bool)> { + let policy = TrustedProxyPolicy::from_values( + env.get(&["adapter", "trusted_proxy_cidrs"]), + env.get(&["adapter", "forwarding_header_family"]), + ) + .map_err(|_error| { + failure( + "settings", + "TRUSTED_PROXY_POLICY", + "invalid literal networks or forwarding family", + ) + })?; + let records = match env.get(&["logging", "request_records"]) { + None | Some("true") => true, + Some("false") => false, + Some(_) => { + return Err(failure( + "settings", + "LOGGING__REQUEST_RECORDS", + "invalid boolean", + )); + } + }; + Ok((policy, records)) } pub(crate) fn failure(stage: &str, subject: &str, category: &str) -> anyhow::Error { @@ -301,6 +373,9 @@ fn fixed_setting(path: &[String]) -> Option<&'static str> { ["adapter", "config_dir"] => Some("CONFIG_DIR"), ["adapter", "data_dir"] => Some("DATA_DIR"), ["adapter", "shutdown_grace_seconds"] => Some("SHUTDOWN_GRACE_SECONDS"), + ["adapter", "trusted_proxy_cidrs"] => Some("TRUSTED_PROXY_CIDRS"), + ["adapter", "forwarding_header_family"] => Some("FORWARDING_HEADER_FAMILY"), + ["logging", "request_records"] => Some("LOGGING__REQUEST_RECORDS"), ["adapter", "json_body_limit_bytes"] => Some("JSON_BODY_LIMIT_BYTES"), ["logging", "level"] => Some("LOGGING__LEVEL"), _ => None, @@ -354,6 +429,59 @@ mod tests { use super::*; use edgezero_core::app::StoreMetadata; + #[test] + fn native_settings_defaults_explicit_builders_and_strict_values() { + let defaults = AxumRunOptions::from_vars(Vec::<(String, String)>::new()).expect("defaults"); + assert!(defaults.request_records()); + assert_eq!( + defaults.trusted_proxy_policy(), + TrustedProxyPolicy::no_trust() + ); + let configured = AxumRunOptions::from_vars([ + ("EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS", "127.0.0.2/32"), + ("EDGEZERO__ADAPTER__FORWARDING_HEADER_FAMILY", "forwarded"), + ("EDGEZERO__LOGGING__REQUEST_RECORDS", "false"), + ]) + .expect("configured"); + assert!(!configured.request_records()); + assert_ne!( + configured.trusted_proxy_policy(), + TrustedProxyPolicy::no_trust() + ); + let explicit = AxumRunOptions::new(defaults.addr()) + .expect("explicit") + .with_request_records(false) + .with_trusted_proxy_policy(configured.trusted_proxy_policy()); + assert!(!explicit.request_records()); + assert_eq!( + explicit.trusted_proxy_policy(), + configured.trusted_proxy_policy() + ); + assert!( + explicit + .env_config() + .get(&["adapter", "trusted_proxy_cidrs"]) + .is_none() + ); + for (key, value) in [ + ("EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS", "127.0.0.2"), + ("EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS", "sentinel-policy"), + ( + "EDGEZERO__ADAPTER__FORWARDING_HEADER_FAMILY", + "sentinel-family", + ), + ("EDGEZERO__LOGGING__REQUEST_RECORDS", "TRUE"), + ("EDGEZERO__LOGGING__REQUEST_RECORDS", "false "), + ("EDGEZERO__LOGGING__REQUEST_RECORDS", ""), + ] { + let error = AxumRunOptions::from_vars([(key, value)]) + .err() + .expect("invalid supplied setting"); + assert_eq!(error.chain().count(), 1); + assert!(!error.to_string().contains("sentinel")); + } + } + #[test] fn json_body_limit_is_positive_decimal_and_representable() { let key = "EDGEZERO__ADAPTER__JSON_BODY_LIMIT_BYTES"; diff --git a/crates/edgezero-adapter-axum/src/service.rs b/crates/edgezero-adapter-axum/src/service.rs index 2edbcf3c4..de3dfd25d 100644 --- a/crates/edgezero-adapter-axum/src/service.rs +++ b/crates/edgezero-adapter-axum/src/service.rs @@ -1,5 +1,6 @@ use std::future::Future; use std::net::SocketAddr; +use std::panic::catch_unwind; use std::pin::Pin; use std::sync::Arc; #[cfg(test)] @@ -8,13 +9,13 @@ use std::task::{Context, Poll}; use axum::body::Body as AxumBody; use axum::extract::connect_info::ConnectInfo; use axum::http::{Request, Response}; -use edgezero_core::app::App; +use edgezero_core::app::{App, FrameworkErrorDetail}; use edgezero_core::config_store::ConfigStoreHandle; use edgezero_core::error::EdgeError; -use edgezero_core::http::Method; +use edgezero_core::http::{Method, RequestParts}; use edgezero_core::ingress::{ IngressBeginOutcome, IngressDispatchOutcome, IngressFraming, IngressHeadAccounting, - IngressHeadParts, validate_normalized_ingress_parts, + IngressHeadParts, PreparedIngress, validate_normalized_ingress_parts_with_summary, }; use edgezero_core::key_value_store::KvHandle; use edgezero_core::probe::{LifecyclePhase, LifecycleReader}; @@ -33,8 +34,16 @@ use tower::Service as TowerService; #[cfg(test)] use crate::connection::{ConnectionExit, serve_http1}; +use crate::context::AxumRequestContext; +#[cfg(test)] +use crate::diagnostics::NativeDiagnosticsHandle; +use crate::diagnostics::{ + ConnectionActivity, ConnectionGuard, IngressDisposition, NativeIngressMetadata, NativeSession, + RequestGuard, +}; #[cfg(test)] use crate::outbound::AxumOutboundClient; +use crate::proxy::{TrustedProxyPolicy, normalize, strip_forwarding_headers}; use crate::request::into_core_request_parts; use crate::response::{AxumEgressBody, EgressConnection, prepare_egress_response}; use crate::run_options::JsonBodyLimit; @@ -47,16 +56,19 @@ pub(crate) struct AxumIngressAbort; /// Shared application state used to construct one private Hyper service per HTTP/1 connection. #[derive(Clone)] pub(crate) struct AxumServiceState { + activity: Option, app: Arc, config_registry: Option, config_store_handle: Option, json_body_limit: JsonBodyLimit, kv_handle: Option, kv_registry: Option, + native: NativeSession, outbound_transport: Option>, phase: Option>, secret_handle: Option, secret_registry: Option, + trusted_proxy_policy: TrustedProxyPolicy, } #[expect( @@ -73,6 +85,8 @@ impl AxumServiceState { app, AxumOutboundClient::try_transport().expect("test transport"), JsonBodyLimit::DEFAULT, + NativeSession::new(NativeDiagnosticsHandle::new(), true).expect("test native identity"), + TrustedProxyPolicy::no_trust(), ) } @@ -80,18 +94,23 @@ impl AxumServiceState { app: App, transport: Arc, json_body_limit: JsonBodyLimit, + native: NativeSession, + trusted_proxy_policy: TrustedProxyPolicy, ) -> Self { Self { + activity: None, app: Arc::new(app), config_registry: None, config_store_handle: None, json_body_limit, kv_handle: None, kv_registry: None, + native, outbound_transport: Some(transport), phase: None, secret_handle: None, secret_registry: None, + trusted_proxy_policy, } } @@ -186,6 +205,10 @@ impl AxumServiceState { self } + pub(crate) fn connection_activity(&self) -> (ConnectionActivity, ConnectionGuard) { + self.native.connection() + } + pub(crate) fn for_connection( &self, remote_addr: SocketAddr, @@ -240,75 +263,25 @@ impl AxumServiceState { .is_none_or(|phase| *phase.borrow() == LifecyclePhase::Ready) } - async fn dispatch_request( + async fn dispatch_prepared( &self, - mut request: Request, + prepared: PreparedIngress, + parts: RequestParts, + native_body: AxumBody, connection: &EgressConnection, - remote_addr: Option, - ) -> Result, AxumIngressAbort> { - // This private gate defines dispatched work. Public extensions and app - // admission policy cannot admit new work after the owner starts draining. - if !self.admission_open() { - return Err(AxumIngressAbort); - } - if let Some(peer_addr) = remote_addr { - request.extensions_mut().insert(ConnectInfo(peer_addr)); - } - let request_start = self.app.monotonic_now(); - let app = Arc::clone(&self.app); - let outbound_transport_handle = self.outbound_transport.clone(); - let (config_registry, kv_registry, secret_registry) = self.resolved_store_registries(); - let (parts, native_body) = request.into_parts(); - let request_method = parts.method.clone(); - if let Err(error) = validate_normalized_ingress_parts(&parts, app.ingress_head_limits()) { - return prepare_detached_ingress_error( - &app, - error, - request_method.clone(), - request_start, - connection, - ); - } - let head_parts = IngressHeadParts::from_parts( - &parts, - IngressHeadAccounting::HostManaged, - IngressFraming::HostManaged, - ); - let prepared = match app.begin_ingress(head_parts, request_start) { - Ok(IngressBeginOutcome::Admitted(prepared)) => prepared, - Ok(IngressBeginOutcome::Refused(response)) => { - return Ok(prepare_egress_response(response, connection)); - } - Ok(IngressBeginOutcome::Aborted) => return Err(AxumIngressAbort), - Ok(_) => { - return prepare_detached_ingress_error( - &app, - EdgeError::internal(anyhow::anyhow!("unsupported ingress admission outcome")), - request_method, - request_start, - connection, - ); - } - Err(error) => { - return prepare_detached_ingress_error( - &app, - error, - request_method.clone(), - request_start, - connection, - ); - } - }; + guard: RequestGuard, + ) -> Response { + let app = &self.app; let read_deadline = prepared.read_deadline(); let monotonic_clock = prepared.monotonic_clock(); - let Some(outbound_transport) = outbound_transport_handle else { + let Some(outbound_transport) = self.outbound_transport.clone() else { let egress = app.admitted_error_egress( prepared, EdgeError::internal(anyhow::anyhow!( "failed to initialize outbound HTTP transport" )), ); - return Ok(prepare_egress_response(egress, connection)); + return prepare_native_egress(egress, connection, guard); }; let mut core_request = match into_core_request_parts( parts, @@ -325,15 +298,15 @@ impl AxumServiceState { "failed to convert inbound request: {error}" )), ); - return Ok(prepare_egress_response(egress, connection)); + return prepare_native_egress(egress, connection, guard); } }; - if let Some(phase) = self.phase.clone() { core_request .extensions_mut() .insert(LifecycleReader::new(move || Some(*phase.borrow()))); } + let (config_registry, kv_registry, secret_registry) = self.resolved_store_registries(); if let Some(registry) = config_registry { core_request.extensions_mut().insert(registry); } @@ -343,9 +316,106 @@ impl AxumServiceState { if let Some(registry) = secret_registry { core_request.extensions_mut().insert(registry); } + let egress = app + .dispatch_admitted_with_framework_error_detail( + prepared, + core_request, + FrameworkErrorDetail::CategoryOnly, + ) + .await; + prepare_native_egress(egress, connection, guard) + } - let egress = app.dispatch_admitted(prepared, core_request).await; - Ok(prepare_egress_response(egress, connection)) + async fn dispatch_request( + &self, + mut request: Request, + connection: &EgressConnection, + remote_addr: Option, + ) -> Result, AxumIngressAbort> { + let request_start = self.app.monotonic_now(); + let mut guard = self.native.request(request.method(), request_start, self.app.monotonic_clock(), self.activity.clone()) + .map_err(|_error| { + let _logged = catch_unwind(|| log::error!(target: "edgezero::native", "event=lifecycle reason=request_identity_exhausted")); + AxumIngressAbort + })?; + // Identity and lifetime start before the gate, without admitting/body polling. + if !self.admission_open() { + guard.finish_ingress(IngressDisposition::Draining); + return Err(AxumIngressAbort); + } + // Only the owning socket supplies native peer facts, including an absent peer. + request.extensions_mut().remove::(); + if let Some(peer_addr) = remote_addr { + request.extensions_mut().insert(ConnectInfo(peer_addr)); + } else { + request.extensions_mut().remove::>(); + } + let app = Arc::clone(&self.app); + let (mut parts, native_body) = request.into_parts(); + let request_method = parts.method.clone(); + let received = + match validate_normalized_ingress_parts_with_summary(&parts, app.ingress_head_limits()) + { + Ok(summary) => summary, + Err(error) => { + return prepare_detached_ingress_error( + &app, + error, + request_method.clone(), + request_start, + connection, + guard, + ); + } + }; + let checked = normalize(&self.trusted_proxy_policy, remote_addr, &parts); + guard.set_forwarding_result(checked.forwarding_result()); + strip_forwarding_headers(&mut parts.headers); + parts.extensions.insert(checked.effective_host().clone()); + parts.extensions.insert(NativeIngressMetadata::new( + guard.request_id(), + checked, + received, + )); + let head_parts = IngressHeadParts::from_parts( + &parts, + IngressHeadAccounting::HostManaged, + IngressFraming::HostManaged, + ); + let prepared = match app.begin_ingress(head_parts, request_start) { + Ok(IngressBeginOutcome::Admitted(prepared)) => prepared, + Ok(IngressBeginOutcome::Refused(response)) => { + guard.refused(); + return Ok(prepare_native_egress(response, connection, guard)); + } + Ok(IngressBeginOutcome::Aborted) => { + guard.finish_ingress(IngressDisposition::Aborted); + return Err(AxumIngressAbort); + } + Ok(_) => { + return prepare_detached_ingress_error( + &app, + EdgeError::internal(anyhow::anyhow!("unsupported ingress admission outcome")), + request_method, + request_start, + connection, + guard, + ); + } + Err(error) => { + return prepare_detached_ingress_error( + &app, + error, + request_method.clone(), + request_start, + connection, + guard, + ); + } + }; + Ok(self + .dispatch_prepared(prepared, parts, native_body, connection, guard) + .await) } } @@ -356,6 +426,13 @@ pub(crate) struct AxumConnectionService { state: AxumServiceState, } +impl AxumConnectionService { + pub(crate) fn with_activity(mut self, activity: ConnectionActivity) -> Self { + self.state.activity = Some(activity); + self + } +} + impl HyperService> for AxumConnectionService { type Error = AxumIngressAbort; type Future = Pin>>>; @@ -401,13 +478,33 @@ fn prepare_detached_ingress_error( request_method: Method, request_start: MonotonicInstant, connection: &EgressConnection, + mut guard: RequestGuard, ) -> Result, AxumIngressAbort> { - match app.detached_ingress_error_egress(error, request_method, request_start) { - IngressDispatchOutcome::Response(envelope) => { - Ok(prepare_egress_response(*envelope, connection)) - } - IngressDispatchOutcome::Aborted | _ => Err(AxumIngressAbort), + guard.set_ingress_error_kind(error.kind()); + if let IngressDispatchOutcome::Response(envelope) = + app.detached_ingress_error_egress(error, request_method, request_start) + { + Ok(prepare_native_egress(*envelope, connection, guard)) + } else { + guard.finish_ingress(IngressDisposition::Aborted); + Err(AxumIngressAbort) + } +} + +fn prepare_native_egress( + envelope: edgezero_core::ResponseEgressEnvelope, + connection: &EgressConnection, + mut guard: RequestGuard, +) -> Response { + if let Some(failure) = envelope.failure_classification() { + guard.set_failure(Some(failure)); } + let (completion, selected) = guard.completion(); + prepare_egress_response( + envelope.with_completion(completion), + connection, + Some(&selected), + ) } #[cfg(test)] @@ -418,6 +515,9 @@ mod tests { )] use super::*; + use crate::diagnostics::{ + NativeRequestFailure, NativeRequestObserver, NativeRequestOutcome, NativeRequestRecord, + }; use bytes::{Bytes, BytesMut}; use edgezero_core::action; use edgezero_core::body::Body; @@ -863,6 +963,17 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn normalized_ingress_error_decision_abort_closes_without_response() { + #[derive(Clone)] + struct NativeRecords(Arc>>); + impl NativeRequestObserver for NativeRecords { + #[inline] + fn observe(&self, record: &NativeRequestRecord) { + self.0.lock().expect("native records").push(record.clone()); + } + } + let records = Arc::new(Mutex::new(Vec::new())); + let diagnostics = NativeDiagnosticsHandle::new() + .with_request_observer(NativeRecords(Arc::clone(&records))); let listener = TcpListener::bind("127.0.0.1:0") .await .expect("bind listener"); @@ -879,7 +990,8 @@ mod tests { observed_factory_calls.fetch_add(1, Ordering::SeqCst); DetachedResponseEgressDecision::Abort }); - let state = AxumServiceState::from_app(app); + let mut state = AxumServiceState::from_app(app); + state.native = NativeSession::new(diagnostics.clone(), false).expect("test native session"); let server = async move { let (stream, remote_addr) = listener.accept().await.expect("accept client"); @@ -913,6 +1025,23 @@ mod tests { } assert!(response.is_empty()); assert_eq!(factory_calls.load(Ordering::SeqCst), 1); + let captured = records.lock().expect("native records"); + assert_eq!(captured.len(), 1); + assert_eq!(captured[0].event, "request_ingress"); + assert_eq!( + captured[0].outcome, + NativeRequestOutcome::Ingress(IngressDisposition::Aborted) + ); + assert_eq!( + captured[0].failure, + Some(NativeRequestFailure::PropagatedError { + kind: "uri_too_long" + }) + ); + assert!(captured[0].status.is_none()); + assert!(captured[0].bytes_written.is_none()); + assert!(captured[0].body_kind.is_none()); + assert_eq!(diagnostics.snapshot().active_requests, 0); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -1977,6 +2106,1501 @@ mod tests { assert_eq!(body_at(&service, "/lookup/missing").await, "present=false"); } + mod managed_native { + use std::future::pending; + use std::net::{IpAddr, SocketAddr}; + + use http_body::Body as _; + use serde_json::{Value, json}; + + use crate::context::AxumRequestContext; + use crate::diagnostics::{ + NativeDiagnosticsSnapshot, NativeRequestFailure, NativeRequestObserver, + NativeRequestOutcome, NativeRequestRecord, + }; + use crate::proxy::{ForwardingHeaderFamily, ForwardingResult, Scheme}; + use edgezero_core::extractor::ForwardedHost; + use edgezero_core::http::{Authority, Uri}; + use edgezero_core::ingress::{CheckedEffectiveHost, CheckedHostSource, IngressHead}; + + use super::*; + + #[derive(Clone, Default)] + struct NativeProbe { + handle: NativeDiagnosticsHandle, + records: Arc>>, + snapshots: Arc>>, + events: Arc>>, + panics: bool, + } + + impl NativeRequestObserver for NativeProbe { + fn observe(&self, record: &NativeRequestRecord) { + self.records + .lock() + .expect("native records") + .push(record.clone()); + self.snapshots + .lock() + .expect("native snapshots") + .push(self.handle.snapshot()); + self.events.lock().expect("hook events").push("native"); + assert!(!self.panics, "native observer fixture panic"); + } + } + + struct OrderedAppObserver { + events: Arc>>, + reports: Arc>>, + } + + impl ResponseEgressObserver for OrderedAppObserver { + fn complete(&self, report: &ResponseEgressReport) { + self.events.lock().expect("hook events").push("observer"); + self.reports + .lock() + .expect("app reports") + .push(report.clone()); + } + } + + #[derive(Clone)] + struct PendingCalls(Arc); + + struct MetadataMiddleware(Arc>>); + + #[async_trait::async_trait(?Send)] + impl Middleware for MetadataMiddleware { + async fn handle( + &self, + ctx: RequestContext, + next: Next<'_>, + ) -> Result { + *self.0.lock().expect("middleware facts") = Some(context_facts(&ctx)); + next.run(ctx).await + } + } + + fn observed_service( + app: App, + policy: TrustedProxyPolicy, + probe: &NativeProbe, + ) -> AxumServiceState { + let handle = probe.handle.clone().with_request_observer(probe.clone()); + AxumServiceState::with_transport( + app, + AxumOutboundClient::try_transport().expect("fixture transport"), + JsonBodyLimit::DEFAULT, + NativeSession::new(handle, false).expect("fixture identity"), + policy, + ) + } + + fn tracked_connection(service: &mut AxumServiceState) -> ConnectionGuard { + let (activity, guard) = service.connection_activity(); + service.activity = Some(activity); + guard + } + + fn source_name(source: CheckedHostSource) -> &'static str { + match source { + CheckedHostSource::Direct => "direct", + CheckedHostSource::TrustedForwarded => "trusted_forwarded", + CheckedHostSource::TrustedXForwarded => "trusted_x_forwarded", + CheckedHostSource::Unavailable => "unavailable", + _ => "unknown", + } + } + + fn metadata_facts( + metadata: &NativeIngressMetadata, + headers: &HeaderMap, + uri: &Uri, + ) -> Value { + let summary = metadata.received_normalized_head(); + json!({ + "request_id": metadata.request_id().to_string(), + "peer": metadata.direct_peer().map(|peer| peer.to_string()), + "client": metadata.effective_client().map(|client| client.to_string()), + "host": metadata.effective_host().authority(), + "scheme": metadata.effective_scheme().as_str(), + "client_source": source_name(metadata.client_source()), + "host_source": source_name(metadata.host_source()), + "scheme_source": source_name(metadata.scheme_source()), + "forwarding_result": metadata.forwarding_result().as_str(), + "host_header": headers.get("host").and_then(|host| host.to_str().ok()), + "uri": uri.to_string(), + "raw_forwarding_absent": headers.keys().all(|name| name.as_str() != "forwarded" && !name.as_str().starts_with("x-forwarded-")), + "normalized": {"target_bytes": summary.target_bytes(), "header_bytes": summary.header_bytes(), "header_count": summary.header_count()}, + }) + } + + fn context_facts(ctx: &RequestContext) -> Value { + metadata_facts( + ctx.extensions() + .get::() + .expect("managed metadata"), + ctx.headers(), + ctx.uri(), + ) + } + + #[action] + async fn metadata_handler( + ctx: RequestContext, + ForwardedHost(host): ForwardedHost, + ) -> Result { + let peer = ctx + .extensions() + .get::() + .and_then(|native| native.remote_addr); + Ok(CoreResponse::new(Body::from( + json!({ + "facts": context_facts(&ctx), "forwarded_host": host, + "direct_context_peer": peer.map(|address| address.to_string()), + }) + .to_string(), + ))) + } + + #[action] + async fn mutate_metadata_handler(mut ctx: RequestContext) -> Result { + let original = ctx + .extensions_mut() + .remove::() + .expect("original metadata"); + ctx.extensions_mut() + .insert(CheckedEffectiveHost::from_ingress( + Some( + "application-mutated.example" + .parse::() + .expect("authority"), + ), + CheckedHostSource::Direct, + )); + Ok(original.request_id().to_string()) + } + + #[action] + async fn ordinary_status_handler(ctx: RequestContext) -> Result { + let status = if ctx.uri().path().ends_with("/413") { + StatusCode::PAYLOAD_TOO_LARGE + } else { + StatusCode::SERVICE_UNAVAILABLE + }; + Ok(response_builder() + .status(status) + .body(Body::from("application status")) + .expect("status response")) + } + + #[action] + async fn success_handler() -> Result<&'static str, EdgeError> { + Ok("ok") + } + + #[action] + async fn internal_error_handler() -> Result<&'static str, EdgeError> { + Err(EdgeError::internal(anyhow::anyhow!( + "PRIVATE_SERVICE_ERROR_SENTINEL" + ))) + } + + #[action] + async fn upstream_error_handler() -> Result<&'static str, EdgeError> { + Err(EdgeError::bad_gateway("PRIVATE_UPSTREAM_SENTINEL")) + } + + #[action] + async fn terminal_stream_handler(ctx: RequestContext) -> Result { + let source_failure = ctx.uri().path().ends_with("/source"); + let mut sent_prefix = false; + let stream = poll_fn(move |_context| { + if !sent_prefix { + sent_prefix = true; + return Poll::Ready(Some(Ok::<_, io::Error>(Bytes::from_static( + b"terminal-prefix", + )))); + } + if source_failure { + Poll::Ready(Some(Err(io::Error::other("PRIVATE_SOURCE_SENTINEL")))) + } else { + Poll::Pending + } + }); + Ok(CoreResponse::new(Body::from_external_stream(stream))) + } + + #[action] + async fn bad_framing_handler() -> Result { + Ok(response_builder() + .header("content-length", "1") + .body(Body::from("mismatched")) + .expect("framing response")) + } + + #[action] + async fn handler_not_found() -> Result<&'static str, EdgeError> { + Err(EdgeError::not_found("APPLICATION_CHOSEN_SENTINEL")) + } + + #[action] + async fn read_body_handler(ctx: RequestContext) -> Result { + ctx.extensions() + .get::() + .expect("handler calls") + .0 + .fetch_add(1, Ordering::SeqCst); + Ok(ctx.body_bytes(1024).await?.len().to_string()) + } + + #[action] + async fn pending_handler(ctx: RequestContext) -> Result<&'static str, EdgeError> { + ctx.extensions() + .get::() + .expect("handler calls") + .0 + .fetch_add(1, Ordering::SeqCst); + pending().await + } + + fn assert_one_released(probe: &NativeProbe) -> NativeRequestRecord { + let records = probe.records.lock().expect("native records"); + assert_eq!(records.len(), 1); + assert_eq!( + probe.snapshots.lock().expect("native snapshots")[0].active_requests, + 0 + ); + assert_eq!(probe.handle.snapshot().active_requests, 0); + records[0].clone() + } + + #[tokio::test] + #[expect( + clippy::too_many_lines, + reason = "The seven-case trust matrix keeps forged inputs, one managed dispatch, and all consumer-witness assertions together so each boundary proof is visible" + )] + async fn owned_peer_checked_metadata_precedes_all_managed_consumers() { + let trusted: SocketAddr = "192.0.2.2:4000".parse().expect("trusted peer"); + let untrusted: SocketAddr = "192.0.2.250:5000".parse().expect("untrusted peer"); + for ( + family, + peer, + enabled, + forwarded, + expected_host, + expected_scheme, + expected_client, + expected_result, + expected_source, + ) in [ + ( + ForwardingHeaderFamily::Forwarded, + Some(trusted), + false, + "for=203.0.113.7;host=public.example;proto=https", + "direct.example", + Scheme::Http, + Some(trusted.ip()), + ForwardingResult::UntrustedPeer, + CheckedHostSource::Direct, + ), + ( + ForwardingHeaderFamily::Forwarded, + Some(trusted), + true, + "for=203.0.113.7;host=public.example;proto=https", + "public.example", + Scheme::Https, + Some("203.0.113.7".parse::().expect("client")), + ForwardingResult::Accepted, + CheckedHostSource::TrustedForwarded, + ), + ( + ForwardingHeaderFamily::Forwarded, + None, + true, + "for=203.0.113.7;host=public.example;proto=https", + "direct.example", + Scheme::Http, + None, + ForwardingResult::UntrustedPeer, + CheckedHostSource::Direct, + ), + ( + ForwardingHeaderFamily::Forwarded, + Some(untrusted), + true, + "for=203.0.113.7;host=public.example;proto=https", + "direct.example", + Scheme::Http, + Some(untrusted.ip()), + ForwardingResult::UntrustedPeer, + CheckedHostSource::Direct, + ), + ( + ForwardingHeaderFamily::XForwarded, + Some(trusted), + true, + "for=203.0.113.7;host=public.example;proto=https", + "x-public.example", + Scheme::Https, + Some("203.0.113.9".parse::().expect("client")), + ForwardingResult::Accepted, + CheckedHostSource::TrustedXForwarded, + ), + ( + ForwardingHeaderFamily::Forwarded, + Some(trusted), + true, + "for=bad-address;host=public.example;proto=https", + "direct.example", + Scheme::Http, + Some(trusted.ip()), + ForwardingResult::Malformed, + CheckedHostSource::Direct, + ), + ( + ForwardingHeaderFamily::Forwarded, + Some(trusted), + true, + "for=unknown;host=public.example;proto=https", + "public.example", + Scheme::Https, + Some(trusted.ip()), + ForwardingResult::UnresolvedChain, + CheckedHostSource::TrustedForwarded, + ), + ] { + let probe = NativeProbe::default(); + let admission = Arc::new(Mutex::new(None)); + let middleware = Arc::new(Mutex::new(None)); + let captured_admission = Arc::clone(&admission); + let admission_handle = probe.handle.clone(); + let router = RouterService::builder() + .get("/native/{id}", metadata_handler) + .middleware(MetadataMiddleware(Arc::clone(&middleware))) + .build(); + let mut app = App::new(router); + app.set_ingress_admission_policy(move |head| { + assert!(matches!( + head.head_accounting(), + IngressHeadAccounting::HostManaged + )); + assert!(matches!(head.framing(), IngressFraming::HostManaged)); + assert_eq!(admission_handle.snapshot().active_requests, 1); + let metadata = head + .extension::() + .expect("admission metadata"); + assert_eq!( + head.extension::() + .expect("checked host"), + metadata.effective_host() + ); + *captured_admission.lock().expect("admission facts") = + Some(metadata_facts(metadata, head.headers(), head.target())); + AdmissionDecision::Admit { + completion: ResponseEgressCompletion::empty(), + grant: IngressGrant::empty(), + read_deadline: head.read_deadline_after(Duration::from_secs(30)), + } + }); + let policy = if enabled { + TrustedProxyPolicy::new(family, ["192.0.2.2"]).expect("policy") + } else { + TrustedProxyPolicy::no_trust() + }; + let mut service = observed_service(app, policy, &probe); + let owned_connection = tracked_connection(&mut service); + let incoming = Request::builder() + .uri("https://direct.example/native/SECRET-PATH?token=SECRET-QUERY") + .header("host", "direct.example") + .header("forwarded", forwarded) + .header("x-forwarded-for", "203.0.113.9") + .header("x-forwarded-host", "x-public.example") + .header("x-forwarded-proto", "https") + .header("X-FoRwArDeD-Private-Secret", "SECRET-HEADER") + .header("x-request-id", "SECRET-VISITOR-ID") + .body(AxumBody::empty()) + .expect("request"); + let (mut parts, body) = incoming.into_parts(); + let received = validate_normalized_ingress_parts_with_summary( + &parts, + IngressHeadLimits::default(), + ) + .expect("original summary"); + let foreign = NativeSession::new(NativeDiagnosticsHandle::new(), false) + .expect("foreign session"); + let foreign_guard = foreign + .request( + &Method::GET, + MonotonicInstant::now(), + MonotonicClock::default(), + None, + ) + .expect("foreign request"); + let foreign_id = foreign_guard.request_id(); + let claimed = normalize( + &TrustedProxyPolicy::new(ForwardingHeaderFamily::Forwarded, ["192.0.2.2"]) + .expect("forged policy"), + Some(trusted), + &parts, + ); + parts + .extensions + .insert(NativeIngressMetadata::new(foreign_id, claimed, received)); + parts.extensions.insert(CheckedEffectiveHost::from_ingress( + Some("FORGED-HOST.example".parse().expect("forged authority")), + CheckedHostSource::TrustedForwarded, + )); + parts + .extensions + .insert(ConnectInfo(if peer == Some(trusted) { + untrusted + } else { + trusted + })); + parts.extensions.insert(AxumRequestContext { + remote_addr: Some(if peer == Some(trusted) { + untrusted + } else { + trusted + }), + }); + drop(foreign_guard); + let response = service + .dispatch_request( + Request::from_parts(parts, body), + &EgressConnection::default(), + peer, + ) + .await + .expect("managed response"); + assert!(!response.headers().contains_key("x-request-id")); + assert!(!response.headers().contains_key("request-id")); + assert_eq!(probe.handle.snapshot().active_requests, 1); + let result: Value = serde_json::from_slice( + &to_bytes(response.into_body(), 8192) + .await + .expect("metadata body"), + ) + .expect("metadata JSON"); + let facts = &result["facts"]; + assert_eq!( + facts, + admission + .lock() + .expect("admission facts") + .as_ref() + .expect("observed admission") + ); + assert_eq!( + facts, + middleware + .lock() + .expect("middleware facts") + .as_ref() + .expect("observed middleware") + ); + assert_eq!( + facts["peer"], + json!(peer.map(|address| address.to_string())) + ); + assert_eq!( + facts["client"], + json!(expected_client.map(|address| address.to_string())) + ); + assert_eq!(facts["host"], expected_host); + assert_eq!(facts["scheme"], expected_scheme.as_str()); + assert_eq!(facts["host_source"], source_name(expected_source)); + assert_eq!(facts["scheme_source"], source_name(expected_source)); + let client_source = if peer.is_none() { + CheckedHostSource::Unavailable + } else if expected_result == ForwardingResult::Accepted { + expected_source + } else { + CheckedHostSource::Direct + }; + assert_eq!(facts["client_source"], source_name(client_source)); + assert_eq!(facts["forwarding_result"], expected_result.as_str()); + assert_eq!(facts["host_header"], "direct.example"); + assert_eq!( + facts["uri"], + "https://direct.example/native/SECRET-PATH?token=SECRET-QUERY" + ); + assert_eq!(facts["raw_forwarding_absent"], true); + assert_eq!( + facts["normalized"], + json!({"target_bytes":received.target_bytes(), "header_bytes":received.header_bytes(), "header_count":received.header_count()}) + ); + assert_eq!(result["forwarded_host"], expected_host); + assert_eq!( + result["direct_context_peer"], + json!(peer.map(|address| address.to_string())) + ); + assert_ne!(facts["request_id"], foreign_id.to_string()); + assert_ne!(facts["request_id"], "SECRET-VISITOR-ID"); + let record = assert_one_released(&probe); + assert_eq!(facts["request_id"], record.request_id.to_string()); + assert_eq!(record.route, "/native/{id}"); + assert_eq!(record.forwarding_result, expected_result); + assert_eq!(record.status, Some(StatusCode::OK)); + assert_eq!( + record.level(), + if expected_result == ForwardingResult::Malformed { + log::Level::Warn + } else { + log::Level::Info + } + ); + assert_eq!(probe.handle.snapshot().idle_connections, 1); + drop(owned_connection); + assert_eq!( + probe.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + } + + #[tokio::test] + async fn original_forwarding_head_overages_reject_before_stripping_or_body_poll() { + for limits in [ + IngressHeadLimits::default() + .with_max_request_header_count(1) + .expect("field limit"), + IngressHeadLimits::default() + .with_max_request_header_bytes(32) + .expect("byte limit"), + ] { + let probe = NativeProbe::default(); + let body_polls = Arc::new(AtomicUsize::new(0)); + let source_drops = Arc::new(AtomicUsize::new(0)); + let source_guard = DropSignal(Arc::clone(&source_drops)); + let counted_polls = Arc::clone(&body_polls); + let body = AxumBody::from_stream(poll_fn(move |_context| { + let _retained = &source_guard; + counted_polls.fetch_add(1, Ordering::SeqCst); + Poll::>>::Pending + })); + let admission_calls = Arc::new(AtomicUsize::new(0)); + let counted_admission = Arc::clone(&admission_calls); + let app_reports = Arc::new(Mutex::new(Vec::new())); + let mut app = App::new( + RouterService::builder() + .post("/head", success_handler) + .build(), + ); + app.set_ingress_head_limits(limits); + app.set_response_egress_observer(RecordingEgressObserver(Arc::clone(&app_reports))); + app.set_ingress_admission_policy(move |_head| { + counted_admission.fetch_add(1, Ordering::SeqCst); + AdmissionDecision::Abort + }); + let service = observed_service(app, TrustedProxyPolicy::no_trust(), &probe); + let incoming = Request::builder() + .method("POST") + .uri("/head") + .header("host", "d.test") + .header( + "forwarded", + "for=203.0.113.8;host=PUBLIC-SECRET.example;proto=https", + ) + .header("x-forwarded-private", "SECRET-HEADER") + .body(body) + .expect("head request"); + let (mut parts, native_body) = incoming.into_parts(); + let saved = parts.headers.clone(); + strip_forwarding_headers(&mut parts.headers); + validate_normalized_ingress_parts_with_summary(&parts, limits) + .expect("stripped head alone would fit"); + parts.headers = saved; + let response = service + .dispatch_request( + Request::from_parts(parts, native_body), + &EgressConnection::default(), + None, + ) + .await + .expect("detached head response"); + assert_eq!( + response.status(), + StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE + ); + assert_eq!(body_polls.load(Ordering::SeqCst), 0); + assert_eq!(admission_calls.load(Ordering::SeqCst), 0); + assert_eq!(source_drops.load(Ordering::SeqCst), 1); + let rendered = to_bytes(response.into_body(), 1024) + .await + .expect("head error body"); + assert!(!String::from_utf8_lossy(&rendered).contains("SECRET")); + let record = assert_one_released(&probe); + assert_eq!( + record.failure, + Some(NativeRequestFailure::PropagatedError { + kind: "request_header_fields_too_large" + }) + ); + assert_eq!( + record.status, + Some(StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE) + ); + assert_eq!(record.forwarding_result, ForwardingResult::Absent); + assert!(app_reports.lock().expect("detached app reports").is_empty()); + } + } + + #[tokio::test] + async fn public_metadata_mutation_cannot_change_private_record_identity_or_add_response_id() + { + let probe = NativeProbe::default(); + let router = RouterService::builder() + .get("/mutate", mutate_metadata_handler) + .build(); + let service = + observed_service(App::new(router), TrustedProxyPolicy::no_trust(), &probe); + let request = Request::builder() + .uri("/mutate") + .header("x-request-id", "VISITOR-SENTINEL") + .body(AxumBody::empty()) + .expect("request"); + let response = service + .dispatch_request(request, &EgressConnection::default(), None) + .await + .expect("response"); + assert!(!response.headers().contains_key("x-request-id")); + assert!(!response.headers().contains_key("request-id")); + let identity = to_bytes(response.into_body(), 1024) + .await + .expect("identity body"); + let record = assert_one_released(&probe); + assert_eq!(identity.as_ref(), record.request_id.to_string().as_bytes()); + assert_eq!(identity.len(), 64); + assert_ne!(identity, "VISITOR-SENTINEL"); + } + + #[tokio::test] + #[expect( + clippy::too_many_lines, + reason = "The abort, drain and refusal matrix shares lifetime fixtures while checking their distinct no-response versus detached-completion contracts" + )] + async fn native_abort_drain_and_refusal_release_once_without_inventing_egress() { + for mode in ["abort", "drain", "refuse"] { + let probe = NativeProbe::default(); + let body_polls = Arc::new(AtomicUsize::new(0)); + let observed_polls = Arc::clone(&body_polls); + let source_drops = Arc::new(AtomicUsize::new(0)); + let source_guard = DropSignal(Arc::clone(&source_drops)); + let body = AxumBody::from_stream(poll_fn(move |_context| { + let _retained = &source_guard; + observed_polls.fetch_add(1, Ordering::SeqCst); + Poll::>>::Pending + })); + let app_reports = Arc::new(Mutex::new(Vec::new())); + let completion_calls = Arc::new(AtomicUsize::new(0)); + let counted_completion = Arc::clone(&completion_calls); + let admission_calls = Arc::new(AtomicUsize::new(0)); + let counted_admission = Arc::clone(&admission_calls); + let admission_handle = probe.handle.clone(); + let mut app = App::new( + RouterService::builder() + .post("/gate", success_handler) + .build(), + ); + app.set_response_egress_observer(RecordingEgressObserver(Arc::clone(&app_reports))); + app.set_ingress_admission_policy(move |_head| { + counted_admission.fetch_add(1, Ordering::SeqCst); + assert_eq!(admission_handle.snapshot().active_requests, 1); + if mode == "refuse" { + let completed = Arc::clone(&counted_completion); + AdmissionDecision::Refuse { + completion: ResponseEgressCompletion::new(move |_report| { + completed.fetch_add(1, Ordering::SeqCst); + }), + response: response_builder() + .status(StatusCode::SERVICE_UNAVAILABLE) + .body(Body::from("admission refusal")) + .expect("refusal"), + } + } else { + AdmissionDecision::Abort + } + }); + let mut service = observed_service(app, TrustedProxyPolicy::no_trust(), &probe); + let (_phase_sender, phase) = watch::channel(if mode == "drain" { + LifecyclePhase::Draining + } else { + LifecyclePhase::Ready + }); + service.phase = Some(phase); + let owned_connection = tracked_connection(&mut service); + let request = Request::builder() + .method("POST") + .uri("/gate") + .body(body) + .expect("request"); + let outcome = service + .dispatch_request(request, &EgressConnection::default(), None) + .await; + if mode == "refuse" { + let response = outcome.expect("refusal response"); + assert_eq!(probe.handle.snapshot().active_requests, 1); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + to_bytes(response.into_body(), 1024) + .await + .expect("refusal body"), + "admission refusal" + ); + let record = assert_one_released(&probe); + assert_eq!(record.status, Some(StatusCode::SERVICE_UNAVAILABLE)); + assert_eq!(record.failure, Some(NativeRequestFailure::AdmissionRefused)); + assert_eq!( + record.outcome, + NativeRequestOutcome::Egress(ResponseEgressOutcome::HostHandoff) + ); + assert_eq!(completion_calls.load(Ordering::SeqCst), 1); + // Refusal retains its completion, not the admitted-request observer. + assert!(app_reports.lock().expect("app reports").is_empty()); + } else { + let Err(_aborted) = outcome else { + panic!("no response expected on abort or drain"); + }; + let record = assert_one_released(&probe); + assert_eq!(record.event, "request_ingress"); + assert_eq!(record.status, None); + assert_eq!(record.bytes_written, None); + assert_eq!(record.body_kind, None); + assert_eq!( + record.outcome, + NativeRequestOutcome::Ingress(if mode == "drain" { + IngressDisposition::Draining + } else { + IngressDisposition::Aborted + }) + ); + assert_eq!(completion_calls.load(Ordering::SeqCst), 0); + assert!(app_reports.lock().expect("app reports").is_empty()); + } + assert_eq!( + admission_calls.load(Ordering::SeqCst), + usize::from(mode != "drain") + ); + assert_eq!(body_polls.load(Ordering::SeqCst), 0); + assert_eq!(source_drops.load(Ordering::SeqCst), 1); + assert_eq!(probe.handle.snapshot().idle_connections, 1); + drop(owned_connection); + assert_eq!( + probe.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + } + + #[tokio::test] + #[expect( + clippy::too_many_lines, + reason = "Three cancellation stages retain one fixture and its before-drop and after-drop assertions for native, body, grant and completion ownership" + )] + async fn cancelled_body_handler_and_fallback_futures_release_native_and_app_resources() { + for stage in ["body", "handler", "fallback"] { + let probe = NativeProbe::default(); + let body_polls = Arc::new(AtomicUsize::new(0)); + let observed_polls = Arc::clone(&body_polls); + let source_drops = Arc::new(AtomicUsize::new(0)); + let source_guard = DropSignal(Arc::clone(&source_drops)); + let body_handle = probe.handle.clone(); + let body = AxumBody::from_stream(poll_fn(move |_context| { + let _retained = &source_guard; + assert_eq!(body_handle.snapshot().active_requests, 1); + observed_polls.fetch_add(1, Ordering::SeqCst); + Poll::>>::Pending + })); + let handler_calls = Arc::new(AtomicUsize::new(0)); + let router = match stage { + "body" => RouterService::builder() + .post("/pending", read_body_handler) + .build(), + "handler" => RouterService::builder() + .post("/pending", pending_handler) + .build(), + _ => RouterService::builder().build(), + }; + let app_reports = Arc::new(Mutex::new(Vec::new())); + let completion_calls = Arc::new(AtomicUsize::new(0)); + let retained_drops = Arc::new(AtomicUsize::new(0)); + let grant_drops = Arc::new(AtomicUsize::new(0)); + let captured_completions = Arc::clone(&completion_calls); + let captured_retained = Arc::clone(&retained_drops); + let captured_grants = Arc::clone(&grant_drops); + let mut app = App::new(router); + app.set_response_egress_observer(RecordingEgressObserver(Arc::clone(&app_reports))); + app.set_ingress_admission_policy(move |head| { + let calls = Arc::clone(&captured_completions); + let retained = DropSignal(Arc::clone(&captured_retained)); + let completion = ResponseEgressCompletion::new(move |_report| { + let _resource = retained; + calls.fetch_add(1, Ordering::SeqCst); + }); + let grant = IngressGrant::new(DropSignal(Arc::clone(&captured_grants))); + if stage == "fallback" { + AdmissionDecision::ReadBodyBeforeFallback { + completion, + grant, + max_body_bytes: 1024, + read_deadline: head.read_deadline_after(Duration::from_secs(30)), + on_exceeded: BufferedIngressResponse::text(StatusCode::OK, "overflow"), + on_timeout: BufferedIngressResponse::text(StatusCode::OK, "timeout"), + } + } else { + AdmissionDecision::Admit { + completion, + grant, + read_deadline: head.read_deadline_after(Duration::from_secs(30)), + } + } + }); + let mut service = observed_service(app, TrustedProxyPolicy::no_trust(), &probe); + let owned_connection = tracked_connection(&mut service); + let mut request = Request::builder() + .method("POST") + .uri("/pending") + .body(body) + .expect("request"); + request + .extensions_mut() + .insert(PendingCalls(Arc::clone(&handler_calls))); + let connection = EgressConnection::default(); + let mut response = Box::pin(service.dispatch_request(request, &connection, None)); + assert!(matches!( + futures_util::poll!(response.as_mut()), + Poll::Pending + )); + assert_eq!( + probe.handle.snapshot(), + NativeDiagnosticsSnapshot { + active_requests: 1, + active_connections: 1, + idle_connections: 0 + } + ); + assert_eq!(source_drops.load(Ordering::SeqCst), 0); + assert_eq!(retained_drops.load(Ordering::SeqCst), 0); + assert_eq!(grant_drops.load(Ordering::SeqCst), 0); + assert_eq!( + handler_calls.load(Ordering::SeqCst), + usize::from(stage != "fallback") + ); + assert_eq!(body_polls.load(Ordering::SeqCst) > 0, stage != "handler"); + drop(response); + assert_eq!(source_drops.load(Ordering::SeqCst), 1); + assert_eq!(retained_drops.load(Ordering::SeqCst), 1); + assert_eq!(grant_drops.load(Ordering::SeqCst), 1); + assert_eq!(completion_calls.load(Ordering::SeqCst), 0); + assert!(app_reports.lock().expect("app reports").is_empty()); + let record = assert_one_released(&probe); + assert_eq!( + record.outcome, + NativeRequestOutcome::Ingress(IngressDisposition::Abandoned) + ); + assert_eq!(record.status, None); + assert_eq!(record.bytes_written, None); + assert_eq!(probe.handle.snapshot().idle_connections, 1); + drop(owned_connection); + assert_eq!( + probe.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + } + + #[tokio::test] + async fn prepared_response_drop_is_one_failing_egress_record_with_original_status() { + let probe = NativeProbe::default(); + let app_reports = Arc::new(Mutex::new(Vec::new())); + let mut app = App::new( + RouterService::builder() + .get("/drop", success_handler) + .build(), + ); + app.set_response_egress_observer(RecordingEgressObserver(Arc::clone(&app_reports))); + let mut service = observed_service(app, TrustedProxyPolicy::no_trust(), &probe); + let owned_connection = tracked_connection(&mut service); + let response = service + .dispatch_request( + Request::builder() + .uri("/drop") + .body(AxumBody::empty()) + .expect("request"), + &EgressConnection::default(), + None, + ) + .await + .expect("response"); + assert_eq!(probe.handle.snapshot().active_requests, 1); + drop(response); + let record = assert_one_released(&probe); + assert_eq!(record.event, "request_terminal"); + assert_eq!(record.status, Some(StatusCode::OK)); + assert_eq!( + record.outcome, + NativeRequestOutcome::Egress(ResponseEgressOutcome::TransportError) + ); + assert_eq!(record.level(), log::Level::Warn); + assert_eq!(app_reports.lock().expect("app reports").len(), 1); + drop(owned_connection); + assert_eq!( + probe.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + + #[tokio::test] + async fn managed_native_observer_panic_preserves_release_and_application_observer() { + for abort in [false, true] { + let probe = NativeProbe { + panics: true, + ..NativeProbe::default() + }; + let app_reports = Arc::new(Mutex::new(Vec::new())); + let mut app = App::new( + RouterService::builder() + .get("/observer", success_handler) + .build(), + ); + app.set_response_egress_observer(RecordingEgressObserver(Arc::clone(&app_reports))); + if abort { + app.set_ingress_admission_policy(|_head| AdmissionDecision::Abort); + } + let service = observed_service(app, TrustedProxyPolicy::no_trust(), &probe); + let request = Request::builder() + .uri("/observer") + .body(AxumBody::empty()) + .expect("request"); + let outcome = service + .dispatch_request(request, &EgressConnection::default(), None) + .await; + if abort { + let Err(_aborted) = outcome else { + panic!("abort produced a response"); + }; + assert_eq!(assert_one_released(&probe).status, None); + assert!(app_reports.lock().expect("app reports").is_empty()); + } else { + let response = outcome.expect("response despite observer panic"); + assert_eq!( + to_bytes(response.into_body(), 1024) + .await + .expect("body despite observer panic"), + "ok" + ); + assert_eq!(assert_one_released(&probe).status, Some(StatusCode::OK)); + assert_eq!(app_reports.lock().expect("app reports").len(), 1); + } + } + } + + #[tokio::test] + async fn configured_fallback_200_has_owner_failure_but_deliberate_413_and_503_do_not() { + for timed_out in [false, true] { + let probe = NativeProbe::default(); + let mut app = App::new(RouterService::builder().build()); + app.set_ingress_admission_policy(move |head| { + AdmissionDecision::ReadBodyBeforeFallback { + completion: ResponseEgressCompletion::empty(), + grant: IngressGrant::empty(), + max_body_bytes: 1, + read_deadline: if timed_out { + Deadline::at_instant(head.request_start()) + } else { + head.read_deadline_after(Duration::from_secs(30)) + }, + on_exceeded: BufferedIngressResponse::text( + StatusCode::OK, + "configured overflow", + ), + on_timeout: BufferedIngressResponse::text( + StatusCode::OK, + "configured timeout", + ), + } + }); + let service = observed_service(app, TrustedProxyPolicy::no_trust(), &probe); + let request = Request::builder() + .method("POST") + .uri("/missing") + .body(AxumBody::from("ab")) + .expect("fallback request"); + let response = service + .dispatch_request(request, &EgressConnection::default(), None) + .await + .expect("configured response"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + to_bytes(response.into_body(), 1024) + .await + .expect("configured body"), + if timed_out { + "configured timeout" + } else { + "configured overflow" + } + ); + let record = assert_one_released(&probe); + assert_eq!( + record.failure, + Some(if timed_out { + NativeRequestFailure::FallbackReadTimedOut + } else { + NativeRequestFailure::FallbackBodyExceeded + }) + ); + assert_eq!(record.status, Some(StatusCode::OK)); + assert_eq!(record.level(), log::Level::Warn); + } + for (path, status) in [ + ("/ordinary/413", StatusCode::PAYLOAD_TOO_LARGE), + ("/ordinary/503", StatusCode::SERVICE_UNAVAILABLE), + ] { + let probe = NativeProbe::default(); + let service = observed_service( + App::new( + RouterService::builder() + .get("/ordinary/{status}", ordinary_status_handler) + .build(), + ), + TrustedProxyPolicy::no_trust(), + &probe, + ); + let request = Request::builder() + .uri(path) + .body(AxumBody::empty()) + .expect("ordinary request"); + let response = service + .dispatch_request(request, &EgressConnection::default(), None) + .await + .expect("ordinary response"); + assert_eq!(response.status(), status); + to_bytes(response.into_body(), 1024) + .await + .expect("ordinary body"); + let record = assert_one_released(&probe); + assert_eq!(record.failure, None); + assert_eq!(record.status, Some(status)); + assert_eq!(record.level(), log::Level::Info); + } + } + + #[tokio::test] + async fn native_json_overflow_observes_owning_413_even_with_custom_error_body() { + let probe = NativeProbe::default(); + let mut app = App::new( + RouterService::builder() + .post("/json", json_limit_probe) + .build(), + ); + app.set_error_response_renderer(|_error| { + CoreResponse::new(Body::from("custom JSON body")) + }); + let mut service = observed_service(app, TrustedProxyPolicy::no_trust(), &probe); + service.json_body_limit = JsonBodyLimit::new(4).expect("JSON limit"); + let request = Request::builder() + .method("POST") + .uri("/json") + .header("content-type", "application/json") + .body(AxumBody::from("\"abcdef\"")) + .expect("JSON request"); + let response = service + .dispatch_request(request, &EgressConnection::default(), None) + .await + .expect("JSON response"); + assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!( + to_bytes(response.into_body(), 1024) + .await + .expect("JSON body"), + "custom JSON body" + ); + let record = assert_one_released(&probe); + assert_eq!(record.status, Some(StatusCode::PAYLOAD_TOO_LARGE)); + assert_eq!( + record.failure, + Some(NativeRequestFailure::PropagatedError { + kind: "payload_too_large" + }) + ); + assert_eq!(record.route, "/json"); + assert_eq!(record.level(), log::Level::Warn); + } + + #[tokio::test] + #[expect( + clippy::too_many_lines, + reason = "Framework routing, handler errors and custom rendering are paired negative controls that keep privacy and provenance assertions in one matrix" + )] + async fn native_routing_privacy_preserves_router_provenance_allow_and_application_control() + { + for (path, method, status, framework, kind, public_sentinel) in [ + ( + "/missing/FRAMEWORK_PATH_SENTINEL", + "GET", + StatusCode::NOT_FOUND, + true, + "not_found", + false, + ), + ( + "/known", + "PRIVATE-METHOD-SENTINEL", + StatusCode::METHOD_NOT_ALLOWED, + true, + "method_not_allowed", + false, + ), + ( + "/handler404", + "GET", + StatusCode::NOT_FOUND, + false, + "not_found", + true, + ), + ( + "/internal", + "GET", + StatusCode::INTERNAL_SERVER_ERROR, + false, + "internal", + false, + ), + ( + "/upstream", + "GET", + StatusCode::BAD_GATEWAY, + false, + "bad_gateway", + false, + ), + ] { + let probe = NativeProbe::default(); + let router = RouterService::builder() + .get("/known", success_handler) + .get("/handler404", handler_not_found) + .get("/internal", internal_error_handler) + .get("/upstream", upstream_error_handler) + .build(); + let service = + observed_service(App::new(router), TrustedProxyPolicy::no_trust(), &probe); + let request = Request::builder() + .method(method) + .uri(path) + .body(AxumBody::empty()) + .expect("error request"); + let response = service + .dispatch_request(request, &EgressConnection::default(), None) + .await + .expect("error response"); + assert_eq!(response.status(), status); + if status == StatusCode::METHOD_NOT_ALLOWED { + assert_eq!(response.headers().get("allow").expect("Allow"), "GET"); + } + let body = to_bytes(response.into_body(), 2048) + .await + .expect("error body"); + assert_eq!( + String::from_utf8_lossy(&body).contains("SENTINEL"), + public_sentinel + ); + let record = assert_one_released(&probe); + assert_eq!( + record.failure, + Some(if framework { + NativeRequestFailure::FrameworkRouting { kind } + } else { + NativeRequestFailure::PropagatedError { kind } + }) + ); + assert_eq!( + record.level(), + if framework { + log::Level::Info + } else { + log::Level::Warn + } + ); + assert_eq!(record.method, if method == "GET" { "GET" } else { "other" }); + } + let probe = NativeProbe::default(); + let mut app = App::new(RouterService::builder().build()); + app.set_error_response_renderer(|error| { + let chosen_body = error.to_string(); + assert!(chosen_body.contains("CUSTOM_ROUTE_SENTINEL")); + CoreResponse::new(Body::from(chosen_body)) + }); + let service = observed_service(app, TrustedProxyPolicy::no_trust(), &probe); + let request = Request::builder() + .uri("/CUSTOM_ROUTE_SENTINEL") + .body(AxumBody::empty()) + .expect("custom request"); + let response = service + .dispatch_request(request, &EgressConnection::default(), None) + .await + .expect("custom response"); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert!( + String::from_utf8_lossy( + &to_bytes(response.into_body(), 1024) + .await + .expect("custom body") + ) + .contains("CUSTOM_ROUTE_SENTINEL") + ); + assert_eq!( + assert_one_released(&probe).failure, + Some(NativeRequestFailure::FrameworkRouting { kind: "not_found" }) + ); + } + + #[tokio::test] + async fn managed_source_and_deadline_failures_keep_selected_200_prefix_accounting_and_one_record() + { + for failure in [ + ResponseEgressOutcome::SourceError, + ResponseEgressOutcome::DeadlineExceeded, + ] { + let probe = NativeProbe::default(); + let app_reports = Arc::new(Mutex::new(Vec::new())); + let start = MonotonicInstant::now(); + let now = Arc::new(Mutex::new(start)); + let captured_clock = Arc::clone(&now); + let mut app = App::new( + RouterService::builder() + .get("/stream/{failure}", terminal_stream_handler) + .build(), + ); + app.set_monotonic_clock(MonotonicClock::new(move || { + *captured_clock.lock().expect("clock") + })); + app.set_response_egress_observer(RecordingEgressObserver(Arc::clone(&app_reports))); + app.set_response_egress_policy(|_head, egress_started_at| ResponseEgressPolicy { + write_deadline: Deadline::at_instant( + egress_started_at + .checked_add(Duration::from_secs(1)) + .expect("write deadline"), + ), + }); + let mut service = observed_service(app, TrustedProxyPolicy::no_trust(), &probe); + let owned_connection = tracked_connection(&mut service); + let path = if failure == ResponseEgressOutcome::SourceError { + "/stream/source" + } else { + "/stream/deadline" + }; + let request = Request::builder() + .uri(path) + .body(AxumBody::empty()) + .expect("stream request"); + let response = service + .dispatch_request(request, &EgressConnection::default(), None) + .await + .expect("stream response"); + assert_eq!(response.status(), StatusCode::OK); + let mut body = response.into_body(); + let prefix = poll_future(|context| Pin::new(&mut body).poll_frame(context)) + .await + .expect("prefix frame") + .expect("prefix data") + .into_data() + .expect("data frame"); + assert_eq!(prefix, "terminal-prefix"); + assert_eq!(probe.handle.snapshot().active_requests, 1); + assert!(probe.records.lock().expect("native records").is_empty()); + if failure == ResponseEgressOutcome::DeadlineExceeded { + *now.lock().expect("clock") = start + .checked_add(Duration::from_secs(1)) + .expect("expired instant"); + } + let terminal_error = poll_future(|context| Pin::new(&mut body).poll_frame(context)) + .await + .expect("terminal frame") + .expect_err("terminal failure"); + assert!(!terminal_error.to_string().contains("SENTINEL")); + // Hyper stops after a body error; an ended body need not be repolled. + assert!(body.is_end_stream()); + drop(body); + let record = assert_one_released(&probe); + assert_eq!(record.status, Some(StatusCode::OK)); + assert_eq!(record.outcome, NativeRequestOutcome::Egress(failure)); + assert_eq!(record.bytes_written, Some(15)); + assert_eq!(record.fallback_disposition, None); + assert_eq!(record.route, "/stream/{failure}"); + assert_eq!(record.level(), log::Level::Warn); + assert_eq!(app_reports.lock().expect("app reports").len(), 1); + assert_eq!(app_reports.lock().expect("app reports")[0].outcome, failure); + drop(owned_connection); + assert_eq!( + probe.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + } + + #[tokio::test] + async fn managed_precommit_framing_fallback_records_selected_500_and_original_cause() { + let probe = NativeProbe::default(); + let service = observed_service( + App::new( + RouterService::builder() + .get("/framing", bad_framing_handler) + .build(), + ), + TrustedProxyPolicy::no_trust(), + &probe, + ); + let request = Request::builder() + .uri("/framing") + .body(AxumBody::empty()) + .expect("request"); + let response = service + .dispatch_request(request, &EgressConnection::default(), None) + .await + .expect("fallback response"); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + let body = to_bytes(response.into_body(), 1024) + .await + .expect("fallback body"); + assert!(!String::from_utf8_lossy(&body).contains("mismatched")); + let record = assert_one_released(&probe); + assert_eq!(record.status, Some(StatusCode::INTERNAL_SERVER_ERROR)); + assert_eq!( + record.outcome, + NativeRequestOutcome::Egress(ResponseEgressOutcome::ConversionError) + ); + assert_eq!( + record.body_kind, + Some(edgezero_core::ResponseEgressBodyKind::Fallback) + ); + assert_eq!( + record.fallback_disposition, + Some(ResponseEgressFallbackDisposition::Completed) + ); + } + + #[tokio::test] + async fn managed_hooks_keep_app_native_observer_order_and_authoritative_total_duration() { + let probe = NativeProbe::default(); + let start = MonotonicInstant::now(); + let now = Arc::new(Mutex::new(start)); + let source = Arc::clone(&now); + let admission_now = Arc::clone(&now); + let completion_now = Arc::clone(&now); + let completion_events = Arc::clone(&probe.events); + let reports = Arc::new(Mutex::new(Vec::new())); + let mut app = App::new( + RouterService::builder() + .get("/timed", success_handler) + .build(), + ); + app.set_monotonic_clock(MonotonicClock::new(move || *source.lock().expect("clock"))); + app.set_ingress_admission_policy(move |head: &IngressHead| { + assert_eq!(head.request_start(), start); + *admission_now.lock().expect("clock") = start + .checked_add(Duration::from_secs(3)) + .expect("pre-egress instant"); + let events = Arc::clone(&completion_events); + let observed_clock = Arc::clone(&completion_now); + AdmissionDecision::Admit { + completion: ResponseEgressCompletion::new(move |_report| { + events.lock().expect("hook events").push("app"); + *observed_clock.lock().expect("clock") = start + .checked_add(Duration::from_secs(80)) + .expect("callback instant"); + }), + grant: IngressGrant::empty(), + read_deadline: head.read_deadline_after(Duration::from_secs(30)), + } + }); + app.set_response_egress_observer(OrderedAppObserver { + events: Arc::clone(&probe.events), + reports: Arc::clone(&reports), + }); + let mut service = observed_service(app, TrustedProxyPolicy::no_trust(), &probe); + let owned_connection = tracked_connection(&mut service); + let response = service + .dispatch_request( + Request::builder() + .uri("/timed") + .body(AxumBody::empty()) + .expect("request"), + &EgressConnection::default(), + None, + ) + .await + .expect("response"); + assert_eq!(probe.handle.snapshot().active_requests, 1); + *now.lock().expect("clock") = start + .checked_add(Duration::from_secs(8)) + .expect("terminal instant"); + to_bytes(response.into_body(), 1024) + .await + .expect("response body"); + let record = assert_one_released(&probe); + assert_eq!(record.duration, Some(Duration::from_secs(8))); + assert_eq!( + reports.lock().expect("app reports")[0].elapsed, + Duration::from_secs(5) + ); + assert_eq!( + probe.events.lock().expect("hook events").as_slice(), + ["app", "native", "observer"] + ); + assert_eq!(probe.handle.snapshot().idle_connections, 1); + drop(owned_connection); + assert_eq!( + probe.handle.snapshot(), + NativeDiagnosticsSnapshot::default() + ); + } + + #[tokio::test] + async fn managed_backwards_request_clock_is_safe_native_fault_without_fake_duration() { + let probe = NativeProbe::default(); + let now = MonotonicInstant::now(); + let request_start = now + .checked_add(Duration::from_secs(1)) + .expect("request start"); + let samples = Arc::new(AtomicUsize::new(0)); + let captured_samples = Arc::clone(&samples); + let mut app = App::new( + RouterService::builder() + .get("/backwards", success_handler) + .build(), + ); + app.set_monotonic_clock(MonotonicClock::new(move || { + if captured_samples.fetch_add(1, Ordering::SeqCst) == 0 { + request_start + } else { + now + } + })); + let service = observed_service(app, TrustedProxyPolicy::no_trust(), &probe); + let request = Request::builder() + .uri("/backwards") + .body(AxumBody::empty()) + .expect("request"); + let response = service + .dispatch_request(request, &EgressConnection::default(), None) + .await + .expect("response"); + to_bytes(response.into_body(), 1024) + .await + .expect("response body"); + let record = assert_one_released(&probe); + assert_eq!(record.status, Some(StatusCode::OK)); + assert_eq!(record.duration, None); + assert_eq!(record.level(), log::Level::Error); + } + } + /// Send a GET request through `service` and return the response body as a UTF-8 string. /// Lifted out of the registry-aware tests so each can stay flat (clippy /// `items_after_statements` rejects nested `async fn` definitions). diff --git a/crates/edgezero-adapter-axum/tests/fixtures/native_proxy.rs b/crates/edgezero-adapter-axum/tests/fixtures/native_proxy.rs new file mode 100644 index 000000000..8d042cacd --- /dev/null +++ b/crates/edgezero-adapter-axum/tests/fixtures/native_proxy.rs @@ -0,0 +1,311 @@ +//! Test-only source-bound proxy and verified TLS ingress. No response is collected here. +#![expect( + clippy::arbitrary_source_item_ordering, + reason = "fixture ownership precedes bounded relay helpers" +)] + +use std::io::{self, ErrorKind}; +use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener as StdTcpListener}; +use std::sync::Arc; +use std::thread::{self, JoinHandle}; +use std::time::Duration; + +use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer, ServerName}; +use rustls::{ClientConfig, RootCertStore, ServerConfig}; +use tokio::io::{ + AsyncRead, AsyncReadExt as _, AsyncWrite, AsyncWriteExt as _, copy_bidirectional_with_sizes, +}; +use tokio::net::{TcpListener, TcpSocket, TcpStream}; +use tokio::runtime::Builder; +use tokio::sync::oneshot; +use tokio::task::JoinSet; +use tokio::time::timeout; +use tokio_rustls::{TlsAcceptor, TlsConnector}; + +const MAX_HEAD: usize = 0x8000; +const RELAY_BUFFER: usize = 0x4000; + +#[derive(Clone, Copy)] +pub(crate) enum Assertions { + /// Keep deliberate malformed/gap inputs unchanged while preserving a real owned peer. + Preserve, + AppendForwarded { + host: &'static str, + scheme: &'static str, + }, + AppendXForwarded { + host: &'static str, + scheme: &'static str, + }, + ReplaceForwarded { + client: IpAddr, + host: &'static str, + scheme: &'static str, + }, +} + +pub(crate) struct LocalProxy { + addr: SocketAddr, + stop: Option>, + worker: Option>, +} + +impl LocalProxy { + #[must_use] + #[inline] + pub(crate) fn addr(&self) -> SocketAddr { + self.addr + } + + #[must_use] + #[inline] + pub(crate) fn start( + upstream: SocketAddr, + source: Ipv4Addr, + assertions: Assertions, + tls: bool, + ) -> Self { + let std_listener = StdTcpListener::bind((Ipv4Addr::LOCALHOST, 0)).expect("proxy listener"); + let addr = std_listener.local_addr().expect("proxy address"); + std_listener + .set_nonblocking(true) + .expect("proxy nonblocking"); + let (stop, mut stopped) = oneshot::channel(); + let worker = thread::spawn(move || { + let runtime = Builder::new_current_thread() + .enable_all() + .build() + .expect("proxy runtime"); + runtime.block_on(async move { + let listener = TcpListener::from_std(std_listener).expect("proxy adoption"); + let tls_acceptor = tls.then(|| TlsAcceptor::from(Arc::new(server_config()))); + let mut tasks = JoinSet::new(); + #[expect(clippy::integer_division_remainder_used, reason = "Tokio select macro branch arithmetic")] + loop { + tokio::select! { + () = async { let _closed = (&mut stopped).await; } => break, + _finished = tasks.join_next(), if !tasks.is_empty() => {}, + accepted = listener.accept(), if tasks.len() < 64 => { + let (socket, peer) = accepted.expect("proxy accept"); + let task_acceptor = tls_acceptor.clone(); + tasks.spawn(async move { + let operation = async { + if let Some(acceptor) = task_acceptor { + if let Ok(mut client) = acceptor.accept(socket).await { + let _relayed = relay(&mut client, peer, upstream, source, assertions).await; + } + } else { + let mut client = socket; + let _relayed = relay(&mut client, peer, upstream, source, assertions).await; + } + }; + let _bounded = timeout(Duration::from_secs(5), operation).await; + }); + } + } + } + tasks.abort_all(); + while tasks.join_next().await.is_some() {} + }); + }); + Self { + addr, + stop: Some(stop), + worker: Some(worker), + } + } +} + +impl Drop for LocalProxy { + fn drop(&mut self) { + if let Some(stop) = self.stop.take() { + let _sent = stop.send(()); + } + if let Some(worker) = self.worker.take() { + worker.join().expect("proxy worker"); + } + } +} + +fn invalid() -> io::Error { + io::Error::new(ErrorKind::InvalidData, "invalid bounded fixture head") +} + +async fn read_head( + client: &mut Client, +) -> io::Result<(String, Vec)> { + let mut received = Vec::with_capacity(1024); + let mut chunk = [0_u8; 512]; + loop { + if let Some(end) = received.windows(4).position(|bytes| bytes == b"\r\n\r\n") { + let body_at = end.checked_add(4).ok_or_else(invalid)?; + let head = String::from_utf8(received.get(..end).ok_or_else(invalid)?.to_vec()) + .map_err(|_error| invalid())?; + let body = received.get(body_at..).ok_or_else(invalid)?.to_vec(); + return Ok((head, body)); + } + let count = client.read(&mut chunk).await?; + if count == 0 { + return Err(io::Error::new( + ErrorKind::UnexpectedEof, + "fixture head incomplete", + )); + } + received + .len() + .checked_add(count) + .filter(|bytes| *bytes <= MAX_HEAD) + .ok_or_else(invalid)?; + received.extend_from_slice(chunk.get(..count).ok_or_else(invalid)?); + } +} + +fn rewrite(head: &str, peer: SocketAddr, assertions: Assertions) -> io::Result { + let mut lines = head.split("\r\n"); + let request = lines + .next() + .filter(|line| !line.is_empty()) + .ok_or_else(invalid)?; + let mut headers = Vec::new(); + for line in lines { + let (name, value) = line.split_once(':').ok_or_else(invalid)?; + headers.push((name.to_owned(), value.trim().to_owned())); + } + let predecessor = match peer.ip() { + IpAddr::V4(ip) => ip.to_string(), + IpAddr::V6(ip) => format!("\"[{ip}]\""), + }; + match assertions { + Assertions::Preserve => {} + Assertions::AppendForwarded { host, scheme } => { + let previous = headers + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case("forwarded")) + .map(|(_, value)| value.as_str()) + .collect::>() + .join(","); + headers.retain(|(name, _)| !name.eq_ignore_ascii_case("forwarded")); + let assertion = format!("for={predecessor};host=\"{host}\";proto={scheme}"); + headers.push(( + "Forwarded".into(), + if previous.is_empty() { + assertion + } else { + format!("{previous},{assertion}") + }, + )); + } + Assertions::AppendXForwarded { host, scheme } => { + let previous = headers + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case("x-forwarded-for")) + .map(|(_, value)| value.as_str()) + .collect::>() + .join(","); + headers.retain(|(name, _)| !name.to_ascii_lowercase().starts_with("x-forwarded-")); + let node = peer.ip().to_string(); + headers.push(( + "X-Forwarded-For".into(), + if previous.is_empty() { + node + } else { + format!("{previous},{node}") + }, + )); + headers.push(("X-Forwarded-Host".into(), host.into())); + headers.push(("X-Forwarded-Proto".into(), scheme.into())); + } + Assertions::ReplaceForwarded { + client, + host, + scheme, + } => { + headers.retain(|(name, _)| { + !name.eq_ignore_ascii_case("forwarded") + && !name.to_ascii_lowercase().starts_with("x-forwarded-") + }); + // The test gateway's client assertion is explicitly supplied by its trusted producer, + // never taken from the visitor's fields. This is not a provider-discovery mechanism. + let node = match client { + IpAddr::V4(ip) => ip.to_string(), + IpAddr::V6(ip) => format!("\"[{ip}]\""), + }; + headers.push(( + "Forwarded".into(), + format!("for={node};host=\"{host}\";proto={scheme}"), + )); + } + } + let mut rewritten = format!("{request}\r\n"); + for (name, value) in headers { + use std::fmt::Write as _; + write!(rewritten, "{name}: {value}\r\n").map_err(|_error| invalid())?; + } + rewritten.push_str("\r\n"); + if rewritten.len() > MAX_HEAD { + return Err(invalid()); + } + Ok(rewritten) +} + +async fn relay( + client: &mut Client, + peer: SocketAddr, + upstream: SocketAddr, + source: Ipv4Addr, + assertions: Assertions, +) -> io::Result<()> { + let (head, body_prefix) = read_head(client).await?; + let rewritten = rewrite(&head, peer, assertions)?; + let socket = TcpSocket::new_v4()?; + socket.bind(SocketAddr::new(IpAddr::V4(source), 0))?; + let mut origin = socket.connect(upstream).await?; + origin.write_all(rewritten.as_bytes()).await?; + origin.write_all(&body_prefix).await?; + // Finite, per-direction relay buffers. Never collect a response or body for proxying. + let _copied = + copy_bidirectional_with_sizes(client, &mut origin, RELAY_BUFFER, RELAY_BUFFER).await?; + Ok(()) +} + +fn server_config() -> ServerConfig { + let certificate = CertificateDer::from(include_bytes!("tls/server.der").to_vec()); + let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from( + include_bytes!("tls/server-key.der").to_vec(), + )); + ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(vec![certificate], key) + .expect("test certificate/key") +} + +#[inline] +pub(crate) fn verified_tls_request( + addr: SocketAddr, + request: &[u8], + server_name: &'static str, +) -> io::Result { + let runtime = Builder::new_current_thread().enable_all().build()?; + runtime.block_on(async move { + let mut roots = RootCertStore::empty(); + roots + .add(CertificateDer::from(include_bytes!("tls/ca.der").to_vec())) + .map_err(|_error| invalid())?; + let config = ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + let connector = TlsConnector::from(Arc::new(config)); + let operation = async { + let socket = TcpStream::connect(addr).await?; + let name = ServerName::try_from(server_name).map_err(|_error| invalid())?; + let mut client = connector.connect(name, socket).await?; + client.write_all(request).await?; + let mut response = String::new(); + client.read_to_string(&mut response).await?; + Ok(response) + }; + timeout(Duration::from_secs(5), operation) + .await + .map_err(|_error| io::Error::new(ErrorKind::TimedOut, "TLS fixture timeout"))? + }) +} diff --git a/crates/edgezero-adapter-axum/tests/fixtures/production_host.rs b/crates/edgezero-adapter-axum/tests/fixtures/production_host.rs index 0f0b72cd2..4b44d7874 100644 --- a/crates/edgezero-adapter-axum/tests/fixtures/production_host.rs +++ b/crates/edgezero-adapter-axum/tests/fixtures/production_host.rs @@ -8,29 +8,171 @@ use std::cell::Cell; use std::env; use std::future; use std::net::SocketAddr; +use std::panic::set_hook; use std::rc::Rc; -use std::sync::{Arc, Mutex}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; use std::time::Duration; +use async_trait::async_trait; use bytes::Bytes; +use edgezero_adapter_axum::context::AxumRequestContext; use edgezero_adapter_axum::dev_server::{ AxumRunOptions, run_app, run_app_with_options_and_initializer, run_production_app, run_production_app_with_initializer, }; +use edgezero_adapter_axum::diagnostics::{ + IngressDisposition, NativeDiagnosticsHandle, NativeDiagnosticsSnapshot, NativeIngressMetadata, + NativeRequestFailure, NativeRequestObserver, NativeRequestOutcome, NativeRequestRecord, +}; use edgezero_core::app::{App, Hooks, StoreMetadata, StoresMetadata}; use edgezero_core::context::RequestContext; -use edgezero_core::extractor::{AppConfig, Json, State}; +use edgezero_core::extractor::{AppConfig, ForwardedHost, FromRequest as _, Json, State}; +use edgezero_core::http::{HeaderMap, HeaderValue, Response, StatusCode, Uri}; +use edgezero_core::ingress::{ + AdmissionDecision, CheckedEffectiveHost, CheckedHostSource, IngressGrant, +}; use edgezero_core::key_value_store::KvHandle; +use edgezero_core::middleware::{Middleware, Next}; use edgezero_core::params::PathParams; use edgezero_core::probe::{self, LifecyclePhase, LifecycleReader}; +use edgezero_core::response::response_with_body; +use edgezero_core::response_egress::{ + ResponseEgressBodyKind, ResponseEgressCompletion, ResponseEgressDeadline, + ResponseEgressFallbackDisposition, ResponseEgressObserver, ResponseEgressOutcome, + ResponseEgressReport, +}; use edgezero_core::router::RouterService; -use edgezero_core::{Body, EdgeError, MonotonicClock, MonotonicInstant, action}; +use edgezero_core::{BadGatewayReason, Body, EdgeError, MonotonicClock, MonotonicInstant, action}; use futures::stream; +use log::{LevelFilter, Log, Metadata, Record}; +use serde_json::{Value, json}; +use tokio::signal; +#[cfg(unix)] +use tokio::signal::unix::{SignalKind, signal as unix_signal}; use tokio::sync::oneshot; use tokio::time::sleep; +static ADMISSION_FACTS: Mutex> = Mutex::new(None); +static MIDDLEWARE_FACTS: Mutex> = Mutex::new(None); +static PANIC_COMPLETION: AtomicBool = AtomicBool::new(false); +static LOGGER_FAULT_REPORTED: AtomicBool = AtomicBool::new(false); +static NATIVE_LOGGER: NativeFixtureLogger = NativeFixtureLogger; + +#[derive(Clone, Default)] +struct NativeFailureInputs { + configuration: String, + secret: String, +} + +struct NativeFixtureLogger; + +impl Log for NativeFixtureLogger { + #[inline] + fn enabled(&self, metadata: &Metadata<'_>) -> bool { + metadata.target().starts_with("edgezero") + } + + #[inline] + fn flush(&self) {} + + #[inline] + #[expect( + clippy::panic, + reason = "regression intentionally panics during the core completion-fault log attempt to prove the joined native completion and app observer still run" + )] + fn log(&self, record: &Record<'_>) { + if !self.enabled(record.metadata()) { + return; + } + let message = record.args().to_string(); + println!( + "FRAMEWORK_LOG {}", + json!({"level": record.level().as_str(), "target": record.target(), "message": message}) + ); + if PANIC_COMPLETION.load(Ordering::Relaxed) + && message == "response-egress completion panicked after terminal transition" + && !LOGGER_FAULT_REPORTED.swap(true, Ordering::Relaxed) + { + println!("LOGGER_FAULT"); + panic!("fixture completion logger fault"); + } + } +} + +struct NativeFixtureObserver(NativeDiagnosticsHandle); + +impl NativeRequestObserver for NativeFixtureObserver { + #[inline] + fn observe(&self, record: &NativeRequestRecord) { + println!( + "NATIVE_OBSERVATION {}", + json!({ + "id": record.request_id.to_string(), + "event": record.event, + "method": record.method, + "route": record.route, + "status": record.status.map(|status| status.as_u16()), + "duration_us": record.duration.map(|duration| duration.as_micros()), + "outcome": native_outcome(record.outcome), + "failure": native_failure(record.failure), + "forwarding_result": record.forwarding_result.as_str(), + "bytes_written": record.bytes_written, + "body_kind": record.body_kind.map(|kind| match kind { + ResponseEgressBodyKind::Application => "application", + ResponseEgressBodyKind::Fallback => "fallback", + _ => "unknown", + }), + "fallback": record.fallback_disposition.map(|disposition| match disposition { + ResponseEgressFallbackDisposition::Aborted => "aborted", + ResponseEgressFallbackDisposition::Completed => "completed", + _ => "unknown", + }), + "level": record.level().as_str(), + "counts": snapshot_json(self.0.snapshot()), + }) + ); + } +} + +struct FixtureEgressObserver; + +impl ResponseEgressObserver for FixtureEgressObserver { + #[inline] + fn complete(&self, report: &ResponseEgressReport) { + println!("APP_OBSERVER {}", egress_outcome(report.outcome)); + } +} + +struct NativeFactsMiddleware; + +#[async_trait(?Send)] +impl Middleware for NativeFactsMiddleware { + #[inline] + async fn handle(&self, ctx: RequestContext, next: Next<'_>) -> Result { + if ctx.uri().path() == "/native-metadata" { + let metadata = ingress_metadata(&ctx)?; + *MIDDLEWARE_FACTS + .lock() + .unwrap_or_else(PoisonError::into_inner) = + Some(metadata_facts(metadata, ctx.headers(), ctx.uri())); + } + next.run(ctx).await + } +} + struct FixtureApp; +struct WorkGuard(&'static str); + +impl Drop for WorkGuard { + fn drop(&mut self) { + println!("{}", self.0); + } +} + +struct StopState(Mutex>>); + #[expect( clippy::missing_trait_methods, reason = "fixture retains unused Hooks defaults" @@ -41,6 +183,10 @@ impl Hooks for FixtureApp { if mode() == "hook-error" { return Err(EdgeError::internal(anyhow::anyhow!("HOOK_SENTINEL"))); } + let selected = mode(); + if selected.starts_with("diagnostics") { + configure_native_fixture(app, &selected); + } if env::var_os("FIXTURE_FREEZE_CLOCK").is_some() { let frozen = MonotonicInstant::now(); app.set_monotonic_clock(MonotonicClock::new(move || frozen)); @@ -74,6 +220,21 @@ impl Hooks for FixtureApp { .get("/distinct", distinct) .get("/snapshot", snapshot) .get("/stop", stop) + .get("/native-metadata", native_metadata) + .get("/native-error", native_error) + .get("/native-upstream", native_upstream) + .get("/native-status503", native_status503) + .get("/native-handler404", native_handler404) + .get("/native-mutated", native_mutated) + .get("/native-source", native_source) + .get("/native-deadline", native_deadline) + .get("/native-expired", native_expired) + .get("/native-conversion", native_conversion) + .post("/native-json-number", native_json_number) + .get("/native-counts", native_counts) + .get("/native-refuse", native_status503) + .get("/native-abort", native_status503) + .middleware(NativeFactsMiddleware) .build() } @@ -86,6 +247,10 @@ impl Hooks for FixtureApp { | "initializer-error" | "starting" | "embedding-bare" + | "diagnostics" + | "diagnostics-panic" + | "diagnostics-optout" + | "diagnostics-custom" ) { return StoresMetadata::default(); } @@ -106,6 +271,195 @@ impl Hooks for FixtureApp { } } +fn configure_native_fixture(app: &mut App, selected: &str) { + let panic_completion = selected == "diagnostics-panic"; + app.set_response_egress_observer(FixtureEgressObserver); + app.set_ingress_admission_policy(move |head| { + let metadata = head.extension::(); + if head.target().path() == "/native-metadata" + && let Some(checked) = metadata + { + *ADMISSION_FACTS + .lock() + .unwrap_or_else(PoisonError::into_inner) = + Some(metadata_facts(checked, head.headers(), head.target())); + } + if head.target().path() == "/native-abort" { + return AdmissionDecision::Abort; + } + let id = metadata.map_or_else( + || "unavailable".to_owned(), + |checked| checked.request_id().to_string(), + ); + let completion = ResponseEgressCompletion::new(move |_report| { + println!("APP_COMPLETION {id}"); + assert!(!panic_completion, "fixture application completion fault"); + }); + if head.target().path() == "/native-refuse" { + let mut response = Response::new(Body::text("fixture admission refused")); + *response.status_mut() = StatusCode::SERVICE_UNAVAILABLE; + return AdmissionDecision::Refuse { + completion, + response, + }; + } + AdmissionDecision::Admit { + completion, + grant: IngressGrant::empty(), + read_deadline: head.read_deadline_after(Duration::from_secs(30)), + } + }); + if selected == "diagnostics-custom" { + app.set_error_response_renderer(|error| { + let mut response = Response::new(Body::text(error.message())); + *response.status_mut() = error.status(); + response + }); + } +} + +fn metadata_facts(metadata: &NativeIngressMetadata, headers: &HeaderMap, uri: &Uri) -> Value { + json!({ + "request_id": metadata.request_id().to_string(), + "peer": metadata.direct_peer().map(|peer| peer.to_string()), + "client": metadata.effective_client().map(|client| client.to_string()), + "host": metadata.effective_host().authority(), + "scheme": metadata.effective_scheme().as_str(), + "client_source": host_source(metadata.client_source()), + "host_source": host_source(metadata.host_source()), + "scheme_source": host_source(metadata.scheme_source()), + "forwarding_result": metadata.forwarding_result().as_str(), + "host_header": headers.get("host").and_then(|value| value.to_str().ok()), + "uri": uri.to_string(), + "raw_forwarding_absent": !headers.keys().any(|name| name.as_str() == "forwarded" || name.as_str().starts_with("x-forwarded-")), + }) +} + +fn ingress_metadata(ctx: &RequestContext) -> Result<&NativeIngressMetadata, EdgeError> { + ctx.extensions() + .get::() + .ok_or_else(|| EdgeError::internal(anyhow::anyhow!("fixture ingress metadata missing"))) +} + +fn host_source(source: CheckedHostSource) -> &'static str { + match source { + CheckedHostSource::Direct => "direct", + CheckedHostSource::TrustedForwarded => "trusted_forwarded", + CheckedHostSource::TrustedXForwarded => "trusted_x_forwarded", + CheckedHostSource::Unavailable => "unavailable", + _ => "unknown", + } +} + +fn egress_outcome(outcome: ResponseEgressOutcome) -> &'static str { + match outcome { + ResponseEgressOutcome::ClientDisconnected => "client_disconnected", + ResponseEgressOutcome::Completed => "completed", + ResponseEgressOutcome::ConversionError => "conversion_error", + ResponseEgressOutcome::DeadlineExceeded => "deadline_exceeded", + ResponseEgressOutcome::HostHandoff => "host_handoff", + ResponseEgressOutcome::RequestCancelled => "request_cancelled", + ResponseEgressOutcome::SourceError => "source_error", + ResponseEgressOutcome::TransportError => "transport_error", + ResponseEgressOutcome::Unspecified => "unspecified", + _ => "unknown", + } +} + +fn native_outcome(outcome: NativeRequestOutcome) -> &'static str { + match outcome { + NativeRequestOutcome::Egress(egress) => egress_outcome(egress), + NativeRequestOutcome::Ingress(IngressDisposition::Aborted) => "aborted", + NativeRequestOutcome::Ingress(IngressDisposition::Draining) => "draining", + NativeRequestOutcome::Ingress(IngressDisposition::Abandoned) => "abandoned", + } +} + +fn native_failure(failure: Option) -> Option<&'static str> { + failure.map(|category| match category { + NativeRequestFailure::FrameworkRouting { kind } + | NativeRequestFailure::PropagatedError { kind } => kind, + NativeRequestFailure::FallbackBodyExceeded => "fallback_body_exceeded", + NativeRequestFailure::FallbackReadTimedOut => "fallback_read_timed_out", + NativeRequestFailure::AdmissionRefused => "admission_refused", + }) +} + +fn json_response(value: &Value) -> Result { + let mut response = response_with_body( + StatusCode::OK, + Body::json(value).map_err(EdgeError::internal)?, + )?; + response + .headers_mut() + .insert("content-type", HeaderValue::from_static("application/json")); + Ok(response) +} + +fn snapshot_json(snapshot: NativeDiagnosticsSnapshot) -> Value { + json!({ + "active_requests": snapshot.active_requests, + "active_connections": snapshot.active_connections, + "idle_connections": snapshot.idle_connections, + }) +} + +#[expect( + clippy::expect_used, + reason = "fixture-owned SIGTERM registration is fatal setup: without that owner this child cannot exercise graceful stop" +)] +#[expect( + clippy::integer_division_remainder_used, + reason = "tokio::select uses internal bounded remainder to randomize branch polling; the fixture itself introduces no integer division" +)] +async fn native_stop() { + #[cfg(unix)] + { + let mut terminate = unix_signal(SignalKind::terminate()).expect("fixture SIGTERM owner"); + tokio::select! { + _interrupted = signal::ctrl_c() => {}, + _terminated = terminate.recv() => {}, + } + } + #[cfg(not(unix))] + { + let _interrupted = signal::ctrl_c().await; + } +} + +fn run_native_fixture(selected: &str) -> anyhow::Result<()> { + PANIC_COMPLETION.store(selected == "diagnostics-panic", Ordering::Relaxed); + set_hook(Box::new(|_info| println!("FIXTURE_PANIC"))); + log::set_logger(&NATIVE_LOGGER)?; + log::set_max_level(LevelFilter::Trace); + println!("APP_LOGGER_INSTALLED"); + let base = NativeDiagnosticsHandle::new(); + // Capture a counter-only clone, not the observer-bearing handle, to avoid a cycle. + let observed = base + .clone() + .with_request_observer(NativeFixtureObserver(base)); + let options = AxumRunOptions::from_env()?.with_diagnostics(observed.clone()); + let state = observed.clone(); + let private_inputs = NativeFailureInputs { + configuration: env::var("PRIVATE_CONFIG_SENTINEL").unwrap_or_default(), + secret: env::var("fixture_token").unwrap_or_default(), + }; + let result = run_app_with_options_and_initializer::( + options, + native_stop(), + move |app, _stores| { + Box::pin(async move { + app.insert_state(state); + app.insert_state(private_inputs); + println!("INITIALIZED"); + Ok(()) + }) + }, + ); + println!("FINAL_SNAPSHOT {}", snapshot_json(observed.snapshot())); + result +} + #[action] async fn buffered_json(Json(value): Json) -> Result { println!("JSON_HANDLER_ENTERED"); @@ -153,15 +507,165 @@ async fn taken_body(ctx: RequestContext) -> Result { Ok(bytes.to_string()) } -struct WorkGuard(&'static str); +#[action] +async fn native_metadata( + ctx: RequestContext, + ForwardedHost(forwarded_host): ForwardedHost, +) -> Result { + let metadata = ingress_metadata(&ctx)?; + let summary = metadata.received_normalized_head(); + let admission = ADMISSION_FACTS + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take() + .ok_or_else(|| EdgeError::internal(anyhow::anyhow!("fixture admission facts missing")))?; + let middleware = MIDDLEWARE_FACTS + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take() + .ok_or_else(|| EdgeError::internal(anyhow::anyhow!("fixture middleware facts missing")))?; + json_response(&json!({ + "admission": admission, + "middleware": middleware, + "handler": metadata_facts(metadata, ctx.headers(), ctx.uri()), + "forwarded_host": forwarded_host, + "direct_context_peer": ctx.extensions().get::().and_then(|context| context.remote_addr).map(|peer| peer.to_string()), + "normalized": { + "target_bytes": summary.target_bytes(), + "header_bytes": summary.header_bytes(), + "header_count": summary.header_count(), + }, + })) +} -impl Drop for WorkGuard { - fn drop(&mut self) { - println!("{}", self.0); - } +#[action] +async fn native_error( + State(inputs): State, +) -> Result<&'static str, EdgeError> { + println!("PRIVATE_INPUTS_USED"); + Err(EdgeError::internal(anyhow::anyhow!( + "PRIVATE_ERROR_SENTINEL {} {}", + inputs.configuration, + inputs.secret, + ))) } -struct StopState(Mutex>>); +#[action] +async fn native_upstream( + State(inputs): State, +) -> Result<&'static str, EdgeError> { + println!("PRIVATE_INPUTS_USED"); + Err(EdgeError::bad_gateway_with_reason( + format!( + "PRIVATE_ORIGIN_SENTINEL {} {}", + inputs.configuration, inputs.secret + ), + BadGatewayReason::Transport, + )) +} + +#[action] +async fn native_status503() -> Result { + response_with_body( + StatusCode::SERVICE_UNAVAILABLE, + Body::text("application-selected 503"), + ) +} + +#[action] +async fn native_handler404() -> Result<&'static str, EdgeError> { + Err(EdgeError::not_found("HANDLER_404_SENTINEL")) +} + +#[action] +async fn native_mutated(mut ctx: RequestContext) -> Result { + let removed = ctx + .extensions_mut() + .remove::() + .ok_or_else(|| EdgeError::internal(anyhow::anyhow!("fixture ingress metadata missing")))?; + let removed_id = removed.request_id().to_string(); + ctx.extensions_mut() + .insert(CheckedEffectiveHost::from_ingress( + Some("replacement.example".parse().map_err(EdgeError::internal)?), + CheckedHostSource::Direct, + )); + let ForwardedHost(after_host) = ForwardedHost::from_request(&ctx).await?; + json_response(&json!({ + "removed_id": removed_id, + "metadata_absent": ctx.extensions().get::().is_none(), + "forwarded_host_after": after_host, + })) +} + +#[action] +async fn native_source() -> Result { + let guard = Rc::new(WorkGuard("SOURCE_DROPPED")); + Ok(Body::from_stream(async_stream::stream! { + let _retained = guard; + yield Ok(Bytes::from_static(b"source-prefix")); + sleep(Duration::from_millis(100)).await; + yield Err(EdgeError::internal(anyhow::anyhow!("SOURCE_SENTINEL"))); + })) +} + +#[action] +async fn native_deadline() -> Result { + let guard = Rc::new(WorkGuard("DEADLINE_DROPPED")); + let body = Body::from_stream(async_stream::stream! { + let _retained = guard; + yield Ok(Bytes::from_static(b"deadline-prefix")); + future::pending::<()>().await; + }); + let mut response = Response::new(body); + response + .extensions_mut() + .insert(ResponseEgressDeadline::after(Duration::from_millis(150))); + Ok(response) +} + +#[action] +async fn native_expired() -> Result { + let mut response = Response::new(Body::text("expired-private-body")); + response + .extensions_mut() + .insert(ResponseEgressDeadline::after(Duration::ZERO)); + Ok(response) +} + +#[action] +async fn native_conversion() -> Result { + let mut response = Response::new(Body::text("CONVERSION_PAYLOAD_SENTINEL")); + response + .headers_mut() + .insert("content-length", HeaderValue::from_static("1")); + Ok(response) +} + +#[action] +async fn native_json_number(Json(value): Json) -> Result { + Ok(value.to_string()) +} + +#[action] +async fn native_counts( + ctx: RequestContext, + State(handle): State, +) -> Result { + let snapshot = handle.snapshot(); + let phase = match reader(&ctx)?.phase() { + Some(LifecyclePhase::Starting) => "starting", + Some(LifecyclePhase::Ready) => "ready", + Some(LifecyclePhase::Draining) => "draining", + Some(LifecyclePhase::Stopped) => "stopped", + None => "unavailable", + }; + json_response(&json!({ + "active_requests": snapshot.active_requests, + "active_connections": snapshot.active_connections, + "idle_connections": snapshot.idle_connections, + "phase": phase, + })) +} fn mode() -> String { env::var("FIXTURE_MODE").unwrap_or_else(|_| "initialized".to_owned()) @@ -310,6 +814,9 @@ async fn stop(State(state): State>) -> Result<&'static str, EdgeE )] fn main() -> anyhow::Result<()> { let selected = mode(); + if selected.starts_with("diagnostics") { + return run_native_fixture(&selected); + } if matches!(selected.as_str(), "development" | "bare-dev") { return run_app::(); } diff --git a/crates/edgezero-adapter-axum/tests/fixtures/tls/README.md b/crates/edgezero-adapter-axum/tests/fixtures/tls/README.md new file mode 100644 index 000000000..0851892ea --- /dev/null +++ b/crates/edgezero-adapter-axum/tests/fixtures/tls/README.md @@ -0,0 +1,11 @@ +# Local proxy TLS fixtures + +These public test assets are only for the local native-host proxy fixture. `server-key.der` is an intentionally published test key, not a credential. Never use it outside tests. + +- `ca.der` is the fixture CA certificate. +- `server.der` is the server certificate for `native-proxy.test`. +- `server-key.der` is its PKCS#8 private key. + +The server certificate is valid from October 6, 2026 to October 3, 2036. The test client trusts only this fixture CA and verifies the certificate and DNS name. A request for a different server name must fail verification. Renew the fixture assets before expiry; do not disable certificate or name checks to extend their lifetime. + +The fixture terminates TLS and relays HTTP to the managed native host. It does not add incoming TLS support to the Axum adapter. diff --git a/crates/edgezero-adapter-axum/tests/fixtures/tls/ca.der b/crates/edgezero-adapter-axum/tests/fixtures/tls/ca.der new file mode 100644 index 000000000..025acfc6c Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/tls/ca.der differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/tls/server-key.der b/crates/edgezero-adapter-axum/tests/fixtures/tls/server-key.der new file mode 100644 index 000000000..c2f0fb2d5 Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/tls/server-key.der differ diff --git a/crates/edgezero-adapter-axum/tests/fixtures/tls/server.der b/crates/edgezero-adapter-axum/tests/fixtures/tls/server.der new file mode 100644 index 000000000..3a7ef3ae8 Binary files /dev/null and b/crates/edgezero-adapter-axum/tests/fixtures/tls/server.der differ diff --git a/crates/edgezero-adapter-axum/tests/production_hosting.rs b/crates/edgezero-adapter-axum/tests/production_hosting.rs index 843b0da94..30c28f2e0 100644 --- a/crates/edgezero-adapter-axum/tests/production_hosting.rs +++ b/crates/edgezero-adapter-axum/tests/production_hosting.rs @@ -1,16 +1,22 @@ //! Real local processes, public entrypoints and ordinary main termination output. #![cfg(all(feature = "axum", feature = "test-utils", unix))] +#[cfg(test)] +#[path = "fixtures/native_proxy.rs"] +mod native_proxy; + #[cfg(test)] #[expect( clippy::arbitrary_source_item_ordering, reason = "subprocess fixtures group setup, process ownership and acceptance scenarios" )] mod tests { + use super::native_proxy::{Assertions, LocalProxy, verified_tls_request}; + use std::collections::HashSet; use std::ffi::OsString; use std::fs; use std::io::{self, BufRead as _, BufReader, Read, Write as _}; - use std::net::{SocketAddr, TcpListener, TcpStream}; + use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener, TcpStream}; use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _, symlink}; use std::os::unix::process::ExitStatusExt as _; use std::process::{Child, Command, ExitStatus, Stdio}; @@ -325,6 +331,1371 @@ mod tests { } } + fn captured_json(log: &str, marker: &str) -> Vec { + log.lines() + .filter_map(|line| line.strip_prefix(marker)) + .map(|value| serde_json::from_str(value).expect("fixture JSON marker")) + .collect() + } + + fn observations(server: &Server) -> Vec { + captured_json(&server.log(), "NATIVE_OBSERVATION ") + } + + fn framework_logs(server: &Server) -> Vec { + captured_json(&server.log(), "FRAMEWORK_LOG ") + } + + fn wait_for(server: &Server, predicate: impl Fn(&str) -> bool) { + let deadline = Instant::now().checked_add(WAIT).expect("observation timer"); + loop { + let log = server.log(); + if predicate(&log) { + return; + } + assert!(Instant::now() < deadline, "observation deadline: {log}"); + thread::sleep(Duration::from_millis(5)); + } + } + + fn route_record(server: &Server, route: &str) -> Value { + wait_for(server, |log| { + captured_json(log, "NATIVE_OBSERVATION ") + .iter() + .any(|record| record["route"] == route) + }); + let records = observations(server) + .into_iter() + .filter(|record| record["route"] == route) + .collect::>(); + assert_eq!(records.len(), 1, "one terminal owner: {}", server.log()); + records.into_iter().next().expect("route record") + } + + fn raw_request(addr: SocketAddr, request: &str) -> String { + let mut socket = TcpStream::connect_timeout(&addr, WAIT).expect("wire connection"); + socket.set_read_timeout(Some(WAIT)).expect("read timeout"); + socket.set_write_timeout(Some(WAIT)).expect("write timeout"); + socket.write_all(request.as_bytes()).expect("wire request"); + let mut response = String::new(); + socket.read_to_string(&mut response).expect("wire response"); + response + } + + fn response_json(response: &str) -> Value { + let (_, body) = response.split_once("\r\n\r\n").expect("response head"); + serde_json::from_str(body).expect("finite fixture JSON response") + } + + fn metadata_request(headers: &str) -> String { + format!( + "GET /native-metadata HTTP/1.1\r\nHost: direct.example:8080\r\n{headers}Connection: close\r\n\r\n" + ) + } + + fn assert_metadata(value: &Value, client: &str, host: &str, scheme: &str, result: &str) { + let facts = &value["handler"]; + assert_eq!(value["admission"], *facts, "admission view"); + assert_eq!(value["middleware"], *facts, "middleware view"); + assert_eq!(facts["client"], client); + assert_eq!(facts["host"], host); + assert_eq!(facts["scheme"], scheme); + assert_eq!(facts["forwarding_result"], result); + assert_eq!(facts["host_header"], "direct.example:8080"); + assert_eq!(facts["uri"], "/native-metadata"); + assert_eq!(facts["raw_forwarding_absent"], true); + assert_eq!(value["forwarded_host"], host); + assert_eq!(value["direct_context_peer"], facts["peer"]); + let id = facts["request_id"].as_str().expect("application ID"); + assert_eq!(id.len(), 64); + assert!(id.bytes().all(|byte| byte.is_ascii_hexdigit())); + } + + fn assert_no_diagnostic_headers(response: &str) { + let (head, _) = response.split_once("\r\n\r\n").expect("response head"); + for line in head.lines().skip(1) { + let (name, _) = line.split_once(':').expect("response header"); + let normalized_name = name.to_ascii_lowercase(); + assert!( + !normalized_name.contains("request-id") + && !normalized_name.starts_with("x-edgezero") + ); + assert!(!matches!( + normalized_name.as_str(), + "traceparent" | "tracestate" | "server-timing" + )); + } + } + + fn native_command( + mode: &str, + roots: &Roots, + addr: SocketAddr, + networks: &str, + family: &str, + ) -> Command { + let mut launch = command(mode, roots, addr); + launch + .env("EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS", networks) + .env("EDGEZERO__ADAPTER__FORWARDING_HEADER_FAMILY", family); + launch + } + + fn assert_final_zero(server: &Server) { + let snapshots = captured_json(&server.log(), "FINAL_SNAPSHOT "); + assert_eq!( + snapshots, + vec![ + json!({"active_requests": 0_u64, "active_connections": 0_u64, "idle_connections": 0_u64}) + ] + ); + } + + fn assert_hook_order(server: &Server, id: &str) { + let log = server.log(); + let completion = log + .find(&format!("APP_COMPLETION {id}\n")) + .expect("application completion"); + let native_line = log + .lines() + .find(|line| line.starts_with("NATIVE_OBSERVATION ") && line.contains(id)) + .expect("native observation"); + let native = log.find(native_line).expect("native offset"); + let observer = log + .get(native..) + .expect("native suffix") + .find("APP_OBSERVER ") + .expect("application observer"); + assert!(completion < native); + assert!(observer > 0); + assert_eq!(log.matches(&format!("APP_COMPLETION {id}\n")).count(), 1); + assert_eq!( + observations(server) + .iter() + .filter(|record| record["id"] == id) + .count(), + 1 + ); + } + + fn assert_record_log(server: &Server, record: &Value) { + let id = record["id"].as_str().expect("record ID"); + let request_logs = framework_logs(server) + .into_iter() + .filter(|entry| { + let message = entry["message"].as_str().expect("framework message"); + (message.starts_with("request_terminal ") + || message.starts_with("request_ingress ")) + && message.contains(id) + }) + .collect::>(); + assert_eq!( + request_logs.len(), + 1, + "facade cardinality: {}", + server.log() + ); + let entry = request_logs + .into_iter() + .next() + .expect("request facade entry"); + assert_eq!(entry["level"], record["level"]); + let message = entry["message"].as_str().expect("safe request text"); + assert!(message.contains(&format!( + "method={}", + record["method"].as_str().expect("method") + ))); + assert!(message.contains("duration_us=")); + assert!(message.contains("bytes_written=") || record["bytes_written"].is_null()); + } + + #[test] + fn native_direct_boundary_strips_spoofs_and_accounts_the_original_head() { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn(command("diagnostics", &roots, addr), addr); + server.ready(); + let spoof = "Forwarded: for=198.51.100.9;host=FORWARDED_SENTINEL;proto=https\r\nX-Forwarded-For: 203.0.113.9\r\nX-Forwarded-Host: XFH_SENTINEL\r\nX-Forwarded-Proto: https\r\nX-Forwarded-Secret: HEADER_SENTINEL\r\nX-Request-ID: VISITOR_ID_SENTINEL\r\n"; + let request = metadata_request(spoof); + let response = raw_request(addr, &request); + assert_no_diagnostic_headers(&response); + let value = response_json(&response); + assert_metadata( + &value, + "127.0.0.1", + "direct.example:8080", + "http", + "untrusted_peer", + ); + for field in ["client_source", "host_source", "scheme_source"] { + assert_eq!(value["handler"][field], "direct"); + } + let peer: SocketAddr = value["handler"]["peer"] + .as_str() + .expect("owned peer") + .parse() + .expect("socket peer"); + assert_eq!(peer.ip(), IpAddr::V4(Ipv4Addr::LOCALHOST)); + let expected_bytes = request + .split("\r\n") + .skip(1) + .filter(|line| !line.is_empty()) + .map(|line| line.len().checked_add(2).expect("normalized line bytes")) + .try_fold(2_usize, usize::checked_add) + .expect("normalized head bytes"); + assert_eq!(value["normalized"]["header_bytes"], expected_bytes); + assert_eq!(value["normalized"]["header_count"], 8_u64); + assert_eq!( + value["normalized"]["target_bytes"], + "/native-metadata".len() + ); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + let record = route_record(&server, "/native-metadata"); + assert_eq!(record["id"], value["handler"]["request_id"]); + assert_record_log(&server, &record); + for entry in framework_logs(&server) { + let message = entry["message"].as_str().expect("framework message"); + assert!(!message.contains("SENTINEL")); + assert!(!message.contains("127.0.0.1") && !message.contains("direct.example")); + } + } + + #[test] + fn native_source_bound_single_proxy_and_mapped_cidr_match_only_owned_peers() { + for networks in [ + "127.0.0.2/32", + "::ffff:127.0.0.2/128", + "::ffff:127.0.0.2/127", + "127.0.0.3/32", + ] { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn( + native_command("diagnostics", &roots, addr, networks, "forwarded"), + addr, + ); + server.ready(); + let proxy = LocalProxy::start( + addr, + Ipv4Addr::new(127, 0, 0, 2), + Assertions::AppendForwarded { + host: "public.example", + scheme: "https", + }, + false, + ); + let response = raw_request( + proxy.addr(), + &metadata_request( + "Forwarded: for=198.51.100.99;host=attacker.example;proto=http\r\nX-Forwarded-Host: conflicting.example\r\n", + ), + ); + let value = response_json(&response); + let trusted = networks != "127.0.0.3/32"; + assert_metadata( + &value, + if trusted { "127.0.0.1" } else { "127.0.0.2" }, + if trusted { + "public.example" + } else { + "direct.example:8080" + }, + if trusted { "https" } else { "http" }, + if trusted { + "accepted" + } else { + "untrusted_peer" + }, + ); + let peer: SocketAddr = value["handler"]["peer"] + .as_str() + .expect("proxy peer") + .parse() + .expect("peer address"); + assert_eq!(peer.ip(), IpAddr::V4(Ipv4Addr::new(127, 0, 0, 2))); + assert_no_diagnostic_headers(&response); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + } + } + + #[test] + fn native_two_proxy_chain_stops_at_untrusted_hop_and_gateway_replaces_prefix() { + for (networks, boundary) in [ + ("127.0.0.2/32,127.0.0.3/32", "127.0.0.1"), + ("127.0.0.3/32", "127.0.0.2"), + ] { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn( + native_command("diagnostics", &roots, addr, networks, "forwarded"), + addr, + ); + server.ready(); + let ingress = LocalProxy::start( + addr, + Ipv4Addr::new(127, 0, 0, 3), + Assertions::AppendForwarded { + host: "final.example", + scheme: "https", + }, + false, + ); + let first = LocalProxy::start( + ingress.addr(), + Ipv4Addr::new(127, 0, 0, 2), + Assertions::AppendForwarded { + host: "earlier.example", + scheme: "http", + }, + false, + ); + let value = response_json(&raw_request( + first.addr(), + &metadata_request( + "Forwarded: for=198.51.100.99;host=attacker.example;proto=http\r\n", + ), + )); + assert_metadata(&value, boundary, "final.example", "https", "accepted"); + assert_eq!(value["handler"]["host_source"], "trusted_forwarded"); + assert!( + value["handler"]["peer"] + .as_str() + .expect("last proxy peer") + .starts_with("127.0.0.3:") + ); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + } + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn( + native_command("diagnostics", &roots, addr, "127.0.0.3/32", "forwarded"), + addr, + ); + server.ready(); + let gateway = LocalProxy::start( + addr, + Ipv4Addr::new(127, 0, 0, 3), + Assertions::ReplaceForwarded { + client: "203.0.113.7".parse().expect("verified gateway client"), + host: "gateway.example", + scheme: "https", + }, + false, + ); + let value = response_json(&raw_request( + gateway.addr(), + &metadata_request( + "Forwarded: for=198.51.100.99;host=attacker.example;proto=http\r\nX-Forwarded-Host: attacker.example\r\n", + ), + )); + assert_metadata( + &value, + "203.0.113.7", + "gateway.example", + "https", + "accepted", + ); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + } + + fn assert_missing_unresolved_and_duplicate_family(addr: SocketAddr, family: &str) { + // This source-bound relay preserves deliberate bad inputs rather than sanitizing them. + let gateway = LocalProxy::start( + addr, + Ipv4Addr::new(127, 0, 0, 2), + Assertions::Preserve, + false, + ); + let absent = if family == "forwarded" { + "Forwarded: host=partial.example;proto=https\r\nX-Forwarded-For: 198.51.100.9\r\n" + } else { + "X-Forwarded-Host: partial.example\r\nX-Forwarded-Proto: https\r\nForwarded: for=198.51.100.9\r\n" + }; + let missing_client = response_json(&raw_request(gateway.addr(), &metadata_request(absent))); + assert_metadata( + &missing_client, + "127.0.0.2", + "partial.example", + "https", + if family == "forwarded" { + "unresolved_chain" + } else { + "partial" + }, + ); + for headers in [ + if family == "forwarded" { + "Forwarded: for=unknown;host=gap.example;proto=https\r\n" + } else { + "X-Forwarded-For: unknown\r\nX-Forwarded-Host: gap.example\r\nX-Forwarded-Proto: https\r\n" + }, + if family == "forwarded" { + "Forwarded: for=127.0.0.2;host=gap.example;proto=https\r\n" + } else { + "X-Forwarded-For: 127.0.0.2\r\nX-Forwarded-Host: gap.example\r\nX-Forwarded-Proto: https\r\n" + }, + ] { + let value = response_json(&raw_request(gateway.addr(), &metadata_request(headers))); + assert_metadata( + &value, + "127.0.0.2", + "gap.example", + "https", + "unresolved_chain", + ); + } + let duplicate = if family == "forwarded" { + "Forwarded: for=198.51.100.8;for=203.0.113.9;host=good.example;proto=https\r\n" + } else { + "X-Forwarded-For: 198.51.100.8\r\nX-Forwarded-Host: good.example\r\nX-Forwarded-Host: duplicate.example\r\nX-Forwarded-Proto: https\r\n" + }; + let malformed_duplicate = + response_json(&raw_request(gateway.addr(), &metadata_request(duplicate))); + assert_metadata( + &malformed_duplicate, + "127.0.0.2", + "direct.example:8080", + "http", + "malformed", + ); + } + + #[test] + fn native_selected_families_conflicts_malformed_sets_gaps_and_exhaustion() { + for family in ["forwarded", "x-forwarded"] { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn( + native_command("diagnostics", &roots, addr, "127.0.0.2/32", family), + addr, + ); + server.ready(); + let assertions = if family == "forwarded" { + Assertions::AppendForwarded { + host: "selected.example", + scheme: "https", + } + } else { + Assertions::AppendXForwarded { + host: "selected.example", + scheme: "https", + } + }; + let proxy = LocalProxy::start(addr, Ipv4Addr::new(127, 0, 0, 2), assertions, false); + for (prefix, expected_result, client, host, scheme) in [ + ( + "Forwarded: for=198.51.100.1;host=wrong.example;proto=http\r\nX-Forwarded-For: 203.0.113.1\r\nX-Forwarded-Host: wrong.example\r\nX-Forwarded-Proto: http\r\n", + "accepted", + "127.0.0.1", + "selected.example", + "https", + ), + ( + if family == "forwarded" { + "Forwarded: for=bad-address;host=good.example;proto=https\r\n" + } else { + "X-Forwarded-For: bad-address\r\n" + }, + "malformed", + "127.0.0.2", + "direct.example:8080", + "http", + ), + ( + if family == "forwarded" { + "Forwarded: for=unknown\r\n" + } else { + "X-Forwarded-For: unknown\r\n" + }, + "accepted", + "127.0.0.1", + "selected.example", + "https", + ), + ] { + let value = response_json(&raw_request(proxy.addr(), &metadata_request(prefix))); + assert_metadata(&value, client, host, scheme, expected_result); + } + assert_missing_unresolved_and_duplicate_family(addr, family); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + let records = observations(&server) + .into_iter() + .filter(|record| record["route"] == "/native-metadata") + .collect::>(); + assert_eq!(records.len(), 7); + for record in records { + assert_record_log(&server, &record); + if record["forwarding_result"] == "malformed" { + assert_eq!(record["level"], "WARN"); + } + } + } + } + + #[test] + fn native_verified_tls_ingress_overwrites_public_assertions_and_rejects_wrong_name() { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn( + native_command("diagnostics", &roots, addr, "127.0.0.2/32", "forwarded"), + addr, + ); + server.ready(); + let proxy = LocalProxy::start( + addr, + Ipv4Addr::new(127, 0, 0, 2), + Assertions::AppendForwarded { + host: "native-proxy.test", + scheme: "https", + }, + true, + ); + let request = metadata_request( + "Forwarded: for=198.51.100.99;host=attacker.example;proto=http\r\nX-Forwarded-Host: attacker.example\r\nX-Forwarded-Proto: http\r\n", + ); + let response = verified_tls_request(proxy.addr(), request.as_bytes(), "native-proxy.test") + .expect("CA and hostname verified TLS"); + let value = response_json(&response); + assert_metadata( + &value, + "127.0.0.1", + "native-proxy.test", + "https", + "accepted", + ); + assert_eq!(value["handler"]["scheme_source"], "trusted_forwarded"); + assert_no_diagnostic_headers(&response); + let wrong_name = verified_tls_request(proxy.addr(), request.as_bytes(), "wrong-name.test") + .expect_err("wrong server name must fail verification"); + assert_eq!( + wrong_name.kind(), + io::ErrorKind::InvalidData, + "certificate verification failure, not a connection timeout" + ); + assert!( + format!("{wrong_name:?}").contains("NotValidForName"), + "hostname verification must be the rejecting boundary" + ); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + assert_eq!( + observations(&server) + .iter() + .filter(|record| record["route"] == "/native-metadata") + .count(), + 1 + ); + } + + fn assert_native_error_record( + server: &Server, + path: &str, + status: u16, + failure: &str, + level: &str, + ) { + if path.starts_with("/PRIVATE_PATH") { + return; + } + let route = path.split('?').next().expect("registered route"); + let record = route_record(server, route); + assert_eq!(record["status"], status); + assert_eq!(record["level"], level); + if !failure.is_empty() { + assert_eq!(record["failure"], failure); + } + if path == "/native-status503" { + assert!(record["failure"].is_null(), "status alone is not rejection"); + } + if matches!(path, "/native-conversion" | "/native-expired") { + assert_eq!(record["body_kind"], "fallback"); + assert_eq!(record["fallback"], "completed"); + assert_ne!(record["outcome"], "completed"); + } + } + + fn assert_default_error_observations(server: &Server) { + let records = observations(server); + let overflows = records + .iter() + .filter(|record| record["status"] == 413_u16) + .collect::>(); + assert_eq!(overflows.len(), 1); + assert_eq!( + overflows + .into_iter() + .next() + .expect("real JSON cap rejection")["failure"], + "payload_too_large" + ); + let extension = records + .iter() + .find(|record| record["method"] == "other") + .expect("extension method bounded category"); + assert_eq!(extension["status"], 405_u16); + assert_eq!(extension["level"], "INFO"); + for entry in framework_logs(server) { + assert!( + !entry["message"] + .as_str() + .expect("framework message") + .contains("SENTINEL"), + "framework log leak: {entry}" + ); + let message = entry["message"].as_str().expect("framework message"); + assert!( + !message.contains("CONFIG_VALUE_SENTINEL") + && !message.contains("SECRET_VALUE_SENTINEL"), + "environment-derived private inputs leaked through the facade: {entry}" + ); + } + for record in records { + assert_record_log(server, &record); + } + } + + #[test] + fn native_default_errors_json_cap_and_logs_keep_sentinels_private() { + let roots = Roots::new(); + let addr = address(); + let mut launch = command("diagnostics", &roots, addr); + launch + .env("EDGEZERO__ADAPTER__JSON_BODY_LIMIT_BYTES", "64") + .env("PRIVATE_CONFIG_SENTINEL", "CONFIG_VALUE_SENTINEL") + .env("fixture_token", "SECRET_VALUE_SENTINEL"); + let mut server = Server::spawn(launch, addr); + server.ready(); + for (path, status, failure, level) in [ + ( + "/PRIVATE_PATH_SENTINEL?token=QUERY_SENTINEL", + 404, + "not_found", + "INFO", + ), + ( + "/native-error?token=QUERY_SENTINEL", + 500, + "internal", + "WARN", + ), + ("/native-upstream", 502, "bad_gateway", "WARN"), + ("/native-conversion", 500, "", "WARN"), + ("/native-expired", 504, "", "WARN"), + ("/native-status503", 503, "", "INFO"), + ] { + let response = raw_request( + addr, + &format!( + "GET {path} HTTP/1.1\r\nHost: HOST_SENTINEL\r\nAuthorization: Bearer AUTH_SENTINEL\r\nCookie: COOKIE_SENTINEL\r\nX-Request-ID: ID_SENTINEL\r\nConnection: close\r\n\r\n" + ), + ); + assert!( + response.starts_with(&format!("HTTP/1.1 {status}")), + "{response}" + ); + assert!( + !response.contains("SENTINEL"), + "default wire leak: {response}" + ); + assert!( + !response.contains("CONFIG_VALUE_SENTINEL") + && !response.contains("SECRET_VALUE_SENTINEL"), + "environment-derived private inputs leaked through the wire: {response}" + ); + assert_no_diagnostic_headers(&response); + assert_native_error_record(&server, path, status, failure, level); + } + let extension_response = raw_request( + addr, + "METHOD_SENTINEL /native-status503?token=QUERY_SENTINEL HTTP/1.1\r\nHost: fixture\r\nConnection: close\r\n\r\n", + ); + assert!(extension_response.starts_with("HTTP/1.1 405")); + assert!(!extension_response.contains("SENTINEL")); + assert!(extension_response.to_ascii_lowercase().contains("allow:")); + let json_response = upload( + &server, + "/native-json-number", + Some("application/json"), + b"\"PAYLOAD_SENTINEL\"".to_vec(), + false, + ); + assert_upload_status(&json_response, 400); + assert!(!json_response.contains("PAYLOAD_SENTINEL")); + assert_eq!( + response_json(&json_response)["error"]["message"], + "invalid JSON payload" + ); + assert_upload_status(&upload(&server, "/json", None, exact_json(64), false), 200); + assert_upload_status(&upload(&server, "/json", None, exact_json(65), true), 413); + let handler_error = server + .request("/native-handler404") + .expect("application error"); + assert!( + handler_error.contains("HANDLER_404_SENTINEL"), + "application error messages remain application policy" + ); + let handler_record = route_record(&server, "/native-handler404"); + assert_eq!(handler_record["level"], "WARN"); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + assert_eq!( + server + .log() + .lines() + .filter(|line| *line == "PRIVATE_INPUTS_USED") + .count(), + 2, + "both error paths consumed the fixture's captured environment-derived configuration and secret inputs" + ); + assert_default_error_observations(&server); + } + + #[test] + fn native_custom_renderer_keeps_ownership_but_framework_json_parse_text_stays_fixed() { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn(command("diagnostics-custom", &roots, addr), addr); + server.ready(); + let custom = server + .request("/CUSTOM_ROUTE_SENTINEL") + .expect("custom route error"); + assert!(custom.starts_with("HTTP/1.1 404")); + assert!( + custom.contains("CUSTOM_ROUTE_SENTINEL"), + "custom renderer retains original routing detail" + ); + let parsed = upload( + &server, + "/native-json-number", + Some("application/json"), + b"\"PAYLOAD_SENTINEL\"".to_vec(), + false, + ); + assert!(parsed.starts_with("HTTP/1.1 400")); + assert!(parsed.ends_with("invalid JSON payload")); + assert!(!parsed.contains("PAYLOAD_SENTINEL")); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + for entry in framework_logs(&server) { + assert!( + !entry["message"] + .as_str() + .expect("framework message") + .contains("SENTINEL") + ); + } + } + + #[test] + fn native_original_head_overage_cannot_be_hidden_by_raw_header_stripping() { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn(command("diagnostics", &roots, addr), addr); + server.ready(); + let request = metadata_request(&format!( + "X-Forwarded-Secret: {}\r\n", + "HEAD_SENTINEL".repeat(6000) + )); + let response = raw_request(addr, &request); + assert!( + response.starts_with("HTTP/1.1 431"), + "original normalized head limit: {response}" + ); + assert!(!response.contains("SENTINEL")); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + let rejections = observations(&server) + .into_iter() + .filter(|record| record["status"] == 431_u16) + .collect::>(); + assert_eq!(rejections.len(), 1); + let record = rejections + .into_iter() + .next() + .expect("head rejection record"); + assert_eq!(record["level"], "WARN"); + assert_eq!( + server.log().matches("APP_COMPLETION ").count(), + 1, + "head failure precedes app admission; only readiness was admitted" + ); + assert_record_log(&server, &record); + } + + #[test] + fn native_ids_remain_private_under_mutation_and_concurrent_hosting_requests() { + let roots = Roots::new(); + let addr = address(); + let other_addr = address(); + let mut server = Server::spawn(command("diagnostics", &roots, addr), addr); + let mut other = Server::spawn(command("diagnostics", &roots, other_addr), other_addr); + server.ready(); + other.ready(); + let response = raw_request( + addr, + "GET /native-mutated HTTP/1.1\r\nHost: fixture\r\nX-Request-ID: VISITOR_ID_SENTINEL\r\nConnection: close\r\n\r\n", + ); + assert_no_diagnostic_headers(&response); + let mutated = response_json(&response); + assert_eq!(mutated["metadata_absent"], true); + assert_eq!(mutated["forwarded_host_after"], "replacement.example"); + let authoritative = route_record(&server, "/native-mutated"); + assert_eq!(authoritative["id"], mutated["removed_id"]); + let mut clients = Vec::new(); + for target in [addr, other_addr] { + for _request in 0..4_u8 { + clients.push(thread::spawn(move || { + let concurrent_response = raw_request(target, "GET /native-status503 HTTP/1.1\r\nHost: fixture\r\nX-Request-ID: VISITOR_ID_SENTINEL\r\nConnection: close\r\n\r\n"); + assert!(concurrent_response.starts_with("HTTP/1.1 503")); + assert_no_diagnostic_headers(&concurrent_response); + })); + } + } + for client in clients { + client.join().expect("bounded concurrent wire client"); + } + for running in [&mut server, &mut other] { + running.signal("TERM"); + assert!(running.wait().success()); + assert_final_zero(running); + } + let mut ids = HashSet::new(); + for running in [&server, &other] { + let records = observations(running); + assert_eq!( + records + .iter() + .filter(|record| record["route"] == "/native-status503") + .count(), + 4 + ); + assert_eq!( + running.log().matches("APP_COMPLETION ").count(), + records.len() + ); + assert_eq!( + running.log().matches("APP_OBSERVER ").count(), + records.len() + ); + for record in records { + let id = record["id"].as_str().expect("generated ID"); + assert_eq!(id.len(), 64); + assert!(id.bytes().all(|byte| byte.is_ascii_hexdigit())); + assert!(ids.insert(id.to_owned()), "no process/request ID collision"); + assert!( + record["counts"]["active_requests"] + .as_u64() + .expect("request count") + <= 3, + "the observed request releases before its sink; only other concurrent requests remain" + ); + assert_record_log(running, &record); + } + assert!(!running.log().contains("VISITOR_ID_SENTINEL")); + } + } + + #[test] + fn native_logger_panic_preserves_order_cardinality_and_resource_release() { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn(command("diagnostics-panic", &roots, addr), addr); + server.ready(); + assert!( + server + .request("/native-status503") + .expect("surviving response") + .starts_with("HTTP/1.1 503") + ); + let selected_record = route_record(&server, "/native-status503"); + wait_for(&server, |log| log.matches("APP_OBSERVER ").count() >= 2); + assert_hook_order(&server, selected_record["id"].as_str().expect("record ID")); + assert_eq!(selected_record["counts"]["active_requests"], 0_u64); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + let log = server.log(); + assert_eq!(log.matches("APP_LOGGER_INSTALLED").count(), 1); + assert_eq!( + log.matches("LOGGER_FAULT").count(), + 1, + "no recursive logger fault reporting" + ); + let records = observations(&server); + assert_eq!(log.matches("APP_COMPLETION ").count(), records.len()); + assert_eq!(log.matches("APP_OBSERVER ").count(), records.len()); + for record in records { + assert_record_log(&server, &record); + } + } + + fn wire_prefix(socket: &mut TcpStream, prefix: &[u8]) -> Vec { + let mut received = Vec::new(); + let deadline = Instant::now().checked_add(WAIT).expect("prefix timer"); + while !received.windows(prefix.len()).any(|bytes| bytes == prefix) { + assert!(Instant::now() < deadline, "wire prefix deadline"); + assert!(received.len() < 0x4000, "finite response head and prefix"); + let mut chunk = [0_u8; 512]; + let count = socket + .read(&mut chunk) + .expect("wire prefix before terminal failure"); + assert_ne!(count, 0, "connection closed before prefix: {received:?}"); + received.extend_from_slice(chunk.get(..count).expect("wire read length")); + } + assert!(received.starts_with(b"HTTP/1.1 200")); + assert!(received.windows(4).any(|bytes| bytes == b"\r\n\r\n")); + received + } + + #[test] + fn native_actual_wire_prefix_precedes_source_and_deadline_failure_without_second_response() { + for (path, prefix, outcome, dropped) in [ + ( + "/native-source", + b"source-prefix".as_slice(), + "source_error", + "SOURCE_DROPPED", + ), + ( + "/native-deadline", + b"deadline-prefix".as_slice(), + "deadline_exceeded", + "DEADLINE_DROPPED", + ), + ] { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn(command("diagnostics", &roots, addr), addr); + server.ready(); + let mut client = server.socket().expect("streaming client"); + write!( + client, + "GET {path} HTTP/1.1\r\nHost: fixture\r\nConnection: close\r\n\r\n" + ) + .expect("stream request"); + let mut wire = wire_prefix(&mut client, prefix); + let mut tail = Vec::new(); + let terminal = client.read_to_end(&mut tail); + if let Err(error) = terminal { + assert!( + matches!( + error.kind(), + io::ErrorKind::ConnectionReset | io::ErrorKind::UnexpectedEof + ), + "failure did not settle before timeout: {error}" + ); + } + wire.extend_from_slice(&tail); + let wire_text = String::from_utf8(wire).expect("ASCII fixture response"); + assert_eq!( + wire_text.matches("HTTP/1.1 ").count(), + 1, + "no replacement response after commitment" + ); + assert!(!wire_text.contains("SENTINEL")); + assert!( + !wire_text.ends_with("0\r\n\r\n"), + "stream failure must not look like completed chunked framing" + ); + let record = route_record(&server, path); + assert_eq!( + record["status"], 200_u16, + "selected status is not successful delivery" + ); + assert_eq!(record["outcome"], outcome); + assert_eq!(record["level"], "WARN"); + assert_eq!(record["bytes_written"], prefix.len()); + assert_eq!(record["body_kind"], "application"); + assert!(record["fallback"].is_null()); + assert_eq!(record["counts"]["active_requests"], 0_u64); + wait_for(&server, |log| { + log.contains(dropped) && log.matches("APP_OBSERVER ").count() >= 2 + }); + assert_hook_order(&server, record["id"].as_str().expect("record ID")); + assert_record_log(&server, &record); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + assert_eq!(server.log().matches(dropped).count(), 1); + assert_eq!( + observations(&server) + .iter() + .filter(|observed| observed["route"] == path) + .count(), + 1 + ); + for entry in framework_logs(&server) { + assert!( + !entry["message"] + .as_str() + .expect("framework message") + .contains("SENTINEL") + ); + } + } + } + + #[test] + fn native_actual_wire_disconnect_keeps_selected_status_and_one_terminal_owner() { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn(command("diagnostics", &roots, addr), addr); + server.ready(); + let mut client = server.socket().expect("stream client"); + client + .write_all(b"GET /stream HTTP/1.1\r\nHost: fixture\r\n\r\n") + .expect("stream request"); + let wire = wire_prefix(&mut client, b"stream-prefix"); + assert_eq!( + String::from_utf8(wire) + .expect("wire head") + .matches("HTTP/1.1 ") + .count(), + 1 + ); + drop(client); + let record = route_record(&server, "/stream"); + assert_eq!(record["status"], 200_u16); + // The existing HTTP/1 connection owner reports this peer loss as a transport failure. + assert_eq!(record["outcome"], "transport_error"); + assert_eq!(record["level"], "WARN"); + assert_eq!(record["counts"]["active_requests"], 0_u64); + wait_for(&server, |log| { + log.contains("STREAM_DROPPED") && log.matches("APP_OBSERVER ").count() >= 2 + }); + assert_hook_order(&server, record["id"].as_str().expect("record ID")); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + assert_eq!(server.log().matches("STREAM_DROPPED").count(), 1); + assert_record_log(&server, &record); + } + + #[test] + fn native_refusal_and_no_response_abort_keep_separate_observer_contracts() { + let roots = Roots::new(); + let addr = address(); + let mut server = Server::spawn(command("diagnostics", &roots, addr), addr); + server.ready(); + let response = server.request("/native-refuse").expect("refusal response"); + assert!(response.starts_with("HTTP/1.1 503")); + let mut client = server.socket().expect("abort connection"); + client + .write_all(b"GET /native-abort HTTP/1.1\r\nHost: fixture\r\nConnection: close\r\n\r\n") + .expect("abort request"); + let mut aborted_wire = Vec::new(); + let abort_read = client.read_to_end(&mut aborted_wire); + if let Err(error) = abort_read { + assert_eq!(error.kind(), io::ErrorKind::ConnectionReset); + } + assert!(aborted_wire.is_empty(), "no fabricated abort response"); + server.signal("TERM"); + assert!(server.wait().success()); + assert_final_zero(&server); + let records = observations(&server); + let refusal = records + .iter() + .filter(|record| record["failure"] == "admission_refused") + .collect::>(); + assert_eq!(refusal.len(), 1); + assert_eq!( + refusal.into_iter().next().expect("refusal record")["status"], + 503_u16 + ); + let abort_records = records + .iter() + .filter(|record| record["outcome"] == "aborted") + .collect::>(); + assert_eq!(abort_records.len(), 1); + let abort_record = abort_records + .into_iter() + .next() + .expect("no response ingress record"); + assert_eq!(abort_record["event"], "request_ingress"); + assert!(abort_record["status"].is_null() && abort_record["bytes_written"].is_null()); + assert_eq!( + server.log().matches("APP_COMPLETION ").count(), + 2, + "readiness plus supplied refusal completion only" + ); + assert_eq!( + server.log().matches("APP_OBSERVER ").count(), + 1, + "refusal is detached; abort has no egress observer" + ); + for record in records { + assert_record_log(&server, &record); + } + } + + fn native_counts(server: &Server, expected_requests: u64, expected_idle: u64) -> Value { + let deadline = Instant::now().checked_add(WAIT).expect("snapshot timer"); + loop { + let counts = response_json(&server.request("/native-counts").expect("counter query")); + if counts["active_requests"] == expected_requests + && counts["idle_connections"] == expected_idle + { + return counts; + } + assert!( + Instant::now() < deadline, + "snapshot did not converge: {counts}; {}", + server.log() + ); + thread::sleep(Duration::from_millis(5)); + } + } + + fn assert_shutdown_observations(server: &Server, path: &str, graceful: bool, optout: bool) { + let logs = framework_logs(server); + for phase in ["starting", "ready", "draining", "stopped"] { + assert!( + logs.iter().any(|entry| entry["message"] + .as_str() + .expect("lifecycle message") + .contains(&format!("event=lifecycle phase={phase}"))), + "missing lifecycle {phase}" + ); + } + if !graceful { + assert!(logs.iter().any(|entry| { + entry["message"] + .as_str() + .expect("fault category") + .contains("reason=grace_exhausted") + })); + } + let records = observations(server); + assert!(!records.is_empty(), "native observer retained"); + assert!(server.log().contains("APP_OBSERVER ") && server.log().contains("APP_COMPLETION ")); + if optout { + assert!( + !logs.iter().any(|entry| { + let message = entry["message"].as_str().expect("framework message"); + message.starts_with("request_terminal ") + || message.starts_with("request_ingress ") + }), + "request-record opt-out" + ); + } + let abandoned = records + .iter() + .filter(|record| record["outcome"] == "abandoned") + .collect::>(); + if matches!(path, "/pending" | "/json") { + assert_eq!(abandoned.len(), 1); + let record = abandoned + .into_iter() + .next() + .expect("pre-attempt abandonment"); + assert!(record["status"].is_null()); + assert_eq!(record["event"], "request_ingress"); + } + } + + #[test] + fn native_idle_and_held_work_snapshots_drain_to_zero_with_request_records_opted_out() { + for optout in [false, true] { + for (path, prefix, marker, graceful) in [ + ("/finite", None, "DISPATCH", true), + ("/pending", None, "DISPATCH", false), + ( + "/stream", + Some(b"stream-prefix".as_slice()), + "STREAM_READY", + false, + ), + ("/json", None, "", false), + ] { + let roots = Roots::new(); + let addr = address(); + let mut launch = command( + if optout { + "diagnostics-optout" + } else { + "diagnostics" + }, + &roots, + addr, + ); + if optout { + launch.env("EDGEZERO__LOGGING__REQUEST_RECORDS", "false"); + } + let mut server = Server::spawn(launch, addr); + server.ready(); + let mut idle = server.socket().expect("owned keepalive connection"); + idle.write_all(b"GET /ready HTTP/1.1\r\nHost: fixture\r\n\r\n") + .expect("keepalive request"); + let mut ready = Vec::new(); + while !ready.ends_with(b"ready") { + let mut buffer = [0_u8; 512]; + let count = idle.read(&mut buffer).expect("keepalive ready response"); + assert_ne!(count, 0); + ready.extend_from_slice(buffer.get(..count).expect("read length")); + } + let idle_counts = native_counts(&server, 1, 1); + assert_eq!( + idle_counts["active_connections"], 2_u64, + "idle connection plus count query" + ); + let mut held = server.socket().expect("held request"); + if path == "/json" { + held.write_all(b"POST /json HTTP/1.1\r\nHost: fixture\r\nContent-Type: application/json\r\nContent-Length: 64\r\n\r\n\"x").expect("partial body held open"); + } else { + write!(held, "GET {path} HTTP/1.1\r\nHost: fixture\r\n\r\n") + .expect("held request"); + server.wait_line(marker); + } + if let Some(expected_prefix) = prefix { + let _wire = wire_prefix(&mut held, expected_prefix); + } + let held_counts = native_counts(&server, 2, 1); + assert_eq!( + held_counts["active_connections"], 3_u64, + "held, idle and query connections" + ); + assert_eq!(held_counts["phase"], "ready"); + let started = Instant::now(); + server.signal("TERM"); + if graceful { + let mut response = String::new(); + held.read_to_string(&mut response) + .expect("graceful finite response"); + assert!( + response.starts_with("HTTP/1.1 200") + && response.contains("finite-complete") + ); + } + let status = server.wait(); + assert_eq!(status.success(), graceful, "{}", server.log()); + assert!(started.elapsed() < Duration::from_secs(4)); + if !graceful { + assert!(started.elapsed() >= Duration::from_millis(700)); + } + assert_final_zero(&server); + assert_eq!(idle.read(&mut [0_u8; 1]).expect("idle close"), 0); + assert_shutdown_observations(&server, path, graceful, optout); + } + } + } + + #[test] + fn native_invalid_proxy_and_record_settings_fail_before_ready_without_input_disclosure() { + let roots = Roots::new(); + for (networks, family, records, category) in [ + ( + Some("NETWORK_SENTINEL.invalid"), + Some("forwarded"), + None, + "TRUSTED_PROXY_POLICY", + ), + ( + Some("0.0.0.0/0"), + Some("forwarded"), + None, + "TRUSTED_PROXY_POLICY", + ), + ( + Some("::/0"), + Some("forwarded"), + None, + "TRUSTED_PROXY_POLICY", + ), + ( + Some("0.0.0.0/1,128.0.0.0/1"), + Some("forwarded"), + None, + "TRUSTED_PROXY_POLICY", + ), + (Some("127.0.0.2/32"), None, None, "TRUSTED_PROXY_POLICY"), + ( + Some("127.0.0.2/32"), + Some("FAMILY_SENTINEL"), + None, + "TRUSTED_PROXY_POLICY", + ), + ( + Some(""), + Some("FAMILY_SENTINEL"), + None, + "TRUSTED_PROXY_POLICY", + ), + (None, None, Some("RECORDS_SENTINEL"), "REQUEST_RECORDS"), + (None, None, Some("FALSE"), "REQUEST_RECORDS"), + ] { + let addr = address(); + let mut launch = command("diagnostics", &roots, addr); + if let Some(value) = networks { + launch.env("EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS", value); + } + if let Some(value) = family { + launch.env("EDGEZERO__ADAPTER__FORWARDING_HEADER_FAMILY", value); + } + if let Some(value) = records { + launch.env("EDGEZERO__LOGGING__REQUEST_RECORDS", value); + } + let log = failed(launch, addr, category); + assert!( + !log.contains("HOOK_ENTERED"), + "invalid settings precede app construction" + ); + assert!(!log.contains("phase=ready")); + if let Some(value) = networks.filter(|text| !text.is_empty()) { + assert!(!log.contains(value), "network input leaked: {log}"); + } + if let Some(value) = records { + assert!(!log.contains(value), "record-setting input leaked: {log}"); + } + } + } + + #[test] + fn native_non_unicode_proxy_and_record_settings_fail_instead_of_defaulting() { + use std::os::unix::ffi::OsStringExt as _; + let roots = Roots::new(); + for (key, category) in [ + ( + "EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS", + "TRUSTED_PROXY_CIDRS", + ), + ( + "EDGEZERO__ADAPTER__FORWARDING_HEADER_FAMILY", + "FORWARDING_HEADER_FAMILY", + ), + ("EDGEZERO__LOGGING__REQUEST_RECORDS", "REQUEST_RECORDS"), + ] { + let addr = address(); + let mut launch = command("diagnostics", &roots, addr); + let mut raw = b"NON_UNICODE_SENTINEL".to_vec(); + raw.push(0xff); + launch.env(key, OsString::from_vec(raw)); + let log = failed(launch, addr, category); + assert!( + log.contains("non-Unicode"), + "captured native setting was rejected: {log}" + ); + assert!(!log.contains("HOOK_ENTERED") && !log.contains("phase=ready")); + assert!( + !log.contains('\u{fffd}'), + "no lossy rendering of non-Unicode input" + ); + } + } + #[test] fn json_body_limit_wire_content_length_and_chunked() { let roots = Roots::new(); diff --git a/crates/edgezero-core/src/app.rs b/crates/edgezero-core/src/app.rs index de77c2cc1..e1d0dd5c4 100644 --- a/crates/edgezero-core/src/app.rs +++ b/crates/edgezero-core/src/app.rs @@ -1,3 +1,4 @@ +use std::panic::catch_unwind; use std::sync::Arc; use crate::body::Body; @@ -12,15 +13,14 @@ use crate::ingress::{ }; use crate::manifest::BakedManifest; use crate::platform::PlatformMetadata; -use crate::response::IntoResponse as _; use crate::response_egress::{ DEFAULT_RESPONSE_WRITE_BUDGET, DetachedResponseEgressDecision, DetachedResponseEgressDecisionFactory, DetachedResponseEgressHead, ResponseEgressCompletion, - ResponseEgressDeadline, ResponseEgressEnvelope, ResponseEgressHead, ResponseEgressObserver, - ResponseEgressObserverHandle, ResponseEgressPolicy, ResponseEgressPolicyCallback, - ResponseEgressRequestMetadata, default_response_egress_policy, + ResponseEgressDeadline, ResponseEgressEnvelope, ResponseEgressFailureClassification, + ResponseEgressHead, ResponseEgressObserver, ResponseEgressObserverHandle, ResponseEgressPolicy, + ResponseEgressPolicyCallback, ResponseEgressRequestMetadata, default_response_egress_policy, }; -use crate::router::{RouteMetadata, RouterService}; +use crate::router::{RouteErrorKind, RouteMetadata, RouterService}; use crate::time::{MonotonicClock, MonotonicInstant}; /// Canonical adapter name for the Axum adapter. @@ -32,7 +32,18 @@ const DEFAULT_APP_NAME: &str = "EdgeZero App"; pub const FASTLY_ADAPTER: &str = "fastly"; /// Canonical adapter name for the Spin adapter. pub const SPIN_ADAPTER: &str = "spin"; -type ErrorResponseRenderer = Arc Response + Send + Sync + 'static>; +enum ErrorResponseRenderer { + Custom(Arc Response + Send + Sync + 'static>), + Default, +} + +/// Presentation of framework-generated routing errors on a managed adapter path. +#[doc(hidden)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FrameworkErrorDetail { + CategoryOnly, + Default, +} /// Lightweight container around a `RouterService` that can be extended via hook implementations. pub struct App { @@ -92,6 +103,7 @@ impl App { let request_start = prepared.request_start(); let route = prepared.route_metadata().cloned(); let (_resolved, _admitted, completion) = prepared.into_parts(); + let failure = ResponseEgressFailureClassification::error(error.kind()); self.response_egress_envelope( self.render_error_response(error), request_method, @@ -99,6 +111,7 @@ impl App { route, completion, ) + .with_failure(Some(failure)) } /// Resolves and admits a normalized request head before an adapter transfers native body @@ -182,6 +195,7 @@ impl App { request_method: Method, request_start: MonotonicInstant, ) -> IngressDispatchOutcome { + let failure = ResponseEgressFailureClassification::error(error.kind()); let head = DetachedResponseEgressHead::new( &request_method, request_start, @@ -198,12 +212,15 @@ impl App { response .extensions_mut() .insert(ResponseEgressDeadline::at(deadline)); - IngressDispatchOutcome::Response(Box::new(self.detached_response_egress( - response, - request_method, - request_start, - completion, - ))) + IngressDispatchOutcome::Response(Box::new( + self.detached_response_egress( + response, + request_method, + request_start, + completion, + ) + .with_failure(Some(failure)), + )) } } } @@ -231,20 +248,55 @@ impl App { &self, prepared: PreparedIngress, request: Request, + ) -> ResponseEgressEnvelope { + self.dispatch_admitted_with_framework_error_detail( + prepared, + request, + FrameworkErrorDetail::Default, + ) + .await + } + + /// Dispatches with native default routing-error presentation, preserving custom renderers. + #[doc(hidden)] + #[inline] + pub async fn dispatch_admitted_with_framework_error_detail( + &self, + prepared: PreparedIngress, + request: Request, + detail: FrameworkErrorDetail, ) -> ResponseEgressEnvelope { let request_method = request.method().clone(); let request_start = prepared.request_start(); let route = prepared.route_metadata().cloned(); let (resolved, admitted, completion) = prepared.into_parts(); - let response = match self + let (result, route_error, mut failure) = self .router - .dispatch_resolved(resolved, request, admitted) + .dispatch_resolved_outcome(resolved, request, admitted) .await - { + .into_parts(); + let response = match result { Ok(response) => response, - Err(error) => self.render_error_response(error), + Err(error) => { + failure = Some(if route_error.is_some() { + ResponseEgressFailureClassification::routing(error.kind()) + } else { + ResponseEgressFailureClassification::error(error.kind()) + }); + let fixed = if detail == FrameworkErrorDetail::CategoryOnly { + match route_error { + Some(RouteErrorKind::NotFound) => Some("route not found"), + Some(RouteErrorKind::MethodNotAllowed) => Some("method not allowed"), + None => None, + } + } else { + None + }; + self.render_error_response_with_message(error, fixed) + } }; self.response_egress_envelope(response, request_method, request_start, route, completion) + .with_failure(failure) } /// Resolves, admits, and dispatches one normalized inbound request into owned egress. @@ -339,12 +391,25 @@ impl App { } fn render_error_response(&self, error: EdgeError) -> Response { + self.render_error_response_with_message(error, None) + } + + fn render_error_response_with_message( + &self, + error: EdgeError, + fixed: Option<&'static str>, + ) -> Response { let status = error.status(); let required_headers = match error.required_response_headers() { Ok(headers) => headers, Err(header_error) => return render_default_error_response(header_error), }; - let mut response = (self.error_response_renderer)(error); + let mut response = match &self.error_response_renderer { + ErrorResponseRenderer::Default => { + render_default_error_response_with_message(error, fixed) + } + ErrorResponseRenderer::Custom(renderer) => renderer(error), + }; *response.status_mut() = status; response.headers_mut().extend(required_headers); response @@ -426,7 +491,7 @@ impl App { where Renderer: Fn(EdgeError) -> Response + Send + Sync + 'static, { - self.error_response_renderer = Arc::new(renderer); + self.error_response_renderer = ErrorResponseRenderer::Custom(Arc::new(renderer)); } /// Installs the synchronous body-blind ingress admission callback. @@ -508,7 +573,7 @@ impl App { deadline: head.write_deadline_after(DEFAULT_RESPONSE_WRITE_BUDGET), } }), - error_response_renderer: Arc::new(render_default_error_response), + error_response_renderer: ErrorResponseRenderer::Default, ingress_head_limits: IngressHeadLimits::default(), ingress_policy: default_admission_policy(), monotonic_clock: MonotonicClock::default(), @@ -608,10 +673,18 @@ pub trait Hooks { } fn render_default_error_response(error: EdgeError) -> Response { - match error.into_response() { + render_default_error_response_with_message(error, None) +} + +fn render_default_error_response_with_message( + error: EdgeError, + fixed: Option<&'static str>, +) -> Response { + match error.into_response_with_message(fixed) { Ok(response) => response, - Err(render_error) => { - log::error!("failed to render typed ingress error response: {render_error}"); + Err(_render_error) => { + let _logged = + catch_unwind(|| log::error!("failed to render typed ingress error response")); // Keep the last-resort response independent of JSON serialization. let mut response = Response::new(Body::from( r#"{"error":{"kind":"internal","message":"internal server error","status":500}}"#, @@ -1858,6 +1931,140 @@ mod tests { ); } + fn managed_dispatch( + app: &App, + method: Method, + path: &str, + detail: FrameworkErrorDetail, + ) -> ResponseEgressEnvelope { + let request = request_builder() + .method(method) + .uri(path) + .body(Body::empty()) + .expect("request"); + let head = IngressHeadParts::from_request( + &request, + IngressHeadAccounting::HostManaged, + IngressFraming::HostManaged, + ); + let IngressBeginOutcome::Admitted(prepared) = app + .begin_ingress(head, app.monotonic_now()) + .expect("admission") + else { + panic!("admission"); + }; + block_on(app.dispatch_admitted_with_framework_error_detail(prepared, request, detail)) + } + + #[test] + fn native_default_routing_errors_do_not_reflect_path_or_custom_method() { + let app = App::new( + RouterService::builder() + .get("/known", |_ctx: RequestContext| async { + Ok::<_, EdgeError>("ok") + }) + .build(), + ); + for (method, path, status, fixed) in [ + ( + Method::GET, + "/private-sentinel", + StatusCode::NOT_FOUND, + "route not found", + ), + ( + Method::from_bytes(b"PRIVATE-SENTINEL").expect("method"), + "/known", + StatusCode::METHOD_NOT_ALLOWED, + "method not allowed", + ), + ] { + let envelope = managed_dispatch(&app, method, path, FrameworkErrorDetail::CategoryOnly); + let failure = envelope.failure_classification().expect("routing category"); + assert!(failure.is_framework_route_error()); + let response = complete_dispatch(IngressDispatchOutcome::Response(Box::new(envelope))); + assert_eq!(response.status(), status); + if status == StatusCode::METHOD_NOT_ALLOWED { + assert_eq!(response.headers()["allow"], "GET"); + } + let payload: serde_json::Value = + serde_json::from_slice(response.body().as_bytes().expect("body")).expect("json"); + assert_eq!(payload["error"]["message"], fixed); + assert!( + !str::from_utf8(response.body().as_bytes().expect("body")) + .expect("utf8") + .contains("sentinel") + ); + } + let legacy = managed_dispatch( + &app, + Method::GET, + "/private-sentinel", + FrameworkErrorDetail::Default, + ); + let response = complete_dispatch(IngressDispatchOutcome::Response(Box::new(legacy))); + assert!( + str::from_utf8(response.body().as_bytes().expect("body")) + .expect("utf8") + .contains("private-sentinel") + ); + } + + #[test] + fn native_routing_detail_preserves_custom_renderer_input_and_handler_errors() { + let mut app = App::new( + RouterService::builder() + .get("/handler", |_ctx: RequestContext| async { + Err::<&'static str, _>(EdgeError::not_found("handler-sentinel")) + }) + .build(), + ); + let handler = managed_dispatch( + &app, + Method::GET, + "/handler", + FrameworkErrorDetail::CategoryOnly, + ); + assert!( + !handler + .failure_classification() + .expect("handler error") + .is_framework_route_error() + ); + let handler_response = + complete_dispatch(IngressDispatchOutcome::Response(Box::new(handler))); + assert!( + str::from_utf8(handler_response.body().as_bytes().expect("body")) + .expect("utf8") + .contains("handler-sentinel") + ); + let seen = Arc::new(Mutex::new(String::new())); + let renderer_seen = Arc::clone(&seen); + app.set_error_response_renderer(move |error| { + *renderer_seen.lock().expect("seen") = error.to_string(); + Response::new(Body::from("application-owned")) + }); + let envelope = managed_dispatch( + &app, + Method::GET, + "/route-sentinel", + FrameworkErrorDetail::CategoryOnly, + ); + assert!( + envelope + .failure_classification() + .expect("routing error") + .is_framework_route_error() + ); + let response = complete_dispatch(IngressDispatchOutcome::Response(Box::new(envelope))); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!( + response.body().as_bytes().expect("body"), + b"application-owned" + ); + assert!(seen.lock().expect("seen").contains("route-sentinel")); + } + #[test] fn custom_error_response_renderer_handles_admitted_dispatch_errors() { let router = RouterService::builder() diff --git a/crates/edgezero-core/src/context.rs b/crates/edgezero-core/src/context.rs index 4fa1c6cae..cfad6d237 100644 --- a/crates/edgezero-core/src/context.rs +++ b/crates/edgezero-core/src/context.rs @@ -462,15 +462,21 @@ impl RequestContext { /// /// # Errors /// Managed JSON-policy overflow returns 413; unmanaged overflow and malformed JSON - /// return 400. Preserves body drain errors. + /// return 400. Managed JSON deserialization failures use fixed text rather than + /// serde details, which may contain payload data. Preserves body drain errors. #[inline] pub async fn json_within(&self, max: usize) -> Result where T: DeserializeOwned, { let bytes = self.buffered_bytes(ReadKind::Json, max).await?; - serde_json::from_slice(bytes.as_ref()) - .map_err(|err| EdgeError::bad_request(format!("invalid JSON payload: {err}"))) + serde_json::from_slice(bytes.as_ref()).map_err(|err| { + if self.body_policy.is_some() { + EdgeError::bad_request("invalid JSON payload") + } else { + EdgeError::bad_request(format!("invalid JSON payload: {err}")) + } + }) } /// Resolve the [`BoundKvStore`] for `id`. Strict lookup: when a @@ -909,6 +915,27 @@ mod tests { ); } + #[test] + fn managed_json_parse_errors_do_not_reflect_payload_but_unmanaged_errors_keep_detail() { + for native in [true, false] { + let payload = serde_json::to_string("PAYLOAD_SENTINEL").expect("payload"); + let body = Body::from(payload); + let selected = if native { + managed(body, 64, true) + } else { + body + }; + let context = ctx("/json", selected, PathParams::default()); + let error = block_on(context.json_within::(64)).expect_err("wrong JSON type"); + assert_eq!(error.status(), StatusCode::BAD_REQUEST); + if native { + assert_eq!(error.message(), "invalid JSON payload"); + } else { + assert!(error.message().contains("PAYLOAD_SENTINEL")); + } + } + } + fn managed(body: Body, cap: usize, is_json: bool) -> Body { body.with_buffered_json_policy(NonZeroUsize::new(cap).expect("positive cap"), is_json) } diff --git a/crates/edgezero-core/src/error.rs b/crates/edgezero-core/src/error.rs index d53c0cb3c..47ad59015 100644 --- a/crates/edgezero-core/src/error.rs +++ b/crates/edgezero-core/src/error.rs @@ -290,6 +290,67 @@ impl EdgeError { } } + pub(crate) fn into_response_with_message( + self, + fixed_message: Option<&'static str>, + ) -> Result { + let required_headers = self.required_response_headers()?; + let kind = self.kind(); + // `ConfigOutOfDate { field_path: String::new(), .. }` (the missing-blob + // path) must OMIT the `field_path` JSON key entirely, not emit + // `"field_path": ""`. Per spec 6.3.1. + let field_path_opt: Option<&str> = match &self { + EdgeError::ConfigOutOfDate { field_path, .. } if !field_path.is_empty() => { + Some(field_path.as_str()) + } + EdgeError::StoreExtraction { + field_path: Some(field_path), + .. + } => Some(field_path.as_str()), + EdgeError::PayloadTooLarge { .. } + | EdgeError::BadGateway { .. } + | EdgeError::BadRequest { .. } + | EdgeError::ConfigOutOfDate { .. } + | EdgeError::GatewayTimeout { .. } + | EdgeError::Internal { .. } + | EdgeError::MethodNotAllowed { .. } + | EdgeError::NotFound { .. } + | EdgeError::NotImplemented { .. } + | EdgeError::RequestHeaderFieldsTooLarge { .. } + | EdgeError::RequestTimeout { .. } + | EdgeError::ResponseTooLarge { .. } + | EdgeError::ServiceUnavailable { .. } + | EdgeError::StoreExtraction { .. } + | EdgeError::UriTooLong { .. } + | EdgeError::Validation { .. } => None, + }; + let status = self.status(); + let message = fixed_message.map_or_else(|| self.wire_message(), str::to_owned); + + let mut payload = json!({ + "error": { + "status": status.as_u16(), + "kind": kind, + "message": message, + }, + }); + if let Some(field_path) = field_path_opt + && let Some(error_obj) = payload + .get_mut("error") + .and_then(serde_json::Value::as_object_mut) + { + error_obj.insert("field_path".into(), json!(field_path)); + } + + let body = json_or_text(&payload); + let mut response = response_with_body(status, body)?; + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + response.headers_mut().extend(required_headers); + Ok(response) + } + /// Returns the stable public category without provider or application diagnostics. #[must_use] #[inline] @@ -572,61 +633,7 @@ impl From for EdgeError { impl IntoResponse for EdgeError { #[inline] fn into_response(self) -> Result { - let required_headers = self.required_response_headers()?; - let kind = self.kind(); - // `ConfigOutOfDate { field_path: String::new(), .. }` (the missing-blob - // path) must OMIT the `field_path` JSON key entirely, not emit - // `"field_path": ""`. Per spec 6.3.1. - let field_path_opt: Option<&str> = match &self { - EdgeError::ConfigOutOfDate { field_path, .. } if !field_path.is_empty() => { - Some(field_path.as_str()) - } - EdgeError::StoreExtraction { - field_path: Some(field_path), - .. - } => Some(field_path.as_str()), - EdgeError::PayloadTooLarge { .. } - | EdgeError::BadGateway { .. } - | EdgeError::BadRequest { .. } - | EdgeError::ConfigOutOfDate { .. } - | EdgeError::GatewayTimeout { .. } - | EdgeError::Internal { .. } - | EdgeError::MethodNotAllowed { .. } - | EdgeError::NotFound { .. } - | EdgeError::NotImplemented { .. } - | EdgeError::RequestHeaderFieldsTooLarge { .. } - | EdgeError::RequestTimeout { .. } - | EdgeError::ResponseTooLarge { .. } - | EdgeError::ServiceUnavailable { .. } - | EdgeError::StoreExtraction { .. } - | EdgeError::UriTooLong { .. } - | EdgeError::Validation { .. } => None, - }; - let status = self.status(); - let message = self.wire_message(); - - let mut payload = json!({ - "error": { - "status": status.as_u16(), - "kind": kind, - "message": message, - }, - }); - if let Some(field_path) = field_path_opt - && let Some(error_obj) = payload - .get_mut("error") - .and_then(serde_json::Value::as_object_mut) - { - error_obj.insert("field_path".into(), json!(field_path)); - } - - let body = json_or_text(&payload); - let mut response = response_with_body(status, body)?; - response - .headers_mut() - .insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); - response.headers_mut().extend(required_headers); - Ok(response) + self.into_response_with_message(None) } } diff --git a/crates/edgezero-core/src/extractor.rs b/crates/edgezero-core/src/extractor.rs index 1c2bc2308..8b529afad 100644 --- a/crates/edgezero-core/src/extractor.rs +++ b/crates/edgezero-core/src/extractor.rs @@ -17,6 +17,7 @@ use crate::config_store::{ConfigExtractionLimits, ConfigStoreError, ConfigStoreH use crate::context::{DEFAULT_INBOUND_FORM_BYTES, RequestContext}; use crate::error::{EdgeError, StoreExtractionReason}; use crate::http::HeaderMap; +use crate::ingress::CheckedEffectiveHost; use crate::secret_store::SecretError; use crate::store_registry::{ BoundConfigStore, BoundKvStore, BoundSecretStore, ConfigRegistry, ConfigStoreBinding, @@ -247,6 +248,10 @@ pub struct ForwardedHost(pub String); impl FromRequest for ForwardedHost { #[inline] async fn from_request(ctx: &RequestContext) -> Result { + if let Some(checked) = ctx.extensions().get::() { + return Ok(Self(checked.authority().unwrap_or("localhost").to_owned())); + } + // Legacy unmanaged paths retain their existing raw-header behavior. let headers = ctx.headers(); let host = headers .get("x-forwarded-host") @@ -2323,6 +2328,56 @@ mod tests { } // ForwardedHost extractor tests + #[test] + fn forwarded_host_checked_value_precedes_both_raw_headers() { + use crate::ingress::CheckedHostSource; + let mut request = request_builder() + .uri("/test") + .body(Body::empty()) + .expect("request"); + request + .headers_mut() + .insert("host", HeaderValue::from_static("direct.example")); + request.headers_mut().insert( + "x-forwarded-host", + HeaderValue::from_static("spoofed.example"), + ); + request + .extensions_mut() + .insert(CheckedEffectiveHost::from_ingress( + Some("verified.example:8443".parse().expect("authority")), + CheckedHostSource::TrustedForwarded, + )); + let ctx = RequestContext::new(request, PathParams::default()); + let host = block_on(ForwardedHost::from_request(&ctx)).expect("host"); + assert_eq!(host.0, "verified.example:8443"); + } + + #[test] + fn forwarded_host_checked_unavailable_never_falls_through_to_raw_headers() { + use crate::ingress::CheckedHostSource; + let mut request = request_builder() + .uri("/test") + .body(Body::empty()) + .expect("request"); + request + .headers_mut() + .insert("host", HeaderValue::from_static("direct.example")); + request.headers_mut().insert( + "x-forwarded-host", + HeaderValue::from_static("spoofed.example"), + ); + request + .extensions_mut() + .insert(CheckedEffectiveHost::from_ingress( + None, + CheckedHostSource::Unavailable, + )); + let ctx = RequestContext::new(request, PathParams::default()); + let host = block_on(ForwardedHost::from_request(&ctx)).expect("host"); + assert_eq!(host.0, "localhost"); + } + #[test] fn forwarded_host_extractor_uses_x_forwarded_host_first() { let mut request = request_builder() diff --git a/crates/edgezero-core/src/ingress.rs b/crates/edgezero-core/src/ingress.rs index 855825966..87998b452 100644 --- a/crates/edgezero-core/src/ingress.rs +++ b/crates/edgezero-core/src/ingress.rs @@ -9,8 +9,8 @@ use crate::body::Body; use crate::config_store::ConfigExtractionLimits; use crate::error::EdgeError; use crate::http::{ - Extensions, HeaderMap, HeaderValue, Method, Request, RequestParts, Response, StatusCode, Uri, - Version, + Authority, Extensions, HeaderMap, HeaderValue, Method, Request, RequestParts, Response, + StatusCode, Uri, Version, header::{CONTENT_LENGTH, CONTENT_TYPE, TRANSFER_ENCODING}, }; use crate::response_egress::{ResponseEgressCompletion, ResponseEgressEnvelope}; @@ -327,6 +327,72 @@ impl AdmittedIngress { } } +/// Provenance of one independently checked effective proxy field. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum CheckedHostSource { + Direct, + TrustedForwarded, + TrustedXForwarded, + Unavailable, +} + +/// Adapter-checked authority for portable host consumers. +/// This witness does not itself authorize forwarding assertions. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CheckedEffectiveHost { + authority: Option, + source: CheckedHostSource, +} + +impl CheckedEffectiveHost { + #[must_use] + #[inline] + pub fn authority(&self) -> Option<&str> { + self.authority.as_ref().map(Authority::as_str) + } + + /// Constructs a witness only after adapter-owned trust and syntax checks. + #[doc(hidden)] + #[must_use] + #[inline] + pub const fn from_ingress(authority: Option, source: CheckedHostSource) -> Self { + Self { authority, source } + } + + #[must_use] + #[inline] + pub const fn source(&self) -> CheckedHostSource { + self.source + } +} + +/// Received normalized-head sizes; never proof of raw-wire accounting. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct NormalizedIngressHeadSummary { + header_bytes: u64, + header_count: u64, + target_bytes: u64, +} + +impl NormalizedIngressHeadSummary { + #[must_use] + #[inline] + pub const fn header_bytes(self) -> u64 { + self.header_bytes + } + #[must_use] + #[inline] + pub const fn header_count(self) -> u64 { + self.header_count + } + #[must_use] + #[inline] + pub const fn target_bytes(self) -> u64 { + self.target_bytes + } +} + /// Whether request-head accounting was enforced before normalized request construction. #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[non_exhaustive] @@ -520,6 +586,7 @@ impl IngressHeadParts { #[inline] pub fn validate_normalized(&self, limits: IngressHeadLimits) -> Result<(), EdgeError> { validate_normalized_head(&self.target, self.version, &self.headers, limits) + .map(|_summary| ()) } #[must_use] @@ -834,6 +901,19 @@ pub fn validate_normalized_ingress_parts( parts: &RequestParts, limits: IngressHeadLimits, ) -> Result<(), EdgeError> { + validate_normalized_ingress_parts_with_summary(parts, limits).map(|_summary| ()) +} + +/// Validates and accounts for the received head in the same bounded pass. +/// Call before stripping assertions; returned sizes remain normalized, not raw-wire. +/// +/// # Errors +/// Returns the same head/framing errors as [`validate_normalized_ingress_parts`]. +#[inline] +pub fn validate_normalized_ingress_parts_with_summary( + parts: &RequestParts, + limits: IngressHeadLimits, +) -> Result { validate_normalized_head(&parts.uri, parts.version, &parts.headers, limits) } @@ -842,7 +922,7 @@ fn validate_normalized_head( version: Version, headers: &HeaderMap, limits: IngressHeadLimits, -) -> Result<(), EdgeError> { +) -> Result { let target_bytes = u64::try_from(target.to_string().len()).map_err(|_length_error| { EdgeError::uri_too_long("normalized request target is too large") })?; @@ -886,7 +966,12 @@ fn validate_normalized_head( } } - validate_normalized_framing(version, headers) + validate_normalized_framing(version, headers)?; + Ok(NormalizedIngressHeadSummary { + header_bytes, + header_count, + target_bytes, + }) } fn validate_normalized_framing(version: Version, headers: &HeaderMap) -> Result<(), EdgeError> { diff --git a/crates/edgezero-core/src/response_egress.rs b/crates/edgezero-core/src/response_egress.rs index f7857b722..5bbf8e2ae 100644 --- a/crates/edgezero-core/src/response_egress.rs +++ b/crates/edgezero-core/src/response_egress.rs @@ -300,7 +300,68 @@ pub struct ResponseEgressReport { pub route: Option, } -type ResponseEgressCompletionCallback = Box; +type ResponseEgressCompletionCallback = + Box; + +/// Bounded reason retained independently of an error renderer's response. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum ResponseEgressFailureKind { + FallbackBodyExceeded, + FallbackReadTimedOut, + PropagatedError, +} + +/// Framework-owned failure classification with no diagnostic source or input text. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ResponseEgressFailureClassification { + error_kind: Option<&'static str>, + framework_route_error: bool, + kind: ResponseEgressFailureKind, +} + +impl ResponseEgressFailureClassification { + pub(crate) const fn error(kind: &'static str) -> Self { + Self { + kind: ResponseEgressFailureKind::PropagatedError, + error_kind: Some(kind), + framework_route_error: false, + } + } + + #[must_use] + #[inline] + pub const fn error_kind(self) -> Option<&'static str> { + self.error_kind + } + + pub(crate) const fn fallback(kind: ResponseEgressFailureKind) -> Self { + Self { + kind, + error_kind: None, + framework_route_error: false, + } + } + + #[must_use] + #[inline] + pub const fn is_framework_route_error(self) -> bool { + self.framework_route_error + } + + #[must_use] + #[inline] + pub const fn kind(self) -> ResponseEgressFailureKind { + self.kind + } + + pub(crate) const fn routing(kind: &'static str) -> Self { + Self { + framework_route_error: true, + ..Self::error(kind) + } + } +} /// Non-clone owner of one response-scoped terminal callback. /// @@ -425,13 +486,18 @@ struct ResponseEgressResourceState { } impl ResponseEgressCompletion { + #[cfg(test)] fn complete(&mut self, report: &ResponseEgressReport) { + self.complete_at(report, report.request_start); + } + + fn complete_at(&mut self, report: &ResponseEgressReport, observed_at: MonotonicInstant) { let Some(callback) = self.callback.take() else { return; }; - let result = catch_unwind(AssertUnwindSafe(|| callback(report))); + let result = catch_unwind(AssertUnwindSafe(|| callback(report, observed_at))); if result.is_err() { - log::error!("response-egress completion panicked after terminal transition"); + log_callback_fault("response-egress completion panicked after terminal transition"); } } @@ -455,9 +521,9 @@ impl ResponseEgressCompletion { #[must_use] #[inline] pub fn join(mut self, mut other: Self) -> Self { - Self::new(move |report| { - self.complete(report); - other.complete(report); + Self::new_with_terminal_time(move |report, observed_at| { + self.complete_at(report, observed_at); + other.complete_at(report, observed_at); }) } @@ -489,6 +555,18 @@ impl ResponseEgressCompletion { pub fn new(complete: Complete) -> Self where Complete: FnOnce(&ResponseEgressReport) + Send + 'static, + { + Self::new_with_terminal_time(move |report, _observed_at| complete(report)) + } + + /// Retains a callback receiving the existing terminal transition's instant. + /// The instant is captured before any application callback or logger runs. + #[doc(hidden)] + #[must_use] + #[inline] + pub fn new_with_terminal_time(complete: Complete) -> Self + where + Complete: FnOnce(&ResponseEgressReport, MonotonicInstant) + Send + 'static, { Self { callback: Some(Box::new(complete)), @@ -527,6 +605,7 @@ pub struct ResponseEgressObserverHandle { pub struct ResponseEgressEnvelope { clock: MonotonicClock, completion: ResponseEgressCompletion, + failure_classification: Option, observer: ResponseEgressObserverHandle, policy: ResponseEgressPolicyCallback, request_method: Method, @@ -616,7 +695,7 @@ impl ResponseEgressEnvelope { let Ok(selected_policy) = catch_unwind(AssertUnwindSafe(|| (self.policy)(&head, egress_started_at))) else { - log::error!("response-egress policy panicked before conversion"); + log_callback_fault("response-egress policy panicked before conversion"); return Err(Box::new(ResponseEgressBeginFailure { attempt, cause: ResponseEgressOutcome::ConversionError, @@ -644,7 +723,7 @@ impl ResponseEgressEnvelope { )), }; let Ok(prepared) = prepare_response_egress(&self.request_method, self.response) else { - log::error!("response-egress framing validation failed before commit"); + log_callback_fault("response-egress framing validation failed before commit"); return Err(Box::new(ResponseEgressBeginFailure { attempt, cause: ResponseEgressOutcome::ConversionError, @@ -671,6 +750,13 @@ impl ResponseEgressEnvelope { ) } + /// Returns the framework-owned category captured before rendering. + #[must_use] + #[inline] + pub const fn failure_classification(&self) -> Option { + self.failure_classification + } + pub(crate) fn new( response: Response, metadata: ResponseEgressRequestMetadata, @@ -680,6 +766,7 @@ impl ResponseEgressEnvelope { clock: MonotonicClock, ) -> Self { Self { + failure_classification: None, clock, completion, observer, @@ -705,13 +792,30 @@ impl ResponseEgressEnvelope { pub fn response_mut(&mut self) -> &mut Response { &mut self.response } + + /// Composes adapter diagnostics without replacing application ownership. + #[doc(hidden)] + #[must_use] + #[inline] + pub fn with_completion(mut self, completion: ResponseEgressCompletion) -> Self { + self.completion = self.completion.join(completion); + self + } + + pub(crate) fn with_failure( + mut self, + failure: Option, + ) -> Self { + self.failure_classification = failure; + self + } } impl ResponseEgressObserverHandle { fn complete(&self, report: &ResponseEgressReport) { let result = catch_unwind(AssertUnwindSafe(|| self.observer.complete(report))); if result.is_err() { - log::error!("response-egress observer panicked after terminal transition"); + log_callback_fault("response-egress observer panicked after terminal transition"); } } @@ -927,7 +1031,7 @@ impl ResponseEgressAttempt { if let Some(elapsed) = observed_at.checked_duration_since(self.egress_started_at) { elapsed } else { - log::error!("response-egress monotonic clock moved backwards"); + log_callback_fault("response-egress monotonic clock moved backwards"); outcome = ResponseEgressOutcome::Unspecified; Duration::ZERO }; @@ -942,7 +1046,7 @@ impl ResponseEgressAttempt { }; self.state = AttemptState::Terminal(report); if let AttemptState::Terminal(terminal_report) = &self.state { - self.completion.complete(terminal_report); + self.completion.complete_at(terminal_report, observed_at); self.observer.complete(terminal_report); true } else { @@ -987,6 +1091,12 @@ pub fn default_response_egress_policy( } } +fn log_callback_fault(category: &'static str) { + // A logger panic must not escape the owner's independent unwind boundary. + // Never retry through a logger that has itself failed. + let _result = catch_unwind(AssertUnwindSafe(|| log::error!("{category}"))); +} + #[cfg(test)] mod tests { use std::cell::Cell; @@ -1787,6 +1897,158 @@ mod tests { assert_eq!(reports[0].outcome, ResponseEgressOutcome::TransportError); } + #[test] + #[expect( + clippy::too_many_lines, + reason = "one isolated facade fixture exercises all three owner fault boundaries" + )] + fn core_fault_reporting_with_panicking_facade_is_isolated() { + use log::{Level, LevelFilter, Log, Metadata, Record, set_logger, set_max_level}; + use std::env::{current_exe, var_os}; + use std::process::Command; + + struct PanickingLogger; + static LOGGER: PanickingLogger = PanickingLogger; + static CALLS: AtomicUsize = AtomicUsize::new(0); + impl Log for PanickingLogger { + fn enabled(&self, metadata: &Metadata<'_>) -> bool { + metadata.level() == Level::Error + } + fn flush(&self) {} + fn log(&self, record: &Record<'_>) { + if self.enabled(record.metadata()) { + CALLS.fetch_add(1, Ordering::SeqCst); + panic!("fixture logger fault"); + } + } + } + const CHILD: &str = "EDGEZERO_TEST_CORE_FAULT_LOGGER"; + if var_os(CHILD).is_none() { + let output = Command::new(current_exe().expect("test executable")) + .args(["--exact", "response_egress::tests::core_fault_reporting_with_panicking_facade_is_isolated", "--nocapture"]) + .env(CHILD, "1").output().expect("isolated child"); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + return; + } + set_logger(&LOGGER).expect("single child logger"); + set_max_level(LevelFilter::Error); + let started = MonotonicInstant::now(); + let terminal = started + .checked_add(Duration::from_millis(42)) + .expect("instant"); + let after_callback = terminal + .checked_add(Duration::from_secs(3)) + .expect("instant"); + let current = Arc::new(Mutex::new(terminal)); + let clock_current = Arc::clone(¤t); + let clock = MonotonicClock::new(move || *clock_current.lock().expect("clock")); + let events = Arc::new(Mutex::new(Vec::new())); + let app_events = Arc::clone(&events); + let native_events = Arc::clone(&events); + let observed = Arc::new(Mutex::new(None)); + let native_observed = Arc::clone(&observed); + let callback_current = Arc::clone(¤t); + let completion = ResponseEgressCompletion::new(move |_| { + app_events.lock().expect("events").push("app"); + *callback_current.lock().expect("clock") = after_callback; + panic!("application completion fault"); + }) + .join(ResponseEgressCompletion::new_with_terminal_time( + move |_, at| { + native_events.lock().expect("events").push("native"); + *native_observed.lock().expect("observed") = Some(at); + }, + )); + let headers = HeaderMap::new(); + let head = ResponseEgressHead::new( + StatusCode::OK, + Version::HTTP_11, + &headers, + None, + started, + None, + ); + let mut attempt = ResponseEgressAttempt::new( + &head, + started, + completion, + ResponseEgressObserverHandle::new(OrderedObserver(Arc::clone(&events))), + clock.clone(), + ); + assert!(attempt.terminate(ResponseEgressOutcome::Completed, terminal)); + assert!(!attempt.terminate(ResponseEgressOutcome::SourceError, after_callback)); + assert_eq!( + *events.lock().expect("events"), + vec!["app", "native", "observer"] + ); + assert_eq!(*observed.lock().expect("observed"), Some(terminal)); + assert_eq!(clock.now(), after_callback); + assert_eq!(CALLS.load(Ordering::SeqCst), 1); + + let mut observer_attempt = ResponseEgressAttempt::new( + &head, + started, + ResponseEgressCompletion::empty(), + ResponseEgressObserverHandle::new(PanickingObserver), + clock.clone(), + ); + assert!(observer_attempt.terminate(ResponseEgressOutcome::Completed, terminal)); + assert!(!observer_attempt.terminate(ResponseEgressOutcome::TransportError, after_callback)); + assert_eq!(CALLS.load(Ordering::SeqCst), 2); + + let policy_calls = Arc::new(Mutex::new(Vec::new())); + let callback_calls = Arc::clone(&policy_calls); + let envelope = ResponseEgressEnvelope::new( + Response::new(Body::empty()), + ResponseEgressRequestMetadata::new(Method::GET, started, None), + ResponseEgressCompletion::new_with_terminal_time(move |_, at| { + callback_calls.lock().expect("calls").push(at); + }), + Arc::new(|_, _| panic!("application policy fault")), + ResponseEgressObserverHandle::default(), + clock, + ); + let failure = envelope.begin().err().expect("bounded policy failure"); + assert!(policy_calls.lock().expect("calls").is_empty()); + let (mut policy_attempt, _clock, cause) = failure.into_parts(); + assert_eq!(cause, ResponseEgressOutcome::ConversionError); + assert!(policy_attempt.terminate(cause, after_callback)); + assert_eq!(*policy_calls.lock().expect("calls"), vec![after_callback]); + assert_eq!(CALLS.load(Ordering::SeqCst), 3); + } + + #[test] + fn timed_nested_completion_preserves_terminal_instant_and_order() { + let events = Arc::new(Mutex::new(Vec::<(u8, MonotonicInstant)>::new())); + let started = MonotonicInstant::now(); + let terminal = started + .checked_add(Duration::from_millis(42)) + .expect("instant"); + let first = Arc::clone(&events); + let second = Arc::clone(&events); + let third = Arc::clone(&events); + let mut completion = ResponseEgressCompletion::new(move |_| { + first.lock().expect("events").push((1, terminal)); + }) + .join(ResponseEgressCompletion::new_with_terminal_time( + move |_, at| second.lock().expect("events").push((2, at)), + )) + .join(ResponseEgressCompletion::new_with_terminal_time( + move |_, at| third.lock().expect("events").push((3, at)), + )); + let report = completed_report(started); + completion.complete_at(&report, terminal); + completion.complete_at(&report, started); + assert_eq!( + *events.lock().expect("events"), + vec![(1, terminal), (2, terminal), (3, terminal)] + ); + } + #[test] fn zero_byte_completion_has_zero_accounting() { let completed_observer = RecordingObserver::default(); diff --git a/crates/edgezero-core/src/router.rs b/crates/edgezero-core/src/router.rs index 1914e5684..bdcc98ef4 100644 --- a/crates/edgezero-core/src/router.rs +++ b/crates/edgezero-core/src/router.rs @@ -14,6 +14,30 @@ use crate::introspection::{ManifestJson, RouteTable}; use crate::middleware::{BoxMiddleware, Middleware, Next}; use crate::params::PathParams; use crate::response::IntoResponse as _; +use crate::response_egress::{ResponseEgressFailureClassification, ResponseEgressFailureKind}; + +pub(crate) enum RouteErrorKind { + MethodNotAllowed, + NotFound, +} + +pub(crate) struct DispatchOutcome { + failure: Option, + response: Result, + route_error: Option, +} + +impl DispatchOutcome { + pub(crate) fn into_parts( + self, + ) -> ( + Result, + Option, + Option, + ) { + (self.response, self.route_error, self.failure) + } +} struct RouteEntry { handler: BoxHandler, @@ -488,45 +512,73 @@ impl RouterService { request: Request, ingress: AdmittedIngress, ) -> Result { - if !Arc::ptr_eq(&self.inner, &resolved.owner) { - return Err(EdgeError::internal(anyhow::anyhow!( - "resolved dispatch token belongs to another router" - ))); - } - if request.method() != resolved.method || request.uri().path() != resolved.path { - return Err(EdgeError::internal(anyhow::anyhow!( - "request method or path changed after ingress route resolution" - ))); - } + self.dispatch_resolved_outcome(resolved, request, ingress) + .await + .response + } - match resolved.target { - ResolvedTarget::Found(entry, params) => { - if ingress.is_fallback() { - return Err(EdgeError::internal(anyhow::anyhow!( - "fallback body policy cannot dispatch a matched route" - ))); - } - self.inner - .dispatch_found(request, &entry, params, Some(ingress)) - .await + pub(crate) async fn dispatch_resolved_outcome( + &self, + resolved: ResolvedDispatch, + request: Request, + ingress: AdmittedIngress, + ) -> DispatchOutcome { + let mut route_error = None; + let mut failure = None; + let response = async { + if !Arc::ptr_eq(&self.inner, &resolved.owner) { + return Err(EdgeError::internal(anyhow::anyhow!( + "resolved dispatch token belongs to another router" + ))); } - ResolvedTarget::MethodNotAllowed(allowed) => { - if let Some(response) = fallback_terminal_response(request, ingress).await? { - return Ok(response); - } - let methods = allowed - .iter() - .map(|metadata| metadata.method().clone()) - .collect::>(); - Err(EdgeError::method_not_allowed(&resolved.method, &methods)) + if request.method() != resolved.method || request.uri().path() != resolved.path { + return Err(EdgeError::internal(anyhow::anyhow!( + "request method or path changed after ingress route resolution" + ))); } - ResolvedTarget::NotFound => { - if let Some(response) = fallback_terminal_response(request, ingress).await? { - return Ok(response); + match resolved.target { + ResolvedTarget::Found(entry, params) => { + if ingress.is_fallback() { + return Err(EdgeError::internal(anyhow::anyhow!( + "fallback body policy cannot dispatch a matched route" + ))); + } + self.inner + .dispatch_found(request, &entry, params, Some(ingress)) + .await + } + ResolvedTarget::MethodNotAllowed(allowed) => { + if let Some((response, category)) = + fallback_terminal_response(request, ingress).await? + { + failure = Some(category); + return Ok(response); + } + let methods = allowed + .iter() + .map(|metadata| metadata.method().clone()) + .collect::>(); + route_error = Some(RouteErrorKind::MethodNotAllowed); + Err(EdgeError::method_not_allowed(&resolved.method, &methods)) + } + ResolvedTarget::NotFound => { + if let Some((response, category)) = + fallback_terminal_response(request, ingress).await? + { + failure = Some(category); + return Ok(response); + } + route_error = Some(RouteErrorKind::NotFound); + Err(EdgeError::not_found(resolved.path)) } - Err(EdgeError::not_found(resolved.path)) } } + .await; + DispatchOutcome { + failure, + response, + route_error, + } } pub(crate) fn insert_state(&mut self, value: T) @@ -605,7 +657,7 @@ impl RouterService { async fn fallback_terminal_response( request: Request, ingress: AdmittedIngress, -) -> Result, EdgeError> { +) -> Result, EdgeError> { let Some(fallback) = ingress.into_fallback() else { return Ok(None); }; @@ -622,8 +674,18 @@ async fn fallback_terminal_response( match outcome? { FallbackDrainOutcome::Complete => Ok(None), - FallbackDrainOutcome::Exceeded => Ok(Some(on_exceeded.into_response())), - FallbackDrainOutcome::TimedOut => Ok(Some(on_timeout.into_response())), + FallbackDrainOutcome::Exceeded => Ok(Some(( + on_exceeded.into_response(), + ResponseEgressFailureClassification::fallback( + ResponseEgressFailureKind::FallbackBodyExceeded, + ), + ))), + FallbackDrainOutcome::TimedOut => Ok(Some(( + on_timeout.into_response(), + ResponseEgressFailureClassification::fallback( + ResponseEgressFailureKind::FallbackReadTimedOut, + ), + ))), } } diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 46f21e821..172ebc1b2 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -50,6 +50,10 @@ export default defineConfig({ { text: 'Cloudflare Workers', link: '/guide/adapters/cloudflare' }, { text: 'Fermyon Spin', link: '/guide/adapters/spin' }, { text: 'Axum (Native)', link: '/guide/adapters/axum' }, + { + text: 'Axum diagnostics and trusted proxies', + link: '/guide/axum-diagnostics', + }, ], }, { diff --git a/docs/guide/adapters/axum.md b/docs/guide/adapters/axum.md index c85fbdefb..330162ed8 100644 --- a/docs/guide/adapters/axum.md +++ b/docs/guide/adapters/axum.md @@ -35,9 +35,15 @@ fn main() -> anyhow::Result<()> { } ``` -`run_app` installs `simple_logger` (unless `owns_logging = true`), builds the app, and reads bind / -store / logging config at runtime from `EDGEZERO__*` environment -variables (see [the migration guide](../manifest-store-migration.md)). +`run_app` installs `simple_logger` unless `owns_logging = true`, builds the app, and reads bind, +store, and logging config at runtime from `EDGEZERO__*` environment +variables. See [the migration guide](../manifest-store-migration.md). + +Incoming proxy trust defaults to no trusted networks. Managed hosting strips raw forwarding +headers before admission and exposes checked ingress metadata instead. See +[Axum diagnostics and trusted proxies](/guide/axum-diagnostics) for configuration, migration, +request records, and work snapshots. + The portable store metadata baked into `App` by the `app!` macro drives which logical stores are exposed; no `edgezero.toml` needs to be loaded by the runtime. @@ -117,6 +123,11 @@ JSON helpers enforce it regardless of Content-Type; generic buffered helpers enf original `application/json` or `application/*+json` ingress classification. Ordinary Form retains its 1 MiB caller cap. Header/extension mutation and context reconstruction cannot loosen policy. +Native-managed JSON deserialization failures return HTTP 400 with the fixed text +`invalid JSON payload`. Custom renderers receive that same fixed error, not serde detail. +Legacy/unmanaged JSON errors and application-authored `EdgeError` messages retain their +existing behavior. See [privacy limits](/guide/axum-diagnostics#timing-failures-and-privacy-limits). + Taken streams and proxy forwarding receive no new aggregate size cap. Manual body collection keeps its supplied cap and existing 400 overflow behavior. The ceiling bounds logical buffered body bytes, not parsed JSON allocations, host buffers, concurrent requests, or process memory. @@ -138,17 +149,26 @@ See [Capabilities](/guide/capabilities). ## Logging -The Axum adapter's `run_app` helper installs `simple_logger` at the level read from -`EDGEZERO__LOGGING__LEVEL`, falling back to `info` when the variable is unset or -unparseable. It does not read `edgezero.toml`, and `echo_stdout` has no effect on -the runtime. To install a different logger, set `owns_logging = true` on your `app!` -declaration so `run_app` skips its own logger, then install yours in `main`. Wiring -`edgezero_core::app::App::build::(edgezero_adapter_axum::AXUM_PLATFORM)` and `AxumDevServer` by hand remains the fallback if you also need -to control the bind address or store setup. +The development `run_app` helper installs `simple_logger` at the level read from +`EDGEZERO__LOGGING__LEVEL`, falling back to `info` when that variable is unset or unparseable. +Production `AxumRunOptions` rejects an invalid supplied level. Neither reads `edgezero.toml` +for logger setup, and `echo_stdout` has no runtime effect. -::: tip Logging status -`run_app` wires logging automatically; custom entrypoints should install a logger explicitly. -::: +To install your own logger, set `owns_logging = true` in the `app!` declaration and initialize +it once in `main` before entering the runner. Ownership controls installation, not framework +facade output. Safe native request and lifecycle records use target `edgezero::native`. + +Request records are on by default. `EDGEZERO__LOGGING__REQUEST_RECORDS` accepts exactly +`true` or `false`; invalid supplied values fail startup in development and production. +`AxumRunOptions::with_request_records(false)` and `AxumDevServer::with_request_records(false)` +suppress only default native request logs. Application/native observers, lifecycle records, +counts, and snapshots remain active. Explicit builders do not merge ambient record settings. + +Use `with_diagnostics` to attach a `NativeDiagnosticsHandle` for snapshots or an optional +native request observer. No exporter or public metrics endpoint is installed. See +[logger ownership](/guide/axum-diagnostics#records-and-logger-ownership) and +[snapshots and metrics](/guide/axum-diagnostics#snapshots-and-optional-metrics) for examples +and the synchronous sink limitations. ## Testing @@ -317,6 +337,7 @@ While Axum provides a convenient development environment, always test on actual ## Next Steps +- Configure [native diagnostics and trusted proxies](/guide/axum-diagnostics) - Deploy to [Fastly Compute](/guide/adapters/fastly) for production - Deploy to [Cloudflare Workers](/guide/adapters/cloudflare) as an alternative - Explore [Configuration](/guide/configuration) for manifest options diff --git a/docs/guide/axum-diagnostics.md b/docs/guide/axum-diagnostics.md new file mode 100644 index 000000000..b0aa64729 --- /dev/null +++ b/docs/guide/axum-diagnostics.md @@ -0,0 +1,443 @@ +# Axum diagnostics and trusted proxies + +The managed native host exposes checked ingress metadata, safe request records, and work +snapshots. Incoming proxy trust is off by default. A forwarding header never establishes +trust by itself. + +These APIs belong to `edgezero_adapter_axum`, with the `axum` feature enabled. They apply to +`run_app`, `run_production_app`, the explicit-options runners, and `AxumDevServer`. Other +adapters retain their existing behavior. + +## Startup settings + +| Setting | Default | Accepted values | +| --------------------------------------------- | ------------------- | ---------------------------------------------------------------------- | +| `EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS` | No trusted networks | Comma-separated literal IPv4/IPv6 addresses or CIDRs | +| `EDGEZERO__ADAPTER__FORWARDING_HEADER_FAMILY` | No selected family | `forwarded` or `x-forwarded`, required when the trust list is nonempty | +| `EDGEZERO__LOGGING__REQUEST_RECORDS` | `true` | Exactly `true` or `false` | + +For example, trust one deployment-owned proxy address and use RFC Forwarded: + +```bash +EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS=10.20.0.8/32 \ +EDGEZERO__ADAPTER__FORWARDING_HEADER_FAMILY=forwarded \ +EDGEZERO__LOGGING__REQUEST_RECORDS=true \ +./my-app-adapter-axum +``` + +An unset trust list, an empty string, or a string containing only spaces/tabs means no trust. +The network parser trims spaces/tabs around entries. It rejects empty entries inside a list, +other control characters, invalid literals, DNS names, and tokens such as `private`, `all`, or +provider names. Family names are lowercase, with surrounding spaces/tabs allowed. Unknown +supplied families fail even with an empty trust list. A valid family alone enables nothing. +Request-record booleans accept no case changes or whitespace. + +Invalid supplied values, including non-Unicode environment values for these settings, fail +startup before readiness. They do not silently revert to defaults. Both development and +production validate these three settings strictly. Production also validates its other hosting +settings; this does not change unrelated development-setting fallback behavior. + +Network entries are truncated to their prefix and deduplicated. IPv4-mapped IPv6 peers and +assertions match their IPv4 equivalents. Mapped networks within `::ffff:0:0/96` become equivalent +IPv4 networks. `/0`, unions covering an entire address family, and IPv6 prefixes that would +trust the entire mapped IPv4 range are rejected. Private, loopback, and container networks +receive no implicit trust. EdgeZero does not discover provider ranges or refresh them. + +The trust configuration has a 16,384-byte text budget, including separators, and at most 256 +supplied entries. Duplicates count before normalization. Explicit policy iterators have the +same bounds and never silently truncate. + +### Explicit hosting + +```rust +use edgezero_adapter_axum::dev_server::AxumRunOptions; +use edgezero_adapter_axum::diagnostics::NativeDiagnosticsHandle; +use edgezero_adapter_axum::proxy::{ForwardingHeaderFamily, TrustedProxyPolicy}; + +let diagnostics = NativeDiagnosticsHandle::new(); +let policy = TrustedProxyPolicy::new( + ForwardingHeaderFamily::Forwarded, + ["10.20.0.8/32"], +)?; +let options = AxumRunOptions::new("127.0.0.1:8787".parse()?)? + .with_trusted_proxy_policy(policy) + .with_diagnostics(diagnostics.clone()) + .with_request_records(false); +``` + +Pass these options and a caller-owned stop future to +`dev_server::run_app_with_options::`. Use +`run_app_with_options_and_initializer` when startup needs application checks. The explicit +runners install no process-wide signal handlers. The corresponding `AxumDevServer` builders +are `with_trusted_proxy_policy`, `with_diagnostics`, and `with_request_records`. + +Explicit options/builders do not merge ambient proxy or request-record settings. Use +`AxumRunOptions::from_env()` to capture environment-driven settings deliberately, or +`from_vars` for supplied string pairs. Each run captures its settings once. +`TrustedProxyPolicy::no_trust()` is the explicit no-trust policy. + +## What makes a forwarding assertion trustworthy + +Protect the proxy-to-application network first. Only deployment-owned proxies should be able +to connect from addresses in the allowlist. A shared private subnet or loopback address is not +proof that every process using it is trustworthy. Trust is neither user authentication nor +permission to use any asserted host as an outbound destination. + +The service matches the socket-owned peer, not `Forwarded by=`, a header, URI, or +caller-supplied `ConnectInfo`. Without a matching peer, it does not parse forwarding assertions. +It uses direct facts instead. + +```mermaid +flowchart TD + A[Owned socket peer and received head] --> B[Validate and account original normalized head] + B --> C[Match explicit proxy policy] + C --> D[Validate selected family within bounds] + D --> E[Compute checked facts and provenance] + E --> F[Remove both raw forwarding families] + F --> G[Admission policy] + G --> H[Middleware and handlers] +``` + +Select exactly one family. `forwarded` reads ordered RFC 7239 elements, including repeated +field lines and quoted IPv6 nodes. `x-forwarded` reads an ordered X-Forwarded-For chain and +singleton X-Forwarded-Host and X-Forwarded-Proto values. Repeated or comma-listed host/proto +values are ambiguous and invalidate that family. The unselected family never fills a gap. + +All recognized assertions are validated before choosing a client boundary, including entries +that the eventual trust walk will ignore. The service walks predecessors right to left while +the current hop is trusted. The first untrusted numeric predecessor is the effective client +boundary. It never selects the leftmost value unconditionally or skips an unknown hop. + +| `ForwardingResult` | Meaning | +| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------- | +| `Absent` | No usable selected assertions | +| `UntrustedPeer` | Forwarding fields are present but the direct peer cannot supply trusted assertions | +| `Accepted` | Client, host, and scheme assertions were accepted | +| `Partial` | Some fields were accepted; missing fields use direct facts | +| `UnresolvedChain` | A valid chain has a missing/unknown/obfuscated predecessor or exhausts only trusted addresses; client falls back to the direct peer | +| `Malformed` | Invalid syntax, recognized values, ambiguity, or bounds invalidate the whole selected assertion | + +An unresolved client chain can still supply valid final-proxy host and scheme assertions. +Malformed client text cannot. With malformed optional forwarding metadata, an otherwise valid +request continues using direct facts for every selected field. Existing HTTP head/framing +violations still follow their normal rejection or abort path. + +The selected assertion has a 16,384-byte budget, counting conceptual separators between +repeated field values, and at most 64 chain/list slots. Forwarded allows at most 16 parameter +slots per element. Empty slots count against bounds. RFC Forwarded tolerates empty list +members and empty semicolon slots. X-Forwarded-For does not: a present-empty value or empty +member invalidates the complete selected family, including otherwise-valid host/proto. +Absent X-Forwarded-For still permits individual host/proto acceptance. + +These bounds cover additional forwarding parsing and retained metadata. They do not cap +Hyper's initial receive allocation, total process memory, or concurrent work. The original +normalized head is validated and counted before stripping, so removed headers cannot evade +head limits. That accounting is normalized head size, not raw wire bytes. + +### Host and scheme belong to the final proxy + +For Forwarded, only the final, rightmost element can supply host/proto. For X-Forwarded-*, only +the singleton host/proto fields can supply them. Earlier host/proto values and the client-address +chain cannot establish the public origin. + +The final trusted ingress must overwrite these fields with values it verified. If another proxy +terminated public TLS, verify that upstream assertion or enforce a known HTTPS-only public +route. The final hop's own connection scheme is not necessarily the visitor's scheme. + +Hosts must be valid nonempty authorities with an optional numeric port, without userinfo, +whitespace, path, query, or fragment. Schemes normalize only `http` and `https`. There is no DNS +or IDNA lookup. Missing fields fall back independently. Direct host comes from the +HTTP-authoritative request head; unavailable authority stays unavailable. The native direct +transport scheme is HTTP, even when an absolute request URI says HTTPS. + +## Nginx terminating public TLS + +This example replaces the chain for a single public ingress. Put the `map` in the existing +Nginx `http` block and add the server alongside your TLS certificate configuration. The map +quotes and brackets IPv6 addresses for Forwarded; IPv4 remains unquoted. + +```nginx +map $remote_addr $edgezero_forwarded_for { + default $remote_addr; + ~: '"[$remote_addr]"'; +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + server_name app.example.com; + # Configure ssl_certificate and ssl_certificate_key here. + + location / { + proxy_pass http://127.0.0.1:8787; + proxy_set_header Host app.example.com; + proxy_set_header Forwarded "for=$edgezero_forwarded_for;host=app.example.com;proto=https"; + proxy_set_header X-Forwarded-For ""; + proxy_set_header X-Forwarded-Host ""; + proxy_set_header X-Forwarded-Proto ""; + } +} +``` + +Configure EdgeZero with `TRUSTED_PROXY_CIDRS=127.0.0.1/32` and +`FORWARDING_HEADER_FAMILY=forwarded`, using the full environment prefixes above. Bind its +listener to loopback and keep untrusted local processes away from this trusted connection. +If Nginx uses IPv6 for the backend hop, configure that actual peer instead. + +The snippet assumes `$remote_addr` is the actual incoming peer, with no header-driven real-IP +rewrite. The HTTPS-only server establishes `proto=https`. It supplies a fixed public host, +not `$http_host`, and replaces visitor-supplied Forwarded values rather than appending them. +Configure your default virtual host to reject unexpected public hosts and handle HTTP +redirects separately. Test both IPv4 and IPv6 visitors in the deployed configuration. +See Nginx's [map](https://nginx.org/en/docs/http/ngx_http_map_module.html) and +[proxy header](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_set_header) +documentation for directive scope and inheritance. + +## Cloudflare plus a load balancer + +For `visitor → Cloudflare → load balancer → EdgeZero`, configure the final load balancer to +append its actual incoming Cloudflare peer to the verified predecessor chain. It must replace +X-Forwarded-Host with the allowed public authority and X-Forwarded-Proto with the verified +visitor scheme. Do not merely pass through a visitor-controlled host/proto field. + +For illustration, suppose the final peer is `10.20.0.8` and the Cloudflare peer is +`198.51.100.10`. These are example addresses, not provider ranges: + +```http +X-Forwarded-For: 203.0.113.7, 198.51.100.10 +X-Forwarded-Host: app.example.com +X-Forwarded-Proto: https +``` + +Select `x-forwarded` and explicitly trust the actual load-balancer addresses and relevant +Cloudflare ranges for your deployment. The walk can then stop at `203.0.113.7`. If only the +load balancer is trusted, it stops at the Cloudflare address. If every represented address is +trusted, it falls back to the direct peer as unresolved. + +Restrict origin access to the intended Cloudflare producers, including verification appropriate +to your zone and route. Restrict EdgeZero access to the final load balancer. Provider defaults, +address membership alone, or an HTTPS load-balancer hop do not prove correct visitor metadata. +Maintain the allowlist and inspect your deployed header mode. EdgeZero has no built-in +Cloudflare/AWS trust policy and does not consume `CF-Connecting-IP` automatically. +Cloudflare's [header documentation](https://developers.cloudflare.com/fundamentals/reference/http-request-headers/) +explains how Workers, transforms, and Pseudo IPv4 can change assertions. + +### A gateway can replace the chain + +If the load balancer cannot produce verified singleton host/proto fields, place a sanitizing +gateway between it and EdgeZero. The gateway must validate its upstream producer and chain, +reject or discard unverified fields, and replace the selected family entirely. For a verified +visitor address, it can emit: + +```http +Forwarded: for=203.0.113.7;host=app.example.com;proto=https +``` + +Trust only that gateway's actual outgoing peer and select `forwarded`. Emit HTTPS only after +verifying the public TLS assertion or enforcing the HTTPS-only public route. If the gateway +cannot establish the client boundary, `for=unknown` is honest; EdgeZero then uses direct client +fallback while retaining valid host/proto. Renaming untouched client input to a new header +does not verify it. + +## Migrate applications from raw headers + +Before admission, the managed host removes Forwarded and every case-insensitive +X-Forwarded-* header, including accepted fields and extensions. It overwrites incoming checked +metadata extensions with its own results. Host, URI, and +`context::AxumRequestContext::remote_addr` remain unchanged; `remote_addr` is always the direct +peer, not the effective client. + +At admission, read `IngressHead::extension::()`. In middleware or a +handler, read the same facts from the request context: + +```rust +use edgezero_adapter_axum::diagnostics::NativeIngressMetadata; + +if let Some(metadata) = ctx.extensions().get::() { + let authority = metadata.effective_host().authority(); + let scheme = metadata.effective_scheme().as_str(); + let client_boundary = metadata.effective_client(); + let source = metadata.host_source(); + // Apply application host authorization before using the public origin. +} +``` + +`request_id`, `direct_peer`, `client_source`, `scheme_source`, `forwarding_result`, and +`received_normalized_head` are also read-only accessors. Per-field sources use +`edgezero_core::ingress::CheckedHostSource`: `Direct`, `TrustedForwarded`, +`TrustedXForwarded`, or `Unavailable`. Read each source independently from the aggregate +forwarding result. Metadata describes original ingress, not later application mutations. + +`edgezero_core::ForwardedHost` now reads `CheckedEffectiveHost` first on the managed native +path, including accepted RFC Forwarded host values. Checked-unavailable uses its existing +`localhost` fallback without consulting raw forwarding fields. Prefer the metadata accessor +when you must distinguish absence from that convenience fallback. Without checked metadata, +other adapters and legacy conversions retain the extractor's old behavior. + +`request::into_core_request` is a low-level bypass. It does not perform managed admission or +socket-owned proxy checking, and raw `ForwardedHost` behavior can remain there. Supplying +ConnectInfo or setting an environment variable does not turn that converter into a trusted +ingress. Use managed hosting for this policy. The converter still attaches the native JSON +buffering policy, so its framework-generated JSON parse errors also use fixed text; the bypass +is for managed admission and proxy checking. Do not recreate removed headers for automatic +outbound propagation; outbound assertions need their own explicit authorization decision. + +## Records and logger ownership + +Default facade records use target `edgezero::native`. `request_terminal` describes begun +egress attempts; `request_ingress` describes no-response abort, draining, or abandonment. +`lifecycle` describes runner transitions and fixed failure categories. + +Request records contain a generated ID, known method or `other`, registered route template or +`unknown`, optional selected status and total duration, typed outcome/failure/forwarding +categories, and optional egress bytes/body/fallback accounting. Default framework request +records do not contain addresses, hosts, raw paths/queries, headers, payloads, config/secret values, +origin URLs, or source error strings. Route templates are application-owned names; do not +put secrets in them. Default output escapes route text to one line. + +Severity follows typed outcomes, not status alone. Ordinary completion, framework route misses, +and expected draining use INFO. Refusal, propagated errors, malformed forwarding, and +failed/abandoned attempts use WARN. Startup/runtime faults and unusable clock/identity state +use ERROR. An application-selected 503 is not automatically a framework failure. + +`Hooks::owns_logging()` controls installation only. Set `owns_logging = true` in the `app!` +declaration and install your logger once before entering the runner. For example, in your +Axum entrypoint with your own logger dependency: + +```rust +use edgezero_adapter_axum::dev_server::run_production_app; +use my_app_core::App; + +fn main() -> anyhow::Result<()> { + // App declares owns_logging = true. + simple_logger::SimpleLogger::new() + .with_level(log::LevelFilter::Info) + .init()?; + run_production_app::() +} +``` + +Keep the `edgezero::native` target enabled in your logger's filters when you want these records. +No competing logger/subscriber is installed when the application owns logging. + +`with_request_records(false)`, or `EDGEZERO__LOGGING__REQUEST_RECORDS=false` on an +environment-driven runner, suppresses default request-terminal and request-ingress logs only. +Native and application observers, lifecycle events, counts, and snapshots remain active. + +### Request identity + +`RequestId` displays 64 hexadecimal characters derived from a process nonce, hosting sequence, +and request sequence. IDs do not repeat within the process; separation between processes is +entropy-based, not a mathematical global uniqueness guarantee. Entropy failure prevents +startup; sequence exhaustion never wraps or reuses an ID. + +Incoming IDs are not adopted. There is no automatic response ID header, tracing-header +propagation, or authentication meaning. Applications can expose an ID deliberately. +Changing the public metadata extension cannot change the framework's private correlation ID. +Never use request IDs as metric labels. + +## Snapshots and optional metrics + +Retain a clone of the handle passed to `with_diagnostics` and sample it from application-owned +management code: + +```rust +let snapshot = diagnostics.snapshot(); +let active_requests = snapshot.active_requests; +let active_connections = snapshot.active_connections; +let idle_connections = snapshot.idle_connections; +``` + +Active requests include managed admission, body read, handlers, and egress until completion or +abandonment releases the guard. Active connections count owned connection tasks, including +idle keep-alive tasks; idle connections are their idle subset. Detached application work, +kernel buffers, and external services are not counted. + +A snapshot coherently reads the three counters at one moment. It is not atomic with lifecycle +phase or subsequent work. A request used to query counts counts itself while active, and its +connection also contributes. Sample externally or account for the query before asserting zero. +Accounting releases before native logger/observer delivery. + +Each runner creates its own handle by default. Explicitly sharing one handle aggregates its +counters across runners; clones of an observer-equipped handle also receive their records. +Each runner still has its own identity namespace and lifecycle. Attach the observer before +cloning the handle for runners: `with_request_observer` configures the returned handle, not +previous clones. + +Implement `NativeRequestObserver` for application metrics. This small example keeps fixed +counters and cumulative duration without per-request labels: + +```rust +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use edgezero_adapter_axum::diagnostics::{ + NativeDiagnosticsHandle, NativeRequestObserver, NativeRequestRecord, +}; + +#[derive(Default)] +struct Totals { + observed: u64, + timed: u64, + duration: Duration, +} + +struct Metrics(Arc>); + +impl NativeRequestObserver for Metrics { + fn observe(&self, record: &NativeRequestRecord) { + if let Ok(mut totals) = self.0.lock() { + totals.observed = totals.observed.saturating_add(1); + if let Some(duration) = record.duration { + totals.timed = totals.timed.saturating_add(1); + totals.duration = totals.duration.saturating_add(duration); + } + } + } +} + +let totals = Arc::new(Mutex::new(Totals::default())); +let diagnostics = NativeDiagnosticsHandle::new() + .with_request_observer(Metrics(Arc::clone(&totals))); +``` + +Use your metrics library for histograms and bounded method, registered-route, status-class, or +outcome labels. Handle missing status/duration and future enum variants. Never label metrics +with IDs, addresses, raw URLs, payloads, or secrets. The native observer adds no exporter, +queue, metrics server, or public diagnostics endpoint and does not replace application egress +observers/completions. Do not capture an observer-bearing handle in its own observer and +create an ownership cycle. + +## Timing, failures, and privacy limits + +Native total duration runs from the parsed service boundary to the existing authoritative +terminal observation on the application monotonic clock. It includes pre-egress work but +excludes subsequent completion-callback and logger delay. `ResponseEgressReport::elapsed` +remains egress-only. A backwards clock produces absent duration and a safe clock-fault +category, not a wall-clock substitute. Optional application timing collectors do not replace +this clock or completion owner. + +Selected status is not proof that headers reached the wire or that the client received them. +A selected 200 may end with `SourceError`, `DeadlineExceeded`, `ClientDisconnected`, or +`TransportError`. A precommit fallback can select 500/504; a post-commit failure does not send +a second error response. `bytes_written` counts egress-accounted bytes, not delivered TCP +bytes. No-response paths have no invented status or egress report. + +On the managed native path, default framework routing 404/405 errors use fixed public text. +Custom renderers retain the original routing errors. Native-managed JSON deserialization +errors return HTTP 400 with the fixed message `invalid JSON payload`, without serde's value, +line, or type detail. Custom renderers also receive this fixed native-generated JSON error; +the serde detail is already discarded. Legacy/unmanaged JSON errors retain their existing +detail. Application-authored `EdgeError` messages and responses are not globally sanitized, +and custom renderers remain responsible for their own output. JSON size overflow still uses +the selected native limit and HTTP 413; it is not a deserialization failure. + +Loggers and observers run synchronously. They can block body progress or shutdown, so keep +them short. Unwind-capable targets isolate diagnostic sink panics independently and release +native accounting, but panic-abort, SIGKILL, and process failure have no completion or log-flush +promise. There is no retry, durable delivery guarantee, or added shutdown flush budget. + +Deployment acceptance remains downstream. Parser tests and local socket/TLS fixtures do not +prove final-image behavior, proxy configuration, client receipt, signal delivery, or bounded +sink latency. Recheck the accepted prerequisite stack and test the actual image and ingress +before relying on these deployment claims. Packaging and final-image acceptance are separate +from this guide's API contract. diff --git a/docs/superpowers/plans/2026-10-06-axum-diagnostics-trusted-proxy.md b/docs/superpowers/plans/2026-10-06-axum-diagnostics-trusted-proxy.md new file mode 100644 index 000000000..b283763a2 --- /dev/null +++ b/docs/superpowers/plans/2026-10-06-axum-diagnostics-trusted-proxy.md @@ -0,0 +1,457 @@ +# Axum diagnostics and trusted incoming proxies implementation plan + +Date: 2026-10-06 +Status: local implementation and integration verification complete after plan approval, an authorized open-stack exception, and the approved native JSON privacy extension. Draft stacked PR preparation is authorized. Accepted-stack and final-image acceptance remain deferred. + +Issue: [#397](https://github.com/stackpop/edgezero/issues/397), under [#391](https://github.com/stackpop/edgezero/issues/391). + +Specification: [Axum safe diagnostics and trusted incoming proxies](../specs/2026-10-06-axum-diagnostics-trusted-proxy-design.md). + +## Goal and approval boundary + +Implement the approved native proxy-trust and safe diagnostic policy using #275's ingress clock and response-owned terminal transition. Keep network parsing, native records, request identity, snapshots, and lifecycle integration in Axum. Add only the portable hooks required by existing consumers and completion ownership. + +The user approved planning after a read-only specification review. Approving this plan will authorize its described implementation only after the prerequisite gate below is resolved. No Git rebase, commit, GitHub action, prerequisite exception, or experiment bypass is implied by plan approval alone. + +The user subsequently authorized the open-stack exception and requested a draft PR using `gh stack`: "yes open a draft pr stacked with `gh stack` when your done." Implementation therefore proceeds above open #393 PR #407. The branch was fast-forwarded, without rebasing or discarding the untracked specification/plan, from the planning HEAD to `3308041b910b648af746c15e809ee86b96adc2e9`. This does not satisfy the accepted-stack gate, authorize the #394/#395 upload experiment, or authorize closing #397. + +- [ ] Confirm the approved integration base and obtain permission for any checkout/rebase operation. Re-audit the accepted #275 revision and completed hosting stack before production implementation. #275 is still open; this gate is not satisfied. +- [ ] Reconcile applicable #393 and #394/#395 work at that base. Observe their accepted failures and lifetimes, not new buffering, transport, deadline, overload, or upload mechanisms. Do not require their entire historical backlog to close before planning #397. +- [ ] Refresh the file/call-site inventory and tests after integration. If hooks materially differ, revise the plan before code. Working on an unaccepted stack requires a separately recorded user decision. +- [x] Obtain approval of this plan's API and compatibility choices, including removal of raw forwarding headers on the managed path. + +Keep #389 optional. Do not add a metrics exporter, HTTP diagnostics endpoint, tracing propagation, vendor integration, generic field-selection framework, request registry, second server, second response finalizer, or portable Send migration. Packaging preparation remains independent. + +## Audited planning baseline + +Repository HEAD is `3ab22096fd117f7506cf437b5ba128e0789e059c`, including the committed #392/#396 hosting implementation. The governing untracked specification has SHA256 `d44dc2a89f44b2e2a2b13843516f8b570c40c9123d9fd6b57142d9663cc8b797`. + +Remote source state inspected during planning: + +- #275 is open at `950cb81d406d0dc269baf94314547329a4c1cef0`. +- #389 is open at `ab444946fcacb1d44c020c6079abb6ba29231402`. +- #393 PR #407 is open at `3308041b910b648af746c15e809ee86b96adc2e9`; its changes are not in this checkout. +- #394/#395's untracked documents remain in their owning worktree. This plan neither copies them nor approves their upload experiment. + +These are source/path observations, not runtime acceptance evidence. A follow-up plan review resolved the callback-fault logging and early fallback-status gaps, made idle/task/callback ownership explicit, hid ipnet from the public constructor, and pinned strict XFF empty-member handling separately from Forwarded's RFC list tolerance. No runtime proof was claimed by that review. + +| Existing boundary | Source and planning consequence | +| --- | --- | +| Owned transport peer | `crates/edgezero-adapter-axum/src/dev_server.rs:593-603` and `service.rs:235-268`. Match only the socket-owned argument, never ConnectInfo supplied by a caller. | +| Managed admission | `service.rs:255-321`. Validate the original head, normalize and strip forwarding fields, then create the admission view and convert the same parts. | +| Existing host consumer | `crates/edgezero-core/src/extractor.rs:228-259`. ForwardedHost currently prefers raw X-Forwarded-Host. It needs a checked-value path; other adapters retain the existing fallback. | +| Completion ownership | `crates/edgezero-core/src/response_egress.rs:445-462,576-690,917-963`. Join callbacks and use transition's existing observed_at. Report elapsed remains egress-only. | +| Error information loss | `crates/edgezero-core/src/app.rs:230-249,341-349`. Capture a bounded category before error rendering consumes EdgeError. | +| Configured fallback responses | `crates/edgezero-core/src/router.rs:599-628`. Body overflow/deadline can return ordinary application-configured responses. Preserve the owning reason separately from status. | +| Residual route-error reflection | `router.rs:513-528` and `error.rs:512-530`. Default framework 404/405 bodies can reflect a raw path or method. Fix the native default presentation, not arbitrary application errors. | +| Lifecycle owner | `dev_server.rs:492-656`. Reuse Starting/Ready/Draining/Stopped, connection tasks, drain budget, and cancellation. | +| Logger ownership | `dev_server.rs:709-712`. owns_logging controls installation only. Facade records and the request-record opt-out remain separate. | + +## Chosen implementation shape + +Two read-only workers investigated proxy ingress and diagnostics independently, then compared solution proposals. The coordinator inspected their cited code and selected the following shapes. + +| Decision | Selected approach | Rejected alternative and reason | +| --- | --- | --- | +| Proxy metadata | Axum policy/parser and native metadata, plus one portable CheckedEffectiveHost value | A generic provider trait adds indirection for one extractor and cannot improve transport provenance. | +| Terminal time | Add a timestamp-aware completion constructor; pass the existing transition instant through join | Sampling in a callback includes earlier callback delay. Adding a public report field unnecessarily breaks downstream report literals. | +| Request records | Attach a native completion to the envelope, with private status/category/identity | Replacing the app observer or installing another finalizer loses ownership and abandonment semantics. | +| Public route errors | Explicit default/custom renderer ownership and native-only framework route detail | New EdgeError variants, fabricated methods, or response-body sanitizers enlarge the contract or bypass custom rendering. | +| Active work | Fixed counters with request/connection lifetime guards | A request map or independent work registry duplicates ownership and is unnecessary for counts. | + +### Proposed API surface + +These names/signatures are the choices submitted for plan approval, not existing APIs. + +Axum exposes public types in `proxy` and `diagnostics` modules; parser functions and accounting guards stay private. Existing AxumRunOptions remains available through dev_server. + +```rust +// Axum proxy module +pub enum ForwardingHeaderFamily { Forwarded, XForwarded } +pub struct TrustedProxyPolicy { /* immutable normalized networks and family */ } +impl TrustedProxyPolicy { + pub fn no_trust() -> Self; + pub fn new(family: ForwardingHeaderFamily, networks: I) + -> Result + where I: IntoIterator, S: AsRef; +} + +// Axum hosting options, also provide the matching AxumDevServer builders +pub fn with_trusted_proxy_policy(self, policy: TrustedProxyPolicy) -> Self; +pub fn with_request_records(self, enabled: bool) -> Self; +pub fn with_diagnostics(self, diagnostics: NativeDiagnosticsHandle) -> Self; + +// Axum diagnostics module +pub struct NativeDiagnosticsHandle { /* cloneable fixed-counter handle */ } +impl NativeDiagnosticsHandle { + pub fn new() -> Self; + pub fn with_request_observer(self, observer: O) -> Self + where O: NativeRequestObserver; + pub fn snapshot(&self) -> NativeDiagnosticsSnapshot; +} +pub struct NativeDiagnosticsSnapshot { + pub active_requests: u64, + pub active_connections: u64, + pub idle_connections: u64, +} +pub trait NativeRequestObserver: Send + Sync + 'static { + fn observe(&self, record: &NativeRequestRecord); +} +``` + +The native ingress metadata accessor is `IngressHead::extension::()` at admission and `RequestContext::extensions().get::()` later. NativeIngressMetadata has read-only methods for request_id, direct_peer, effective_client, effective_host, effective_scheme, per-field provenance, forwarding_result, and received_normalized_head. Its fields are private. RequestId is opaque and has bounded Display output; it is not an authentication or distributed-tracing token. + +Core adds: + +- `CheckedEffectiveHost`, containing an optional validated core Authority and host provenance. Its hidden adapter constructor is `from_ingress(authority: Option, source: CheckedHostSource) -> Self`; accessors are `authority() -> Option<&str>` and `source() -> CheckedHostSource`. The source enum is Direct, TrustedForwarded, TrustedXForwarded, or Unavailable. Construction is descriptive metadata, not authorization. ForwardedHost reads it first. Checked-unavailable returns the existing localhost fallback without consulting raw forwarding fields. Absence of this type preserves the legacy extractor path. +- `ResponseEgressCompletion::new_with_terminal_time(callback)`, accepting `FnOnce(&ResponseEgressReport, MonotonicInstant) + Send + 'static`. Existing new, report fields, observer signatures, elapsed meaning, and completion ownership remain unchanged. +- `ResponseEgressEnvelope::with_completion(self, completion) -> Self`, joining existing application completion first and native completion second. +- `ResponseEgressEnvelope::failure_classification(&self) -> Option`. The private-field, copyable classification exposes a fixed kind of PropagatedError, FallbackBodyExceeded, or FallbackReadTimedOut, and an optional static `error_kind()` populated only from EdgeError::kind. Only core/owning boundaries construct it; it holds no EdgeError/source/message. The read-only `is_framework_route_error()` bit distinguishes actual router 404/405 from handler errors of the same kind, so native severity does not confuse their provenance. +- Hidden `App::dispatch_admitted_with_framework_error_detail(&self, prepared: PreparedIngress, request: Request, detail: FrameworkErrorDetail) -> ResponseEgressEnvelope`, an async adapter seam. FrameworkErrorDetail has Default and CategoryOnly variants. Existing dispatch_admitted delegates to the same implementation with Default. No duplicated dispatch pipeline. +- `validate_normalized_ingress_parts_with_summary(&RequestParts, IngressHeadLimits) -> Result`, returning normalized target bytes, header bytes, and field count computed by the existing validator. Existing validation entrypoints retain their signatures and discard that summary. Summary accessors are read-only. + +Use workspace dependency declarations for `ipnet = "2.12"` and `getrandom = "0.4.2"`, matching packages already in Cargo.lock. Only Axum's existing axum feature enables these dependencies. Core and other adapters do not depend on either. Do not install/upgrade unrelated packages or change the toolchain. + +### Configuration choices + +| Setting | Behavior | +| --- | --- | +| `EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS` | Comma-separated literal IP/CIDR entries. Unset or empty after trimming spaces/tabs means no trust. Empty interior entries, controls, names, DNS/provider/private/all tokens, invalid CIDRs, and exceeded bounds fail startup. | +| `EDGEZERO__ADAPTER__FORWARDING_HEADER_FAMILY` | `forwarded` or `x-forwarded`. Required for a nonempty trust list. Unknown supplied values fail even when trust is empty. A valid family without networks is inert. | +| `EDGEZERO__LOGGING__REQUEST_RECORDS` | Strict `true` or `false`; omitted means true. Invalid or non-Unicode input fails startup. Controls only default native per-request facade records. | + +Production from_env/from_vars uses the captured configuration once. The development run_app path validates these same new settings strictly while retaining its documented behavior for unrelated development settings. It must capture relevant OsString values once so non-Unicode inputs cannot disappear into omission defaults. Explicit options/builders never merge ambient proxy or record settings. + +Policy construction accepts literal IP/CIDR strings, keeping ipnet private rather than committing its type to the public API. It consumes at most 256 supplied entries, including duplicates before normalization, and at most 16,384 aggregate text bytes including conceptual separators. Environment network text is checked against that limit before allocating the entry vector. Bound explicit iterators too, and never silently truncate. Empty explicit policy is no trust. Error Display/Debug contain fixed setting/category names only. + +The default native diagnostics handle is created per managed runner. Explicitly sharing a handle between runners aggregates its counters and observer records; document that scope. Each runner still owns a separate identity namespace and lifecycle. Snapshot carries no duplicated lifecycle state; lifecycle phase remains owned by the existing runner/reader. + +## Proxy parser and trust invariants + +### Normalization and matching + +1. Canonicalize IPv4-mapped IPv6 socket peers and asserted numeric nodes to IPv4. +2. Truncate networks to their prefix, deduplicate, and project mapped IPv6 networks inside `::ffff:0:0/96` to equivalent IPv4 prefixes. For example, mapped /120 becomes IPv4 /24. +3. A mapped /96 becomes IPv4 /0 and is rejected. A shorter IPv6 prefix intersecting the mapped block contains the entire mapped IPv4 range; reject it rather than silently drop or broaden its IPv4 meaning. Ordinary disjoint IPv6 networks remain IPv6. +4. Check the normalized IPv4 and IPv6 unions separately for complete-family coverage. Reject explicit /0 and equivalent unions. Use bounded numeric interval merging with checked end-point handling, including u128::MAX; do not enumerate addresses. +5. Never infer trust from private/loopback/container ranges. Walk predecessors right-to-left only while the current hop is trusted. Stop at the first untrusted numeric predecessor. A missing/unknown/obfuscated node or all-trusted exhaustion means unresolved direct-client fallback, not leftmost selection. + +### Bounds and grammar + +The parser is an ordered byte scanner over received field values, not a concatenated string, serde tree, DNS lookup, or provider detector. + +- Maximum selected assertion input is 16,384 bytes, counting a conceptual delimiter between repeated fields. Preflight this sum with checked arithmetic before retaining parse data. +- Maximum chain/list resource slots is 64. Each value has one initial slot plus one per comma outside quoting, including tolerated empty members. Summing those slots across repeated field lines gives the bound; a repeated-field boundary is a conceptual comma joining its two neighboring members, not an extra empty member. Thus 32 two-member lines use exactly 64 slots. Maximum parameter slots per Forwarded element is 16, one initial slot plus one per unquoted semicolon, including empty slots. The first exceeded bound invalidates the whole selected assertion. +- Retain at most 64 fixed-size predecessor descriptions, one final host, one scheme, and bounded duplicate-name scratch. Quoted-value decoding and retained host text are each constrained by the same assertion byte budget. Unknown extension payloads are never retained. Scan duplicate parameter names case-insensitively with at most 16 names per element, not an unbounded set. +- Forwarded follows RFC 7239 token/quoted-string grammar, repeated-field order, escapes, duplicate-parameter rules, and its permitted empty semicolon slots. Tolerate empty HTTP list members under the HTTP list rule; they are not invented proxy hops. A substantive element lacking for remains a predecessor gap. Empty-only assertions yield no effective asserted facts. Test these cases explicitly. +- Numeric nodes support RFC IPv4 and quoted bracketed IPv6 with optional numeric or valid obfuscated node ports. Numeric ports are ASCII decimal u16 values. Valid obfuscated ports do not hide the accompanying numeric IP and never affect trust matching. Valid unknown/obfuscated nodes have no usable client IP. +- XFF supports bare IPv4/IPv6, IPv4:port, bracketed IPv6 with optional port, and unknown/obfuscated unresolved nodes. Numeric ports follow the same bounds; accept valid obfuscated node ports as unresolved port detail attached to a numeric node. No hostnames or zone identifiers. Iterate repeated XFF lines in received order. Unlike Forwarded's RFC list tolerance, an empty XFF value/member is malformed and invalidates the entire selected assertion; never skip an empty predecessor slot. +- Scan raw bytes. Do not reject an otherwise valid unknown quoted extension merely because it contains RFC-permitted obs-text; do not use HeaderValue::to_str on an entire Forwarded field. Recognized assertions must satisfy their own node/authority/scheme grammar. +- XFH and XFP are singleton assertions. Repetition or comma-list ambiguity invalidates the selected assertion. Forwarded host/proto may come only from its final element; do not borrow earlier values or another family. +- Validate all recognized assertions before the chain walk, including prefixes that will not be selected. Invalid node syntax, duplicates, host authority, scheme, or parser bounds cause whole-set direct fallback. Unknown/missing nodes are unresolved, not malformed, and may coexist with valid final-proxy host/scheme assertions. +- Host is a nonempty authority without userinfo, whitespace, path, query, or fragment, with a valid optional numeric port. Schemes normalize only http/https. There are no DNS or IDNA lookups. Direct host facts use the HTTP-authoritative head authority, parsing them without inferring scheme from an absolute URI; unavailable direct authority stays unavailable. Direct transport scheme remains HTTP. + +The 16KiB/64/16 bounds constrain additional forwarding parsing and retained metadata only. Current locked Hyper 1.10.1 has `DEFAULT_MAX_BUFFER_SIZE = 8192 + 4096 * 100`, or 417,792 bytes, in dependency `src/proto/h1/io.rs:23`. Reconfirm its effective use at the accepted revision. Do not alter the transport parser under #397 or claim the forwarding bound controls Hyper's initial receive allocation, total process memory, or raw parser accounting. + +### Managed order and compatibility + +```mermaid +flowchart TD + A[Parsed request and owned socket peer] --> B[Allocate private identity and request lifetime guard] + B --> C[Existing private draining gate] + C --> D[Validate and account original normalized head] + D --> E[Parse selected assertion within explicit bounds] + E --> F[Compute checked facts and remove both raw families] + F --> G[Existing admission using checked extensions] + G --> H[Convert same parts and dispatch] + H --> I[Attach native completion to existing egress owner] + I --> J[Existing terminal transition supplies instant and report] +``` + +The gate still aborts before admission/body work. Moving identity/start capture ahead of it does not normalize or admit a draining request. Head-validation failures retain their existing detached send/abort policy. + +With a missing, nonmatching, or disabled trusted-peer policy, do not parse forwarding assertions. Use direct facts and a bounded absent/untrusted-peer result, then strip both families. Only a matching socket peer enables selected-family parsing. This does not let any ignored family escape original-head accounting. + +Keep IngressHeadAccounting::HostManaged. The original normalized size summary is captured by the same validation loop before stripping and carried separately; do not call it raw wire bytes or upgrade capabilities. Admission sees checked headers and can read that original normalized summary. Head limits cannot be bypassed by stripping. + +Remove Forwarded and every case-insensitive X-Forwarded-* header, trusted or untrusted, before admission. Preserve Host, URI, direct AxumRequestContext.remote_addr, and other adapters. Overwrite any incoming checked metadata extension with the service-owned result. The public low-level into_core_request converter remains a documented bypass, not a policy-aware trust entrypoint. + +Document per-field fallback/provenance independently from the aggregate forwarding result. Incomplete client information must not discard valid final-proxy host/proto as though it were malformed; conversely malformed client text must discard all selected assertions. No raw invalid value enters a reason or record. + +## Diagnostics ownership and privacy + +### Identity and lifetime + +Generate a process nonce using getrandom once, before Ready. Pair it with a process-wide checked hosting sequence and a checked per-host request sequence. RequestId is the fixed-width concatenation of 128 nonce bits, 64 hosting-sequence bits, and 64 request-sequence bits, rendered as 64 hex characters. + +This gives deterministic non-reuse within a process, including concurrent hosting instances, and entropy-based separation between processes. It is not a mathematical global uniqueness guarantee or a security credential. No caller-selected namespace or incoming request ID participates. Entropy failure fails startup with a safe category; hosting-sequence exhaustion fails startup; request-sequence exhaustion emits a safe runtime fault and aborts without reusing an ID or inventing status. Inject entropy/sequence sources only in tests. No predictable fallback, wraparound, response header, or propagation is added. + +Keep the authoritative ID and method in the private request guard. Publish only a copy in NativeIngressMetadata. Use the existing app monotonic clock/start for both ingress and diagnostics; capture it once at the parsed service boundary, including drain-gate exits. + +A request guard is non-clone. It counts admission, body read, handler execution, and egress. On response paths it moves into the joined native completion; completion or abandonment releases it. On no-response Abort/draining/cancellation, release it through the actual ingress/drop path and emit a separately named, status-less record. Unbegun envelope abandonment invokes no fabricated egress callback/report; do not log its merely prepared status as a transmitted response. + +Use fixed counters, not a work map. active_connections is the total runner-owned connection-task count, including idle keep-alive tasks; idle_connections is its idle subset. Store a small connection-local activity count with the existing connection task. First active request makes that connection busy; the last released request makes it idle. Construct and increment the task guard before submitting the future to JoinSet, then move it into that future. It tracks connection lifetime and releases on return, cancellation, or panic, including an unpolled aborted task. The request guard and its counter/activity links use Arc/Mutex state satisfying the existing completion Send bound; EgressConnection and portable body streams keep their current non-Send ownership. Updates use short synchronous locks in a fixed connection-then-counter order; snapshot takes only the counter lock. No lock is held through a callback, logger, await, or body poll. Closed connection state prevents a late request release from recreating idle connections. + +Snapshot reads the three counters coherently under one short lock. It is a momentary observation, not an atomic combination with lifecycle phase, kernel buffers, detached application work, or external services. Sharing a diagnostics handle explicitly aggregates these counts without creating a registry. + +### Terminal time, selected status, and categories + +- Store one private timestamp-aware callback signature. Existing new adapts a report-only callback by ignoring the supplied time; new_with_terminal_time stores its callback directly. No callback-kind enum or duplicated completion path is needed. transition supplies its existing observed_at after storing terminal state and before invoking callbacks. join forwards the same instant through every nested child and retains independent unwind boundaries. Existing application completion runs before native completion; existing application observer still runs afterward. Guard the existing core callback/observer/policy panic-report log attempt with its own catch_unwind. Attempt that fixed-category report once, suppress a logger panic, and never retry/log that logging failure. The current unguarded log after catching a callback panic can otherwise skip the next joined child or recurse; add a focused regression for that exact path. +- Do not change ResponseEgressReport or elapsed. Native total duration uses terminal_instant.checked_duration_since(request_start). Preserve backwards-clock classification and report a safe clock fault rather than falling back to wall time. +- Attach native completion on every response envelope, including refusal, detached errors, conversion failure, dispatch errors, fallback, and ordinary success. A private selected-status slot captures the envelope's initial selected status before begin and is updated at the actual prepared/fallback selection before any terminal call. Set fallback 500/504 even on failed fallback registration/preparation before settling the attempt. In prepare_fallback's fail-closed begin_fallback == false branch, store the returned detached 500 before terminating any still-live attempt. An already-terminal attempt cannot emit another record or have its previous record rewritten; prove the managed path reaches fallback only with an Initial, unterminalized attempt and include defensive-state tests. The slot is never a public response extension or another finalizer. +- Preserve typed failure provenance before rendering. A private router dispatch result marks only router-owned NotFound/MethodNotAllowed and configured fallback-body exceeded/timed-out branches. Existing public router dispatch APIs unwrap that result; App uses its classification. Propagated EdgeError captures only a bounded kind/reason, never Display/Debug/message/source. Admission refusal is classified at its owning outcome; deliberate application 413/503 statuses alone carry no framework rejection. +- NativeRequestRecord is a fixed allowlist: event, ID, known-method/other category, route template or fixed unmatched/unknown, optional selected status, total duration, typed terminal/ingress outcome, bounded failure/forwarding category, and optional egress byte/body/fallback accounting. No full report copy containing arbitrary route class, no addresses, hostnames, raw target/query, headers, payload, config/secret values, origin URLs, or source strings. +- A selected 200 and SourceError/DeadlineExceeded/TransportError remain separate facts. Egress-accounted bytes are not delivered TCP bytes; selected status is not wire commitment or receipt. Wire fixtures observe those separately. +- Release request accounting before native sinks. Default request facade output and the optional native observer are isolated individually on unwind-capable targets, including Drop paths. A broken logger must not be called recursively to report its own panic. Existing app panic-abort limitations remain. + +Default events use target `edgezero::native`: `request_terminal` for begun attempts, `request_ingress` for no-response/abandonment, and `lifecycle` for owner transitions/failures. Escape application-owned route-template text for one-line output without logging raw paths. Known HTTP methods are fixed names; every extension method becomes `other`. + +Ordinary completion/canonical route miss and expected draining events use INFO. Typed refusal, malformed forwarding, propagated request errors, and failing/abandoned attempts use WARN. Startup/runner faults and unusable identity/clock state use ERROR. Status alone never selects a level. Malformed-forwarding reason is included in the eventual request record, not a second per-request terminal record. + +Request-record opt-out suppresses default request_terminal/request_ingress logs only. Native/app observers, lifecycle records, counts, and snapshots remain active. Hooks::owns_logging is unchanged: use log facade without installing a competing logger/subscriber. A synchronous logger/observer can block; add no queue, retry, flush wait, timer, or shutdown budget. Process abort/SIGKILL has no completion or flush promise. + +### Residual default public route errors + +Keep one error serializer. Internally represent renderer ownership as `Default | Custom(callback)`, replacing the current always-callback representation, not adding a synchronized boolean. + +Managed native dispatch requests CategoryOnly detail. Router-owned 404/405 errors get explicit private provenance tags at their construction branches. Only the default renderer uses a fixed public message for those tags while the same serializer keeps status/kind/required headers, including Allow. Custom renderers receive the unchanged original error and retain control as today. Legacy dispatch on other adapters retains existing text. Handler-generated NotFound/MethodNotAllowed and application validation messages are not silently treated as framework routing errors. + +Audit other framework call sites for source-bearing messages in categories that #275 intentionally leaves application-controlled. Fix only demonstrated native framework boundaries with safe fixed messages or the established renderer, preserving custom response ownership. Do not add a sanitizer, regex masking, source-string logging, or universal application-response redaction. Any wider change discovered here returns to plan review. + +## Delivery checklist + +### Task 0: Refresh the accepted base and acceptance inventory + +Files to inspect: core app/ingress/router/error/response_egress; Axum service/request/response/connection/dev_server/run_options; accepted #393 body policy; existing hosting/contract fixtures. + +- [ ] Resolve the approval/prerequisite gate without discarding untracked documents. +- [ ] Inventory existing versus missing cases from the specification's acceptance table. Retain Unsupported/BestEffort classifications. +- [ ] Record exact base SHA, relevant uncommitted inputs, dependency/tool versions, and fixture/logging environment. Confirm any shared-build command owner. +- [x] Establish focused baseline using `cargo test -p edgezero-core` and `cargo test -p edgezero-adapter-axum --all-targets --all-features`. Both passed at the open #407 head before implementation. Logs: `/tmp/edgezero-impl-397/baseline-core.log` and `baseline-axum.log`. + +### Task 1: Add minimal portable completion, provenance, and renderer hooks + +Modify: `crates/edgezero-core/src/{response_egress,app,router,error}.rs`; re-export only required adapter-facing types in lib.rs. + +- [x] Add red tests for timestamp-aware completion, nested joins, exact supplied terminal instant, app/native/observer ordering, left/right panic isolation, repeated terminal signals, and unbegun abandonment with resource release but no report. +- [x] Implement the additive timestamp-aware constructor and envelope composition. Keep report shape, report-only callbacks, elapsed, sole transition, and observer API unchanged. Contain the existing core callback/observer/policy panic-report log attempt independently, without retry or recursive logging; test that a left callback plus its logger can panic without suppressing the right timed child or the app observer. Extend the same focused matrix to policy/observer panic-report logging so those newly guarded paths are exercised too. Use an isolated logger fixture or thread-scoped test logger, not competing process-global logger initialization. +- [x] Add private dispatch-result provenance and immutable envelope classification before rendering. Tag configured fallback overflow/deadline responses at their owner, without altering their selected body/status or public router signature. +- [x] Add the default/custom renderer distinction and native dispatch detail seam. Factor message selection into the existing serializer. Test default native fixed 404/405, required Allow, unchanged legacy output, unchanged custom renderer inputs, and no tag on handler-selected errors. +- [x] Run `cargo test -p edgezero-core --lib response_egress`, `cargo test -p edgezero-core --lib app`, `cargo test -p edgezero-core --lib router`, `cargo test -p edgezero-core --lib error`, and `cargo test -p edgezero-core --doc`. + +### Task 2: Implement native trust configuration and bounded parsing + +Create: `crates/edgezero-adapter-axum/src/proxy.rs`. Modify: root/Axum Cargo manifests and Cargo.lock only as needed for direct gated dependencies; Axum run_options.rs/lib.rs. + +- [x] Add pure red tests for no-trust defaults, strict env/family parsing, safe error formatting, raw entry/byte caps, CIDR truncation/deduplication, mapped equivalence, entire-family unions, and trust-all rejection. +- [x] Implement validated immutable policy using ipnet. Bound explicit iterators as well as environment input; no DNS or provider discovery. +- [x] Add parser tables for repeated fields/order, quotes/escapes/obs-text extensions, case-insensitive duplicates, valid empty slots, IPv4/IPv6/node ports, unknown/obfuscated gaps, host/scheme validation, selected-family isolation, and singleton XFH/XFP. +- [x] Prove exactly 16KiB/64 slots/16 parameters succeeds when semantically valid and each next unit yields whole-set direct fallback. Include mixed good host/proto plus malformed client and unknown-but-valid client cases. +- [x] Cover right-to-left spoofed prefixes, partial trust, gaps, and all-trusted exhaustion with pure tests. Pin strict XFF empty-value/member handling: present-empty or mixed-empty XFF invalidates the complete family, including otherwise-valid XFH/XFP, while absent XFF permits individual host/proto acceptance. Forwarded's RFC empty-list tolerance remains separate. +- [x] Run `cargo test -p edgezero-adapter-axum --lib proxy` and `cargo test -p edgezero-adapter-axum --lib run_options` after each coherent change. + +### Task 3: Integrate checked metadata before managed consumers + +Modify: core ingress.rs/extractor.rs/lib.rs; Axum service.rs/dev_server.rs/run_options.rs/proxy.rs. Request conversion remains unchanged except documented bypass. + +- [x] Add CheckedEffectiveHost and accounted normalized-head helper using the existing validator's loop. Preserve old signatures and HostManaged markers. +- [x] Carry native metadata through existing extensions. Normalize/strip after original-head validation and before IngressHead construction. Match the private peer argument even if a test supplies fake ConnectInfo/metadata. +- [x] Wire the same policy into standalone production, development, explicit-options hosting, and AxumDevServer builders, validating before Ready. Explicit APIs ignore ambient policy settings. +- [x] Add service tests proving admission/middleware/handler/ForwardedHost agree, received normalized summary still includes removed fields, pre-strip overages still reject, Host/URI/direct peer stay unchanged, and raw fields are absent even when trusted. +- [x] Retain legacy extractor tests without checked metadata, including nonnative/default/local fallback. Test checked-unavailable cannot fall through to raw forwarding text. +- [x] Run `cargo test -p edgezero-core --lib ingress`, `cargo test -p edgezero-core --lib extractor`, `cargo test -p edgezero-adapter-axum --lib service`, and `cargo test -p edgezero-adapter-axum --lib dev_server`. + +### Task 4: Add native identity, lifetime accounting, and safe observer data + +Create: `crates/edgezero-adapter-axum/src/diagnostics.rs`. Modify: Axum lib.rs/run_options.rs/dev_server.rs/service.rs/proxy.rs. Add getrandom gated dependency if not added in Task 2. + +- [x] Add red tests for deterministic hosting/request non-reuse, independent nonce sources, entropy failure, sequence exhaustion without wrap, incoming-ID/public-metadata mutation, and no automatic response header. +- [x] Implement native session initialization before Ready and generated ID/start capture before the private gate. Public metadata copies cannot change private correlation. +- [x] Implement Send-safe request/task counter links and fixed counter snapshots, including idle/busy connection transitions, two pipelined request lifetimes on one connection, and closed-connection late cleanup. Construct the connection guard before JoinSet submission. Prove an idle owned task contributes one active_connection and one idle_connection but no active_requests. +- [x] Prove release on completion, explicit Abort, pending-admission/body/handler cancellation, unbegun envelope drop, task aborted before polling, and observer/logger panic. No fabricated report/status on those no-response paths. +- [x] Add the one typed native observer and strict request-record setting. Opt-out must not disable counts or either application observer. +- [x] Run `cargo test -p edgezero-adapter-axum --lib diagnostics`, `cargo test -p edgezero-adapter-axum --lib run_options`, and affected service/dev_server tests. + +### Task 5: Attach correlated terminal records at the existing owner + +Modify: Axum service.rs/response.rs/diagnostics.rs/dev_server.rs. Use Task 1's shared hooks; do not replace App policies or observers. + +- [x] Attach native completion to all envelopes and classify status-less explicit branches separately. Ensure abandonment Drop cannot duplicate an explicit record. +- [x] Capture selected status privately before begin; update it at prepared/fallback selection before any settlement, including the early begin_fallback == false detached-500 branch and fallback preparation/registration failure. Prove the managed precommit fallback entry invariant and test defensive live/already-terminal states without a second record. Post-commit failures keep the original selected status. +- [x] Add deterministic clock tests with distinct pre-egress, egress, application-callback, and logger delays. Total duration must exclude callback/logger delay and differ from egress-only elapsed. +- [x] Add record/level/cardinality tests for propagated errors, configured fallback reasons, custom 413/503, source/deadline/disconnect failures, malformed forwarding, refusal, cancellation, and opt-out. Use existing clock fixtures and no source-message matching. +- [x] Audit existing per-request Axum warnings. The old source-failure warning was removed; native records remain the sole request-terminal record and opt-out controls that sink. Preserve core callback-fault diagnostics. A dedicated source-error-under-optout socket combination remains untested, as recorded below. +- [x] Add safe lifecycle events at the actual runner transitions and startup/runner/grace-exhaustion/forced-stop returns, retaining safe returned startup errors when a logger has not yet been installed. Do not change signals, readiness, acceptance, drain budget, or teardown. +- [x] Run `cargo test -p edgezero-adapter-axum --lib response`, `cargo test -p edgezero-adapter-axum --lib service`, `cargo test -p edgezero-adapter-axum --lib connection`, and `cargo test -p edgezero-adapter-axum --lib dev_server`. + +### Task 6: Prove the managed production path with existing sockets/fixtures + +Modify: `crates/edgezero-adapter-axum/tests/production_hosting.rs`, `tests/fixtures/production_host.rs`, and nearby service tests. Add a test-only `tests/fixtures/native_proxy.rs` helper and static local TLS certificate/key fixtures. Reuse outbound contract origins where relevant; no new runtime server. + +For the local TLS fixture, declare the already-locked rustls 0.23.40 and tokio-rustls 0.26.4 families through workspace dependencies and Axum dev-dependencies only. Match the accepted reqwest crypto backend; add no competing crypto provider or production incoming-TLS dependency. The test client trusts only the fixture CA and verifies the fixture server name; never disable TLS verification. Fixture private keys are test-only, not application secrets. + +- [x] Extend the existing subprocess fixture with test-only metadata observation and a captured application-owned log implementation installed once per process. Include a mode whose logger panics while reporting an app completion panic; the timed native child and app observer must still run, counters must release, and logging must not recurse. Keep fixture diagnostics routes test-only. Assert no competing logger/subscriber initialization and original app hook counts/order. +- [x] Exercise default spoofed input, matching/nonmatching peers, both selected families, conflicting/missing/malformed assertions, head overages, and explicitly trusted mapped/CIDR cases. Missing private peer is a managed-service fixture case, not falsely claimed as a real accepted socket condition. +- [x] Add local forwarding fixtures with actual accepted socket peers for single/multi-proxy chains, partial trust, spoofed prefixes, gaps, and a sanitizing gateway. Bind outgoing proxy sockets to distinct loopback addresses with TcpSocket, so each hop's actual predecessor differs from the visitor. Configure only those proxy /32 entries as trusted. Pure fabricated header chains alone do not prove proxy peer ownership. Observe admission, middleware, handler, and ForwardedHost. Keep proxy-head buffering finite and use a bounded duplex relay, not a collected response. +- [x] Use a controlled local TLS-terminating ingress fixture for HTTPS public assertions over an HTTP native hop. Keep that TLS fixture test-only; do not add native incoming TLS. Prove overwrite obligations and that client/earlier-hop host/proto cannot win. +- [x] Inject sentinel secrets into unmatched path, query, extension method, headers, payload, configuration, conversion/dispatch/internal/upstream errors, and failing response sources. Assert absence in default client errors and all ordinary framework logs, including DEBUG paths; application/custom-renderer output remains explicitly outside that default guarantee. +- [x] Observe actual wire head/prefix before source/deadline/disconnect failure. Assert one native terminal record, selected status plus failure, preserved app callbacks, and no second response. Reuse existing zero-read/deadline/disconnect fixtures rather than re-proving #394 transport claims here. +- [x] Exercise #393 413 and cleanup on the authorized open base through managed hosting, not just error construction. Preserve its configured ceiling and body semantics. +- [x] Hold body/handler/egress work and idle keep-alive sockets, accounting for the query itself, and verify graceful or budget-forced cleanup. Admission cancellation and unpolled task ownership are separately executed service/unit tests, not a held asynchronous-admission socket witness. +- [x] Run `cargo test -p edgezero-adapter-axum --all-features --test production_hosting`, `cargo test -p edgezero-adapter-axum --all-targets --all-features`, and the focused affected outbound contract cases. Do not label local socket results final-image acceptance; #398/#399 reuse them later. + +### Task 7: Publish operator and compatibility guidance + +Create: `docs/guide/axum-diagnostics.md`. Modify only related sections/links in `docs/guide/adapters/axum.md` and the documentation navigation if required. + +- [x] Document settings/defaults/bounds, all-or-nothing versus unresolved fallback, network isolation, explicitly trusted chains, final-proxy overwrite/verification, and no automatic provider-range discovery. +- [x] Include Nginx, Cloudflare-plus-load-balancer, and gateway-replacement examples. Explain which operator configuration supplies verified assertions; do not promise every provider default fits the policy. +- [x] Document raw-header application migration to NativeIngressMetadata/ForwardedHost, direct peer preservation, unavailable authority, and low-level converter bypass. No raw outbound propagation recipe without a separate decision. +- [x] Show custom logging initialization, request-record opt-out, snapshot access, and safe native-observer metric counters/durations. IDs, addresses, payloads, and secret values are not metric labels. Ship no exporter or public endpoint. +- [x] Explain timestamp/status/byte guarantees, synchronous logger blocking, panic-abort/SIGKILL, identity scope, and shared snapshot-handle scope. Keep #389 optional and label final-image validation downstream. +- [x] Do not fix unrelated stale Docker/storage/CLI documentation under this task; record those with their owners if encountered. +- [x] Run `cd docs && npm run lint && npm run format && npm run build` for published docs changes. Internal superpowers plans/specs are excluded from Prettier; formatting is not acceptance evidence. + +### Task 8: Verify integrated state and prepare acceptance evidence + +Run one command owner per shared Cargo target directory. Reuse checks only when their tested state and affected inputs remain current. + +- [x] `cargo fmt --all -- --check` +- [x] `cargo clippy --workspace --all-targets --all-features -- -D warnings` +- [x] `cargo test --workspace --all-targets` +- [x] `cargo check --workspace --all-targets --features "fastly cloudflare spin"` +- [x] `cargo check -p edgezero-adapter-fastly --target wasm32-wasip1 --features fastly` +- [x] `cargo check -p edgezero-adapter-cloudflare --target wasm32-unknown-unknown --features cloudflare` +- [x] `cargo check -p edgezero-adapter-spin --target wasm32-wasip2 --features spin` +- [x] Published docs checks from Task 7 and a compiled API/options/snapshot smoke. The existing ignored generated-workspace compile smoke remains excluded; no CLI template change was made. +- [x] Hand integrated verification to a read-only build/acceptance worker and independently inspect its evidence. Record unavailable SDK/target/tool checks as gaps, not passes. +- [x] Record each specification acceptance row as covered-by-existing-test, newly-proved, or explicitly outstanding, with command, exact tested revision/untracked inputs, environment, elapsed time, result, and log/wire artifact. + +## Acceptance ownership and stop conditions + +Task 1 proves portable clock/composition/provenance and native default route privacy. Tasks 2-3 prove configuration, syntax, trust, consumer order, and extractor compatibility. Tasks 4-5 prove identity, counts, safe fields, correlation, levels, observer ownership, and lifecycle diagnostics. Task 6 is the production/managed socket evidence required by every applicable specification row. Tasks 7-8 document and verify the integrated result. + +The two highest-risk assumptions are now proven at the local Runtime boundary, not at accepted-stack or final-image acceptance: + +1. Socket-peer-gated checked facts reach admission, middleware, handlers, and ForwardedHost. The production-hosting tests use actual source-bound .2/.3 proxy sockets, direct and nonmatching controls, both header families, and verified TLS with a wrong-name negative control. +2. Native observations preserve the sole terminal owner and application hooks. The production-hosting tests observe actual head/prefix before source/deadline/disconnect failures and assert selected status, exact record/callback counts, order, and released work. Precise terminal-time duration and callback-delay exclusion are separately Executed with injected Core/service clocks. + +Stop and return to plan review if the accepted stack requires another finalizer/clock, blanket Send changes, transport/parser changes to satisfy a new memory guarantee, a generic observability registry, broader application-message redaction, or a new policy-aware low-level converter. Do not select new workload concurrency/deadline defaults to finish diagnostics. + +Planning verification was read-only source/issue inspection and document consistency. No runtime proof was claimed at plan approval. Implementation checks are recorded below; they do not discharge deferred gates. #397 must not close until its accepted-stack and runtime evidence gates are actually satisfied. + +## Intermediate implementation evidence + +The coordinator owns shared Cargo commands and global formatting. Proxy and diagnostics workers edit exclusive new modules, return ownership before integration, and run no shared Cargo commands. Their investigations/proposals and plan reviews are retained in `/tmp/edgezero-plan-397/`; module handoffs are in `/tmp/edgezero-impl-397/`. + +Environment: Rust/Cargo 1.95.0, Node 24.12.0, native `x86_64-unknown-linux-gnu`; all three planned WASM targets are installed. Native dependencies enable only already-locked ipnet 2.12.0 and getrandom 0.4.2. The two direct lockfile edges are the only dependency changes so far. + +Current local evidence, on uncommitted implementation above `3308041`: + +- Core library checkpoint passed 727 tests before adding the isolated fault-logger regression, log `core-integrated.log`, test execution 0.21s. +- Portable completion suite passes after formatting and the new isolated subprocess regression. The fixture proves completion, policy, and observer fault reporting remains contained when the installed facade logger panics. It also proves the native child receives the original terminal instant after the application callback advances the clock. Log `core-completion-formatted.log`. +- Axum library checkpoint passed 198 tests, including 24 proxy and 21 diagnostics cases. Log `native-formatted.log`, test execution 0.72s. +- `cargo clippy -p edgezero-core -p edgezero-adapter-axum --all-targets --all-features -- -D warnings` passed, log `clippy-formatted.log`, 4.56s. +- Shared formatting has run. Final formatting, workspace/WASM checks, managed socket evidence, operator docs, independent verification, and PR submission are not yet complete. + +The summary-helper name was clarified during integration to `validate_normalized_ingress_parts_with_summary`. Failure classification gained only a read-only router-origin bit, keeping its original bounded kind and existing report shape. Neither change adds another validator, clock, finalizer, or native registry. + +### Approved additional native JSON privacy boundary + +A direct executable reproduction of the existing native-budgeted JSON reader found that `RequestContext::json_within::` maps serde's wrong-type error into an application-controlled BadRequest message. Input `"PAYLOAD_SENTINEL"` yields HTTP 400 with `invalid JSON payload: invalid type: string "PAYLOAD_SENTINEL", expected u64 ...` in the default serializer. Source: `crates/edgezero-core/src/context.rs`, JSON deserialization error mapping. Reproduction and output are `/tmp/edgezero-impl-397/repro-json-privacy.rs` and `repro-json-privacy.log`. + +This is a pre-existing framework-owned payload disclosure, not a native record/log disclosure or a regression introduced by these hooks. The user explicitly approved native-only fixed JSON parse text after clarification of the compatibility change. `json_within` now maps deserialization errors to `invalid JSON payload` when the request carries the existing managed body policy. It preserves HTTP 400, nonnative serde detail, and body-drain or application-authored EdgeErrors. Custom renderers retain response ownership but receive fixed text for these native deserialization failures. No message/status matching, universal sanitizer, or new provenance flag was added. + +The new wrong-type/payload sentinel regression failed before the fix, log `json-privacy-red.log`, and passed afterward with all 731 Core library tests and doctests, log `core-json-privacy-green.log`. Managed wire coverage uses a fixture-only POST `/native-json-number` handler. The native TLS test dependencies use already-locked rustls 0.23.40 and tokio-rustls 0.26.4 as Axum dev-dependencies only; no provider package was upgraded. + +The new service test handoff also exposed two incorrect test assumptions: refusal intentionally has no application egress observer, and a terminal-error private body is checked with `is_end_stream`/drop rather than repolling a pending source. Corrections are confined to those assertions, preserving the prerequisite's production lifetime behavior. The first service run timed out after 600s; its output is retained as a failed check, not runtime proof. + +After those assertion corrections and scoped fixture lint fixes, all 42 service cases passed in 0.55s (`service-managed-lint-fixed.log`), all 16 existing production-hosting socket scenarios passed in 3.13s (`hosting-fixture-lint-fixed.log`), and the Core/Axum all-target/all-feature Clippy command passed in 1.84s (`clippy-fixtures-fixed.log`). The 16 hosting cases establish regression coverage for the extended fixture; they do not yet exercise the new proxy/TLS scenarios. The test-only proxy helper and certificate assets exist but are not wired into a test module or dependency manifest yet. The user subsequently approved the JSON compatibility decision and its focused Core regression passed. The following final local checkpoint supersedes these intermediate results. + +## Final local integration checkpoint + +Date: 2026-10-07. Base: `3308041b910b648af746c15e809ee86b96adc2e9`, the still-open #407 head, rechecked before PR preparation. #275 also remains open at `950cb81d406d0dc269baf94314547329a4c1cef0`. No accepted-stack or final-image evidence is claimed. + +Environment: Rust/Cargo 1.95.0, Node 24.12.0, Linux `x86_64-unknown-linux-gnu`, local subprocess and loopback HTTP/1/TLS fixtures. Test keys are explicitly public fixtures, not deployment credentials. Dependency changes are four direct Axum edges to already-locked ipnet 2.12.0, getrandom 0.4.2, rustls 0.23.40 and tokio-rustls 0.26.4. The last two are dev-only. No package version changed. + +The final runtime-input manifest covers modified Rust/Cargo/published docs and new parser/diagnostics/proxy/TLS files, excluding the changing internal plan and immutable specification. Its SHA256 is `ac40934b17bb6c521951f15090e2accf3059abcb4d386c6da74a69d95de8c9d7`, stored at `/tmp/edgezero-impl-397/final-runtime-inputs.sha256`. Bind these tests to the resulting implementation commit; no runtime input changed after this checkpoint. + +All logs below are under `/tmp/edgezero-impl-397/`. Broader passing suites supersede the checklist's narrower filters; those checkboxes describe covered test groups, not an assertion that every listed filter was run separately. + +| Check | Result | Time | Log | +| --- | --- | --- | --- | +| `cargo fmt --all -- --check`; `git diff --check` | Pass | Check-only | `fmt-final.log` | +| `cargo clippy --workspace --all-targets --all-features -- -D warnings` | Pass | 4.35s real | `workspace-clippy-final.log` | +| `cargo test --workspace --all-targets` | Pass; 1,938 tests across 27 suites, one existing ignored compile smoke | 51.85s real | `workspace-tests-final.log` | +| `cargo test -p edgezero-adapter-axum --all-targets --all-features` | Pass; 255 all-feature library, 13 standalone contract and 32 hosting tests | 12.84s real; hosting 6.54s | `axum-all-final.log` | +| `cargo test -p edgezero-core` | Pass; 731 library, two integration cases, two ordinary and four compile-fail doctests; 13 existing ignored examples | Unit and doctest times in log | `core-final.log` | +| `cargo check --workspace --all-targets --features "fastly cloudflare spin"` | Pass | 5.66s real | `workspace-check-final.log` | +| Fastly `wasm32-wasip1` check | Pass | 0.66s real | `wasm-fastly-final.log` | +| Cloudflare `wasm32-unknown-unknown` check | Pass | 0.67s real | `wasm-cloudflare-final.log` | +| Spin `wasm32-wasip2` check | Pass | 0.75s real | `wasm-spin-final.log` | +| Locked `npm ci`, docs lint, format and build | Pass | Build 3.13s | `docs-{install,lint,format,build}.log` | +| Compiled guide APIs, explicit options and empty snapshot smoke | Pass | Local executable | `docs-api-smoke.log` | + +Final fixture hashes: hosting test `5c9d6e6170289933d064e49a9280428c01fb9bf6d1e3a955577acf599d952368`, host executable source `4e7e11d6b04d16992ec34642ad914e80b714454ea8a402605b595d5bced5b6fe`, relay helper `0dc5d7a839a1e47cde609cd73158c981517f8ca9f052c71d165f30b5ab791adc`. All are covered by `axum-all-final.log` after final formatting. The three defensive response tests retained their handoff hash `bad0422ea9e9daf24e7729ad3c6826a340f396fc6c5dd948c78f596ef3e79b93`; global formatting did not change that file. + +### Independent inspection and corrections + +Four read-only lanes inspected the stable 20-file production/API/doc target at manifest `0f6c234f6506f0f541dc4b78017b1485bb66c2bead136e02b8a3fe0d76aeb6c3`. The changing executable/socket/helper fixtures were excluded from that source target and inspected separately by the acceptance worker and runtime tests. Architecture found no defects. Correctness found C1, a lost rejection category on detached Abort. A real-socket regression failed with `failure=None`, then passed after retaining only `error.kind()` in the private native guard before core consumed the error. It still emits one status-less record with no bytes/body report and releases work. A focused post-format review confirmed C1 resolved. Style T1 was resolved by global formatting and its final passing check. Maintenance-only S1 notes duplicated IPv6 bracket splitting in node/authority parsers. It remains unchanged to avoid a parsing refactor unrelated to behavior; their port grammars intentionally differ. + +The acceptance worker mapped all 22 specification rows, inspected concrete assertions and actual parent logs, and exposed the unused config/secret sentinel and defensive-status gaps. The fixture now captures actual environment-derived configuration/secret values into immutable state, composes them into internal/upstream errors, and proves consumption with exactly two nonsecret markers. Default wire errors and framework facade logs exclude those values. This is not declared-store/backend or actual outbound-origin proof. Three added private response tests execute live duplicate fallback, already-terminal no-rewrite, and failed fallback registration for selected 500/504 with one callback/record and zero work. Static fixed fallback framing is valid by construction; its unreachable preparation-failure branch was not artificially injected. + +### Acceptance row inventory + +All named native socket tests are in `tests/production_hosting.rs`; the 32-test suite contains 16 existing hosting cases and 16 new cases. The following abbreviated W keys identify those new tests: + +| Key | Test name after `native_` | +| --- | --- | +| W01 | `direct_boundary_strips_spoofs_and_accounts_the_original_head` | +| W02 | `source_bound_single_proxy_and_mapped_cidr_match_only_owned_peers` | +| W03 | `two_proxy_chain_stops_at_untrusted_hop_and_gateway_replaces_prefix` | +| W04 | `selected_families_conflicts_malformed_sets_gaps_and_exhaustion` | +| W05 | `verified_tls_ingress_overwrites_public_assertions_and_rejects_wrong_name` | +| W06 | `default_errors_json_cap_and_logs_keep_sentinels_private` | +| W07 | `custom_renderer_keeps_ownership_but_framework_json_parse_text_stays_fixed` | +| W08 | `original_head_overage_cannot_be_hidden_by_raw_header_stripping` | +| W09 | `ids_remain_private_under_mutation_and_concurrent_hosting_requests` | +| W10 | `logger_panic_preserves_order_cardinality_and_resource_release` | +| W11 | `actual_wire_prefix_precedes_source_and_deadline_failure_without_second_response` | +| W12 | `actual_wire_disconnect_keeps_selected_status_and_one_terminal_owner` | +| W13 | `refusal_and_no_response_abort_keep_separate_observer_contracts` | +| W14 | `idle_and_held_work_snapshots_drain_to_zero_with_request_records_opted_out` | +| W15 | `invalid_proxy_and_record_settings_fail_before_ready_without_input_disclosure` | +| W16 | `non_unicode_proxy_and_record_settings_fail_instead_of_defaulting` | + +| Specification row | Local coverage and boundary | +| --- | --- | +| Default no trust | Runtime W01; managed forged-extension matrix separately executed. | +| Explicit trusted peer | Runtime W02-W04 with actual .2/.3 sources and nonmatching control; mapped CIDRs on IPv4 sockets are not IPv6 transport proof. | +| Peer unavailable | Executed managed service/private-peer-none matrix; an accepted TCP socket cannot provide a real missing-peer case. Converter bypass is documented. | +| Family selection | Runtime W04, both families and missing/conflicting assertions without borrowing. | +| Parsing | Runtime representative W04/W08 plus 24 parser cases. Exact byte/slot/parameter bounds, obs-text, escapes and exhaustive IPv6/port grammar are pure parser evidence. | +| Spoofed prefix | Runtime W02-W04, actual multi-hop/partial trust/gaps/exhaustion and replacing gateway. Gateway client is operator supplied, not discovered/authenticated by EdgeZero. | +| Malformed whole-set fallback | Runtime W04 verifies bad client text discards otherwise-valid final host/HTTPS and emits no raw reason. | +| Missing/unknown values | Runtime W04 distinguishes independent fallback/unresolved client from malformed whole-set rejection. | +| Host/HTTPS provenance | Runtime W03/W05, verified fixture CA/name and wrong-name negative control, final proxy versus visitor/earlier-hop assertions. Not deployed Nginx/Cloudflare/LB proof. | +| Configuration | Runtime W15/W16 production `from_env` failures before Hooks/Ready/bind, including each new non-Unicode setting. Exhaustive bounds/unions and explicit options are unit evidence. New-key development capture remains source-reviewed, not a dedicated process witness. | +| Consumer order/compatibility | Runtime W01-W05/W08 compares admission/middleware/handler/ForwardedHost and original normalized head accounting. Legacy/unavailable extractor behavior is Core test evidence. | +| Safe errors/records | Runtime W01/W06-W08/W11 with private path/query/method/header/payload/config/secret/error/source values; custom/application output deliberately remains owned by the application. Constructed BadGateway is not actual outbound-origin proof. | +| Correlation/levels | Runtime W01/W06, typed levels versus deliberate status. Parameterized `/native/{id}` template and escaped one-line route text are service/unit evidence. | +| Identity ownership | Runtime W09 concurrent requests/two processes/mutation/no auto headers. Same-process namespace/entropy/exhaustion is deterministic unit evidence; sample noncollision is not global uniqueness proof. | +| Rejection/Abort | Runtime W06/W08/W13 plus the real-socket C1 regression retains `uri_too_long` without invented status/report. Drain-gate semantics have managed service coverage. | +| Failure after headers | Runtime W11/W12 observes actual 200 head/prefix before failure, one response/record and selected status. Disconnect preserves existing TransportError classification. | +| Fallback/abandonment | Runtime W06/W14 plus Executed service/Core drops and three private defensive response tests. Unbegun envelopes have no egress report; prepared native responses have begun. | +| Application logging | Runtime W10-W13 one owned logger, app-completion/logger panic, ordering and exact counts. Native-observer panic is executed service/unit evidence. | +| Request-record opt-out | Runtime W14 paired workloads keeps observers/lifecycle/counts while suppressing request records. Dedicated source-error-under-optout socket combination remains untested; legacy warning removal is source-reviewed. | +| Active work/shutdown | Runtime W14 idle/body/handler/stream and graceful/grace-forced final zeros. Admission/unpolled ownership is executed service/unit. Convenience second-signal behavior is existing socket proof, not combined native snapshot proof. | +| Total duration | Executed injected Core/service clocks prove pre-egress plus egress timing and exclude callback/logger delay. Wire records prove correlation/presence, not deterministic elapsed equality. | +| Optional timing/other adapters | #389 remains optional; Core legacy tests and three WASM compilation checks pass. No provider runtime or portable Tokio/Send claim was added. | + +### Failed checks, exclusions and downstream gates + +Retain the initial 600s service failure, fixture compile failure, 39 test-lint failures, pre-format failure, JSON and C1 red regressions, and first combined socket suite failure as historical checks, not acceptance passes. The first socket suite passed 30 cases and failed the unsupported second-signal/custom-stop assertion. Explicit-options runners do not own convenience second-signal handling. That case was removed with the limitation disclosed, not by weakening its timing threshold or changing production behavior. The supported convenience second-signal test remains, while native counters are proven under graceful/budget-forced cleanup. + +The existing generated-workspace compile smoke and 13 Core doc examples remain ignored. The excluded example-app workspace was not tested. Docs installation reports seven existing dependency audit vulnerabilities, two moderate and five high; no docs package was changed or upgraded. Initial docs checking lacked installed packages and an early ad hoc Rust smoke used mismatched compiled dependencies; installing locked packages and correcting the smoke scaffold resolved those harness failures, not production defects. + +Accepted prerequisite integration, #394/#395 upload experiments, final image, deployed ingress/operator configuration, provider-origin/store-backend experiments, panic-abort/SIGKILL and durable log delivery remain outside this local checkpoint. No log-flush or bounded synchronous-sink latency guarantee is claimed. Keep #397 open. Prepare a draft PR stacked above #407 using `gh stack`, without an auto-closing issue keyword. diff --git a/docs/superpowers/specs/2026-10-06-axum-diagnostics-trusted-proxy-design.md b/docs/superpowers/specs/2026-10-06-axum-diagnostics-trusted-proxy-design.md new file mode 100644 index 000000000..d4c2ee5a2 --- /dev/null +++ b/docs/superpowers/specs/2026-10-06-axum-diagnostics-trusted-proxy-design.md @@ -0,0 +1,320 @@ +# Axum safe diagnostics and trusted incoming proxies + +Date: 2026-10-06 +Status: finalized design. The policy decisions below are approved, including the seven-item closing batch. Exact API signatures, configuration spellings, and integration hooks require a separate implementation-plan review. No runtime implementation or acceptance proof has been performed. + +Issue: [#397](https://github.com/stackpop/edgezero/issues/397), under [Epic #391](https://github.com/stackpop/edgezero/issues/391). + +Primary prerequisite: [PR #275](https://github.com/stackpop/edgezero/pull/275). Related timing work: [PR #389](https://github.com/stackpop/edgezero/pull/389). Coordinate portable client metadata with [#92](https://github.com/stackpop/edgezero/issues/92) and observability with [#118](https://github.com/stackpop/edgezero/issues/118) through [#122](https://github.com/stackpop/edgezero/issues/122). + +Branch: `issue-397-diagnostics-proxy-trust-spec`, created from `issue-394-395-http-acceptance-spec` at `3ab22096fd117f7506cf437b5ba128e0789e059c`. + +Worktree: `/home/pav/worktrees/issue-397-diagnostics-proxy-trust-spec`. + +The branch inherits the committed #392/#396 hosting implementation and its #275 base, `5527b58e2a7cb6307d5c7a8926cd5dfc6f4f8b4a`. The #394/#395 spec and plan were untracked when this worktree was created. They remain in their original worktree and were not copied here. Rebase onto the completed preceding work later, with approval, and refresh the affected audit and evidence. This specification does not approve the incremental-upload mechanism or bypass its experiment and implementation gates. + +## Goal and scope + +Give operators correlated evidence of native request outcomes and process shutdown. Keep framework-generated public errors and ordinary diagnostics free of source strings, secrets, bodies, and raw request targets. Make effective client, host, and scheme metadata depend on an explicit incoming-proxy trust policy. + +Keep runtime implementation in the Axum adapter. Permit only small, reviewed portable hooks needed to reuse #275's completion or expose normalized metadata to existing extractors. Fastly, Cloudflare, and Spin keep their existing behavior. + +| Owner | Responsibility | +| -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| #397 | Native diagnostic records, proxy-trust configuration, safe metadata integration, custom logger ownership, and operator documentation | +| #275 | Public error categories, ingress disposition, monotonic clock, response-owned completion, framing, and post-commit failure behavior | +| #392/#396 | Startup barrier, private drain gate, process lifecycle, prepared stores/transport, configuration loading, and shutdown budget | +| #393 | JSON buffering limit and typed HTTP 413; #397 observes its rejection without selecting a new limit | +| #394/#395 | Streaming, outbound HTTP semantics, deadlines, pool reuse, and transport cleanup; #397 consumes their safe failure categories | +| #389 | Optional request timing collector; integration requires compatible clock origins and terminal boundaries | +| #398/#399/#400 | Packaging, deployment, and later final-image acceptance using the evidence defined here | +| Application/operator | Authentication, destination and host authorization, logging/exporter setup, privacy policy for additional fields, and correct proxy/network configuration | + +Exclude a replacement observability framework, tracing vendor, metrics server, default public diagnostics endpoint, distributed tracing propagation, rate limiter, incoming TLS implementation, PROXY protocol, new response finalizer, and portable `Send` migration. Preserve the established admission, stream deadline, and shutdown contracts. + +## Approved policy decisions + +| Area | Decision | +| -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Proxy topology | Support single proxies and proxy chains, including Nginx, Cloudflare, AWS load balancers, and combinations. No provider guessing. | +| Trust | Explicit deployment address/network configuration. Default to no trust in development and production. | +| Header family | Configure one family, Forwarded or X-Forwarded-*. Ignore the other; do not combine conflicting assertions. | +| Malformed metadata | Ignore the whole selected forwarding assertion and continue using direct connection details. Do not reject an otherwise valid request for malformed optional forwarding metadata. | +| Missing metadata | Fall back individually for absent fields. Do not repair them using the unselected header family. | +| Application view | Remove original forwarding headers after checking them; expose checked effective metadata separately. | +| Host and scheme | The final trusted ingress proxy supplies checked public host/HTTPS assertions, replacing client-supplied values. Do not infer them from the client-address chain. | +| Configuration errors | Invalid supplied settings fail startup. Reject trust-every-address policies. No automatic trust for private networks or localhost. | +| Request logs | Generated request identity, method, route pattern, selected status, duration, egress-accounted bytes, and outcome. No default addresses, raw URLs/queries, headers, bodies, or source errors. | +| Logging ownership | Safe request records enabled by default with an opt-out; preserve application logger initialization and existing observers. | +| Record levels | INFO for ordinary records; WARN for noteworthy request failures; ERROR for startup/runtime faults. Include bounded failure categories rather than raw error text. | +| Request identity | Generate within EdgeZero and expose to application code. Do not adopt visitor-supplied IDs or automatically add response headers. | +| Shutdown visibility | Log lifecycle transitions and shutdown outcomes. Expose native active-request and connection snapshots, with idle connections counted separately. No default public endpoint. | +| Timing | Reuse the existing monotonic clock and completion owner. Keep #389 optional unless compatibility is proved; do not delay this issue for it. | + +These are design decisions, not claims about current runtime behavior. The acceptance checks below must prove them before #397 closes. + +## Baseline and residual gaps + +Source audit on 2026-10-06 used the branch base above. #275 was open at inspected head `950cb81d406d0dc269baf94314547329a4c1cef0`. #389 was open at `ab444946fcacb1d44c020c6079abb6ba29231402`; its source was inspected in the matching clean timing worktree. These are source facts, not acceptance results. + +| Area | Current source | Consequence for this issue | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Public errors | Core `src/error.rs`, `wire_message`, hides Internal, BadGateway, GatewayTimeout, and upstream-size details. Other application-facing categories can still carry chosen messages. Axum `src/response.rs` has fixed fallback bodies and category-only body errors. | Reuse the accepted renderer. Audit residual framework call sites; do not add another sanitizer or promise to redact arbitrary application responses. | +| Native service | Axum `src/service.rs`, `dispatch_request`, uses a private readiness/drain gate, then normalized head validation, `begin_ingress`, conversion, and admitted dispatch. | Normalize metadata before consumers. Include early returns and no-response aborts in diagnostics without changing their dispositions. | +| Forwarded host | Core `src/extractor.rs`, `ForwardedHost::from_request`, prefers raw X-Forwarded-Host over Host without checking peer trust. | A production trust filter cannot live only in a new client-IP extractor. Existing host consumers must see the normalized result. | +| Socket peer | Axum `src/context.rs`, `AxumRequestContext::remote_addr`, records the direct socket peer from ConnectInfo. | Preserve this meaning; expose asserted client information separately. | +| Completion | Core `src/response_egress.rs`, `ResponseEgressCompletion::join` and `ResponseEgressAttempt`, compose callbacks and report one terminal outcome. | Attach native diagnostics to the existing owner and preserve application callback/observer behavior. | +| Report fields | `ResponseEgressReport` has route, request start, outcome, body kind, bytes, and fallback disposition, but no request ID, method, or selected status. Its `elapsed` is measured from `egress_started_at`, not request start. | A global observer alone cannot produce the required record. Native total duration must not be mislabeled egress elapsed. Review the smallest metadata/composition hook. | +| Lifecycle | Axum `src/dev_server.rs`, `run_owned`, owns Starting/Ready/Draining/Stopped and connection tasks; connection egress tracks live response deadlines. | Observe existing state and owners. A connection count is not an active-request count. | +| Logging | `init_logging` uses simple_logger only when `Hooks::owns_logging()` is false. Native startup already emits readiness and safe failure categories. | Add records through the facade, not another logger/subscriber. | +| Timing | #389's `RequestTimings` starts its own web_time instant at construction and has explicit best-effort completion marks. | It is not automatically compatible with #275's injected clock/origin and cannot become another completion owner. | + +Re-audit the accepted #275 revision, current hosting stack, and any accepted #389 integration before planning implementation. Inventory existing error-leakage and lifecycle tests first; add only missing real-path evidence. + +## Incoming-proxy trust boundary + +### Match the transport peer + +Trust only an explicit IP/CIDR allowlist matched against the peer supplied by the owning socket service. Header text, `Forwarded by=`, request URI, and caller-supplied extensions cannot establish that identity. A missing or nonmatching transport peer means no forwarding trust. + +Never infer trust from private, loopback, container-network, or same-host addresses. Match IPv4-mapped IPv6 peers consistently with their IPv4 equivalents. The implementation plan must specify normalization of equivalent configuration entries and prove those matches. + +A trusted proxy can assert selected metadata; it does not authenticate the end user or authorize a Host value. Operators must protect the proxy-to-application network. Appending proxies must record the actual immediate predecessor. A gateway replacing the chain must supply a verified assertion. Neither can make untouched client-supplied host/scheme values trustworthy merely by forwarding them. + +[RFC 7239 sections 7.1 and 8.1](https://www.rfc-editor.org/rfc/rfc7239.txt) describe repeated/list-valued fields and warn that a trusted last proxy does not make earlier client-supplied entries trustworthy. A bare allowlist plus "take the first address" is insufficient. + +### Select and validate one header family + +Each deployment selects exactly one family: + +- `forwarded`: an ordered RFC 7239 list, including repeated field lines, with correct quoting/escaping, IPv4 or quoted IPv6 node syntax, and no duplicate parameters within an element. +- `x-forwarded`: an ordered X-Forwarded-For address chain plus singleton X-Forwarded-Host and X-Forwarded-Proto assertions written by the final trusted ingress proxy. + +Do not automatically prefer one family, combine them, or use the unselected family when selected fields are missing or malformed. Remove both raw families from the ordinary managed application view regardless of which one was accepted. + +Respect repeated-field order and existing received-head limits. The implementation plan must give a finite parsing/allocation bound derived from those limits and the selected parser, including any stricter chain bound. Do not allocate an unbounded vector of forwarding elements or accept a queue/item-count argument as a byte bound. + +Malformed syntax, ambiguous duplicate fields, invalid recognized values, or exceeding the forwarding-parser bound invalidates the whole selected assertion. Continue the otherwise valid request with direct client/host/scheme metadata. No partially accepted host or HTTPS assertion survives that malformed result. Existing HTTP head/framing violations still use their established rejection/abort path. + +Absent fields fall back individually to direct facts. Syntactically valid unknown/obfuscated client nodes do not become IP addresses and do not permit traversal past that node. Record only a bounded ignored/unresolved reason, never the raw invalid value. + +### Walk the client-address chain backward + +Start from the direct socket peer. Consume asserted predecessor addresses from right to left only while the current hop is explicitly trusted. The first untrusted address is the effective client boundary. Ignore all assertions farther left as claims about the visitor. + +For `Visitor → Cloudflare → AWS load balancer → EdgeZero`, trust the relevant load-balancer and Cloudflare networks to walk through both. If Cloudflare is not configured as trusted, its address is the effective boundary, not an invented original visitor address. A gateway may instead verify its upstream chain and provide one sanitized assertion. + +Never use universal leftmost selection or a fixed hop count. Missing predecessor information, an unknown/obfuscated node, or exhaustion while every represented address is still trusted cannot establish another original visitor. Stop and use direct client fallback, marked as unresolved. Do not skip the gap. A malformed address invalidates the whole selected assertion under the rule above. + +Conceptual pseudocode, not implementation: + +```text +client_boundary(peer, ordered_predecessors, trusted_networks): + if peer is absent or peer is not trusted: + return direct_client(peer) + current = peer.ip + for predecessor in ordered_predecessors from right to left: + if predecessor has no usable numeric IP: + return direct_client(peer, reason = unresolved_chain) + current = predecessor.ip + if current is not trusted: + return asserted_client(current) + return direct_client(peer, reason = unresolved_chain) +``` + +Parse/validate the selected assertion before this walk. This pseudocode does not authorize skipping malformed input or borrowing another header family. A boundary address is not proof of the original visitor when another untrusted proxy stands in front of it. + +### Public host and HTTPS assertions + +The final trusted ingress proxy is responsible for supplying verified public host and scheme information. For Forwarded, use host/proto only from its final, rightmost element. For X-Forwarded-*, use only its singleton host/proto fields. Do not zip independent lists or search earlier hops for convenient values. + +The operator must configure that proxy to overwrite those assertions and, when needed, verify the public values received through upstream proxies. CIDR matching alone cannot prove that proxy configuration. Document this deployment obligation for each supported example topology. + +Only accept http/https as effective schemes. Validate host as an authority without userinfo, path, query, or fragment, and validate any numeric port. Missing host/proto fields use direct authority/transport scheme. Invalid values invalidate the whole selected assertion. + +For example, a visitor may use HTTPS while the load balancer connects to EdgeZero over HTTP. EdgeZero may expose HTTPS only through the checked final-proxy assertion. It must not infer HTTPS from a client-controlled absolute URI or from the visitor-address chain. + +### Checked metadata and application compatibility + +Preserve direct facts and per-field provenance separately from effective values: + +```text +DirectPeer = known socket address | unavailable +EffectiveClient = direct IP | trusted-proxy asserted boundary IP | unavailable +EffectiveHost = validated direct authority | trusted-proxy asserted authority | unavailable +EffectiveScheme = known transport scheme | trusted-proxy asserted http/https +FieldSource = direct | trusted_forwarded | trusted_x_forwarded | unavailable +ForwardingResult = absent | accepted | partial | untrusted_peer | malformed | unresolved_chain +``` + +This is a conceptual model, not a public type layout. Coordinate naming and portable access with #92. Preserve `AxumRequestContext::remote_addr` as the direct peer. The direct authority comes from the validated request head; the current direct transport scheme is HTTP. Host metadata remains separate from application host authorization. + +Validate/account for the received head before removing forwarding fields so stripping cannot bypass head-size limits. Normalize before admission policy, middleware, handlers, or host/scheme/client metadata consumers. Carry the checked ingress facts in a bounded metadata value. + +Make `ForwardedHost` use that checked value on the managed native path, including accepted Forwarded host assertions. Retain its existing fallback behavior for other adapters. Do not rewrite Host or the request URI merely to change an extractor result. + +Remove original Forwarded and X-Forwarded-* headers after checking, including accepted trusted-peer assertions. Applications that inspect or forward those raw fields on the managed native path must migrate to checked metadata. Do not preserve unsafe values under another name or automatically propagate them outbound. Raw access or outbound assertion generation would require a separate explicit decision. + +Checked metadata describes ingress, not later application mutations. Framework policy cannot stop hostile application code synthesizing headers or bypassing the managed host; the guarantee applies to the supported ingress boundary and normal metadata accessors. + +```mermaid +flowchart TD + A[Owned socket peer and received HTTP head] --> B[Existing head validation and accounting] + B --> C[Explicit peer trust and selected family] + C --> D[Bounded validation and client-chain walk] + D --> E[Checked effective facts with provenance] + E --> F[Remove original forwarding headers] + F --> G[Existing admission policy] + G --> H[Middleware and metadata extractors] + H --> I[Handler and existing egress owner] +``` + +Head-validation failure still uses the existing detached error/abort path. Rendering that error safely does not require forwarded metadata. + +### Configuration and bypass rules + +Configure trusted literal addresses/network ranges and the selected family through standalone environment settings or explicit hosting APIs. Load standalone environment once. Explicit embedding uses its supplied policy and does not silently merge environment values. Provide the same no-trust default in development and production. No manifest section or CLI flag is required. + +The following spellings are implementation candidates, not approved public API names: + +| Setting | Required behavior | +| --------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS` | Literal IP/CIDR list. Unset or an explicitly empty list means no trust. No DNS lookups or provider/private/all tokens. | +| `EDGEZERO__ADAPTER__FORWARDING_HEADER_FAMILY` | Explicit `forwarded` or `x-forwarded` when trust is nonempty. A valid family with an empty trust list does not enable trust. | +| Explicit hosting API | Validated policy installed in production, development, and embedded managed hosting. Final signatures are part of plan review. | + +Validate supplied settings before Ready. Invalid entries, unknown modes, a nonempty trust list without a selected family, and trust-every-address policies fail startup with safe configuration categories, not input values or silent fallback. Reject `/0` entries and equivalent allowlists covering an entire address family. No dangerous trust-all override is included. + +Low-level `into_core_request` bypasses production admission and the owning socket boundary today. Document that bypass. It cannot claim production proxy trust merely because a request carries ConnectInfo or an environment variable exists. This issue does not require a new policy-aware low-level converter; supported trust guarantees belong to managed hosting. Any additional converter API requires plan review. + +## Native request diagnostics + +### Identity, clock, and sole completion owner + +Create a bounded framework-owned identity at the native parsed-request service boundary before admission can return early. It must distinguish requests across concurrently running hosting instances and within each instance. The concrete generated representation, sequence-overflow behavior, and public accessor are implementation-plan details. + +Expose the generated identity to application code through checked request metadata. Retain a private authoritative copy for framework records so overwriting a public extension cannot change correlation. Never adopt an incoming X-Request-ID as that identity. Do not automatically add a response header or propagate tracing headers; applications may expose an ID deliberately. + +Use the same #275 monotonic clock and request-start snapshot used by native ingress. Define total request duration from that start to the authoritative terminal observation. The audited `ResponseEgressReport::elapsed` measures egress only and must retain its existing meaning. It is not total request duration. + +The implementation plan must select a narrow way to obtain the terminal snapshot or equivalent total duration at the existing transition, before potentially slow application callbacks or logger output. Sampling after callbacks would mismeasure request duration. Do not add another clock origin, deadline, timer task, or terminal owner to obtain the timestamp. + +For each response attempt, emit one native terminal record from #275's existing completion boundary. Compose with application completion using `ResponseEgressCompletion::join` or its accepted equivalent. Preserve application invocation counts, ordering, and panic isolation; do not replace the application observer or admission policy with telemetry. + +No-response admission Abort, the private draining gate, and cancellation before an egress attempt need safe ingress/abandonment records and resource release. They must not receive a fabricated ResponseEgressReport, HTTP status, or successful completion callback. Unbegun envelope abandonment retains #275's no-report distinction. Process abort and SIGKILL provide no callback or log-flush guarantee. + +Capture identity and method privately; carry route, typed rejection category, and selected status through the existing owner. A global observer alone lacks these facts in the audited baseline. A pre-conversion application status is not authoritative when framing failure selects a fallback. Do not infer rejection from status alone; an application can deliberately return 413 or 503. + +### Fields, levels, and opt-out + +Enable safe request records by default through the existing log facade. Provide an explicit request-record opt-out for applications that already produce their own records. `Hooks::owns_logging()` controls logger initialization, not whether framework facade events exist. Never install a competing logger/subscriber or overwrite custom observers. + +| Field | Default rule | +| -------------------------- | ---------------------------------------------------------------------------------------------------------------- | +| Event and request identity | Fixed event name and generated bounded identity | +| Method | Known method or bounded other category; no arbitrary token dump | +| Route | Registered route template, never raw path or parameters; fixed unmatched/unknown value when unresolved | +| Status | Final selected response status, absent for no-response paths; not proof of receipt | +| Duration | Total monotonic request duration at the existing terminal boundary; not egress-only elapsed or log-emission time | +| Outcome | Existing bounded egress outcome or separately named ingress disposition | +| Rejection/failure category | Typed bounded reason from its owning boundary; never source-message matching | +| Response accounting | Egress-accounted bytes, existing body kind, and fallback disposition; never claim delivered TCP bytes | +| Forwarding result | Bounded accepted/ignored/unresolved reason, without raw values or client address | + +For example, record `GET /orders/{id} status=200 duration=42ms outcome=completed`, not `/orders/123?token=secret` or the visitor's address. This illustrates content, not an approved serialization format. + +Use INFO for ordinary completions and lifecycle transitions; WARN for noteworthy request rejections/failures, including upstream/stream failure and ignored malformed forwarding assertions; ERROR for startup/runtime faults. Choose request severity from the typed disposition/outcome, not HTTP status alone. An application-selected status does not automatically become a framework fault. Record a malformed-forwarding reason with the request outcome rather than duplicating terminal records. + +Omit client/peer IPs, hostnames, raw paths, queries, headers, cookies, authorization values, secret/config values, bodies, arbitrary application timing data, origin URLs, and provider error strings. Trusted proxy metadata is not permission to log it. Debug level is not permission to dump secrets. Registered route templates are application-owned diagnostic names and must not contain secrets. + +Applications may deliberately add fields through their own observer/logger and privacy policy. Do not grow a generic field-selection/redaction framework. Document application metrics integration using existing observers plus safe native hooks for bounded route/status/outcome counters and durations. IDs, addresses, bodies, and secret values are never default metric labels. Install no exporter, metrics server, or public endpoint. + +The request-record opt-out suppresses default native per-request records, including no-response/abandonment records. It does not disable application observers, lifecycle events, resource accounting, or snapshot access. + +### Selected status and failures after headers + +A selected 200 response can terminate with SourceError, DeadlineExceeded, ClientDisconnected, or TransportError. Keep both the selected status and the failing terminal outcome. Never call it successful merely because the handler returned 200. + +For precommit framing/policy failure, report the selected fixed 500/504 fallback status, original cause, body kind, and completed/aborted disposition. Never replace a committed response with a second error body to improve diagnostics. + +Distinguish selected status from proven wire commitment. The body-poll boundary does not prove a headers-sent timestamp or end-client receipt. Acceptance fixtures must observe the actual wire separately. The plan must identify where final status becomes authoritative without another finalizer. + +### Public errors and residual leakage + +Reuse #275's category-only internal/upstream renderer and fixed native egress fallback bodies. Audit conversion, route/dispatch errors, upstream failures, body-source errors, framing fallback, and startup/shutdown errors for residual exposure. Log typed categories, not Display/Debug of error sources or requests. Safe reason labels such as `invalid_forwarding` are permitted; raw source strings are not. + +The accepted renderer intentionally preserves some application-controlled messages in validation and related categories. Do not claim every EdgeError message is redacted. Framework call sites must use safe fixed messages where they create those categories. Custom renderers and application-generated responses remain application policy and can override the safe defaults. + +No source detail is permitted in ordinary framework request records. Any separately requested internal detail requires a named field and explicit privacy approval. Regex masking after formatting a raw error is not a substitute for bounded categories. + +## Active work and shutdown visibility + +Observe the existing Starting/Ready/Draining/Stopped owner and private admission gate. Emit transition/failure events at their actual boundaries, including startup failure before Ready, graceful completion, grace exhaustion, and forced stop. Use fixed reasons and configured durations, not supplied environment values or source errors. + +Expose a read-only native snapshot of active managed requests and connections, with shutdown summaries through the facade. Reuse compatible #118 work rather than creating another registry. Expose no public diagnostics endpoint by default. + +Define the counts precisely: + +- Active requests count managed parsed requests in admission, body read, handler execution, and response egress until the managed lifetime guard is released by terminal completion or pre-attempt abandonment. +- Active connections count the runner's owned connection tasks. Idle keep-alive connections count here but not as active requests. +- Application-spawned work, arbitrary detached tasks, kernel buffers, and external services are outside these counts. + +Use a request-scoped accounting guard moved with the existing completion owner. Drop releases accounting on cancellation or abandonment before an attempt; terminal completion releases it on response paths. The guard must not manufacture another egress terminal state. Preserve application resource release and release accounting even if a diagnostic callback panics. Document snapshot consistency under concurrency; do not imply a cross-counter transactional snapshot without providing one. + +Diagnostics add no async producer, retry loop, unbounded queue, or flush wait beyond the existing shutdown budget. A user-supplied synchronous logger can block; the runtime cannot promise bounded logger delivery. Document that limitation, and retain existing process-abort/no-flush distinctions. + +## Optional #389 timing integration + +#389 currently starts a collector clock at construction, not native ingress, and uses best-effort explicit marks. Do not merge unrelated origins, restart deadlines, finalize at middleware return, or substitute that collector for #275's request/completion owner. + +Before native integration, prove a compatible accepted clock/origin and marks from the existing terminal boundary. Otherwise retain optional application phase timing and obtain native duration through the same #275 clock/transition. #397 can close without adopting the collector. Never copy arbitrary collector data into default logs, response headers, or labels. + +## Acceptance evidence + +Exercise the supported production/managed native host over real HTTP/1 sockets with local fixtures and a captured test log sink. Parser/converter tests alone do not prove the trust boundary. Reuse hosting, service, and contract fixtures; do not add a second server. + +| Case | Required observation | +| ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Default no trust | Spoofed Forwarded/X-Forwarded-* cannot change effective client/host/scheme, including ForwardedHost. Original forwarding headers are absent from the managed application view. | +| Explicit trusted peer | Matching peers supply accepted selected-family metadata. Nonmatching peers get direct fallback. Preserve the direct socket address. | +| Peer unavailable | Headers/extensions cannot grant trust when managed transport peer information is absent. Document low-level bypass separately. | +| Family selection | Conflicting families, missing selected fields, and an apparently valid unselected family follow configured selection with no borrowing. | +| Parsing | Repeated field order, lists, quoting/escaping, IPv6, duplicates, unknown/obfuscated nodes, invalid ports/schemes/authorities, and parser limits follow deterministic policy. | +| Spoofed prefix | Local single-/multi-proxy fixtures cannot select attacker-added prefixes beyond the first untrusted hop. Cover partial trust, gaps, all-trusted exhaustion, and a gateway replacing the chain. | +| Malformed whole-set fallback | A trusted proxy supplies a bad client address plus otherwise valid host/HTTPS assertions. The request continues, but all selected forwarding metadata falls back to direct facts, with a safe reason and no raw value. | +| Missing/unknown values | Absent fields fall back individually. Unknown nodes or missing predecessor links cannot enable traversal past the gap. No fallback address is labeled as a verified original visitor. | +| Host/HTTPS provenance | Final-proxy assertions survive a controlled HTTPS-at-ingress/HTTP-at-EdgeZero deployment. Client-supplied values and earlier-hop host/proto assertions cannot override them. | +| Configuration | Empty/default trust, CIDR edges, mapped IPv6 equivalence, invalid entries/modes, missing family, `/0`, and equivalent trust-all policies are exercised. Invalid configuration fails before Ready. | +| Consumer order and compatibility | Admission, middleware, handlers, and ForwardedHost agree on checked facts. Received-head accounting does not shrink after stripping. Raw-header applications have documented migration. | +| Safe errors and records | Sentinel secrets in conversion, internal/dispatch, upstream, and source failures appear in neither default client errors nor ordinary logs. Also test query/path parameters, headers, secret/config values, and payloads. | +| Ordinary correlation and levels | One generated ID correlates safe method/route/status/duration/bytes/outcome. Log `/orders/{id}`, not its supplied ID. Typed outcomes select levels; deliberate application statuses are not invented framework faults. | +| Identity ownership | Concurrent hosting/request identities do not collide within the specified scope. Incoming IDs/public-extension mutation cannot change the framework ID. Application access works; no automatic response ID header appears. | +| Rejection and Abort | #393 413, admitted refusal, head failure, no-response Abort, and draining refusal expose bounded reasons without invented status/completion. | +| Failure after headers | Client observes head and prefix, then source/deadline/disconnect failure. One terminal record retains selected status plus failure outcome; no second response. | +| Fallback and abandonment | Precommit failure reports fallback status/cause. Unbegun envelope/drop/cancellation preserves report/no-report distinctions and releases work. | +| Application logging | App-owned logger gets facade records without competing initialization. Custom admission, completion, and observer hooks retain counts/order and unwind panic isolation. | +| Request-record opt-out | Default per-request records stop; application observers, lifecycle events, accounting, and snapshots still operate. | +| Active work and shutdown | Hold body/handler/stream work and idle keep-alive connections. Snapshots distinguish requests/connections. Graceful drain and forced stop release managed work and report outcomes without added budget. | +| Total duration | Inject separate pre-egress and egress delays. Record total native duration on the established clock, not egress-only elapsed or callback/logger delay. | +| Optional timing and other adapters | Optional timing cannot replace the native clock/completion owner. Small shared hooks preserve nonnative behavior and add no native dependencies or Send requirements to portable code. | + +Test record cardinality under concurrent requests, refusal, cancellation, and fallback. Document any log-sink drops; claim no durable delivery guarantee. + +Current evidence for the two highest-risk assumptions remains Source/Path only and unproven: + +1. Every managed metadata consumer uses the peer-gated checked view. The next proof is a production socket request with spoofed forwarding fields observed by admission, middleware, and ForwardedHost, including a local proxy chain. +2. Native diagnostics preserve the sole terminal owner and application hooks. The next proof is a delayed response failing after its head, with native and application sinks asserting exact counts, final status, total duration, and cleanup. + +Record command, exact tested revision and uncommitted inputs, environment, result, elapsed time, and captured evidence. Refresh affected proofs after prerequisite integration or rebase. Markdown formatting is not runtime evidence. + +## Implementation-plan gate + +Policy discussion is complete. Before production code: + +1. Rebase onto the approved completed stack and re-audit accepted #275 error/framing/completion behavior. Record existing versus missing coverage, including egress elapsed versus total duration. +2. Present the smallest native implementation and compatibility footprint. Specify parser/config normalization and finite bounds; finalize environment spellings, hosting/accessor signatures, ID representation, and snapshot consistency. +3. Identify narrow shared hooks for checked extractor metadata and completion attachment/status/terminal timing. Preserve observer order, abandonment distinctions, and existing report semantics. Do not implement a second finalizer. +4. Keep #389 optional unless compatible evidence exists. Its acceptance must not introduce another clock origin or block native diagnostics unnecessarily. +5. Write and obtain approval of a separate implementation plan with the acceptance fixtures above and the repository's required test, formatting, lint, and feature/WASM checks. + +No implementation plan, prototype, runtime changes, commits, or issue-closing evidence is authorized by finalizing this specification. Earlier specifications and the #394 upload proof gate remain unchanged.