diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index 7972b34700..1bfb1c2aaa 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -14,6 +14,11 @@ repository.workspace = true name = "openshell-sandbox" path = "src/main.rs" +[[bin]] +name = "openshell-seccomp-perf" +path = "src/bin/seccomp-perf.rs" +required-features = ["perf-harness"] + [features] perf-harness = [] diff --git a/crates/openshell-sandbox/src/bin/seccomp-perf.rs b/crates/openshell-sandbox/src/bin/seccomp-perf.rs new file mode 100644 index 0000000000..f1354087cb --- /dev/null +++ b/crates/openshell-sandbox/src/bin/seccomp-perf.rs @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Microbenchmark entry point for the production seccomp network broker. + +use std::net::SocketAddr; + +use clap::{Parser, Subcommand}; +use openshell_sandbox::perf::{BenchmarkOptions, Layer, Protocol}; + +#[derive(Debug, Parser)] +#[command( + about = "Measure native and seccomp-filtered socket performance", + long_about = "Measure native and seccomp-filtered socket performance. UDP unconnected means destination-bearing SOCK_DGRAM traffic, not SOCK_RAW. General external UDP and SOCK_RAW are currently denied by the sandbox." +)] +struct Cli { + /// Benchmark layer: native, filtered, or all. + #[arg(long, default_value = "all", value_parser = ["native", "filtered", "all"])] + layer: String, + /// Protocol: tcp-connect, tcp-stream, udp-connected, udp-unconnected, or all. + #[arg( + long, + default_value = "all", + value_parser = ["tcp-connect", "tcp-stream", "udp-connected", "udp-unconnected", "all"] + )] + protocol: String, + #[arg(long, default_value_t = 10_000)] + iterations: u64, + #[arg(long, default_value_t = 1_000)] + warmup: u64, + #[arg(long, default_value_t = 1)] + concurrency: usize, + #[arg(long, default_value_t = 64)] + payload_bytes: usize, + #[command(subcommand)] + command: Option, +} + +#[derive(Debug, Subcommand)] +enum Command { + #[command(hide = true)] + Worker { + #[arg(long)] + protocol: Protocol, + #[arg(long)] + target: SocketAddr, + #[arg(long)] + iterations: u64, + #[arg(long)] + warmup: u64, + #[arg(long)] + concurrency: usize, + #[arg(long)] + payload_bytes: usize, + }, +} + +fn main() -> anyhow::Result<()> { + let cli = Cli::parse(); + if let Some(Command::Worker { + protocol, + target, + iterations, + warmup, + concurrency, + payload_bytes, + }) = cli.command + { + let report = openshell_sandbox::perf::run_worker( + protocol, + target, + iterations, + warmup, + concurrency, + payload_bytes, + )?; + println!("{}", serde_json::to_string(&report)?); + return Ok(()); + } + + let options = BenchmarkOptions { + layers: Layer::selection(&cli.layer)?, + protocols: Protocol::selection(&cli.protocol)?, + iterations: cli.iterations, + warmup: cli.warmup, + concurrency: cli.concurrency, + payload_bytes: cli.payload_bytes, + }; + for report in openshell_sandbox::perf::run(options)? { + println!("{}", serde_json::to_string(&report)?); + } + Ok(()) +} diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 78bc3d2629..4928fe6f40 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -18,6 +18,8 @@ pub mod main_session; pub mod managed_children; #[cfg(target_os = "linux")] mod network_broker; +#[cfg(all(target_os = "linux", feature = "perf-harness"))] +pub mod perf; #[cfg(unix)] pub mod process; mod pty; diff --git a/crates/openshell-sandbox/src/perf.rs b/crates/openshell-sandbox/src/perf.rs new file mode 100644 index 0000000000..cc3b7da10f --- /dev/null +++ b/crates/openshell-sandbox/src/perf.rs @@ -0,0 +1,329 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Opt-in microbenchmarks for the production seccomp network listener. + +use std::io::{self, Read as _, Write as _}; +use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener, TcpStream}; +use std::process::{Command, Stdio}; +use std::sync::{Arc, Barrier, mpsc}; +use std::thread; +use std::time::Instant; + +use anyhow::{Context as _, bail}; +use clap::ValueEnum; +use openshell_isolation_interface::linux::workload_launcher; +use serde::{Deserialize, Serialize}; +use socket2::{Domain, Socket, Type}; + +use crate::network_broker::NetworkBroker; + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, ValueEnum)] +#[serde(rename_all = "kebab-case")] +pub enum Layer { + Native, + Filtered, +} + +impl Layer { + pub fn selection(value: &str) -> anyhow::Result> { + match value { + "all" => Ok(vec![Self::Native, Self::Filtered]), + "native" => Ok(vec![Self::Native]), + "filtered" => Ok(vec![Self::Filtered]), + _ => bail!("unknown layer {value}"), + } + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, ValueEnum)] +#[serde(rename_all = "kebab-case")] +pub enum Protocol { + TcpConnect, + TcpStream, +} + +impl Protocol { + pub fn selection(value: &str) -> anyhow::Result> { + match value { + "all" => Ok(vec![Self::TcpConnect, Self::TcpStream]), + _ => Ok(vec![ + Self::from_str(value, true).map_err(|error| anyhow::anyhow!("{error}"))?, + ]), + } + } +} + +#[derive(Debug)] +pub struct BenchmarkOptions { + pub layers: Vec, + pub protocols: Vec, + pub iterations: u64, + pub warmup: u64, + pub concurrency: usize, + pub payload_bytes: usize, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct BenchmarkReport { + pub layer: Layer, + pub protocol: Protocol, + pub iterations: u64, + pub concurrency: usize, + pub payload_bytes: usize, + pub capability_scope: String, + pub elapsed_ms: f64, + pub operations_per_second: f64, + pub throughput_mbit_per_second: f64, + pub latency_ns_p50: u64, + pub latency_ns_p95: u64, + pub latency_ns_p99: u64, +} + +pub fn run(options: BenchmarkOptions) -> anyhow::Result> { + if options.iterations == 0 || options.concurrency == 0 { + bail!("iterations and concurrency must be greater than zero"); + } + if options.payload_bytes == 0 || options.payload_bytes > 65_507 { + bail!("payload-bytes must be between 1 and 65507"); + } + + let executable = std::env::current_exe().context("resolve benchmark executable")?; + let mut reports = Vec::new(); + let mut filtered_runtime = None; + for layer in options.layers { + if matches!(layer, Layer::Filtered) && filtered_runtime.is_none() { + let (launcher, listener) = workload_launcher::start()?; + let broker = NetworkBroker::start_for_test(listener)?; + broker.confirm_healthy()?; + filtered_runtime = Some((launcher, broker)); + } + for protocol in &options.protocols { + let fixture = Fixture::start(*protocol, filtered_runtime.as_ref().map(|(_, b)| b))?; + let mut command = Command::new(&executable); + command + .arg("worker") + .arg("--protocol") + .arg( + protocol + .to_possible_value() + .expect("protocol value") + .get_name(), + ) + .arg("--target") + .arg(fixture.target.to_string()) + .arg("--iterations") + .arg(options.iterations.to_string()) + .arg("--warmup") + .arg(options.warmup.to_string()) + .arg("--concurrency") + .arg(options.concurrency.to_string()) + .arg("--payload-bytes") + .arg(options.payload_bytes.to_string()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()); + let output = match layer { + Layer::Native => command.output(), + Layer::Filtered => filtered_runtime + .as_ref() + .expect("filtered runtime") + .0 + .execute(move || command.output())?, + } + .context("run benchmark worker")?; + if !output.status.success() { + bail!("benchmark worker exited with {}", output.status); + } + let mut report: BenchmarkReport = + serde_json::from_slice(&output.stdout).context("decode worker report")?; + report.layer = layer; + reports.push(report); + } + } + Ok(reports) +} + +struct Fixture { + target: SocketAddr, +} + +impl Fixture { + fn start(protocol: Protocol, _broker: Option<&NetworkBroker>) -> io::Result { + match protocol { + Protocol::TcpConnect => start_tcp_fixture(false), + Protocol::TcpStream => start_tcp_fixture(true), + } + } +} + +fn start_tcp_fixture(echo: bool) -> io::Result { + let socket = Socket::new(Domain::IPV4, Type::STREAM, None)?; + socket.set_reuse_address(true)?; + socket.bind(&SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0).into())?; + socket.listen(4_096)?; + let listener: TcpListener = socket.into(); + let target = listener.local_addr()?; + thread::Builder::new() + .name("seccomp-perf-tcp-fixture".into()) + .spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { break }; + let _ = stream.set_nodelay(true); + if echo { + let _ = thread::Builder::new() + .name("seccomp-perf-tcp-echo".into()) + .spawn(move || { + let mut buffer = vec![0_u8; 65_507]; + while let Ok(length) = stream.read(&mut buffer) { + if length == 0 || stream.write_all(&buffer[..length]).is_err() { + break; + } + } + }); + } + } + })?; + Ok(Fixture { target }) +} + +#[allow(clippy::cast_precision_loss)] +pub fn run_worker( + protocol: Protocol, + target: SocketAddr, + iterations: u64, + warmup: u64, + concurrency: usize, + payload_bytes: usize, +) -> anyhow::Result { + let barrier = Arc::new(Barrier::new(concurrency + 1)); + let (sender, receiver) = mpsc::channel(); + let mut threads = Vec::with_capacity(concurrency); + for _ in 0..concurrency { + let barrier = Arc::clone(&barrier); + let sender = sender.clone(); + threads.push(thread::spawn(move || { + let result = worker_loop( + protocol, + target, + iterations, + warmup, + payload_bytes, + &barrier, + ); + let _ = sender.send(result); + })); + } + drop(sender); + let started = Instant::now(); + barrier.wait(); + let mut samples = Vec::new(); + for result in receiver { + samples.extend(result?); + } + let elapsed = started.elapsed(); + for worker in threads { + worker + .join() + .map_err(|_| anyhow::anyhow!("worker panicked"))?; + } + samples.sort_unstable(); + let operations = iterations.saturating_mul(concurrency as u64); + let seconds = elapsed.as_secs_f64(); + Ok(BenchmarkReport { + layer: Layer::Native, + protocol, + iterations: operations, + concurrency, + payload_bytes, + capability_scope: protocol.capability_scope().to_string(), + elapsed_ms: seconds * 1_000.0, + operations_per_second: operations as f64 / seconds, + throughput_mbit_per_second: operations as f64 * payload_bytes as f64 * 8.0 + / seconds + / 1_000_000.0, + latency_ns_p50: percentile(&samples, 50), + latency_ns_p95: percentile(&samples, 95), + latency_ns_p99: percentile(&samples, 99), + }) +} + +impl Protocol { + const fn capability_scope(self) -> &'static str { + match self { + Self::TcpConnect => "implemented local TCP socket/connect interception", + Self::TcpStream => "implemented established TCP fast path", + } + } +} + +fn worker_loop( + protocol: Protocol, + target: SocketAddr, + iterations: u64, + warmup: u64, + payload_bytes: usize, + barrier: &Barrier, +) -> anyhow::Result> { + let payload = vec![0x5a; payload_bytes]; + let mut tcp = if matches!(protocol, Protocol::TcpStream) { + let stream = TcpStream::connect(target)?; + let _ = stream.set_nodelay(true); + Some(stream) + } else { + None + }; + let mut response = vec![0_u8; payload_bytes]; + for _ in 0..warmup { + one_operation(protocol, target, &payload, &mut response, tcp.as_mut())?; + } + barrier.wait(); + let mut samples = Vec::with_capacity(usize::try_from(iterations).unwrap_or(0)); + for _ in 0..iterations { + let started = Instant::now(); + one_operation(protocol, target, &payload, &mut response, tcp.as_mut())?; + samples.push(u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX)); + } + Ok(samples) +} + +fn one_operation( + protocol: Protocol, + target: SocketAddr, + payload: &[u8], + response: &mut [u8], + tcp: Option<&mut TcpStream>, +) -> io::Result<()> { + match protocol { + Protocol::TcpConnect => { + let stream = TcpStream::connect(target)?; + let _ = stream.set_nodelay(true); + } + Protocol::TcpStream => { + let stream = tcp.expect("TCP stream initialized"); + stream.write_all(payload)?; + stream.read_exact(response)?; + } + } + Ok(()) +} + +fn percentile(samples: &[u64], percentile: usize) -> u64 { + if samples.is_empty() { + return 0; + } + let index = (samples.len() - 1) * percentile / 100; + samples[index] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn percentiles_are_stable() { + let samples = (1..=100).collect::>(); + assert_eq!(percentile(&samples, 50), 50); + assert_eq!(percentile(&samples, 95), 95); + assert_eq!(percentile(&samples, 99), 99); + } +} diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 8eb3403c61..189e65ad8a 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -243,3 +243,7 @@ module_name_repetitions = "allow" must_use_candidate = "allow" missing_errors_doc = "allow" missing_panics_doc = "allow" +[[test]] +name = "live_internet_traffic_perf" +path = "tests/live_internet_traffic_perf.rs" +required-features = ["e2e-host-gateway"] diff --git a/e2e/rust/tests/internet_network_perf.rs b/e2e/rust/tests/internet_network_perf.rs new file mode 100644 index 0000000000..70c0f1af25 --- /dev/null +++ b/e2e/rust/tests/internet_network_perf.rs @@ -0,0 +1,170 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Opt-in Internet benchmark for the complete sandbox-to-supervisor data path. + +#![cfg(feature = "e2e-host-gateway")] + +use std::io::Write as _; +use std::process::Stdio; + +use openshell_e2e::harness::sandbox::SandboxGuard; +use tempfile::NamedTempFile; + +const BENCHMARK: &str = r#" +import http.client +import json +import os +import socket +import ssl +import statistics +import time + +HOST = "example.com" +ITERATIONS = int(os.environ.get("OPENSHELL_INET_PERF_ITERATIONS", "20")) + +def percentile(values, fraction): + values = sorted(values) + return values[min(len(values) - 1, int((len(values) - 1) * fraction))] + +def measure(name, operation, iterations=ITERATIONS): + operation() + samples = [] + started = time.perf_counter_ns() + for _ in range(iterations): + before = time.perf_counter_ns() + operation() + samples.append((time.perf_counter_ns() - before) / 1_000_000) + elapsed = (time.perf_counter_ns() - started) / 1_000_000_000 + return { + "name": name, + "iterations": iterations, + "mean_ms": statistics.fmean(samples), + "p50_ms": percentile(samples, 0.50), + "p95_ms": percentile(samples, 0.95), + "ops_per_second": iterations / elapsed, + } + +def dns_lookup(): + result = socket.getaddrinfo(HOST, 443, socket.AF_INET, socket.SOCK_STREAM) + if not result: + raise RuntimeError("DNS returned no IPv4 addresses") + +def tcp_connect(): + with socket.create_connection((HOST, 443), timeout=10): + pass + +tls_context = ssl.create_default_context() + +def https_cold(): + connection = http.client.HTTPSConnection(HOST, 443, timeout=10, context=tls_context) + try: + connection.request("HEAD", "/", headers={"Connection": "close"}) + response = connection.getresponse() + response.read() + if response.status != 200: + raise RuntimeError(f"unexpected HTTP status {response.status}") + finally: + connection.close() + +warm_connection = http.client.HTTPSConnection(HOST, 443, timeout=10, context=tls_context) + +def https_reuse(): + warm_connection.request("HEAD", "/", headers={"Connection": "keep-alive"}) + response = warm_connection.getresponse() + response.read() + if response.status != 200: + raise RuntimeError(f"unexpected HTTP status {response.status}") + +try: + metrics = [ + measure("dns_lookup", dns_lookup), + measure("tcp_connect", tcp_connect), + measure("https_cold", https_cold, max(5, ITERATIONS // 2)), + measure("https_reuse", https_reuse), + ] +finally: + warm_connection.close() + +print(json.dumps({"metrics": metrics}, separators=(",", ":"))) +"#; + +fn write_policy() -> Result { + let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; + write!( + file, + r#"version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] + +landlock: + compatibility: best_effort + +network_policies: + internet_performance: + name: internet_performance + endpoints: + - host: example.com + port: 80 + protocol: tcp + - host: example.com + port: 443 + protocol: tcp + binaries: + - path: "/**" +"# + ) + .map_err(|error| format!("write policy: {error}"))?; + file.flush() + .map_err(|error| format!("flush policy: {error}"))?; + Ok(file) +} + +async fn run_host_benchmark() -> Result { + let output = tokio::process::Command::new("python3") + .args(["-c", BENCHMARK]) + .env("OPENSHELL_INET_PERF_ITERATIONS", "20") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .map_err(|error| format!("run host benchmark: {error}"))?; + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + if !output.status.success() { + return Err(format!("host benchmark failed: {combined}")); + } + Ok(combined) +} + +#[tokio::test] +#[ignore = "manual Internet performance benchmark"] +async fn benchmark_complete_internet_path() { + let policy = write_policy().expect("write Internet benchmark policy"); + let policy_path = policy.path().to_str().expect("UTF-8 policy path"); + let sandbox = SandboxGuard::create(&["--policy", policy_path]) + .await + .expect("create benchmark sandbox"); + + for round in 1..=3 { + let host = run_host_benchmark().await.expect("host benchmark"); + println!("INTERNET_PERF host round={round} {}", host.trim()); + let mediated = sandbox + .exec(&[ + "sh", + "-c", + "OPENSHELL_INET_PERF_ITERATIONS=20 python3 -c \"$1\"", + "openshell-internet-perf", + BENCHMARK, + ]) + .await + .expect("sandbox benchmark"); + println!("INTERNET_PERF sandbox round={round} {}", mediated.trim()); + } +} diff --git a/e2e/rust/tests/live_internet_traffic_perf.rs b/e2e/rust/tests/live_internet_traffic_perf.rs new file mode 100644 index 0000000000..e8727bbce3 --- /dev/null +++ b/e2e/rust/tests/live_internet_traffic_perf.rs @@ -0,0 +1,337 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Opt-in live-Internet benchmark shaped like a short coding-agent session. + +#![cfg(feature = "e2e-host-gateway")] + +use std::io::Write as _; +use std::process::Stdio; +use std::time::Instant; + +use openshell_e2e::harness::sandbox::SandboxGuard; +use tempfile::NamedTempFile; + +const DEFAULT_ROUNDS: usize = 10; + +const BENCHMARK: &str = r#" +import concurrent.futures +import http.client +import json +import os +import shutil +import socket +import ssl +import statistics +import subprocess +import tempfile +import time +import urllib.parse + +MODE = os.environ.get("OPENSHELL_LIVE_PERF_MODE", "direct") +TLS = ssl.create_default_context() +USER_AGENT = "OpenShell-live-Internet-benchmark/1" + +METADATA_URLS = [ + "https://raw.githubusercontent.com/NVIDIA/OpenShell/main/README.md", + "https://pypi.org/pypi/requests/json", + "https://registry.npmjs.org/typescript/latest", + "https://docs.python.org/3/", +] +DNS_HOSTS = [ + "github.com", + "raw.githubusercontent.com", + "pypi.org", + "registry.npmjs.org", + "crates.io", + "docs.python.org", + "speed.cloudflare.com", +] + +def percentile(values, fraction): + values = sorted(values) + return values[min(len(values) - 1, int((len(values) - 1) * fraction))] + +def request(url, method="GET", max_bytes=None): + parsed = urllib.parse.urlsplit(url) + connection = http.client.HTTPSConnection(parsed.hostname, parsed.port or 443, timeout=20, context=TLS) + path = parsed.path or "/" + if parsed.query: + path += "?" + parsed.query + started = time.perf_counter_ns() + connection.request(method, path, headers={"User-Agent": USER_AGENT, "Connection": "close"}) + response = connection.getresponse() + first = response.read(1) + first_byte_ms = (time.perf_counter_ns() - started) / 1_000_000 + body_bytes = len(first) + while max_bytes is None or body_bytes < max_bytes: + remaining = None if max_bytes is None else max_bytes - body_bytes + chunk = response.read(65536 if remaining is None else min(65536, remaining)) + if not chunk: + break + body_bytes += len(chunk) + total_ms = (time.perf_counter_ns() - started) / 1_000_000 + status = response.status + connection.close() + if status < 200 or status >= 400: + raise RuntimeError(f"{url} returned HTTP {status}") + return {"status": status, "bytes": body_bytes, "first_byte_ms": first_byte_ms, "total_ms": total_ms} + +def metric(name, operation): + started = time.perf_counter_ns() + try: + detail = operation() + return { + "name": name, + "ok": True, + "elapsed_ms": (time.perf_counter_ns() - started) / 1_000_000, + "detail": detail, + } + except Exception as error: + return { + "name": name, + "ok": False, + "elapsed_ms": (time.perf_counter_ns() - started) / 1_000_000, + "error": f"{type(error).__name__}: {error}", + } + +def dns_set(): + samples = [] + for host in DNS_HOSTS: + started = time.perf_counter_ns() + addresses = socket.getaddrinfo(host, 443, socket.AF_UNSPEC, socket.SOCK_STREAM) + samples.append({ + "host": host, + "elapsed_ms": (time.perf_counter_ns() - started) / 1_000_000, + "addresses": len(addresses), + }) + timings = [sample["elapsed_ms"] for sample in samples] + return { + "lookups": samples, + "p50_ms": percentile(timings, 0.50), + "p95_ms": percentile(timings, 0.95), + } + +def metadata_serial(): + return {"requests": [request(url, max_bytes=2_000_000) for url in METADATA_URLS]} + +def metadata_concurrent(): + urls = METADATA_URLS + METADATA_URLS + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: + responses = list(pool.map(lambda url: request(url, max_bytes=2_000_000), urls)) + return {"requests": responses} + +def https_reuse(): + connection = http.client.HTTPSConnection("example.com", 443, timeout=20, context=TLS) + samples = [] + try: + for _ in range(20): + started = time.perf_counter_ns() + connection.request("HEAD", "/", headers={"User-Agent": USER_AGENT, "Connection": "keep-alive"}) + response = connection.getresponse() + response.read() + if response.status != 200: + raise RuntimeError(f"example.com returned HTTP {response.status}") + samples.append((time.perf_counter_ns() - started) / 1_000_000) + finally: + connection.close() + return { + "requests": len(samples), + "p50_ms": percentile(samples, 0.50), + "p95_ms": percentile(samples, 0.95), + "mean_ms": statistics.fmean(samples), + } + +def git_clone(): + if shutil.which("git") is None: + return {"skipped": "git is not installed in the workload image"} + with tempfile.TemporaryDirectory(prefix="openshell-live-git-") as directory: + checkout = os.path.join(directory, "sampleproject") + command = [ + "git", "-c", "advice.detachedHead=false", "clone", "--quiet", + "--depth", "1", "--filter=blob:none", + "https://github.com/pypa/sampleproject.git", checkout, + ] + output = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60, check=False) + if output.returncode != 0: + raise RuntimeError(output.stderr.decode(errors="replace")[-500:]) + files = sum(len(names) for _, _, names in os.walk(checkout)) + size = sum(os.path.getsize(os.path.join(root, name)) for root, _, names in os.walk(checkout) for name in names) + return {"files": files, "checkout_bytes": size} + +def package_download(): + metadata = request("https://pypi.org/pypi/idna/json", max_bytes=2_000_000) + connection = http.client.HTTPSConnection("pypi.org", 443, timeout=20, context=TLS) + connection.request("GET", "/pypi/idna/json", headers={"User-Agent": USER_AGENT}) + response = connection.getresponse() + document = json.loads(response.read()) + connection.close() + wheels = [entry for entry in document["urls"] if entry["packagetype"] == "bdist_wheel"] + if not wheels: + raise RuntimeError("PyPI returned no idna wheel") + artifact = request(wheels[0]["url"], max_bytes=5_000_000) + return {"metadata": metadata, "artifact": artifact, "filename": wheels[0]["filename"]} + +def bulk_download(): + return request("https://speed.cloudflare.com/__down?bytes=5242880", max_bytes=5242880) + +def denied_destination(): + if MODE != "sandbox": + return {"skipped": "policy denial applies only to sandbox mode"} + started = time.perf_counter_ns() + try: + socket.create_connection(("example.net", 443), timeout=5).close() + except OSError as error: + return { + "denied": True, + "latency_ms": (time.perf_counter_ns() - started) / 1_000_000, + "error": str(error), + } + raise RuntimeError("destination omitted from policy was reachable") + +started = time.perf_counter_ns() +metrics = [ + metric("dns_service_set", dns_set), + metric("https_metadata_serial", metadata_serial), + metric("https_metadata_concurrent", metadata_concurrent), + metric("https_reuse", https_reuse), + metric("git_clone", git_clone), + metric("package_download", package_download), + metric("bulk_download_5mib", bulk_download), + metric("policy_denial", denied_destination), +] +document = { + "schema": "openshell.live-internet-perf.v1", + "mode": MODE, + "total_ms": (time.perf_counter_ns() - started) / 1_000_000, + "metrics": metrics, +} +print(json.dumps(document, separators=(",", ":"))) +if any(item["name"] == "policy_denial" and not item["ok"] for item in metrics): + raise SystemExit(1) +"#; + +fn rounds() -> usize { + std::env::var("OPENSHELL_LIVE_PERF_ROUNDS") + .ok() + .and_then(|value| value.parse().ok()) + .filter(|value| *value > 0) + .unwrap_or(DEFAULT_ROUNDS) +} + +fn write_policy() -> Result { + let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; + write!( + file, + r#"version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] + +landlock: + compatibility: best_effort + +network_policies: + live_internet_performance: + name: live_internet_performance + endpoints: + - {{ host: example.com, port: 443, protocol: tcp }} + - {{ host: github.com, port: 443, protocol: tcp }} + - {{ host: raw.githubusercontent.com, port: 443, protocol: tcp }} + - {{ host: pypi.org, port: 443, protocol: tcp }} + - {{ host: files.pythonhosted.org, port: 443, protocol: tcp }} + - {{ host: registry.npmjs.org, port: 443, protocol: tcp }} + - {{ host: crates.io, port: 443, protocol: tcp }} + - {{ host: docs.python.org, port: 443, protocol: tcp }} + - {{ host: speed.cloudflare.com, port: 443, protocol: tcp }} + binaries: + - path: "/**" +"# + ) + .map_err(|error| format!("write policy: {error}"))?; + file.flush() + .map_err(|error| format!("flush policy: {error}"))?; + Ok(file) +} + +async fn run_direct() -> Result { + run_python("direct").await +} + +async fn run_python(mode: &str) -> Result { + let output = tokio::process::Command::new("python3") + .args(["-c", BENCHMARK]) + .env("OPENSHELL_LIVE_PERF_MODE", mode) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .map_err(|error| format!("run {mode} live Internet benchmark: {error}"))?; + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + if !output.status.success() { + return Err(format!( + "{mode} live Internet benchmark failed (exit {:?}): {combined}", + output.status.code() + )); + } + Ok(combined) +} + +async fn run_sandbox(sandbox: &SandboxGuard) -> Result { + sandbox + .exec(&[ + "sh", + "-c", + "OPENSHELL_LIVE_PERF_MODE=sandbox python3 -c \"$1\"", + "openshell-live-internet-perf", + BENCHMARK, + ]) + .await +} + +#[tokio::test] +#[ignore = "manual live-Internet performance benchmark"] +async fn benchmark_live_internet_agent_traffic() { + let policy = write_policy().expect("write live Internet benchmark policy"); + let policy_path = policy.path().to_str().expect("UTF-8 policy path"); + let create_started = Instant::now(); + let mut sandbox = SandboxGuard::create(&["--policy", policy_path]) + .await + .expect("create live Internet benchmark sandbox"); + println!( + "LIVE_INTERNET_PERF create {{\"elapsed_ms\":{}}}", + create_started.elapsed().as_secs_f64() * 1000.0 + ); + + for round in 1..=rounds() { + if round % 2 == 1 { + let direct = run_direct().await.expect("direct live Internet benchmark"); + println!("LIVE_INTERNET_PERF direct round={round} {}", direct.trim()); + let mediated = run_sandbox(&sandbox) + .await + .expect("sandbox live Internet benchmark"); + println!( + "LIVE_INTERNET_PERF sandbox round={round} {}", + mediated.trim() + ); + } else { + let mediated = run_sandbox(&sandbox) + .await + .expect("sandbox live Internet benchmark"); + println!( + "LIVE_INTERNET_PERF sandbox round={round} {}", + mediated.trim() + ); + let direct = run_direct().await.expect("direct live Internet benchmark"); + println!("LIVE_INTERNET_PERF direct round={round} {}", direct.trim()); + } + } + + sandbox.cleanup().await; +} diff --git a/tasks/rust.toml b/tasks/rust.toml index 50eda4118c..a1ecfc7890 100644 --- a/tasks/rust.toml +++ b/tasks/rust.toml @@ -3,6 +3,10 @@ # Rust check, lint, and format tasks +["perf:seccomp"] +description = "Compare native and sandbox-filtered TCP and DNS performance" +run = "cargo run --release -p openshell-sandbox --features perf-harness --bin openshell-seccomp-perf --" + ["rust:check"] description = "Check all Rust crates for errors" run = "cargo check --workspace"