From 289a9eba90e1cf596ddef769881f573f495ccbb6 Mon Sep 17 00:00:00 2001 From: Christian Date: Fri, 2 Oct 2026 20:12:17 -0500 Subject: [PATCH 1/6] Scaffold hardened Axum application Docker recipes --- crates/edgezero-cli/src/generator.rs | 166 ++++++++++++++- crates/edgezero-cli/src/scaffold.rs | 12 ++ .../src/templates/root/Dockerfile.hbs | 62 ++++++ .../src/templates/root/README.md.hbs | 39 ++++ .../src/templates/root/dockerignore.hbs | 29 +++ docs/guide/adapters/axum.md | 145 ++++++++++++- scripts/test_generated_axum_container.sh | 194 ++++++++++++++++++ 7 files changed, 629 insertions(+), 18 deletions(-) create mode 100644 crates/edgezero-cli/src/templates/root/Dockerfile.hbs create mode 100644 crates/edgezero-cli/src/templates/root/dockerignore.hbs create mode 100755 scripts/test_generated_axum_container.sh diff --git a/crates/edgezero-cli/src/generator.rs b/crates/edgezero-cli/src/generator.rs index f566e76bc..a10850efa 100644 --- a/crates/edgezero-cli/src/generator.rs +++ b/crates/edgezero-cli/src/generator.rs @@ -16,6 +16,8 @@ use std::path::{Path, PathBuf}; use std::process::Command; use thiserror::Error; +const GENERATED_RUST_VERSION: &str = "1.95.0"; + /// Errors produced by `edgezero new`. #[derive(Debug, Error)] pub enum GeneratorError { @@ -566,6 +568,14 @@ fn build_base_data( workspace_dependencies: &BTreeMap, ) -> Map { let mut data = Map::new(); + data.insert( + "has_axum".into(), + Value::Bool(artifacts.adapter_ids.iter().any(|id| id == "axum")), + ); + data.insert( + "rust_version".into(), + Value::String(GENERATED_RUST_VERSION.to_owned()), + ); data.insert("name".into(), Value::String(layout.name.clone())); data.insert("proj_core".into(), Value::String(layout.core_name.clone())); data.insert("proj_cli".into(), Value::String(layout.cli_name.clone())); @@ -635,8 +645,7 @@ fn build_base_data( /// /// `asdf install` reads this file to pin per-tool versions. Every /// generated project gets `rust` pinned (it's a Rust workspace). -/// Per-adapter pins are added ONLY when the operator selected -/// that adapter at `edgezero new` time: +/// Per-adapter pins are added only for adapters registered in this CLI build: /// /// - `cloudflare` → `nodejs` (wrangler is a Node binary). /// - `fastly` → `fastly` (the Fastly CLI we shell out to for @@ -648,8 +657,7 @@ fn build_base_data( /// they don't wonder why everything else is pinned but spin. /// - `axum` → no extra pin (uses the host Rust toolchain only). /// -/// Versions are pulled from this repo's own `.tool-versions` (see -/// the repo root). When we bump those, we bump these. +/// Keep these supported pins aligned with the repository's `.tool-versions`. fn build_tool_versions(adapter_ids: &[String]) -> String { let has = |id: &str| adapter_ids.iter().any(|adapter| adapter == id); let mut lines: Vec = Vec::new(); @@ -660,7 +668,7 @@ fn build_tool_versions(adapter_ids: &[String]) -> String { lines.push("fastly 15.1.0".to_owned()); lines.push("viceroy 0.17.0".to_owned()); } - lines.push("rust 1.95.0".to_owned()); + lines.push(format!("rust {GENERATED_RUST_VERSION}")); // Sort + dedup so the file is stable regardless of adapter // declaration order (and asdf doesn't care). lines.sort(); @@ -676,8 +684,7 @@ fn build_tool_versions(adapter_ids: &[String]) -> String { body } -/// Render the six workspace-root files (Cargo.toml, edgezero.toml, -/// README.md, .gitignore, clippy.toml, .tool-versions). +/// Render shared workspace-root files and the optional Axum container recipe. /// /// Split out of `render_templates` so the parent stays under the /// project's `too_many_lines` clippy cap; the order of writes is @@ -697,6 +704,14 @@ fn write_root_files( ] { write_tmpl(hbs, template, data_value, &layout.out_dir.join(rel))?; } + if data_value["has_axum"].as_bool().unwrap_or(false) { + for (template, rel) in [ + ("root_Dockerfile", "Dockerfile"), + ("root_dockerignore", ".dockerignore"), + ] { + write_tmpl(hbs, template, data_value, &layout.out_dir.join(rel))?; + } + } Ok(()) } @@ -977,6 +992,138 @@ mod tests { ); } + #[test] + fn root_container_files_are_conditional_on_axum() { + let temp = TempDir::new().expect("temporary directory"); + let mut hbs = Handlebars::new(); + register_templates(&mut hbs); + for has_axum in [false, true] { + let layout = ProjectLayout::new(&NewArgs { + dir: Some(temp.path().to_string_lossy().into_owned()), + name: if has_axum { "My App" } else { "no-axum" }.to_owned(), + }) + .expect("project layout"); + let data = serde_json::json!({ + "has_axum": has_axum, + "name": layout.name, + "proj_axum": format!("{}-adapter-axum", layout.name), + "rust_version": "1.95.0", + }); + write_root_files(&hbs, &layout, &data).expect("root files"); + for file in ["Dockerfile", ".dockerignore"] { + assert_eq!(layout.out_dir.join(file).is_file(), has_axum, "{file}"); + } + if has_axum { + let dockerfile = + fs::read_to_string(layout.out_dir.join("Dockerfile")).expect("Dockerfile"); + assert!(dockerfile.contains("ARG RUST_VERSION=1.95.0")); + assert!(dockerfile.contains("-p my-app-adapter-axum --bin my-app-adapter-axum")); + assert!(dockerfile.contains("--release --locked")); + assert!(dockerfile.contains("USER 10001:10001")); + assert!(dockerfile.contains("ENTRYPOINT [\"/usr/local/bin/app\"]")); + assert!(dockerfile.contains("EDGEZERO__ADAPTER__HOST=0.0.0.0")); + assert!(dockerfile.contains("EDGEZERO__ADAPTER__PORT=8787")); + assert!(!dockerfile.contains("{{")); + let ignore = + fs::read_to_string(layout.out_dir.join(".dockerignore")).expect("dockerignore"); + assert!(ignore.lines().any(|line| line == "my-app.toml")); + assert!( + !ignore + .lines() + .any(|line| matches!(line, "*.toml" | "*.json" | "crates")) + ); + assert!(!layout.out_dir.join("Cargo.lock").exists()); + } + } + } + + #[cfg(unix)] + #[test] + fn container_toolchain_check_validates_declared_and_installed_versions() { + use std::os::unix::fs::PermissionsExt as _; + + let temp = TempDir::new().expect("temporary directory"); + let bin = temp.path().join("bin"); + fs::create_dir_all(&bin).expect("bin directory"); + let rustup = bin.join("rustup"); + fs::write( + &rustup, + "#!/bin/sh\nprintf 'rustc %s (fixture)\\n' \"$MOCK_RUST_VERSION\"\n", + ) + .expect("mock rustup"); + fs::set_permissions(&rustup, fs::Permissions::from_mode(0o755)).expect("executable mock"); + fs::write(temp.path().join("Cargo.lock"), "").expect("application lockfile"); + let template = include_str!("templates/root/Dockerfile.hbs"); + let check = template + .split("RUN <<'SH'\n") + .nth(1) + .expect("toolchain check") + .split("\nSH") + .next() + .expect("heredoc end"); + let path = format!("{}:{}", bin.display(), env::var("PATH").expect("PATH")); + for (versions, toolchain, installed, expected_error) in [ + ("rust 1.95.0\n", None, "1.95.0", ""), + ( + "rust 1.95.0\n", + Some("[toolchain]\nchannel = \"1.95.0\" # pin\ntargets = []\n"), + "1.95.0", + "", + ), + ("rust stable\n", None, "1.95.0", "exact X.Y.Z"), + ("rust 1.94.0\n", None, "1.95.0", "Docker Rust pin"), + ("rust 1.95.0\nrust 1.95.0\n", None, "1.95.0", "one rust"), + ("nodejs 24.12.0\n", None, "1.95.0", "one rust"), + ( + "rust 1.95.0\n", + Some("[toolchain]\nchannel = \"1.94.0\"\n"), + "1.95.0", + "conflicts", + ), + ( + "rust 1.95.0\n", + Some("[toolchain]\nchannel = \"stable\"\n"), + "1.95.0", + "one quoted", + ), + ( + "rust 1.95.0\n", + Some("[other]\nchannel = \"1.95.0\"\n"), + "1.95.0", + "one quoted", + ), + ( + "rust 1.95.0\n", + None, + "1.94.0", + "installed builder compiler", + ), + ] { + fs::write(temp.path().join(".tool-versions"), versions).expect("tool versions"); + let file = temp.path().join("rust-toolchain.toml"); + match toolchain { + Some(contents) => fs::write(&file, contents).expect("toolchain file"), + None if file.exists() => fs::remove_file(&file).expect("remove toolchain file"), + None => {} + } + let output = Command::new("sh") + .args(["-c", check]) + .current_dir(temp.path()) + .env("PATH", &path) + .env("RUST_VERSION", GENERATED_RUST_VERSION) + .env("MOCK_RUST_VERSION", installed) + .output() + .expect("run container check"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!( + output.status.success(), + expected_error.is_empty(), + "{stderr}" + ); + assert!(stderr.contains(expected_error), "{stderr}"); + } + } + #[test] fn generator_error_format_displays_underlying_fmt_error() { // `writeln!`-to-`String` cannot actually fail in production, but the @@ -1019,6 +1166,11 @@ mod tests { assert!(project_dir.join(".gitignore").exists()); assert!(project_dir.join(".tool-versions").exists()); assert!(project_dir.join("README.md").exists()); + assert!(project_dir.join("Dockerfile").exists()); + assert!(project_dir.join(".dockerignore").exists()); + let dockerfile = fs::read_to_string(project_dir.join("Dockerfile")).expect("Dockerfile"); + assert!(dockerfile.contains(&format!("ARG RUST_VERSION={GENERATED_RUST_VERSION}"))); + assert!(build_tool_versions(&[]).contains(GENERATED_RUST_VERSION)); assert!(project_dir.join("crates/demo-app-core/src/lib.rs").exists()); assert!( project_dir.join("crates/demo-app-cli/Cargo.toml").exists(), diff --git a/crates/edgezero-cli/src/scaffold.rs b/crates/edgezero-cli/src/scaffold.rs index d0b5f3456..26e7ae920 100644 --- a/crates/edgezero-cli/src/scaffold.rs +++ b/crates/edgezero-cli/src/scaffold.rs @@ -84,6 +84,16 @@ pub fn register_templates(hbs: &mut Handlebars) { include_str!("templates/root/tool-versions.hbs"), ) .expect("compiled-in template is valid"); + hbs.register_template_string( + "root_Dockerfile", + include_str!("templates/root/Dockerfile.hbs"), + ) + .expect("compiled-in template is valid"); + hbs.register_template_string( + "root_dockerignore", + include_str!("templates/root/dockerignore.hbs"), + ) + .expect("compiled-in template is valid"); // Core hbs.register_template_string( "core_Cargo_toml", @@ -263,6 +273,8 @@ mod tests { "root_gitignore", "root_clippy_toml", "root_tool_versions", + "root_Dockerfile", + "root_dockerignore", "core_Cargo_toml", "core_src_lib_rs", "core_src_handlers_rs", diff --git a/crates/edgezero-cli/src/templates/root/Dockerfile.hbs b/crates/edgezero-cli/src/templates/root/Dockerfile.hbs new file mode 100644 index 000000000..035ec446e --- /dev/null +++ b/crates/edgezero-cli/src/templates/root/Dockerfile.hbs @@ -0,0 +1,62 @@ +# Build from the workspace root after preparing portable dependencies and Cargo.lock. +# Keep this pin aligned with .tool-versions when updating the application toolchain. +ARG RUST_VERSION={{rust_version}} +FROM rust:${RUST_VERSION}-bookworm@sha256:6258907abe69656e41cd992e0b705cdcfabcbbe3db374f92ed2d47121282d4a1 AS builder +ARG RUST_VERSION +ENV RUSTUP_TOOLCHAIN=${RUST_VERSION} +WORKDIR /build +COPY . . + +# Support exact release pins and the conventional [toolchain] channel declaration. +# Validate before invoking rustup, which could otherwise download a directory override. +RUN <<'SH' +set -eu +fail() { echo "container toolchain: $*" >&2; exit 1; } +test -f Cargo.lock || fail 'prepare and commit the application Cargo.lock first' +pin=$(awk '$1 == "rust" { if (NF != 2 || ++count != 1) exit 1; pin = $2 } + END { if (count != 1) exit 1; print pin }' .tool-versions) \ + || fail 'expected one rust X.Y.Z entry in .tool-versions' +printf '%s\n' "$pin" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$' \ + || fail 'Rust must be an exact X.Y.Z release' +test "$pin" = "$RUST_VERSION" || fail 'update the Docker Rust pin to match .tool-versions' +test ! -e rust-toolchain || fail 'use rust-toolchain.toml with an exact channel instead of rust-toolchain' +if test -f rust-toolchain.toml; then + channel=$(awk ' + { sub(/#.*/, ""); gsub(/^[ \t]+|[ \t]+$/, "") } + /^\[/ { section = $0; next } + section == "[toolchain]" && /^channel[ \t]*=/ { + if (++count != 1 || $0 !~ /^channel[ \t]*=[ \t]*"[0-9]+\.[0-9]+\.[0-9]+"$/) exit 1 + sub(/^channel[ \t]*=[ \t]*"/, ""); sub(/"$/, ""); channel = $0 + } + END { if (count != 1) exit 1; print channel } + ' rust-toolchain.toml) || fail 'expected one quoted X.Y.Z channel under [toolchain]' + test "$channel" = "$pin" || fail 'rust-toolchain.toml conflicts with .tool-versions' +fi +actual=$(rustup run "$RUST_VERSION" rustc --version | awk '{print $2}') +test "$actual" = "$pin" || fail 'the installed builder compiler does not match the application pin' +SH + +# AWS-LC may build bundled native sources. These tools stay in the builder. +RUN apt-get update \ + && apt-get install -y --no-install-recommends cmake \ + && rm -rf /var/lib/apt/lists/* +RUN <<'SH' +set -eu +host=$(rustc -vV | awk '$1 == "host:" {print $2}') +cargo build -p {{proj_axum}} --bin {{proj_axum}} --release --locked --target "$host" +mkdir /out +cp "target/$host/release/{{proj_axum}}" /out/app +strip --strip-debug /out/app +SH + +FROM debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251 AS runtime +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates libgcc-s1 \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /app +COPY --from=builder /out/app /usr/local/bin/app +ENV EDGEZERO__ADAPTER__HOST=0.0.0.0 \ + EDGEZERO__ADAPTER__PORT=8787 +USER 10001:10001 +EXPOSE 8787 +ENTRYPOINT ["/usr/local/bin/app"] diff --git a/crates/edgezero-cli/src/templates/root/README.md.hbs b/crates/edgezero-cli/src/templates/root/README.md.hbs index 810a010b3..ee917682f 100644 --- a/crates/edgezero-cli/src/templates/root/README.md.hbs +++ b/crates/edgezero-cli/src/templates/root/README.md.hbs @@ -19,6 +19,45 @@ This workspace demonstrates a multi-target EdgeZero app. {{{readme_adapter_dev}}} +{{#if has_axum}} +## Container build + +The root `Dockerfile` builds only `{{proj_axum}}`. Prepare the workspace first: + +1. Check `[workspace.dependencies]` in `Cargo.toml`. `edgezero new` may have + generated paths into the developer's EdgeZero checkout or unpinned Git + references. Replace every EdgeZero entry with the same approved release tag + or commit at `https://github.com/stackpop/edgezero.git`, preserving its + feature settings. Keep paths between this application's crates. +2. Run `cargo generate-lockfile`, review the dependency sources with + `cargo metadata --locked --format-version 1`, and commit this application's + manifests and `Cargo.lock`. Do not copy EdgeZero's lockfile. These preparation + commands may fetch dependencies; scaffolding itself does not. +3. Keep the Docker builder's Rust pin aligned with `.tool-versions`. If adding + `rust-toolchain.toml`, use a matching exact quoted `[toolchain].channel`. + Update both declarations and the reviewed builder image when changing Rust. +4. Review `.dockerignore` and add your app-specific config, secrets and state + filenames. Keep required workspace manifests and build-time assets. + +```sh +docker build -t {{name}}:local . +docker run --rm --read-only --cap-drop=ALL \ + --security-opt=no-new-privileges:true \ + -p 127.0.0.1:8787:8787 {{name}}:local +``` + +The image runs as UID/GID 10001 and binds `0.0.0.0:8787` inside the container. +Local development still defaults to localhost. The default app has no stores +and needs no writable mount. Apps using current embedded stores must provision +an explicit `/app/.edgezero` mount with permissions for UID 10001. No config or +keys are generated by the image. Embedded redb requires one writer. + +This recipe proves packaging, not production readiness or bounded SIGTERM +shutdown. Application owners publish their own images. See the +[Axum guide](https://stackpop.github.io/edgezero/guide/adapters/axum) +for runtime limitations, optional asset builds and container checks. + +{{/if}} ## Configuration Environment variables are declared in `edgezero.toml`. Set `API_BASE_URL` to the upstream origin you want `/proxy/...` to target. Adapter-specific secrets are opt-in: uncomment the `#[secret]` field in `crates/{{proj_core}}/src/config.rs` and the matching `[stores.secrets]` block in `edgezero.toml`, then provide the secret value through the wired backend (Cloudflare Worker secret, Fastly Secret Store, Spin secret variable, ...). diff --git a/crates/edgezero-cli/src/templates/root/dockerignore.hbs b/crates/edgezero-cli/src/templates/root/dockerignore.hbs new file mode 100644 index 000000000..8fccd7ef2 --- /dev/null +++ b/crates/edgezero-cli/src/templates/root/dockerignore.hbs @@ -0,0 +1,29 @@ +# Host build output and dependencies. Keep workspace manifests and source assets. +**/target +/bin +/pkg +**/node_modules + +# Operator inputs and local runtime state do not belong in a build context. +{{name}}.toml +**/.env +**/.env.* +**/.dev.vars +**/.dev.vars.* +**/.cargo/credentials +**/.cargo/credentials.toml +**/.edgezero +**/.wrangler +**/.spin +**/.*.edgezero-lock +**/.*.edgezero-*.tmp + +# Repository/editor metadata. +**/.git +**/.vscode +**/.idea +**/.specstory +**/.DS_Store + +# Add application-specific config, signing-key and state filenames here. +# Do not exclude Cargo.lock, edgezero.toml, workspace crates, or required build assets. diff --git a/docs/guide/adapters/axum.md b/docs/guide/adapters/axum.md index a21364614..22233773e 100644 --- a/docs/guide/adapters/axum.md +++ b/docs/guide/adapters/axum.md @@ -216,22 +216,145 @@ flow — the bundled `edgezero config push` errors), which writes the same `.edgezero/local-config-.json` files the runtime reads — no shell-out, no server to authenticate against. -## Container Deployment +## Container packaging -Build and deploy as a standard container: +`edgezero new` generates a workspace-root `Dockerfile` and `.dockerignore` when +Axum is registered in the CLI build. The recipe builds only the native Axum +application binary with `--release --locked`, using a GNU/Linux Rust builder and +compatible Debian-slim runtime. The base images have reviewed digest pins; +package installation still needs network access and is not a byte-reproducible +build claim. + +### Prepare the application + +Generated EdgeZero dependencies may point into the developer's local checkout. +Those paths are not portable Docker inputs. Before building, edit the root +`[workspace.dependencies]` entries to use the same approved release tag or +commit for every EdgeZero crate. For example: + +```toml +# Replace the placeholder with a reviewed release compatible with this app. +edgezero-core = { git = "https://github.com/stackpop/edgezero.git", tag = "", default-features = false } +edgezero-adapter-axum = { git = "https://github.com/stackpop/edgezero.git", tag = "", default-features = false } +edgezero-cli = { git = "https://github.com/stackpop/edgezero.git", tag = "" } +``` + +Apply that ref to the other generated EdgeZero adapter entries too. Preserve +member feature settings, including Axum's `axum` feature and the operator CLI's +default features. Keep relative paths between the application's own crates. +There is no new dependency-selection flag; this is ordinary manifest preparation. + +```sh +cargo generate-lockfile +cargo metadata --locked --format-version 1 +# Review dependency sources, then commit the app manifests and Cargo.lock. +``` + +Do not copy EdgeZero's workspace lockfile. Scaffolding stays offline; these +explicit preparation commands may fetch dependencies. Docker requires a present, +current application lockfile and never repairs dependencies during the build. + +The builder compiler must match the exact Rust pin in the generated +`.tool-versions`. An optional `rust-toolchain.toml` must have a matching quoted +`channel = "X.Y.Z"` under `[toolchain]`. The small container check supports that +conventional format, not arbitrary TOML/channel syntax; legacy `rust-toolchain` +files are rejected. Update the declarations, Docker Rust version and reviewed +builder digest together when upgrading. The image uses the installed compiler +without downloading toolchain-file targets/components. It selects the native +host target explicitly rather than inheriting a consumer's WASM build target. + +### Build and run + +Run from the application workspace root, which must retain all workspace member +manifests, `edgezero.toml` and required build-time sources/assets: + +```sh +docker build -t my-app:local . +docker run --rm --read-only --cap-drop=ALL \ + --security-opt=no-new-privileges:true \ + -p 127.0.0.1:8787:8787 my-app:local +# From another terminal: +curl --fail http://127.0.0.1:8787/ +``` + +The runtime contains the binary, system CA trust and required native libraries, +not the source workspace, Rust, Node or the operator CLI. It runs as UID/GID +10001:10001 with exec startup and binds `0.0.0.0:8787` inside the container. The +publication above is localhost-only. EXPOSE does not publish ports, and the +read-only/capability settings are Docker run options, not Dockerfile guarantees. +To change the internal port, set `EDGEZERO__ADAPTER__PORT` and adjust the published +port mapping. Local development still defaults to localhost. + +The default generated app declares no stores and needs no writable mount. Apps +using current embedded stores must provide `/app/.edgezero`, owned/readable as +needed by UID 10001. Provision permissions outside startup; do not run a root +chown wrapper or use world-writable state. Config-only mounts can be read-only. +Embedded redb needs one writer; a shared volume does not establish multi-replica +safety. Runtime configuration, secrets, signing keys and mutable state remain +external inputs. The image does not generate them. + +Review `.dockerignore` before building. It excludes common credentials, local +state, Git and host build output, but cannot identify custom secret filenames. +Keep required manifests and assets. Private build credentials need an explicit +app-owned BuildKit secret/SSH setup, never COPY, ARG or ENV. + +### Optional application asset builds + +Node is not required for the generic template. Applications that embed JS during +Rust compilation can add a separate pinned asset stage before the Rust builder. +Trusted Server is one such consumer. Its app-owned extension can start with: ```dockerfile -FROM rust:1.75 as builder -WORKDIR /app -COPY . . -RUN cargo build -p my-app-adapter-axum --release - -FROM debian:bookworm-slim -COPY --from=builder /app/target/release/my-app-adapter-axum /usr/local/bin/ -EXPOSE 8787 -CMD ["my-app-adapter-axum"] +FROM node:24.12.0-bookworm-slim AS assets +WORKDIR /assets/lib +COPY crates/trusted-server-js/lib/ ./ +RUN npm ci && npm run build + +# In the existing Rust builder, before Cargo compilation: +# COPY --from=assets /assets/dist /build/crates/trusted-server-js/dist +# ENV TSJS_SKIP_BUILD=1 +``` + +Choose Node from the consumer's declared pin, retain its npm lockfile, and build +fresh bundles rather than reuse host dist output. Trusted Server's build.rs +embeds those bundles; after the explicit asset build, `TSJS_SKIP_BUILD=1` avoids +an implicit npm retry. Select its native package `trusted-server-adapter-axum` +and binary `trusted-server-axum`, not its Fastly default member or operator CLI. +Check bundle completeness and embedded bytes. This customization is guidance, +not a claim that Trusted Server's bind/config/health behavior or image has been +validated. Node stays out of the final runtime. + +### Packaging verification and limits + +From an EdgeZero checkout, the opt-in smoke accepts a **disposable generated +workspace** after the preparation above: + +```sh +./scripts/test_generated_axum_container.sh /path/to/container-probe ``` +It requires Docker, Python 3.11+, curl, OpenSSL, readelf and preparation network access. +It tests actual context exclusions, final filesystem/native libraries, UID, +read-only root, dropped capabilities, explicit mount permissions, published-port +HTTP and the real proxy's controlled TLS trust and hostname rejection. It uses +private test CA material, not production secrets or public upstreams, and cleans +up its containers/network/image. Failure diagnostics are retained in a temporary +directory, with private test keys removed. + +For unmerged framework changes, a separate disposable fixture can explicitly +stage EdgeZero sources and point test-only dependencies within the context. +Include all needed workspace manifests and the CLI's optional app-demo source +path. Resolve that fixture's own lockfile. This local-source evidence is not a +substitute for a standalone consumer with portable Git references. + +These are packaging checks, not production certification. The current runtime +still needs accepted PR #275 integration and the lifecycle/config work tracked +by #392/#396 for readiness, required-config failure, durable restart and bounded +SIGTERM draining. A response from `/` is not application readiness, and exec +startup alone does not implement signal handling. #399 owns native amd64/arm64 +CI and publishing guidance; #400 owns Docker/Kubernetes/cloud operations. +Application owners publish their own images. No public EdgeZero image is needed. + ## Configuration Configure the Axum adapter in `edgezero.toml`. See [Configuration](/guide/configuration) for the full diff --git a/scripts/test_generated_axum_container.sh b/scripts/test_generated_axum_container.sh new file mode 100755 index 000000000..122dcc251 --- /dev/null +++ b/scripts/test_generated_axum_container.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Opt-in packaging check for a disposable generated app, NOT a production app. +# Prepare portable dependencies and the app's own Cargo.lock before calling this. +# Usage: ./scripts/test_generated_axum_container.sh /path/to/container-probe +# Requires Docker, Python 3.11+, curl, OpenSSL, readelf and network access for image builds. +APP_DIR="$(cd "${1:?pass a prepared disposable generated workspace}" && pwd)" +for tool in docker python3 curl openssl readelf sha256sum; do + command -v "$tool" >/dev/null || { echo "Missing prerequisite: $tool" >&2; exit 1; } +done +docker info >/dev/null +NAME="$(python3 - "$APP_DIR/edgezero.toml" <<'PY' +import sys, tomllib +with open(sys.argv[1], 'rb') as f: + print(tomllib.load(f)['app']['name']) +PY +)" +[[ "$NAME" =~ ^[a-zA-Z0-9_-]+$ ]] || { echo 'Invalid generated app name' >&2; exit 1; } +for file in Dockerfile .dockerignore Cargo.lock .tool-versions; do + test -f "$APP_DIR/$file" || { echo "Prepare $file first" >&2; exit 1; } +done +WORK="$(mktemp -d -t edgezero-container.XXXXXX)" +ID="edgezero-smoke-$$-$RANDOM" +IMAGE="$ID:app" +PROBE="$ID:context" +NETWORK="$ID" +CONTAINERS=() +cleanup() { + status=$? + for container in "${CONTAINERS[@]}"; do + docker logs "$container" > "$WORK/$container.log" 2>&1 || true + docker rm -f "$container" >/dev/null 2>&1 || true + done + docker network rm "$NETWORK" >/dev/null 2>&1 || true + docker image rm "$IMAGE" "$PROBE" >/dev/null 2>&1 || true + # Always remove disposable private keys, including on failure. + rm -f "$WORK/tls/ca.key" "$WORK/tls/server.key" + if test "$status" -eq 0; then + rm -rf "$WORK" + else + echo "Failed; diagnostics retained at $WORK" >&2 + fi + exit "$status" +} +trap cleanup EXIT + +echo "==> Packaging fixture $NAME; host $(docker info --format '{{.Architecture}}')" +sha256sum "$APP_DIR/Cargo.lock" "$APP_DIR/.tool-versions" +python3 - "$APP_DIR/Cargo.toml" <<'PY' +import sys, tomllib +with open(sys.argv[1], 'rb') as f: + deps = tomllib.load(f)['workspace']['dependencies'] +for name, dep in deps.items(): + if name.startswith('edgezero-'): + print(name, dep) +PY + +# Work on a copy so sentinels never alter the supplied workspace. +mkdir "$WORK/context" +cp -a "$APP_DIR/." "$WORK/context/" +CONTEXT="$WORK/context" +for directory in .git .edgezero .wrangler .spin target bin pkg node_modules; do + mkdir -p "$CONTEXT/$directory" + printf 'disposable forbidden input\n' > "$CONTEXT/$directory/container-sentinel" +done +mkdir -p "$CONTEXT/.cargo" "$CONTEXT/assets" "$CONTEXT/crates/$NAME-core/src/bin" +printf 'build asset\n' > "$CONTEXT/assets/container-build-asset" +printf 'build source\n' > "$CONTEXT/crates/$NAME-core/src/bin/container-build-source" +printf 'disposable secret\n' > "$CONTEXT/.env.local" +printf 'disposable secret\n' > "$CONTEXT/.dev.vars" +printf 'disposable secret\n' > "$CONTEXT/.cargo/credentials.toml" +printf 'disposable config\n' > "$CONTEXT/$NAME.toml" +# Test Docker's actual context rules, not a substring interpretation of ignore patterns. +cat > "$WORK/context.Dockerfile" <<'DOCKER' +FROM debian:bookworm-slim +COPY . /context +ARG APP_NAME +RUN test ! -e "/context/$APP_NAME.toml" \ + && test -f /context/assets/container-build-asset \ + && test -f "/context/crates/$APP_NAME-core/src/bin/container-build-source" \ + && test -f /context/Cargo.toml && test -f /context/Cargo.lock \ + && test -f /context/edgezero.toml && test -d /context/crates \ + && test -f /context/.tool-versions \ + && test ! -e /context/.env.local && test ! -e /context/.dev.vars \ + && test ! -e /context/.cargo/credentials.toml \ + && ! find /context -name container-sentinel | grep . +DOCKER +docker build --progress=plain --build-arg "APP_NAME=$NAME" -f "$WORK/context.Dockerfile" -t "$PROBE" "$CONTEXT" 2>&1 | tee "$WORK/context.log" +docker build --progress=plain -t "$IMAGE" "$CONTEXT" 2>&1 | tee "$WORK/build.log" +docker image inspect "$IMAGE" --format 'image={{.Id}} architecture={{.Architecture}} user={{.Config.User}}' +docker history --no-trunc "$IMAGE" > "$WORK/history.log" + +CONTAINERS+=("$ID-export") +docker create --name "$ID-export" "$IMAGE" >/dev/null +docker export "$ID-export" -o "$WORK/runtime.tar" +docker cp "$ID-export:/etc/ssl/certs/ca-certificates.crt" "$WORK/os-roots.pem" +docker cp "$ID-export:/usr/local/bin/app" "$WORK/app" +readelf -h -l -d "$WORK/app" > "$WORK/elf.log" +grep -E 'Machine:|interpreter|NEEDED' "$WORK/elf.log" +test -s "$WORK/os-roots.pem" +python3 - "$WORK/runtime.tar" <<'PY' +import sys, tarfile +with tarfile.open(sys.argv[1]) as archive: + paths = {m.name.lstrip('./') for m in archive} + forbidden = ('build/', 'context/', 'usr/local/cargo/', 'usr/local/rustup/') + assert not any(p.startswith(forbidden) or '/.git/' in p or p.endswith('container-sentinel') for p in paths) + for path in ('usr/bin/cargo', 'usr/bin/rustc', 'usr/bin/node', 'usr/bin/npm', 'app/Cargo.toml'): + assert path not in paths, path + assert 'usr/local/bin/app' in paths + header = archive.extractfile('usr/local/bin/app').read(20) + assert header[:4] == b'\x7fELF', 'not a native ELF executable' + print('ELF class', header[4], 'machine', int.from_bytes(header[18:20], 'little')) +PY +docker run --rm --read-only --cap-drop=ALL --security-opt=no-new-privileges:true \ + --entrypoint sh "$IMAGE" -ec ' + id; test "$(id -u):$(id -g)" = 10001:10001 + grep -q "^CapEff:[[:space:]]*0000000000000000$" /proc/self/status + grep -q "^NoNewPrivs:[[:space:]]*1$" /proc/self/status + if touch /app/forbidden 2>/dev/null; then exit 1; fi + ' 2>&1 | tee "$WORK/identity-libraries.log" +# ldd may return success despite unresolved entries; check its complete output separately. +docker run --rm --read-only --cap-drop=ALL --entrypoint ldd "$IMAGE" /usr/local/bin/app \ + | tee "$WORK/ldd.log" +if grep -q 'not found' "$WORK/ldd.log"; then echo 'Unresolved runtime library' >&2; exit 1; fi + +# Explicit tmpfs ownership works; wrong-owner state remains unwritable. +docker run --rm --read-only --cap-drop=ALL --security-opt=no-new-privileges:true \ + --tmpfs /app/.edgezero:rw,noexec,nosuid,size=1m,uid=10001,gid=10001,mode=0700 \ + --entrypoint sh "$IMAGE" -ec 'touch /app/.edgezero/check; if touch /app/forbidden 2>/dev/null; then exit 1; fi' +docker run --rm --read-only --cap-drop=ALL --security-opt=no-new-privileges:true \ + --tmpfs /app/.edgezero:rw,noexec,nosuid,size=1m,uid=0,gid=0,mode=0700 \ + --entrypoint sh "$IMAGE" -ec '! touch /app/.edgezero/check 2>/dev/null' + +# A private bridge supports loopback publication. Docker internal networks may not. +docker network create "$NETWORK" >/dev/null +mkdir "$WORK/tls" +openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=EdgeZero disposable test CA' \ + -keyout "$WORK/tls/ca.key" -out "$WORK/tls/ca.crt" > "$WORK/certificates.log" 2>&1 +openssl req -new -newkey rsa:2048 -nodes -subj '/CN=upstream' \ + -keyout "$WORK/tls/server.key" -out "$WORK/tls/server.csr" >> "$WORK/certificates.log" 2>&1 +printf 'subjectAltName=DNS:upstream\n' > "$WORK/tls/extensions" +openssl x509 -req -days 1 -in "$WORK/tls/server.csr" -CA "$WORK/tls/ca.crt" \ + -CAkey "$WORK/tls/ca.key" -CAcreateserial -extfile "$WORK/tls/extensions" \ + -out "$WORK/tls/server.crt" >> "$WORK/certificates.log" 2>&1 +printf 'edgezero controlled TLS\n' > "$WORK/tls/index.html" +cat "$WORK/os-roots.pem" "$WORK/tls/ca.crt" > "$WORK/trusted.pem" +chmod 644 "$WORK/trusted.pem" +CONTAINERS+=("$ID-upstream") +docker run -d --name "$ID-upstream" --network "$NETWORK" --network-alias upstream \ + --network-alias wrong-host --user "$(id -u):$(id -g)" --read-only --cap-drop=ALL \ + --security-opt=no-new-privileges:true --mount "type=bind,src=$WORK/tls,dst=/tls,readonly" \ + --workdir /tls --entrypoint openssl "$IMAGE" s_server -quiet -accept 8443 \ + -cert /tls/server.crt -key /tls/server.key -WWW >/dev/null + +start_app() { + local case_name="$1" origin="$2" + shift 2 + local container="$ID-$case_name" + CONTAINERS+=("$container") + docker run -d --name "$container" --network "$NETWORK" --read-only --cap-drop=ALL \ + --security-opt=no-new-privileges:true -p 127.0.0.1::8787 \ + -e "API_BASE_URL=$origin" "$@" "$IMAGE" >/dev/null + PORT="$(docker port "$container" 8787/tcp | awk -F: '{print $NF}')" + local ready=false + for _ in $(seq 1 60); do + if curl --silent --fail --max-time 2 "http://127.0.0.1:$PORT/" > "$WORK/root-body"; then + ready=true; break + fi + sleep 1 + done + "$ready" || { echo "$case_name never served its published port" >&2; exit 1; } + test "$(cat "$WORK/root-body")" = "$NAME app" + docker inspect "$container" --format '{{.HostConfig.ReadonlyRootfs}} {{.HostConfig.CapDrop}} {{.HostConfig.SecurityOpt}}' +} +start_app trusted https://upstream:8443 --mount "type=bind,src=$WORK/trusted.pem,dst=/etc/ssl/certs/ca-certificates.crt,readonly" +# Poll the controlled upstream too; its first request may race server startup. +for _ in $(seq 1 30); do + curl --silent --fail --max-time 5 "http://127.0.0.1:$PORT/proxy/index.html" > "$WORK/trusted-body" \ + && grep -q 'edgezero controlled TLS' "$WORK/trusted-body" && break + sleep 1 +done +grep -q 'edgezero controlled TLS' "$WORK/trusted-body" +# Only test rejection after proving the same endpoint is reachable and serving TLS. +start_app untrusted https://upstream:8443 +status="$(curl --silent --max-time 35 -o "$WORK/untrusted-body" -w '%{http_code}' "http://127.0.0.1:$PORT/proxy/index.html")" +[[ "$status" =~ ^5[0-9][0-9]$ ]] +if grep -q 'edgezero controlled TLS' "$WORK/untrusted-body"; then exit 1; fi +start_app wrong-host https://wrong-host:8443 --mount "type=bind,src=$WORK/trusted.pem,dst=/etc/ssl/certs/ca-certificates.crt,readonly" +status="$(curl --silent --max-time 35 -o "$WORK/wrong-host-body" -w '%{http_code}' "http://127.0.0.1:$PORT/proxy/index.html")" +[[ "$status" =~ ^5[0-9][0-9]$ ]] +if grep -q 'edgezero controlled TLS' "$WORK/wrong-host-body"; then exit 1; fi +echo 'PASS: generated app context, native libraries, UID, read-only root, capabilities, mounts, HTTP and controlled TLS.' +echo 'Packaging only; no readiness, durable restart, SIGTERM or multi-architecture certification.' From 7e03616fcc6049430c8ced9990e9ed693d18ca6a Mon Sep 17 00:00:00 2001 From: Christian Date: Fri, 2 Oct 2026 22:17:59 -0500 Subject: [PATCH 2/6] review fix --- crates/edgezero-cli/src/generator.rs | 20 +++++++++++++++++++ .../src/templates/root/README.md.hbs | 4 ++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/crates/edgezero-cli/src/generator.rs b/crates/edgezero-cli/src/generator.rs index a10850efa..e0ef990ae 100644 --- a/crates/edgezero-cli/src/generator.rs +++ b/crates/edgezero-cli/src/generator.rs @@ -1037,6 +1037,26 @@ mod tests { } } + #[test] + fn container_readme_uses_valid_image_name_for_digit_leading_app() { + let mut hbs = Handlebars::new(); + register_templates(&mut hbs); + let name = sanitize_crate_name("123app"); + let readme = hbs + .render( + "root_README_md", + &serde_json::json!({ + "has_axum": true, + "name": name, + "proj_axum": format!("{name}-adapter-axum"), + }), + ) + .expect("render README"); + assert!(readme.contains("docker build -t app:local .")); + assert!(readme.contains("-p 127.0.0.1:8787:8787 app:local")); + assert!(readme.contains("`_123app-adapter-axum`")); + } + #[cfg(unix)] #[test] fn container_toolchain_check_validates_declared_and_installed_versions() { diff --git a/crates/edgezero-cli/src/templates/root/README.md.hbs b/crates/edgezero-cli/src/templates/root/README.md.hbs index ee917682f..d1d479135 100644 --- a/crates/edgezero-cli/src/templates/root/README.md.hbs +++ b/crates/edgezero-cli/src/templates/root/README.md.hbs @@ -40,10 +40,10 @@ The root `Dockerfile` builds only `{{proj_axum}}`. Prepare the workspace first: filenames. Keep required workspace manifests and build-time assets. ```sh -docker build -t {{name}}:local . +docker build -t app:local . docker run --rm --read-only --cap-drop=ALL \ --security-opt=no-new-privileges:true \ - -p 127.0.0.1:8787:8787 {{name}}:local + -p 127.0.0.1:8787:8787 app:local ``` The image runs as UID/GID 10001 and binds `0.0.0.0:8787` inside the container. From 0ea5874d90c308b14b9f565489121d694bbe8c30 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 7 Oct 2026 15:33:50 -0500 Subject: [PATCH 3/6] ci(axum): prepare native OCI validation and app-owned publishing --- .github/workflows/axum-container.yml | 173 ++++++++ .github/workflows/deploy-action.yml | 24 +- .gitignore | 3 + docs/.vitepress/config.mts | 8 + docs/guide/adapters/axum.md | 6 + docs/guide/deploying-axum-containers.md | 166 +++++++ docs/guide/publishing-axum-containers.md | 200 +++++++++ examples/axum-deployment/compose.yaml | 18 + examples/container-publishing/publish.sh | 101 +++++ examples/container-publishing/release.yml | 307 +++++++++++++ examples/container-publishing/tools.env | 9 + .../container-publishing/verify_artifacts.py | 411 ++++++++++++++++++ scripts/prepare_generated_axum_container.sh | 89 ++++ scripts/test_axum_container_deployment.sh | 49 +++ scripts/test_axum_container_handoff.sh | 129 ++++++ scripts/test_generated_axum_container.sh | 62 ++- .../tests/test_axum_container_artifacts.py | 368 ++++++++++++++++ .../tests/test_axum_container_preparation.py | 49 +++ 18 files changed, 2161 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/axum-container.yml create mode 100644 docs/guide/deploying-axum-containers.md create mode 100644 docs/guide/publishing-axum-containers.md create mode 100644 examples/axum-deployment/compose.yaml create mode 100755 examples/container-publishing/publish.sh create mode 100644 examples/container-publishing/release.yml create mode 100644 examples/container-publishing/tools.env create mode 100644 examples/container-publishing/verify_artifacts.py create mode 100755 scripts/prepare_generated_axum_container.sh create mode 100755 scripts/test_axum_container_deployment.sh create mode 100755 scripts/test_axum_container_handoff.sh create mode 100644 scripts/tests/test_axum_container_artifacts.py create mode 100644 scripts/tests/test_axum_container_preparation.py diff --git a/.github/workflows/axum-container.yml b/.github/workflows/axum-container.yml new file mode 100644 index 000000000..00992248c --- /dev/null +++ b/.github/workflows/axum-container.yml @@ -0,0 +1,173 @@ +name: Axum container preparation + +on: + pull_request: + branches: [main] + paths: + - .github/workflows/axum-container.yml + - Cargo.toml + - Cargo.lock + - .tool-versions + - crates/** + - examples/app-demo/** + - examples/container-publishing/** + - examples/axum-deployment/** + - scripts/*axum_container*.sh + - scripts/tests/test_axum_container*.py + - docs/guide/*axum-containers.md + - docs/guide/adapters/axum.md + push: + branches: [main] + paths: + - .github/workflows/axum-container.yml + - Cargo.toml + - Cargo.lock + - .tool-versions + - crates/** + - examples/app-demo/** + - examples/container-publishing/** + - examples/axum-deployment/** + - scripts/*axum_container*.sh + - scripts/tests/test_axum_container*.py + - docs/guide/*axum-containers.md + - docs/guide/adapters/axum.md + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + prepare: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Use the declared Rust toolchain + run: | + version=$(awk '$1 == "rust" { print $2 }' .tool-versions) + rustup toolchain install "$version" --profile minimal + echo "RUSTUP_TOOLCHAIN=$version" >> "$GITHUB_ENV" + - name: Prepare one application and lockfile + run: | + ./scripts/prepare_generated_axum_container.sh "$RUNNER_TEMP/fixture" + AXUM_PREPARED_CONTEXT="$RUNNER_TEMP/fixture/container-probe" \ + python3 -m unittest discover -s scripts/tests -p 'test_axum_container*.py' -v + tar -C "$RUNNER_TEMP/fixture" -cf "$RUNNER_TEMP/context.tar" container-probe + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: axum-context-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/context.tar + if-no-files-found: error + retention-days: 7 + + native: + needs: prepare + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: axum-context-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/context + digest-mismatch: error + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + with: + version: v0.37.0 + use: false + driver-opts: image=moby/buildkit:v0.25.1@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6 + - name: Native image, packaging, scan and OCI identity checks + env: + ARCH: ${{ matrix.arch }} + run: | + tar -C "$RUNNER_TEMP/context" -xf "$RUNNER_TEMP/context/context.tar" --no-same-owner + ./scripts/test_axum_container_handoff.sh build \ + "$RUNNER_TEMP/context/container-probe" "$RUNNER_TEMP/accepted" \ + "linux/$ARCH" "$GITHUB_SHA" "$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: accepted-${{ matrix.arch }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/accepted/image.tar + ${{ runner.temp }}/accepted/receipt.json + ${{ runner.temp }}/accepted/inputs.json + ${{ runner.temp }}/accepted/scan.json + ${{ runner.temp }}/accepted/scanner.json + ${{ runner.temp }}/accepted/sbom.json + ${{ runner.temp }}/accepted/provenance.json + if-no-files-found: error + retention-days: 7 + - name: Retain sanitized failure evidence separately + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: failed-${{ matrix.arch }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/accepted/*.log + ${{ runner.temp }}/accepted/scan.json + ${{ runner.temp }}/accepted/scanner.json + ${{ runner.temp }}/accepted/image.json + ${{ runner.temp }}/accepted/inputs.json + ${{ runner.temp }}/accepted/smoke + if-no-files-found: warn + retention-days: 7 + + handoff: + needs: native + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + actions: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Select exact immutable artifacts from this run and attempt + id: artifacts + env: + GH_TOKEN: ${{ github.token }} + run: | + gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100" > "$RUNNER_TEMP/artifacts.json" + ids=$(python3 examples/container-publishing/verify_artifacts.py artifact-ids \ + "$RUNNER_TEMP/artifacts.json" --run-id "$GITHUB_RUN_ID" --attempt "$GITHUB_RUN_ATTEMPT") + echo "ids=$ids" >> "$GITHUB_OUTPUT" + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + artifact-ids: ${{ steps.artifacts.outputs.ids }} + path: ${{ runner.temp }}/bundles + digest-mismatch: error + - name: Import accepted variants in a separate job without rebuilding + run: | + for arch in amd64 arm64; do + ./scripts/test_axum_container_handoff.sh import \ + "$RUNNER_TEMP/bundles/accepted-$arch-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" \ + "$RUNNER_TEMP/import-$arch" "linux/$arch" "$GITHUB_SHA" "$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + done + - name: Assemble and verify a localhost runtime index without external publication + run: | + source examples/container-publishing/tools.env + registry="axum-index-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + trap 'docker rm -f "$registry" >/dev/null 2>&1 || true' EXIT + docker run -d --name "$registry" -p 127.0.0.1::5000 "$REGISTRY_IMAGE" + port=$(docker port "$registry" 5000/tcp | awk -F: '{print $NF}') + for _ in $(seq 1 30); do curl --fail --silent "http://127.0.0.1:$port/v2/" && break; sleep 1; done + ./examples/container-publishing/publish.sh "127.0.0.1:$port/fixture" fixture \ + "$RUNNER_TEMP/bundles/accepted-amd64-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" \ + "$RUNNER_TEMP/bundles/accepted-arm64-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" \ + "$RUNNER_TEMP/index" "$GITHUB_SHA" "$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" diff --git a/.github/workflows/deploy-action.yml b/.github/workflows/deploy-action.yml index eff8262de..f47b05240 100644 --- a/.github/workflows/deploy-action.yml +++ b/.github/workflows/deploy-action.yml @@ -3,6 +3,11 @@ name: Deploy actions on: pull_request: paths: + - scripts/*axum_container*.sh + - scripts/tests/test_axum_container*.py + - examples/container-publishing/** + - examples/axum-deployment/** + - docs/guide/*axum-containers.md # ANY workflow or action change must start the repository-wide pin gate (and # the smokes) — including a brand-new action directory or an unrelated # workflow like test.yml that could introduce a floating ref. @@ -27,6 +32,11 @@ on: push: branches: [main] paths: + - scripts/*axum_container*.sh + - scripts/tests/test_axum_container*.py + - examples/container-publishing/** + - examples/axum-deployment/** + - docs/guide/*axum-containers.md # ANY workflow or action change must start the repository-wide pin gate (and # the smokes) — including a brand-new action directory or an unrelated # workflow like test.yml that could introduce a floating ref. @@ -97,15 +107,20 @@ jobs: # warning floor (info-level notes in unrelated workflows are not failures). run: | "$RUNNER_TEMP/tools/bin/actionlint" -shellcheck='shellcheck -S warning' + "$RUNNER_TEMP/tools/bin/actionlint" -shellcheck='shellcheck -S warning' examples/container-publishing/release.yml - name: Third-party actions pinned to a ref - run: .github/actions/deploy-core/tests/check-action-pins.sh + run: | + .github/actions/deploy-core/tests/check-action-pins.sh + .github/actions/deploy-core/tests/check-action-pins.sh examples/container-publishing/release.yml - name: Zizmor security scan run: | "$RUNNER_TEMP/tools/bin/zizmor" --offline \ .github/workflows/deploy-action.yml \ .github/workflows/fastly-installer-check.yml \ + .github/workflows/axum-container.yml \ + examples/container-publishing/release.yml \ .github/actions/build-app-cli/action.yml \ .github/actions/deploy-fastly/action.yml \ .github/actions/healthcheck-fastly/action.yml \ @@ -121,7 +136,12 @@ jobs: .github/actions/*/scripts/*.sh \ .github/actions/deploy-core/tests/*.sh \ scripts/install-actionlint.sh \ - scripts/install-yq.sh + scripts/install-yq.sh \ + scripts/*axum_container*.sh \ + examples/container-publishing/publish.sh + + - name: Container helper unit tests + run: python3 -m unittest discover -s scripts/tests -p 'test_axum_container*.py' -v - name: Bash contract tests run: .github/actions/deploy-core/tests/run.sh diff --git a/.gitignore b/.gitignore index beba4261f..010271bfe 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,9 @@ target/ .*.edgezero-lock .*.edgezero-*.tmp +# Standard-library container-helper test bytecode. +__pycache__/ + # env .env diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 4dd30b7b0..89d8de7eb 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -49,6 +49,14 @@ export default defineConfig({ { text: 'Cloudflare Workers', link: '/guide/adapters/cloudflare' }, { text: 'Fermyon Spin', link: '/guide/adapters/spin' }, { text: 'Axum (Native)', link: '/guide/adapters/axum' }, + { + text: 'Publishing Axum containers', + link: '/guide/publishing-axum-containers', + }, + { + text: 'Deploying Axum containers', + link: '/guide/deploying-axum-containers', + }, ], }, { diff --git a/docs/guide/adapters/axum.md b/docs/guide/adapters/axum.md index 22233773e..6be1fd4b3 100644 --- a/docs/guide/adapters/axum.md +++ b/docs/guide/adapters/axum.md @@ -355,6 +355,12 @@ startup alone does not implement signal handling. #399 owns native amd64/arm64 CI and publishing guidance; #400 owns Docker/Kubernetes/cloud operations. Application owners publish their own images. No public EdgeZero image is needed. +See [Publishing Axum containers](../publishing-axum-containers.md) for native +architecture CI, artifact identity, the strict scan gate and the app-owned GHCR +recipe. [Deploying Axum containers](../deploying-axum-containers.md) covers the +no-store Docker/Compose example and the remaining runtime, Kubernetes and +ECS/Fargate acceptance gates. + ## Configuration Configure the Axum adapter in `edgezero.toml`. See [Configuration](/guide/configuration) for the full diff --git a/docs/guide/deploying-axum-containers.md b/docs/guide/deploying-axum-containers.md new file mode 100644 index 000000000..2332b9473 --- /dev/null +++ b/docs/guide/deploying-axum-containers.md @@ -0,0 +1,166 @@ +# Deploying Axum containers + +Start with a tested, policy-accepted digest from +[the application-owned publisher](./publishing-axum-containers.md). Pin the +multi-platform index for scheduler-selected architecture, or the exact native +child for a known host. Do not rebuild on the deployment host. + +The examples here prepare Docker and Compose operations. They do not claim +production readiness, persistent recovery or graceful SIGTERM drain. The current +generated entrypoint is the development `run_app`; `/` is a packaging response, +not readiness. Production runtime work must be accepted before the remaining +[#400](https://github.com/stackpop/edgezero/issues/400) deployment checks. + +## No-store Docker and Compose + +Use a no-store application first. On a host matching an accepted native variant: + +```sh +export AXUM_IMAGE='ghcr.io/OWNER/APP@sha256:ACCEPTED_DIGEST' +docker pull "$AXUM_IMAGE" +docker run --rm --read-only --cap-drop=ALL \ + --security-opt=no-new-privileges:true \ + -e EDGEZERO__ADAPTER__HOST=0.0.0.0 -e EDGEZERO__ADAPTER__PORT=8787 \ + -p 127.0.0.1:8787:8787 "$AXUM_IMAGE" +``` + +The image runs as UID/GID 10001:10001. The bind address inside the container must +be externally reachable; publishing a port does not fix a localhost-only bind. +The Docker publication above is localhost-only. For external traffic, use an +approved proxy/TLS termination and restrict ingress explicitly. + +`examples/axum-deployment/compose.yaml` applies the same user, read-only root, +dropped capabilities, no-new-privileges and bind environment. It publishes to +localhost and deliberately has no healthcheck binary or dependency on in-image +curl/wget. Set `AXUM_PORT` to change the host port. The internal port remains 8787. + +```sh +docker compose -f examples/axum-deployment/compose.yaml config +docker compose -f examples/axum-deployment/compose.yaml up -d --no-build +# Inspect host-side HTTP behavior and logs, then: +docker compose -f examples/axum-deployment/compose.yaml down +``` + +From an EdgeZero checkout, a disposable generated fixture can exercise the +no-store packaging settings against an already loaded digest: + +```sh +./scripts/test_axum_container_deployment.sh "$AXUM_IMAGE" container-probe +``` + +The check verifies HTTP, UID, read-only root, capabilities, no-new-privileges, +stop/start and teardown. It never builds or deletes the supplied image. Plain +stop/start is not durable-state or bounded-drain evidence. The Compose 30-second +stop budget is a placeholder; it must exceed the application's accepted drain +budget before production use. + +## External state, configuration and secrets + +The generic image does not load an external `edgezero.toml` at runtime. Routing +and logical store declarations are compiled into `App`. Use supported +`EDGEZERO__*` runtime overrides and the app's accepted runtime contract. Do not +invent container-only configuration or secret loaders. + +Current embedded Axum stores use `/app/.edgezero`. Provide this directory outside +the image. Provision ownership and restrictive permissions for UID/GID 10001 +before startup. Do not add a root startup chown wrapper or world-writable mount. +Mount config-only files read-only where possible. A read-only root must remain +read-only when writable state is attached. + +Typed config uses the existing BlobEnvelope contract. Prepare per-id files with +the application's own config CLI, not EdgeZero's generic CLI. Actual secret +values, signing keys and mutable state stay outside image layers and build +contexts. The current environment-backed secret store reads actual process +environment values; merely mounting a secret file does not make it readable via +that API. Any file-based secret or signing-key loader needs an accepted +application/runtime implementation first. + +Embedded redb is single-writer state. One persistent volume does not make replicas +safe. Keep one owner/writer and verify shutdown, restart/recovery and backup/ +restore against the same backend. A Kubernetes shared-volume example must not +suggest replicated local redb or distributed leases. + +## Upgrades, restarts and recovery + +Retain the previous accepted image digest and its release evidence before an +upgrade. Pull and verify the new digest, restart with the reviewed external +inputs, then verify actual readiness and traffic through the approved ingress. +Changing environment config, secrets or signing keys requires a controlled +restart unless the accepted application implements reload explicitly. + +An old image digest does not undo incompatible data/config/key changes. Check +those compatibility boundaries before rollback and restore the matching external +inputs only under the application's recovery policy. + +For redb, stop and confirm the sole writer has exited before a file-level backup. +Keep a tested restore procedure and sufficient free disk for database growth and +backup copies. Deleting keys does not imply the database file shrinks. Do not +claim online snapshots, zero-loss failover or automated repair from these recipes. +Recovery and restore behavior remain acceptance checks, not documentation proof. + +## Runtime acceptance before deployment claims + +```mermaid +flowchart TD + A[Accepted native container digest] --> B[Accepted production runtime integration] + B --> C[Valid config becomes ready] + B --> D[Required config failure never becomes ready] + B --> E[Persistent write survives restart and recovery] + B --> F[SIGTERM drains active work within the configured budget] + C --> G[Docker and Compose acceptance] + D --> G + E --> G + F --> G + G --> H[Kubernetes single-writer deployment] + G --> I[ECS Fargate no-store deployment] +``` + +Run these checks in the actual candidate image after the accepted runtime stack +is integrated. Include timeout boundaries and failure status, not just successful +startup. Test readiness through the real service/probe endpoint, missing required +configuration, a persisted value after restart, an in-flight request during +SIGTERM, a request exceeding the drain budget and final process exit behavior. +The orchestrator termination budget must be longer than the configured runtime +drain budget. Preserve logs, configs and artifact identities without secrets. + +The production-runtime draft stack is not accepted integration evidence. Changes +to runtime behavior require a fresh candidate and repeat acceptance before +publishing that new digest. + +## Kubernetes follow-up + +Kubernetes acceptance remains separate. It needs the same index or explicit native +child digest, correct node architecture, non-root/read-only settings and separate +startup/readiness/liveness probes. Use HTTP probes or an already accepted probe +mechanism; do not assume curl exists in the image. Check Service/ingress routing +and termination timing under load. + +The persistent example must use single-writer scheduling and storage ownership, +then prove restart/recovery. Private production secrets/configuration must come +from approved resources through the existing runtime contract. No Helm chart, +operator or new deployment CLI is required by this preparation. + +## First managed target: ECS/Fargate + +The selected first managed example is a no-store ECS/Fargate service. Fargate +provides explicit `X86_64`/`ARM64` task architecture, read-only root settings and a +configurable stop timeout. This fits the native image and avoids pretending a +container's local disk is durable redb storage. + +Before an authorized test, an owner must specify the AWS account, region, cluster, +network/security groups, execution/task roles, registry credential path, service +and load-balancer/TLS resources, secret/config inputs, maximum cost and cleanup +ownership. GHCR package publication approval is not AWS deployment approval. +There is no AWS credential or resource-creation step in the publishing recipe. + +Then bind task architecture to the accepted child/index, set `readonlyRootFilesystem`, +UID/GID and the runtime bind environment, and choose a stop timeout longer than +the accepted drain budget within Fargate's supported limit. A private GHCR image +needs an approved `repositoryCredentials` secret and execution-role access, or a +separately approved digest-preserving registry copy. Verify its raw manifest +identity after any copy. Do not rebuild into another registry. + +Cloud acceptance must prove the approved pull path, traffic routing, actual +readiness, required-config failure and SIGTERM termination behavior, plus final +resource cleanup. Keep the first example stateless. State-backed ECS behavior +would require a separately accepted backend and replication contract. diff --git a/docs/guide/publishing-axum-containers.md b/docs/guide/publishing-axum-containers.md new file mode 100644 index 000000000..88cce7ef4 --- /dev/null +++ b/docs/guide/publishing-axum-containers.md @@ -0,0 +1,200 @@ +# Publishing Axum containers + +Application owners publish application images. EdgeZero does not need a public +application image. This guide adds publishing preparation to the +[Axum packaging recipe](./adapters/axum.md#container-packaging). It does not certify +production readiness, storage recovery or graceful shutdown. + +## Build once, test the published bytes + +The repository's `.github/workflows/axum-container.yml` prepares one generated +application and one application lockfile. Separate ephemeral Ubuntu amd64 and +arm64 runners build native GNU images. Neither uses QEMU as acceptance evidence. + +```mermaid +flowchart TD + A[One prepared context and lockfile] --> B[Native amd64 build and app checks] + A --> C[Native arm64 build and app checks] + B --> D[Scan and retain single-platform OCI bytes] + C --> E[Scan and retain single-platform OCI bytes] + D --> F[Same-run immutable artifact selection and verification] + E --> F + F --> G[Credential-free localhost import and index check] + F --> H[Application-owned protected publisher] + H --> I[Copy unchanged children and create runtime index] + I --> J[Authenticate and verify each digest] +``` + +The exporter disables automatic BuildKit attestation descriptors. The retained +archive has one runnable OCI child, its original config and layers. The handoff +uses a digest-pinned localhost registry and Skopeo's `--preserve-digests`. It checks +raw registry manifest bytes and Docker's pulled reference, filesystem identities +and runtime config. Docker's classic and containerd image stores report different +kinds of `.Id`; that field alone is not the config identity check. + +The final runtime index contains exactly the accepted amd64 and arm64 children. +Labels are set before testing. Publication never rebuilds, edits a child manifest +or silently converts its media type. A registry/tool that rewrites the manifest +must fail the gate. OCI attestations attach as separate referrers after publication. +They are not runnable children in the runtime index. + +## Reproduce the framework preparation locally + +Use a new output directory and a checkout you intend to test: + +```sh +./scripts/prepare_generated_axum_container.sh /tmp/axum-context +./scripts/test_axum_container_handoff.sh build \ + /tmp/axum-context/container-probe /tmp/axum-amd64 \ + linux/amd64 "$(git rev-parse HEAD)" local +``` + +The fixture stages framework sources under `vendor/edgezero`, preserves generated +member dependency features and excludes that nested framework workspace from the +application workspace. It resolves its own lockfile and rejects local packages +outside the application context. This is explicit test preparation, not a new +scaffold dependency-selection option or consumer release recipe. + +The build helper uses the existing generated-app HTTP/TLS packaging smoke and the +no-store Compose check. An application-owned `APP_SMOKE_SCRIPT` can replace those +fixture checks in a credential-free build job. It receives `IMAGE_REF PLATFORM`. +It must execute that image, fail on unmet requirements and clean up its containers. +The publisher never executes this script or an application binary. + +A successful bundle contains `image.tar`, `receipt.json`, `inputs.json`, +`scan.json`, `scanner.json`, `sbom.json` and `provenance.json`. Import verifies a +transferred bundle without rebuilding or running it: + +```sh +./scripts/test_axum_container_handoff.sh import \ + /path/to/transferred-bundle /tmp/axum-import \ + linux/amd64 "$(git rev-parse HEAD)" local +``` + +`local` permits explicitly dirty local preparation. Trusted release runs require +clean inputs. Local receipts, archive hashes and declared revisions prove +consistency, not producer authenticity. Do not feed an arbitrary PR bundle to a +credentialed publisher merely because its receipt verifies. + +## Application-owned GHCR recipe + +`examples/container-publishing/release.yml` is an inactive, copyable recipe. It +runs only from `workflow_dispatch` on `refs/heads/main`. Copy it as +`.github/workflows/release.yml` into the application repository, along with: + +- `examples/container-publishing/tools.env` +- `examples/container-publishing/verify_artifacts.py` +- `examples/container-publishing/publish.sh` +- `scripts/test_axum_container_handoff.sh` + +Keep those relative locations. Commit the app's portable dependency manifests, +`Cargo.lock`, `.tool-versions`, digest-pinned Dockerfile and executable +`.ci/test-container.sh`. The context must contain required assets and workspace +members. This sample uses a committed repository-root context without source +symlinks or submodule-only contents. `fixture-inputs.json` is reserved for the +preparation job. Private build credentials and generated assets need a separate +reviewed application-owned extension, never credentials in COPY, ARG or ENV. + +Before enabling the workflow, an owner must: + +1. Protect `main` and review changes to the workflow, Dockerfile, tools and tests. +2. Configure the `container-release` environment with required owner-designated + reviewers, prevent self-review and disable administrator bypass. Restrict its + deployment branches to `main`. Naming an environment alone does not protect it. + The publisher checks the required-reviewer settings and protected main via the + GitHub API and fails closed if they are missing or unreadable. +3. Approve the concrete `ghcr.io//` namespace, package access, + visibility and retention policy. The recipe derives this name from the app + repository. Adapt the fixed image name if the repository slug is not a valid + registry name. Do not use EdgeZero's namespace for a consumer application. +4. Reserve the version-tag namespace to this serialized workflow. The release + input accepts `vMAJOR.MINOR.PATCH` with an optional prerelease suffix, not + `latest`, `stable` or `prod`. GHCR tags are mutable. Preflight refuses an + existing different tag, permits creation only on confirmed registry not-found, + and checks the tag again after copying. Other writers can still race or move a + tag later. Deploy by digest, never by that alias. + +Build jobs have read-only repository permissions, nonpersistent checkout +credentials and no registry/OIDC authority. The publisher needs both native jobs +and protected-environment approval. Only its job has package-write, attestation +and OIDC permissions. OIDC authority applies to that whole job. + +The publisher queries this run's artifact metadata, selects exact immutable IDs +whose names bind platform/run/attempt, checks source SHA and artifact digests, and +uses same-run downloads with digest mismatch treated as an error. Uploads never +overwrite artifacts and missing successful outputs are errors. Failed scan +reports are diagnostic artifacts, never accepted release inputs. No +`pull_request_target`, cross-run latest-artifact discovery or automatic PR +promotion is involved. + +The publisher checks out its exact trusted commit. Before login, its verifier +recreates the committed source archive and recomputes context, Cargo.lock, +Dockerfile and toolchain traces. It verifies the OCI bundle, subject-bound scan, +sidecar hashes and policy. Registry credentials then live only in a disposable +publisher HOME so the copy tool and `actions/attest` use the same Docker auth file. +Cleanup removes that HOME even on failure. + +The recipe pins the GitHub CLI verifier by archive checksum. It attaches and +verifies separate SLSA publisher provenance for each child and the index, plus +CycloneDX SBOM attestations for each child. Standard publisher provenance does +not authenticate test assertions. Separate app-owned acceptance predicates bind +the independently checked input hashes, immutable artifact IDs/digests, child and +index digests, scanner version/database metadata and report/SBOM hashes. The +publisher generates them from verified inputs, never from an artifact-provided +predicate. They describe the application checks, not a SLSA level or framework +production certification. + +Consumer verification must constrain repository, signer workflow, source ref, +source SHA, digest and predicate type. For example, using the recipe's pinned CLI: + +```sh +gh attestation verify 'oci://ghcr.io/OWNER/APP@sha256:INDEX_DIGEST' \ + --repo OWNER/APP --signer-workflow OWNER/APP/.github/workflows/release.yml \ + --source-ref refs/heads/main --source-digest SOURCE_SHA \ + --predicate-type https://github.com/OWNER/APP/container-acceptance/v1 +``` + +Repeat for the two child digests. Verify SLSA provenance separately with +`https://slsa.dev/provenance/v1`, and child SBOMs with `https://cyclonedx.org/bom`. +A valid signature from some other workflow/repository is not acceptance. + +Keep accepted rollback digests and metadata under an application-owned retention +policy. Security updates require a reviewed source/base change, a fresh build and +the same native test/scan/identity gates before publication. Never relabel an old +accepted child after tests. A failed publication may leave immutable children or +a version alias without all required attestations. Treat that as incomplete, +verify the retained bytes and use an explicit owner-approved recovery path. Do +not rebuild with the same version tag and overwrite the partial release. + +Because selection binds the run attempt, rerun all preparation/native jobs for a +new attempt. The recipe does not mix a successful old platform with a new retry +or discover a prior successful run automatically. + +## Scanner policy and current blocker + +Trivy scans the verified OCI layout, which came directly from the retained +archive. The verifier requires a nonempty supported report with matching image +config identity and a manifest-bound scanner record. High and Critical findings +block acceptance, including findings without a fix. It does not use +`--ignore-unfixed` or an unreviewed ignore file. + +Exceptions must come from an independently trusted, owner-reviewed policy file, +not the artifact. Each entry requires `manifest_digest`, `vulnerability`, +`package`, `owner`, `reason` and ISO-date `expires`. Broad wildcards, missing scope +and expired exceptions fail. The standalone verifier/import/publisher helpers +accept `--exceptions` or `SCAN_EXCEPTIONS`; the sample workflow deliberately has +no automatic waiver path. Review the exact existing candidate bytes and an +explicit approval path before adapting the workflow for an exception. Do not +rebuild a waived image and assume the old digest exemption still applies. + +The October 7, 2026 local scan of the current Debian recipe reported 55 High and +4 Critical findings. The strict policy blocks that candidate. No exceptions have +been authorized. Image remediation or explicit scoped risk decisions are still +needed. The image scan found no language-specific files; its SBOM is not a claim +that every statically linked Rust dependency was independently analyzed. + +Native arm64 runner execution, accepted artifact import/index assembly in hosted +CI and one separately authorized disposable GHCR publish/attestation verification +remain execution gates. Keep [#399](https://github.com/stackpop/edgezero/issues/399) +open until those acceptance checks pass. Production runtime and deployment gates +are described in [Deploying Axum containers](./deploying-axum-containers.md). diff --git a/examples/axum-deployment/compose.yaml b/examples/axum-deployment/compose.yaml new file mode 100644 index 000000000..dcef292e5 --- /dev/null +++ b/examples/axum-deployment/compose.yaml @@ -0,0 +1,18 @@ +# Packaging example for a no-store generated app. No readiness/SIGTERM claim. +services: + app: + image: ${AXUM_IMAGE:?Set AXUM_IMAGE to a tested image digest} + user: "10001:10001" + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + ports: + - "127.0.0.1:${AXUM_PORT:-8787}:8787" + environment: + EDGEZERO__ADAPTER__HOST: "0.0.0.0" + EDGEZERO__ADAPTER__PORT: "8787" + # Production must set this longer than the accepted app drain budget. + # Current packaging fixture uses the development entrypoint. + stop_grace_period: 30s diff --git a/examples/container-publishing/publish.sh b/examples/container-publishing/publish.sh new file mode 100755 index 000000000..8c755b6ef --- /dev/null +++ b/examples/container-publishing/publish.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Trusted publisher only: verify two same-run bundles, copy their exact OCI bytes, +# and publish a two-platform index. Never builds or executes application images. +# Usage: publish.sh REPOSITORY TAG AMD64_BUNDLE ARM64_BUNDLE OUTPUT REVISION RUN_ID +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tools.env +source "$HERE/tools.env" +REPO="${1:?registry repository}"; TAG="${2:?version tag}" +AMD="${3:?amd64 bundle}"; ARM="${4:?arm64 bundle}"; OUTPUT="${5:?new output directory}" +REVISION="${6:?approved source SHA}"; RUN_ID="${7:?approved workflow run}" +[[ $# -eq 7 && "$TAG" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$ ]] || { echo 'Invalid publication arguments' >&2; exit 2; } +LOCAL=false +if [[ "$REPO" =~ ^127\.0\.0\.1:[0-9]+/[a-z0-9/-]+$ ]]; then + LOCAL=true +elif [[ ! "$REPO" =~ ^ghcr\.io/[a-z0-9_.-]+/[a-z0-9_.-]+$ ]]; then + echo 'This recipe supports app-owned GHCR or a disposable loopback registry only' >&2; exit 2 +elif [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.-]+)?$ ]]; then + echo 'GHCR publication requires a version tag, not a reserved alias' >&2; exit 2 +fi +[[ ! -e "$OUTPUT" ]] || { echo 'Output must not exist' >&2; exit 2; } +AMD="$(cd "$AMD" && pwd)"; ARM="$(cd "$ARM" && pwd)" +mkdir -p "$OUTPUT" +OUTPUT="$(cd "$OUTPUT" && pwd)" +WORK="$(mktemp -d -t axum-publisher.XXXXXX)" +trap 'rm -rf "$WORK"' EXIT +EXCEPTIONS=() +if [[ -n "${SCAN_EXCEPTIONS:-}" ]]; then EXCEPTIONS+=(--exceptions "$SCAN_EXCEPTIONS"); fi +python3 "$HERE/verify_artifacts.py" verify "$AMD" --revision "$REVISION" --run-id "$RUN_ID" \ + --platform linux/amd64 "${EXCEPTIONS[@]}" > "$OUTPUT/amd64.json" +python3 "$HERE/verify_artifacts.py" verify "$ARM" --revision "$REVISION" --run-id "$RUN_ID" \ + --platform linux/arm64 "${EXCEPTIONS[@]}" > "$OUTPUT/arm64.json" +python3 "$HERE/verify_artifacts.py" extract "$AMD/image.tar" "$WORK/amd64" --platform linux/amd64 >/dev/null +python3 "$HERE/verify_artifacts.py" extract "$ARM/image.tar" "$WORK/arm64" --platform linux/arm64 >/dev/null +python3 "$HERE/verify_artifacts.py" index "$AMD" "$ARM" --revision "$REVISION" --run-id "$RUN_ID" \ + "${EXCEPTIONS[@]}" > "$OUTPUT/index.json" +python3 - "$WORK" "$OUTPUT" <<'PY' +import hashlib, json, pathlib, shutil, sys +work, output = map(pathlib.Path, sys.argv[1:]) +merged = work / 'merged' +(merged / 'blobs/sha256').mkdir(parents=True) +for arch in ('amd64', 'arm64'): + for blob in (work / arch / 'blobs/sha256').iterdir(): + target = merged / 'blobs/sha256' / blob.name + if target.exists(): + if target.read_bytes() != blob.read_bytes(): + raise ValueError('Conflicting content-addressed blob') + else: + shutil.copyfile(blob, target) +raw = (output / 'index.json').read_bytes() +digest = hashlib.sha256(raw).hexdigest() +(merged / 'blobs/sha256' / digest).write_bytes(raw) +(merged / 'oci-layout').write_text('{"imageLayoutVersion":"1.0.0"}') +(merged / 'index.json').write_text(json.dumps({'schemaVersion': 2, 'manifests': [{ + 'mediaType': 'application/vnd.oci.image.index.v1+json', 'digest': 'sha256:' + digest, 'size': len(raw), + 'annotations': {'org.opencontainers.image.ref.name': 'release'}, +}]})) +(output / 'digest').write_text('sha256:' + digest + '\n') +PY +AUTH=(); TLS=() +if "$LOCAL"; then + TLS=(--dest-tls-verify=false) +else + # Docker-compatible auth file must live outside artifacts. Login is workflow-owned. + AUTH_FILE="${REGISTRY_AUTH_FILE:?provide a protected GHCR auth file}" + [[ -f "$AUTH_FILE" && "$AUTH_FILE" == /* ]] || { echo 'Expected absolute auth file' >&2; exit 2; } + AUTH=(--mount "type=bind,src=$AUTH_FILE,dst=/auth.json,readonly") +fi +COPY_AUTH=() +if ! "$LOCAL"; then COPY_AUTH=(--authfile /auth.json); fi +INSPECT_TLS=() +if "$LOCAL"; then INSPECT_TLS=(--tls-verify=false); fi +# Same version may be retried. Only HTTP 404 with a registry not-found code +# permits creation; all other preflight errors abort before any publication. +TAG_AUTH=() +if ! "$LOCAL"; then TAG_AUTH=(--auth-file "$AUTH_FILE"); fi +python3 "$HERE/verify_artifacts.py" tag-check "$REPO" "$TAG" "$(< "$OUTPUT/digest")" \ + "${TAG_AUTH[@]}" > "$OUTPUT/prior-tag.json" +docker run --rm --network host --user "$(id -u):$(id -g)" --cap-drop=ALL \ + --security-opt=no-new-privileges:true --mount "type=bind,src=$WORK,dst=/work,readonly" \ + "${AUTH[@]}" "$SKOPEO_IMAGE" copy --all --preserve-digests "${TLS[@]}" "${COPY_AUTH[@]}" \ + oci:/work/merged:release "docker://$REPO:$TAG" +docker run --rm --network host --user "$(id -u):$(id -g)" --cap-drop=ALL \ + --security-opt=no-new-privileges:true "${AUTH[@]}" "$SKOPEO_IMAGE" inspect --raw \ + "${INSPECT_TLS[@]}" "${COPY_AUTH[@]}" "docker://$REPO:$TAG" > "$OUTPUT/tag.json" +[[ "sha256:$(sha256sum "$OUTPUT/tag.json" | awk '{print $1}')" == "$(< "$OUTPUT/digest")" ]] \ + || { echo 'Version tag moved during publication' >&2; exit 1; } +for subject in "$(< "$OUTPUT/digest")" \ + "$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["manifest_digest"])' "$OUTPUT/amd64.json")" \ + "$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["manifest_digest"])' "$OUTPUT/arm64.json")"; do + raw="$OUTPUT/${subject#sha256:}.json" + docker run --rm --network host --user "$(id -u):$(id -g)" --cap-drop=ALL \ + --security-opt=no-new-privileges:true "${AUTH[@]}" "$SKOPEO_IMAGE" inspect --raw \ + "${INSPECT_TLS[@]}" "${COPY_AUTH[@]}" "docker://$REPO@$subject" > "$raw" + [[ "sha256:$(sha256sum "$raw" | awk '{print $1}')" == "$subject" ]] \ + || { echo 'Published manifest digest mismatch' >&2; exit 1; } +done +INDEX_DIGEST="$(< "$OUTPUT/digest")" +cmp "$OUTPUT/index.json" "$OUTPUT/${INDEX_DIGEST#sha256:}.json" +echo "Published unchanged runtime children and index: $REPO@$(< "$OUTPUT/digest")" diff --git a/examples/container-publishing/release.yml b/examples/container-publishing/release.yml new file mode 100644 index 000000000..bb0ed1258 --- /dev/null +++ b/examples/container-publishing/release.yml @@ -0,0 +1,307 @@ +# Copy with tools.env, verify_artifacts.py, publish.sh and the handoff helper. +# The application supplies .ci/test-container.sh IMAGE_REF PLATFORM, its own +# committed Cargo.lock, declared toolchain, and digest-pinned Dockerfile. +name: Application container release + +on: + workflow_dispatch: + inputs: + version: + description: New vMAJOR.MINOR.PATCH tag owned by this workflow + required: true + type: string + +permissions: + contents: read + +# Reserve the tag namespace to this workflow; GHCR tags themselves are mutable. +concurrency: + group: application-container-release-${{ github.repository }} + cancel-in-progress: false + +jobs: + prepare: + if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ github.sha }} + - name: Prepare the committed application context without rewriting dependencies + env: + VERSION_TAG: ${{ inputs.version }} + run: | + [[ "$VERSION_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.-]+)?$ && ${#VERSION_TAG} -le 128 ]] + test ! -e fixture-inputs.json + test -f Cargo.lock + test -x .ci/test-container.sh + git archive --format=tar "$GITHUB_SHA" > "$RUNNER_TEMP/source.tar" + mkdir "$RUNNER_TEMP/context" + tar -C "$RUNNER_TEMP/context" -xf "$RUNNER_TEMP/source.tar" --no-same-owner + python3 - "$RUNNER_TEMP/context" "$RUNNER_TEMP/source.tar" <<'PY' + import hashlib, json, os, pathlib, re, sys + app = pathlib.Path(sys.argv[1]) + source = pathlib.Path(sys.argv[2]) + hash_file = lambda p: hashlib.sha256(p.read_bytes()).hexdigest() + context = hashlib.sha256() + for path in sorted(app.rglob('*'), key=lambda path: path.relative_to(app).as_posix()): + if path.is_file(): + context.update(path.relative_to(app).as_posix().encode() + b'\0' + path.read_bytes() + b'\0') + (app / 'fixture-inputs.json').write_text(json.dumps({ + 'revision': os.environ['GITHUB_SHA'], 'dirty': False, + 'dependency_mode': 'application-owned-committed-lock', + 'source_sha256': hash_file(source), 'context_sha256': context.hexdigest(), + 'lock_sha256': hash_file(app / 'Cargo.lock'), + 'dockerfile_sha256': hash_file(app / 'Dockerfile'), + 'base_images': re.findall(r'^FROM (\S+)', (app / 'Dockerfile').read_text(), re.M), + 'toolchain': (app / '.tool-versions').read_text(), + }, indent=2) + '\n') + PY + tar -C "$RUNNER_TEMP" -cf "$RUNNER_TEMP/context.tar" context + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: application-context-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/context.tar + if-no-files-found: error + retention-days: 7 + + native: + needs: prepare + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ github.sha }} + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: application-context-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/download + digest-mismatch: error + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + with: + version: v0.37.0 + use: false + driver-opts: image=moby/buildkit:v0.25.1@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6 + - name: Build, run app-owned checks and scan without registry authority + env: + ARCH: ${{ matrix.arch }} + run: | + tar -C "$RUNNER_TEMP" -xf "$RUNNER_TEMP/download/context.tar" --no-same-owner + export APP_SMOKE_SCRIPT="$GITHUB_WORKSPACE/.ci/test-container.sh" + ./scripts/test_axum_container_handoff.sh build "$RUNNER_TEMP/context" \ + "$RUNNER_TEMP/accepted" "linux/$ARCH" "$GITHUB_SHA" "$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: accepted-${{ matrix.arch }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/accepted/image.tar + ${{ runner.temp }}/accepted/receipt.json + ${{ runner.temp }}/accepted/inputs.json + ${{ runner.temp }}/accepted/scan.json + ${{ runner.temp }}/accepted/scanner.json + ${{ runner.temp }}/accepted/sbom.json + ${{ runner.temp }}/accepted/provenance.json + if-no-files-found: error + retention-days: 7 + + publish: + needs: native + if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' + runs-on: ubuntu-24.04 + timeout-minutes: 20 + environment: container-release + permissions: + contents: read + actions: read + packages: write + attestations: write + id-token: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ github.sha }} + - name: Pin the publisher verifier + run: | + source examples/container-publishing/tools.env + curl --fail --location --retry 3 \ + "https://github.com/cli/cli/releases/download/v$GH_VERSION/gh_${GH_VERSION}_linux_amd64.tar.gz" \ + > "$RUNNER_TEMP/gh.tar.gz" + echo "$GH_AMD64_SHA256 $RUNNER_TEMP/gh.tar.gz" | sha256sum --check --strict + mkdir "$RUNNER_TEMP/ghcli" + tar -C "$RUNNER_TEMP/ghcli" -xf "$RUNNER_TEMP/gh.tar.gz" --strip-components=1 --no-same-owner + echo "$RUNNER_TEMP/ghcli/bin" >> "$GITHUB_PATH" + - name: Fail closed if release protection has not been configured + env: + GH_TOKEN: ${{ github.token }} + run: | + gh api "repos/$GITHUB_REPOSITORY/environments/container-release" > "$RUNNER_TEMP/environment.json" + gh api "repos/$GITHUB_REPOSITORY/branches/main" > "$RUNNER_TEMP/branch.json" + python3 - "$RUNNER_TEMP/environment.json" "$RUNNER_TEMP/branch.json" <<'PY' + import json, sys + environment, branch = [json.load(open(path)) for path in sys.argv[1:]] + rules = environment.get('protection_rules', []) + if not (branch.get('protected') and environment.get('can_admins_bypass') is False and any( + rule.get('type') == 'required_reviewers' and rule.get('reviewers') and rule.get('prevent_self_review') + for rule in rules + )): + raise SystemExit('Configure protected main, required release reviewers, no self-review and no admin bypass') + PY + - name: Select immutable artifacts from this trusted run and commit + id: artifacts + env: + GH_TOKEN: ${{ github.token }} + run: | + gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100" > "$RUNNER_TEMP/artifacts.json" + ids=$(python3 examples/container-publishing/verify_artifacts.py artifact-ids \ + "$RUNNER_TEMP/artifacts.json" --run-id "$GITHUB_RUN_ID" --attempt "$GITHUB_RUN_ATTEMPT" --revision "$GITHUB_SHA") + echo "ids=$ids" >> "$GITHUB_OUTPUT" + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + artifact-ids: ${{ steps.artifacts.outputs.ids }} + path: ${{ runner.temp }}/bundles + digest-mismatch: error + - name: Verify before exposing registry credentials + id: verified + run: | + for arch in amd64 arm64; do + bundle="$RUNNER_TEMP/bundles/accepted-$arch-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + python3 examples/container-publishing/verify_artifacts.py verify "$bundle" \ + --revision "$GITHUB_SHA" --run-id "$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" --platform "linux/$arch" \ + > "$RUNNER_TEMP/$arch.json" + digest=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["manifest_digest"])' "$RUNNER_TEMP/$arch.json") + echo "$arch=$digest" >> "$GITHUB_OUTPUT" + done + python3 examples/container-publishing/verify_artifacts.py release-record "$GITHUB_WORKSPACE" \ + "$RUNNER_TEMP/bundles/accepted-amd64-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" \ + "$RUNNER_TEMP/bundles/accepted-arm64-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" \ + "$RUNNER_TEMP/artifacts.json" --revision "$GITHUB_SHA" --run-id "$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" \ + > "$RUNNER_TEMP/release-record.json" + echo "image=ghcr.io/${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT" + echo "predicate_type=https://github.com/$GITHUB_REPOSITORY/container-acceptance/v1" >> "$GITHUB_OUTPUT" + - name: Login only in the approved publisher + env: + REGISTRY_TOKEN: ${{ github.token }} + REGISTRY_USER: ${{ github.actor }} + run: | + mkdir -m 700 "$RUNNER_TEMP/publisher-home" + mkdir -m 700 "$RUNNER_TEMP/publisher-home/.docker" + echo "HOME=$RUNNER_TEMP/publisher-home" >> "$GITHUB_ENV" + printf '%s' "$REGISTRY_TOKEN" | docker --config "$RUNNER_TEMP/publisher-home/.docker" login ghcr.io \ + --username "$REGISTRY_USER" --password-stdin + - name: Copy tested children and construct their exact runtime index, never rebuild + id: release + env: + VERSION_TAG: ${{ inputs.version }} + IMAGE: ${{ steps.verified.outputs.image }} + run: | + export REGISTRY_AUTH_FILE="$RUNNER_TEMP/publisher-home/.docker/config.json" + ./examples/container-publishing/publish.sh "$IMAGE" "$VERSION_TAG" \ + "$RUNNER_TEMP/bundles/accepted-amd64-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" \ + "$RUNNER_TEMP/bundles/accepted-arm64-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" \ + "$RUNNER_TEMP/release" "$GITHUB_SHA" "$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + digest=$(cat "$RUNNER_TEMP/release/digest") + expected=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["index_digest"])' "$RUNNER_TEMP/release-record.json") + [[ "$digest" == "$expected" ]] + echo "digest=$digest" >> "$GITHUB_OUTPUT" + - name: Authenticate amd64 publisher provenance + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-name: ${{ steps.verified.outputs.image }} + subject-digest: ${{ steps.verified.outputs.amd64 }} + push-to-registry: true + create-storage-record: false + - name: Authenticate arm64 publisher provenance + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-name: ${{ steps.verified.outputs.image }} + subject-digest: ${{ steps.verified.outputs.arm64 }} + push-to-registry: true + create-storage-record: false + - name: Authenticate index publisher provenance + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-name: ${{ steps.verified.outputs.image }} + subject-digest: ${{ steps.release.outputs.digest }} + push-to-registry: true + create-storage-record: false + - name: Authenticate amd64 SBOM + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-name: ${{ steps.verified.outputs.image }} + subject-digest: ${{ steps.verified.outputs.amd64 }} + sbom-path: ${{ runner.temp }}/bundles/accepted-amd64-${{ github.run_id }}-${{ github.run_attempt }}/sbom.json + push-to-registry: true + create-storage-record: false + - name: Authenticate arm64 SBOM + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-name: ${{ steps.verified.outputs.image }} + subject-digest: ${{ steps.verified.outputs.arm64 }} + sbom-path: ${{ runner.temp }}/bundles/accepted-arm64-${{ github.run_id }}-${{ github.run_attempt }}/sbom.json + push-to-registry: true + create-storage-record: false + - name: Authenticate independently verified amd64 acceptance trace + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-name: ${{ steps.verified.outputs.image }} + subject-digest: ${{ steps.verified.outputs.amd64 }} + predicate-type: ${{ steps.verified.outputs.predicate_type }} + predicate-path: ${{ runner.temp }}/release-record.json + push-to-registry: true + create-storage-record: false + - name: Authenticate independently verified arm64 acceptance trace + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-name: ${{ steps.verified.outputs.image }} + subject-digest: ${{ steps.verified.outputs.arm64 }} + predicate-type: ${{ steps.verified.outputs.predicate_type }} + predicate-path: ${{ runner.temp }}/release-record.json + push-to-registry: true + create-storage-record: false + - name: Authenticate independently verified index acceptance trace + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-name: ${{ steps.verified.outputs.image }} + subject-digest: ${{ steps.release.outputs.digest }} + predicate-type: ${{ steps.verified.outputs.predicate_type }} + predicate-path: ${{ runner.temp }}/release-record.json + push-to-registry: true + create-storage-record: false + - name: Verify exact subjects and expected publisher workflow identity + env: + GH_TOKEN: ${{ github.token }} + IMAGE: ${{ steps.verified.outputs.image }} + AMD64: ${{ steps.verified.outputs.amd64 }} + ARM64: ${{ steps.verified.outputs.arm64 }} + INDEX: ${{ steps.release.outputs.digest }} + PREDICATE_TYPE: ${{ steps.verified.outputs.predicate_type }} + run: | + signer="${GITHUB_WORKFLOW_REF%@*}" + for digest in "$AMD64" "$ARM64" "$INDEX"; do + for predicate in https://slsa.dev/provenance/v1 "$PREDICATE_TYPE"; do + gh attestation verify "oci://$IMAGE@$digest" --repo "$GITHUB_REPOSITORY" \ + --signer-workflow "$signer" --source-ref refs/heads/main --source-digest "$GITHUB_SHA" \ + --predicate-type "$predicate" + done + done + for digest in "$AMD64" "$ARM64"; do + gh attestation verify "oci://$IMAGE@$digest" --repo "$GITHUB_REPOSITORY" \ + --signer-workflow "$signer" --source-ref refs/heads/main --source-digest "$GITHUB_SHA" \ + --predicate-type https://cyclonedx.org/bom + done + - name: Remove publisher authentication even on failure + if: always() + run: rm -rf "$RUNNER_TEMP/publisher-home" diff --git a/examples/container-publishing/tools.env b/examples/container-publishing/tools.env new file mode 100644 index 000000000..6c4bdf69e --- /dev/null +++ b/examples/container-publishing/tools.env @@ -0,0 +1,9 @@ +# Multi-architecture OCI index pins, resolved and inspected 2026-10-07. +# Update versions and digests together, then rerun the native image/handoff checks. +# Publisher verifier; install this checksum-verified archive in release.yml. +GH_VERSION=2.102.0 +GH_AMD64_SHA256=bb766f710eef8ede859c18578c72c327597cd4c8a85b06001b1f3843c6019386 +REGISTRY_IMAGE=registry:3.0.0@sha256:6c5666b861f3505b116bb9aa9b25175e71210414bd010d92035ff64018f9457e +SKOPEO_IMAGE=quay.io/skopeo/stable:v1.21.0@sha256:a585e4a3b8a045baa87c7f1b2f940d6d299ebede85ab3f2419d52d2264eefc93 +BUILDKIT_IMAGE=moby/buildkit:v0.25.1@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6 +TRIVY_IMAGE=aquasec/trivy:0.75.0@sha256:af6acf9a6b85dfe389a1941505c0ce9efef52a4719635e1a962f022a3d855daa diff --git a/examples/container-publishing/verify_artifacts.py b/examples/container-publishing/verify_artifacts.py new file mode 100644 index 000000000..529e8fd12 --- /dev/null +++ b/examples/container-publishing/verify_artifacts.py @@ -0,0 +1,411 @@ +#!/usr/bin/env python3 +"""Verify this recipe's single-platform OCI bundles. No third-party dependencies. + +Receipts/checksums are integrity checks, not authentication. The publisher must +select immutable artifacts from its approved same-run producer before calling us. +""" + +import argparse +import datetime +import hashlib +import io +import json +import pathlib +import re +import subprocess +import sys +import tarfile +import tempfile +import urllib.error +import urllib.parse +import urllib.request + +MANIFEST = 'application/vnd.oci.image.manifest.v1+json' +CONFIG = 'application/vnd.oci.image.config.v1+json' +INDEX = 'application/vnd.oci.image.index.v1+json' +LAYERS = {'application/vnd.oci.image.layer.v1.tar', 'application/vnd.oci.image.layer.v1.tar+gzip', + 'application/vnd.oci.image.layer.v1.tar+zstd'} +DIGEST = re.compile(r'sha256:[0-9a-f]{64}\Z') +PLATFORMS = {'linux/amd64', 'linux/arm64'} +SIDECARS = ('inputs.json', 'scan.json', 'scanner.json', 'sbom.json', 'provenance.json') + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def checksum(path): + require(path.is_file() and not path.is_symlink(), 'Expected regular artifact file') + digest = hashlib.sha256() + with open(path, 'rb') as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b''): + digest.update(chunk) + return digest.hexdigest() + + +def load(path): + require(path.is_file() and not path.is_symlink(), 'Expected regular JSON file') + require(path.stat().st_size <= 16 * 1024 * 1024, f'JSON too large: {path.name}') + return json.loads(path.read_text()) + + +def unpack(archive, destination): + # New output only. Reject links, special files, duplicate names and arbitrary payloads. + destination.mkdir() + seen = set() + total = 0 + with tarfile.open(archive, 'r:*') as tar: + for member in tar: + name = member.name.removeprefix('./') + if name in ('', '.') and member.isdir(): + continue + path = pathlib.PurePosixPath(name) + require(not path.is_absolute() and '..' not in path.parts and str(path) == name.rstrip('/'), 'Unsafe archive path') + require(name.rstrip('/') not in seen, 'Duplicate archive member') + seen.add(name.rstrip('/')) + require(len(seen) <= 10000, 'Too many archive members') + valid_dir = name.rstrip('/') in ('blobs', 'blobs/sha256') + valid_file = name in ('oci-layout', 'index.json') or re.fullmatch(r'blobs/sha256/[0-9a-f]{64}', name) + require((member.isdir() and valid_dir) or (member.isreg() and valid_file and not member.sparse), 'Unexpected archive member') + total += member.size + require(total <= 4 * 1024 ** 3, 'Archive exceeds 4 GiB limit') + target = destination.joinpath(*path.parts) + if member.isdir(): + target.mkdir(exist_ok=True) + else: + target.parent.mkdir(parents=True, exist_ok=True) + with tar.extractfile(member) as source, target.open('xb') as output: + for chunk in iter(lambda: source.read(1024 * 1024), b''): + output.write(chunk) + require(target.stat().st_size == member.size, 'Truncated archive member') + + +def blob(layout, descriptor, media_types): + digest = descriptor['digest'] + require(isinstance(digest, str) and DIGEST.fullmatch(digest), 'Unsupported blob digest') + require(descriptor['mediaType'] in media_types, 'Unexpected blob media type') + size = descriptor['size'] + require(type(size) is int and size >= 0, 'Invalid descriptor size') + path = layout / 'blobs' / 'sha256' / digest[7:] + require(path.is_file() and not path.is_symlink(), 'Missing regular blob') + require(path.stat().st_size == size and checksum(path) == digest[7:], 'Blob digest/size mismatch') + return path + + +def describe(layout, platform): + require(platform in PLATFORMS, 'Unsupported platform') + require(load(layout / 'oci-layout') == {'imageLayoutVersion': '1.0.0'}, 'Unsupported OCI layout') + index = load(layout / 'index.json') + require(index['schemaVersion'] == 2 and index.get('mediaType', INDEX) == INDEX, 'Invalid OCI index') + require(len(index['manifests']) == 1, 'Expected exactly one runtime manifest, no attestations') + descriptor = index['manifests'][0] + manifest = load(blob(layout, descriptor, {MANIFEST})) + require(manifest['schemaVersion'] == 2 and manifest['mediaType'] == MANIFEST, 'Invalid runtime manifest') + config = load(blob(layout, manifest['config'], {CONFIG})) + actual = f'{config["os"]}/{config["architecture"]}' + require(actual == platform, 'Config/platform mismatch') + if 'platform' in descriptor: + require(f'{descriptor["platform"]["os"]}/{descriptor["platform"]["architecture"]}' == platform, 'Descriptor/platform mismatch') + require(manifest['layers'], 'Runtime image has no layers') + for layer in manifest['layers']: + blob(layout, layer, LAYERS) + return {'platform': platform, 'manifest_digest': descriptor['digest'], + 'manifest_size': descriptor['size'], 'config_digest': manifest['config']['digest'], + 'revision_label': config.get('config', {}).get('Labels', {}).get('org.opencontainers.image.revision')} + + +def verify_pulled(layout, inspection, platform, reference): + info = describe(layout, platform) + image = inspection[0] + require(reference.endswith('@' + info['manifest_digest']), 'Expected pulled child digest mismatch') + require(reference in image['RepoDigests'], 'Pulled digest reference mismatch') + require(f'{image["Os"]}/{image["Architecture"]}' == platform, 'Pulled platform mismatch') + config = load(layout / 'blobs/sha256' / info['config_digest'][7:]) + require(image['RootFS']['Layers'] == config['rootfs']['diff_ids'], 'Pulled filesystem identity mismatch') + for key in ('User', 'Env', 'Entrypoint', 'Cmd', 'Labels', 'WorkingDir', 'ExposedPorts', 'StopSignal'): + require(image['Config'].get(key) == config['config'].get(key), 'Pulled runtime config mismatch') + # Docker's classic store uses config digest for Id; its containerd store uses + # the manifest digest. Neither representation is an identity conversion. + if 'Descriptor' in image: + require(image['Descriptor']['digest'] == info['manifest_digest'], 'Pulled manifest mismatch') + return info + + +def scan_policy(report, exceptions, manifest_digest, today=None): + today = today or datetime.datetime.now(datetime.timezone.utc).date() + require(report.get('SchemaVersion') == 2 and isinstance(report.get('Results'), list) and report['Results'], 'Missing Trivy results') + require(isinstance(exceptions, list), 'Exceptions must be a list') + allowed = set() + for entry in exceptions: + require(set(entry) == {'manifest_digest', 'vulnerability', 'package', 'owner', 'reason', 'expires'}, 'Invalid exception fields') + require(all(isinstance(v, str) and v.strip() for v in entry.values()), 'Empty exception field') + require(DIGEST.fullmatch(entry['manifest_digest']), 'Invalid exception artifact scope') + require(not any(c in entry['package'] + entry['vulnerability'] for c in '*?[]'), 'Wildcard scan exception') + require(datetime.date.fromisoformat(entry['expires']) >= today, 'Expired scan exception') + key = (entry['manifest_digest'], entry['vulnerability'], entry['package']) + require(key not in allowed, 'Duplicate scan exception') + allowed.add(key) + for result in report['Results']: + for finding in result.get('Vulnerabilities') or []: + require(finding['Severity'] in ('UNKNOWN', 'LOW', 'MEDIUM', 'HIGH', 'CRITICAL'), 'Invalid vulnerability severity') + if finding['Severity'] in ('HIGH', 'CRITICAL'): + key = (manifest_digest, finding['VulnerabilityID'], finding['PkgName']) + require(key in allowed, f'Blocked {finding["Severity"]} vulnerability: {finding["VulnerabilityID"]} in {finding["PkgName"]}') + + +def seal(bundle, revision, run_id, platform): + require(re.fullmatch(r'[0-9a-f]{40}', revision), 'Expected immutable source commit') + require(re.fullmatch(r'[0-9]+(?:-[0-9]+)?|local', run_id), 'Invalid run identity') + info = describe(bundle / 'oci', platform) + require(info['revision_label'] == revision, 'Image/source revision mismatch') + inputs = load(bundle / 'inputs.json') + check_inputs(inputs, revision, run_id) + provenance = load(bundle / 'provenance.json') + require(provenance['subject'] == info['manifest_digest'] and provenance['revision'] == revision, 'Provenance subject/source mismatch') + hashes = {name: checksum(bundle / name) for name in SIDECARS} + receipt = {'schema': 1, 'revision': revision, 'run_id': run_id, 'checks': 'packaging-passed', + **info, 'archive_sha256': checksum(bundle / 'image.tar'), 'sidecars': hashes} + (bundle / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + return receipt + + +def check_inputs(inputs, revision, run_id): + require(inputs['revision'] == revision, 'Input/source revision mismatch') + require(type(inputs['dirty']) is bool, 'Input state missing') + require(run_id == 'local' or not inputs['dirty'], 'Dirty source cannot be a trusted release input') + for name in ('source_sha256', 'context_sha256', 'lock_sha256', 'dockerfile_sha256'): + require(isinstance(inputs[name], str) and re.fullmatch(r'[0-9a-f]{64}', inputs[name]), 'Input hash missing') + require(isinstance(inputs['toolchain'], str) and re.search(r'^rust [0-9]+\.[0-9]+\.[0-9]+$', inputs['toolchain'], re.M), 'Toolchain trace missing') + bases = inputs['base_images'] + require(isinstance(bases, list) and bases and all(isinstance(base, str) and + re.fullmatch(r'[^\s@]+@sha256:[0-9a-f]{64}', base) for base in bases), 'Pinned base trace missing') + + +def verify(bundle, revision, run_id, platform, exceptions): + receipt = load(bundle / 'receipt.json') + require(receipt['schema'] == 1 and receipt['checks'] == 'packaging-passed', 'Acceptance missing') + require(receipt['revision'] == revision and receipt['run_id'] == run_id, 'Unauthorized source/run') + require(receipt['platform'] == platform, 'Receipt/platform mismatch') + require(checksum(bundle / 'image.tar') == receipt['archive_sha256'], 'Archive checksum mismatch') + # Never use artifact-provided unpacked paths. Recreate from the retained archive. + with tempfile.TemporaryDirectory() as scratch: + layout = pathlib.Path(scratch) / 'oci' + unpack(bundle / 'image.tar', layout) + info = describe(layout, platform) + config = load(layout / 'blobs/sha256' / info['config_digest'][7:]) + require(all(receipt[k] == v for k, v in info.items()), 'Receipt/image mismatch') + require(info['revision_label'] == revision, 'Image revision label mismatch') + require(set(receipt['sidecars']) == set(SIDECARS), 'Missing sidecar identities') + for name, expected in receipt['sidecars'].items(): + require(checksum(bundle / name) == expected, 'Metadata checksum mismatch') + inputs = load(bundle / 'inputs.json') + check_inputs(inputs, revision, run_id) + provenance = load(bundle / 'provenance.json') + require(provenance['subject'] == info['manifest_digest'] and provenance['revision'] == revision, 'Provenance subject/source mismatch') + require(provenance['inputs'] == inputs and provenance['run_id'] == run_id, 'Provenance input/run mismatch') + scanner = load(bundle / 'scanner.json') + require(scanner.get('Version') and scanner.get('VulnerabilityDB', {}).get('UpdatedAt'), 'Scanner/database identity missing') + require(scanner.get('SubjectManifestDigest') == info['manifest_digest'], 'Scanner subject mismatch') + report = load(bundle / 'scan.json') + require(report.get('Metadata', {}).get('ImageID') == info['config_digest'], 'Scan/image config mismatch') + require(report.get('Metadata', {}).get('DiffIDs') == config['rootfs']['diff_ids'], 'Scan/filesystem mismatch') + family = report.get('Metadata', {}).get('OS', {}).get('Family') + require(family and any(r.get('Class') == 'os-pkgs' and r.get('Type') == family for r in report['Results']), + 'Supported OS scan missing') + require(all(r.get('Class') in ('os-pkgs', 'lang-pkgs') and r.get('Target') for r in report['Results']), + 'Incomplete supported scan results') + scan_policy(report, exceptions, info['manifest_digest']) + sbom = load(bundle / 'sbom.json') + require(sbom.get('bomFormat') == 'CycloneDX' and sbom.get('components'), 'SBOM missing') + properties = sbom.get('metadata', {}).get('component', {}).get('properties', []) + require(any(p.get('name') == 'aquasecurity:trivy:ImageID' and p.get('value') == info['config_digest'] + for p in properties), 'SBOM/image config mismatch') + return receipt + + +def runtime_index(receipts): + require(len(receipts) == 2 and {r['platform'] for r in receipts} == PLATFORMS, 'Both unique native variants required') + require(len({(r['revision'], r['run_id']) for r in receipts}) == 1, 'Variant source/run mismatch') + require(len({r['sidecars']['inputs.json'] for r in receipts}) == 1, 'Variant prepared input mismatch') + return {'schemaVersion': 2, 'mediaType': INDEX, 'manifests': [ + {'mediaType': MANIFEST, 'digest': r['manifest_digest'], 'size': r['manifest_size'], + 'platform': {'os': 'linux', 'architecture': r['platform'].split('/')[1]}} + for r in sorted(receipts, key=lambda r: r['platform'])]} + + +def release_record(source, bundles, metadata, revision, run_id, exceptions): + # Recreate the preparation job's inputs from the trusted commit, never by + # executing or unpacking application-provided code in the publisher. + require(re.fullmatch(r'[0-9a-f]{40}', revision), 'Expected immutable source commit') + raw = subprocess.run(['git', 'archive', '--format=tar', revision], cwd=source, + check=True, stdout=subprocess.PIPE).stdout + files = {} + with tarfile.open(fileobj=io.BytesIO(raw)) as archive: + for entry in archive: + require(entry.isdir() or entry.isfile(), 'Release source must not contain links') + if entry.isfile(): + files[entry.name] = archive.extractfile(entry).read() + context = hashlib.sha256() + for name, data in sorted(files.items()): + context.update(name.encode() + b'\0' + data + b'\0') + expected = {'source_sha256': hashlib.sha256(raw).hexdigest(), 'context_sha256': context.hexdigest(), + 'lock_sha256': hashlib.sha256(files['Cargo.lock']).hexdigest(), + 'dockerfile_sha256': hashlib.sha256(files['Dockerfile']).hexdigest(), + 'toolchain': files['.tool-versions'].decode(), + 'base_images': re.findall(r'^FROM (\S+)', files['Dockerfile'].decode(), re.M), 'dirty': False} + run, attempt = run_id.split('-') + artifact_ids(metadata, run, attempt, revision) + receipts, children = [], [] + for bundle, platform in zip(bundles, ('linux/amd64', 'linux/arm64')): + receipt = verify(bundle, revision, run_id, platform, exceptions) + inputs = load(bundle / 'inputs.json') + require(all(inputs[key] == value for key, value in expected.items()), 'Prepared context differs from trusted source') + receipts.append(receipt) + scanner = load(bundle / 'scanner.json') + arch = platform.split('/')[1] + artifact = next(a for a in metadata['artifacts'] if a['name'] == f'accepted-{arch}-{run}-{attempt}') + children.append({'platform': platform, 'manifest_digest': receipt['manifest_digest'], + 'config_digest': receipt['config_digest'], 'archive_sha256': receipt['archive_sha256'], + 'sidecars': receipt['sidecars'], + 'actions_artifact': {'id': artifact['id'], 'digest': artifact['digest'], 'name': artifact['name']}, + 'scanner': {'version': scanner['Version'], 'database': scanner['VulnerabilityDB']}, + 'tests': 'application-owned checks succeeded in native job'}) + serialized = (json.dumps(runtime_index(receipts), indent=2) + '\n').encode() + return {'version': 1, 'scope': 'application container acceptance, not framework production certification', + 'source': {'revision': revision, **expected}, 'run_id': run_id, 'children': children, + 'index_digest': 'sha256:' + hashlib.sha256(serialized).hexdigest(), + 'scan_policy': 'High/Critical including unfixed; only independently trusted scoped exceptions', + 'exceptions': exceptions} + + +def check_tag(repository, tag, digest, auth_file=None): + require(DIGEST.fullmatch(digest), 'Invalid expected index digest') + require(re.fullmatch(r'[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}', tag), 'Invalid version tag') + host, name = repository.split('/', 1) + require(re.fullmatch(r'[a-z0-9]+(?:[._/-][a-z0-9]+)*', name), 'Invalid repository path') + require(host == 'ghcr.io' or re.fullmatch(r'127\.0\.0\.1:[0-9]+', host), 'Unsupported registry') + headers = {'Accept': ', '.join((INDEX, MANIFEST, 'application/vnd.docker.distribution.manifest.list.v2+json', + 'application/vnd.docker.distribution.manifest.v2+json'))} + + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise ValueError('Registry redirect is not permitted') + + opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect()) + if host == 'ghcr.io': + require(auth_file is not None, 'GHCR auth file required') + auth = load(auth_file)['auths']['ghcr.io']['auth'] + query = urllib.parse.urlencode({'service': host, 'scope': f'repository:{name}:pull'}) + request = urllib.request.Request(f'https://{host}/token?{query}', headers={'Authorization': 'Basic ' + auth}) + with opener.open(request, timeout=30) as response: + token = json.loads(response.read(1_000_001))['token'] + require(isinstance(token, str) and token, 'Registry token missing') + headers['Authorization'] = 'Bearer ' + token + scheme = 'https' if host == 'ghcr.io' else 'http' + request = urllib.request.Request(f'{scheme}://{host}/v2/{name}/manifests/{tag}', headers=headers) + try: + with opener.open(request, timeout=30) as response: + raw = response.read(16_000_001) + require(len(raw) <= 16_000_000, 'Registry manifest too large') + except urllib.error.HTTPError as error: + # Only a confirmed registry not-found response permits a first publish; + # authentication, rate-limit, transport and service errors fail closed. + require(error.code == 404, f'Registry tag check failed: HTTP {error.code}') + errors = json.loads(error.read(1_000_001)).get('errors', []) + require(errors and all(e.get('code') in ('MANIFEST_UNKNOWN', 'NAME_UNKNOWN') for e in errors), + 'Registry did not confirm tag absence') + return {'state': 'absent'} + require('sha256:' + hashlib.sha256(raw).hexdigest() == digest, 'Version tag identifies a different release') + return {'state': 'same-index'} + + +def artifact_ids(metadata, run_id, attempt, revision=None): + names = {f'accepted-{arch}-{run_id}-{attempt}' for arch in ('amd64', 'arm64')} + selected = [a for a in metadata['artifacts'] if a['name'] in names] + require(len(selected) == 2 and {a['name'] for a in selected} == names, 'Accepted artifacts missing or duplicated') + for artifact in selected: + require(type(artifact['id']) is int and artifact['id'] > 0 and not artifact['expired'], 'Invalid artifact identity') + require(DIGEST.fullmatch(artifact['digest']), 'Actions artifact digest missing') + require(str(artifact['workflow_run']['id']) == run_id, 'Artifact run mismatch') + if revision is not None: + require(artifact['workflow_run']['head_sha'] == revision, 'Artifact producer source mismatch') + return ','.join(str(a['id']) for a in sorted(selected, key=lambda a: a['name'])) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest='command', required=True) + extract = sub.add_parser('extract') + extract.add_argument('archive', type=pathlib.Path) + extract.add_argument('destination', type=pathlib.Path) + extract.add_argument('--platform', required=True, choices=sorted(PLATFORMS)) + for command in ('seal', 'verify'): + child = sub.add_parser(command) + child.add_argument('bundle', type=pathlib.Path) + child.add_argument('--revision', required=True) + child.add_argument('--run-id', required=True) + child.add_argument('--platform', required=True, choices=sorted(PLATFORMS)) + if command == 'verify': + child.add_argument('--exceptions', type=pathlib.Path) + index = sub.add_parser('index') + index.add_argument('bundles', nargs=2, type=pathlib.Path) + index.add_argument('--revision', required=True) + index.add_argument('--run-id', required=True) + index.add_argument('--exceptions', type=pathlib.Path) + pulled = sub.add_parser('pulled') + pulled.add_argument('layout', type=pathlib.Path) + pulled.add_argument('inspection', type=pathlib.Path) + pulled.add_argument('--platform', required=True, choices=sorted(PLATFORMS)) + pulled.add_argument('--reference', required=True) + artifacts = sub.add_parser('artifact-ids') + artifacts.add_argument('metadata', type=pathlib.Path) + artifacts.add_argument('--run-id', required=True) + artifacts.add_argument('--attempt', required=True) + artifacts.add_argument('--revision') + record = sub.add_parser('release-record') + record.add_argument('source', type=pathlib.Path) + record.add_argument('bundles', nargs=2, type=pathlib.Path) + record.add_argument('metadata', type=pathlib.Path) + record.add_argument('--revision', required=True) + record.add_argument('--run-id', required=True) + record.add_argument('--exceptions', type=pathlib.Path) + tag = sub.add_parser('tag-check') + tag.add_argument('repository') + tag.add_argument('tag') + tag.add_argument('digest') + tag.add_argument('--auth-file', type=pathlib.Path) + args = parser.parse_args() + try: + if args.command == 'release-record': + exceptions = load(args.exceptions) if args.exceptions else [] + print(json.dumps(release_record(args.source, args.bundles, load(args.metadata), + args.revision, args.run_id, exceptions), sort_keys=True)) + return + if args.command == 'tag-check': + print(json.dumps(check_tag(args.repository, args.tag, args.digest, args.auth_file))) + return + if args.command == 'artifact-ids': + print(artifact_ids(load(args.metadata), args.run_id, args.attempt, args.revision)) + return + if args.command == 'extract': + unpack(args.archive, args.destination) + result = describe(args.destination, args.platform) + elif args.command == 'seal': + result = seal(args.bundle, args.revision, args.run_id, args.platform) + elif args.command == 'verify': + exceptions = load(args.exceptions) if args.exceptions else [] + result = verify(args.bundle, args.revision, args.run_id, args.platform, exceptions) + elif args.command == 'pulled': + result = verify_pulled(args.layout, load(args.inspection), args.platform, args.reference) + else: + exceptions = load(args.exceptions) if args.exceptions else [] + result = runtime_index([verify(bundle, args.revision, args.run_id, platform, exceptions) + for bundle, platform in zip(args.bundles, ('linux/amd64', 'linux/arm64'))]) + print(json.dumps(result, indent=2)) + except (ValueError, KeyError, TypeError, IndexError, OSError, tarfile.TarError, subprocess.CalledProcessError) as error: + parser.exit(1, f'Artifact verification failed: {error}\n') + + +if __name__ == '__main__': + main() diff --git a/scripts/prepare_generated_axum_container.sh b/scripts/prepare_generated_axum_container.sh new file mode 100755 index 000000000..173b1184e --- /dev/null +++ b/scripts/prepare_generated_axum_container.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Test-only source staging. Scaffolding and production dependency policy are unchanged. +# Usage: prepare_generated_axum_container.sh /empty/output/parent +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PARENT="${1:?pass a new disposable output directory}" +[[ $# -eq 1 ]] || { echo 'Expected one output directory' >&2; exit 2; } +[[ ! -e "$PARENT" ]] || { echo 'Output directory must not exist' >&2; exit 2; } +mkdir -p "$PARENT" +PARENT="$(cd "$PARENT" && pwd)" +[[ "$PARENT/" != "$ROOT/"* ]] || { echo 'Fixture must be outside the source checkout' >&2; exit 2; } +RUSTUP_TOOLCHAIN="$(awk '$1 == "rust" { print $2 }' "$ROOT/.tool-versions")" +export RUSTUP_TOOLCHAIN +[[ "$RUSTUP_TOOLCHAIN" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo 'Expected declared Rust pin' >&2; exit 2; } +cargo run --locked --manifest-path "$ROOT/Cargo.toml" -p edgezero-cli --bin edgezero -- \ + new container-probe --dir "$PARENT" +APP="$PARENT/container-probe" +python3 - "$ROOT" "$APP" <<'PY' +import hashlib, json, pathlib, re, shutil, subprocess, sys, tomllib +root, app = map(pathlib.Path, sys.argv[1:]) +vendor = app / 'vendor' / 'edgezero' +paths = subprocess.check_output(['git', 'ls-files', '-z', '--cached', '--others', '--exclude-standard'], cwd=root).decode().split('\0') +source_hash = hashlib.sha256() +for name in sorted(set(filter(None, paths))): + # Only the crate graph and its required optional example sources are needed. + if name not in ('Cargo.toml', 'Cargo.lock', '.tool-versions') and not name.startswith(('crates/', 'examples/app-demo/')): + continue + source = root / name + if not source.is_file() or source.is_symlink() or not source.resolve().is_relative_to(root): + raise ValueError(f'Expected regular source file: {name}') + destination = vendor / name + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + source_hash.update(name.encode() + b'\0' + source.read_bytes() + b'\0') +manifest = app / 'Cargo.toml' +text = manifest.read_text() +deps = tomllib.loads(text)['workspace']['dependencies'] +for name, dep in deps.items(): + if not name.startswith('edgezero-'): + continue + if not isinstance(dep, dict) or 'path' not in dep or not (vendor / 'crates' / name / 'Cargo.toml').is_file(): + raise ValueError(f'Expected generated checkout dependency: {name}') + pattern = rf'(?m)^({re.escape(name)}\s*=\s*\{{[^\n]*?\bpath\s*=\s*)"[^"]*"' + text, count = re.subn(pattern, lambda m: m[1] + f'"vendor/edgezero/crates/{name}"', text) + if count != 1: + raise ValueError(f'Cannot stage generated dependency: {name}') +# Cargo otherwise auto-enrolls in-tree path dependencies into the app workspace, +# losing the staged framework's own workspace dependency/lint inheritance. +require_resolver = 'resolver = "2"' +if text.count(require_resolver) != 1: + raise ValueError('Unexpected generated workspace resolver') +text = text.replace(require_resolver, 'exclude = ["vendor/edgezero"]\n' + require_resolver, 1) +manifest.write_text(text) +shutil.rmtree(app / '.git') +revision = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=root, text=True).strip() +(app / 'fixture-source.json').write_text(json.dumps({ + 'revision': revision, 'source_sha256': source_hash.hexdigest(), + 'dirty': bool(subprocess.check_output(['git', 'status', '--porcelain'], cwd=root)), + 'dependency_mode': 'staged-test-sources', +}, indent=2) + '\n') +PY +( + cd "$APP" + cargo generate-lockfile + cargo metadata --locked --format-version 1 > "$PARENT/metadata.json" +) +python3 - "$APP" "$PARENT/metadata.json" <<'PY' +import hashlib, json, pathlib, re, sys +app = pathlib.Path(sys.argv[1]) +metadata = json.loads(pathlib.Path(sys.argv[2]).read_text()) +for package in metadata['packages']: + if package['source'] is None and not pathlib.Path(package['manifest_path']).resolve().is_relative_to(app): + raise ValueError(f'Out-of-context dependency: {package["name"]}') +inputs = json.loads((app / 'fixture-source.json').read_text()) +inputs['package'] = 'container-probe-adapter-axum' +inputs['binary'] = 'container-probe-adapter-axum' +inputs['lock_sha256'] = hashlib.sha256((app / 'Cargo.lock').read_bytes()).hexdigest() +inputs['toolchain'] = (app / '.tool-versions').read_text() +inputs['dockerfile_sha256'] = hashlib.sha256((app / 'Dockerfile').read_bytes()).hexdigest() +inputs['base_images'] = re.findall(r'^FROM (\S+)', (app / 'Dockerfile').read_text(), re.M) +context = hashlib.sha256() +for path in sorted(app.rglob('*'), key=lambda path: path.relative_to(app).as_posix()): + if path.is_file(): + context.update(path.relative_to(app).as_posix().encode() + b'\0' + path.read_bytes() + b'\0') +inputs['context_sha256'] = context.hexdigest() +(app / 'fixture-inputs.json').write_text(json.dumps(inputs, indent=2) + '\n') +PY +printf 'Prepared disposable fixture: %s\n' "$APP" diff --git a/scripts/test_axum_container_deployment.sh b/scripts/test_axum_container_deployment.sh new file mode 100755 index 000000000..4c6cf7acc --- /dev/null +++ b/scripts/test_axum_container_deployment.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Opt-in no-store Compose packaging check; not readiness/storage/drain proof. +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +export AXUM_IMAGE="${1:?pass a local loaded image digest}" +NAME="${2:-container-probe}" +[[ $# -le 2 && "$AXUM_IMAGE" == *@sha256:* && "$NAME" =~ ^[a-zA-Z0-9_-]+$ ]] \ + || { echo 'Expected image digest and generated app name' >&2; exit 2; } +export AXUM_PORT=0 +PROJECT="edgezero-compose-$$-$RANDOM" +WORK="$(mktemp -d -t edgezero-compose.XXXXXX)" +compose() { docker compose -p "$PROJECT" -f "$ROOT/examples/axum-deployment/compose.yaml" "$@"; } +cleanup() { + status=$? + compose logs > "$WORK/compose.log" 2>&1 || true + compose down >/dev/null 2>&1 || true + if [[ "$status" -eq 0 ]]; then rm -rf "$WORK"; else echo "Diagnostics: $WORK" >&2; fi + exit "$status" +} +trap cleanup EXIT +compose config > "$WORK/config.yaml" +compose up -d --no-build --pull never +CONTAINER="$(compose ps -q app)" +PORT="$(compose port app 8787 | awk -F: '{print $NF}')" +for _ in $(seq 1 30); do + curl --fail --silent --max-time 2 "http://127.0.0.1:$PORT/" > "$WORK/body" && break + sleep 1 +done +curl --fail --silent --max-time 2 "http://127.0.0.1:$PORT/" > "$WORK/body" +[[ "$(< "$WORK/body")" == "$NAME app" ]] +[[ "$(docker inspect "$CONTAINER" --format '{{.HostConfig.ReadonlyRootfs}}')" == true ]] +docker exec "$CONTAINER" sh -ec ' + test "$(id -u):$(id -g)" = 10001:10001 + grep -q "^CapEff:[[:space:]]*0000000000000000$" /proc/self/status + grep -q "^NoNewPrivs:[[:space:]]*1$" /proc/self/status + ! touch /app/forbidden 2>/dev/null +' +compose stop +compose start +PORT="$(compose port app 8787 | awk -F: '{print $NF}')" +for _ in $(seq 1 30); do + curl --fail --silent --max-time 2 "http://127.0.0.1:$PORT/" > "$WORK/body" && break + sleep 1 +done +curl --fail --silent --max-time 2 "http://127.0.0.1:$PORT/" > "$WORK/body" +[[ "$(< "$WORK/body")" == "$NAME app" ]] +echo 'PASS: no-store Compose config, restricted HTTP startup, stop/start and teardown.' +echo 'No production readiness, durable state or bounded SIGTERM acceptance is implied.' diff --git a/scripts/test_axum_container_handoff.sh b/scripts/test_axum_container_handoff.sh new file mode 100755 index 000000000..ecd56ac40 --- /dev/null +++ b/scripts/test_axum_container_handoff.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Native build/test or credential-free import of this recipe's accepted OCI bundle. +# build APP_DIR OUTPUT PLATFORM REVISION RUN_ID +# import BUNDLE_DIR OUTPUT PLATFORM REVISION RUN_ID +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=../examples/container-publishing/tools.env +source "$ROOT/examples/container-publishing/tools.env" +VERIFY="$ROOT/examples/container-publishing/verify_artifacts.py" +MODE="${1:?build or import}"; INPUT="${2:?input directory}"; OUTPUT="${3:?new output directory}" +PLATFORM="${4:?linux/amd64 or linux/arm64}"; REVISION="${5:?source SHA}"; RUN_ID="${6:?run identity}" +[[ $# -eq 6 && ( "$MODE" == build || "$MODE" == import ) ]] || { echo 'Invalid handoff arguments' >&2; exit 2; } +[[ "$PLATFORM" == linux/amd64 || "$PLATFORM" == linux/arm64 ]] || { echo 'Unsupported platform' >&2; exit 2; } +if [[ "$MODE" == build ]]; then + case "$(docker info --format '{{.Architecture}}')" in + x86_64|amd64) HOST_PLATFORM=linux/amd64 ;; + aarch64|arm64) HOST_PLATFORM=linux/arm64 ;; + *) echo 'Unsupported native Docker host' >&2; exit 2 ;; + esac + [[ "$PLATFORM" == "$HOST_PLATFORM" ]] || { echo 'Native build required; emulation is not acceptance' >&2; exit 2; } +fi +VERIFY_ARGS=() +if [[ -n "${SCAN_EXCEPTIONS:-}" ]]; then VERIFY_ARGS+=(--exceptions "$SCAN_EXCEPTIONS"); fi +[[ ! -e "$OUTPUT" ]] || { echo 'Output must not exist' >&2; exit 2; } +INPUT="$(cd "$INPUT" && pwd)" +mkdir -p "$OUTPUT" +OUTPUT="$(cd "$OUTPUT" && pwd)" +WORK="$(mktemp -d -t edgezero-handoff.XXXXXX)" +ID="edgezero-handoff-$$-$RANDOM" +BUILDER="$ID-builder" +REGISTRY="$ID-registry" +IMAGE="" +cleanup() { + status=$? + docker logs "$REGISTRY" > "$OUTPUT/registry.log" 2>&1 || true + docker rm -f "$REGISTRY" >/dev/null 2>&1 || true + if [[ "$MODE" == build ]]; then docker buildx rm "$BUILDER" >/dev/null 2>&1 || true; fi + if [[ -n "$IMAGE" ]]; then docker image rm "$IMAGE" >/dev/null 2>&1 || true; fi + rm -rf "$WORK" + exit "$status" +} +trap cleanup EXIT + +if [[ "$MODE" == build ]]; then + docker buildx create --name "$BUILDER" --driver docker-container --driver-opt "image=$BUILDKIT_IMAGE" > "$OUTPUT/builder.log" + docker buildx build --builder "$BUILDER" --platform "$PLATFORM" --provenance=false --sbom=false \ + --label "org.opencontainers.image.revision=$REVISION" \ + --label "org.opencontainers.image.source=${GITHUB_SERVER_URL:-https://github.com}/${GITHUB_REPOSITORY:-stackpop/edgezero}" \ + --output "type=oci,dest=$OUTPUT/image.tar,name=app" "$INPUT" 2>&1 | tee "$OUTPUT/build.log" + cp "$INPUT/fixture-inputs.json" "$OUTPUT/inputs.json" +else + python3 "$VERIFY" verify "$INPUT" --revision "$REVISION" --run-id "$RUN_ID" --platform "$PLATFORM" \ + "${VERIFY_ARGS[@]}" > "$OUTPUT/verified.json" + cp "$INPUT/image.tar" "$OUTPUT/image.tar" +fi +python3 "$VERIFY" extract "$OUTPUT/image.tar" "$WORK/oci" --platform "$PLATFORM" > "$OUTPUT/image.json" +DIGEST="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["manifest_digest"])' "$OUTPUT/image.json")" +docker run -d --name "$REGISTRY" -p 127.0.0.1::5000 "$REGISTRY_IMAGE" > "$OUTPUT/registry-id" +PORT="$(docker port "$REGISTRY" 5000/tcp | awk -F: '{print $NF}')" +REG="127.0.0.1:$PORT" +for _ in $(seq 1 30); do + curl --fail --silent "http://$REG/v2/" >/dev/null && break + sleep 1 +done +curl --fail --silent "http://$REG/v2/" >/dev/null +# Skopeo talks to loopback in the host namespace; only disposable local registry data is exposed. +docker run --rm --network host --user "$(id -u):$(id -g)" --cap-drop=ALL \ + --security-opt=no-new-privileges:true --mount "type=bind,src=$WORK,dst=/work,readonly" \ + "$SKOPEO_IMAGE" copy --all --preserve-digests --dest-tls-verify=false \ + oci:/work/oci "docker://$REG/app:candidate" 2>&1 | tee "$OUTPUT/copy.log" +curl --fail --silent -H 'Accept: application/vnd.oci.image.manifest.v1+json' \ + "http://$REG/v2/app/manifests/$DIGEST" > "$OUTPUT/registry-manifest.json" +[[ "sha256:$(sha256sum "$OUTPUT/registry-manifest.json" | awk '{print $1}')" == "$DIGEST" ]] \ + || { echo 'Registry changed manifest bytes' >&2; exit 1; } +IMAGE="$REG/app@$DIGEST" +docker pull --platform "$PLATFORM" "$IMAGE" > "$OUTPUT/pull.log" +docker image inspect "$IMAGE" > "$OUTPUT/pulled-image.json" +python3 "$VERIFY" pulled "$WORK/oci" "$OUTPUT/pulled-image.json" --platform "$PLATFORM" \ + --reference "$IMAGE" > "$OUTPUT/pulled-verified.json" + +if [[ "$MODE" == build ]]; then + if [[ -n "${APP_SMOKE_SCRIPT:-}" ]]; then + # Application-owned test script is permitted only in the credential-free build job. + "$APP_SMOKE_SCRIPT" "$IMAGE" "$PLATFORM" + else + "$ROOT/scripts/test_generated_axum_container.sh" "$INPUT" --image "$IMAGE" \ + --platform "$PLATFORM" --evidence-dir "$OUTPUT/smoke" + APP_NAME="$(python3 -c 'import sys,tomllib; print(tomllib.load(open(sys.argv[1], "rb"))["app"]["name"])' "$INPUT/edgezero.toml")" + "$ROOT/scripts/test_axum_container_deployment.sh" "$IMAGE" "$APP_NAME" + fi + mkdir "$OUTPUT/scanner-cache" + docker run --rm --user "$(id -u):$(id -g)" --mount "type=bind,src=$OUTPUT,dst=/work" \ + --mount "type=bind,src=$WORK/oci,dst=/oci,readonly" \ + "$TRIVY_IMAGE" image --cache-dir /work/scanner-cache --input /oci \ + --scanners vuln --ignorefile /dev/null --format json --output /work/scan.json + docker run --rm --user "$(id -u):$(id -g)" --mount "type=bind,src=$OUTPUT,dst=/work" \ + "$TRIVY_IMAGE" version --cache-dir /work/scanner-cache --format json > "$OUTPUT/scanner.json" + docker run --rm --user "$(id -u):$(id -g)" --mount "type=bind,src=$OUTPUT,dst=/work" \ + --mount "type=bind,src=$WORK/oci,dst=/oci,readonly" \ + "$TRIVY_IMAGE" image --cache-dir /work/scanner-cache --input /oci \ + --format cyclonedx --output /work/sbom.json + python3 - "$OUTPUT" "$DIGEST" "$REVISION" "$RUN_ID" "$BUILDKIT_IMAGE" <<'PY' +import json, pathlib, sys +out = pathlib.Path(sys.argv[1]) +inputs = json.loads((out / 'inputs.json').read_text()) +(out / 'provenance.json').write_text(json.dumps({ + 'subject': sys.argv[2], 'revision': sys.argv[3], 'run_id': sys.argv[4], + 'builder': sys.argv[5], 'inputs': inputs, + 'note': 'Unsigned build receipt. Trusted publisher authentication is separate.', +}, indent=2) + '\n') +PY + python3 - "$OUTPUT/scanner.json" "$DIGEST" <<'PY' +import json, pathlib, sys +path = pathlib.Path(sys.argv[1]) +metadata = json.loads(path.read_text()) +metadata['SubjectManifestDigest'] = sys.argv[2] +path.write_text(json.dumps(metadata, indent=2) + '\n') +PY + # Seal needs the validated layout, but only the archive/sidecars are transferred. + ln -s "$WORK/oci" "$OUTPUT/oci" + python3 "$VERIFY" seal "$OUTPUT" --revision "$REVISION" --run-id "$RUN_ID" --platform "$PLATFORM" > "$OUTPUT/accepted.json" + rm "$OUTPUT/oci" + python3 "$VERIFY" verify "$OUTPUT" --revision "$REVISION" --run-id "$RUN_ID" --platform "$PLATFORM" "${VERIFY_ARGS[@]}" > "$OUTPUT/verified.json" + rm -rf "$OUTPUT/scanner-cache" +fi +# Transfer into a distinct directory and verify/import there in a later job with MODE=import. +printf '%s\n' "$DIGEST" > "$OUTPUT/digest" +echo "PASS: preserved $PLATFORM manifest $DIGEST through OCI registry copy and Docker pull." diff --git a/scripts/test_generated_axum_container.sh b/scripts/test_generated_axum_container.sh index 122dcc251..e39af6b6e 100755 --- a/scripts/test_generated_axum_container.sh +++ b/scripts/test_generated_axum_container.sh @@ -6,6 +6,26 @@ set -euo pipefail # Usage: ./scripts/test_generated_axum_container.sh /path/to/container-probe # Requires Docker, Python 3.11+, curl, OpenSSL, readelf and network access for image builds. APP_DIR="$(cd "${1:?pass a prepared disposable generated workspace}" && pwd)" +shift +BORROWED_IMAGE="" +PLATFORM="" +EVIDENCE="" +while [[ $# -gt 0 ]]; do + case "$1" in + --image) BORROWED_IMAGE="${2:?--image requires a reference}"; shift 2 ;; + --platform) PLATFORM="${2:?--platform requires linux/amd64 or linux/arm64}"; shift 2 ;; + --evidence-dir) EVIDENCE="${2:?--evidence-dir requires a directory}"; shift 2 ;; + *) echo "Unknown option: $1" >&2; exit 2 ;; + esac +done +[[ -z "$PLATFORM" || "$PLATFORM" == linux/amd64 || "$PLATFORM" == linux/arm64 ]] \ + || { echo 'Unsupported platform' >&2; exit 2; } +[[ "$BORROWED_IMAGE" != -* ]] || { echo 'Invalid image reference' >&2; exit 2; } +if [[ -n "$EVIDENCE" ]]; then + [[ ! -e "$EVIDENCE" ]] || { echo 'Evidence directory must not exist' >&2; exit 2; } + mkdir -p "$EVIDENCE" + EVIDENCE="$(cd "$EVIDENCE" && pwd)" +fi for tool in docker python3 curl openssl readelf sha256sum; do command -v "$tool" >/dev/null || { echo "Missing prerequisite: $tool" >&2; exit 1; } done @@ -22,7 +42,7 @@ for file in Dockerfile .dockerignore Cargo.lock .tool-versions; do done WORK="$(mktemp -d -t edgezero-container.XXXXXX)" ID="edgezero-smoke-$$-$RANDOM" -IMAGE="$ID:app" +IMAGE="${BORROWED_IMAGE:-$ID:app}" PROBE="$ID:context" NETWORK="$ID" CONTAINERS=() @@ -33,9 +53,17 @@ cleanup() { docker rm -f "$container" >/dev/null 2>&1 || true done docker network rm "$NETWORK" >/dev/null 2>&1 || true - docker image rm "$IMAGE" "$PROBE" >/dev/null 2>&1 || true + docker image rm "$PROBE" >/dev/null 2>&1 || true + if [[ -z "$BORROWED_IMAGE" ]]; then docker image rm "$IMAGE" >/dev/null 2>&1 || true; fi # Always remove disposable private keys, including on failure. rm -f "$WORK/tls/ca.key" "$WORK/tls/server.key" + if [[ -n "$EVIDENCE" ]]; then + # Contexts and TLS directories are not uploadable evidence. + for log in "$WORK"/*.log "$WORK"/*-body "$WORK/root-body"; do + [[ ! -f "$log" ]] || cp "$log" "$EVIDENCE/" + done + printf '%s\n' "$status" > "$EVIDENCE/exit-status" + fi if test "$status" -eq 0; then rm -rf "$WORK" else @@ -45,7 +73,15 @@ cleanup() { } trap cleanup EXIT -echo "==> Packaging fixture $NAME; host $(docker info --format '{{.Architecture}}')" +HOST_ARCH="$(docker info --format '{{.Architecture}}')" +case "$HOST_ARCH" in + x86_64|amd64) HOST_PLATFORM=linux/amd64 ;; + aarch64|arm64) HOST_PLATFORM=linux/arm64 ;; + *) echo "Unsupported native host: $HOST_ARCH" >&2; exit 1 ;; +esac +PLATFORM="${PLATFORM:-$HOST_PLATFORM}" +[[ "$PLATFORM" == "$HOST_PLATFORM" ]] || { echo 'Native host/platform mismatch' >&2; exit 1; } +echo "==> Packaging fixture $NAME; native $PLATFORM" sha256sum "$APP_DIR/Cargo.lock" "$APP_DIR/.tool-versions" python3 - "$APP_DIR/Cargo.toml" <<'PY' import sys, tomllib @@ -73,7 +109,7 @@ printf 'disposable secret\n' > "$CONTEXT/.cargo/credentials.toml" printf 'disposable config\n' > "$CONTEXT/$NAME.toml" # Test Docker's actual context rules, not a substring interpretation of ignore patterns. cat > "$WORK/context.Dockerfile" <<'DOCKER' -FROM debian:bookworm-slim +FROM debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251 COPY . /context ARG APP_NAME RUN test ! -e "/context/$APP_NAME.toml" \ @@ -87,8 +123,13 @@ RUN test ! -e "/context/$APP_NAME.toml" \ && ! find /context -name container-sentinel | grep . DOCKER docker build --progress=plain --build-arg "APP_NAME=$NAME" -f "$WORK/context.Dockerfile" -t "$PROBE" "$CONTEXT" 2>&1 | tee "$WORK/context.log" -docker build --progress=plain -t "$IMAGE" "$CONTEXT" 2>&1 | tee "$WORK/build.log" -docker image inspect "$IMAGE" --format 'image={{.Id}} architecture={{.Architecture}} user={{.Config.User}}' +if [[ -z "$BORROWED_IMAGE" ]]; then + docker build --progress=plain -t "$IMAGE" "$CONTEXT" 2>&1 | tee "$WORK/build.log" +fi +IMAGE_PLATFORM="$(docker image inspect "$IMAGE" --format '{{.Os}}/{{.Architecture}}')" +[[ "$IMAGE_PLATFORM" == "$PLATFORM" ]] || { echo 'Image/platform mismatch' >&2; exit 1; } +docker image inspect "$IMAGE" --format 'image={{.Id}} architecture={{.Architecture}} user={{.Config.User}}' \ + | tee "$WORK/image.log" docker history --no-trunc "$IMAGE" > "$WORK/history.log" CONTAINERS+=("$ID-export") @@ -99,7 +140,7 @@ docker cp "$ID-export:/usr/local/bin/app" "$WORK/app" readelf -h -l -d "$WORK/app" > "$WORK/elf.log" grep -E 'Machine:|interpreter|NEEDED' "$WORK/elf.log" test -s "$WORK/os-roots.pem" -python3 - "$WORK/runtime.tar" <<'PY' +python3 - "$WORK/runtime.tar" "$PLATFORM" <<'PY' import sys, tarfile with tarfile.open(sys.argv[1]) as archive: paths = {m.name.lstrip('./') for m in archive} @@ -110,7 +151,10 @@ with tarfile.open(sys.argv[1]) as archive: assert 'usr/local/bin/app' in paths header = archive.extractfile('usr/local/bin/app').read(20) assert header[:4] == b'\x7fELF', 'not a native ELF executable' - print('ELF class', header[4], 'machine', int.from_bytes(header[18:20], 'little')) + machine = int.from_bytes(header[18:20], 'little') + assert header[4:6] == bytes((2, 1)), 'expected ELF64 little-endian' + assert machine == {'linux/amd64': 62, 'linux/arm64': 183}[sys.argv[2]], 'ELF/platform mismatch' + print('ELF class', header[4], 'machine', machine) PY docker run --rm --read-only --cap-drop=ALL --security-opt=no-new-privileges:true \ --entrypoint sh "$IMAGE" -ec ' @@ -191,4 +235,4 @@ status="$(curl --silent --max-time 35 -o "$WORK/wrong-host-body" -w '%{http_code [[ "$status" =~ ^5[0-9][0-9]$ ]] if grep -q 'edgezero controlled TLS' "$WORK/wrong-host-body"; then exit 1; fi echo 'PASS: generated app context, native libraries, UID, read-only root, capabilities, mounts, HTTP and controlled TLS.' -echo 'Packaging only; no readiness, durable restart, SIGTERM or multi-architecture certification.' +echo 'Packaging only; no readiness, durable restart, SIGTERM or production certification.' diff --git a/scripts/tests/test_axum_container_artifacts.py b/scripts/tests/test_axum_container_artifacts.py new file mode 100644 index 000000000..53ca210d8 --- /dev/null +++ b/scripts/tests/test_axum_container_artifacts.py @@ -0,0 +1,368 @@ +import gzip +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path +import subprocess +import tarfile +import tempfile +import unittest +from unittest import mock +import urllib.error + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location('verify', ROOT / 'examples/container-publishing/verify_artifacts.py') +v = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(v) +REV = 'a' * 40 + + +def put_json(path, value): + path.write_text(json.dumps(value)) + + +def fixture(root, arch='amd64', revision=REV): + layout = root / 'oci' + (layout / 'blobs/sha256').mkdir(parents=True) + + def store(value, media_type): + data = json.dumps(value).encode() if isinstance(value, dict) else value + digest = hashlib.sha256(data).hexdigest() + (layout / 'blobs/sha256' / digest).write_bytes(data) + return {'mediaType': media_type, 'size': len(data), 'digest': 'sha256:' + digest} + + raw_layer = io.BytesIO() + with tarfile.open(fileobj=raw_layer, mode='w') as tar: + entry = tarfile.TarInfo('transport-fixture') + entry.size = 5 + tar.addfile(entry, io.BytesIO(b'layer')) + diff_id = 'sha256:' + hashlib.sha256(raw_layer.getvalue()).hexdigest() + config = store({'os': 'linux', 'architecture': arch, + 'rootfs': {'type': 'layers', 'diff_ids': [diff_id]}, + 'config': {'Labels': {'org.opencontainers.image.revision': revision}}}, v.CONFIG) + layer = store(gzip.compress(raw_layer.getvalue(), mtime=0), 'application/vnd.oci.image.layer.v1.tar+gzip') + image = store({'schemaVersion': 2, 'mediaType': v.MANIFEST, 'config': config, 'layers': [layer]}, v.MANIFEST) + put_json(layout / 'oci-layout', {'imageLayoutVersion': '1.0.0'}) + put_json(layout / 'index.json', {'schemaVersion': 2, 'mediaType': v.INDEX, 'manifests': [image]}) + with tarfile.open(root / 'image.tar', 'w') as archive: + for path in sorted(layout.rglob('*')): + archive.add(path, arcname=path.relative_to(layout).as_posix(), recursive=False) + put_json(root / 'inputs.json', {'revision': revision, 'dirty': False, 'toolchain': 'rust 1.95.0\n', + 'source_sha256': 'a' * 64, 'context_sha256': 'b' * 64, + 'lock_sha256': 'c' * 64, 'dockerfile_sha256': 'd' * 64, + 'base_images': ['debian@sha256:' + 'e' * 64]}) + put_json(root / 'scan.json', {'SchemaVersion': 2, 'Metadata': {'ImageID': config['digest'], + 'DiffIDs': [diff_id], 'OS': {'Family': 'debian'}}, + 'Results': [{'Class': 'os-pkgs', 'Type': 'debian', 'Target': 'fixture', 'Vulnerabilities': []}]}) + put_json(root / 'scanner.json', {'Version': 'test', 'VulnerabilityDB': {'UpdatedAt': '2026-10-07'}, + 'SubjectManifestDigest': image['digest']}) + put_json(root / 'sbom.json', {'bomFormat': 'CycloneDX', 'specVersion': '1.6', 'version': 1, + 'components': [{'type': 'library', 'name': 'transport-fixture'}], + 'metadata': {'component': {'type': 'container', 'name': 'fixture', 'properties': [ + {'name': 'aquasecurity:trivy:ImageID', 'value': config['digest']} ]}}}) + put_json(root / 'provenance.json', {'revision': revision, 'subject': image['digest'], + 'run_id': '123-1', 'inputs': v.load(root / 'inputs.json')}) + return image + + +class Artifacts(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.image = fixture(self.root) + + def seal(self): + return v.seal(self.root, REV, '123-1', 'linux/amd64') + + def verify(self, revision=REV, run_id='123-1', platform='linux/amd64'): + return v.verify(self.root, revision, run_id, platform, []) + + def test_receipt_and_archive_roundtrip(self): + receipt = self.seal() + self.assertEqual(self.verify(), receipt) + v.unpack(self.root / 'image.tar', self.root / 'copy') + self.assertEqual(v.describe(self.root / 'copy', 'linux/amd64')['manifest_digest'], self.image['digest']) + + def test_arm64(self): + other = self.root / 'arm' + other.mkdir() + fixture(other, 'arm64') + v.seal(other, REV, '123-1', 'linux/arm64') + v.verify(other, REV, '123-1', 'linux/arm64', []) + + def test_wrong_run_source_platform_and_acceptance(self): + receipt = self.seal() + for kwargs in ({'revision': 'b' * 40}, {'run_id': '124-1'}, {'platform': 'linux/arm64'}): + with self.subTest(kwargs=kwargs), self.assertRaises(ValueError): + self.verify(**kwargs) + receipt['checks'] = 'failed' + put_json(self.root / 'receipt.json', receipt) + with self.assertRaises(ValueError): + self.verify() + + def test_mutated_archive_or_sidecar(self): + self.seal() + with open(self.root / 'image.tar', 'ab') as stream: + stream.write(b'changed') + with self.assertRaises(ValueError): + self.verify() + self.seal() + put_json(self.root / 'sbom.json', {'bomFormat': 'changed'}) + with self.assertRaises(ValueError): + self.verify() + + def test_empty_scan_wrong_subject_and_dirty_source(self): + for changes in ({'Results': []}, {'Metadata': {'ImageID': 'sha256:' + 'f' * 64}}): + report = v.load(self.root / 'scan.json') + report.update(changes) + put_json(self.root / 'scan.json', report) + self.seal() + with self.assertRaises(ValueError): + self.verify() + inputs = v.load(self.root / 'inputs.json') + inputs['dirty'] = True + put_json(self.root / 'inputs.json', inputs) + with self.assertRaises(ValueError): + self.seal() + + def test_pulled_identity_on_both_docker_stores(self): + info = v.describe(self.root / 'oci', 'linux/amd64') + config = v.load(self.root / 'oci/blobs/sha256' / info['config_digest'][7:]) + reference = '127.0.0.1/app@' + info['manifest_digest'] + inspection = {'Id': info['manifest_digest'], 'RepoDigests': [reference], + 'Os': 'linux', 'Architecture': 'amd64', 'Config': config['config'], + 'RootFS': {'Layers': config['rootfs']['diff_ids']}, + 'Descriptor': {'digest': info['manifest_digest']}} + v.verify_pulled(self.root / 'oci', [inspection], 'linux/amd64', reference) + inspection['Id'] = info['config_digest'] + inspection.pop('Descriptor') + v.verify_pulled(self.root / 'oci', [inspection], 'linux/amd64', reference) + inspection['RootFS']['Layers'] = [] + with self.assertRaises(ValueError): + v.verify_pulled(self.root / 'oci', [inspection], 'linux/amd64', reference) + + def test_invalid_base_pins_and_symlink_sidecars(self): + original = v.load(self.root / 'inputs.json') + for bases in ('not-a-list', [], ['debian@sha256:garbage'], [5]): + with self.assertRaises(ValueError): + v.check_inputs(dict(original, base_images=bases), REV, '123-1') + self.seal() + (self.root / 'sbom.json').rename(self.root / 'other.json') + (self.root / 'sbom.json').symlink_to(self.root / 'other.json') + with self.assertRaises(ValueError): + self.verify() + + def test_sbom_wrong_subject_and_scan_wrong_filesystem(self): + self.seal() + bill = v.load(self.root / 'sbom.json') + bill['metadata']['component']['properties'][0]['value'] = 'sha256:' + 'f' * 64 + put_json(self.root / 'sbom.json', bill) + self.seal() + with self.assertRaises(ValueError): + self.verify() + report = v.load(self.root / 'scan.json') + report['Metadata']['DiffIDs'] = [] + put_json(self.root / 'scan.json', report) + self.seal() + with self.assertRaises(ValueError): + self.verify() + + def test_index_cli_rejects_unverified_receipts_and_mutated_bundles(self): + self.seal() + arm = self.root / 'arm' + arm.mkdir() + fixture(arm, 'arm64') + v.seal(arm, REV, '123-1', 'linux/arm64') + command = ['python3', str(SPEC.origin), 'index', str(self.root), str(arm), '--revision', REV, '--run-id', '123-1'] + result = subprocess.run(command, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len(json.loads(result.stdout)['manifests']), 2) + put_json(arm / 'scan.json', {}) + self.assertNotEqual(subprocess.run(command, capture_output=True).returncode, 0) + + def test_artifact_selection_binds_attempt_source_and_digest(self): + metadata = {'artifacts': [dict(id=i, name=f'accepted-{arch}-123-1', expired=False, + digest='sha256:' + str(i) * 64, workflow_run={'id': 123, 'head_sha': REV}) + for i, arch in enumerate(('amd64', 'arm64'), start=1)]} + self.assertEqual(v.artifact_ids(metadata, '123', '1', REV), '1,2') + for run, attempt, revision in (('124', '1', REV), ('123', '2', REV), ('123', '1', 'b' * 40)): + with self.assertRaises(ValueError): + v.artifact_ids(metadata, run, attempt, revision) + metadata['artifacts'][0]['digest'] = None + with self.assertRaises((ValueError, TypeError)): + v.artifact_ids(metadata, '123', '1', REV) + + def test_registry_tag_preflight_fails_closed(self): + expected = 'sha256:' + hashlib.sha256(b'index').hexdigest() + for status, body, succeeds in ((404, {'errors': [{'code': 'MANIFEST_UNKNOWN'}]}, True), + (404, {'errors': [{'code': 'UNAUTHORIZED'}]}, False), + (401, {}, False), (500, {}, False)): + error = urllib.error.HTTPError('http://registry', status, 'test', {}, io.BytesIO(json.dumps(body).encode())) + with mock.patch.object(v.urllib.request, 'build_opener') as factory: + factory.return_value.open.side_effect = error + if succeeds: + self.assertEqual(v.check_tag('127.0.0.1:5000/app', 'fixture', expected)['state'], 'absent') + else: + with self.assertRaises(ValueError): + v.check_tag('127.0.0.1:5000/app', 'fixture', expected) + with mock.patch.object(v.urllib.request, 'build_opener') as factory: + factory.return_value.open.return_value.__enter__.return_value.read.return_value = b'index' + self.assertEqual(v.check_tag('127.0.0.1:5000/app', 'fixture', expected)['state'], 'same-index') + with self.assertRaises(ValueError): + v.check_tag('127.0.0.1:5000/app', 'fixture', 'sha256:' + 'f' * 64) + + def test_release_record_recomputes_committed_source(self): + source = self.root / 'source' + source.mkdir() + for path, text in {'Cargo.lock': '# locked\n', '.tool-versions': 'rust 1.95.0\n', + 'Dockerfile': 'FROM debian@sha256:' + 'e' * 64 + '\n', + 'a/b': 'nested', 'a-extra/c': 'ordering'}.items(): + target = source / path + target.parent.mkdir(exist_ok=True) + target.write_text(text) + def git(*args): + return subprocess.check_output(['git', '-c', 'core.hooksPath=/dev/null', '-c', 'commit.gpgsign=false', *args], cwd=source, stderr=subprocess.DEVNULL) + git('init', '-q') + git('add', '.') + git('-c', 'user.name=Test', '-c', 'user.email=test@example.invalid', 'commit', '-qm', 'fixture') + revision = git('rev-parse', 'HEAD').decode().strip() + archive = git('archive', '--format=tar', revision) + context = hashlib.sha256() + for path in sorted(source.rglob('*'), key=lambda p: p.relative_to(source).as_posix()): + if path.is_file() and '.git' not in path.relative_to(source).parts: + context.update(path.relative_to(source).as_posix().encode() + b'\0' + path.read_bytes() + b'\0') + bundles = [] + for arch in ('amd64', 'arm64'): + bundle = self.root / ('release-' + arch) + bundle.mkdir() + fixture(bundle, arch, revision) + inputs = v.load(bundle / 'inputs.json') + inputs.update(source_sha256=hashlib.sha256(archive).hexdigest(), context_sha256=context.hexdigest(), + lock_sha256=v.checksum(source / 'Cargo.lock'), dockerfile_sha256=v.checksum(source / 'Dockerfile')) + put_json(bundle / 'inputs.json', inputs) + provenance = v.load(bundle / 'provenance.json') + provenance['inputs'] = inputs + put_json(bundle / 'provenance.json', provenance) + v.seal(bundle, revision, '123-1', 'linux/' + arch) + bundles.append(bundle) + metadata = {'artifacts': [dict(id=i, name=f'accepted-{arch}-123-1', expired=False, + digest='sha256:' + str(i) * 64, workflow_run={'id': 123, 'head_sha': revision}) + for i, arch in enumerate(('amd64', 'arm64'), start=1)]} + record = v.release_record(source, bundles, metadata, revision, '123-1', []) + self.assertEqual(record['source']['lock_sha256'], v.checksum(source / 'Cargo.lock')) + self.assertEqual(len(record['children']), 2) + inputs = v.load(bundles[0] / 'inputs.json') + inputs['lock_sha256'] = 'f' * 64 + put_json(bundles[0] / 'inputs.json', inputs) + provenance = v.load(bundles[0] / 'provenance.json') + provenance['inputs'] = inputs + put_json(bundles[0] / 'provenance.json', provenance) + v.seal(bundles[0], revision, '123-1', 'linux/amd64') + with self.assertRaises(ValueError): + v.release_record(source, bundles, metadata, revision, '123-1', []) + + def test_blob_digest_and_size(self): + path = v.blob(self.root / 'oci', self.image, {v.MANIFEST}) + path.write_bytes(b'changed') + with self.assertRaises(ValueError): + v.describe(self.root / 'oci', 'linux/amd64') + + def test_no_attestations_in_runtime_index(self): + put_json(self.root / 'oci/index.json', {'schemaVersion': 2, 'mediaType': v.INDEX, 'manifests': [self.image, self.image]}) + with self.assertRaises(ValueError): + v.describe(self.root / 'oci', 'linux/amd64') + + def test_config_platform_is_authority(self): + with self.assertRaises(ValueError): + v.describe(self.root / 'oci', 'linux/arm64') + + def test_provenance_subject(self): + put_json(self.root / 'provenance.json', {'revision': REV, 'subject': 'sha256:' + 'b' * 64}) + with self.assertRaises(ValueError): + self.seal() + + def test_index_requires_two_same_run_variants(self): + one = self.seal() + two = dict(one, platform='linux/arm64', manifest_digest='sha256:' + 'c' * 64) + index = v.runtime_index([one, two]) + self.assertEqual({m['platform']['architecture'] for m in index['manifests']}, {'amd64', 'arm64'}) + for receipts in ([one], [one, one], [one, dict(two, run_id='124-1')], + [one, dict(two, sidecars={**two['sidecars'], 'inputs.json': 'f' * 64})]): + with self.assertRaises(ValueError): + v.runtime_index(receipts) + + def test_unsafe_archive_members(self): + for name, kind in [('../escape', tarfile.REGTYPE), ('/absolute', tarfile.REGTYPE), + ('blobs/sha256/' + 'a' * 64, tarfile.SYMTYPE), ('index.json', tarfile.LNKTYPE), + ('script.sh', tarfile.REGTYPE), ('index.json', tarfile.FIFOTYPE)]: + with self.subTest(name=name, kind=kind): + archive = self.root / 'unsafe.tar' + with tarfile.open(archive, 'w') as stream: + member = tarfile.TarInfo(name) + member.type = kind + member.linkname = '/escape' + stream.addfile(member) + destination = self.root / ('unsafe-' + str(len(list(self.root.iterdir())))) + with self.assertRaises(ValueError): + v.unpack(archive, destination) + + def test_duplicate_archive_member(self): + with tarfile.open(self.root / 'unsafe.tar', 'w') as stream: + for _ in range(2): + member = tarfile.TarInfo('index.json') + member.size = 2 + stream.addfile(member, io.BytesIO(b'{}')) + with self.assertRaises(ValueError): + v.unpack(self.root / 'unsafe.tar', self.root / 'unsafe') + + def test_scan_policy_includes_unfixed(self): + report = {'SchemaVersion': 2, 'Results': [{'Vulnerabilities': [{ + 'Severity': 'HIGH', 'VulnerabilityID': 'CVE-test', 'PkgName': 'lib-test', 'FixedVersion': ''}]}]} + with self.assertRaises(ValueError): + v.scan_policy(report, [], self.image['digest']) + exception = {'manifest_digest': self.image['digest'], 'vulnerability': 'CVE-test', 'package': 'lib-test', + 'owner': 'operator', 'reason': 'reviewed risk', 'expires': '2099-01-01'} + v.scan_policy(report, [exception], self.image['digest']) + for bad in [dict(exception, expires='2000-01-01'), dict(exception, reason=''), dict(exception, package='*'), dict(exception, typo='value')]: + with self.assertRaises(ValueError): + v.scan_policy(report, [bad], self.image['digest']) + with self.assertRaises(ValueError): + v.scan_policy(report, [dict(exception, manifest_digest='sha256:' + 'd' * 64)], self.image['digest']) + with self.assertRaises(ValueError): + v.scan_policy({}, [], self.image['digest']) + + def test_smoke_borrowed_image_not_built_or_deleted(self): + app = self.root / 'app' + app.mkdir() + for name in ('Dockerfile', '.dockerignore', 'Cargo.lock', '.tool-versions'): + (app / name).touch() + (app / 'edgezero.toml').write_text('[app]\nname="test-app"\n') + (app / 'Cargo.toml').write_text('[workspace.dependencies]\n') + binaries = self.root / 'bin' + binaries.mkdir() + log = self.root / 'docker-calls' + docker = binaries / 'docker' + docker.write_text('''#!/usr/bin/env bash +printf '%s\\n' "$*" >> "$CALLS" +case "$*" in + 'info --format '*) echo x86_64 ;; + 'image inspect supplied-image --format '*) echo linux/arm64 ;; +esac +''') + docker.chmod(0o755) + result = subprocess.run(['bash', str(ROOT / 'scripts/test_generated_axum_container.sh'), str(app), + '--image', 'supplied-image', '--platform', 'linux/amd64'], + env=dict(os.environ, PATH=str(binaries) + ':' + os.environ['PATH'], CALLS=str(log)), + capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn('Image/platform mismatch', result.stderr) + calls = log.read_text() + self.assertNotIn('image rm supplied-image', calls) + self.assertNotIn('-t supplied-image', calls) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/tests/test_axum_container_preparation.py b/scripts/tests/test_axum_container_preparation.py new file mode 100644 index 000000000..6675ef52f --- /dev/null +++ b/scripts/tests/test_axum_container_preparation.py @@ -0,0 +1,49 @@ +"""Opt-in checks of the actual generated/staged context, without rebuilding it.""" +import hashlib +import json +import os +from pathlib import Path +import tomllib +import unittest + +APP = os.environ.get('AXUM_PREPARED_CONTEXT') + + +@unittest.skipUnless(APP, 'set AXUM_PREPARED_CONTEXT to the freshly prepared fixture') +class PreparedContext(unittest.TestCase): + def setUp(self): + self.app = Path(APP).resolve() + self.manifest = tomllib.loads((self.app / 'Cargo.toml').read_text()) + self.inputs = json.loads((self.app / 'fixture-inputs.json').read_text()) + + def test_nested_workspace_and_features_preserved(self): + self.assertEqual(self.manifest['workspace']['exclude'], ['vendor/edgezero']) + for name, dep in self.manifest['workspace']['dependencies'].items(): + if name.startswith('edgezero-'): + self.assertEqual(dep['path'], f'vendor/edgezero/crates/{name}') + axum = tomllib.loads((self.app / 'crates/container-probe-adapter-axum/Cargo.toml').read_text()) + self.assertEqual(axum['dependencies']['edgezero-adapter-axum']['features'], ['axum']) + cli = tomllib.loads((self.app / 'crates/container-probe-cli/Cargo.toml').read_text()) + self.assertNotIn('default-features', cli['dependencies']['edgezero-cli']) + framework = tomllib.loads((self.app / 'vendor/edgezero/Cargo.toml').read_text()) + self.assertIn('async-compression', framework['workspace']['dependencies']) + + def test_own_lockfile_and_confined_local_packages(self): + self.assertFalse((self.app / '.git').exists()) + self.assertEqual(hashlib.sha256((self.app / 'Cargo.lock').read_bytes()).hexdigest(), self.inputs['lock_sha256']) + self.assertNotEqual((self.app / 'Cargo.lock').read_bytes(), (self.app / 'vendor/edgezero/Cargo.lock').read_bytes()) + metadata = json.loads((self.app.parent / 'metadata.json').read_text()) + for package in metadata['packages']: + if package['source'] is None: + self.assertTrue(Path(package['manifest_path']).resolve().is_relative_to(self.app), package['name']) + + def test_no_git_or_host_build_output_staged(self): + for path in (self.app / 'vendor').rglob('*'): + relative = path.relative_to(self.app) + self.assertNotIn('.git', relative.parts) + self.assertNotIn('target', relative.parts) + self.assertEqual(self.inputs['dependency_mode'], 'staged-test-sources') + + +if __name__ == '__main__': + unittest.main() From 05d3c6e88076ae62fe0d0b8a0552a3bfbc1221a4 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 7 Oct 2026 19:16:43 -0500 Subject: [PATCH 4/6] fix(ci): separate container packaging scans from release approval Retain full vulnerability evidence in framework packaging checks without treating the fixture as an approved release. Keep strict defaults and reject report-only GHCR publication. Update inherited Bookworm packages and cover the policy boundary with regression tests. --- .github/workflows/axum-container.yml | 5 + crates/edgezero-cli/src/generator.rs | 8 ++ .../src/templates/root/Dockerfile.hbs | 2 + docs/guide/publishing-axum-containers.md | 49 +++++++--- examples/container-publishing/publish.sh | 16 +++- .../container-publishing/verify_artifacts.py | 25 +++-- scripts/test_axum_container_handoff.sh | 5 +- .../tests/test_axum_container_artifacts.py | 95 +++++++++++++++++++ 8 files changed, 178 insertions(+), 27 deletions(-) diff --git a/.github/workflows/axum-container.yml b/.github/workflows/axum-container.yml index 00992248c..870b0dcc4 100644 --- a/.github/workflows/axum-container.yml +++ b/.github/workflows/axum-container.yml @@ -68,6 +68,9 @@ jobs: native: needs: prepare timeout-minutes: 30 + # Framework packaging evidence is not application release approval. + env: + SCAN_POLICY: report-only strategy: fail-fast: false matrix: @@ -131,6 +134,8 @@ jobs: needs: native runs-on: ubuntu-24.04 timeout-minutes: 15 + env: + SCAN_POLICY: report-only permissions: contents: read actions: read diff --git a/crates/edgezero-cli/src/generator.rs b/crates/edgezero-cli/src/generator.rs index e0ef990ae..626e20131 100644 --- a/crates/edgezero-cli/src/generator.rs +++ b/crates/edgezero-cli/src/generator.rs @@ -1019,6 +1019,14 @@ mod tests { assert!(dockerfile.contains("ARG RUST_VERSION=1.95.0")); assert!(dockerfile.contains("-p my-app-adapter-axum --bin my-app-adapter-axum")); assert!(dockerfile.contains("--release --locked")); + let runtime = dockerfile + .split(" AS runtime\n") + .nth(1) + .expect("runtime stage"); + assert!(runtime.contains("apt-get upgrade -y --no-install-recommends")); + assert!(runtime.contains( + "apt-get install -y --no-install-recommends ca-certificates libgcc-s1" + )); assert!(dockerfile.contains("USER 10001:10001")); assert!(dockerfile.contains("ENTRYPOINT [\"/usr/local/bin/app\"]")); assert!(dockerfile.contains("EDGEZERO__ADAPTER__HOST=0.0.0.0")); diff --git a/crates/edgezero-cli/src/templates/root/Dockerfile.hbs b/crates/edgezero-cli/src/templates/root/Dockerfile.hbs index 035ec446e..1653cd9da 100644 --- a/crates/edgezero-cli/src/templates/root/Dockerfile.hbs +++ b/crates/edgezero-cli/src/templates/root/Dockerfile.hbs @@ -50,7 +50,9 @@ strip --strip-debug /out/app SH FROM debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251 AS runtime +# Apply available Bookworm fixes to packages inherited from the pinned base too. RUN apt-get update \ + && apt-get upgrade -y --no-install-recommends \ && apt-get install -y --no-install-recommends ca-certificates libgcc-s1 \ && rm -rf /var/lib/apt/lists/* WORKDIR /app diff --git a/docs/guide/publishing-axum-containers.md b/docs/guide/publishing-axum-containers.md index 88cce7ef4..b176c479f 100644 --- a/docs/guide/publishing-axum-containers.md +++ b/docs/guide/publishing-axum-containers.md @@ -10,6 +10,9 @@ production readiness, storage recovery or graceful shutdown. The repository's `.github/workflows/axum-container.yml` prepares one generated application and one application lockfile. Separate ephemeral Ubuntu amd64 and arm64 runners build native GNU images. Neither uses QEMU as acceptance evidence. +Framework CI validates packaging and artifact integrity, and retains vulnerability +reports without treating the fixture as an approved application release. The +application-owned publisher separately enforces its strict vulnerability policy. ```mermaid flowchart TD @@ -170,13 +173,26 @@ Because selection binds the run attempt, rerun all preparation/native jobs for a new attempt. The recipe does not mix a successful old platform with a new retry or discover a prior successful run automatically. -## Scanner policy and current blocker +## Scanner policies Trivy scans the verified OCI layout, which came directly from the retained -archive. The verifier requires a nonempty supported report with matching image -config identity and a manifest-bound scanner record. High and Critical findings -block acceptance, including findings without a fix. It does not use -`--ignore-unfixed` or an unreviewed ignore file. +archive. Both policies require a nonempty supported report with matching image +config and filesystem identities, plus a manifest-bound scanner record. Scan +execution errors, malformed reports and artifact-integrity failures still block. + +Framework packaging CI sets `SCAN_POLICY=report-only` for its native build and +credential-free handoff jobs. The helpers retain the complete report and warn +with High/Critical counts and how many findings have available fixes. Those +findings do not block packaging validation or the disposable localhost index +check. A passing framework job does not approve an image for application release. +The verifier's `verify` and `index` commands accept `--scan-policy report-only`; +policy comes from the caller, never from a bundle's receipt or scan metadata. + +The default is `strict`. The application release workflow and `release-record` +always use strict verification. High and Critical findings block release, +including findings without a fix. The publishing helper rejects report-only +policy for GHCR before any publication; only its disposable loopback destination +permits it. Neither policy uses `--ignore-unfixed` or an unreviewed ignore file. Exceptions must come from an independently trusted, owner-reviewed policy file, not the artifact. Each entry requires `manifest_digest`, `vulnerability`, @@ -187,14 +203,19 @@ no automatic waiver path. Review the exact existing candidate bytes and an explicit approval path before adapting the workflow for an exception. Do not rebuild a waived image and assume the old digest exemption still applies. -The October 7, 2026 local scan of the current Debian recipe reported 55 High and -4 Critical findings. The strict policy blocks that candidate. No exceptions have -been authorized. Image remediation or explicit scoped risk decisions are still -needed. The image scan found no language-specific files; its SBOM is not a claim +The October 7, 2026 scan before the runtime package update reported 55 High and +4 Critical package findings across 20 CVEs on each architecture. Eight findings +had available fixes. The runtime recipe now upgrades inherited Bookworm packages +as well as installing required libraries. A native amd64 scan after this update +reported 50 High and 1 Critical finding, none with an available fix. Strict +verification still rejected that same image. No exceptions have been authorized, +and report-only packaging results do not waive the release policy. The image +scan found no language-specific files; its SBOM is not a claim that every statically linked Rust dependency was independently analyzed. -Native arm64 runner execution, accepted artifact import/index assembly in hosted -CI and one separately authorized disposable GHCR publish/attestation verification -remain execution gates. Keep [#399](https://github.com/stackpop/edgezero/issues/399) -open until those acceptance checks pass. Production runtime and deployment gates -are described in [Deploying Axum containers](./deploying-axum-containers.md). +Both hosted native jobs passed packaging and Compose checks on October 7. +Hosted artifact import/index assembly under the packaging policy and one separately +authorized disposable GHCR publish/attestation verification still need execution +evidence. Keep [#399](https://github.com/stackpop/edgezero/issues/399) open until +those acceptance checks pass. Production runtime and deployment gates are +described in [Deploying Axum containers](./deploying-axum-containers.md). diff --git a/examples/container-publishing/publish.sh b/examples/container-publishing/publish.sh index 8c755b6ef..74cdbfc09 100755 --- a/examples/container-publishing/publish.sh +++ b/examples/container-publishing/publish.sh @@ -19,22 +19,28 @@ elif [[ ! "$REPO" =~ ^ghcr\.io/[a-z0-9_.-]+/[a-z0-9_.-]+$ ]]; then elif [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.-]+)?$ ]]; then echo 'GHCR publication requires a version tag, not a reserved alias' >&2; exit 2 fi +SCAN_POLICY="${SCAN_POLICY:-strict}" +[[ "$SCAN_POLICY" == strict || "$SCAN_POLICY" == report-only ]] || { echo 'Invalid scan policy' >&2; exit 2; } +# Report-only evidence may exercise a disposable loopback index, never GHCR. +if ! "$LOCAL" && [[ "$SCAN_POLICY" != strict ]]; then + echo 'GHCR publication requires strict scan policy' >&2; exit 2 +fi [[ ! -e "$OUTPUT" ]] || { echo 'Output must not exist' >&2; exit 2; } AMD="$(cd "$AMD" && pwd)"; ARM="$(cd "$ARM" && pwd)" mkdir -p "$OUTPUT" OUTPUT="$(cd "$OUTPUT" && pwd)" WORK="$(mktemp -d -t axum-publisher.XXXXXX)" trap 'rm -rf "$WORK"' EXIT -EXCEPTIONS=() -if [[ -n "${SCAN_EXCEPTIONS:-}" ]]; then EXCEPTIONS+=(--exceptions "$SCAN_EXCEPTIONS"); fi +VERIFY_ARGS=(--scan-policy "$SCAN_POLICY") +if [[ -n "${SCAN_EXCEPTIONS:-}" ]]; then VERIFY_ARGS+=(--exceptions "$SCAN_EXCEPTIONS"); fi python3 "$HERE/verify_artifacts.py" verify "$AMD" --revision "$REVISION" --run-id "$RUN_ID" \ - --platform linux/amd64 "${EXCEPTIONS[@]}" > "$OUTPUT/amd64.json" + --platform linux/amd64 "${VERIFY_ARGS[@]}" > "$OUTPUT/amd64.json" python3 "$HERE/verify_artifacts.py" verify "$ARM" --revision "$REVISION" --run-id "$RUN_ID" \ - --platform linux/arm64 "${EXCEPTIONS[@]}" > "$OUTPUT/arm64.json" + --platform linux/arm64 "${VERIFY_ARGS[@]}" > "$OUTPUT/arm64.json" python3 "$HERE/verify_artifacts.py" extract "$AMD/image.tar" "$WORK/amd64" --platform linux/amd64 >/dev/null python3 "$HERE/verify_artifacts.py" extract "$ARM/image.tar" "$WORK/arm64" --platform linux/arm64 >/dev/null python3 "$HERE/verify_artifacts.py" index "$AMD" "$ARM" --revision "$REVISION" --run-id "$RUN_ID" \ - "${EXCEPTIONS[@]}" > "$OUTPUT/index.json" + "${VERIFY_ARGS[@]}" > "$OUTPUT/index.json" python3 - "$WORK" "$OUTPUT" <<'PY' import hashlib, json, pathlib, shutil, sys work, output = map(pathlib.Path, sys.argv[1:]) diff --git a/examples/container-publishing/verify_artifacts.py b/examples/container-publishing/verify_artifacts.py index 529e8fd12..80a517537 100644 --- a/examples/container-publishing/verify_artifacts.py +++ b/examples/container-publishing/verify_artifacts.py @@ -132,7 +132,8 @@ def verify_pulled(layout, inspection, platform, reference): return info -def scan_policy(report, exceptions, manifest_digest, today=None): +def scan_policy(report, exceptions, manifest_digest, today=None, *, policy='strict'): + require(policy in ('strict', 'report-only'), 'Invalid scan policy') today = today or datetime.datetime.now(datetime.timezone.utc).date() require(report.get('SchemaVersion') == 2 and isinstance(report.get('Results'), list) and report['Results'], 'Missing Trivy results') require(isinstance(exceptions, list), 'Exceptions must be a list') @@ -146,12 +147,20 @@ def scan_policy(report, exceptions, manifest_digest, today=None): key = (entry['manifest_digest'], entry['vulnerability'], entry['package']) require(key not in allowed, 'Duplicate scan exception') allowed.add(key) + counts = {'HIGH': 0, 'CRITICAL': 0} + fixable = 0 for result in report['Results']: for finding in result.get('Vulnerabilities') or []: require(finding['Severity'] in ('UNKNOWN', 'LOW', 'MEDIUM', 'HIGH', 'CRITICAL'), 'Invalid vulnerability severity') - if finding['Severity'] in ('HIGH', 'CRITICAL'): + if finding['Severity'] in counts: key = (manifest_digest, finding['VulnerabilityID'], finding['PkgName']) - require(key in allowed, f'Blocked {finding["Severity"]} vulnerability: {finding["VulnerabilityID"]} in {finding["PkgName"]}') + counts[finding['Severity']] += 1 + fixable += bool(finding.get('FixedVersion')) + if policy == 'strict': + require(key in allowed, f'Blocked {finding["Severity"]} vulnerability: {finding["VulnerabilityID"]} in {finding["PkgName"]}') + if policy == 'report-only': + print(f'WARNING: report-only scan: {counts["HIGH"]} HIGH, {counts["CRITICAL"]} CRITICAL package findings, ' + f'{fixable} with fixes. Packaging evidence only; not approved for release.', file=sys.stderr) def seal(bundle, revision, run_id, platform): @@ -182,7 +191,7 @@ def check_inputs(inputs, revision, run_id): re.fullmatch(r'[^\s@]+@sha256:[0-9a-f]{64}', base) for base in bases), 'Pinned base trace missing') -def verify(bundle, revision, run_id, platform, exceptions): +def verify(bundle, revision, run_id, platform, exceptions, *, policy='strict'): receipt = load(bundle / 'receipt.json') require(receipt['schema'] == 1 and receipt['checks'] == 'packaging-passed', 'Acceptance missing') require(receipt['revision'] == revision and receipt['run_id'] == run_id, 'Unauthorized source/run') @@ -215,7 +224,7 @@ def verify(bundle, revision, run_id, platform, exceptions): 'Supported OS scan missing') require(all(r.get('Class') in ('os-pkgs', 'lang-pkgs') and r.get('Target') for r in report['Results']), 'Incomplete supported scan results') - scan_policy(report, exceptions, info['manifest_digest']) + scan_policy(report, exceptions, info['manifest_digest'], policy=policy) sbom = load(bundle / 'sbom.json') require(sbom.get('bomFormat') == 'CycloneDX' and sbom.get('components'), 'SBOM missing') properties = sbom.get('metadata', {}).get('component', {}).get('properties', []) @@ -348,11 +357,13 @@ def main(): child.add_argument('--platform', required=True, choices=sorted(PLATFORMS)) if command == 'verify': child.add_argument('--exceptions', type=pathlib.Path) + child.add_argument('--scan-policy', choices=('strict', 'report-only'), default='strict') index = sub.add_parser('index') index.add_argument('bundles', nargs=2, type=pathlib.Path) index.add_argument('--revision', required=True) index.add_argument('--run-id', required=True) index.add_argument('--exceptions', type=pathlib.Path) + index.add_argument('--scan-policy', choices=('strict', 'report-only'), default='strict') pulled = sub.add_parser('pulled') pulled.add_argument('layout', type=pathlib.Path) pulled.add_argument('inspection', type=pathlib.Path) @@ -395,12 +406,12 @@ def main(): result = seal(args.bundle, args.revision, args.run_id, args.platform) elif args.command == 'verify': exceptions = load(args.exceptions) if args.exceptions else [] - result = verify(args.bundle, args.revision, args.run_id, args.platform, exceptions) + result = verify(args.bundle, args.revision, args.run_id, args.platform, exceptions, policy=args.scan_policy) elif args.command == 'pulled': result = verify_pulled(args.layout, load(args.inspection), args.platform, args.reference) else: exceptions = load(args.exceptions) if args.exceptions else [] - result = runtime_index([verify(bundle, args.revision, args.run_id, platform, exceptions) + result = runtime_index([verify(bundle, args.revision, args.run_id, platform, exceptions, policy=args.scan_policy) for bundle, platform in zip(args.bundles, ('linux/amd64', 'linux/arm64'))]) print(json.dumps(result, indent=2)) except (ValueError, KeyError, TypeError, IndexError, OSError, tarfile.TarError, subprocess.CalledProcessError) as error: diff --git a/scripts/test_axum_container_handoff.sh b/scripts/test_axum_container_handoff.sh index ecd56ac40..e24fee37a 100755 --- a/scripts/test_axum_container_handoff.sh +++ b/scripts/test_axum_container_handoff.sh @@ -11,6 +11,9 @@ VERIFY="$ROOT/examples/container-publishing/verify_artifacts.py" MODE="${1:?build or import}"; INPUT="${2:?input directory}"; OUTPUT="${3:?new output directory}" PLATFORM="${4:?linux/amd64 or linux/arm64}"; REVISION="${5:?source SHA}"; RUN_ID="${6:?run identity}" [[ $# -eq 6 && ( "$MODE" == build || "$MODE" == import ) ]] || { echo 'Invalid handoff arguments' >&2; exit 2; } +# Framework packaging CI may retain scan evidence without approving a release. +SCAN_POLICY="${SCAN_POLICY:-strict}" +[[ "$SCAN_POLICY" == strict || "$SCAN_POLICY" == report-only ]] || { echo 'Invalid scan policy' >&2; exit 2; } [[ "$PLATFORM" == linux/amd64 || "$PLATFORM" == linux/arm64 ]] || { echo 'Unsupported platform' >&2; exit 2; } if [[ "$MODE" == build ]]; then case "$(docker info --format '{{.Architecture}}')" in @@ -20,7 +23,7 @@ if [[ "$MODE" == build ]]; then esac [[ "$PLATFORM" == "$HOST_PLATFORM" ]] || { echo 'Native build required; emulation is not acceptance' >&2; exit 2; } fi -VERIFY_ARGS=() +VERIFY_ARGS=(--scan-policy "$SCAN_POLICY") if [[ -n "${SCAN_EXCEPTIONS:-}" ]]; then VERIFY_ARGS+=(--exceptions "$SCAN_EXCEPTIONS"); fi [[ ! -e "$OUTPUT" ]] || { echo 'Output must not exist' >&2; exit 2; } INPUT="$(cd "$INPUT" && pwd)" diff --git a/scripts/tests/test_axum_container_artifacts.py b/scripts/tests/test_axum_container_artifacts.py index 53ca210d8..adadc49df 100644 --- a/scripts/tests/test_axum_container_artifacts.py +++ b/scripts/tests/test_axum_container_artifacts.py @@ -1,3 +1,4 @@ +import contextlib import gzip import hashlib import importlib.util @@ -254,6 +255,15 @@ def git(*args): record = v.release_record(source, bundles, metadata, revision, '123-1', []) self.assertEqual(record['source']['lock_sha256'], v.checksum(source / 'Cargo.lock')) self.assertEqual(len(record['children']), 2) + report = v.load(bundles[0] / 'scan.json') + report['Results'][0]['Vulnerabilities'] = [{'Severity': 'HIGH', 'VulnerabilityID': 'CVE-test', + 'PkgName': 'lib-test', 'FixedVersion': ''}] + put_json(bundles[0] / 'scan.json', report) + v.seal(bundles[0], revision, '123-1', 'linux/amd64') + with self.assertRaisesRegex(ValueError, 'Blocked HIGH'): + v.release_record(source, bundles, metadata, revision, '123-1', []) + report['Results'][0]['Vulnerabilities'] = [] + put_json(bundles[0] / 'scan.json', report) inputs = v.load(bundles[0] / 'inputs.json') inputs['lock_sha256'] = 'f' * 64 put_json(bundles[0] / 'inputs.json', inputs) @@ -334,6 +344,91 @@ def test_scan_policy_includes_unfixed(self): with self.assertRaises(ValueError): v.scan_policy({}, [], self.image['digest']) + def test_report_only_scan_does_not_approve_release(self): + report = v.load(self.root / 'scan.json') + report['Results'][0]['Vulnerabilities'] = [ + {'Severity': severity, 'VulnerabilityID': 'CVE-' + severity, + 'PkgName': 'lib-test', 'FixedVersion': fixed} + for severity, fixed in [('HIGH', ''), ('CRITICAL', '2.0')] + ] + put_json(self.root / 'scan.json', report) + receipt = self.seal() + with contextlib.redirect_stderr(io.StringIO()) as warnings: + self.assertEqual(v.verify(self.root, REV, '123-1', 'linux/amd64', [], policy='report-only'), receipt) + self.assertIn('not approved for release', warnings.getvalue()) + with self.assertRaisesRegex(ValueError, 'Blocked HIGH'): + self.verify() + command = ['python3', str(SPEC.origin), 'verify', str(self.root), '--revision', REV, + '--run-id', '123-1', '--platform', 'linux/amd64'] + self.assertNotEqual(subprocess.run(command, env=dict(os.environ, SCAN_POLICY='report-only'), + capture_output=True).returncode, 0) + result = subprocess.run(command + ['--scan-policy', 'report-only'], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout), receipt) + arm = self.root / 'arm' + arm.mkdir() + fixture(arm, 'arm64') + v.seal(arm, REV, '123-1', 'linux/arm64') + index = ['python3', str(SPEC.origin), 'index', str(self.root), str(arm), '--revision', REV, '--run-id', '123-1'] + self.assertNotEqual(subprocess.run(index, capture_output=True).returncode, 0) + self.assertEqual(subprocess.run(index + ['--scan-policy', 'report-only'], capture_output=True).returncode, 0) + release = ['python3', str(SPEC.origin), 'release-record', str(self.root), str(self.root), str(arm), + str(self.root / 'metadata.json'), '--revision', REV, '--run-id', '123-1', + '--scan-policy', 'report-only'] + self.assertIn('unrecognized arguments', subprocess.run(release, capture_output=True, text=True).stderr) + + def test_report_only_preserves_scan_validation(self): + original = v.load(self.root / 'scan.json') + for change in ({'Results': []}, {'SchemaVersion': 1}, + {'Metadata': dict(original['Metadata'], ImageID='sha256:' + 'f' * 64)}, + {'Metadata': dict(original['Metadata'], DiffIDs=[])}, + {'Results': [{'Class': 'unsupported', 'Target': 'fixture'}]}, + {'Results': [dict(original['Results'][0], Vulnerabilities=[{ + 'Severity': 'INVALID', 'VulnerabilityID': 'CVE-test', 'PkgName': 'lib-test'}])]}): + with self.subTest(change=change): + put_json(self.root / 'scan.json', dict(original, **change)) + self.seal() + with self.assertRaises(ValueError): + v.verify(self.root, REV, '123-1', 'linux/amd64', [], policy='report-only') + for severity in ('HIGH', 'CRITICAL'): + for missing in ('VulnerabilityID', 'PkgName'): + finding = {'Severity': severity, 'VulnerabilityID': 'CVE-test', 'PkgName': 'lib-test'} + del finding[missing] + report = dict(original, Results=[dict(original['Results'][0], Vulnerabilities=[finding])]) + put_json(self.root / 'scan.json', report) + self.seal() + with self.subTest(severity=severity, missing=missing), self.assertRaises(KeyError): + v.verify(self.root, REV, '123-1', 'linux/amd64', [], policy='report-only') + result = subprocess.run(['python3', str(SPEC.origin), 'verify', str(self.root), '--revision', REV, + '--run-id', '123-1', '--platform', 'linux/amd64', '--scan-policy', 'report-only'], + capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, '') + with self.assertRaisesRegex(ValueError, 'Invalid scan policy'): + v.scan_policy(original, [], self.image['digest'], policy='typo') + with self.assertRaisesRegex(ValueError, 'Expired scan exception'): + v.scan_policy(original, [{'manifest_digest': self.image['digest'], 'vulnerability': 'CVE-test', + 'package': 'lib-test', 'owner': 'operator', 'reason': 'reviewed risk', + 'expires': '2000-01-01'}], self.image['digest'], policy='report-only') + + def test_ghcr_publication_rejects_report_only_before_side_effects(self): + output = self.root / 'published' + command = ['bash', str(ROOT / 'examples/container-publishing/publish.sh'), 'ghcr.io/owner/app', 'v1.0.0', + str(self.root), str(self.root), str(output), REV, '123-1'] + result = subprocess.run(command, env=dict(os.environ, SCAN_POLICY='report-only'), capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn('GHCR publication requires strict scan policy', result.stderr) + self.assertFalse(output.exists()) + for script, arguments in ( + (ROOT / 'examples/container-publishing/publish.sh', command[2:]), + (ROOT / 'scripts/test_axum_container_handoff.sh', + ['build', str(self.root), str(output), 'linux/amd64', REV, '123-1'])): + result = subprocess.run(['bash', str(script), *arguments], env=dict(os.environ, SCAN_POLICY='typo'), + capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn('Invalid scan policy', result.stderr) + self.assertFalse(output.exists()) + def test_smoke_borrowed_image_not_built_or_deleted(self): app = self.root / 'app' app.mkdir() From 68894a05f6f4a8c49a8a2b7bd7fee2bed24efcfe Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 7 Oct 2026 19:25:33 -0500 Subject: [PATCH 5/6] fix(ci): validate report-only finding identities Reject empty and non-string CVE/package identities in both scan policies. Record the successful hosted native builds and artifact/index handoff. --- docs/guide/publishing-axum-containers.md | 13 ++++++++----- examples/container-publishing/verify_artifacts.py | 2 ++ scripts/tests/test_axum_container_artifacts.py | 9 +++++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/docs/guide/publishing-axum-containers.md b/docs/guide/publishing-axum-containers.md index b176c479f..25bac8bc6 100644 --- a/docs/guide/publishing-axum-containers.md +++ b/docs/guide/publishing-axum-containers.md @@ -213,9 +213,12 @@ and report-only packaging results do not waive the release policy. The image scan found no language-specific files; its SBOM is not a claim that every statically linked Rust dependency was independently analyzed. -Both hosted native jobs passed packaging and Compose checks on October 7. -Hosted artifact import/index assembly under the packaging policy and one separately -authorized disposable GHCR publish/attestation verification still need execution -evidence. Keep [#399](https://github.com/stackpop/edgezero/issues/399) open until -those acceptance checks pass. Production runtime and deployment gates are +Both hosted native jobs passed packaging and Compose checks on October 7. The +[follow-up CI run](https://github.com/stackpop/edgezero/actions/runs/37706921831) +also passed native amd64/arm64 builds, transferred artifact imports and localhost +index assembly under the packaging policy. Both scans reported 50 High and +1 Critical finding, with no available fixes. One separately authorized disposable +GHCR publish/attestation verification still needs execution evidence. Keep +[#399](https://github.com/stackpop/edgezero/issues/399) open until that acceptance +check passes. Production runtime and deployment gates are described in [Deploying Axum containers](./deploying-axum-containers.md). diff --git a/examples/container-publishing/verify_artifacts.py b/examples/container-publishing/verify_artifacts.py index 80a517537..250f42b6b 100644 --- a/examples/container-publishing/verify_artifacts.py +++ b/examples/container-publishing/verify_artifacts.py @@ -153,6 +153,8 @@ def scan_policy(report, exceptions, manifest_digest, today=None, *, policy='stri for finding in result.get('Vulnerabilities') or []: require(finding['Severity'] in ('UNKNOWN', 'LOW', 'MEDIUM', 'HIGH', 'CRITICAL'), 'Invalid vulnerability severity') if finding['Severity'] in counts: + require(all(isinstance(finding[field], str) and finding[field].strip() + for field in ('VulnerabilityID', 'PkgName')), 'Invalid vulnerability/package identity') key = (manifest_digest, finding['VulnerabilityID'], finding['PkgName']) counts[finding['Severity']] += 1 fixable += bool(finding.get('FixedVersion')) diff --git a/scripts/tests/test_axum_container_artifacts.py b/scripts/tests/test_axum_container_artifacts.py index adadc49df..0035e39f2 100644 --- a/scripts/tests/test_axum_container_artifacts.py +++ b/scripts/tests/test_axum_container_artifacts.py @@ -404,6 +404,15 @@ def test_report_only_preserves_scan_validation(self): capture_output=True, text=True) self.assertNotEqual(result.returncode, 0) self.assertEqual(result.stdout, '') + for severity in ('HIGH', 'CRITICAL'): + for field in ('VulnerabilityID', 'PkgName'): + for invalid in (None, '', ' ', []): + finding = {'Severity': severity, 'VulnerabilityID': 'CVE-test', 'PkgName': 'lib-test', field: invalid} + report = dict(original, Results=[dict(original['Results'][0], Vulnerabilities=[finding])]) + with self.subTest(severity=severity, field=field, invalid=invalid): + for policy in ('strict', 'report-only'): + with self.assertRaisesRegex(ValueError, 'Invalid vulnerability/package identity'): + v.scan_policy(report, [], self.image['digest'], policy=policy) with self.assertRaisesRegex(ValueError, 'Invalid scan policy'): v.scan_policy(original, [], self.image['digest'], policy='typo') with self.assertRaisesRegex(ValueError, 'Expired scan exception'): From 7f846ae02cf2ff334673c59ccd2f6eef076139ba Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 14 Oct 2026 13:48:32 -0500 Subject: [PATCH 6/6] docs: draft Kubernetes and AWS deployment guides --- docs/.vitepress/config.mts | 4 + docs/guide/deploying-axum-containers.md | 25 ++- docs/guide/deploying-axum-kubernetes.md | 244 ++++++++++++++++++++++++ docs/specs/aws-reference-deployment.md | 225 ++++++++++++++++++++++ 4 files changed, 488 insertions(+), 10 deletions(-) create mode 100644 docs/guide/deploying-axum-kubernetes.md create mode 100644 docs/specs/aws-reference-deployment.md diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 89d8de7eb..7ddfc16c0 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -57,6 +57,10 @@ export default defineConfig({ text: 'Deploying Axum containers', link: '/guide/deploying-axum-containers', }, + { + text: 'Deploying Axum on Kubernetes', + link: '/guide/deploying-axum-kubernetes', + }, ], }, { diff --git a/docs/guide/deploying-axum-containers.md b/docs/guide/deploying-axum-containers.md index 2332b9473..8e4e21f5c 100644 --- a/docs/guide/deploying-axum-containers.md +++ b/docs/guide/deploying-axum-containers.md @@ -127,18 +127,18 @@ The production-runtime draft stack is not accepted integration evidence. Changes to runtime behavior require a fresh candidate and repeat acceptance before publishing that new digest. -## Kubernetes follow-up +## Kubernetes -Kubernetes acceptance remains separate. It needs the same index or explicit native -child digest, correct node architecture, non-root/read-only settings and separate -startup/readiness/liveness probes. Use HTTP probes or an already accepted probe -mechanism; do not assume curl exists in the image. Check Service/ingress routing -and termination timing under load. +The [Kubernetes guide draft](./deploying-axum-kubernetes.md) selects kind and +Envoy Gateway for disposable-cluster verification, with plain manifests and +separate local stateless, AWS-provider and single-writer redb profiles. The planned +AWS profile includes AppConfig Agent, Secrets Manager and DynamoDB after accepted +provider integration. No cluster or AWS execution is claimed by the draft. -The persistent example must use single-writer scheduling and storage ownership, -then prove restart/recovery. Private production secrets/configuration must come -from approved resources through the existing runtime contract. No Helm chart, -operator or new deployment CLI is required by this preparation. +Use the same accepted digest and runtime contract, matching node architecture, +restricted execution and actual application probes. Kubernetes acceptance still +needs routed HTTPS/proxy checks, bounded shutdown, restart/recovery, upgrades and +rollback. Local cluster results do not qualify EKS or real AWS services. ## First managed target: ECS/Fargate @@ -147,6 +147,11 @@ provides explicit `X86_64`/`ARM64` task architecture, read-only root settings an configurable stop timeout. This fits the native image and avoids pretending a container's local disk is durable redb storage. +The [AWS reference deployment draft](../specs/aws-reference-deployment.md) +extends this direction with AppConfig Agent, Secrets Manager, an ECR publishing +option and an optional DynamoDB profile. It defines acceptance checks and open +design decisions, not verified cloud support. + Before an authorized test, an owner must specify the AWS account, region, cluster, network/security groups, execution/task roles, registry credential path, service and load-balancer/TLS resources, secret/config inputs, maximum cost and cleanup diff --git a/docs/guide/deploying-axum-kubernetes.md b/docs/guide/deploying-axum-kubernetes.md new file mode 100644 index 000000000..6d28a14be --- /dev/null +++ b/docs/guide/deploying-axum-kubernetes.md @@ -0,0 +1,244 @@ +# Deploying Axum applications on Kubernetes + +::: warning Draft, not deployment acceptance +This guide defines the Kubernetes recipe for [#400](https://github.com/stackpop/edgezero/issues/400). Runnable manifests, controller/image pins and disposable-cluster execution are not delivered yet. The YAML fragments below are design examples, not a complete deployment. Do not treat them as production verification. + +Production entrypoints, probes, startup validation and shutdown must use the accepted runtime contracts. The current container packaging response at `/` is not readiness. +::: + +## Deployment scope + +Start with a disposable **kind** cluster and **Envoy Gateway** implementing Gateway API. Use plain application manifests. Envoy Gateway supports installation from Kubernetes YAML, so the initial recipe does not require an application Helm chart, a service mesh or an operator. + +ECS/Fargate remains the first managed AWS deployment target. A local Kubernetes run does not establish EKS, IAM, cloud storage or AWS load-balancer support. + +Use three separate profiles: + +| Profile | Configuration and secrets | Mutable application state | Acceptance boundary | +| --------------------------- | --------------------------------------------------- | -------------------------------------------------- | ------------------------------------------------------------------------------------- | +| Local stateless fixture | Reviewed local inputs and synthetic fixture secrets | None | Credential-free disposable-cluster checks | +| AWS provider profile | AppConfig Agent and Secrets Manager | DynamoDB through the accepted ordinary-KV provider | Real AWS calls require separate authorization and a reviewed workload-credential path | +| Local single-writer fixture | Reviewed local inputs and synthetic fixture secrets | Embedded redb on a durable volume | One writer, tested storage permissions and stop-before-start replacement | + +DynamoDB is included in the planned AWS profile from the outset, once its provider is accepted and integrated. It is not required for applications without mutable state. The default local checks require no AWS credentials and must not silently switch to cloud services. + +EdgeZero owns the generic fixture, manifests and checks. Application repositories own images, required inputs, signing policy, infrastructure, workload sizing, releases and deployment approval. Trusted Server's identity/CAS and signing requirements remain downstream. + +## Why Envoy Gateway + +Gateway API separates infrastructure-owned listeners from application routing. The application uses a `Service` and `HTTPRoute`; the local test supplies a `GatewayClass`, `Gateway` and TLS certificate. Those application resources can be reviewed independently of a particular cloud load balancer. + +Envoy Gateway is the selected reference controller, not a guarantee that every Gateway API implementation behaves identically. Its [YAML installation](https://gateway.envoyproxy.io/docs/install/install-yaml/) and [compatibility matrix](https://gateway.envoyproxy.io/news/releases/matrix/) provide the starting point for choosing a supported test combination. + +Before execution, record the exact kind release, node image digest, Kubernetes version, Gateway API CRDs, Envoy Gateway release and controller/data-plane image digests. Do not install from `latest`, blindly replace existing CRDs or copy an upstream quickstart's echo application as EdgeZero runtime evidence. A public release number alone is not a verified image pin. + +```mermaid +flowchart TD + Client[Fixture client] --> Gateway[Envoy Gateway HTTPS listener] + Gateway --> Route[HTTPRoute] + Route --> Service[ClusterIP Service] + Service --> App[Axum application Pod] + Kubelet[Startup, readiness and liveness probes] --> App + App --> Inputs[Validated configuration and required secrets] + App -. AWS profile .-> Agent[Loopback AppConfig Agent] + Agent --> AppConfig[AWS AppConfig] + App -. AWS profile .-> Secrets[AWS Secrets Manager] + App -. AWS profile .-> DynamoDB[Amazon DynamoDB] +``` + +## Image and runtime prerequisites + +Use the [container packaging contract](./deploying-axum-containers.md) and [application-owned publishing contract](./publishing-axum-containers.md). Deploy the accepted native child digest for a homogeneous test cluster, or a verified runtime index whose children support the selected nodes. Never rebuild during deployment. + +A provider-enabled image is a new candidate if its binary or build inputs differ from the packaging fixture. Build and test it through the same artifact handoff, then retain its own release evidence. Report-only packaging scans do not authorize a deployment release. + +Before writing runnable manifests, settle: + +- The production entrypoint and initializer that validate required typed config and nested secret references before listening. +- Actual application-owned startup/readiness/liveness routes, their port and measured probe budgets. +- Image/node architecture agreement and the registry pull path inside kind. Imported tags or a host Docker image ID alone do not prove retained manifest identity. +- Supported local input paths and native provider bindings, with no runtime loading of an invented `edgezero.toml`. +- Fixture resource requests/limits, termination budget and bounded client/check timeouts. These are measured fixture settings, not universal workload defaults. + +No fixed probe paths, resource limits or image digests are invented by this draft. + +## Workload and security settings + +The planned manifest set lives under `examples/axum-deployment/kubernetes/`. That directory is not delivered yet. Keep the default fixture, AWS additions and persistent profile separate rather than maintaining three independent deployment frameworks. + +The default set needs a Namespace, ServiceAccount, Deployment, ClusterIP Service, reviewed ConfigMap/Secret references and the Gateway API resources. Keep cluster-scoped controller installation separate from application resources. + +Use application UID/GID 10001 and restricted execution. This container fragment expresses the intended restrictions, but omits the image, probes, resources, volumes and deployment metadata: + +```yaml +securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] +``` + +Set Pod `seccompProfile.type: RuntimeDefault`. The local fixture does not need Kubernetes API access, so disable automatic ServiceAccount token mounting there. An AWS identity mechanism may need a specific projected token and must be configured separately, not assumed from the local profile. + +Provision only required writable paths. Do not attach an emptyDir or persistent volume to make the entire application root writable. Do not use a privileged startup wrapper, root chown init container or world-writable permissions to hide a storage ownership problem. + +ConfigMaps can carry non-secret native bindings and accepted local configuration files. Kubernetes Secret references can supply synthetic local fixture values through the supported environment mechanism. Mounting a secret file alone does not populate an environment-backed secret store. Kubernetes Secret base64 encoding is not encryption or a replacement for production secret-access policy. + +## Probes and initialization failures + +Use kubelet HTTP probes against actual application routes. They need no curl/wget inside the image. A successful TCP connection or the packaging response at `/` does not establish initialized application readiness. + +| Probe | What it establishes | What it must not imply | +| --------- | -------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------ | +| Startup | The process has completed its bounded startup path sufficiently to begin normal probes | Unbounded waiting for missing configuration | +| Readiness | This initialized application can accept the work defined by its dependency policy | Every possible downstream service is healthy or every client will receive a response | +| Liveness | The process needs replacement under the app's documented health policy | A transient AWS failure should restart every replica | + +Make the startup window accommodate the accepted initialization budget without hiding permanent failure. Required input or validation failure must exit or remain non-serving with an observable finite outcome. It must never load development defaults after selecting an explicit provider. + +Exercise missing configuration, malformed envelopes, unresolved required secrets and incompatible release inputs. Observe the process and client traffic, not merely `kubectl rollout status`. Document whether a withdrawing listener produces refusal/reset or a probe response instead of promising a particular status code. + +## HTTPS routing and proxy trust + +Use an HTTPS `Gateway` listener and `HTTPRoute`, with TLS terminated at Envoy and the accepted Axum backend protocol configured explicitly. Client HTTP/2 negotiation does not prove HTTP/2 support on the backend connection. + +The following routing fragment uses fictional fixture names. It assumes an installed matching GatewayClass, a same-namespace TLS Secret and an existing `axum-fixture` Service on port 8787. It does not create those prerequisites: + +```yaml +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: axum-fixture + namespace: edgezero-kubernetes-probe +spec: + gatewayClassName: axum-fixture + listeners: + - name: https + protocol: HTTPS + port: 443 + hostname: axum-fixture.example.test + tls: + mode: Terminate + certificateRefs: + - kind: Secret + name: axum-fixture-tls + allowedRoutes: + namespaces: + from: Same +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: axum-fixture + namespace: edgezero-kubernetes-probe +spec: + parentRefs: + - name: axum-fixture + sectionName: https + hostnames: [axum-fixture.example.test] + rules: + - backendRefs: + - name: axum-fixture + port: 8787 +``` + +Use a disposable certificate whose SAN covers the fixture hostname and an explicit test CA. Do not disable client certificate verification. Production certificate issuance, renewal and private keys are operator-owned, not generated application-image contents. + +Check Gateway `Accepted`/`Programmed` and the matching HTTPRoute parent `Accepted`/`ResolvedRefs` conditions, then exercise actual HTTPS traffic. A kind cluster needs an explicit local exposure mechanism. For NodePort exposure, the kind node mapping's container port must match the Service's `nodePort`, and the host binding must stay loopback-only; do not assume a LoadBalancer address appears automatically. Record that mechanism and its limitations. + +Forwarding to the application Service bypasses Envoy. Forwarding to Envoy can diagnose TLS/routing, but still does not prove the complete configured exposure path. Acceptance must use that path and test wrong-hostname and untrusted-CA rejection. + +Restrict trusted forwarded metadata to the accepted proxy boundary. Do not trust the entire Pod CIDR, every namespace or any client-supplied `X-Forwarded-*` header. Test direct requests from an unrelated Pod and spoofed metadata through Envoy. + +If NetworkPolicy is part of the isolation claim, select and test a CNI that enforces it. Creating policy objects is not enforcement evidence. Account for kubelet probes and required egress rather than accidentally blocking startup, DNS or the approved AWS access path. Keep controller/admin ports outside client routing. + +## AWS profile with DynamoDB + +Use the [AWS reference design](../specs/aws-reference-deployment.md) for provider, release and IAM ownership. The first AWS Kubernetes profile includes AppConfig Agent, Secrets Manager and DynamoDB through their accepted portable interfaces. Do not add Redis or a shared redb volume to bridge these services. + +Mount non-secret native resource mappings through a reviewed ConfigMap and select the accepted bindings file explicitly. Keep actual credentials, application secrets and signing keys out of image layers, ConfigMaps and source-controlled YAML. Build features must include the selected providers; a bindings file cannot enable code absent from the binary. + +Run AppConfig Agent in the application Pod's network namespace. Explicitly configure a literal-loopback endpoint and reviewed bearer-token authentication. Do not rely on its container-environment bind default or expose port 2772 through a Service/Gateway. Disable disk backups/preload in the initial profile and review both application and Agent logging for secret/token disclosure. + +A loopback-only Agent cannot be checked by an ordinary HTTP probe targeting the Pod IP. Select a supported in-container probe for the reviewed Agent image, or an accepted bounded initialization/restart path. Do not invent an Agent health command or add curl merely to complete the YAML. Regular Pod containers have no implicit startup order; native sidecar ordering is available only under the pinned Kubernetes contract and still requires verified wiring. + +Application configuration and secret values are immutable startup snapshots initially. Agent polling does not hot-reload typed application state. Restart to adopt a reviewed config revision or secret version. Secret deletion/revocation does not erase already loaded values, so emergency replacement remains an operator procedure. + +DynamoDB is different: ordinary KV operations perform live remote I/O. Scope runtime access to the approved table and required operations. Runtime state writes are allowed; image publishing, configuration deployment, secret mutation and infrastructure changes remain operator-only. Apply the accepted provider's payload, key, TTL, pagination, timeout and retry limits. Asynchronous DynamoDB TTL deletion is not logical expiry, and a failed write may have an unknown effect. + +A Kubernetes ServiceAccount alone does not grant AWS access. On a future EKS deployment, select and qualify a workload-identity mechanism supported by both the pinned SDK and Agent. Containers sharing the Pod's credential delivery are not independent security principals. Do not copy an ECS task-role example into Kubernetes or add long-lived AWS keys to a Secret. + +The local baseline must run without AWS credentials or service calls. AWS wiring can be reviewed and tested with deterministic fixtures, but that does not prove real IAM, table access or version adoption. Real AWS acceptance needs an approved credential/environment path, resource and cost limits, and cleanup ownership. EKS deployment is not included in the first local milestone. + +DynamoDB externalizes state; it does not certify application concurrency. Do not enable multi-replica identity mutation on the strength of ordinary CRUD/list tests. Trusted Server's conditional creation, generation/CAS and consent-withdrawal semantics need their separate backend acceptance. + +## Single-writer redb profile + +Keep embedded persistence separate from the stateless/AWS profiles. Use one application replica, no HPA and a verified stop-before-start rollout. A PVC, ReadWriteOnce access mode, a StatefulSet or `Recreate` alone does not prove that an old writer has stopped, especially after node isolation or force deletion. + +Provision the volume for the runtime UID/GID with the chosen storage driver's supported ownership mechanism. Keep the root filesystem read-only. Record the actual storage class/driver and limit durability claims to its tested behavior. + +Test container restart and replacement Pod reads from the same volume, denied ownership, competing-writer rejection and recovery after interrupted work. Confirm old process exit and lock release before replacement. If an old writer cannot be proven stopped, stop the replacement procedure and follow the operator's fencing/recovery policy. Do not force-delete a stuck Pod and call API-object disappearance proof of process exit. + +Stop the sole writer before a file-level backup. Restore a known fixture value and check it through the application. Volume retention/deletion policy and backups belong to the application operator; deleting a namespace is not permission to delete production data. + +## Shutdown, upgrades and rollback + +Set `terminationGracePeriodSeconds` longer than the accepted runtime drain budget plus any selected hook/sidecar overhead. A `preStop` hook consumes the same grace period. Do not add a sleep hook and claim graceful shutdown from elapsed time. + +EndpointSlice and proxy updates are asynchronous with termination. The application must enforce its own withdrawal/drain contract rather than assume all new connections disappear instantly. Test finite in-flight work, never-ending work, stalled readers and final forced termination through the actual routing path. Capture responses, connection outcomes, exit status and elapsed time. + +A rolling update is allowed only for a fixture whose state and release inputs permit overlapping replicas. Use bounded rollout deadlines and record the selected availability policy. The redb profile must not overlap writers. + +Retain the previous image digest and compatible config/secret identities before upgrading. Apply the reviewed image and external inputs together, then observe startup, readiness and client behavior. For a failed upgrade, preserve usable old capacity where the profile supports it and inspect the failure before rollback. + +An image-only `kubectl rollout undo` does not restore a changed ConfigMap, secret stage or data schema. Restore the compatible release inputs explicitly. Use new immutable input references where practical, and validate expected revisions before serving. Stateful schema changes and signing-key rotation remain application-owned policies. + +## Disposable-cluster acceptance + +The repeatable test will be a bounded, opt-in exercise using the same retained fixture artifacts as container acceptance. It must target an explicitly selected disposable context, use unique run identities and clean up after partial failure. Do not run against the operator's current context by default. + +| Check | Required observation | Evidence boundary | +| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | +| Setup and identity | Compatible pinned node/controller inputs; actual application/native manifest identity agrees with scheduled architecture | Local cluster; each architecture claimed must execute natively | +| Restricted execution | Non-root/read-only application root, explicit writable paths, reviewed service-account/token access and no exposed Agent/admin ports | Actual Pod behavior, not manifest inspection alone | +| Valid and invalid startup | Valid required inputs initialize before traffic; missing/malformed/mismatched inputs do not become ready or admit application work | Process, probe and client observations | +| Probe distinction | Startup, readiness and liveness follow their documented meanings without masking permanent initialization failure | Actual kubelet behavior and events | +| HTTPS and routing | Gateway/route conditions agree with successful client traffic; wrong-hostname and untrusted-CA requests fail | Configured local exposure path, not direct app port-forward | +| Proxy boundary | Spoofed metadata is ignored/rejected under the accepted contract; claimed network isolation is enforced by the tested CNI | Gateway and unrelated-Pod requests | +| Drain and forced exit | Active finite work has the documented outcome; new work does not dispatch after drain begins; over-budget work reaches a bounded process-exit outcome | Real Pod termination under routed traffic | +| Replacement | A replacement Pod recovers the documented stateless behavior without relying on predecessor-local files | Local stateless profile | +| Upgrade and rollback | Compatible release becomes ready; incompatible release fails safely; retained image and matching external inputs restore expected behavior | Two recorded fixture releases | +| Persistent restart/recovery | Same durable value survives container and Pod replacement; wrong ownership and competing writer fail; stopped-writer restore works | Separate redb profile and named storage driver | +| AWS providers | Real AppConfig/secret mapping, pinned/current-version adoption, denied access and redacted startup failure match the accepted provider contract | Separately authorized AWS run; local fixtures are insufficient | +| DynamoDB | Writes survive Pod replacement and are readable by another approved replica; denial/TTL/list/limit/failure behavior matches the provider contract | Real table plus deterministic failure tests; no identity/CAS certification | +| Cleanup | Test cluster, local published ports, registry containers, fixture secrets and disposable CA private keys are removed; external resources follow approved retention policy | Inventory after success and partial failure | + +Record checked source/input hashes, native digests, config/secret identities, controller/CNI/storage versions, commands, client observations, sanitized events/logs, timing and cleanup outcomes. Do not publish fixture secrets or private keys in failure artifacts. + +## Troubleshooting + +- **Pending/ImagePullBackOff:** check node architecture, digest availability and the cluster's actual registry access path. A successful pull on the host is not a successful pull on a kind node. +- **CrashLoopBackOff or never ready:** inspect redacted initializer/provider failures, selected bindings and missing required inputs. Do not remove readiness checks or introduce development fallback to make the rollout green. +- **Gateway or route rejected:** check the matching parent status, installed CRD/controller versions, hostname, certificate reference and Service port. A ready application Pod does not prove accepted routing. +- **TLS failure:** check SAN, hostname, CA trust and the configured exposure path. Do not use insecure curl flags as the remedy. +- **Agent startup failure:** check loopback binding, supported startup ordering, authentication, compiled provider features and the approved credential path. Do not expose the Agent listener to make a probe succeed. +- **Storage permission/lock failure:** check UID/GID, mount ownership and whether another writer is still running. Do not chmod 777 or force-delete the predecessor as a substitute for fencing. +- **Termination or rollback failure:** compare the complete hook/drain/grace budget and external input compatibility. Control-plane deletion and image history alone are not recovery proof. + +## Remaining implementation + +Select and verify exact cluster/controller/registry inputs, integrate the accepted production/provider fixture, deliver plain manifests and a repeatable exercise, then execute the matrix. This guide records that work; it does not imply it has run. + +Local Kubernetes acceptance can proceed independently of EKS qualification. #400 still needs its Docker/Compose and authorized managed-target requirements, and the AWS provider issues retain their own acceptance criteria. Documentation alone does not close them. + +## References + +- [AWS reference deployment and acceptance](../specs/aws-reference-deployment.md). +- [Envoy Gateway YAML installation](https://gateway.envoyproxy.io/docs/install/install-yaml/), [compatibility matrix](https://gateway.envoyproxy.io/news/releases/matrix/) and [HTTPS secure gateways](https://gateway.envoyproxy.io/docs/tasks/security/secure-gateways/). +- [kind configuration and local port mappings](https://kind.sigs.k8s.io/docs/user/configuration/). +- [Kubernetes probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/), [Pod termination](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-termination) and [NetworkPolicy](https://kubernetes.io/docs/concepts/services-networking/network-policies/). +- [EdgeZero #400](https://github.com/stackpop/edgezero/issues/400), [AWS provider issue #410](https://github.com/stackpop/edgezero/issues/410) and [draft provider PR #414](https://github.com/stackpop/edgezero/pull/414). diff --git a/docs/specs/aws-reference-deployment.md b/docs/specs/aws-reference-deployment.md new file mode 100644 index 000000000..d95385231 --- /dev/null +++ b/docs/specs/aws-reference-deployment.md @@ -0,0 +1,225 @@ +# AWS reference deployment and acceptance + +- Status: Draft for review. No deployed support claim. +- Date: 2026-10-14 +- Epic: [#391](https://github.com/stackpop/edgezero/issues/391) +- Related work: [#400](https://github.com/stackpop/edgezero/issues/400), [#410](https://github.com/stackpop/edgezero/issues/410), [#401](https://github.com/stackpop/edgezero/pull/401) and [#414](https://github.com/stackpop/edgezero/pull/414) + +## Purpose + +Define an AWS-first deployment recipe for native Axum applications, with one acceptance contract shared by Docker, ECS/Fargate and Kubernetes. Prefer AWS-managed configuration, secrets and durable storage over operating additional services merely to supply those functions. + +The first cloud target is ECS/Fargate with AppConfig Agent and Secrets Manager. DynamoDB is an optional application-state extension, not a requirement for applications without mutable state. Kubernetes remains part of EdgeZero #400, but is not a prerequisite for deploying on AWS. + +This is a draft for discussion, not authorization to provision resources, publish images, merge branches or close issues. It extends [the container deployment guide](../guide/deploying-axum-containers.md) with an AWS-first design. Existing delivery and security approvals remain unchanged unless explicitly revised. + +## Outcome and boundaries + +A disposable generated application can use reviewed AWS resources through EdgeZero's existing portable interfaces, validate required inputs before serving, and run as a replaceable Fargate task. Operators can upgrade and roll back using retained image digests and compatible external inputs. + +EdgeZero owns reusable providers, runtime contracts, generic examples and fixture verification. Application repositories own deployable images, configuration schemas, required secrets, signing policy, workload sizing, infrastructure, credentials and release approval. Trusted Server's actual deployment and identity backend remain downstream. + +The reference demonstrates a deployment mechanism. It does not certify arbitrary applications, unlimited concurrency, end-client receipt, all AWS regions or every target architecture. + +## Current evidence + +Reviewed on 2026-10-14: + +| Work | Observed state | Remaining proof | +| ---------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- | +| #401 at `68894a05` | Draft packaging PR with passing native amd64/arm64 and artifact-handoff CI; Docker/Compose preparation delivered | Accepted-runtime final-image cases, production deployment acceptance, branch conflicts and failing Fastly check | +| #410 / #414 at published head `b764a883` | Draft AppConfig Agent and Secrets Manager providers, per-ID selection and immutable startup snapshots | Accepted prerequisite integration and authorized real AWS/IAM acceptance | +| DynamoDB provider worktree | Uncommitted ordinary-KV implementation and plan present in `issue-410-native-store-providers` | Implementation review, test evidence, publication and real service acceptance | +| #275 and #406 | Open draft HTTP and production startup/lifecycle work | Accepted integration and revalidation through candidate images | +| Kubernetes and AWS deployments | No execution evidence established in this review | Disposable-cluster and separately authorized AWS runs | + +These are inspected records, not tests executed for this document. Packaging scans are report-only; strict application-release verification still rejects the recorded candidate. No vulnerability exception is authorized by this draft. + +## Proposed decisions + +| Concern | First choice | Boundary | +| ----------------- | ---------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | +| Compute | ECS/Fargate, one region | No EKS dependency; AZ placement and scaling claims require separate execution evidence | +| TLS and ingress | Application Load Balancer with ACM | Tasks accept application traffic only from the approved ingress boundary | +| Configuration | AppConfig Agent sidecar, free-form documents | Whole-document mapping; no arbitrary JSON flattening or configuration publishing in the runtime | +| Secrets | Secrets Manager through the native provider | Whole values and explicit version selectors; no automatic signing-key rotation | +| Application state | No embedded mutable store in the first fixture; optional DynamoDB KV profile | No shared redb filesystem or Redis requirement; ordinary KV does not supply CAS | +| Images | ECR option for AWS deployments | Reuse tested OCI artifacts; do not replace or duplicate the existing GHCR recipe | +| Observability | Application stdout/stderr to CloudWatch Logs | Reviewed retention and redaction; AWS dependency DEBUG/TRACE and sensitive-body logging off unless separately reviewed | +| Updates | Validated startup snapshots and controlled task replacement | No live application reload in the initial version | +| Kubernetes | Plain manifests consuming the same runtime/fixture contract | Separate disposable-cluster proof; no Helm, operator or EKS qualification implied | + +ECR support is a proposed extension. Digest preservation, metadata association and verification must execute against ECR before that path is described as tested. The existing publisher's GHCR-specific authentication and attestation steps cannot simply be assumed to work unchanged. + +## Reference architecture + +```mermaid +flowchart TD + Clients[Clients] --> ALB[ALB with ACM TLS] + ALB --> App[Native Axum application on ECS Fargate] + ECR[Accepted digest-pinned image in ECR] --> App + App --> Agent[Loopback AppConfig Agent sidecar] + Agent --> Config[AWS AppConfig] + App --> Secrets[AWS Secrets Manager] + App -. Optional mutable KV .-> DynamoDB[Amazon DynamoDB] + App --> Logs[CloudWatch Logs] + Agent --> Logs +``` + +Use non-root application identity, read-only root, dropped capabilities where supported, and explicit port/bind settings. Verify the actual task restrictions rather than translating Docker flags mechanically. Give the Agent its own reviewed security/resource settings and operator-verified image digest. + +The first fixture has no required local mutable data. AppConfig and Secrets Manager snapshots do not make an application stateful in the durable-storage sense. Fargate ephemeral storage is not a redb persistence recipe. + +## Bootstrap, IAM and network ownership + +Keep logical store declarations in the portable application definition. Put deployment resource mappings in the native bindings file selected by `EDGEZERO__STORE_BINDINGS_FILE`. Region, resource identifiers and credential discovery must be available before fetching configuration. Never fetch bootstrap credentials from the provider they are needed to authenticate. + +Settle how the non-secret bindings file reaches the task before implementing the example. An application-owned file in the reviewed release artifact is one option. Fargate has no Kubernetes ConfigMap equivalent to assume. Any delivery helper must preserve exec startup, file permissions, immutable-input identity and the read-only application root. Secret values do not belong in that file or image. + +Separate IAM responsibilities: + +- Task execution role for approved ECR pulls and CloudWatch logging. Add secret-read permission here only for a deliberately selected ECS-injected secret or registry-credential path. +- Task role for application Secrets Manager reads, Agent AppConfig reads and optional DynamoDB operations. Grant access only to reviewed resources and the required KMS decrypt scope. +- Operator/release roles for configuration deployments, secret mutation, image publication and infrastructure changes. Runtime roles do not inherit these powers. + +Containers in one ECS task share the task-role authority. The Agent and application are not isolated IAM principals. Scope the combined task permissions accordingly. + +Choose private task networking and document the approved outbound route or VPC endpoints for image pull, logs, configuration and secrets. Include those resources in cost and cleanup approval. Do not expose Agent port 2772 through the ALB or task ingress rules. + +The provider requires a literal-loopback Agent endpoint. Review the selected Agent's binding, token and cache settings. Resolve local endpoint authentication without storing tokens in source, URLs or diagnostic output. Agent disk backup/preload remains disabled for the initial recipe; an acceptable cached response is not independent proof of current AWS reachability. + +## Startup and update contract + +```mermaid +flowchart TD + Release[Capture release and native bindings] --> Sources[Validate mappings and compiled providers] + Sources --> Agent[Agent starts and supplies the selected document] + Agent --> Snapshots[Prepare config and secret snapshots within startup budgets] + Snapshots --> AppInit[Validate typed config and required nested secrets] + AppInit --> Listener[Bind listener and report application readiness] + Listener --> Traffic[Serve from validated immutable snapshots] + Change[Remote config or secret changes] --> Replace[Controlled task replacement] + Replace --> Release +``` + +Every required mapping and application-declared dependency must succeed before readiness. Missing data, denied access, invalid envelopes, failed secret resolution and incompatible release inputs must not silently fall back to development files or environment values. + +The production initializer validates typed application state. Successful provider preparation alone does not prove that the application's required configuration or secrets were checked. + +AppConfig Agent polling does not update a running typed application. Secret rotation, remote deletion and permission revocation do not erase bytes already loaded into a running process. Restart-based adoption is explicit, including an emergency replacement procedure for secret revocation. + +Use ECS startup dependency wiring only after identifying a health-check mechanism actually supported by the selected Agent image. Do not assume curl, wget or an undocumented endpoint exists. `HEALTHY` dependencies apply at startup, not as a continuing readiness guarantee. Agent health also does not replace application validation. + +Application readiness and process liveness are different checks. ALB health checking is a routing signal, not a security boundary. ALB can fail open when every target is unhealthy, so startup and draining safety must be enforced by the application/listener contract and tested through real ingress. + +## Release identity, upgrades and rollback + +Record one application release with: + +- Immutable source revision and build context, lockfile, toolchain and base-image identities. +- Accepted native child digests, runtime index digest and authenticated scan/SBOM/provenance evidence. +- Reviewed AppConfig application/environment/profile mappings and expected document revision or digest. +- Selected secret version IDs or stages, without secret values. +- Bindings-file identity, Agent image digest, task-definition revision and architecture. +- Application compatibility decisions, startup/drain budgets and operator approval. + +AppConfig profile identifiers do not pin arbitrary historical document versions. Choose release-specific profiles or explicitly deploy and validate the required document revision. Reject mismatched revisions when an application requires coordinated inputs. Multiple config profiles and secrets are not an atomic cross-service snapshot. + +Prefer explicit secret version IDs in the rollback fixture. If production uses `AWSCURRENT`, record how stage movement affects replacement tasks and rollback. Do not mistake an old image plus today's secret stage for the old release. + +Publication preserves tested manifests and blobs. Verify destination digests and metadata subjects after any registry copy. Tags are aliases, not release identity. Report-only packaging scans cannot authorize a deployment release; strict release policy and any independently approved exceptions still apply. + +Retain the previous compatible release before upgrade. Failed initialization must not become healthy or replace usable capacity without a documented failure outcome. Rollback restores compatible image and external inputs, not just an image tag. Database schema migration and signing-key rotation need application-owned policies beyond this fixture. + +Align ALB deregistration delay, runtime drain budget and ECS stop timeout. Their mechanisms differ; no single timeout proves drain. Fargate's documented stop timeout maximum is 120 seconds. Test finite work, long-lived streams, stalled clients and forced termination against the selected budgets. Preserve BestEffort/Unsupported and protocol limitations of the accepted runtime. + +## Acceptance matrix + +Every row below is pending execution for this reference deployment. Reuse prior evidence only where the exact inputs, command and environment remain applicable. Capture client observations, sanitized logs and release identity, not merely control-plane success. + +| ID | Scenario | Required observation | Test location | +| --- | ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------ | +| A1 | Independent provider selection | AppConfig plus local secrets, local config plus Secrets Manager, and both AWS providers work without replacing unrelated KV selection | Deterministic native/generated-consumer tests, then selected AWS profile | +| A2 | Valid startup | Complete envelope and required secret resolution finish before listener/readiness; requests use validated snapshots | Candidate image and AWS through ALB | +| A3 | Missing or invalid inputs | Missing profile/secret, malformed config and unresolved nested secret fail startup without development fallback | Deterministic fixtures and disposable AWS resources | +| A4 | IAM and service failures | Denied access and relevant decryption/network failures remain redacted errors, not successful misses; startup has an observable finite failure | Fixtures plus least-privilege AWS negative tests | +| A5 | Agent boundary | Verified image, loopback access, reviewed authentication/cache policy and supported startup ordering; endpoint unavailable through external ingress | Actual ECS task | +| A6 | Snapshot adoption | Remote updates do not silently alter a running app; replacement adopts the reviewed config/secret revision | Real AppConfig/Secrets Manager and task replacement | +| A7 | Mixed revisions | Incompatible config/secret/release combination is rejected by the fixture initializer before readiness | Candidate image and AWS negative release | +| A8 | Runtime credentials | Workload role supplies provider access; no static keys in image/config; runtime cannot publish images, deploy config, mutate secrets or change infrastructure; optional DynamoDB writes are limited to the approved application table | Approved IAM policy review and live allowed/denied operations | +| A9 | Image handoff | ECR receives the exact accepted manifests/blobs; digest pull and subject/producer verification succeed; wrong artifacts are rejected | Authorized publisher and separate deployment job | +| A10 | Container constraints | Non-root identity, read-only application root, explicit writable paths if any, and no exposed Agent listener | Actual ECS task inspection and runtime checks | +| A11 | Probes and ingress | Application readiness differs from liveness; failed/all-unhealthy startup does not admit application work; spoofed proxy metadata is rejected or ignored under the accepted policy | Actual ALB client traffic and task observations | +| A12 | Graceful termination | Dispatched finite work completes within budget; new work does not dispatch after drain begins; over-budget work reaches a bounded exit outcome | Actual ECS replacement/stop under traffic | +| A13 | HTTP boundaries | Claimed client/backend protocols, buffering/stream behavior, idle timeouts and unsupported capabilities match deployed evidence | Actual ALB and accepted final-image checks | +| A14 | Upgrade and rollback | New compatible release becomes ready; failed upgrade has a documented outcome; retained image and matching external inputs restore the old behavior | ECS service with two fixture releases | +| A15 | Task replacement | Replace or kill a task and recover the documented capacity without relying on its ephemeral files | ECS service, with outage/duration recorded | +| A16 | Architecture | Each claimed task architecture pulls and executes its accepted native variant; untested architectures stay labelled unverified | Fargate per claimed architecture | +| A17 | Logging safety | Startup/provider/ingress failures reveal no fixture secret or token; dependency logging configuration and log retention reviewed | Captured application and Agent logs | +| A18 | Cleanup | Disposable tasks and all approved billable dependencies are deleted or explicitly retained with an owner and expiry | AWS resource inventory after run | +| D1 | Optional DynamoDB persistence | Data written through the real provider survives replacement and is read by another replica; binary/TTL/list/limit behavior matches the provider contract | Separate DynamoDB-enabled image and real table | +| D2 | Optional DynamoDB failures | Denial/throttling and ambiguous writes remain explicit failures; no local fallback or unsafe retry guarantee | SDK fixtures and approved AWS negative tests | +| K1 | Kubernetes deployment | Plain manifests run in a disposable cluster with actual probes, failed initialization, shutdown, replacement, digest upgrade and rollback | Separate cluster run | +| K2 | Kubernetes persistent redb profile | One writer, durable state and verified stop-before-start rollout; storage ownership and fencing/recovery limitations explicit | Separate single-writer cluster profile | + +DynamoDB rows do not certify Trusted Server's identity backend. Its conditional creation, generation/CAS, strong withdrawal lookup and bounded prefix-list semantics require a separate backend design and acceptance. Ordinary read-modify-write is not atomic, and a shared managed database alone does not make an application safe to replicate. + +## Kubernetes profile decisions + +The [Kubernetes guide draft](../guide/deploying-axum-kubernetes.md) selects kind and Envoy Gateway with Gateway API for the first credential-free disposable-cluster proof. Plain manifests remain the application deployment format. EKS qualification is separate; ECS/Fargate remains the first managed AWS target. + +The planned AWS Kubernetes profile includes AppConfig Agent, Secrets Manager and DynamoDB from its first delivery, once the providers are accepted and integrated. This does not make DynamoDB mandatory for stateless applications or change the initial no-store Fargate fixture. Local baseline checks do not contact AWS, and real provider/table acceptance still needs separate authorization. + +## Delivery sequence and likely artifacts + +1. Agree this draft's release-input policy, bindings-file delivery, Agent health/authentication wiring and AWS authorization checklist. Keep unresolved choices visible rather than fill examples with invented commands or pins. +2. Reconcile accepted HTTP/lifecycle/provider contracts. Resolve overlapping lifecycle work in #406/#414 at its owning boundary, not with another implementation in #401. Extend the disposable fixture to validate typed config and secrets before readiness. +3. Build and test the candidate final images using #401's existing preparation/handoff checks. Keep the portable consumer proof separate from staged-source fixture builds. A provider-enabled image is a new candidate, not the already-tested packaging artifact. +4. Draft generic task/service, role and bindings examples under `examples/axum-deployment/ecs-fargate/`; extend `docs/guide/deploying-axum-containers.md` and link the provider guide. Add an ECR publishing option only after its preserving path and metadata checks are specified. +5. Add bounded opt-in execution/cleanup helpers only where they make acceptance repeatable. Re-running after partial failure must use stable run identities and avoid duplicate resources. Credential-free PR checks must never gain publisher/deployment authority. +6. Execute the approved AWS matrix, then repeat affected native, provider, generated-consumer, docs and WASM compatibility gates. Record the exact checked state and gaps. +7. Add plain Kubernetes manifests and a kind/Envoy Gateway exercise using the same accepted fixture/runtime contract. Default local checks require no AWS credentials. Prepare a separate AWS profile with AppConfig Agent, Secrets Manager and DynamoDB, and preserve the single-writer redb profile required by #400. +8. Run the DynamoDB-enabled AWS profile after accepted provider integration and separately authorized service access. Qualify Trusted Server's stronger identity contract independently; neither local Kubernetes nor ordinary KV evidence replaces it. + +The proposed integration keeps deployment preparation in #401 and provider implementation in #414. Kubernetes preparation may join #401 using the same accepted fixture contract. Implementation, GitHub metadata and issue-closing decisions remain separate from this document draft. + +AWS execution can begin without waiting for Kubernetes completion. #400 still cannot close until Docker/Compose, Kubernetes and the selected managed-target requirements are all met. #410 still needs real provider/IAM acceptance; deployment smoke alone does not close its entire contract. + +## Authorization and execution record + +Before any AWS run, obtain approval for account, region, allowed architectures, principal/role usage, resource names, VPC/subnets/security groups, registry and publication rights, ALB/TLS inputs, AppConfig/secret/table resources, log retention, maximum spend, maximum duration, cleanup owner and permitted retained artifacts. Approval for one service is not approval for the others. + +Each test record must include `ID | command | checked revision/input hashes | environment | image/config/secret identities | observed result | elapsed time | sanitized evidence | cleanup outcome`. Never include secret values, tokens or credential material. Mark unrun or failed cases explicitly. + +No AWS resources, credentials, publication or runtime tests were used while drafting this document. No performance, cost, IAM or production-support claim is established here. + +## Decisions needed before implementation + +- Approve ECS/Fargate as the primary AWS path and ECR as an additional registry example, without replacing existing GHCR evidence requirements. +- Select the reviewed Agent release/digest, supported health check and local authentication policy. +- Select non-secret bindings-file delivery and the fixture's compatible config/secret release-identity checks. +- Agree the first AWS test's resource/cost/time/cleanup authorization and architecture scope. +- Confirm the resource mappings and permitted IAM operations for the DynamoDB-enabled AWS Kubernetes profile after provider integration. Keep identity/CAS acceptance separate. + +## Sources + +- [EdgeZero #391, self-hosted runtime epic](https://github.com/stackpop/edgezero/issues/391). +- [EdgeZero #400, deployment guidance and acceptance](https://github.com/stackpop/edgezero/issues/400). +- [EdgeZero #410, AWS provider scope](https://github.com/stackpop/edgezero/issues/410), [draft #414](https://github.com/stackpop/edgezero/pull/414) and its [published native AWS guide](https://github.com/stackpop/edgezero/blob/b764a883ce57a323250d9d7a20787dc5937ef90e/docs/guide/native-aws-stores.md). +- [Draft #401, packaging/release/deployment evidence](https://github.com/stackpop/edgezero/pull/401). +- [AppConfig Agent on ECS](https://docs.aws.amazon.com/appconfig/latest/userguide/appconfig-integration-containers-agent-starting-ecs.html). +- [ECS IAM role responsibilities](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/security-ecs-iam-role-overview.html). +- [Fargate task parameters, startup dependencies and termination limits](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html). +- [ALB health checks and fail-open behavior](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/target-group-health-checks.html). +- [ECS load-balancer connection draining](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/load-balancer-connection-draining.html). +- DynamoDB implementation context was inspected in uncommitted work on the #414 branch. It is not published or accepted provider evidence. + +## Related documentation + +- [Deploying Axum containers](../guide/deploying-axum-containers.md). +- [Deploying Axum on Kubernetes](../guide/deploying-axum-kubernetes.md). +- [Publishing Axum containers](../guide/publishing-axum-containers.md). +- [Axum adapter guide](../guide/adapters/axum.md). +- [Production lifecycle/configuration work, draft #406](https://github.com/stackpop/edgezero/pull/406). +- [Downstream AWS adoption, Trusted Server #1236](https://github.com/IABTechLab/trusted-server/issues/1236). +- [Downstream image/deployment ownership, Trusted Server #897](https://github.com/IABTechLab/trusted-server/issues/897).