diff --git a/.github/workflows/check-digdir.yml b/.github/workflows/check-digdir.yml index c357ce99b..23150f2bd 100644 --- a/.github/workflows/check-digdir.yml +++ b/.github/workflows/check-digdir.yml @@ -4,9 +4,11 @@ name: Check Digdir # matrices remain outside downstream CI. The hardware-virtualization lanes are # upstream's, run on GitHub-hosted runners, which expose KVM. # -# Runs on pull requests and on demand. Caches are saved only in runs on -# main-digdir, which the release runbook in MAINTAINING-digdir.md makes part -# of every release. +# Runs on pull requests and on demand. It builds the runtime once, with the +# release profile, runs the integration and smoke tests against those +# binaries, and assembles them into the release assets, so that +# release-digdir.yml publishes exactly what this run tested without building +# anything. Caches are saved only in runs on main-digdir. on: pull_request: @@ -28,6 +30,10 @@ env: CARGO_INCREMENTAL: "0" LIBKRUNFW_VERSION: "5.6.1" LIBKRUNFW_ABI: "5" + # Every shipped Linux ELF is compiled or linked inside the architecture-native + # manylinux 2.28 container, so the published runtime does not import glibc + # symbols newer than the hosts it is installed on. + LINUX_GLIBC_BASELINE: "2.28" jobs: select-run: @@ -110,6 +116,26 @@ jobs: ;; esac + - name: Check the upstream base + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + version=$(sed -n 's/^version = "\([^"]*\)"/\1/p' Cargo.toml | head -1) + upstream_tag="v${version%-digdir.*}" + upstream_commit=$(git ls-remote https://github.com/superradcompany/microsandbox.git \ + "refs/tags/${upstream_tag}" "refs/tags/${upstream_tag}^{}" | awk 'END { print $1 }') + if [ -z "${upstream_commit}" ]; then + echo "::error::upstream release tag ${upstream_tag} not found" + exit 1 + fi + status=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${upstream_commit}...${GITHUB_SHA}" --jq .status) + if [ "${status}" != ahead ]; then + echo "::error::${GITHUB_SHA} is not based on upstream ${upstream_tag} (${upstream_commit}): ${status}" + exit 1 + fi + - uses: ./.github/actions/pinned-rust-toolchain-digdir - name: Check the locked dependency graph @@ -219,16 +245,16 @@ jobs: cache-bin: false cache-targets: true save-if: ${{ github.ref == 'refs/heads/main-digdir' }} - shared-key: digdir-agentd-${{ matrix.arch }} + shared-key: digdir-release-agentd-${{ matrix.arch }} - name: Install musl build dependencies run: sudo apt-get update && sudo apt-get install -y musl-tools - name: Build agentd run: | - cargo build --locked --profile ci --manifest-path crates/agentd/Cargo.toml --target ${{ matrix.target }} + cargo build --locked --release --manifest-path crates/agentd/Cargo.toml --target ${{ matrix.target }} mkdir -p build - cp target/${{ matrix.target }}/ci/agentd build/agentd + cp target/${{ matrix.target }}/release/agentd build/agentd - name: Upload guest agent uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -253,11 +279,13 @@ jobs: arch: x86_64 manylinux_image: quay.io/pypa/manylinux_2_28_x86_64@sha256:0c87ccb5996dab6c3b7612ee4fda7b80c4ab3c44a86c2541e4a872afdf4f131b libkrunfw_file: libkrunfw.so.5.6.1 + libkrunfw_asset: libkrunfw-linux-x86_64.so - target: linux-aarch64 runner: ubuntu-24.04-arm arch: aarch64 manylinux_image: quay.io/pypa/manylinux_2_28_aarch64@sha256:561427136aabf3787bffb294b3515748241e0962d1527ae28bea1e076bfb9d99 libkrunfw_file: libkrunfw.so.5.6.1 + libkrunfw_asset: libkrunfw-linux-aarch64.so steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -268,7 +296,7 @@ jobs: cache-bin: false cache-targets: true save-if: ${{ github.ref == 'refs/heads/main-digdir' }} - shared-key: digdir-runtime-${{ matrix.target }} + shared-key: digdir-release-${{ matrix.target }} - name: Install build dependencies run: dnf install -y libcap-ng-devel @@ -291,15 +319,22 @@ jobs: cc -fPIC -DABI_VERSION=${{ env.LIBKRUNFW_ABI }} -shared \ -Wl,-soname,libkrunfw.so.${{ env.LIBKRUNFW_ABI }} \ -o build/${{ matrix.libkrunfw_file }} firmware/kernel.c + strip build/${{ matrix.libkrunfw_file }} cd build ln -sf ${{ matrix.libkrunfw_file }} libkrunfw.so.${{ env.LIBKRUNFW_ABI }} ln -sf libkrunfw.so.${{ env.LIBKRUNFW_ABI }} libkrunfw.so - name: Build msb run: | - cargo build --locked --profile ci --no-default-features --features embed-binaries,net,ssh -p microsandbox-cli - cp target/ci/msb build/msb + cargo build --locked --release --no-default-features --features embed-binaries,net,ssh -p microsandbox-cli + cp target/release/msb build/msb + + - name: Verify runtime and glibc baseline + run: | build/msb --version + python3 scripts/ci/validate_linux_glibc.py \ + --max-version ${{ env.LINUX_GLIBC_BASELINE }} \ + build/agentd build/msb build/${{ matrix.libkrunfw_file }} - name: Smoke test installed runtime discovery run: | @@ -314,6 +349,24 @@ jobs: path: build/ compression-level: 0 + - name: Stage release assets + run: | + set -euo pipefail + mkdir -p artifacts + # The bundle is created here, before any artifact round trip can + # normalize modes; assert the executable bit anyway. + chmod 755 build/msb + cp build/${{ matrix.libkrunfw_file }} artifacts/${{ matrix.libkrunfw_asset }} + tar -czf artifacts/microsandbox-${{ matrix.target }}.tar.gz \ + -C build msb ${{ matrix.libkrunfw_file }} + + - name: Upload release assets + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-runtime-${{ matrix.target }} + path: artifacts/ + compression-level: 0 + build-macos: name: Build runtime (darwin-aarch64) needs: [validate, build-kernel, build-agentd] @@ -329,7 +382,7 @@ jobs: cache-bin: false cache-targets: true save-if: ${{ github.ref == 'refs/heads/main-digdir' }} - shared-key: digdir-runtime-darwin-aarch64 + shared-key: digdir-release-darwin-aarch64 - name: Download guest agent uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -353,8 +406,8 @@ jobs: - name: Build and codesign msb run: | - cargo build --locked --profile ci --no-default-features --features embed-binaries,net,ssh -p microsandbox-cli - cp target/ci/msb build/msb + cargo build --locked --release --no-default-features --features embed-binaries,net,ssh -p microsandbox-cli + cp target/release/msb build/msb codesign --entitlements msb-entitlements.plist --force -s - build/msb build/msb --version @@ -370,6 +423,26 @@ jobs: path: build/ compression-level: 0 + - name: Stage release assets + # bsdtar would otherwise add AppleDouble entries for extended attributes, + # which the runtime installer rejects. + env: + COPYFILE_DISABLE: "1" + run: | + set -euo pipefail + mkdir -p artifacts + chmod 755 build/msb + cp build/libkrunfw.${{ env.LIBKRUNFW_ABI }}.dylib artifacts/libkrunfw-darwin-aarch64.dylib + tar -czf artifacts/microsandbox-darwin-aarch64.tar.gz \ + -C build msb libkrunfw.${{ env.LIBKRUNFW_ABI }}.dylib + + - name: Upload release assets + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-runtime-darwin-aarch64 + path: artifacts/ + compression-level: 0 + rust-quality: name: Rust quality needs: build-linux @@ -754,8 +827,8 @@ jobs: path: ${{ runner.temp }}/snapshot-branch-smoke-${{ github.run_id }}-${{ github.run_attempt }} if-no-files-found: warn - windows: - name: Rust runtime and tests (${{ matrix.target }}) + build-windows: + name: Build runtime (${{ matrix.target }}) needs: [validate, build-kernel, build-agentd] runs-on: ${{ matrix.runner }} timeout-minutes: 60 @@ -769,22 +842,24 @@ jobs: rust_target: x86_64-pc-windows-msvc vs_arch: amd64 vs_host_arch: amd64 + libkrunfw_asset: libkrunfw-windows-x86_64.dll - target: windows-aarch64 runner: windows-11-arm arch: aarch64 rust_target: aarch64-pc-windows-msvc vs_arch: arm64 vs_host_arch: arm64 + libkrunfw_asset: libkrunfw-windows-aarch64.dll steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # Only Windows needs libkrunfw's MSVC helper scripts. - name: Checkout libkrunfw sources shell: pwsh run: git submodule update --init --depth 1 vendor/libkrunfw - uses: ./.github/actions/pinned-rust-toolchain-digdir with: - components: clippy targets: ${{ matrix.rust_target }} - uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 @@ -792,7 +867,7 @@ jobs: cache-bin: false cache-targets: true save-if: ${{ github.ref == 'refs/heads/main-digdir' }} - shared-key: digdir-windows-${{ matrix.target }} + shared-key: digdir-release-${{ matrix.target }} - name: Download kernel firmware bundle uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -828,8 +903,8 @@ jobs: $ErrorActionPreference = "Stop" . "$env:GITHUB_WORKSPACE\vendor\libkrunfw\scripts\msvc-env.ps1" Set-MsvcEnvironment -Architecture ${{ matrix.vs_arch }} -HostArchitecture ${{ matrix.vs_host_arch }} - cargo build --locked --profile ci --no-default-features --features embed-binaries,net,ssh -p microsandbox-cli --target ${{ matrix.rust_target }} - Copy-Item target\${{ matrix.rust_target }}\ci\msb.exe build\msb.exe -Force + cargo build --locked --release --no-default-features --features embed-binaries,net,ssh -p microsandbox-cli --target ${{ matrix.rust_target }} + Copy-Item target\${{ matrix.rust_target }}\release\msb.exe build\msb.exe -Force & build\msb.exe --version - name: Smoke test installed runtime discovery @@ -839,6 +914,87 @@ jobs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } python scripts/smoke/cli/runtime-discovery.py --msb build/msb.exe --libkrunfw build/libkrunfw.dll + - name: Stage release assets + shell: pwsh + run: | + $ErrorActionPreference = "Stop" + New-Item -ItemType Directory -Force -Path artifacts | Out-Null + Copy-Item build\libkrunfw.dll artifacts\${{ matrix.libkrunfw_asset }} -Force + tar.exe -czf artifacts\microsandbox-${{ matrix.target }}.tar.gz -C build msb.exe libkrunfw.dll + + - name: Upload release assets + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-runtime-${{ matrix.target }} + path: artifacts/ + compression-level: 0 + + windows: + name: Rust runtime and tests (${{ matrix.target }}) + needs: [validate, build-kernel, build-agentd] + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - target: windows-x86_64 + runner: windows-latest + arch: x86_64 + rust_target: x86_64-pc-windows-msvc + vs_arch: amd64 + vs_host_arch: amd64 + - target: windows-aarch64 + runner: windows-11-arm + arch: aarch64 + rust_target: aarch64-pc-windows-msvc + vs_arch: arm64 + vs_host_arch: arm64 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Checkout libkrunfw sources + shell: pwsh + run: git submodule update --init --depth 1 vendor/libkrunfw + + - uses: ./.github/actions/pinned-rust-toolchain-digdir + with: + components: clippy + targets: ${{ matrix.rust_target }} + + - uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 + with: + cache-bin: false + cache-targets: true + save-if: ${{ github.ref == 'refs/heads/main-digdir' }} + shared-key: digdir-windows-${{ matrix.target }} + + - name: Download kernel firmware bundle + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: digdir-kernel-${{ matrix.arch }} + path: vendor/libkrunfw/ + + - name: Download guest agent + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: digdir-agentd-${{ matrix.arch }} + path: build/ + + - name: Build libkrunfw + shell: pwsh + run: | + $ErrorActionPreference = "Stop" + & .\vendor\libkrunfw\scripts\build-windows.ps1 ` + -SkipKernelBundle ` + -AbiVersion ${{ env.LIBKRUNFW_ABI }} ` + -Architecture ${{ matrix.vs_arch }} ` + -HostArchitecture ${{ matrix.vs_host_arch }} ` + -Output libkrunfw.dll ` + -ImportLibrary libkrunfw.lib + New-Item -ItemType Directory -Force -Path build | Out-Null + Copy-Item vendor\libkrunfw\libkrunfw.dll build\libkrunfw.dll -Force + - name: Check msb shell: pwsh run: | @@ -950,3 +1106,219 @@ jobs: . "$env:GITHUB_WORKSPACE\vendor\libkrunfw\scripts\msvc-env.ps1" Set-MsvcEnvironment -Architecture ${{ matrix.vs_arch }} -HostArchitecture ${{ matrix.vs_host_arch }} cargo test --locked -p microsandbox-network --lib --target ${{ matrix.rust_target }} dns::upstream::system::windows:: + + # Only the source decides the notices, so they are generated while the runtime builds. + notices: + name: Third-party notices + needs: validate + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Checkout libkrunfw sources + run: git submodule update --init --depth 1 vendor/libkrunfw + + - uses: ./.github/actions/pinned-rust-toolchain-digdir + + - name: Install cargo-about + run: cargo install cargo-about --locked --features cli --version 0.9.2 + + - name: Generate third-party notices + run: | + mkdir -p notices + scripts/digdir/third-party-notices.sh notices/THIRD_PARTY_NOTICES.md + + - name: Upload third-party notices + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: digdir-third-party-notices + path: notices/THIRD_PARTY_NOTICES.md + + # The release assets that release-digdir.yml publishes once this tree lands on main-digdir. + assemble: + name: Assemble downstream runtime release + needs: [validate, notices, build-agentd, build-linux, build-macos, build-windows] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Checkout libkrunfw sources + run: git submodule update --init --depth 1 vendor/libkrunfw + + - name: Download runtime artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: release-artifacts + pattern: release-runtime-* + merge-multiple: true + + - name: Download guest agents + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: guest-agents + pattern: digdir-agentd-* + + - name: Validate artifact set + shell: bash + run: | + set -euo pipefail + cp guest-agents/digdir-agentd-x86_64/agentd release-artifacts/agentd-x86_64 + cp guest-agents/digdir-agentd-aarch64/agentd release-artifacts/agentd-aarch64 + expected=( + agentd-aarch64 + agentd-x86_64 + libkrunfw-darwin-aarch64.dylib + libkrunfw-linux-aarch64.so + libkrunfw-linux-x86_64.so + libkrunfw-windows-aarch64.dll + libkrunfw-windows-x86_64.dll + microsandbox-darwin-aarch64.tar.gz + microsandbox-linux-aarch64.tar.gz + microsandbox-linux-x86_64.tar.gz + microsandbox-windows-aarch64.tar.gz + microsandbox-windows-x86_64.tar.gz + ) + for artifact in "${expected[@]}"; do + if [ ! -s "release-artifacts/${artifact}" ]; then + echo "::error::missing or empty release artifact: ${artifact}" + exit 1 + fi + done + actual_count=$(find release-artifacts -maxdepth 1 -type f | wc -l) + if [ "${actual_count}" -ne "${#expected[@]}" ]; then + echo "::error::expected ${#expected[@]} artifacts, found ${actual_count}" + find release-artifacts -maxdepth 1 -type f -printf '%f\n' | sort + exit 1 + fi + + # The runtime installer accepts exactly the two regular files of a bundle. + - name: Validate bundle entries + shell: bash + run: | + set -euo pipefail + python3 - <<'PY' + import sys + import tarfile + from pathlib import Path + + expected = { + "microsandbox-linux-x86_64.tar.gz": {"msb", "libkrunfw.so.5.6.1"}, + "microsandbox-linux-aarch64.tar.gz": {"msb", "libkrunfw.so.5.6.1"}, + "microsandbox-darwin-aarch64.tar.gz": {"msb", "libkrunfw.5.dylib"}, + "microsandbox-windows-x86_64.tar.gz": {"msb.exe", "libkrunfw.dll"}, + "microsandbox-windows-aarch64.tar.gz": {"msb.exe", "libkrunfw.dll"}, + } + failed = False + for bundle, names in expected.items(): + with tarfile.open(Path("release-artifacts") / bundle) as archive: + members = archive.getmembers() + actual = {member.name for member in members} + irregular = [member.name for member in members if not member.isreg()] + if actual != names or len(members) != len(names) or irregular: + print(f"::error::{bundle} has entries {sorted(actual)}; expected {sorted(names)}") + failed = True + sys.exit(1 if failed else 0) + PY + + # Unix bundles must extract an executable msb; artifact round trips normalize modes. + - name: Validate bundled msb executable mode + shell: bash + run: | + set -euo pipefail + python3 - <<'PY' + import sys + from pathlib import Path + sys.path.insert(0, "scripts/ci") + import importlib.util + spec = importlib.util.spec_from_file_location( + "validate_release_artifacts", "scripts/ci/validate-release-artifacts.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + errors: list[str] = [] + for bundle in ( + "microsandbox-linux-x86_64.tar.gz", + "microsandbox-linux-aarch64.tar.gz", + "microsandbox-darwin-aarch64.tar.gz", + ): + module.validate_bundle_executable(Path("release-artifacts") / bundle, errors) + if errors: + for error in errors: + print(f"::error::{error}") + sys.exit(1) + print("bundled msb is executable in every Unix bundle") + PY + + # Gate over the published files, not only the build-job outputs: the + # earlier check protects the build, this one protects the release. + - name: Validate Linux glibc baseline + run: | + python3 -m unittest scripts.ci.test_validate_linux_glibc + python3 scripts/ci/validate_linux_glibc.py \ + --max-version ${{ env.LINUX_GLIBC_BASELINE }} \ + release-artifacts + + - name: Resolve firmware source + id: firmware + shell: bash + run: | + set -euo pipefail + kernel_version=$(awk '$1 == "KERNEL_VERSION" { print $3; exit }' vendor/libkrunfw/Makefile) + if [ -z "${kernel_version}" ]; then + echo "::error::vendor/libkrunfw/Makefile must define KERNEL_VERSION" + exit 1 + fi + echo "kernel-version=${kernel_version}" >> "${GITHUB_OUTPUT}" + + # Downloads the kernel tarball and verifies it against upstream's pinned + # checksum, as the kernel build jobs do. + - name: Fetch kernel source + uses: ./.github/actions/cache-libkrunfw-kernel + + # Named by commit rather than tag, so that the assets are final before libkrunfw is tagged. + - name: Add firmware source + shell: bash + env: + KERNEL_VERSION: ${{ steps.firmware.outputs.kernel-version }} + run: | + set -euo pipefail + commit=$(git -C vendor/libkrunfw rev-parse HEAD) + git -C vendor/libkrunfw archive --format=tar.gz --prefix="libkrunfw-${commit:0:12}/" \ + -o "${GITHUB_WORKSPACE}/release-artifacts/libkrunfw-source.tar.gz" HEAD + cp "vendor/libkrunfw/tarballs/${KERNEL_VERSION}.tar.gz" "release-artifacts/${KERNEL_VERSION}.tar.gz" + + - name: Download third-party notices + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: digdir-third-party-notices + path: release-artifacts + + - name: Add licenses + shell: bash + run: | + set -euo pipefail + cp LICENSE release-artifacts/LICENSE + cp vendor/libkrunfw/LICENSE-GPL-2.0-only release-artifacts/LICENSE-libkrunfw-GPL-2.0-only + cp vendor/libkrunfw/LICENSE-LGPL-2.1-only release-artifacts/LICENSE-libkrunfw-LGPL-2.1-only + + - name: Generate checksums + shell: bash + run: | + set -euo pipefail + (cd release-artifacts && sha256sum -- * > checksums.sha256) + (cd release-artifacts && sha256sum --check --strict checksums.sha256) + + # A pull request run builds its merge with main-digdir. Named by that tree, the assets are + # found by the tree that lands, whatever the pull request's own head. + - name: Resolve the source tree + id: tree + run: echo "tree=$(git rev-parse 'HEAD^{tree}')" >> "${GITHUB_OUTPUT}" + + # Kept as long as the repository keeps artifacts: a release can land well after its review. + - name: Upload release assets + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-${{ steps.tree.outputs.tree }} + path: release-artifacts/ + compression-level: 0 diff --git a/.github/workflows/release-digdir.yml b/.github/workflows/release-digdir.yml index 399486617..d22249e7c 100644 --- a/.github/workflows/release-digdir.yml +++ b/.github/workflows/release-digdir.yml @@ -1,689 +1,189 @@ name: Release Digdir Rust SDK runtime -# Downstream copy of the upstream release lanes that publishes only the host -# runtime artifacts consumed by the Rust SDK: the `msb` bundle, the standalone -# libkrunfw library and the guest agent. It inherits none of upstream's later -# workflow fixes automatically; compare it against `release.yml`, -# `release-linux.yml`, `release-macos.yml` and `release-windows.yml` on every -# upstream synchronization (see MAINTAINING-digdir.md). +# Publishes the runtime when main-digdir moves to a version that has no release yet. It builds +# nothing: it publishes the release assets that check-digdir.yml built, tested and assembled from +# exactly the tree that landed, and creates the v tag on the landed commit as it +# publishes the release. Tags and published releases are immutable, so a run that fails spends no +# version. A push whose version is already released is not a release, and the run ends at once. on: push: - # Source-only tags (v-source.) reuse an already published - # runtime and must not start a release. - tags: ["v*-digdir.*", "!v*-digdir.*-source.*"] - # A manual run builds and validates every release artifact without - # publishing, so that a failing release is caught before a tag is spent. + branches: [main-digdir] + # Retries a release that failed, such as one that landed before libkrunfw was tagged. workflow_dispatch: concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }} cancel-in-progress: false -env: - CARGO_TERM_COLOR: always - CARGO_NET_RETRY: "10" - CARGO_HTTP_TIMEOUT: "120" - CARGO_HTTP_MULTIPLEXING: "false" - CARGO_INCREMENTAL: "0" - LIBKRUNFW_VERSION: "5.6.1" - LIBKRUNFW_ABI: "5" - # Every shipped Linux ELF is compiled or linked inside the architecture-native - # manylinux 2.28 container, so the published runtime does not import glibc - # symbols newer than the hosts it is installed on. Validated below. - LINUX_GLIBC_BASELINE: "2.28" - permissions: contents: read jobs: - validate: - name: Validate downstream release + release: + name: Publish downstream runtime release if: github.repository == 'digdir/microsandbox' runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + # To find the check-digdir.yml run of this tree and download its release assets. + actions: read + contents: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Checkout libkrunfw sources - run: git submodule update --init --depth 1 vendor/libkrunfw - - - name: Validate version and firmware source + - name: Resolve the release + id: release shell: bash run: | set -euo pipefail - version=$(sed -n 's/^version = "\([^"]*\)"/\1/p' Cargo.toml | head -1) - expected_tag="v${version}" - if [ "${GITHUB_EVENT_NAME}" = push ] && [ "${GITHUB_REF_NAME}" != "${expected_tag}" ]; then - echo "::error::tag ${GITHUB_REF_NAME} does not match workspace version ${version}" + if [ "${GITHUB_REF}" != refs/heads/main-digdir ]; then + echo "::error::a release is published from main-digdir, not ${GITHUB_REF}" exit 1 fi - case "${version}" in - *-digdir.*) ;; - *) - echo "::error::workspace version must contain the -digdir.N prerelease suffix" - exit 1 - ;; - esac - expected_remote=https://github.com/digdir/libkrunfw.git - actual_remote=$(git config --file .gitmodules --get submodule.vendor/libkrunfw.url) - if [ "${actual_remote}" != "${expected_remote}" ]; then - echo "::error::libkrunfw submodule uses ${actual_remote}, expected ${expected_remote}" - exit 1 - fi - status=$(git submodule status vendor/libkrunfw) + version=$(sed -n 's/^version = "\([^"]*\)"/\1/p' Cargo.toml | head -1) + tag="v${version}" + status=0 + git ls-remote --exit-code origin "refs/tags/${tag}" > /dev/null || status=$? case "${status}" in - [-+]*) - echo "::error::libkrunfw submodule is not at its committed revision: ${status}" - exit 1 + 0) + echo "::notice::${tag} is already released; this push publishes nothing" + echo "publish=false" >> "${GITHUB_OUTPUT}" + exit 0 ;; + 2) ;; + *) echo "::error::could not check whether ${tag} exists"; exit 1 ;; esac + { + echo "publish=true" + echo "version=${version}" + echo "tag=${tag}" + } >> "${GITHUB_OUTPUT}" - - name: Validate the release validators - run: python3 -m unittest scripts.ci.test_validate_linux_glibc scripts.ci.test_validate_release_artifacts - - build-kernel: - name: Build guest kernel (${{ matrix.arch }}) - needs: validate - runs-on: ${{ matrix.runner }} - strategy: - fail-fast: false - matrix: - include: - - arch: x86_64 - runner: ubuntu-latest - - arch: aarch64 - runner: ubuntu-24.04-arm - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Checkout libkrunfw sources - run: git submodule update --init --depth 1 vendor/libkrunfw - - - name: Install kernel build dependencies - run: sudo apt-get update && sudo apt-get install -y bc bison flex gcc libelf-dev make python3-pyelftools - - - name: Prepare kernel source - uses: ./.github/actions/cache-libkrunfw-kernel - - # Only the generated kernel.c bundle leaves this job. Every host lane - # links its own libkrunfw from it so the library shares that platform's - # toolchain baseline. - - name: Build kernel firmware bundle - working-directory: vendor/libkrunfw - run: make -j$(nproc) kernel.c - - - name: Upload kernel firmware bundle - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: runtime-kernel-${{ matrix.arch }} - path: vendor/libkrunfw/kernel.c - compression-level: 0 - - build-agentd: - name: Build guest agent (${{ matrix.arch }}) - needs: validate - runs-on: ${{ matrix.runner }} - strategy: - fail-fast: false - matrix: - include: - - arch: x86_64 - target: x86_64-unknown-linux-musl - runner: ubuntu-latest - - arch: aarch64 - target: aarch64-unknown-linux-musl - runner: ubuntu-24.04-arm - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - uses: ./.github/actions/pinned-rust-toolchain-digdir - with: - targets: ${{ matrix.target }} - - - uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 - with: - cache-bin: false - cache-targets: true - shared-key: release-agentd-${{ matrix.arch }} - - - name: Install musl build dependencies - run: sudo apt-get update && sudo apt-get install -y musl-tools - - - name: Build agentd - run: | - cargo build --locked --release --manifest-path crates/agentd/Cargo.toml --target ${{ matrix.target }} - mkdir -p build - cp target/${{ matrix.target }}/release/agentd build/agentd - - - name: Upload agentd - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: runtime-agentd-${{ matrix.arch }} - path: build/agentd - compression-level: 0 - - build-linux: - name: Build runtime (${{ matrix.target }}) - needs: [validate, build-kernel, build-agentd] - runs-on: ${{ matrix.runner }} - # Digest-pinned manylinux 2.28 images, kept identical to upstream release.yml. - container: - image: ${{ matrix.manylinux_image }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - target: linux-x86_64 - runner: ubuntu-latest - arch: x86_64 - manylinux_image: quay.io/pypa/manylinux_2_28_x86_64@sha256:0c87ccb5996dab6c3b7612ee4fda7b80c4ab3c44a86c2541e4a872afdf4f131b - libkrunfw_file: libkrunfw.so.5.6.1 - libkrunfw_asset: libkrunfw-linux-x86_64.so - - target: linux-aarch64 - runner: ubuntu-24.04-arm - arch: aarch64 - manylinux_image: quay.io/pypa/manylinux_2_28_aarch64@sha256:561427136aabf3787bffb294b3515748241e0962d1527ae28bea1e076bfb9d99 - libkrunfw_file: libkrunfw.so.5.6.1 - libkrunfw_asset: libkrunfw-linux-aarch64.so - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - uses: ./.github/actions/pinned-rust-toolchain-digdir - - - uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 - with: - cache-bin: false - cache-targets: true - shared-key: release-runtime-${{ matrix.target }} - - - name: Install build dependencies - run: dnf install -y libcap-ng-devel - - - name: Download guest agent - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: runtime-agentd-${{ matrix.arch }} - path: build/ - - - name: Download kernel firmware bundle - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: runtime-kernel-${{ matrix.arch }} - path: firmware/ - - - name: Link libkrunfw against the baseline - run: | - mkdir -p build - cc -fPIC -DABI_VERSION=${{ env.LIBKRUNFW_ABI }} -shared \ - -Wl,-soname,libkrunfw.so.${{ env.LIBKRUNFW_ABI }} \ - -o build/${{ matrix.libkrunfw_file }} firmware/kernel.c - strip build/${{ matrix.libkrunfw_file }} - - - name: Build msb - run: | - cargo build --locked --release --no-default-features --features embed-binaries,net,ssh -p microsandbox-cli - cp target/release/msb build/msb - - - name: Verify runtime and glibc baseline - run: | - build/msb --version - python3 scripts/ci/validate_linux_glibc.py \ - --max-version ${{ env.LINUX_GLIBC_BASELINE }} \ - build/agentd build/msb build/${{ matrix.libkrunfw_file }} - - - name: Smoke test installed runtime discovery - run: | - python3 scripts/smoke/cli/runtime-discovery.py \ - --msb build/msb --libkrunfw build/${{ matrix.libkrunfw_file }} - - - name: Stage runtime artifacts - run: | - set -euo pipefail - mkdir -p artifacts - # The bundle is created in the build job, before any artifact round - # trip can normalize modes; assert the executable bit anyway. - chmod 755 build/msb - cp build/${{ matrix.libkrunfw_file }} artifacts/${{ matrix.libkrunfw_asset }} - tar -czf artifacts/microsandbox-${{ matrix.target }}.tar.gz \ - -C build msb ${{ matrix.libkrunfw_file }} - - - name: Upload runtime artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: release-runtime-${{ matrix.target }} - path: artifacts/ - compression-level: 0 - - build-macos: - name: Build runtime (${{ matrix.target }}) - needs: [validate, build-kernel, build-agentd] - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - target: darwin-aarch64 - runner: macos-14 - arch: aarch64 - libkrunfw_file: libkrunfw.5.dylib - libkrunfw_asset: libkrunfw-darwin-aarch64.dylib - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - uses: ./.github/actions/pinned-rust-toolchain-digdir - - - uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 - with: - cache-bin: false - cache-targets: true - shared-key: release-runtime-${{ matrix.target }} - - - name: Download guest agent - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: runtime-agentd-${{ matrix.arch }} - path: build/ - - - name: Download kernel firmware bundle - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: runtime-kernel-${{ matrix.arch }} - path: firmware/ - - - name: Build libkrunfw - run: | - mkdir -p build - cc -fPIC -DABI_VERSION=${{ env.LIBKRUNFW_ABI }} -shared \ - -o build/${{ matrix.libkrunfw_file }} firmware/kernel.c - - - name: Build msb - run: | - cargo build --locked --release --no-default-features --features embed-binaries,net,ssh -p microsandbox-cli - cp target/release/msb build/msb - - - name: Codesign msb - run: codesign --entitlements msb-entitlements.plist --force -s - build/msb - - - name: Verify runtime binary - run: build/msb --version - - - name: Smoke test installed runtime discovery - run: | - python3 scripts/smoke/cli/runtime-discovery.py \ - --msb build/msb --libkrunfw build/${{ matrix.libkrunfw_file }} - - - name: Stage runtime artifacts - # bsdtar would otherwise add AppleDouble entries for extended attributes, - # which the runtime installer rejects. - env: - COPYFILE_DISABLE: "1" - run: | - set -euo pipefail - mkdir -p artifacts - chmod 755 build/msb - cp build/${{ matrix.libkrunfw_file }} artifacts/${{ matrix.libkrunfw_asset }} - tar -czf artifacts/microsandbox-${{ matrix.target }}.tar.gz \ - -C build msb ${{ matrix.libkrunfw_file }} - - - name: Upload runtime artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: release-runtime-${{ matrix.target }} - path: artifacts/ - compression-level: 0 - - build-windows: - name: Build runtime (${{ matrix.target }}) - needs: [validate, build-kernel, build-agentd] - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - target: windows-x86_64 - runner: windows-latest - arch: x86_64 - rust_target: x86_64-pc-windows-msvc - vs_arch: amd64 - vs_host_arch: amd64 - libkrunfw_asset: libkrunfw-windows-x86_64.dll - - target: windows-aarch64 - runner: windows-11-arm - arch: aarch64 - rust_target: aarch64-pc-windows-msvc - vs_arch: arm64 - vs_host_arch: arm64 - libkrunfw_asset: libkrunfw-windows-aarch64.dll - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - # Only Windows needs libkrunfw's MSVC helper scripts. - - name: Checkout libkrunfw sources - shell: pwsh - run: git submodule update --init --depth 1 vendor/libkrunfw - - - uses: ./.github/actions/pinned-rust-toolchain-digdir - with: - targets: ${{ matrix.rust_target }} - - - uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 - with: - cache-bin: false - cache-targets: true - shared-key: release-runtime-${{ matrix.target }} - - - name: Download kernel firmware bundle - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: runtime-kernel-${{ matrix.arch }} - path: vendor/libkrunfw/ - - - name: Download guest agent - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: runtime-agentd-${{ matrix.arch }} - path: build/ - - - name: Build libkrunfw - shell: pwsh - run: | - $ErrorActionPreference = "Stop" - & .\vendor\libkrunfw\scripts\build-windows.ps1 ` - -SkipKernelBundle ` - -AbiVersion ${{ env.LIBKRUNFW_ABI }} ` - -Architecture ${{ matrix.vs_arch }} ` - -HostArchitecture ${{ matrix.vs_host_arch }} ` - -Output libkrunfw.dll ` - -ImportLibrary libkrunfw.lib - New-Item -ItemType Directory -Force -Path build | Out-Null - Copy-Item vendor\libkrunfw\libkrunfw.dll build\libkrunfw.dll -Force - - - name: Build msb - shell: pwsh - env: - RUSTFLAGS: -C target-feature=+crt-static - run: | - $ErrorActionPreference = "Stop" - . "$env:GITHUB_WORKSPACE\vendor\libkrunfw\scripts\msvc-env.ps1" - Set-MsvcEnvironment -Architecture ${{ matrix.vs_arch }} -HostArchitecture ${{ matrix.vs_host_arch }} - cargo build --locked --release --no-default-features --features embed-binaries,net,ssh -p microsandbox-cli --target ${{ matrix.rust_target }} - Copy-Item target\${{ matrix.rust_target }}\release\msb.exe build\msb.exe -Force - & build\msb.exe --version - - - name: Smoke test installed runtime discovery - shell: pwsh - run: python scripts/smoke/cli/runtime-discovery.py --msb build/msb.exe --libkrunfw build/libkrunfw.dll - - - name: Stage runtime artifacts - shell: pwsh - run: | - $ErrorActionPreference = "Stop" - New-Item -ItemType Directory -Force -Path artifacts | Out-Null - Copy-Item build\libkrunfw.dll artifacts\${{ matrix.libkrunfw_asset }} -Force - tar.exe -czf artifacts\microsandbox-${{ matrix.target }}.tar.gz -C build msb.exe libkrunfw.dll - - - name: Upload runtime artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: release-runtime-${{ matrix.target }} - path: artifacts/ - compression-level: 0 - - assemble: - name: Assemble downstream runtime release - needs: [validate, build-agentd, build-linux, build-macos, build-windows] - runs-on: ubuntu-latest - # Only this job publishes the release. - permissions: - contents: write - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Checkout libkrunfw sources - run: git submodule update --init --depth 1 vendor/libkrunfw - - - name: Download runtime artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - path: release-artifacts - pattern: release-runtime-* - merge-multiple: true - - - name: Download guest agents - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - path: guest-agents - pattern: runtime-agentd-* - - - name: Validate artifact set - shell: bash - run: | - set -euo pipefail - cp guest-agents/runtime-agentd-x86_64/agentd release-artifacts/agentd-x86_64 - cp guest-agents/runtime-agentd-aarch64/agentd release-artifacts/agentd-aarch64 - expected=( - agentd-aarch64 - agentd-x86_64 - libkrunfw-darwin-aarch64.dylib - libkrunfw-linux-aarch64.so - libkrunfw-linux-x86_64.so - libkrunfw-windows-aarch64.dll - libkrunfw-windows-x86_64.dll - microsandbox-darwin-aarch64.tar.gz - microsandbox-linux-aarch64.tar.gz - microsandbox-linux-x86_64.tar.gz - microsandbox-windows-aarch64.tar.gz - microsandbox-windows-x86_64.tar.gz - ) - for artifact in "${expected[@]}"; do - if [ ! -s "release-artifacts/${artifact}" ]; then - echo "::error::missing or empty release artifact: ${artifact}" - exit 1 - fi - done - actual_count=$(find release-artifacts -maxdepth 1 -type f | wc -l) - if [ "${actual_count}" -ne "${#expected[@]}" ]; then - echo "::error::expected ${#expected[@]} artifacts, found ${actual_count}" - find release-artifacts -maxdepth 1 -type f -printf '%f\n' | sort - exit 1 - fi - - # The runtime installer accepts exactly the two regular files of a bundle. - - name: Validate bundle entries - shell: bash - run: | - set -euo pipefail - python3 - <<'PY' - import sys - import tarfile - from pathlib import Path - - expected = { - "microsandbox-linux-x86_64.tar.gz": {"msb", "libkrunfw.so.5.6.1"}, - "microsandbox-linux-aarch64.tar.gz": {"msb", "libkrunfw.so.5.6.1"}, - "microsandbox-darwin-aarch64.tar.gz": {"msb", "libkrunfw.5.dylib"}, - "microsandbox-windows-x86_64.tar.gz": {"msb.exe", "libkrunfw.dll"}, - "microsandbox-windows-aarch64.tar.gz": {"msb.exe", "libkrunfw.dll"}, - } - failed = False - for bundle, names in expected.items(): - with tarfile.open(Path("release-artifacts") / bundle) as archive: - members = archive.getmembers() - actual = {member.name for member in members} - irregular = [member.name for member in members if not member.isreg()] - if actual != names or len(members) != len(names) or irregular: - print(f"::error::{bundle} has entries {sorted(actual)}; expected {sorted(names)}") - failed = True - sys.exit(1 if failed else 0) - PY - - # Unix bundles must extract an executable msb; artifact round trips normalize modes. - - name: Validate bundled msb executable mode - shell: bash - run: | - set -euo pipefail - python3 - <<'PY' - import sys - from pathlib import Path - sys.path.insert(0, "scripts/ci") - import importlib.util - spec = importlib.util.spec_from_file_location( - "validate_release_artifacts", "scripts/ci/validate-release-artifacts.py" - ) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - errors: list[str] = [] - for bundle in ( - "microsandbox-linux-x86_64.tar.gz", - "microsandbox-linux-aarch64.tar.gz", - "microsandbox-darwin-aarch64.tar.gz", - ): - module.validate_bundle_executable(Path("release-artifacts") / bundle, errors) - if errors: - for error in errors: - print(f"::error::{error}") - sys.exit(1) - print("bundled msb is executable in every Unix bundle") - PY - - # Gate over the published files, not only the build-job outputs: the - # earlier check protects the build, this one protects the release. - - name: Validate Linux glibc baseline - run: | - python3 -m unittest scripts.ci.test_validate_linux_glibc - python3 scripts/ci/validate_linux_glibc.py \ - --max-version ${{ env.LINUX_GLIBC_BASELINE }} \ - release-artifacts - - # The firmware is GPL-2.0 software, so its corresponding source is - # published with it, named by an immutable libkrunfw tag (see MAINTAINING-digdir.md). - - name: Resolve firmware source + # The firmware is GPL-2.0 software, published with its corresponding source and named by an + # immutable libkrunfw tag (see MAINTAINING-digdir.md). + - name: Resolve the firmware tag id: firmware + if: steps.release.outputs.publish == 'true' shell: bash run: | set -euo pipefail + git submodule update --init --depth 1 vendor/libkrunfw commit=$(git -C vendor/libkrunfw rev-parse HEAD) tag=$(git ls-remote --tags https://github.com/digdir/libkrunfw.git 'refs/tags/v*-digdir.*' \ | awk -v commit="${commit}" '$1 == commit { sub("refs/tags/", "", $2); sub("\\^\\{\\}$", "", $2); print $2; exit }') if [ -z "${tag}" ]; then - if [ "${GITHUB_EVENT_NAME}" = push ]; then - echo "::error::libkrunfw commit ${commit} has no v*-digdir.* tag in digdir/libkrunfw; tag it before releasing" - exit 1 - fi - echo "::warning::libkrunfw commit ${commit} has no v*-digdir.* tag yet; tag it before the release" - tag=${commit:0:12} - fi - kernel_version=$(awk '$1 == "KERNEL_VERSION" { print $3; exit }' vendor/libkrunfw/Makefile) - if [ -z "${kernel_version}" ]; then - echo "::error::vendor/libkrunfw/Makefile must define KERNEL_VERSION" + echo "::error::libkrunfw commit ${commit} has no v*-digdir.* tag in digdir/libkrunfw; tag it, then run this workflow again" exit 1 fi + kernel_version=$(awk '$1 == "KERNEL_VERSION" { print $3; exit }' vendor/libkrunfw/Makefile) { - echo "commit=${commit}" echo "tag=${tag}" echo "kernel-version=${kernel_version}" } >> "${GITHUB_OUTPUT}" - # Downloads the kernel tarball and verifies it against upstream's pinned - # checksum, as the kernel build jobs do. - - name: Fetch kernel source - uses: ./.github/actions/cache-libkrunfw-kernel - - - name: Add firmware source - id: firmware-source + # check-digdir.yml names the assets by the tree it built. Only a successful run in this + # repository counts, since a fork's pull request runs its own workflow file. The oldest is + # the one whose digests a consumer could already have pinned: the bundles are not + # byte-reproducible, so a later run of the same tree has other digests. + - name: Find the check run that built this tree + id: build + if: steps.release.outputs.publish == 'true' shell: bash env: - FIRMWARE_TAG: ${{ steps.firmware.outputs.tag }} - KERNEL_VERSION: ${{ steps.firmware.outputs.kernel-version }} - run: | - set -euo pipefail - git -C vendor/libkrunfw archive --format=tar.gz --prefix="libkrunfw-${FIRMWARE_TAG}/" \ - -o "${GITHUB_WORKSPACE}/release-artifacts/libkrunfw-source.tar.gz" HEAD - cp "vendor/libkrunfw/tarballs/${KERNEL_VERSION}.tar.gz" "release-artifacts/${KERNEL_VERSION}.tar.gz" - kernel_sha256=$(sha256sum "release-artifacts/${KERNEL_VERSION}.tar.gz" | cut -d ' ' -f 1) - echo "kernel-sha256=${kernel_sha256}" >> "${GITHUB_OUTPUT}" - - - uses: ./.github/actions/pinned-rust-toolchain-digdir - - - name: Install cargo-about - run: cargo install cargo-about --locked --features cli --version 0.9.2 - - - name: Add licenses and third-party notices - shell: bash - run: | - set -euo pipefail - cp LICENSE release-artifacts/LICENSE - cp vendor/libkrunfw/LICENSE-GPL-2.0-only release-artifacts/LICENSE-libkrunfw-GPL-2.0-only - cp vendor/libkrunfw/LICENSE-LGPL-2.1-only release-artifacts/LICENSE-libkrunfw-LGPL-2.1-only - scripts/digdir/third-party-notices.sh release-artifacts/THIRD_PARTY_NOTICES.md - - - name: Generate checksums - shell: bash + GH_TOKEN: ${{ github.token }} + REPOSITORY_ID: ${{ github.repository_id }} run: | set -euo pipefail - (cd release-artifacts && sha256sum * > checksums.sha256) - (cd release-artifacts && sha256sum --check --strict checksums.sha256) + name="release-$(git rev-parse 'HEAD^{tree}')" + run_id="" + while read -r id; do + run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${id}" --jq '"\(.status) \(.conclusion) \(.path)"') + case "${run}" in + "completed success .github/workflows/check-digdir.yml") run_id="${id}"; break ;; + # A newer run has other digests, so wait for this one. + *" .github/workflows/check-digdir.yml") + if [ "${run%% *}" != completed ]; then + echo "::error::check run ${id} of this tree has not finished; run this workflow again once it has" + exit 1 + fi + ;; + esac + done < <(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${name}&per_page=100" \ + --jq "[.artifacts[] | select((.expired | not) and .workflow_run.head_repository_id == ${REPOSITORY_ID})] + | sort_by(.created_at) | .[].workflow_run.id") + if [ -z "${run_id}" ]; then + echo "::error::no successful check-digdir.yml run built this tree (${name}). If main-digdir moved after a pull request's checks ran, they built another tree. Run gh workflow run check-digdir.yml --ref main-digdir, wait for it to pass, then run gh workflow run release-digdir.yml --ref main-digdir" + exit 1 + fi + echo "Publishing the release assets of ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${run_id}" + { + echo "run-id=${run_id}" + echo "name=${name}" + } >> "${GITHUB_OUTPUT}" - - name: Upload release assets for inspection - if: github.event_name != 'push' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + - name: Download the release assets + if: steps.release.outputs.publish == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: release-assets - path: release-artifacts/ - compression-level: 0 - retention-days: 7 + name: ${{ steps.build.outputs.name }} + path: release-artifacts + run-id: ${{ steps.build.outputs.run-id }} + github-token: ${{ github.token }} + + - name: Verify the release assets + if: steps.release.outputs.publish == 'true' + run: (cd release-artifacts && sha256sum --check --strict checksums.sha256) - # Manual runs write the notes too, to the job summary, so that the - # upstream base check below runs before a tag is spent. - name: Write release notes + if: steps.release.outputs.publish == 'true' shell: bash env: - GH_TOKEN: ${{ github.token }} + VERSION: ${{ steps.release.outputs.version }} FIRMWARE_TAG: ${{ steps.firmware.outputs.tag }} KERNEL_VERSION: ${{ steps.firmware.outputs.kernel-version }} - KERNEL_SHA256: ${{ steps.firmware-source.outputs.kernel-sha256 }} run: | set -euo pipefail - version=$(sed -n 's/^version = "\([^"]*\)"/\1/p' Cargo.toml | head -1) - upstream_tag="v${version%-digdir.*}" + upstream_tag="v${VERSION%-digdir.*}" upstream_commit=$(git ls-remote https://github.com/superradcompany/microsandbox.git \ "refs/tags/${upstream_tag}" "refs/tags/${upstream_tag}^{}" | awk 'END { print $1 }') - if [ -z "${upstream_commit}" ]; then - echo "::error::upstream release tag ${upstream_tag} not found" - exit 1 - fi - status=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${upstream_commit}...${GITHUB_SHA}" --jq .status) - if [ "${status}" != ahead ]; then - echo "::error::${GITHUB_SHA} is not based on upstream ${upstream_tag} (${upstream_commit}): ${status}" - exit 1 - fi + kernel_sha256=$(awk -v name="${KERNEL_VERSION}.tar.gz" '$2 == name { print $1 }' release-artifacts/checksums.sha256) cat > release-notes.md <> "${GITHUB_STEP_SUMMARY}" - # Immutable releases lock their assets on publication, so upload every - # asset to a draft first and publish it last. A draft left by a failed - # run is not immutable and is replaced. Manual runs stop before this. - - name: Create GitHub release - if: github.event_name == 'push' + # Immutable releases lock their assets on publication, so upload every asset to a draft + # first and publish it last. Publishing the draft creates its tag on the commit that + # landed. A draft left by a failed run has no tag yet and is not immutable, and is replaced. + - name: Publish the release + if: steps.release.outputs.publish == 'true' env: GH_TOKEN: ${{ github.token }} + TAG: ${{ steps.release.outputs.tag }} run: | set -euo pipefail - if [ "$(gh release view "${GITHUB_REF_NAME}" --json isDraft --jq .isDraft 2>/dev/null)" = true ]; then - gh release delete "${GITHUB_REF_NAME}" --yes + # Every draft for the tag goes, so that the commands below, which find the release by + # its tag, find only this run's. + drafts=$(gh api --paginate "repos/${GITHUB_REPOSITORY}/releases?per_page=100" \ + --jq ".[] | select(.draft and .tag_name == \"${TAG}\") | .id") + for id in ${drafts}; do + gh api --method DELETE "repos/${GITHUB_REPOSITORY}/releases/${id}" + done + gh release create "${TAG}" --draft --prerelease --target "${GITHUB_SHA}" \ + --title "${TAG}" --notes-file release-notes.md + gh release upload "${TAG}" release-artifacts/* + # A tag pushed since this run started would take the release in place of this commit. + status=0 + git ls-remote --exit-code origin "refs/tags/${TAG}" > /dev/null || status=$? + if [ "${status}" != 2 ]; then + echo "::error::${TAG} appeared during this run, or could not be checked. The draft stays unpublished" + exit 1 fi - gh release create "${GITHUB_REF_NAME}" --draft --prerelease \ - --title "${GITHUB_REF_NAME}" --notes-file release-notes.md - gh release upload "${GITHUB_REF_NAME}" release-artifacts/* - gh release edit "${GITHUB_REF_NAME}" --draft=false + gh release edit "${TAG}" --draft=false diff --git a/MAINTAINING-digdir.md b/MAINTAINING-digdir.md index 240e912cc..a95585747 100644 --- a/MAINTAINING-digdir.md +++ b/MAINTAINING-digdir.md @@ -45,12 +45,14 @@ upstream surfaces the fork does not ship to the script. Upstream's workflow files stay unchanged and are disabled with `gh workflow disable `. The Digdir workflows are: -- `check-digdir.yml`: the checks, on pull requests and on demand. Caches are saved only in runs on `main-digdir`. It +- `check-digdir.yml`: the checks, on pull requests and on demand. It builds the runtime once, with the release + profile, runs the integration and smoke tests against those binaries, and assembles them into the release assets, + which the run keeps as an artifact. Caches are saved only in runs on `main-digdir`. It includes upstream's KVM lanes from `check.yml`: the integration tests with their fixtures, and the CLI smoke tests. Upstream runs them on a self-hosted runner; here they run on GitHub-hosted Linux runners, which expose KVM. - `test-platform-digdir.yml`: a copy of upstream's `test-platform.yml`, called by `check-digdir.yml`. -- `release-digdir.yml`: a copy of upstream's release lanes. It publishes the runtime for a `v` tag; a manual - run builds and validates everything without publishing. +- `release-digdir.yml`: runs when `main-digdir` moves to a version that has no release yet, and publishes the release + assets of the `check-digdir.yml` run that built exactly that tree. It builds nothing (see [Release](#6-release)). When synchronizing, compare upstream's release workflows, `test-platform.yml` and the KVM lanes in `check.yml` with the Digdir copies and adopt build and test changes. Disable new upstream workflows unless they are useful for our CI; remove @@ -72,8 +74,8 @@ lives in upstream's `cache-libkrunfw-kernel` action. ## Review gates 1. **After triage:** review the triage record before rebasing. -2. **Before the tag:** review the `sync/digdir-X.Y.Z` pull request: the `range-diff` against the triage record, the - tree diff from the upstream tag, and the test results. +2. **Before landing, which releases the runtime:** review the `sync/digdir-X.Y.Z` pull request: the `range-diff` + against the triage record, the tree diff from the upstream tag, and the test results. A rewritten sync branch never gets `pull_request` checks, so run them with `workflow_dispatch` after every push. Land the approved queue by moving the branch; GitHub then marks the pull request as merged: @@ -155,19 +157,38 @@ git range-diff "$old_msb_base"..origin/main-digdir "$target_msb_tag"..sync/digdi ### 6. Release -`check-digdir.yml` runs the workspace checks and the hardware-backed integration tests. Then: - -1. Run `gh workflow run check-digdir.yml --ref main-digdir` and wait for it to pass. -2. Run `gh workflow run release-digdir.yml --ref main-digdir`, a dry run that builds and validates every artifact - without publishing. -3. Tag the libkrunfw commit, as libkrunfw's MAINTAINING-digdir.md describes. -4. Tag `v`. The release workflow builds and publishes the runtime for Linux, macOS and Windows. - -The release also attaches the license texts, third-party notices and the firmware's corresponding source, and refuses -to publish while the libkrunfw commit is untagged. - -A failed or partial release is not repaired; publish a new Digdir revision. After publishing, verify the checksums and -the Linux glibc baseline, and record the bundle digests for consumers. +Landing a commit with a new version on `main-digdir` releases it. `release-digdir.yml` runs on the push, finds the +oldest successful `check-digdir.yml` run in this repository that built the same tree, and publishes that run's release +assets. It creates the `v` tag on the landed commit as it publishes the release. Nothing is built twice, and +the published runtime is the one the checks tested. A push whose version already has a release publishes nothing. + +1. Before landing, `check-digdir.yml` must pass on exactly the tree that lands. A sync branch's run counts, since + moving `main-digdir` to it keeps the commit; a rewritten branch gets no `pull_request` checks, so start it with + `gh workflow run check-digdir.yml --ref sync/digdir-X.Y.Z`. A pull request's run builds it merged into + `main-digdir` as it was when the run started, which is the tree that lands unless `main-digdir` moves before the + merge. If it moves, rebase the pull request onto `main-digdir` and force-push it, and let its checks run again + before merging it. +2. Tag the libkrunfw commit, as libkrunfw's MAINTAINING-digdir.md describes. +3. Land it. A synchronization lands by moving `main-digdir` to the approved sync branch, as + [Review gates](#review-gates) shows. Never use its pull request's merge button, which would replay the rebuilt + queue onto the old `main-digdir`. Once `main-digdir` points at the sync branch's head, GitHub marks the pull request + as merged by itself. Don't close or merge it. Any other change lands through its pull request's merge button, which + rebase-merges it. Either way, the push releases the runtime if its `Cargo.toml` version has no release yet. Only a + version bump does: the last commit of a synchronization, or a pull request that bumps `-digdir.` between + synchronizations. +4. After a synchronization, run `gh workflow run check-digdir.yml --ref main-digdir`, which saves the caches that pull + requests restore. Nothing waits for it. Run it also after a change to a cache key lands: until then, pull requests + build without that cache. + +The release assets, and so the bundle digests consumers pin, are final once the first check run of the tree passes, +before anything lands. The bundles are not byte-reproducible, so a later run of the same tree, such as the one in step +4, has other digests, and the release takes the oldest. The firmware's corresponding source is named by its libkrunfw +commit, so that the assets don't depend on whether libkrunfw is tagged yet. The release notes name its tag. + +The release refuses, without creating anything, when the libkrunfw commit is untagged or no check run built the tree. +Tag libkrunfw, or run `gh workflow run check-digdir.yml --ref main-digdir` and wait for it to pass, then run +`gh workflow run release-digdir.yml --ref main-digdir` to release the same version. A published release is not +repaired. Publish a new Digdir revision instead. After publishing, verify the checksums and the Linux glibc baseline. ## Updating consumers