[ci] verify native release artifacts before publishing (STAR-104)

.github/workflows/release.yml

@@ -1,39 +1,162 @@
 name: Release
 
-# Build and publish cross-platform binaries when a vX.Y.Z tag is pushed.
-# The tag itself is still cut by a human (after maintainer confirmation) — this
-# workflow only turns an existing tag into prebuilt archives + a GitHub Release.
+# Build once, test those archives natively, then publish the same bytes.
+# Maintainers approve the annotated tag and its release notes before pushing it.
 on:
   push:
-    tags:
-      - "v*"
+    branches: ["master"]
+    tags: ["v*"]
+  pull_request:
+    branches: ["master"]
 
 permissions:
-  contents: write
+  contents: read
 
 jobs:
-  goreleaser:
+  package:
     runs-on: ubuntu-22.04
+    timeout-minutes: 20
+    outputs:
+      version: ${{ steps.version.outputs.version }}
     steps:
       - uses: actions/checkout@v5
         with:
-          fetch-depth: 0 # full history for the changelog
+          fetch-depth: 0
+          persist-credentials: false
       - uses: actions/setup-go@v6
         with:
           go-version: "1.27.1"
       - name: Check source vulnerabilities
         run: go run golang.org/x/vuln/cmd/govulncheck@v1.8.0 ./...
-      - name: Capture Go version
-        # GoReleaser has no built-in go-version var; expose it for the ldflags.
-        # Use `go env GOVERSION` (e.g. go1.25.9) — a single token, since ldflag
-        # values must not contain spaces (`go version ...` would break linking).
-        run: echo "GOVERSION=$(go env GOVERSION)" >> "$GITHUB_ENV"
-      - name: Run GoReleaser
-        # Third-party action pinned to a full commit SHA (supply-chain policy).
+      - name: Select release or snapshot build
+        run: |
+          echo "GOVERSION=$(go env GOVERSION)" >> "$GITHUB_ENV"
+          if [[ "$GITHUB_REF" == refs/tags/* ]]; then
+            test "$(git cat-file -t "$GITHUB_REF")" = tag
+            echo 'RELEASE_ARGS=release --clean --skip=publish' >> "$GITHUB_ENV"
+          else
+            echo 'RELEASE_ARGS=release --snapshot --clean --skip=publish' >> "$GITHUB_ENV"
+          fi
+      - name: Build archives without publishing
         uses: goreleaser/goreleaser-action@5daf1e915a5f0af01ddbcd89a43b8061ff4f1a89 # v7.2.2
         with:
           distribution: goreleaser
           version: "v2.18.2"
-          args: release --clean
+          args: ${{ env.RELEASE_ARGS }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Read the packaged version
+        id: version
+        run: |
+          python3 - <<'PY'
+          import json, os
+          with open('dist/metadata.json') as source:
+              version = json.load(source)['version']
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as output:
+              print('version=' + version, file=output)
+          PY
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: release-archives
+          if-no-files-found: error
+          retention-days: 7
+          path: |
+            dist/*.tar.gz
+            dist/*.zip
+            dist/checksums.txt
+
+  install:
+    needs: package
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-22.04
+            target: linux_amd64.tar.gz
+          - os: ubuntu-22.04-arm
+            target: linux_arm64.tar.gz
+          - os: macos-15-intel
+            target: macOS_amd64.tar.gz
+          - os: macos-14
+            target: macOS_arm64.tar.gz
+          - os: windows-2022
+            target: windows_amd64.zip
+          - os: windows-11-arm
+            target: windows_arm64.zip
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+      - uses: actions/setup-go@v6
+        with:
+          go-version: "1.27.1"
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
+        with:
+          python-version: "3.13"
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: release-archives
+          path: dist
+      - name: Download the rollback baseline package
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PREVIOUS_ARCHIVE: starcli_0.1.2_${{ matrix.target }}
+        run: |
+          gh release download v0.1.2 --repo "$GITHUB_REPOSITORY" --dir previous \
+            --pattern checksums.txt --pattern "$PREVIOUS_ARCHIVE"
+      - name: Verify checksums, installation, upgrade and rollback
+        shell: bash
+        env:
+          RELEASE_VERSION: ${{ needs.package.outputs.version }}
+        run: python e2e/verify-release.py dist "$RELEASE_VERSION" previous 0.1.2
+
+  publish:
+    if: startsWith(github.ref, 'refs/tags/')
+    needs: [package, install]
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    permissions:
+      contents: write
+      id-token: write
+      attestations: write
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: release-archives
+          path: dist
+      - name: Verify the bytes received for publication
+        working-directory: dist
+        run: sha256sum --check checksums.txt
+      - name: Attest the tested archives
+        uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
+        with:
+          subject-path: |
+            dist/*.tar.gz
+            dist/*.zip
+            dist/checksums.txt
+      - name: Publish with the approved annotated tag notes
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          test "$(git cat-file -t "$GITHUB_REF")" = tag
+          release_title=$(git for-each-ref --format='%(contents:subject)' "$GITHUB_REF")
+          git for-each-ref --format='%(contents:body)' "$GITHUB_REF" > release-notes.md
+          test -n "$release_title"
+          test -n "$(tr -d '[:space:]' < release-notes.md)"
+          # No --clobber: a retry cannot overwrite an existing release.
+          extra=()
+          if [[ "$GITHUB_REF_NAME" == *-* ]]; then extra+=(--prerelease); fi
+          gh release create "$GITHUB_REF_NAME" dist/*.tar.gz dist/*.zip dist/checksums.txt \
+            --verify-tag --title "$release_title" --notes-file release-notes.md --draft "${extra[@]}"
+          if [[ "$GITHUB_REF_NAME" == *-* ]]; then
+            gh release edit "$GITHUB_REF_NAME" --draft=false --prerelease
+          else
+            gh release edit "$GITHUB_REF_NAME" --draft=false --latest
+          fi

.goreleaser.yaml

@@ -2,13 +2,14 @@
 # CLI can be installed without a Go toolchain. Validate/iterate locally with:
 #   go run github.com/goreleaser/goreleaser/v2@v2.18.2 check
 #   go run github.com/goreleaser/goreleaser/v2@v2.18.2 release --snapshot --clean --skip=publish
-# Releases are produced on a vX.Y.Z tag by .github/workflows/release.yml.
+# .github/workflows/release.yml tests these archives before uploading them on
+# an approved annotated vX.Y.Z tag; local snapshots never publish.
 version: 2
 
 project_name: starcli
 
 # GoVersion isn't a built-in GoReleaser template var, so the release workflow
-# exports `GOVERSION=$(go version)` into the env; this entry surfaces it (and
+# exports `GOVERSION=$(go env GOVERSION)` into the env; this entry surfaces it (and
 # stays empty for a local `--snapshot` where it isn't set, rather than erroring).
 env:
   - GOVERSION={{ if index .Env "GOVERSION" }}{{ .Env.GOVERSION }}{{ end }}

Makefile

@@ -51,7 +51,7 @@ build_windows:
 # nothing is published). Inspect the artifacts under dist/. `make build` stays
 # the fast single-platform dev build; this is for checking what a release ships.
 snapshot:
-	GOVERSION=`go env GOVERSION` $(GORUN) github.com/goreleaser/goreleaser/v2@latest release --snapshot --clean
+	GOVERSION=`go env GOVERSION` $(GORUN) github.com/goreleaser/goreleaser/v2@v2.18.2 release --snapshot --clean --skip=publish
 
 run: build
 	./$(BINARY)

README.md

@@ -33,6 +33,40 @@ StarCLI is a versatile tool that provides a convenient environment for running S
 
 ## Installation
 
+### Prebuilt archives
+
+Download the archive for your system and `checksums.txt` from the same
+[GitHub Release](https://github.com/1set/starcli/releases). No Go installation is
+needed to run a release binary.
+
+| System | Architecture | Archive suffix |
+|---|---|---|
+| Linux | amd64 / arm64 | `linux_amd64.tar.gz` / `linux_arm64.tar.gz` |
+| macOS | Intel / Apple Silicon | `macOS_amd64.tar.gz` / `macOS_arm64.tar.gz` |
+| Windows | x64 / ARM64 | `windows_amd64.zip` / `windows_arm64.zip` |
+
+1. Compute the archive's SHA-256 (`sha256sum FILE` on Linux,
+   `shasum -a 256 FILE` on macOS, or `Get-FileHash FILE -Algorithm SHA256` in
+   PowerShell) and compare it with the matching filename in `checksums.txt`.
+2. Extract the archive. Keep its README and license; copy `starcli` (or
+   `starcli.exe`) into a directory on your PATH.
+3. Run `starcli --version` and `starcli -c 'print(6 * 7)'`. The version must
+   match the downloaded release and the script must print `42`.
+
+Before upgrading, keep a copy of the currently installed binary and record its
+version. Stop running CLI processes, replace the binary, and repeat the checks
+above. To roll back, stop those processes and restore the saved binary. This
+rehearses binary replacement; backing up script-owned files or databases is the
+host application's responsibility.
+
+The release workflow verifies all six archive layouts and checksums, then runs
+installation, upgrade from v0.1.2, the CLI end-to-end suite, and binary rollback
+on all six native targets. It publishes those same archives after the checks
+pass and adds GitHub artifact attestations. For releases produced by this
+workflow, verify provenance with `gh attestation verify FILE --repo 1set/starcli`.
+Older releases may not include attestations. Binaries are not Apple notarized
+or Windows Authenticode signed.
+
 ### From Source
 
 Clone the repository and build from source:

e2e/e2e_test.go

@@ -25,10 +25,18 @@ import (
 	"time"
 )
 
-// binPath is the freshly built starcli binary, set up once in TestMain.
+// binPath is a freshly built binary or the installed release candidate.
 var binPath string
 
 func TestMain(m *testing.M) {
+	if candidate := os.Getenv("STARCLI_TEST_BINARY"); candidate != "" {
+		var err error
+		binPath, err = filepath.Abs(candidate)
+		if err != nil {
+			panic(err)
+		}
+		os.Exit(m.Run())
+	}
 	dir, err := os.MkdirTemp("", "starcli-e2e")
 	if err != nil {
 		panic(err)

e2e/verify-release.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+"""Verify real release archives, then rehearse install, upgrade, and rollback.
+
+Usage: python3 e2e/verify-release.py DIST VERSION PREVIOUS_DIST PREVIOUS_VERSION
+Only a temporary installation directory is changed. No system install or
+release publication occurs. Run from the repository root with Go on PATH.
+"""
+
+import hashlib
+import io
+import os
+from pathlib import Path
+import platform
+import re
+import subprocess
+import sys
+import tarfile
+import tempfile
+import zipfile
+
+
+TARGETS = {
+    ("linux", "amd64"), ("linux", "arm64"),
+    ("macOS", "amd64"), ("macOS", "arm64"),
+    ("windows", "amd64"), ("windows", "arm64"),
+}
+
+
+def native_target():
+    system = {"Linux": "linux", "Darwin": "macOS", "Windows": "windows"}[platform.system()]
+    arch = {"x86_64": "amd64", "amd64": "amd64", "arm64": "arm64", "aarch64": "arm64"}[platform.machine().lower()]
+    return system, arch
+
+
+def verify_archives(directory, version, *, native_only=False):
+    """Pin the complete archive/checksum/member contract for all six targets."""
+    if not re.fullmatch(r"\d+\.\d+\.\d+(?:-[A-Za-z0-9.-]+)?", version):
+        raise ValueError(f"invalid version: {version!r}")
+    expected = {
+        f"starcli_{version}_{system}_{arch}." + ("zip" if system == "windows" else "tar.gz"): (system, arch)
+        for system, arch in TARGETS
+    }
+    checksums = {}
+    for line in (directory / "checksums.txt").read_text().splitlines():
+        match = re.fullmatch(r"([a-f0-9]{64})  (\S+)", line)
+        if not match or match[2] in checksums:
+            raise ValueError("invalid or duplicate checksum entry")
+        checksums[match[2]] = match[1]
+    if checksums.keys() != expected.keys():
+        raise ValueError("checksum manifest must contain exactly the six supported archives")
+    target = native_target()
+    archives = {name for name, value in expected.items() if not native_only or value == target}
+    actual = {p.name for p in directory.iterdir() if p.is_file()}
+    if actual != archives | {"checksums.txt"}:
+        raise ValueError(f"unexpected or missing distribution files: {actual ^ (archives | {'checksums.txt'})}")
+    native_binary = None
+    for name in sorted(archives):
+        archive = (directory / name).read_bytes()
+        if hashlib.sha256(archive).hexdigest() != checksums[name]:
+            raise ValueError(f"checksum mismatch: {name}")
+        system, arch = expected[name]
+        binary = "starcli.exe" if system == "windows" else "starcli"
+        members = {}
+        if system == "windows":
+            with zipfile.ZipFile(io.BytesIO(archive)) as package:
+                for member in package.infolist():
+                    if member.is_dir() or member.filename in members:
+                        raise ValueError(f"non-file or duplicate ZIP member: {member.filename}")
+                    members[member.filename] = package.read(member)
+        else:
+            with tarfile.open(fileobj=io.BytesIO(archive), mode="r:gz") as package:
+                for member in package:
+                    if not member.isfile() or member.name in members:
+                        raise ValueError(f"non-file or duplicate TAR member: {member.name}")
+                    if member.name == binary and not member.mode & 0o111:
+                        raise ValueError(f"binary is not executable: {name}")
+                    members[member.name] = package.extractfile(member).read()
+        if members.keys() != {binary, "README.md", "LICENSE"} or not all(members.values()):
+            raise ValueError(f"archive must contain only a binary, README and license: {name}")
+        if (system, arch) == target:
+            native_binary = members[binary]
+        print(f"verified {name}", flush=True)
+    if native_binary is None:
+        raise ValueError(f"missing native binary: {target}")
+    return native_binary
+
+
+def run(binary, *args, expected_stdout=None, success=True):
+    result = subprocess.run([str(binary), *args], capture_output=True, text=True, timeout=30)
+    if (result.returncode == 0) != success or (expected_stdout is not None and result.stdout != expected_stdout):
+        raise RuntimeError(f"{args!r}: exit={result.returncode}\n{result.stdout}\n{result.stderr}")
+    return result.stdout
+
+
+def smoke(binary, version):
+    banner = re.sub(r"\x1b\[[0-9;]*m", "", run(binary, "--version"))
+    # v0.1.2 printed a bare version; current releases include the v prefix.
+    summaries = [line.split("GitSummary: ", 1)[1] for line in banner.splitlines() if "GitSummary: " in line]
+    if len(summaries) != 1 or summaries[0].removeprefix("v") != version:
+        raise ValueError(f"installed binary does not identify v{version}: {banner}")
+    run(binary, "-c", "print(6 * 7)", expected_stdout="42\n")
+    run(binary, "-c", 'fail("installation test")', success=False)
+
+
+def main():
+    directory, version, previous, previous_version = sys.argv[1:]
+    candidate = verify_archives(Path(directory), version)
+    baseline = verify_archives(Path(previous), previous_version, native_only=True)
+    with tempfile.TemporaryDirectory(prefix="starcli install with spaces ") as temp:
+        binary = Path(temp) / ("starcli.exe" if os.name == "nt" else "starcli")
+        for label, data, expected_version in (
+            ("install baseline", baseline, previous_version),
+            ("upgrade candidate", candidate, version),
+            ("rollback baseline", baseline, previous_version),
+        ):
+            pending = binary.with_suffix(".pending")
+            pending.write_bytes(data)
+            pending.chmod(0o755)
+            pending.replace(binary)
+            smoke(binary, expected_version)
+            if label == "upgrade candidate":
+                subprocess.run(
+                    ["go", "test", "-v", "./e2e", "-count=1"], check=True,
+                    env={**os.environ, "STARCLI_TEST_BINARY": str(binary)}, timeout=180,
+                )
+            if binary.read_bytes() != data:
+                raise ValueError(f"installed bytes changed during {label}")
+            print(f"passed {label}: v{expected_version}", flush=True)
+
+
+if __name__ == "__main__":
+    main()