From ff5635a9dc06ef5bcac98c6fe905e68f4a93b54b Mon Sep 17 00:00:00 2001 From: RodCor Date: Tue, 13 Oct 2026 14:20:26 -0300 Subject: [PATCH 1/2] fix(release): stage npm packages for maintainer approval --- .github/workflows/ci.yml | 14 ++ .github/workflows/npm-stage.yml | 104 +++++++++++ .github/workflows/release.yml | 295 +------------------------------- docs/npm-staged-publishing.md | 73 ++++++++ scripts/assemble-npm.sh | 160 +++++++++++++++++ scripts/stage-package.mjs | 72 ++++++++ scripts/stage-package.test.mjs | 85 +++++++++ scripts/test-assemble-npm.py | 73 ++++++++ 8 files changed, 588 insertions(+), 288 deletions(-) create mode 100644 .github/workflows/npm-stage.yml create mode 100644 docs/npm-staged-publishing.md create mode 100644 scripts/assemble-npm.sh create mode 100644 scripts/stage-package.mjs create mode 100644 scripts/stage-package.test.mjs create mode 100644 scripts/test-assemble-npm.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e83e8e..4c72575 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,20 @@ env: RUST_BACKTRACE: 1 jobs: + npm-staging: + name: npm staging regression tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + - uses: actions/setup-node@v5 + with: + node-version: 24 + package-manager-cache: false + - run: node --test scripts/stage-package.test.mjs + - run: python3 scripts/test-assemble-npm.py + decision-examples: name: decision provider examples runs-on: ubuntu-latest diff --git a/.github/workflows/npm-stage.yml b/.github/workflows/npm-stage.yml new file mode 100644 index 0000000..4be477a --- /dev/null +++ b/.github/workflows/npm-stage.yml @@ -0,0 +1,104 @@ +name: stage npm release + +on: + workflow_call: + inputs: + release_tag: + required: true + type: string + secrets: + NPM_TOKEN: + required: true + workflow_dispatch: + inputs: + release_tag: + description: Existing stable GitHub release tag (for example v2.9.0) + required: true + type: string + +permissions: + contents: read + id-token: write + +concurrency: + group: npm-stage-${{ inputs.release_tag }} + cancel-in-progress: false + +jobs: + stage: + if: ${{ vars.PUBLISH_NPM == 'true' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/main') }} + runs-on: ubuntu-latest + env: + RELEASE_TAG: ${{ inputs.release_tag }} + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: validate stable release tag + run: '[[ "$RELEASE_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]' + + - uses: actions/checkout@v5 + with: + ref: ${{ inputs.release_tag }} + path: release-source + fetch-depth: 0 + persist-credentials: false + + - name: verify release source and download existing assets + working-directory: release-source + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + git merge-base --is-ancestor HEAD origin/main + python3 - <<'PY' + import os, tomllib + with open('Cargo.toml', 'rb') as f: + assert tomllib.load(f)['workspace']['package']['version'] == os.environ['RELEASE_TAG'][1:] + PY + gh release view "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" --json isDraft,isPrerelease > /tmp/npm-release.json + jq -e '.isDraft == false and .isPrerelease == false' /tmp/npm-release.json + gh release download "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" --dir dist + gh attestation verify dist/checksums.txt --repo "$GITHUB_REPOSITORY" \ + --signer-workflow "$GITHUB_REPOSITORY/.github/workflows/release.yml" \ + --source-ref "refs/tags/$RELEASE_TAG" --source-digest "$(git rev-parse HEAD)" + (cd dist && sha256sum --check --strict checksums.txt) + + - uses: actions/setup-node@v5 + with: + node-version: 24 + registry-url: https://registry.npmjs.org + package-manager-cache: false + + - name: install staging-capable npm + run: | + npm install --global npm@11.21.0 --ignore-scripts + test "$(npm --version)" = 11.21.0 + npm stage --help + + - name: assemble packages without registry credentials + working-directory: release-source + run: bash "$GITHUB_WORKSPACE/scripts/assemble-npm.sh" "${RELEASE_TAG#v}" + + - name: build independently versioned SDK without registry credentials + working-directory: release-source/npm/kimetsu-sdk + run: | + npm ci --ignore-scripts + npm run build + + - name: stage packages (never approve) + working-directory: release-source + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + set -euo pipefail + if [ -z "$NODE_AUTH_TOKEN" ]; then + echo '::error::NPM_TOKEN is not configured' + exit 1 + fi + echo '## npm stages awaiting maintainer approval' >> "$GITHUB_STEP_SUMMARY" + for directory in stage/* stage-remote/* npm/kimetsu npm/kimetsu-remote npm/kimetsu-sdk; do + node "$GITHUB_WORKSPACE/scripts/stage-package.mjs" "$directory" + done + echo 'Staging does not make a version public. Review and approve platform packages before their wrappers; see docs/npm-staged-publishing.md.' >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e1cd690..c15c867 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -343,296 +343,15 @@ jobs: echo "kimetsu uninstall --dry-run" >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY - publish-npm: - # Publish the `kimetsu` npm package after the binary matrix + GH Release - # succeed. esbuild/turbo style: one per-platform package per target (lean - # binary inside, os/cpu set) plus the main `kimetsu` package that lists - # them as optionalDependencies. npm installs only the matching platform - # package; the bin/cli.js launcher execs its binary. No postinstall. - # - # Embeddings is NOT shipped as a package — it's fetched on demand by the - # launcher when KIMETSU_NPM_FLAVOR=embeddings is set (see npm/kimetsu). - # - # Gated, like publish-crates, on: - # (a) tag push (handled by `if`) - # (b) the repo variable PUBLISH_NPM == 'true' - # (c) the NPM_TOKEN secret being set - # To enable: `gh variable set PUBLISH_NPM --body true` and - # `gh secret set NPM_TOKEN` (an npm automation token). - name: publish to npm + stage-npm: + name: stage npm packages for maintainer approval needs: release - runs-on: ubuntu-latest if: ${{ startsWith(github.ref, 'refs/tags/v') && vars.PUBLISH_NPM == 'true' }} - # id-token lets npm generate a provenance attestation (Sigstore): every - # published package is publicly verifiable as built by THIS workflow from - # THIS repo at THIS commit — the counter to "random npm binary" distrust. + uses: ./.github/workflows/npm-stage.yml + with: + release_tag: ${{ github.ref_name }} + secrets: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} permissions: contents: read id-token: write - steps: - - uses: actions/checkout@v5 - - - name: setup node - uses: actions/setup-node@v5 - with: - node-version: 20 - registry-url: https://registry.npmjs.org - - - name: download all artifacts - uses: actions/download-artifact@v5 - with: - path: dist - - - name: verify NPM_TOKEN secret is set - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - if [ -z "$NODE_AUTH_TOKEN" ]; then - echo "::error::NPM_TOKEN secret is not set. Skipping npm publish." - echo "::error::Add the secret via: gh secret set NPM_TOKEN" - exit 1 - fi - echo "NPM_TOKEN is set (length: ${#NODE_AUTH_TOKEN})" - - - name: assemble + publish platform packages - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - set -euo pipefail - VERSION="${GITHUB_REF_NAME#v}" - - # npm-key target-triple os cpu binname ext - PLATFORMS=( - "linux-x64 x86_64-unknown-linux-gnu linux x64 kimetsu tar.gz" - "darwin-x64 x86_64-apple-darwin darwin x64 kimetsu tar.gz" - "darwin-arm64 aarch64-apple-darwin darwin arm64 kimetsu tar.gz" - "win32-x64 x86_64-pc-windows-msvc win32 x64 kimetsu.exe zip" - ) - - mkdir -p stage - for row in "${PLATFORMS[@]}"; do - read -r key target osv cpuv binname ext <<<"$row" - stem="kimetsu-${VERSION}-${target}-lean" - archive="dist/kimetsu-${target}-lean/${stem}.${ext}" - if [ ! -f "$archive" ]; then - echo "::error::missing lean artifact for ${target}: ${archive}" - exit 1 - fi - - tmp="$(mktemp -d)" - if [ "$ext" = "zip" ]; then - # PowerShell `Compress-Archive` writes ZIP entries with backslash - # separators; Info-ZIP `unzip` flattens those into a literal - # filename so the binary can't be found by path. `7z` (preinstalled - # on ubuntu-latest) normalizes backslashes into real directories. - 7z x -y -o"$tmp" "$archive" >/dev/null - else - tar -xf "$archive" -C "$tmp" - fi - - pkgdir="stage/${key}" - mkdir -p "$pkgdir/bin" - # Prefer the expected layout (`/`), then fall back to - # a recursive search so the step is robust to archive quirks. - if [ -f "$tmp/${stem}/${binname}" ]; then - src="$tmp/${stem}/${binname}" - elif [ -f "$tmp/${binname}" ]; then - src="$tmp/${binname}" - else - # Exact match, then suffix match — the latter catches a - # backslash-flattened basename like `\kimetsu.exe` if some - # extractor still mangles the Windows archive. - src="$(find "$tmp" -type f -name "${binname}" | head -n1)" - [ -z "$src" ] && src="$(find "$tmp" -type f -name "*${binname}" | head -n1)" - fi - if [ -z "${src:-}" ] || [ ! -f "$src" ]; then - echo "::error::binary ${binname} not found inside ${archive}" - echo "::error::extracted tree:"; find "$tmp" -maxdepth 3 -print - exit 1 - fi - cp "$src" "$pkgdir/bin/${binname}" - [ "$ext" = "zip" ] || chmod +x "$pkgdir/bin/${binname}" - - node -e ' - const fs = require("fs"); - const [dir, name, version, osv, cpuv] = process.argv.slice(1); - fs.writeFileSync(dir + "/package.json", JSON.stringify({ - name, - version, - description: name + " — prebuilt kimetsu binary", - repository: { type: "git", url: "git+https://github.com/RodCor/kimetsu.git" }, - license: "MIT OR Apache-2.0", - os: [osv], - cpu: [cpuv], - files: ["bin"], - }, null, 2) + "\n"); - ' "$pkgdir" "@kimetsu-ai/${key}" "$VERSION" "$osv" "$cpuv" - - # Idempotent: skip if this exact version is already on npm, so a - # re-run after a partial failure doesn't choke on EPUBLISHCONFLICT. - if npm view "@kimetsu-ai/${key}@${VERSION}" version >/dev/null 2>&1; then - echo "skip: @kimetsu-ai/${key}@${VERSION} already published" - else - echo "publishing @kimetsu-ai/${key}@${VERSION}" - ( cd "$pkgdir" && npm publish --access public --provenance ) - fi - done - - - name: stamp + publish main package - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - set -euo pipefail - VERSION="${GITHUB_REF_NAME#v}" - node -e ' - const fs = require("fs"); - const version = process.argv[1]; - const p = "npm/kimetsu/package.json"; - const j = JSON.parse(fs.readFileSync(p, "utf8")); - j.version = version; - for (const k of Object.keys(j.optionalDependencies || {})) { - j.optionalDependencies[k] = version; - } - fs.writeFileSync(p, JSON.stringify(j, null, 2) + "\n"); - ' "$VERSION" - if npm view "kimetsu-ai@${VERSION}" version >/dev/null 2>&1; then - echo "skip: kimetsu-ai@${VERSION} already published" - else - echo "publishing kimetsu-ai@${VERSION}" - ( cd npm/kimetsu && npm publish --access public --provenance ) - fi - - - name: assemble + publish kimetsu-remote platform packages - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - set -euo pipefail - VERSION="${GITHUB_REF_NAME#v}" - - # The server ships only where an embeddings build exists (ONNX prebuilts). - # npm-key target-triple os cpu binname ext - PLATFORMS=( - "linux-x64 x86_64-unknown-linux-gnu linux x64 kimetsu-remote tar.gz" - "darwin-arm64 aarch64-apple-darwin darwin arm64 kimetsu-remote tar.gz" - "win32-x64 x86_64-pc-windows-msvc win32 x64 kimetsu-remote.exe zip" - ) - - mkdir -p stage-remote - for row in "${PLATFORMS[@]}"; do - read -r key target osv cpuv binname ext <<<"$row" - stem="kimetsu-remote-${VERSION}-${target}" - # The remote archive is uploaded alongside the embeddings kimetsu archive. - archive="dist/kimetsu-${target}-embeddings/${stem}.${ext}" - if [ ! -f "$archive" ]; then - echo "::error::missing kimetsu-remote artifact for ${target}: ${archive}" - exit 1 - fi - - tmp="$(mktemp -d)" - if [ "$ext" = "zip" ]; then - 7z x -y -o"$tmp" "$archive" >/dev/null - else - tar -xf "$archive" -C "$tmp" - fi - - pkgdir="stage-remote/${key}" - mkdir -p "$pkgdir/bin" - if [ -f "$tmp/${stem}/${binname}" ]; then - src="$tmp/${stem}/${binname}" - elif [ -f "$tmp/${binname}" ]; then - src="$tmp/${binname}" - else - src="$(find "$tmp" -type f -name "${binname}" | head -n1)" - [ -z "$src" ] && src="$(find "$tmp" -type f -name "*${binname}" | head -n1)" - fi - if [ -z "${src:-}" ] || [ ! -f "$src" ]; then - echo "::error::binary ${binname} not found inside ${archive}" - find "$tmp" -maxdepth 3 -print - exit 1 - fi - cp "$src" "$pkgdir/bin/${binname}" - [ "$ext" = "zip" ] || chmod +x "$pkgdir/bin/${binname}" - - node -e ' - const fs = require("fs"); - const [dir, name, version, osv, cpuv] = process.argv.slice(1); - fs.writeFileSync(dir + "/package.json", JSON.stringify({ - name, - version, - description: name + " — prebuilt kimetsu-remote server binary", - repository: { type: "git", url: "git+https://github.com/RodCor/kimetsu.git" }, - license: "MIT OR Apache-2.0", - os: [osv], - cpu: [cpuv], - files: ["bin"], - }, null, 2) + "\n"); - ' "$pkgdir" "@kimetsu-ai/remote-${key}" "$VERSION" "$osv" "$cpuv" - - if npm view "@kimetsu-ai/remote-${key}@${VERSION}" version >/dev/null 2>&1; then - echo "skip: @kimetsu-ai/remote-${key}@${VERSION} already published" - else - echo "publishing @kimetsu-ai/remote-${key}@${VERSION}" - ( cd "$pkgdir" && npm publish --access public --provenance ) - fi - done - - - name: stamp + publish kimetsu-remote main package - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - set -euo pipefail - VERSION="${GITHUB_REF_NAME#v}" - node -e ' - const fs = require("fs"); - const version = process.argv[1]; - const p = "npm/kimetsu-remote/package.json"; - const j = JSON.parse(fs.readFileSync(p, "utf8")); - j.version = version; - for (const k of Object.keys(j.optionalDependencies || {})) { - j.optionalDependencies[k] = version; - } - fs.writeFileSync(p, JSON.stringify(j, null, 2) + "\n"); - ' "$VERSION" - if npm view "kimetsu-remote@${VERSION}" version >/dev/null 2>&1; then - echo "skip: kimetsu-remote@${VERSION} already published" - else - echo "publishing kimetsu-remote@${VERSION}" - ( cd npm/kimetsu-remote && npm publish --access public --provenance ) - fi - - - name: build + publish @kimetsu-ai/sdk - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - set -euo pipefail - # Deliberately NOT versioned off the tag. The SDK is a client - # library whose API moves independently of the binary's, and its - # sibling `kimetsu` on PyPI is versioned the same way — pinning it - # to the CLI version would promise a lockstep that does not exist. - # So it publishes at whatever package.json says, and a release that - # did not touch the SDK simply skips. - SDK_VERSION="$(node -p "require('./npm/kimetsu-sdk/package.json').version")" - if npm view "@kimetsu-ai/sdk@${SDK_VERSION}" version >/dev/null 2>&1; then - echo "skip: @kimetsu-ai/sdk@${SDK_VERSION} already published" - else - echo "publishing @kimetsu-ai/sdk@${SDK_VERSION}" - ( cd npm/kimetsu-sdk && npm ci && npm publish --access public --provenance ) - fi - - - name: summary - run: | - VERSION="${GITHUB_REF_NAME#v}" - echo "## npm publish — v${VERSION} ✅" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Published the main package + 4 platform packages:" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "- [\`kimetsu-ai\`](https://www.npmjs.com/package/kimetsu-ai/v/${VERSION})" >> $GITHUB_STEP_SUMMARY - for k in linux-x64 darwin-x64 darwin-arm64 win32-x64; do - echo "- [\`@kimetsu-ai/${k}\`](https://www.npmjs.com/package/@kimetsu-ai/${k}/v/${VERSION})" >> $GITHUB_STEP_SUMMARY - done - echo "" >> $GITHUB_STEP_SUMMARY - echo "Users can now install with:" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "npm install -g kimetsu-ai # lean build" >> $GITHUB_STEP_SUMMARY - echo "KIMETSU_NPM_FLAVOR=embeddings npm install -g kimetsu-ai # semantic build" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY diff --git a/docs/npm-staged-publishing.md b/docs/npm-staged-publishing.md new file mode 100644 index 0000000..c44877a --- /dev/null +++ b/docs/npm-staged-publishing.md @@ -0,0 +1,73 @@ +# npm staged publishing and release recovery + +The release workflow stages npm packages for human approval. A successful staging +job does **not** mean the packages are publicly available. GitHub release assets +and crates.io publication remain separate. + +We use Node 24 and npm 11.21.0. npm documents Node 22.14+ and npm 11.15+ as the +minimum for staged publishing. The existing `NPM_TOKEN` can stay stage-only; +this workflow does not require direct-publish permission, change account or +trusted-publisher settings, or approve/reject stages. `PUBLISH_NPM=true` remains +the repository gate. Lifecycle scripts are disabled while packing/staging. The +SDK is built separately without the npm credential and retains its own version. + +## Recover an existing release + +After this workflow is merged and its checks pass, dispatch **stage npm release** +from `main`, setting `release_tag` to the existing stable tag (for example +`v2.9.0`). Equivalently: + +```sh +gh workflow run npm-stage.yml --ref main -f release_tag=v2.9.0 +``` + +This uses package sources from the tag and downloads the existing release assets. +It checks that the tag's commit belongs to main, that Cargo's version agrees, +and that the release is public and stable. It verifies the checksum manifest's +GitHub attestation against that exact tag/commit and the original release workflow, +then verifies the asset checksums. It never rebuilds binaries, changes tags or +release assets, or reruns crates.io publication. npm provenance describes the +staging workflow run; the verified original asset attestation binds binaries to +the release commit. + +The job summary reports each package as already public, newly staged, or already +staged, including stage IDs where applicable. Public versions are skipped. +An existing stage is reused only if its package, version, tarball SHA-1 and +`latest` tag match. Conflicts and registry/authentication errors stop the job; +they are not treated as permission to resubmit, reject, or bypass approval. +If a submission's result is uncertain, inspect the stage list before retrying. +Partial stages remain private until approved. + +## Maintainer approval + +Review the stage IDs in the job summary in npm's **Staged Packages** interface, +or inspect them using npm 11.21.0: + +```sh +npm stage list @kimetsu-ai/linux-x64 --json +npm stage view +npm stage download +npm stage approve +``` + +Approval is an authenticated maintainer action with interactive 2FA. Do not put +an OTP or credentials in a workflow, issue, or chat. The workflow never invokes +`approve`. Review downloaded tarballs, versions, metadata, provenance and binaries +before approval; a matching SHA-1 only establishes retry identity. + +Approve the four CLI platform packages and three remote platform packages first. +Verify each exact version is public using `npm view @ version` +before approving the `kimetsu-ai` and `kimetsu-remote` wrappers that depend on them. +The SDK is independent. Approval is per package, not an atomic multi-package +transaction; a failed or partial approval must be resolved before approving a +dependent wrapper. Finally verify all public versions and test installation. + +The helper tests simulate registry and npm responses. They do not demonstrate +real-service staging, provenance acceptance, or human approval; those require a +successful authorized workflow run and maintainer approval. + +Official references: +- [Staged publishing](https://docs.npmjs.com/staged-publishing/) +- [npm stage commands](https://docs.npmjs.com/cli/v11/commands/npm-stage/) +- [Trusted publishing](https://docs.npmjs.com/trusted-publishers/) (a separate setup, + not enabled by this change) diff --git a/scripts/assemble-npm.sh b/scripts/assemble-npm.sh new file mode 100644 index 0000000..9571ddd --- /dev/null +++ b/scripts/assemble-npm.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# Run from the verified release-source checkout, with flat release assets in dist/. +set -euo pipefail +VERSION="${1:?stable release version required}" +[[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || exit 1 + +# npm-key target-triple os cpu binname ext +PLATFORMS=( + "linux-x64 x86_64-unknown-linux-gnu linux x64 kimetsu tar.gz" + "darwin-x64 x86_64-apple-darwin darwin x64 kimetsu tar.gz" + "darwin-arm64 aarch64-apple-darwin darwin arm64 kimetsu tar.gz" + "win32-x64 x86_64-pc-windows-msvc win32 x64 kimetsu.exe zip" +) + +mkdir -p stage +for row in "${PLATFORMS[@]}"; do + read -r key target osv cpuv binname ext <<<"$row" + stem="kimetsu-${VERSION}-${target}-lean" + archive="dist/${stem}.${ext}" + if [ ! -f "$archive" ]; then + echo "::error::missing lean artifact for ${target}: ${archive}" + exit 1 + fi + + tmp="$(mktemp -d)" + if [ "$ext" = "zip" ]; then + # PowerShell `Compress-Archive` writes ZIP entries with backslash + # separators; Info-ZIP `unzip` flattens those into a literal + # filename so the binary can't be found by path. `7z` (preinstalled + # on ubuntu-latest) normalizes backslashes into real directories. + 7z x -y -o"$tmp" "$archive" >/dev/null + else + tar -xf "$archive" -C "$tmp" + fi + + pkgdir="stage/${key}" + mkdir -p "$pkgdir/bin" + # Prefer the expected layout (`/`), then fall back to + # a recursive search so the step is robust to archive quirks. + if [ -f "$tmp/${stem}/${binname}" ]; then + src="$tmp/${stem}/${binname}" + elif [ -f "$tmp/${binname}" ]; then + src="$tmp/${binname}" + else + # Exact match, then suffix match — the latter catches a + # backslash-flattened basename like `\kimetsu.exe` if some + # extractor still mangles the Windows archive. + src="$(find "$tmp" -type f -name "${binname}" | head -n1)" + [ -z "$src" ] && src="$(find "$tmp" -type f -name "*${binname}" | head -n1)" + fi + if [ -z "${src:-}" ] || [ ! -f "$src" ]; then + echo "::error::binary ${binname} not found inside ${archive}" + echo "::error::extracted tree:"; find "$tmp" -maxdepth 3 -print + exit 1 + fi + cp "$src" "$pkgdir/bin/${binname}" + [ "$ext" = "zip" ] || chmod +x "$pkgdir/bin/${binname}" + + node -e ' + const fs = require("fs"); + const [dir, name, version, osv, cpuv] = process.argv.slice(1); + fs.writeFileSync(dir + "/package.json", JSON.stringify({ + name, + version, + description: name + " — prebuilt kimetsu binary", + repository: { type: "git", url: "git+https://github.com/RodCor/kimetsu.git" }, + license: "MIT OR Apache-2.0", + os: [osv], + cpu: [cpuv], + files: ["bin"], + }, null, 2) + "\n"); + ' "$pkgdir" "@kimetsu-ai/${key}" "$VERSION" "$osv" "$cpuv" + +done + +node -e ' + const fs = require("fs"); + const version = process.argv[1]; + const p = "npm/kimetsu/package.json"; + const j = JSON.parse(fs.readFileSync(p, "utf8")); + j.version = version; + for (const k of Object.keys(j.optionalDependencies || {})) { + j.optionalDependencies[k] = version; + } + fs.writeFileSync(p, JSON.stringify(j, null, 2) + "\n"); +' "$VERSION" + +# The server ships only where an embeddings build exists (ONNX prebuilts). +# npm-key target-triple os cpu binname ext +PLATFORMS=( + "linux-x64 x86_64-unknown-linux-gnu linux x64 kimetsu-remote tar.gz" + "darwin-arm64 aarch64-apple-darwin darwin arm64 kimetsu-remote tar.gz" + "win32-x64 x86_64-pc-windows-msvc win32 x64 kimetsu-remote.exe zip" +) + +mkdir -p stage-remote +for row in "${PLATFORMS[@]}"; do + read -r key target osv cpuv binname ext <<<"$row" + stem="kimetsu-remote-${VERSION}-${target}" + # The remote archive is uploaded alongside the embeddings kimetsu archive. + archive="dist/${stem}.${ext}" + if [ ! -f "$archive" ]; then + echo "::error::missing kimetsu-remote artifact for ${target}: ${archive}" + exit 1 + fi + + tmp="$(mktemp -d)" + if [ "$ext" = "zip" ]; then + 7z x -y -o"$tmp" "$archive" >/dev/null + else + tar -xf "$archive" -C "$tmp" + fi + + pkgdir="stage-remote/${key}" + mkdir -p "$pkgdir/bin" + if [ -f "$tmp/${stem}/${binname}" ]; then + src="$tmp/${stem}/${binname}" + elif [ -f "$tmp/${binname}" ]; then + src="$tmp/${binname}" + else + src="$(find "$tmp" -type f -name "${binname}" | head -n1)" + [ -z "$src" ] && src="$(find "$tmp" -type f -name "*${binname}" | head -n1)" + fi + if [ -z "${src:-}" ] || [ ! -f "$src" ]; then + echo "::error::binary ${binname} not found inside ${archive}" + find "$tmp" -maxdepth 3 -print + exit 1 + fi + cp "$src" "$pkgdir/bin/${binname}" + [ "$ext" = "zip" ] || chmod +x "$pkgdir/bin/${binname}" + + node -e ' + const fs = require("fs"); + const [dir, name, version, osv, cpuv] = process.argv.slice(1); + fs.writeFileSync(dir + "/package.json", JSON.stringify({ + name, + version, + description: name + " — prebuilt kimetsu-remote server binary", + repository: { type: "git", url: "git+https://github.com/RodCor/kimetsu.git" }, + license: "MIT OR Apache-2.0", + os: [osv], + cpu: [cpuv], + files: ["bin"], + }, null, 2) + "\n"); + ' "$pkgdir" "@kimetsu-ai/remote-${key}" "$VERSION" "$osv" "$cpuv" + +done + +node -e ' + const fs = require("fs"); + const version = process.argv[1]; + const p = "npm/kimetsu-remote/package.json"; + const j = JSON.parse(fs.readFileSync(p, "utf8")); + j.version = version; + for (const k of Object.keys(j.optionalDependencies || {})) { + j.optionalDependencies[k] = version; + } + fs.writeFileSync(p, JSON.stringify(j, null, 2) + "\n"); +' "$VERSION" + diff --git a/scripts/stage-package.mjs b/scripts/stage-package.mjs new file mode 100644 index 0000000..49c96f8 --- /dev/null +++ b/scripts/stage-package.mjs @@ -0,0 +1,72 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { pathToFileURL } from 'node:url'; + +const registry = 'https://registry.npmjs.org'; + +function liveIO(directory) { + return { + manifest: () => JSON.parse(fs.readFileSync(path.join(directory, 'package.json'), 'utf8')), + published: async (name, version) => { + const response = await fetch(`${registry}/${encodeURIComponent(name)}/${version}`, { + signal: AbortSignal.timeout(30000), redirect: 'error', + }); + if (response.status === 200) { + const body = await response.json(); + if (body.name !== name || body.version !== version) throw new Error('Unexpected registry version'); + } + return { status: response.status }; + }, + run: (args) => { + const result = spawnSync('npm', [...args, `--registry=${registry}`], { + cwd: directory, encoding: 'utf8', stdio: ['ignore', 'pipe', 'inherit'], + }); + if (result.error || result.status !== 0) throw new Error(`npm ${args.slice(0, 2).join(' ')} failed; no fallback attempted`); + return JSON.parse(result.stdout); + }, + }; +} + +export async function stagePackage(directory, io = liveIO(directory)) { + const { name, version } = io.manifest(); + if (!/^(?:@kimetsu-ai\/[a-z0-9-]+|kimetsu-ai|kimetsu-remote)$/.test(name) + || !/^\d+\.\d+\.\d+$/.test(version)) throw new Error('Unexpected package name/version'); + const published = await io.published(name, version); + if (published.status === 200) return { name, version, state: 'published' }; + if (published.status !== 404) throw new Error(`Registry lookup failed (${published.status}); no staging attempted`); + + const packed = io.run(['pack', '--json', '--ignore-scripts']); + if (!Array.isArray(packed) || packed.length !== 1 || !/^[a-f0-9]{40}$/.test(packed[0].shasum) + || typeof packed[0].filename !== 'string' + || path.basename(packed[0].filename) !== packed[0].filename) throw new Error('Invalid npm pack result'); + // npm 11.21 stage list accepts package NAME, not name@version. + const stages = io.run(['stage', 'list', name, '--json']); + if (!Array.isArray(stages)) throw new Error('Invalid stage list; no staging attempted'); + const matches = stages.filter(item => item.packageName === name && item.version === version); + if (matches.length) { + if (matches.length !== 1 || matches[0].shasum !== packed[0].shasum || matches[0].tag !== 'latest') { + throw new Error('Existing stage differs; maintainer must review it before any retry'); + } + if (typeof matches[0].id !== 'string' || !matches[0].id) throw new Error('Missing stage ID'); + return { name, version, state: 'already-staged', stageId: matches[0].id, shasum: packed[0].shasum }; + } + const result = io.run(['stage', 'publish', path.resolve(directory, packed[0].filename), + '--tag', 'latest', '--access', 'public', '--provenance', '--ignore-scripts', '--json']); + if (typeof result.stageId !== 'string' || !result.stageId) throw new Error('Missing stage ID; inspect npm before retrying'); + return { name, version, state: 'staged', stageId: result.stageId, shasum: packed[0].shasum }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + const result = await stagePackage(path.resolve(process.argv[2] ?? '.')); + console.log(JSON.stringify(result)); + if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, + `- ${result.name}@${result.version}: ${result.state}${result.stageId ? `; stage ID: ${result.stageId}; tarball SHA-1: ${result.shasum}` : ''}\n`); + } + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } +} diff --git a/scripts/stage-package.test.mjs b/scripts/stage-package.test.mjs new file mode 100644 index 0000000..fb7bdac --- /dev/null +++ b/scripts/stage-package.test.mjs @@ -0,0 +1,85 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { stagePackage } from './stage-package.mjs'; + +const shasum = 'a'.repeat(40); + +function fixture({ status = 404, stages = [], response = { stageId: 'new-stage' } } = {}) { + const calls = []; + return { + calls, + io: { + manifest: () => ({ name: '@kimetsu-ai/linux-x64', version: '2.9.0' }), + published: async () => ({ status }), + run: (args) => { + calls.push(args); + if (args[0] === 'pack') return [{ filename: 'package.tgz', shasum }]; + if (args[1] === 'list') return stages; + if (args[1] === 'publish') return response; + throw new Error('Unexpected command'); + }, + }, + }; +} + +test('public version skips without packing or authenticating', async () => { + const f = fixture({ status: 200 }); + assert.equal((await stagePackage('/package', f.io)).state, 'published'); + assert.deepEqual(f.calls, []); +}); +test('registry errors cannot be mistaken for an unpublished version', async () => { + for (const status of [401, 403, 429, 500]) { + const f = fixture({ status }); + await assert.rejects(stagePackage('/package', f.io), /Registry lookup failed/); + assert.deepEqual(f.calls, []); + } +}); +test('new version stages exact packed tarball and never approves', async () => { + const f = fixture(); + const result = await stagePackage('/package', f.io); + assert.equal(result.stageId, 'new-stage'); + assert.equal(result.state, 'staged'); + assert.deepEqual(f.calls[1].slice(0, 3), ['stage', 'list', '@kimetsu-ai/linux-x64']); + assert.deepEqual(f.calls[2].slice(0, 2), ['stage', 'publish']); + assert.ok(f.calls[2].includes('--ignore-scripts')); + assert.ok(!f.calls.some(args => args.includes('approve'))); +}); +test('identical existing stage is reused without resubmission', async () => { + const f = fixture({ stages: [{ packageName: '@kimetsu-ai/linux-x64', version: '2.9.0', id: 'existing', tag: 'latest', shasum }] }); + assert.equal((await stagePackage('/package', f.io)).stageId, 'existing'); + assert.equal(f.calls.length, 2); +}); +test('conflicting staged bytes or tag fail without submission', async () => { + for (const changes of [{ shasum: 'different' }, { tag: 'next' }]) { + const f = fixture({ stages: [{ packageName: '@kimetsu-ai/linux-x64', version: '2.9.0', id: 'existing', tag: 'latest', shasum, ...changes }] }); + await assert.rejects(stagePackage('/package', f.io), /Existing stage differs/); + assert.equal(f.calls.length, 2); + } +}); +test('malformed or failed stage listing never triggers submission', async () => { + const f = fixture({ stages: {} }); + await assert.rejects(stagePackage('/package', f.io), /Invalid stage list/); + assert.equal(f.calls.length, 2); +}); +test('a successful response without a stage ID is not called staged', async () => { + const f = fixture({ response: {} }); + await assert.rejects(stagePackage('/package', f.io), /Missing stage ID/); +}); + +test('stage-list failure is propagated without submission', async () => { + const f = fixture(); + const run = f.io.run; + f.io.run = args => { + if (args[1] === 'list') throw new Error('Authentication failed'); + return run(args); + }; + await assert.rejects(stagePackage('/package', f.io), /Authentication failed/); + assert.equal(f.calls.length, 1); +}); + +test('ambiguous duplicate stages stop without submission', async () => { + const entry = { packageName: '@kimetsu-ai/linux-x64', version: '2.9.0', id: 'existing', tag: 'latest', shasum }; + const f = fixture({ stages: [entry, entry] }); + await assert.rejects(stagePackage('/package', f.io), /Existing stage differs/); + assert.equal(f.calls.length, 2); +}); diff --git a/scripts/test-assemble-npm.py b/scripts/test-assemble-npm.py new file mode 100644 index 0000000..cda84e9 --- /dev/null +++ b/scripts/test-assemble-npm.py @@ -0,0 +1,73 @@ +"""Exercise production assembly with synthetic archives; never contact npm.""" +import io +import json +import os +from pathlib import Path +import shutil +import subprocess +import tarfile +import tempfile +import unittest +import zipfile + +ROOT = Path(__file__).resolve().parents[1] +VERSION = '2.9.0' +PLATFORMS = [ + ('linux-x64', 'x86_64-unknown-linux-gnu', 'linux', 'x64', False), + ('darwin-x64', 'x86_64-apple-darwin', 'darwin', 'x64', False), + ('darwin-arm64', 'aarch64-apple-darwin', 'darwin', 'arm64', False), + ('win32-x64', 'x86_64-pc-windows-msvc', 'win32', 'x64', True), +] + + +class AssemblyTest(unittest.TestCase): + def test_all_platforms_wrappers_and_missing_asset(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / 'dist').mkdir() + for wrapper in ['kimetsu', 'kimetsu-remote']: + shutil.copytree(ROOT / 'npm' / wrapper, root / 'npm' / wrapper) + expected = [] + for remote in [False, True]: + for key, target, system, cpu, windows in PLATFORMS: + if remote and key == 'darwin-x64': + continue + binary = ('kimetsu-remote' if remote else 'kimetsu') + ('.exe' if windows else '') + stem = f'kimetsu-remote-{VERSION}-{target}' if remote else f'kimetsu-{VERSION}-{target}-lean' + payload = f'fixture:{stem}'.encode() + if windows: + # Match the backslash entries from PowerShell Compress-Archive. + with zipfile.ZipFile(root / 'dist' / f'{stem}.zip', 'w') as archive: + archive.writestr(f'{stem}\\{binary}', payload) + else: + with tarfile.open(root / 'dist' / f'{stem}.tar.gz', 'w:gz') as archive: + info = tarfile.TarInfo(f'{stem}/{binary}') + info.size = len(payload) + archive.addfile(info, io.BytesIO(payload)) + expected.append((remote, key, system, cpu, binary, payload)) + command = [os.environ.get('KIMETSU_TEST_BASH', 'bash'), + (ROOT / 'scripts/assemble-npm.sh').as_posix(), VERSION] + subprocess.run(command, cwd=root, check=True) + for remote, key, system, cpu, binary, payload in expected: + directory = root / ('stage-remote' if remote else 'stage') / key + manifest = json.loads((directory / 'package.json').read_text()) + self.assertEqual(manifest['name'], '@kimetsu-ai/' + ('remote-' if remote else '') + key) + self.assertEqual(manifest['version'], VERSION) + self.assertEqual(manifest['os'], [system]) + self.assertEqual(manifest['cpu'], [cpu]) + self.assertEqual((directory / 'bin' / binary).read_bytes(), payload) + if os.name != 'nt' and system != 'win32': + self.assertTrue((directory / 'bin' / binary).stat().st_mode & 0o111) + for wrapper, count in [('kimetsu', 4), ('kimetsu-remote', 3)]: + manifest = json.loads((root / 'npm' / wrapper / 'package.json').read_text()) + self.assertEqual(manifest['version'], VERSION) + self.assertEqual(len(manifest['optionalDependencies']), count) + self.assertEqual(set(manifest['optionalDependencies'].values()), {VERSION}) + (root / 'dist' / f'kimetsu-{VERSION}-x86_64-unknown-linux-gnu-lean.tar.gz').unlink() + result = subprocess.run(command, cwd=root, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn('missing lean artifact', result.stdout) + + +if __name__ == '__main__': + unittest.main() From bfefd61cdba6a700bfcd7b6ab72d8bcd10dce2ba Mon Sep 17 00:00:00 2001 From: RodCor Date: Tue, 13 Oct 2026 14:24:49 -0300 Subject: [PATCH 2/2] fix(release): verify every archive and parse npm stage result --- .github/workflows/npm-stage.yml | 7 +++++++ docs/npm-staged-publishing.md | 4 +++- scripts/assemble-npm.sh | 1 - scripts/stage-package.mjs | 9 +++++++-- scripts/stage-package.test.mjs | 12 +++++++++++- 5 files changed, 28 insertions(+), 5 deletions(-) diff --git a/.github/workflows/npm-stage.yml b/.github/workflows/npm-stage.yml index 4be477a..3b1808d 100644 --- a/.github/workflows/npm-stage.yml +++ b/.github/workflows/npm-stage.yml @@ -64,6 +64,13 @@ jobs: --signer-workflow "$GITHUB_REPOSITORY/.github/workflows/release.yml" \ --source-ref "refs/tags/$RELEASE_TAG" --source-digest "$(git rev-parse HEAD)" (cd dist && sha256sum --check --strict checksums.txt) + # A signed manifest only authenticates files it actually lists. Also + # authenticate every archive, including any asset absent from that list. + for archive in dist/*.tar.gz dist/*.zip; do + gh attestation verify "$archive" --repo "$GITHUB_REPOSITORY" \ + --signer-workflow "$GITHUB_REPOSITORY/.github/workflows/release.yml" \ + --source-ref "refs/tags/$RELEASE_TAG" --source-digest "$(git rev-parse HEAD)" + done - uses: actions/setup-node@v5 with: diff --git a/docs/npm-staged-publishing.md b/docs/npm-staged-publishing.md index c44877a..8dafedf 100644 --- a/docs/npm-staged-publishing.md +++ b/docs/npm-staged-publishing.md @@ -25,7 +25,9 @@ This uses package sources from the tag and downloads the existing release assets It checks that the tag's commit belongs to main, that Cargo's version agrees, and that the release is public and stable. It verifies the checksum manifest's GitHub attestation against that exact tag/commit and the original release workflow, -then verifies the asset checksums. It never rebuilds binaries, changes tags or +then verifies the asset checksums and each archive's own attestation. This also +rejects an unauthenticated archive that was omitted from the checksum list. +It never rebuilds binaries, changes tags or release assets, or reruns crates.io publication. npm provenance describes the staging workflow run; the verified original asset attestation binds binaries to the release commit. diff --git a/scripts/assemble-npm.sh b/scripts/assemble-npm.sh index 9571ddd..1907bcc 100644 --- a/scripts/assemble-npm.sh +++ b/scripts/assemble-npm.sh @@ -157,4 +157,3 @@ node -e ' } fs.writeFileSync(p, JSON.stringify(j, null, 2) + "\n"); ' "$VERSION" - diff --git a/scripts/stage-package.mjs b/scripts/stage-package.mjs index 49c96f8..beb0cae 100644 --- a/scripts/stage-package.mjs +++ b/scripts/stage-package.mjs @@ -53,8 +53,13 @@ export async function stagePackage(directory, io = liveIO(directory)) { } const result = io.run(['stage', 'publish', path.resolve(directory, packed[0].filename), '--tag', 'latest', '--access', 'public', '--provenance', '--ignore-scripts', '--json']); - if (typeof result.stageId !== 'string' || !result.stageId) throw new Error('Missing stage ID; inspect npm before retrying'); - return { name, version, state: 'staged', stageId: result.stageId, shasum: packed[0].shasum }; + // npm 11.21 publish JSON is keyed by package name (unlike stage list). + const staged = result?.[name]; + if (typeof staged?.stageId !== 'string' || !staged.stageId) throw new Error('Missing stage ID; inspect npm before retrying'); + if (staged.name !== name || staged.version !== version || staged.shasum !== packed[0].shasum) { + throw new Error('Unexpected staged package metadata; inspect npm before retrying'); + } + return { name, version, state: 'staged', stageId: staged.stageId, shasum: packed[0].shasum }; } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { diff --git a/scripts/stage-package.test.mjs b/scripts/stage-package.test.mjs index fb7bdac..4d8a111 100644 --- a/scripts/stage-package.test.mjs +++ b/scripts/stage-package.test.mjs @@ -4,7 +4,10 @@ import { stagePackage } from './stage-package.mjs'; const shasum = 'a'.repeat(40); -function fixture({ status = 404, stages = [], response = { stageId: 'new-stage' } } = {}) { +const packageName = '@kimetsu-ai/linux-x64'; +const stagedResponse = { [packageName]: { name: packageName, version: '2.9.0', shasum, stageId: 'new-stage' } }; + +function fixture({ status = 404, stages = [], response = stagedResponse } = {}) { const calls = []; return { calls, @@ -66,6 +69,13 @@ test('a successful response without a stage ID is not called staged', async () = await assert.rejects(stagePackage('/package', f.io), /Missing stage ID/); }); +test('incorrect staged package identity fails instead of claiming success', async () => { + for (const changes of [{ name: 'kimetsu-ai' }, { version: '2.8.0' }, { shasum: 'b'.repeat(40) }]) { + const f = fixture({ response: { [packageName]: { ...stagedResponse[packageName], ...changes } } }); + await assert.rejects(stagePackage('/package', f.io), /Unexpected staged package metadata/); + } +}); + test('stage-list failure is propagated without submission', async () => { const f = fixture(); const run = f.io.run;