diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e83e8e..4c72575 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,20 @@ env: RUST_BACKTRACE: 1 jobs: + npm-staging: + name: npm staging regression tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + - uses: actions/setup-node@v5 + with: + node-version: 24 + package-manager-cache: false + - run: node --test scripts/stage-package.test.mjs + - run: python3 scripts/test-assemble-npm.py + decision-examples: name: decision provider examples runs-on: ubuntu-latest diff --git a/.github/workflows/npm-stage.yml b/.github/workflows/npm-stage.yml new file mode 100644 index 0000000..3b1808d --- /dev/null +++ b/.github/workflows/npm-stage.yml @@ -0,0 +1,111 @@ +name: stage npm release + +on: + workflow_call: + inputs: + release_tag: + required: true + type: string + secrets: + NPM_TOKEN: + required: true + workflow_dispatch: + inputs: + release_tag: + description: Existing stable GitHub release tag (for example v2.9.0) + required: true + type: string + +permissions: + contents: read + id-token: write + +concurrency: + group: npm-stage-${{ inputs.release_tag }} + cancel-in-progress: false + +jobs: + stage: + if: ${{ vars.PUBLISH_NPM == 'true' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/main') }} + runs-on: ubuntu-latest + env: + RELEASE_TAG: ${{ inputs.release_tag }} + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: validate stable release tag + run: '[[ "$RELEASE_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]' + + - uses: actions/checkout@v5 + with: + ref: ${{ inputs.release_tag }} + path: release-source + fetch-depth: 0 + persist-credentials: false + + - name: verify release source and download existing assets + working-directory: release-source + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + git merge-base --is-ancestor HEAD origin/main + python3 - <<'PY' + import os, tomllib + with open('Cargo.toml', 'rb') as f: + assert tomllib.load(f)['workspace']['package']['version'] == os.environ['RELEASE_TAG'][1:] + PY + gh release view "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" --json isDraft,isPrerelease > /tmp/npm-release.json + jq -e '.isDraft == false and .isPrerelease == false' /tmp/npm-release.json + gh release download "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" --dir dist + gh attestation verify dist/checksums.txt --repo "$GITHUB_REPOSITORY" \ + --signer-workflow "$GITHUB_REPOSITORY/.github/workflows/release.yml" \ + --source-ref "refs/tags/$RELEASE_TAG" --source-digest "$(git rev-parse HEAD)" + (cd dist && sha256sum --check --strict checksums.txt) + # A signed manifest only authenticates files it actually lists. Also + # authenticate every archive, including any asset absent from that list. + for archive in dist/*.tar.gz dist/*.zip; do + gh attestation verify "$archive" --repo "$GITHUB_REPOSITORY" \ + --signer-workflow "$GITHUB_REPOSITORY/.github/workflows/release.yml" \ + --source-ref "refs/tags/$RELEASE_TAG" --source-digest "$(git rev-parse HEAD)" + done + + - uses: actions/setup-node@v5 + with: + node-version: 24 + registry-url: https://registry.npmjs.org + package-manager-cache: false + + - name: install staging-capable npm + run: | + npm install --global npm@11.21.0 --ignore-scripts + test "$(npm --version)" = 11.21.0 + npm stage --help + + - name: assemble packages without registry credentials + working-directory: release-source + run: bash "$GITHUB_WORKSPACE/scripts/assemble-npm.sh" "${RELEASE_TAG#v}" + + - name: build independently versioned SDK without registry credentials + working-directory: release-source/npm/kimetsu-sdk + run: | + npm ci --ignore-scripts + npm run build + + - name: stage packages (never approve) + working-directory: release-source + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + set -euo pipefail + if [ -z "$NODE_AUTH_TOKEN" ]; then + echo '::error::NPM_TOKEN is not configured' + exit 1 + fi + echo '## npm stages awaiting maintainer approval' >> "$GITHUB_STEP_SUMMARY" + for directory in stage/* stage-remote/* npm/kimetsu npm/kimetsu-remote npm/kimetsu-sdk; do + node "$GITHUB_WORKSPACE/scripts/stage-package.mjs" "$directory" + done + echo 'Staging does not make a version public. Review and approve platform packages before their wrappers; see docs/npm-staged-publishing.md.' >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e1cd690..c15c867 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -343,296 +343,15 @@ jobs: echo "kimetsu uninstall --dry-run" >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY - publish-npm: - # Publish the `kimetsu` npm package after the binary matrix + GH Release - # succeed. esbuild/turbo style: one per-platform package per target (lean - # binary inside, os/cpu set) plus the main `kimetsu` package that lists - # them as optionalDependencies. npm installs only the matching platform - # package; the bin/cli.js launcher execs its binary. No postinstall. - # - # Embeddings is NOT shipped as a package — it's fetched on demand by the - # launcher when KIMETSU_NPM_FLAVOR=embeddings is set (see npm/kimetsu). - # - # Gated, like publish-crates, on: - # (a) tag push (handled by `if`) - # (b) the repo variable PUBLISH_NPM == 'true' - # (c) the NPM_TOKEN secret being set - # To enable: `gh variable set PUBLISH_NPM --body true` and - # `gh secret set NPM_TOKEN` (an npm automation token). - name: publish to npm + stage-npm: + name: stage npm packages for maintainer approval needs: release - runs-on: ubuntu-latest if: ${{ startsWith(github.ref, 'refs/tags/v') && vars.PUBLISH_NPM == 'true' }} - # id-token lets npm generate a provenance attestation (Sigstore): every - # published package is publicly verifiable as built by THIS workflow from - # THIS repo at THIS commit — the counter to "random npm binary" distrust. + uses: ./.github/workflows/npm-stage.yml + with: + release_tag: ${{ github.ref_name }} + secrets: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} permissions: contents: read id-token: write - steps: - - uses: actions/checkout@v5 - - - name: setup node - uses: actions/setup-node@v5 - with: - node-version: 20 - registry-url: https://registry.npmjs.org - - - name: download all artifacts - uses: actions/download-artifact@v5 - with: - path: dist - - - name: verify NPM_TOKEN secret is set - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - if [ -z "$NODE_AUTH_TOKEN" ]; then - echo "::error::NPM_TOKEN secret is not set. Skipping npm publish." - echo "::error::Add the secret via: gh secret set NPM_TOKEN" - exit 1 - fi - echo "NPM_TOKEN is set (length: ${#NODE_AUTH_TOKEN})" - - - name: assemble + publish platform packages - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - set -euo pipefail - VERSION="${GITHUB_REF_NAME#v}" - - # npm-key target-triple os cpu binname ext - PLATFORMS=( - "linux-x64 x86_64-unknown-linux-gnu linux x64 kimetsu tar.gz" - "darwin-x64 x86_64-apple-darwin darwin x64 kimetsu tar.gz" - "darwin-arm64 aarch64-apple-darwin darwin arm64 kimetsu tar.gz" - "win32-x64 x86_64-pc-windows-msvc win32 x64 kimetsu.exe zip" - ) - - mkdir -p stage - for row in "${PLATFORMS[@]}"; do - read -r key target osv cpuv binname ext <<<"$row" - stem="kimetsu-${VERSION}-${target}-lean" - archive="dist/kimetsu-${target}-lean/${stem}.${ext}" - if [ ! -f "$archive" ]; then - echo "::error::missing lean artifact for ${target}: ${archive}" - exit 1 - fi - - tmp="$(mktemp -d)" - if [ "$ext" = "zip" ]; then - # PowerShell `Compress-Archive` writes ZIP entries with backslash - # separators; Info-ZIP `unzip` flattens those into a literal - # filename so the binary can't be found by path. `7z` (preinstalled - # on ubuntu-latest) normalizes backslashes into real directories. - 7z x -y -o"$tmp" "$archive" >/dev/null - else - tar -xf "$archive" -C "$tmp" - fi - - pkgdir="stage/${key}" - mkdir -p "$pkgdir/bin" - # Prefer the expected layout (`/`), then fall back to - # a recursive search so the step is robust to archive quirks. - if [ -f "$tmp/${stem}/${binname}" ]; then - src="$tmp/${stem}/${binname}" - elif [ -f "$tmp/${binname}" ]; then - src="$tmp/${binname}" - else - # Exact match, then suffix match — the latter catches a - # backslash-flattened basename like `\kimetsu.exe` if some - # extractor still mangles the Windows archive. - src="$(find "$tmp" -type f -name "${binname}" | head -n1)" - [ -z "$src" ] && src="$(find "$tmp" -type f -name "*${binname}" | head -n1)" - fi - if [ -z "${src:-}" ] || [ ! -f "$src" ]; then - echo "::error::binary ${binname} not found inside ${archive}" - echo "::error::extracted tree:"; find "$tmp" -maxdepth 3 -print - exit 1 - fi - cp "$src" "$pkgdir/bin/${binname}" - [ "$ext" = "zip" ] || chmod +x "$pkgdir/bin/${binname}" - - node -e ' - const fs = require("fs"); - const [dir, name, version, osv, cpuv] = process.argv.slice(1); - fs.writeFileSync(dir + "/package.json", JSON.stringify({ - name, - version, - description: name + " — prebuilt kimetsu binary", - repository: { type: "git", url: "git+https://github.com/RodCor/kimetsu.git" }, - license: "MIT OR Apache-2.0", - os: [osv], - cpu: [cpuv], - files: ["bin"], - }, null, 2) + "\n"); - ' "$pkgdir" "@kimetsu-ai/${key}" "$VERSION" "$osv" "$cpuv" - - # Idempotent: skip if this exact version is already on npm, so a - # re-run after a partial failure doesn't choke on EPUBLISHCONFLICT. - if npm view "@kimetsu-ai/${key}@${VERSION}" version >/dev/null 2>&1; then - echo "skip: @kimetsu-ai/${key}@${VERSION} already published" - else - echo "publishing @kimetsu-ai/${key}@${VERSION}" - ( cd "$pkgdir" && npm publish --access public --provenance ) - fi - done - - - name: stamp + publish main package - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - set -euo pipefail - VERSION="${GITHUB_REF_NAME#v}" - node -e ' - const fs = require("fs"); - const version = process.argv[1]; - const p = "npm/kimetsu/package.json"; - const j = JSON.parse(fs.readFileSync(p, "utf8")); - j.version = version; - for (const k of Object.keys(j.optionalDependencies || {})) { - j.optionalDependencies[k] = version; - } - fs.writeFileSync(p, JSON.stringify(j, null, 2) + "\n"); - ' "$VERSION" - if npm view "kimetsu-ai@${VERSION}" version >/dev/null 2>&1; then - echo "skip: kimetsu-ai@${VERSION} already published" - else - echo "publishing kimetsu-ai@${VERSION}" - ( cd npm/kimetsu && npm publish --access public --provenance ) - fi - - - name: assemble + publish kimetsu-remote platform packages - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - set -euo pipefail - VERSION="${GITHUB_REF_NAME#v}" - - # The server ships only where an embeddings build exists (ONNX prebuilts). - # npm-key target-triple os cpu binname ext - PLATFORMS=( - "linux-x64 x86_64-unknown-linux-gnu linux x64 kimetsu-remote tar.gz" - "darwin-arm64 aarch64-apple-darwin darwin arm64 kimetsu-remote tar.gz" - "win32-x64 x86_64-pc-windows-msvc win32 x64 kimetsu-remote.exe zip" - ) - - mkdir -p stage-remote - for row in "${PLATFORMS[@]}"; do - read -r key target osv cpuv binname ext <<<"$row" - stem="kimetsu-remote-${VERSION}-${target}" - # The remote archive is uploaded alongside the embeddings kimetsu archive. - archive="dist/kimetsu-${target}-embeddings/${stem}.${ext}" - if [ ! -f "$archive" ]; then - echo "::error::missing kimetsu-remote artifact for ${target}: ${archive}" - exit 1 - fi - - tmp="$(mktemp -d)" - if [ "$ext" = "zip" ]; then - 7z x -y -o"$tmp" "$archive" >/dev/null - else - tar -xf "$archive" -C "$tmp" - fi - - pkgdir="stage-remote/${key}" - mkdir -p "$pkgdir/bin" - if [ -f "$tmp/${stem}/${binname}" ]; then - src="$tmp/${stem}/${binname}" - elif [ -f "$tmp/${binname}" ]; then - src="$tmp/${binname}" - else - src="$(find "$tmp" -type f -name "${binname}" | head -n1)" - [ -z "$src" ] && src="$(find "$tmp" -type f -name "*${binname}" | head -n1)" - fi - if [ -z "${src:-}" ] || [ ! -f "$src" ]; then - echo "::error::binary ${binname} not found inside ${archive}" - find "$tmp" -maxdepth 3 -print - exit 1 - fi - cp "$src" "$pkgdir/bin/${binname}" - [ "$ext" = "zip" ] || chmod +x "$pkgdir/bin/${binname}" - - node -e ' - const fs = require("fs"); - const [dir, name, version, osv, cpuv] = process.argv.slice(1); - fs.writeFileSync(dir + "/package.json", JSON.stringify({ - name, - version, - description: name + " — prebuilt kimetsu-remote server binary", - repository: { type: "git", url: "git+https://github.com/RodCor/kimetsu.git" }, - license: "MIT OR Apache-2.0", - os: [osv], - cpu: [cpuv], - files: ["bin"], - }, null, 2) + "\n"); - ' "$pkgdir" "@kimetsu-ai/remote-${key}" "$VERSION" "$osv" "$cpuv" - - if npm view "@kimetsu-ai/remote-${key}@${VERSION}" version >/dev/null 2>&1; then - echo "skip: @kimetsu-ai/remote-${key}@${VERSION} already published" - else - echo "publishing @kimetsu-ai/remote-${key}@${VERSION}" - ( cd "$pkgdir" && npm publish --access public --provenance ) - fi - done - - - name: stamp + publish kimetsu-remote main package - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - set -euo pipefail - VERSION="${GITHUB_REF_NAME#v}" - node -e ' - const fs = require("fs"); - const version = process.argv[1]; - const p = "npm/kimetsu-remote/package.json"; - const j = JSON.parse(fs.readFileSync(p, "utf8")); - j.version = version; - for (const k of Object.keys(j.optionalDependencies || {})) { - j.optionalDependencies[k] = version; - } - fs.writeFileSync(p, JSON.stringify(j, null, 2) + "\n"); - ' "$VERSION" - if npm view "kimetsu-remote@${VERSION}" version >/dev/null 2>&1; then - echo "skip: kimetsu-remote@${VERSION} already published" - else - echo "publishing kimetsu-remote@${VERSION}" - ( cd npm/kimetsu-remote && npm publish --access public --provenance ) - fi - - - name: build + publish @kimetsu-ai/sdk - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - set -euo pipefail - # Deliberately NOT versioned off the tag. The SDK is a client - # library whose API moves independently of the binary's, and its - # sibling `kimetsu` on PyPI is versioned the same way — pinning it - # to the CLI version would promise a lockstep that does not exist. - # So it publishes at whatever package.json says, and a release that - # did not touch the SDK simply skips. - SDK_VERSION="$(node -p "require('./npm/kimetsu-sdk/package.json').version")" - if npm view "@kimetsu-ai/sdk@${SDK_VERSION}" version >/dev/null 2>&1; then - echo "skip: @kimetsu-ai/sdk@${SDK_VERSION} already published" - else - echo "publishing @kimetsu-ai/sdk@${SDK_VERSION}" - ( cd npm/kimetsu-sdk && npm ci && npm publish --access public --provenance ) - fi - - - name: summary - run: | - VERSION="${GITHUB_REF_NAME#v}" - echo "## npm publish — v${VERSION} ✅" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Published the main package + 4 platform packages:" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "- [\`kimetsu-ai\`](https://www.npmjs.com/package/kimetsu-ai/v/${VERSION})" >> $GITHUB_STEP_SUMMARY - for k in linux-x64 darwin-x64 darwin-arm64 win32-x64; do - echo "- [\`@kimetsu-ai/${k}\`](https://www.npmjs.com/package/@kimetsu-ai/${k}/v/${VERSION})" >> $GITHUB_STEP_SUMMARY - done - echo "" >> $GITHUB_STEP_SUMMARY - echo "Users can now install with:" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "npm install -g kimetsu-ai # lean build" >> $GITHUB_STEP_SUMMARY - echo "KIMETSU_NPM_FLAVOR=embeddings npm install -g kimetsu-ai # semantic build" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY diff --git a/docs/npm-staged-publishing.md b/docs/npm-staged-publishing.md new file mode 100644 index 0000000..8dafedf --- /dev/null +++ b/docs/npm-staged-publishing.md @@ -0,0 +1,75 @@ +# npm staged publishing and release recovery + +The release workflow stages npm packages for human approval. A successful staging +job does **not** mean the packages are publicly available. GitHub release assets +and crates.io publication remain separate. + +We use Node 24 and npm 11.21.0. npm documents Node 22.14+ and npm 11.15+ as the +minimum for staged publishing. The existing `NPM_TOKEN` can stay stage-only; +this workflow does not require direct-publish permission, change account or +trusted-publisher settings, or approve/reject stages. `PUBLISH_NPM=true` remains +the repository gate. Lifecycle scripts are disabled while packing/staging. The +SDK is built separately without the npm credential and retains its own version. + +## Recover an existing release + +After this workflow is merged and its checks pass, dispatch **stage npm release** +from `main`, setting `release_tag` to the existing stable tag (for example +`v2.9.0`). Equivalently: + +```sh +gh workflow run npm-stage.yml --ref main -f release_tag=v2.9.0 +``` + +This uses package sources from the tag and downloads the existing release assets. +It checks that the tag's commit belongs to main, that Cargo's version agrees, +and that the release is public and stable. It verifies the checksum manifest's +GitHub attestation against that exact tag/commit and the original release workflow, +then verifies the asset checksums and each archive's own attestation. This also +rejects an unauthenticated archive that was omitted from the checksum list. +It never rebuilds binaries, changes tags or +release assets, or reruns crates.io publication. npm provenance describes the +staging workflow run; the verified original asset attestation binds binaries to +the release commit. + +The job summary reports each package as already public, newly staged, or already +staged, including stage IDs where applicable. Public versions are skipped. +An existing stage is reused only if its package, version, tarball SHA-1 and +`latest` tag match. Conflicts and registry/authentication errors stop the job; +they are not treated as permission to resubmit, reject, or bypass approval. +If a submission's result is uncertain, inspect the stage list before retrying. +Partial stages remain private until approved. + +## Maintainer approval + +Review the stage IDs in the job summary in npm's **Staged Packages** interface, +or inspect them using npm 11.21.0: + +```sh +npm stage list @kimetsu-ai/linux-x64 --json +npm stage view +npm stage download +npm stage approve +``` + +Approval is an authenticated maintainer action with interactive 2FA. Do not put +an OTP or credentials in a workflow, issue, or chat. The workflow never invokes +`approve`. Review downloaded tarballs, versions, metadata, provenance and binaries +before approval; a matching SHA-1 only establishes retry identity. + +Approve the four CLI platform packages and three remote platform packages first. +Verify each exact version is public using `npm view @ version` +before approving the `kimetsu-ai` and `kimetsu-remote` wrappers that depend on them. +The SDK is independent. Approval is per package, not an atomic multi-package +transaction; a failed or partial approval must be resolved before approving a +dependent wrapper. Finally verify all public versions and test installation. + +The helper tests simulate registry and npm responses. They do not demonstrate +real-service staging, provenance acceptance, or human approval; those require a +successful authorized workflow run and maintainer approval. + +Official references: +- [Staged publishing](https://docs.npmjs.com/staged-publishing/) +- [npm stage commands](https://docs.npmjs.com/cli/v11/commands/npm-stage/) +- [Trusted publishing](https://docs.npmjs.com/trusted-publishers/) (a separate setup, + not enabled by this change) diff --git a/scripts/assemble-npm.sh b/scripts/assemble-npm.sh new file mode 100644 index 0000000..1907bcc --- /dev/null +++ b/scripts/assemble-npm.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# Run from the verified release-source checkout, with flat release assets in dist/. +set -euo pipefail +VERSION="${1:?stable release version required}" +[[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || exit 1 + +# npm-key target-triple os cpu binname ext +PLATFORMS=( + "linux-x64 x86_64-unknown-linux-gnu linux x64 kimetsu tar.gz" + "darwin-x64 x86_64-apple-darwin darwin x64 kimetsu tar.gz" + "darwin-arm64 aarch64-apple-darwin darwin arm64 kimetsu tar.gz" + "win32-x64 x86_64-pc-windows-msvc win32 x64 kimetsu.exe zip" +) + +mkdir -p stage +for row in "${PLATFORMS[@]}"; do + read -r key target osv cpuv binname ext <<<"$row" + stem="kimetsu-${VERSION}-${target}-lean" + archive="dist/${stem}.${ext}" + if [ ! -f "$archive" ]; then + echo "::error::missing lean artifact for ${target}: ${archive}" + exit 1 + fi + + tmp="$(mktemp -d)" + if [ "$ext" = "zip" ]; then + # PowerShell `Compress-Archive` writes ZIP entries with backslash + # separators; Info-ZIP `unzip` flattens those into a literal + # filename so the binary can't be found by path. `7z` (preinstalled + # on ubuntu-latest) normalizes backslashes into real directories. + 7z x -y -o"$tmp" "$archive" >/dev/null + else + tar -xf "$archive" -C "$tmp" + fi + + pkgdir="stage/${key}" + mkdir -p "$pkgdir/bin" + # Prefer the expected layout (`/`), then fall back to + # a recursive search so the step is robust to archive quirks. + if [ -f "$tmp/${stem}/${binname}" ]; then + src="$tmp/${stem}/${binname}" + elif [ -f "$tmp/${binname}" ]; then + src="$tmp/${binname}" + else + # Exact match, then suffix match — the latter catches a + # backslash-flattened basename like `\kimetsu.exe` if some + # extractor still mangles the Windows archive. + src="$(find "$tmp" -type f -name "${binname}" | head -n1)" + [ -z "$src" ] && src="$(find "$tmp" -type f -name "*${binname}" | head -n1)" + fi + if [ -z "${src:-}" ] || [ ! -f "$src" ]; then + echo "::error::binary ${binname} not found inside ${archive}" + echo "::error::extracted tree:"; find "$tmp" -maxdepth 3 -print + exit 1 + fi + cp "$src" "$pkgdir/bin/${binname}" + [ "$ext" = "zip" ] || chmod +x "$pkgdir/bin/${binname}" + + node -e ' + const fs = require("fs"); + const [dir, name, version, osv, cpuv] = process.argv.slice(1); + fs.writeFileSync(dir + "/package.json", JSON.stringify({ + name, + version, + description: name + " — prebuilt kimetsu binary", + repository: { type: "git", url: "git+https://github.com/RodCor/kimetsu.git" }, + license: "MIT OR Apache-2.0", + os: [osv], + cpu: [cpuv], + files: ["bin"], + }, null, 2) + "\n"); + ' "$pkgdir" "@kimetsu-ai/${key}" "$VERSION" "$osv" "$cpuv" + +done + +node -e ' + const fs = require("fs"); + const version = process.argv[1]; + const p = "npm/kimetsu/package.json"; + const j = JSON.parse(fs.readFileSync(p, "utf8")); + j.version = version; + for (const k of Object.keys(j.optionalDependencies || {})) { + j.optionalDependencies[k] = version; + } + fs.writeFileSync(p, JSON.stringify(j, null, 2) + "\n"); +' "$VERSION" + +# The server ships only where an embeddings build exists (ONNX prebuilts). +# npm-key target-triple os cpu binname ext +PLATFORMS=( + "linux-x64 x86_64-unknown-linux-gnu linux x64 kimetsu-remote tar.gz" + "darwin-arm64 aarch64-apple-darwin darwin arm64 kimetsu-remote tar.gz" + "win32-x64 x86_64-pc-windows-msvc win32 x64 kimetsu-remote.exe zip" +) + +mkdir -p stage-remote +for row in "${PLATFORMS[@]}"; do + read -r key target osv cpuv binname ext <<<"$row" + stem="kimetsu-remote-${VERSION}-${target}" + # The remote archive is uploaded alongside the embeddings kimetsu archive. + archive="dist/${stem}.${ext}" + if [ ! -f "$archive" ]; then + echo "::error::missing kimetsu-remote artifact for ${target}: ${archive}" + exit 1 + fi + + tmp="$(mktemp -d)" + if [ "$ext" = "zip" ]; then + 7z x -y -o"$tmp" "$archive" >/dev/null + else + tar -xf "$archive" -C "$tmp" + fi + + pkgdir="stage-remote/${key}" + mkdir -p "$pkgdir/bin" + if [ -f "$tmp/${stem}/${binname}" ]; then + src="$tmp/${stem}/${binname}" + elif [ -f "$tmp/${binname}" ]; then + src="$tmp/${binname}" + else + src="$(find "$tmp" -type f -name "${binname}" | head -n1)" + [ -z "$src" ] && src="$(find "$tmp" -type f -name "*${binname}" | head -n1)" + fi + if [ -z "${src:-}" ] || [ ! -f "$src" ]; then + echo "::error::binary ${binname} not found inside ${archive}" + find "$tmp" -maxdepth 3 -print + exit 1 + fi + cp "$src" "$pkgdir/bin/${binname}" + [ "$ext" = "zip" ] || chmod +x "$pkgdir/bin/${binname}" + + node -e ' + const fs = require("fs"); + const [dir, name, version, osv, cpuv] = process.argv.slice(1); + fs.writeFileSync(dir + "/package.json", JSON.stringify({ + name, + version, + description: name + " — prebuilt kimetsu-remote server binary", + repository: { type: "git", url: "git+https://github.com/RodCor/kimetsu.git" }, + license: "MIT OR Apache-2.0", + os: [osv], + cpu: [cpuv], + files: ["bin"], + }, null, 2) + "\n"); + ' "$pkgdir" "@kimetsu-ai/remote-${key}" "$VERSION" "$osv" "$cpuv" + +done + +node -e ' + const fs = require("fs"); + const version = process.argv[1]; + const p = "npm/kimetsu-remote/package.json"; + const j = JSON.parse(fs.readFileSync(p, "utf8")); + j.version = version; + for (const k of Object.keys(j.optionalDependencies || {})) { + j.optionalDependencies[k] = version; + } + fs.writeFileSync(p, JSON.stringify(j, null, 2) + "\n"); +' "$VERSION" diff --git a/scripts/stage-package.mjs b/scripts/stage-package.mjs new file mode 100644 index 0000000..beb0cae --- /dev/null +++ b/scripts/stage-package.mjs @@ -0,0 +1,77 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { pathToFileURL } from 'node:url'; + +const registry = 'https://registry.npmjs.org'; + +function liveIO(directory) { + return { + manifest: () => JSON.parse(fs.readFileSync(path.join(directory, 'package.json'), 'utf8')), + published: async (name, version) => { + const response = await fetch(`${registry}/${encodeURIComponent(name)}/${version}`, { + signal: AbortSignal.timeout(30000), redirect: 'error', + }); + if (response.status === 200) { + const body = await response.json(); + if (body.name !== name || body.version !== version) throw new Error('Unexpected registry version'); + } + return { status: response.status }; + }, + run: (args) => { + const result = spawnSync('npm', [...args, `--registry=${registry}`], { + cwd: directory, encoding: 'utf8', stdio: ['ignore', 'pipe', 'inherit'], + }); + if (result.error || result.status !== 0) throw new Error(`npm ${args.slice(0, 2).join(' ')} failed; no fallback attempted`); + return JSON.parse(result.stdout); + }, + }; +} + +export async function stagePackage(directory, io = liveIO(directory)) { + const { name, version } = io.manifest(); + if (!/^(?:@kimetsu-ai\/[a-z0-9-]+|kimetsu-ai|kimetsu-remote)$/.test(name) + || !/^\d+\.\d+\.\d+$/.test(version)) throw new Error('Unexpected package name/version'); + const published = await io.published(name, version); + if (published.status === 200) return { name, version, state: 'published' }; + if (published.status !== 404) throw new Error(`Registry lookup failed (${published.status}); no staging attempted`); + + const packed = io.run(['pack', '--json', '--ignore-scripts']); + if (!Array.isArray(packed) || packed.length !== 1 || !/^[a-f0-9]{40}$/.test(packed[0].shasum) + || typeof packed[0].filename !== 'string' + || path.basename(packed[0].filename) !== packed[0].filename) throw new Error('Invalid npm pack result'); + // npm 11.21 stage list accepts package NAME, not name@version. + const stages = io.run(['stage', 'list', name, '--json']); + if (!Array.isArray(stages)) throw new Error('Invalid stage list; no staging attempted'); + const matches = stages.filter(item => item.packageName === name && item.version === version); + if (matches.length) { + if (matches.length !== 1 || matches[0].shasum !== packed[0].shasum || matches[0].tag !== 'latest') { + throw new Error('Existing stage differs; maintainer must review it before any retry'); + } + if (typeof matches[0].id !== 'string' || !matches[0].id) throw new Error('Missing stage ID'); + return { name, version, state: 'already-staged', stageId: matches[0].id, shasum: packed[0].shasum }; + } + const result = io.run(['stage', 'publish', path.resolve(directory, packed[0].filename), + '--tag', 'latest', '--access', 'public', '--provenance', '--ignore-scripts', '--json']); + // npm 11.21 publish JSON is keyed by package name (unlike stage list). + const staged = result?.[name]; + if (typeof staged?.stageId !== 'string' || !staged.stageId) throw new Error('Missing stage ID; inspect npm before retrying'); + if (staged.name !== name || staged.version !== version || staged.shasum !== packed[0].shasum) { + throw new Error('Unexpected staged package metadata; inspect npm before retrying'); + } + return { name, version, state: 'staged', stageId: staged.stageId, shasum: packed[0].shasum }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + const result = await stagePackage(path.resolve(process.argv[2] ?? '.')); + console.log(JSON.stringify(result)); + if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, + `- ${result.name}@${result.version}: ${result.state}${result.stageId ? `; stage ID: ${result.stageId}; tarball SHA-1: ${result.shasum}` : ''}\n`); + } + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } +} diff --git a/scripts/stage-package.test.mjs b/scripts/stage-package.test.mjs new file mode 100644 index 0000000..4d8a111 --- /dev/null +++ b/scripts/stage-package.test.mjs @@ -0,0 +1,95 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { stagePackage } from './stage-package.mjs'; + +const shasum = 'a'.repeat(40); + +const packageName = '@kimetsu-ai/linux-x64'; +const stagedResponse = { [packageName]: { name: packageName, version: '2.9.0', shasum, stageId: 'new-stage' } }; + +function fixture({ status = 404, stages = [], response = stagedResponse } = {}) { + const calls = []; + return { + calls, + io: { + manifest: () => ({ name: '@kimetsu-ai/linux-x64', version: '2.9.0' }), + published: async () => ({ status }), + run: (args) => { + calls.push(args); + if (args[0] === 'pack') return [{ filename: 'package.tgz', shasum }]; + if (args[1] === 'list') return stages; + if (args[1] === 'publish') return response; + throw new Error('Unexpected command'); + }, + }, + }; +} + +test('public version skips without packing or authenticating', async () => { + const f = fixture({ status: 200 }); + assert.equal((await stagePackage('/package', f.io)).state, 'published'); + assert.deepEqual(f.calls, []); +}); +test('registry errors cannot be mistaken for an unpublished version', async () => { + for (const status of [401, 403, 429, 500]) { + const f = fixture({ status }); + await assert.rejects(stagePackage('/package', f.io), /Registry lookup failed/); + assert.deepEqual(f.calls, []); + } +}); +test('new version stages exact packed tarball and never approves', async () => { + const f = fixture(); + const result = await stagePackage('/package', f.io); + assert.equal(result.stageId, 'new-stage'); + assert.equal(result.state, 'staged'); + assert.deepEqual(f.calls[1].slice(0, 3), ['stage', 'list', '@kimetsu-ai/linux-x64']); + assert.deepEqual(f.calls[2].slice(0, 2), ['stage', 'publish']); + assert.ok(f.calls[2].includes('--ignore-scripts')); + assert.ok(!f.calls.some(args => args.includes('approve'))); +}); +test('identical existing stage is reused without resubmission', async () => { + const f = fixture({ stages: [{ packageName: '@kimetsu-ai/linux-x64', version: '2.9.0', id: 'existing', tag: 'latest', shasum }] }); + assert.equal((await stagePackage('/package', f.io)).stageId, 'existing'); + assert.equal(f.calls.length, 2); +}); +test('conflicting staged bytes or tag fail without submission', async () => { + for (const changes of [{ shasum: 'different' }, { tag: 'next' }]) { + const f = fixture({ stages: [{ packageName: '@kimetsu-ai/linux-x64', version: '2.9.0', id: 'existing', tag: 'latest', shasum, ...changes }] }); + await assert.rejects(stagePackage('/package', f.io), /Existing stage differs/); + assert.equal(f.calls.length, 2); + } +}); +test('malformed or failed stage listing never triggers submission', async () => { + const f = fixture({ stages: {} }); + await assert.rejects(stagePackage('/package', f.io), /Invalid stage list/); + assert.equal(f.calls.length, 2); +}); +test('a successful response without a stage ID is not called staged', async () => { + const f = fixture({ response: {} }); + await assert.rejects(stagePackage('/package', f.io), /Missing stage ID/); +}); + +test('incorrect staged package identity fails instead of claiming success', async () => { + for (const changes of [{ name: 'kimetsu-ai' }, { version: '2.8.0' }, { shasum: 'b'.repeat(40) }]) { + const f = fixture({ response: { [packageName]: { ...stagedResponse[packageName], ...changes } } }); + await assert.rejects(stagePackage('/package', f.io), /Unexpected staged package metadata/); + } +}); + +test('stage-list failure is propagated without submission', async () => { + const f = fixture(); + const run = f.io.run; + f.io.run = args => { + if (args[1] === 'list') throw new Error('Authentication failed'); + return run(args); + }; + await assert.rejects(stagePackage('/package', f.io), /Authentication failed/); + assert.equal(f.calls.length, 1); +}); + +test('ambiguous duplicate stages stop without submission', async () => { + const entry = { packageName: '@kimetsu-ai/linux-x64', version: '2.9.0', id: 'existing', tag: 'latest', shasum }; + const f = fixture({ stages: [entry, entry] }); + await assert.rejects(stagePackage('/package', f.io), /Existing stage differs/); + assert.equal(f.calls.length, 2); +}); diff --git a/scripts/test-assemble-npm.py b/scripts/test-assemble-npm.py new file mode 100644 index 0000000..cda84e9 --- /dev/null +++ b/scripts/test-assemble-npm.py @@ -0,0 +1,73 @@ +"""Exercise production assembly with synthetic archives; never contact npm.""" +import io +import json +import os +from pathlib import Path +import shutil +import subprocess +import tarfile +import tempfile +import unittest +import zipfile + +ROOT = Path(__file__).resolve().parents[1] +VERSION = '2.9.0' +PLATFORMS = [ + ('linux-x64', 'x86_64-unknown-linux-gnu', 'linux', 'x64', False), + ('darwin-x64', 'x86_64-apple-darwin', 'darwin', 'x64', False), + ('darwin-arm64', 'aarch64-apple-darwin', 'darwin', 'arm64', False), + ('win32-x64', 'x86_64-pc-windows-msvc', 'win32', 'x64', True), +] + + +class AssemblyTest(unittest.TestCase): + def test_all_platforms_wrappers_and_missing_asset(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / 'dist').mkdir() + for wrapper in ['kimetsu', 'kimetsu-remote']: + shutil.copytree(ROOT / 'npm' / wrapper, root / 'npm' / wrapper) + expected = [] + for remote in [False, True]: + for key, target, system, cpu, windows in PLATFORMS: + if remote and key == 'darwin-x64': + continue + binary = ('kimetsu-remote' if remote else 'kimetsu') + ('.exe' if windows else '') + stem = f'kimetsu-remote-{VERSION}-{target}' if remote else f'kimetsu-{VERSION}-{target}-lean' + payload = f'fixture:{stem}'.encode() + if windows: + # Match the backslash entries from PowerShell Compress-Archive. + with zipfile.ZipFile(root / 'dist' / f'{stem}.zip', 'w') as archive: + archive.writestr(f'{stem}\\{binary}', payload) + else: + with tarfile.open(root / 'dist' / f'{stem}.tar.gz', 'w:gz') as archive: + info = tarfile.TarInfo(f'{stem}/{binary}') + info.size = len(payload) + archive.addfile(info, io.BytesIO(payload)) + expected.append((remote, key, system, cpu, binary, payload)) + command = [os.environ.get('KIMETSU_TEST_BASH', 'bash'), + (ROOT / 'scripts/assemble-npm.sh').as_posix(), VERSION] + subprocess.run(command, cwd=root, check=True) + for remote, key, system, cpu, binary, payload in expected: + directory = root / ('stage-remote' if remote else 'stage') / key + manifest = json.loads((directory / 'package.json').read_text()) + self.assertEqual(manifest['name'], '@kimetsu-ai/' + ('remote-' if remote else '') + key) + self.assertEqual(manifest['version'], VERSION) + self.assertEqual(manifest['os'], [system]) + self.assertEqual(manifest['cpu'], [cpu]) + self.assertEqual((directory / 'bin' / binary).read_bytes(), payload) + if os.name != 'nt' and system != 'win32': + self.assertTrue((directory / 'bin' / binary).stat().st_mode & 0o111) + for wrapper, count in [('kimetsu', 4), ('kimetsu-remote', 3)]: + manifest = json.loads((root / 'npm' / wrapper / 'package.json').read_text()) + self.assertEqual(manifest['version'], VERSION) + self.assertEqual(len(manifest['optionalDependencies']), count) + self.assertEqual(set(manifest['optionalDependencies'].values()), {VERSION}) + (root / 'dist' / f'kimetsu-{VERSION}-x86_64-unknown-linux-gnu-lean.tar.gz').unlink() + result = subprocess.run(command, cwd=root, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn('missing lean artifact', result.stdout) + + +if __name__ == '__main__': + unittest.main()