feat: expand key controls protocol and rollback compatibility

.dockerignore

@@ -40,3 +40,5 @@ task_plan.md
 test_*.py
 test_*.sh
 tests/helpers/workbench_preview.py
+
+.release-wrangler-*

.github/workflows/ci.yml

@@ -17,6 +17,7 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
+          fetch-depth: 0
       - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.11"
@@ -50,6 +51,7 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
+          fetch-depth: 0
       - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.11"
@@ -84,6 +86,7 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
+          fetch-depth: 0
       - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.11"
@@ -106,6 +109,7 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
+          fetch-depth: 0
       - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
           node-version: "24"
@@ -119,11 +123,15 @@ jobs:
         run: git diff --exit-code -- static/css/style.css
       - name: Frontend build contracts
         run: node --test tests/test_frontend_build.mjs
+      - name: Check release configuration
+        run: node scripts/key_data_preflight.mjs --check
+      # Bundle-only checks cannot attest remote deployment prerequisites. The
+      # normal Wrangler build enforces those before a real upload.
       - name: Validate Knowledge Worker bundle
         env:
           WRANGLER_SEND_METRICS: "false"
-        run: pnpm exec wrangler deploy --config wrangler.knowledge.jsonc --dry-run --outdir /tmp/knowledge-worker
+        run: node scripts/check_worker_bundle.mjs --config wrangler.knowledge.jsonc --outdir /tmp/knowledge-worker
       - name: Validate main Worker bundle
         env:
           WRANGLER_SEND_METRICS: "false"
-        run: pnpm exec wrangler deploy --dry-run --containers-rollout=none --outdir /tmp/main-worker
+        run: node scripts/check_worker_bundle.mjs --config wrangler.jsonc --outdir /tmp/main-worker

.github/workflows/deploy.yml

@@ -121,6 +121,8 @@ jobs:
         run: npm install --global "$(node -p "require('./package.json').packageManager")"
       - name: Install deployment dependencies
         run: pnpm install --frozen-lockfile
+      - name: Check release configuration
+        run: node scripts/key_data_preflight.mjs --check
       - name: Check migration files
         run: node scripts/verify_d1_migrations.mjs --check
       - name: List pending D1 migrations
@@ -136,19 +138,37 @@ jobs:
         run: |
           pnpm exec wrangler d1 migrations apply "$D1_DATABASE" --remote
           node scripts/verify_d1_migrations.mjs "$D1_DATABASE"
+      - name: Check migrated schema and required bindings
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+        run: node scripts/key_data_preflight.mjs --prepare
       - name: Deploy Knowledge Worker
         id: knowledge
         env:
           CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
           CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
         run: pnpm exec wrangler deploy --config wrangler.knowledge.jsonc --tag "$SHA" --message "$DEPLOY_MESSAGE"
+      - name: Capture Container rollout baseline
+        if: vars.DEPLOY_MAIN_WORKER == 'true'
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+        run: node scripts/container_rollout.mjs --prepare --state "$RUNNER_TEMP/container-before.json" --output .release-wrangler-actions.jsonc
       - name: Deploy main Worker and Container
         id: main
         if: vars.DEPLOY_MAIN_WORKER == 'true'
         env:
           CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
           CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
-        run: pnpm exec wrangler deploy --tag "$SHA" --message "$DEPLOY_MESSAGE"
+        run: pnpm exec wrangler deploy --config .release-wrangler-actions.jsonc --tag "$SHA" --message "$DEPLOY_MESSAGE"
+      - name: Verify actual Container rollout
+        if: vars.DEPLOY_MAIN_WORKER == 'true'
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          DEPLOY_STATUS_URL: ${{ vars.DEPLOY_STATUS_URL }}
+        run: node scripts/container_rollout.mjs --verify --state "$RUNNER_TEMP/container-before.json"
       - name: Verify deployment
         id: verify
         env:

.gitignore

@@ -151,3 +151,6 @@ user_data.json
 .container-build-id
 # Local browser verification artifacts
 output/playwright/
+
+# Temporary deployment configuration (build arguments only).
+.release-wrangler-*

Dockerfile

@@ -25,6 +25,11 @@ RUN addgroup --system multillm && \
     chmod -R a+rX /app && \
     chmod 0750 /app/instance /tmp/multillm
 
+# A fresh release identifier prevents stale application-list comparisons from
+# treating a reused image as an unchanged Container. It is not a source stamp.
+ARG MULTILLM_RELEASE_IMAGE_ID=local
+LABEL multillm.release_image_id=$MULTILLM_RELEASE_IMAGE_ID
+
 USER multillm
 
 EXPOSE 8080

cloudflare-worker.mjs

@@ -1,3 +1,7 @@
+import { BUILD_ID } from "./worker/build-id.mjs";
+import { admitContainerDelegation } from "./worker/peer-controls.mjs";
+import { dispatchFailure } from "./worker/provider-dispatch.mjs";
+import { KEY_CONTROLS } from "./worker/key-controls-capability.mjs";
 import { nativeGenerationFetch, withForwardedCorrelation, withNativeMetrics, nativeCacheHeader, nativeGenerationSetup, generationErrorResponse, forwardedGenerationHeaders, runScheduledMaintenance, scheduledMaintenanceEnabled, handleRoleplayContextPageRequest, handleRealtimeRequest } from "./worker/gateway-extensions.mjs";
 import { batchesEnabled, runScheduledBatches } from "./worker/batch-jobs.mjs";
 import { tickNativeRevisionSync } from "./worker/native-config-sync.mjs";
@@ -282,7 +286,7 @@ function jsonResponse(body, init = {}) {
 
 function buildFallbackHealthResponse() {
   return jsonResponse({
-    status: "healthy",
+    status: "healthy", key_controls: KEY_CONTROLS,
     mode: "worker-fallback",
   });
 }
@@ -2079,6 +2083,10 @@ export default {
 
     try {
       const container = getContainer(env.MULTILLM_PROXY_CONTAINER, "primary");
+      if ((apiPath || isKnowledgeEdgePath(requestUrl.pathname)) && !readyPath) {
+        try { await admitContainerDelegation(request, env, container); }
+        catch (error) { return applyCorsHeaders(request, dispatchFailure(error), env); }
+      }
       const bodyAllowed = request.method !== "GET" && request.method !== "HEAD";
       const headers = forwardedGenerationHeaders(request, env);
       // Client copies of the Access identity headers never reach the Container.
@@ -2102,6 +2110,12 @@ export default {
         ...(bodyAllowed && request.body ? { duplex: "half" } : {}),
       });
       let response = await container.fetch(forwardedRequest);
+      if (readyPath && response.ok) {
+        const health = await response.clone().json();
+        const releaseHeaders = new Headers(response.headers);
+        releaseHeaders.set("X-MultiLLM-Worker-Build", BUILD_ID);
+        response = jsonResponse({ ...health, main_key_controls: KEY_CONTROLS }, { headers: releaseHeaders });
+      }
       if (requestUrl.pathname === "/v1/metrics/prometheus") {
         response = await withNativeMetrics(response, env);
       }

docs/deployment-cloudflare.md

@@ -383,14 +383,12 @@ the code, and nothing ships unless ci passed.
 3. Add *Containers Edit* to the API token, and run the `deploy` workflow once from the
    **Actions** tab to deploy the current head of `main`.
 
-**(b) Workers Builds keeps deploying the main Worker.** Leave `DEPLOY_MAIN_WORKER` unset
-(the default) or `false`, and keep the Workers Builds deploy command as
-`npx wrangler deploy`; do not change it to `npm run deploy`, which would apply the
-migrations and deploy the Knowledge Worker a second time. Actions applies migrations and
-deploys the Knowledge Worker after ci passes. Workers Builds starts at push time, so after
-a push that adds a migration the new main Worker usually runs for a few minutes before
-its tables exist, and `/ready` answers 503 `d1_schema_missing` until they do. Code must
-tolerate a missing new table, and a push whose ci fails still ships the main Worker.
+**(b) Keeping automatic promotion is not supported for the key-controls rollout.**
+Workers Builds can publish the main Worker and Container before migrations or the
+private Worker, even when GitHub CI fails. Disconnect it or use `wrangler versions upload`
+without automatic promotion, and deploy through `scripts/deploy.sh`. The preflight's
+D1, secret-list and deployments-read permissions inside Builds have not been verified.
+See [the staged rollout and rollback procedure](key-controls-rollout.md).
 
 ## One-time setup
 
@@ -440,13 +438,19 @@ which can show no commit until the next deploy.
 
 ## Manual deploys
 
-`npm run deploy` (`scripts/deploy.sh`) runs the same order from your machine: the
-migration checks, `wrangler d1 migrations apply`, the remote check, the Knowledge
-Worker and the main Worker. From a clean checkout it tags both versions with the commit.
-Extra arguments go to the final deploy, for example
-`npm run deploy -- --containers-rollout immediate` for a one-shot rollout of a changed
-Container image. Use it only when the workflow cannot run, and never while a deploy run
-is in progress; with option (a) the next push to `main` replaces it.
+Set `D1_DATABASE` explicitly to the database in both selected Worker configurations,
+then run `npm run deploy` (`scripts/deploy.sh`). For another environment set
+`MAIN_WORKER_CONFIG`, `KNOWLEDGE_WORKER_CONFIG` and `DEPLOY_STATUS_URL` as well. The
+script checks migrations, applies them, deploys the private Worker, then the main
+Worker and Container. From a clean checkout it tags both versions with the commit.
+Extra arguments go to the final deploy. Never run it concurrently with another deploy.
+
+Each release uses a fresh build-only image identifier. The command does not finish
+successfully until application detail shows a newer Container version and image, and
+`/ready` returns matching Worker/Container artifact fingerprints. "No changes" is not
+rollout proof. See [deployment verification](key-controls-rollout.md#deployment-verification).
+Status reads also require `--database` or `D1_DATABASE`; alternate configs are selected
+with `--config` and `--knowledge-config`. No default database is queried.
 
 **Alerts.** Worker logs are sampled at 100% (`head_sampling_rate: 1`): the Worker
 writes little besides structured failure lines such as `account_storage_failed`,
@@ -471,7 +475,11 @@ With D1, more Gunicorn worker processes behave like more instances: each keeps i
 
 ## Rollback
 
-Revert the commit on `main` so the deploy workflow ships the previous code, or use `npx wrangler rollback` for an immediate rollback that lasts until the next deploy. D1 migrations are not rolled back; because they are additive, the previous code keeps working with the newer schema. Container SQLite files are ephemeral and need no rollback; restore D1 data itself with D1 Time Travel.
+Use the [Container-first rollback procedure](key-controls-rollout.md#deployment-verification)
+until A/B rollback has passed real staging. A Worker-only rollback across earlier strict
+RPC decoders can fail unrestricted requests. Keep migrations and stored restrictions;
+A refuses controlled dispatch while preserving unrestricted and management access.
+Do not clear policies or roll back D1 migrations to make a rollback appear healthy.
 
 ## Sources