From 7b3509c2bdc68e608b309ac8187ada95f3193f9d Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 22 Oct 2026 19:34:59 -0600 Subject: [PATCH] fix: retain sequential pool roll history and image bindings --- README.md | 10 ++++++-- scripts/verify-pool-runtime-roll.ts | 32 +++++++++++++++++++++++ src/runtime/pool-maintenance.ts | 40 ++++++++++++++++++++++------- 3 files changed, 71 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index d89e125..58d57b5 100644 --- a/README.md +++ b/README.md @@ -1240,7 +1240,8 @@ Retained native-session API jobs do not yet enter this one-shot capture scope. ### Individually maintained Docker pool slots The bearer-protected `/admin/pools/status` endpoint reports explicit adopted pool -identities and durable roll receipts. Maintenance requires `BRIDGE_BEARER`, even +identities, per-slot immutable image bindings, and durable roll receipts/history. +Maintenance requires `BRIDGE_BEARER`, even on loopback. Adoption requires Linux with the existing `ss` socket inventory tool; unavailable or ambiguous listener evidence refuses ownership transfer. Run adoption from a native Linux bridge service in the same host network namespace as the @@ -1270,7 +1271,12 @@ No image is pulled and no volume is deleted by this interface. `pool-maintenance.json` under the bridge data directory records exact old and candidate identities before each Docker transition. After an interrupted roll, -retry the same original ID and image; a different intent is refused. After owner +retry the same original ID and image; a different unfinished intent is refused. +Once a transaction is promoted, drain its current candidate ID and request a +different immutable local image to begin the next transaction. The owner appends +the completed receipt to history before recording the new intent, retaining the +original adoption identities and every stopped predecessor. Missing local images +refuse without changing the completed journal. After owner restart, adopted slots remain drained until the same roll request verifies and re-admits each completed candidate. The journal contains no credential values. Use the registered service deployer to activate merged source; an unchanged diff --git a/scripts/verify-pool-runtime-roll.ts b/scripts/verify-pool-runtime-roll.ts index 3913291..11fade9 100644 --- a/scripts/verify-pool-runtime-roll.ts +++ b/scripts/verify-pool-runtime-roll.ts @@ -27,6 +27,7 @@ const owned = new Set() const volumes = new Set() const checks: string[] = [] let oldImage = '' +let nextImage = '' let pool: ContainerPool | undefined let appOwner: Awaited> | undefined let server: ReturnType | undefined @@ -64,6 +65,7 @@ try { const seed = await command(['create', '--name', `${prefix}-seed`, image]) owned.add(seed) oldImage = await command(['commit', '--change', `LABEL cli-bridge.proof=${suffix}`, seed]) + nextImage = await command(['commit', '--change', `LABEL cli-bridge.proof-next=${suffix}`, seed]) await command(['rm', '-v', seed]); owned.delete(seed) pool = await ContainerPool.create({ size: 2, image: oldImage, namePrefix: `${prefix}-leases`, @@ -144,7 +146,36 @@ try { assert.ok(status.pools[0]!.slots.every((slot) => !slot.draining)) assert.equal(readFileSync(join(data, 'pool-maintenance.json'), 'utf8').includes(token), false) checks.push('actual HTTP owner restart reconciles completed receipts and re-admits verified slots without secrets in journal') + const firstRoll = JSON.parse(readFileSync(join(data, 'pool-maintenance.json'), 'utf8')).pools[0].rolls['0'] as RollReceipt + const currentId = firstRoll.candidateId! + assert.equal((await api(`/admin/pools/${prefix}/slots/0/drain`, { expectedId: currentId })).status, 200) + const completedJournal = readFileSync(join(data, 'pool-maintenance.json'), 'utf8') + assert.equal((await api(`/admin/pools/${prefix}/slots/0/roll`, { expectedId: currentId, imageId: `sha256:${'0'.repeat(64)}` })).status, 409) + assert.equal(readFileSync(join(data, 'pool-maintenance.json'), 'utf8'), completedJournal) + const secondResponse = await api(`/admin/pools/${prefix}/slots/0/roll`, { expectedId: currentId, imageId: nextImage }) + assert.equal(secondResponse.status, 200) + const second = (await secondResponse.json() as { receipt: RollReceipt }).receipt + owned.add(second.candidateId!) + assert.notEqual(second.nonce, firstRoll.nonce) + assert.equal(second.oldId, currentId) + assert.equal((await dockerEngine('GET', `/containers/${currentId}/json`)).State.Running, false) + assert.equal((await dockerEngine('GET', `/containers/${slots[0]!.id}/json`)).State.Running, false) + assert.equal(await command(['exec', second.candidateId!, 'cat', '/root/.config/opencode/proof-marker']), 'retained') + await stopApp() + await startApp() + assert.equal((await api(`/admin/pools/${prefix}/slots/0/roll`, { expectedId: currentId, imageId: nextImage })).status, 200) + assert.equal((await api(`/admin/pools/${prefix}/slots/0/roll`, { expectedId: slots[0]!.id, imageId: image })).status, 409) + const journal = JSON.parse(readFileSync(join(data, 'pool-maintenance.json'), 'utf8')).pools[0] + assert.deepEqual(journal.slots, slots) + assert.deepEqual(journal.history['0'], [firstRoll]) + assert.equal(journal.rolls['0'].candidateId, second.candidateId) + const repeatedStatus = await (await api('/admin/pools/status')).json() as { pools: Array<{ slots: Array<{ containerId: string; draining: boolean }> }> } + assert.equal(repeatedStatus.pools[0]!.slots[0]!.containerId, second.candidateId) + assert.equal(repeatedStatus.pools[0]!.slots[0]!.draining, false) + checks.push('second immutable artifact appends prior receipt, preserves original identities and volumes, rejects stale intent, and reconciles after owner restart') + checks.push('missing second artifact refuses without changing completed journal') const result = { complete: true, checks, old_fixture_image: oldImage, candidate_image: image, + second_candidate_image: nextImage, production_node_upgrade_proven: false, no_model_calls: true } const output = process.env.CLI_ROLL_PROOF_OUTPUT if (output) writeFileSync(output, `${JSON.stringify(result, null, 2)}\n`, { mode: 0o600 }) @@ -156,5 +187,6 @@ try { for (const id of owned) await dockerCli(['rm', '-f', '-v', id]) for (const volume of volumes) await dockerCli(['volume', 'rm', volume]) if (oldImage) await dockerCli(['image', 'rm', oldImage]) + if (nextImage) await dockerCli(['image', 'rm', nextImage]) rmSync(root, { recursive: true, force: true }) } diff --git a/src/runtime/pool-maintenance.ts b/src/runtime/pool-maintenance.ts index 85eb339..8f27e0a 100644 --- a/src/runtime/pool-maintenance.ts +++ b/src/runtime/pool-maintenance.ts @@ -6,7 +6,7 @@ import { execFile } from 'node:child_process' import { promisify } from 'node:util' import { z } from 'zod' import { ContainerPool } from '../executors/container-pool.js' -import { inspectIdle, replaceRuntime, type RollReceipt } from '../executors/docker-runtime-roll.js' +import { dockerEngine, inspectIdle, replaceRuntime, type RollReceipt } from '../executors/docker-runtime-roll.js' import { registeredPools } from '../routes/metrics.js' import { acquireDataDirectoryLock, type DataDirectoryLock } from './single-instance.js' @@ -21,7 +21,8 @@ const rollSchema = z.object({ oldId: identity, imageId: z.string().regex(/^sha25 nonce: z.string().regex(/^[a-f0-9]{32}$/u), candidateId: identity.optional(), retainedName: z.string().optional(), phase: z.enum(['intent', 'candidate', 'verified', 'stopped', 'retained', 'promoted']), node: z.string().optional(), }).strict() -const recordSchema = adoptionRequest.extend({ rolls: z.record(z.string(), rollSchema), image: z.string() }).strict() +const recordSchema = adoptionRequest.extend({ rolls: z.record(z.string(), rollSchema), image: z.string(), + history: z.record(z.string(), z.array(rollSchema)).default({}), }).strict() type Record = z.infer /** Linux's kernel socket inventory covers IPv4, IPv6 and non-loopback binds. */ @@ -71,7 +72,7 @@ export class PoolMaintenance { if (this.pools.has(record.prefix)) return this.pools.get(record.prefix)! const ids = record.slots.map((slot) => { const roll = record.rolls[String(slot.index)] - return roll?.phase === 'promoted' ? roll.candidateId! : slot.id + return roll ? (roll.phase === 'promoted' ? roll.candidateId! : roll.oldId) : slot.id }) const pool = ContainerPool.adopt({ size: ids.length, image: record.image, namePrefix: record.prefix, oauthMode: 'share', slotMaxHoldMs: 0 }, ids) @@ -91,8 +92,16 @@ export class PoolMaintenance { } snapshot() { - return [...this.pools].map(([name, pool]) => ({ prefix: name, slots: pool.slotSnapshot(), - rolls: Object.values(this.records.find((r) => r.prefix === name)!.rolls) })) + return [...this.pools].map(([name, pool]) => { + const record = this.records.find((r) => r.prefix === name)! + const slots = pool.slotSnapshot().map((slot) => { + const transactions = [...(record.history[String(slot.index)] ?? []), record.rolls[String(slot.index)]] + const receipt = transactions.find((roll) => roll?.candidateId === slot.containerId) + const imageId = receipt?.imageId ?? (record.slots[slot.index]?.id === slot.containerId ? record.expectedImage : null) + return { ...slot, imageId } + }) + return { prefix: name, slots, rolls: Object.values(record.rolls), history: record.history } + }) } async adopt(input: z.infer) { @@ -119,7 +128,7 @@ export class PoolMaintenance { if (image && image !== info.Image) throw new Error('adoption requires one known runtime image') image = info.Image } - const record = { ...input, legacyDataDir: realpathSync(input.legacyDataDir), image, rolls: {} } + const record = { ...input, legacyDataDir: realpathSync(input.legacyDataDir), image, rolls: {}, history: {} } this.records.push(record) this.save() this.restore(record) @@ -148,18 +157,31 @@ export class PoolMaintenance { const slot = record.slots[index] if (!slot || slot.index !== index) throw new Error('unknown slot') const previous = record.rolls[String(index)] - if (previous && (previous.oldId !== expectedId || previous.imageId !== imageId)) throw new Error('roll intent differs from durable receipt') - const receipt: RollReceipt = previous ?? { oldId: expectedId, imageId, + const retry = previous?.oldId === expectedId && previous.imageId === imageId + if (previous && !retry && (previous.phase !== 'promoted' || previous.candidateId !== expectedId || previous.imageId === imageId)) { + throw new Error('new roll requires the completed transaction current identity and a different image') + } + const receipt: RollReceipt = retry ? previous! : { oldId: expectedId, imageId, name: `${name}-${index}`, nonce: randomBytes(16).toString('hex'), phase: 'intent' } this.operating.add(name) try { - if (previous?.phase === 'promoted') { + if (retry && previous?.phase === 'promoted') { const current = pool.slotSnapshot()[index]! pool.drainSlot(index, current.containerId) await pool.rollSlot(index, current.containerId, imageId, async (_id, arm) => await replaceRuntime(previous, () => this.save(), arm)) return previous } + if (!retry) { + const current = pool.slotSnapshot()[index]! + if (current.containerId !== expectedId || !current.draining || current.busy || current.maintenance) { + throw new Error('new roll requires the drained idle current slot') + } + // Missing local artifacts never replace a completed durable receipt. + const image = await dockerEngine('GET', `/images/${imageId}/json`) + if (image.Id !== imageId) throw new Error('runtime image identity differs') + if (previous) (record.history[String(index)] ??= []).push(structuredClone(previous)) + } // Persist the identity before the first Docker write; failures stay drained. record.rolls[String(index)] = receipt this.save()