From 1991e67a362d33f22c8bebed92e2284d5cfa0bf7 Mon Sep 17 00:00:00 2001 From: lihautan Date: Thu, 5 Nov 2026 12:33:59 +0800 Subject: [PATCH 1/9] Layer 3 design: agent-driven onboarding runs on a box Design for simular-ai/box-plugin#11. An agent plays the new user on a box through box's MCP tools, every step is recorded to a run folder, and a separate judge gives the verdict and files issues in box-portal. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/publish-cli.yml | 1 + cli/e2e/layer3/DESIGN.md | 150 ++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 cli/e2e/layer3/DESIGN.md diff --git a/.github/workflows/publish-cli.yml b/.github/workflows/publish-cli.yml index 34dc8d3..5bb2063 100644 --- a/.github/workflows/publish-cli.yml +++ b/.github/workflows/publish-cli.yml @@ -26,6 +26,7 @@ name: Publish CLI # The repo is public, so npm attaches a provenance attestation. # # TODO(layer 3): nightly run against the latest client versions plus a real agent task on a box. +# Design: cli/e2e/layer3/DESIGN.md (simular-ai/box-plugin#11). on: pull_request: diff --git a/cli/e2e/layer3/DESIGN.md b/cli/e2e/layer3/DESIGN.md new file mode 100644 index 0000000..d77cbd3 --- /dev/null +++ b/cli/e2e/layer3/DESIGN.md @@ -0,0 +1,150 @@ +# Layer 3: agent-driven onboarding runs + +Status: design, nothing built. Tracks [simular-ai/box-plugin#11](https://github.com/simular-ai/box-plugin/issues/11). + +## Goal + +Find out when a new user can no longer get from "I want box" to a first working tool call, before a user tells us. The main cause we want to catch is upstream change: Claude Code, Codex, Cursor, Hermes and the desktop apps ship often, and any release can move a path, a button or an auth flow. Our own regressions are a secondary target. Layers 1 and 2 already cover the parts that don't change. + +A run must not depend on steps we wrote down in advance. Scripted steps break when a client's UI moves, so they test the script, not the onboarding. Instead an agent plays the new user and works the steps out each time from what a user would see. + +## Roles + +| Role | What it is | First version | +|---|---|---| +| Driver | An agent acting as the new user. It sees the screen, clicks and types. | Claude Code with the box MCP server | +| Machine under test | The "user's computer". The client gets installed here. | A box (Ubuntu desktop), reset from a snapshot per run | +| Client under test | The app the user is setting box up in | Claude Code CLI, then Codex CLI, Cursor, ChatGPT desktop (Linux) | +| Box under test | The box the client reaches over MCP after sign-in | The test account's box on prod | +| Judge | A separate agent that reads the recording afterwards | Claude Code, fresh session per run | + +Later versions add a Mac mini as the machine under test for macOS builds, and Parallels VMs on it for Windows and Linux. The roles and the run folder below stay the same. Only the driver's tool set changes. + +## How a run works + +1. **Reset.** Restore the machine under test from a known snapshot. Apply the cell's seeds (for example, a second `codex` on PATH, or two Google accounts signed in to Chrome). +2. **Hand over.** Give the driver only what a real user gets for this entry point: the landing line `Set up box from https://box.simular.ai/install`, an "Open in" link, or the install.md URL. The driver also gets the rules below. +3. **Act.** The driver works through the onboarding on the box's screen until it gets a reply to one real task through box, or decides it is stuck. +4. **Collect.** The harness saves the run folder, including the server-side records for the run. +5. **Judge.** A fresh judge agent reads the run folder and writes a verdict and improvements. +6. **File.** Improvements become issues or draft PRs (see Filing). +7. **Clean up.** Revoke the run's grant and delete anything the run created on the test account. + +### Driver rules + +The driver must act as a user would. It controls the box through `screenshot`, `computer_act` and `computer_state`. It types into the box's own terminal window when a user would use a terminal (for example, `npx @simular-ai/box connect`), so the step is on screen and in the recording. + +The driver must not: + +- use `shell`, `write_file`, `edit_file` or `browser_*` during the user flow. These tools are for steps 1 and 7 only, run by the harness. +- edit config files by hand, read our source code, or use any URL that the user-facing pages don't show. +- retry the same failing step more than three times. + +If the driver breaks a rule to get past a problem, it must say so in a step note. The judge counts every such step as a workaround. + +## Recording + +The recording must not depend on the driver remembering to take notes. The driver reaches the box MCP server through a recording proxy. The proxy writes every tool call, its arguments, its result and every screenshot into the run folder. The driver's own notes ("I clicked Login because...") come from its transcript. + +### Run folder + +``` +runs// + run.json cell, machine, snapshot id, client and version, driver model, + account label, start and end time, the driver's own claim + steps.jsonl one line per tool call: index, time, tool, arguments, result + summary, screenshot file, driver note + screenshots/ 000.png, 001.png, ... + transcript.jsonl the driver's full Claude Code transcript + funnel.json server-side records for this run's user and time window + judge.json written by the judge +``` + +The run folder is the contract between the driver and the judge. Either side can change as long as the folder keeps this shape. + +### Server-side funnel + +`funnel.json` lists which onboarding steps the box service saw for the run: + +1. consent page served and approved (`/oauth/authorize`) +2. token exchange +3. `initialize` on `/mcp` +4. `tools/list` +5. first `tools/call` + +Each entry records the user and grant. Steps 1, 3, 4 and 5 are in box api logs today. Step 2, and any 401 responses, are not logged yet. That logging is a follow-up in box-portal. + +## Judge + +The judge is a fresh agent. It gets only the run folder, never the driver's live session, so the driver's claim can't sway it. It writes `judge.json`: + +```json +{ + "verdict": "pass | fail | blocked", + "reasons": [{ "text": "...", "steps": [12, 13] }], + "workarounds": [{ "text": "...", "steps": [20] }], + "improvements": [ + { + "target": "install.md | cli | portal | client-docs | harness", + "title": "...", + "detail": "...", + "steps": [8, 9], + "proposed": "issue | draft-pr" + } + ] +} +``` + +Verdicts: + +- **pass.** `funnel.json` shows a first `tools/call` as the intended account. The driver used no workarounds. The user saw an example prompt after connecting. +- **fail.** The onboarding or its docs failed a user. Only this verdict files product issues. +- **blocked.** The harness failed: the box didn't wake, the snapshot didn't restore, or the model hit a limit. It files nothing about the product, so harness problems don't fill the tracker. + +Any workaround makes the run a fail, even when the funnel completed. An agent that hand-edits a config still connects, but a real user would not. + +## Filing + +Recordings show the test account's email, the Google account chooser and sometimes tokens. This repo is public, so evidence never goes here. + +- Issues go to `simular-ai/box-portal` (private) with screenshots and the relevant transcript lines. Run folders live in private storage, and issues link to them. +- Before filing, the judge searches open issues for the same failure. If it finds one, it adds a comment ("seen again in run X") instead of opening a new issue. +- At most three new issues per run. +- PRs only for doc and copy fixes, for example install.md. They open as drafts and never merge without review. +- `harness` improvements (driver prompt, seeds, judge rubric) go to this repo, with no screenshots. + +## Test account + +A team member's account is the test account for now. It moves to `dev@simular.ai` later. The account is one config value, so the switch is one line. + +Until then: + +- Cleanup (step 7) matters more, because runs create grants on a real account. +- Free-tier cells (caps, `max_boxes`) wait for the dedicated account, unless the current account is on the free tier. +- The box's Chrome profile is signed in only to the test account, so no other account appears in screenshots. + +## First run + +One cell, driven by hand once to test the recording and the judge before building a runner: + +- Entry point: the landing line pasted into Claude Code CLI on the box +- Client: Claude Code CLI, latest version +- Auth: MCP OAuth +- Environment: clean snapshot, with the preinstalled `codex` and `claude` removed + +The run counts if the judge can reach a verdict from the folder alone, and every reason it gives points at a step and a screenshot. + +After that, in order: + +1. Codex CLI with a second `codex` on PATH. A stock box already has `codex` and `claude` in `/usr/local/bin`, so this seed is the snapshot as it is today. +2. Two Google accounts signed in to Chrome, asserting the grant user. +3. ChatGPT desktop (Linux preview). +4. A nightly schedule over the cells, with an alert when a scheduled run posts no result. + +## Open questions + +1. **One box or two.** The machine under test and the box under test both belong to the test account. If an account can have only one box, the client's first tool call acts on the same box the driver is using. Do we allow a second box for the test account, or accept a first task that only reads (for example, a screenshot)? +2. **Where the driver runs.** A local Claude Code session for the first run. For nightly runs, a GitHub workflow or the Mac mini. A GitHub workflow needs a box key or OAuth token as a secret. +3. **Pass rate.** One run is a sample. How many runs per cell per night, and what pass rate counts as green? +4. **Recording proxy.** Build a small MCP proxy, or rely on the driver's transcript, which already holds each tool call and screenshot? The transcript is simpler but ties the format to Claude Code. +5. **Sign-in.** Does Google sign-in accept a computer-use driver on a real Chrome window, or does the profile need signing in by hand once and saving into the snapshot? From 671302aa453683f0ea890c17547086bc7a450aeb Mon Sep 17 00:00:00 2001 From: lihautan Date: Thu, 5 Nov 2026 13:52:23 +0800 Subject: [PATCH 2/9] Layer 3: agent-driven onboarding run on a box, with a separate judge run.mjs resets a box over its shell tool, drives the onboarding with a headless claude -p limited to screenshot, computer_act and computer_state, extracts steps and screenshots from the stream-json transcript, then runs a fresh judge session that writes judge.json. The first cell is Claude Code CLI from the landing line. mcp-tools.mjs now uses a shared mcp-client.mjs. Co-Authored-By: Claude Opus 5.5 --- cli/e2e/layer3/DESIGN.md | 13 +++- cli/e2e/layer3/cells.json | 18 +++++ cli/e2e/layer3/driver.md | 31 ++++++++ cli/e2e/layer3/extract.mjs | 63 ++++++++++++++++ cli/e2e/layer3/judge.md | 30 ++++++++ cli/e2e/layer3/judge.schema.json | 36 +++++++++ cli/e2e/layer3/run.mjs | 123 +++++++++++++++++++++++++++++++ cli/e2e/mcp-client.mjs | 41 +++++++++++ cli/e2e/mcp-tools.mjs | 40 ++-------- cli/test/layer3-extract.test.js | 34 +++++++++ 10 files changed, 391 insertions(+), 38 deletions(-) create mode 100644 cli/e2e/layer3/cells.json create mode 100644 cli/e2e/layer3/driver.md create mode 100644 cli/e2e/layer3/extract.mjs create mode 100644 cli/e2e/layer3/judge.md create mode 100644 cli/e2e/layer3/judge.schema.json create mode 100644 cli/e2e/layer3/run.mjs create mode 100644 cli/e2e/mcp-client.mjs create mode 100644 cli/test/layer3-extract.test.js diff --git a/cli/e2e/layer3/DESIGN.md b/cli/e2e/layer3/DESIGN.md index d77cbd3..241364a 100644 --- a/cli/e2e/layer3/DESIGN.md +++ b/cli/e2e/layer3/DESIGN.md @@ -1,6 +1,6 @@ # Layer 3: agent-driven onboarding runs -Status: design, nothing built. Tracks [simular-ai/box-plugin#11](https://github.com/simular-ai/box-plugin/issues/11). +Status: first cell built (`run.mjs`, `cells.json`, `driver.md`, `judge.md`), not yet run live. Tracks [simular-ai/box-plugin#11](https://github.com/simular-ai/box-plugin/issues/11). ## Goal @@ -44,7 +44,11 @@ If the driver breaks a rule to get past a problem, it must say so in a step note ## Recording -The recording must not depend on the driver remembering to take notes. The driver reaches the box MCP server through a recording proxy. The proxy writes every tool call, its arguments, its result and every screenshot into the run folder. The driver's own notes ("I clicked Login because...") come from its transcript. +The recording must not depend on the driver remembering to take notes. The driver is a headless `claude -p` session with `--output-format stream-json`, which writes every tool call, its arguments, its result and every screenshot to `transcript.jsonl`. `extract.mjs` turns that into `steps.jsonl` and `screenshots/`. The driver's notes ("I clicked Login because...") are the text it writes before each call. + +The driver rules are enforced by the CLI, not only by the prompt. `--tools ""` removes every built-in tool, `--strict-mcp-config` loads only the box server, `--allowedTools` allows only `screenshot`, `computer_act` and `computer_state`, and `--disallowedTools` denies every other box tool. A denied attempt still shows in the transcript, so the judge sees it. + +Runs are written to `~/.box-layer3/runs/` (or `LAYER3_RUNS_DIR`), outside the repo, because they show the test account's screen. ### Run folder @@ -146,5 +150,6 @@ After that, in order: 1. **One box or two.** The machine under test and the box under test both belong to the test account. If an account can have only one box, the client's first tool call acts on the same box the driver is using. Do we allow a second box for the test account, or accept a first task that only reads (for example, a screenshot)? 2. **Where the driver runs.** A local Claude Code session for the first run. For nightly runs, a GitHub workflow or the Mac mini. A GitHub workflow needs a box key or OAuth token as a secret. 3. **Pass rate.** One run is a sample. How many runs per cell per night, and what pass rate counts as green? -4. **Recording proxy.** Build a small MCP proxy, or rely on the driver's transcript, which already holds each tool call and screenshot? The transcript is simpler but ties the format to Claude Code. -5. **Sign-in.** Does Google sign-in accept a computer-use driver on a real Chrome window, or does the profile need signing in by hand once and saving into the snapshot? +4. **Grant cleanup.** Step 7 needs a way to revoke the run's grant. There is no API for it from the harness yet, so runs leave grants behind on the test account. +5. **Funnel.** `funnel.json` is not collected yet. The judge decides from screenshots until box api exposes the funnel per run. +6. **Sign-in.** Does Google sign-in accept a computer-use driver on a real Chrome window, or does the profile need signing in by hand once and saving into the snapshot? diff --git a/cli/e2e/layer3/cells.json b/cli/e2e/layer3/cells.json new file mode 100644 index 0000000..c3fbee0 --- /dev/null +++ b/cli/e2e/layer3/cells.json @@ -0,0 +1,18 @@ +{ + "//": "Layer 3 cells. Each is one onboarding a new user might do. reset runs on the box over its shell tool before the driver starts; precheck must print OK or the run is blocked.", + "claude-code-landing": { + "entry": "landing-line", + "client": "claude-code", + "auth": "oauth", + "environment": "clean", + "reset": [ + "npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1 || sudo npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1", + "for s in user local project; do claude mcp remove box -s $s >/dev/null 2>&1; done; true", + "claude --version" + ], + "precheck": "test -s ~/.claude/.credentials.json && ! claude mcp list 2>/dev/null | grep -q '^box' && echo OK", + "precheckHelp": "Sign Claude Code in on the box once by hand (open a terminal on the box, run claude, finish /login). The run keeps that sign-in and removes only box's MCP entry.", + "handover": "You use Claude Code in a terminal on this computer. You found box by Simular and the landing page told you to paste this into your agent:\n\n Set up box from https://box.simular.ai/install", + "task": "Once box is set up, ask Claude Code to use box to take a screenshot of the box and describe what is on it." + } +} diff --git a/cli/e2e/layer3/driver.md b/cli/e2e/layer3/driver.md new file mode 100644 index 0000000..b14cf9f --- /dev/null +++ b/cli/e2e/layer3/driver.md @@ -0,0 +1,31 @@ +You are testing an onboarding flow by acting as a new user on a Linux desktop. You control the desktop only through its screen: take screenshots, click, type and press keys. You have no shell and no file access, on purpose. A real user has only the screen, and this test exists to find out whether the screen is enough. + +## Your situation + +{{handover}} + +## Your goal + +{{task}} + +## How to act + +- Do what a reasonable new user would do with what is on the screen and what the pages and apps tell you. Read instructions as you meet them. Do not use anything you know about how box works inside. +- When a user would use a terminal, open the terminal window on the desktop and type there. +- When a browser page asks you to sign in or approve access, do it if the account is already signed in or offered for one click. Never type a password. If a password is needed, stop and report it. +- Before each action, say in one short sentence what you see and why you are doing it. These notes are the record a reviewer reads later. +- If the same step fails three times, stop. + +## What not to do + +- Do not hand-edit config files or settings files, even if you could open them in an editor on screen. +- Do not guess URLs that no page or app showed you. +- If you do anything a new user would not, say so plainly in your note with the word WORKAROUND. + +## When you finish + +End your final message with exactly one line: + + RESULT: done +or + RESULT: stuck: diff --git a/cli/e2e/layer3/extract.mjs b/cli/e2e/layer3/extract.mjs new file mode 100644 index 0000000..3031683 --- /dev/null +++ b/cli/e2e/layer3/extract.mjs @@ -0,0 +1,63 @@ +// Turn a Claude Code stream-json transcript into the run folder's steps.jsonl and screenshots/. +import fs from 'node:fs' +import path from 'node:path' + +const EXT = { 'image/png': 'png', 'image/jpeg': 'jpg', 'image/webp': 'webp', 'image/gif': 'gif' } +const blocks = (m) => (Array.isArray(m?.message?.content) ? m.message.content : []) + +function summarize(content) { + if (typeof content === 'string') return content.slice(0, 2000) + return (content || []).filter((b) => b.type === 'text').map((b) => b.text).join('\n').slice(0, 2000) +} + +/** Parse stream-json lines into { steps, images, final }. Pure: no file access. */ +export function parseTranscript(lines) { + const steps = [] + const byId = new Map() + const images = [] + let note = '' + let final = null + for (const line of lines) { + if (!line.trim()) continue + const msg = JSON.parse(line) + if (msg.type === 'assistant') { + for (const b of blocks(msg)) { + if (b.type === 'text') note = note ? `${note}\n${b.text}` : b.text + if (b.type === 'tool_use') { + const step = { i: steps.length, tool: b.name, input: b.input, note: note.trim(), result: null, isError: false, screenshots: [] } + steps.push(step) + byId.set(b.id, step) + note = '' + } + } + } else if (msg.type === 'user') { + for (const b of blocks(msg)) { + if (b.type !== 'tool_result') continue + const step = byId.get(b.tool_use_id) + if (!step) continue + step.result = summarize(b.content) + step.isError = Boolean(b.is_error) + for (const c of Array.isArray(b.content) ? b.content : []) { + if (c.type !== 'image' || c.source?.type !== 'base64') continue + const file = `screenshots/${String(step.i).padStart(3, '0')}${step.screenshots.length ? `-${step.screenshots.length}` : ''}.${EXT[c.source.media_type] || 'png'}` + step.screenshots.push(file) + images.push({ file, data: c.source.data }) + } + } + } else if (msg.type === 'result') { + final = { text: msg.result ?? '', subtype: msg.subtype, turns: msg.num_turns, costUsd: msg.total_cost_usd, durationMs: msg.duration_ms } + } + } + const claim = /RESULT:\s*(done|stuck(?::.*)?)\s*$/im.exec(final?.text || '')?.[1] ?? null + return { steps, images, final: final && { ...final, claim, trailingNote: note.trim() || undefined } } +} + +/** Write steps.jsonl and screenshots/ into runDir from its transcript.jsonl. */ +export function extractRun(runDir) { + const lines = fs.readFileSync(path.join(runDir, 'transcript.jsonl'), 'utf8').split('\n') + const parsed = parseTranscript(lines) + fs.mkdirSync(path.join(runDir, 'screenshots'), { recursive: true }) + for (const img of parsed.images) fs.writeFileSync(path.join(runDir, img.file), Buffer.from(img.data, 'base64')) + fs.writeFileSync(path.join(runDir, 'steps.jsonl'), parsed.steps.map((s) => JSON.stringify(s)).join('\n') + (parsed.steps.length ? '\n' : '')) + return parsed +} diff --git a/cli/e2e/layer3/judge.md b/cli/e2e/layer3/judge.md new file mode 100644 index 0000000..ca02007 --- /dev/null +++ b/cli/e2e/layer3/judge.md @@ -0,0 +1,30 @@ +You review one recorded onboarding run. An agent (the driver) acted as a new user on a Linux desktop and tried to set up box by Simular in an AI client, then use it for one task. You decide whether a real new user would have succeeded, and what should change. + +The run folder is `{{runDir}}`. Read, in this order: + +1. `run.json` for the cell, the client version, what the driver was handed, and the driver's own claim. +2. `steps.jsonl`, one line per driver action, with the driver's note, the tool input, a result summary and the screenshot file. +3. The screenshots the steps point to. Look at them. The driver's notes can be wrong. +4. `funnel.json` if it exists, for what the box server saw. If it is missing, judge from the screenshots alone and say so in a reason. + +Do not trust the driver's claim. Verify every part of it against the screenshots. + +## Verdict + +- `pass` when all hold: a screenshot shows the client's reply to the task using box (for example, a description of the box's screen that came from a box tool); the driver needed no workaround; after connecting, the user was shown at least one example prompt or a clear next step. +- `fail` when the onboarding, its docs or a client let the user down: an error, a dead end, a confusing instruction the driver had to guess past, a missing next step, or any workaround. +- `blocked` when the test setup failed, not the product: the desktop never responded, the driver ran out of turns or budget before reaching the client, a precondition was missing. Explain which. + +A workaround is any step where the driver did something a new user would not: edited a config file, guessed an unlisted URL, used knowledge no page showed. Any workaround makes the run `fail`, even if box worked at the end. + +## Improvements + +For each problem, propose one improvement a maintainer can act on. Name the target: + +- `install.md` for the setup instructions the agent reads (box-portal) +- `portal` for box.simular.ai pages, the consent page, sign-in +- `cli` for `@simular-ai/box` +- `client-docs` for something only the client's maker can fix +- `harness` for this test itself: the driver prompt, the cell, the judge + +Every reason and improvement must cite step indexes. Prefer a few sharp improvements over many vague ones. If nothing should change, return no improvements. diff --git a/cli/e2e/layer3/judge.schema.json b/cli/e2e/layer3/judge.schema.json new file mode 100644 index 0000000..88a041b --- /dev/null +++ b/cli/e2e/layer3/judge.schema.json @@ -0,0 +1,36 @@ +{ + "type": "object", + "additionalProperties": false, + "required": ["verdict", "reasons", "workarounds", "improvements"], + "properties": { + "verdict": { "enum": ["pass", "fail", "blocked"] }, + "reasons": { "type": "array", "items": { "$ref": "#/$defs/cited" } }, + "workarounds": { "type": "array", "items": { "$ref": "#/$defs/cited" } }, + "improvements": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["target", "title", "detail", "steps", "proposed"], + "properties": { + "target": { "enum": ["install.md", "portal", "cli", "client-docs", "harness"] }, + "title": { "type": "string" }, + "detail": { "type": "string" }, + "steps": { "type": "array", "items": { "type": "integer" } }, + "proposed": { "enum": ["issue", "draft-pr"] } + } + } + } + }, + "$defs": { + "cited": { + "type": "object", + "additionalProperties": false, + "required": ["text", "steps"], + "properties": { + "text": { "type": "string" }, + "steps": { "type": "array", "items": { "type": "integer" } } + } + } + } +} diff --git a/cli/e2e/layer3/run.mjs b/cli/e2e/layer3/run.mjs new file mode 100644 index 0000000..5c9665b --- /dev/null +++ b/cli/e2e/layer3/run.mjs @@ -0,0 +1,123 @@ +// Layer 3: an agent plays a new user onboarding box on a box, then a separate judge reviews the recording. +// BOX_TOKEN=cb_... node e2e/layer3/run.mjs reset, drive, extract, judge +// node e2e/layer3/run.mjs --judge judge an existing run again +// Needs a signed-in `claude` CLI on this machine. Recordings hold the test account's screen, so they +// are written outside the repo (LAYER3_RUNS_DIR, default ~/.box-layer3/runs) and never committed. +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import spawn from 'cross-spawn' +import { DEFAULT_MCP_URL } from '../../src/constants.js' +import { connectMcp } from '../mcp-client.mjs' +import { extractRun } from './extract.mjs' + +const HERE = path.dirname(fileURLToPath(import.meta.url)) +const SERVER = 'box' +const DRIVER_TOOLS = ['screenshot', 'computer_act', 'computer_state'] +const read = (f) => fs.readFileSync(path.join(HERE, f), 'utf8') +const fill = (tpl, vars) => tpl.replace(/\{\{(\w+)\}\}/g, (_, k) => vars[k] ?? '') +const writeJson = (dir, name, obj) => fs.writeFileSync(path.join(dir, name), JSON.stringify(obj, null, 2) + '\n') + +function claude(args, { cwd, env, stdoutFile }) { + const out = stdoutFile ? fs.openSync(stdoutFile, 'w') : 'pipe' + const res = spawn.sync('claude', args, { cwd, env, stdio: ['ignore', out, 'pipe'], encoding: 'utf8', maxBuffer: 1 << 28 }) + if (stdoutFile) fs.closeSync(out) + if (res.error) throw res.error + return res +} + +function textOf(result) { + return (result?.content || []).filter((c) => c.type === 'text').map((c) => c.text).join('\n') +} + +async function drive(cellName) { + const cells = JSON.parse(read('cells.json')) + const cell = cells[cellName] + if (!cell) throw new Error(`unknown cell "${cellName}". Cells: ${Object.keys(cells).filter((k) => k !== '//').join(', ')}`) + const key = process.env.BOX_TOKEN + if (!key) throw new Error('set BOX_TOKEN to a box key for the test account (https://box.simular.ai/keys)') + const url = process.env.BOX_MCP_URL || DEFAULT_MCP_URL + + const runId = `${new Date().toISOString().replace(/[:.]/g, '-')}-${cellName}` + const runDir = path.join(process.env.LAYER3_RUNS_DIR || path.join(os.homedir(), '.box-layer3', 'runs'), runId) + fs.mkdirSync(runDir, { recursive: true }) + const run = { runId, cell: { name: cellName, ...cell }, mcpUrl: url, account: process.env.LAYER3_ACCOUNT || 'test-account', startedAt: new Date().toISOString(), funnel: 'unavailable' } + const finish = (extra) => { + Object.assign(run, extra, { endedAt: new Date().toISOString() }) + writeJson(runDir, 'run.json', run) + return runDir + } + + const mcp = await connectMcp(url, key, { clientName: 'box-layer3-harness' }) + const toolNames = (await mcp.listTools()).map((t) => t.name) + const missing = [...DRIVER_TOOLS, 'shell'].filter((t) => !toolNames.includes(t)) + if (missing.length) return finish({ blocked: `box MCP server lacks tools: ${missing.join(', ')}` }) + + const log = [] + for (const command of [...cell.reset, cell.precheck]) { + const res = await mcp.callTool('shell', { command, timeout_s: 300 }) + log.push(`$ ${command}\n${textOf(res)}`) + } + fs.writeFileSync(path.join(runDir, 'reset.log'), log.join('\n\n') + '\n') + run.clientVersion = log.at(-2)?.split('\n')[1]?.trim() + if (!/\bOK\b/.test(log.at(-1))) return finish({ blocked: `precheck failed. ${cell.precheckHelp}` }) + + writeJson(runDir, 'mcp.json', { mcpServers: { [SERVER]: { type: 'http', url, headers: { Authorization: 'Bearer ${BOX_TOKEN}' } } } }) + const allowed = DRIVER_TOOLS.map((t) => `mcp__${SERVER}__${t}`) + const denied = toolNames.filter((t) => !DRIVER_TOOLS.includes(t)).map((t) => `mcp__${SERVER}__${t}`) + const prompt = fill(read('driver.md'), cell) + const driverArgs = [ + '-p', prompt, + '--mcp-config', path.join(runDir, 'mcp.json'), '--strict-mcp-config', + '--tools', '', + '--allowedTools', allowed.join(','), + '--disallowedTools', denied.join(','), + '--output-format', 'stream-json', '--verbose', + '--max-turns', process.env.LAYER3_MAX_TURNS || '150', + '--max-budget-usd', process.env.LAYER3_MAX_USD || '10', + '--no-session-persistence', + ...(process.env.LAYER3_DRIVER_MODEL ? ['--model', process.env.LAYER3_DRIVER_MODEL] : []), + ] + console.log(`driving ${cellName} on ${url} (recording to ${runDir})`) + const transcript = path.join(runDir, 'transcript.jsonl') + const res = claude(driverArgs, { cwd: runDir, env: { ...process.env, BOX_TOKEN: key }, stdoutFile: transcript }) + fs.writeFileSync(transcript, fs.readFileSync(transcript, 'utf8').split(key).join('***')) + if (res.stderr) fs.writeFileSync(path.join(runDir, 'driver.stderr'), res.stderr.split(key).join('***')) + + const { steps, final } = extractRun(runDir) + return finish({ driver: { exit: res.status, model: process.env.LAYER3_DRIVER_MODEL || 'default', steps: steps.length, ...final } }) +} + +function judge(runDir) { + const run = JSON.parse(fs.readFileSync(path.join(runDir, 'run.json'), 'utf8')) + if (run.blocked) { + const verdict = { verdict: 'blocked', reasons: [{ text: run.blocked, steps: [] }], workarounds: [], improvements: [] } + writeJson(runDir, 'judge.json', verdict) + return verdict + } + const res = claude([ + '-p', fill(read('judge.md'), { runDir }), + '--tools', 'Read,Glob,Grep', + '--allowedTools', 'Read,Glob,Grep', + '--add-dir', runDir, + '--strict-mcp-config', + '--output-format', 'json', + '--json-schema', read('judge.schema.json'), + '--no-session-persistence', + ...(process.env.LAYER3_JUDGE_MODEL ? ['--model', process.env.LAYER3_JUDGE_MODEL] : []), + ], { cwd: runDir, env: process.env }) + if (res.status !== 0) throw new Error(`judge exited ${res.status}: ${res.stderr || res.stdout}`) + const out = JSON.parse(res.stdout) + const verdict = out.structured_output ?? JSON.parse(out.result) + writeJson(runDir, 'judge.json', verdict) + return verdict +} + +const argv = process.argv.slice(2) +const runDir = argv[0] === '--judge' ? path.resolve(argv[1]) : await drive(argv[0] || 'claude-code-landing') +const verdict = judge(runDir) +console.log(`${verdict.verdict.toUpperCase()} ${runDir}`) +for (const r of verdict.reasons) console.log(` - ${r.text} (steps ${r.steps.join(', ') || 'none'})`) +for (const imp of verdict.improvements) console.log(` * [${imp.target}] ${imp.title}`) +process.exitCode = verdict.verdict === 'pass' ? 0 : verdict.verdict === 'blocked' ? 2 : 1 diff --git a/cli/e2e/mcp-client.mjs b/cli/e2e/mcp-client.mjs new file mode 100644 index 0000000..760efa9 --- /dev/null +++ b/cli/e2e/mcp-client.mjs @@ -0,0 +1,41 @@ +// Minimal MCP client over streamable HTTP with a bearer key. +export async function connectMcp(url, key, { clientName = 'box-ci' } = {}) { + let session + let nextId = 1 + async function rpc(method, params, { notify = false, timeoutMs = 60000 } = {}) { + const id = notify ? undefined : nextId++ + const res = await fetch(url, { + method: 'POST', + headers: { + 'content-type': 'application/json', + accept: 'application/json, text/event-stream', + authorization: `Bearer ${key}`, + ...(session ? { 'mcp-session-id': session } : {}), + ...(method !== 'initialize' ? { 'mcp-protocol-version': '2025-06-18' } : {}), + }, + body: JSON.stringify({ jsonrpc: '2.0', method, ...(params ? { params } : {}), ...(id !== undefined ? { id } : {}) }), + signal: AbortSignal.timeout(timeoutMs), + }) + if (res.status === 401) throw new Error('401: the key was rejected') + if (!res.ok && res.status !== 202) throw new Error(`${method}: HTTP ${res.status}`) + session = res.headers.get('mcp-session-id') || session + if (id === undefined) return null + const type = res.headers.get('content-type') || '' + const text = await res.text() + const messages = type.includes('text/event-stream') + ? text.split('\n').filter((l) => l.startsWith('data:')).map((l) => JSON.parse(l.slice(5))) + : [JSON.parse(text)] + const reply = messages.find((m) => m.id === id) + if (!reply) throw new Error(`${method}: no reply`) + if (reply.error) throw new Error(`${method}: ${reply.error.message}`) + return reply.result + } + + const init = await rpc('initialize', { protocolVersion: '2025-06-18', capabilities: {}, clientInfo: { name: clientName, version: '0' } }) + await rpc('notifications/initialized', undefined, { notify: true }) + return { + serverInfo: init.serverInfo, + listTools: async () => (await rpc('tools/list', {})).tools, + callTool: (name, args, { timeoutMs = 180000 } = {}) => rpc('tools/call', { name, arguments: args }, { timeoutMs }), + } +} diff --git a/cli/e2e/mcp-tools.mjs b/cli/e2e/mcp-tools.mjs index 732e9bc..34001b7 100644 --- a/cli/e2e/mcp-tools.mjs +++ b/cli/e2e/mcp-tools.mjs @@ -1,41 +1,13 @@ -// Minimal MCP client: initialize + tools/list over streamable HTTP with a bearer key. +// Minimal MCP check: initialize + tools/list over streamable HTTP with a bearer key. // Lists tools only; never calls one (calling would wake the box or act on it). // BOX_MCP_URL=... BOX_TOKEN=cb_... node e2e/mcp-tools.mjs +import { connectMcp } from './mcp-client.mjs' + const url = process.env.BOX_MCP_URL const key = process.env.BOX_TOKEN if (!url || !key) throw new Error('set BOX_MCP_URL and BOX_TOKEN') -let session -async function rpc(method, params, id) { - const res = await fetch(url, { - method: 'POST', - headers: { - 'content-type': 'application/json', - accept: 'application/json, text/event-stream', - authorization: `Bearer ${key}`, - ...(session ? { 'mcp-session-id': session } : {}), - ...(method !== 'initialize' ? { 'mcp-protocol-version': '2025-06-18' } : {}), - }, - body: JSON.stringify({ jsonrpc: '2.0', method, ...(params ? { params } : {}), ...(id !== undefined ? { id } : {}) }), - signal: AbortSignal.timeout(60000), - }) - if (res.status === 401) throw new Error('401: the key was rejected') - if (!res.ok && res.status !== 202) throw new Error(`${method}: HTTP ${res.status}`) - session = res.headers.get('mcp-session-id') || session - if (id === undefined) return null - const type = res.headers.get('content-type') || '' - const text = await res.text() - const messages = type.includes('text/event-stream') - ? text.split('\n').filter((l) => l.startsWith('data:')).map((l) => JSON.parse(l.slice(5))) - : [JSON.parse(text)] - const reply = messages.find((m) => m.id === id) - if (!reply) throw new Error(`${method}: no reply`) - if (reply.error) throw new Error(`${method}: ${reply.error.message}`) - return reply.result -} - -const init = await rpc('initialize', { protocolVersion: '2025-06-18', capabilities: {}, clientInfo: { name: 'box-ci', version: '0' } }, 1) -await rpc('notifications/initialized') -const { tools } = await rpc('tools/list', {}, 2) -console.log(`server ${init.serverInfo?.name} ${init.serverInfo?.version}: ${tools.length} tools: ${tools.map((t) => t.name).join(', ')}`) +const mcp = await connectMcp(url, key) +const tools = await mcp.listTools() +console.log(`server ${mcp.serverInfo?.name} ${mcp.serverInfo?.version}: ${tools.length} tools: ${tools.map((t) => t.name).join(', ')}`) if (!tools.length) throw new Error('no tools listed') diff --git a/cli/test/layer3-extract.test.js b/cli/test/layer3-extract.test.js new file mode 100644 index 0000000..bf75a9d --- /dev/null +++ b/cli/test/layer3-extract.test.js @@ -0,0 +1,34 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import test from 'node:test' +import { extractRun } from '../e2e/layer3/extract.mjs' + +const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==' +const line = (o) => JSON.stringify(o) +const transcript = [ + line({ type: 'system', subtype: 'init' }), + line({ type: 'assistant', message: { content: [{ type: 'text', text: 'The desktop is empty. I will open a terminal.' }, { type: 'tool_use', id: 't1', name: 'mcp__box__screenshot', input: {} }] } }), + line({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 't1', content: [{ type: 'text', text: '1280x800' }, { type: 'image', source: { type: 'base64', media_type: 'image/png', data: PNG } }] }] } }), + line({ type: 'assistant', message: { content: [{ type: 'tool_use', id: 't2', name: 'mcp__box__computer_act', input: { action: 'type', text: 'claude' } }] } }), + line({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 't2', is_error: true, content: 'denied' }] } }), + line({ type: 'result', subtype: 'success', result: 'Connected.\nRESULT: stuck: the consent page never loaded', num_turns: 3, total_cost_usd: 0.1 }), +].join('\n') + +test('extractRun writes one step per tool call, its screenshots, and the driver claim', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'layer3-')) + fs.writeFileSync(path.join(dir, 'transcript.jsonl'), transcript) + + const { final } = extractRun(dir) + const steps = fs.readFileSync(path.join(dir, 'steps.jsonl'), 'utf8').trim().split('\n').map((l) => JSON.parse(l)) + + assert.equal(steps.length, 2) + assert.equal(steps[0].note, 'The desktop is empty. I will open a terminal.') + assert.deepEqual(steps[0].screenshots, ['screenshots/000.png']) + assert.equal(steps[0].result, '1280x800') + assert.ok(fs.readFileSync(path.join(dir, 'screenshots/000.png')).subarray(1, 4).equals(Buffer.from('PNG'))) + assert.equal(steps[1].isError, true) + assert.equal(steps[1].result, 'denied') + assert.equal(final.claim, 'stuck: the consent page never loaded') +}) From 4b0e9d7f0df415e4df6e8d412822af1a04260ad5 Mon Sep 17 00:00:00 2001 From: lihautan Date: Thu, 5 Nov 2026 14:18:13 +0800 Subject: [PATCH 3/9] Layer 3: read BOX_TOKEN from e2e/layer3/.env Co-Authored-By: Claude Opus 5.5 --- cli/.gitignore | 1 + cli/e2e/layer3/.env.example | 3 +++ cli/e2e/layer3/run.mjs | 6 ++++-- 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 cli/e2e/layer3/.env.example diff --git a/cli/.gitignore b/cli/.gitignore index 6b1d0bf..d35f03d 100644 --- a/cli/.gitignore +++ b/cli/.gitignore @@ -1 +1,2 @@ LICENSE +.env diff --git a/cli/e2e/layer3/.env.example b/cli/e2e/layer3/.env.example new file mode 100644 index 0000000..add119f --- /dev/null +++ b/cli/e2e/layer3/.env.example @@ -0,0 +1,3 @@ +# Copy to .env and fill in. .env is gitignored. +# A box key for the test account, from https://box.simular.ai/keys +BOX_TOKEN= diff --git a/cli/e2e/layer3/run.mjs b/cli/e2e/layer3/run.mjs index 5c9665b..04350a9 100644 --- a/cli/e2e/layer3/run.mjs +++ b/cli/e2e/layer3/run.mjs @@ -1,5 +1,6 @@ // Layer 3: an agent plays a new user onboarding box on a box, then a separate judge reviews the recording. -// BOX_TOKEN=cb_... node e2e/layer3/run.mjs reset, drive, extract, judge +// node e2e/layer3/run.mjs reset, drive, extract, judge +// BOX_TOKEN comes from the environment or e2e/layer3/.env (see .env.example). // node e2e/layer3/run.mjs --judge judge an existing run again // Needs a signed-in `claude` CLI on this machine. Recordings hold the test account's screen, so they // are written outside the repo (LAYER3_RUNS_DIR, default ~/.box-layer3/runs) and never committed. @@ -13,6 +14,7 @@ import { connectMcp } from '../mcp-client.mjs' import { extractRun } from './extract.mjs' const HERE = path.dirname(fileURLToPath(import.meta.url)) +if (fs.existsSync(path.join(HERE, '.env'))) process.loadEnvFile(path.join(HERE, '.env')) const SERVER = 'box' const DRIVER_TOOLS = ['screenshot', 'computer_act', 'computer_state'] const read = (f) => fs.readFileSync(path.join(HERE, f), 'utf8') @@ -36,7 +38,7 @@ async function drive(cellName) { const cell = cells[cellName] if (!cell) throw new Error(`unknown cell "${cellName}". Cells: ${Object.keys(cells).filter((k) => k !== '//').join(', ')}`) const key = process.env.BOX_TOKEN - if (!key) throw new Error('set BOX_TOKEN to a box key for the test account (https://box.simular.ai/keys)') + if (!key) throw new Error(`set BOX_TOKEN in ${path.join(HERE, '.env')} to a box key for the test account (https://box.simular.ai/keys)`) const url = process.env.BOX_MCP_URL || DEFAULT_MCP_URL const runId = `${new Date().toISOString().replace(/[:.]/g, '-')}-${cellName}` From 26b3219260f7a30b16788e012f683afa8c53316b Mon Sep 17 00:00:00 2001 From: lihautan Date: Thu, 5 Nov 2026 14:27:34 +0800 Subject: [PATCH 4/9] Layer 3: turn off claude.ai connectors in the clean cell, add a connector cell The first live run found Claude Code on the box loading the signed-in claude.ai account's box connector, so the clean cell never installed anything. The clean cell now sets ENABLE_CLAUDEAI_MCP_SERVERS=false and its precheck matches box's URL, not the name box. The connector path is its own cell. The driver no longer types ahead into a starting program. Co-Authored-By: Claude Opus 5.5 --- cli/e2e/layer3/cells.json | 25 ++++++++++++++++++++++--- cli/e2e/layer3/driver.md | 1 + 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/cli/e2e/layer3/cells.json b/cli/e2e/layer3/cells.json index c3fbee0..e1e13d6 100644 --- a/cli/e2e/layer3/cells.json +++ b/cli/e2e/layer3/cells.json @@ -1,17 +1,36 @@ { - "//": "Layer 3 cells. Each is one onboarding a new user might do. reset runs on the box over its shell tool before the driver starts; precheck must print OK or the run is blocked.", + "//": "Layer 3 cells. Each is one onboarding a new user might do. reset runs on the box over its shell tool before the driver starts; precheck must print OK or the run is blocked. Claude Code on the box loads the signed-in claude.ai account's connectors, so a clean cell turns them off with ENABLE_CLAUDEAI_MCP_SERVERS=false in ~/.bashrc.", "claude-code-landing": { "entry": "landing-line", "client": "claude-code", "auth": "oauth", "environment": "clean", "reset": [ + "pkill -x claude >/dev/null 2>&1; pkill -x xfce4-terminal >/dev/null 2>&1; true", "npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1 || sudo npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1", "for s in user local project; do claude mcp remove box -s $s >/dev/null 2>&1; done; true", + "grep -q '^export ENABLE_CLAUDEAI_MCP_SERVERS=false$' ~/.bashrc || echo 'export ENABLE_CLAUDEAI_MCP_SERVERS=false' >> ~/.bashrc", "claude --version" ], - "precheck": "test -s ~/.claude/.credentials.json && ! claude mcp list 2>/dev/null | grep -q '^box' && echo OK", - "precheckHelp": "Sign Claude Code in on the box once by hand (open a terminal on the box, run claude, finish /login). The run keeps that sign-in and removes only box's MCP entry.", + "precheck": "cd ~ && test -s ~/.claude/.credentials.json && grep -q '^export ENABLE_CLAUDEAI_MCP_SERVERS=false$' ~/.bashrc && ! ENABLE_CLAUDEAI_MCP_SERVERS=false claude mcp list 2>/dev/null | grep -q 'mcp.box.simular.ai' && echo OK", + "precheckHelp": "Sign Claude Code in on the box once by hand (open a terminal on the box, run claude, finish /login). The run keeps that sign-in, turns off claude.ai connectors, and removes any local box MCP entry.", + "handover": "You use Claude Code in a terminal on this computer. You found box by Simular and the landing page told you to paste this into your agent:\n\n Set up box from https://box.simular.ai/install", + "task": "Once box is set up, ask Claude Code to use box to take a screenshot of the box and describe what is on it." + }, + "claude-code-landing-connector": { + "entry": "landing-line", + "client": "claude-code", + "auth": "claude.ai-connector", + "environment": "claude.ai account already has the box connector", + "reset": [ + "pkill -x claude >/dev/null 2>&1; pkill -x xfce4-terminal >/dev/null 2>&1; true", + "npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1 || sudo npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1", + "for s in user local project; do claude mcp remove box -s $s >/dev/null 2>&1; done; true", + "sed -i '/^export ENABLE_CLAUDEAI_MCP_SERVERS=false$/d' ~/.bashrc", + "claude --version" + ], + "precheck": "cd ~ && test -s ~/.claude/.credentials.json && claude mcp list 2>/dev/null | grep -q '^claude.ai .*mcp.box.simular.ai.*Connected' && echo OK", + "precheckHelp": "Sign Claude Code in on the box with a claude.ai account that has the box by Simular connector.", "handover": "You use Claude Code in a terminal on this computer. You found box by Simular and the landing page told you to paste this into your agent:\n\n Set up box from https://box.simular.ai/install", "task": "Once box is set up, ask Claude Code to use box to take a screenshot of the box and describe what is on it." } diff --git a/cli/e2e/layer3/driver.md b/cli/e2e/layer3/driver.md index b14cf9f..193b99f 100644 --- a/cli/e2e/layer3/driver.md +++ b/cli/e2e/layer3/driver.md @@ -14,6 +14,7 @@ You are testing an onboarding flow by acting as a new user on a Linux desktop. Y - When a user would use a terminal, open the terminal window on the desktop and type there. - When a browser page asks you to sign in or approve access, do it if the account is already signed in or offered for one click. Never type a password. If a password is needed, stop and report it. - Before each action, say in one short sentence what you see and why you are doing it. These notes are the record a reviewer reads later. +- Do not type ahead into a program that is still starting. Take a screenshot first, because a prompt you didn't expect may take the keys. - If the same step fails three times, stop. ## What not to do From de4d601601b574ed85cb5d36f56d3560dcd1d060 Mon Sep 17 00:00:00 2001 From: lihautan Date: Thu, 5 Nov 2026 14:40:36 +0800 Subject: [PATCH 5/9] Layer 3: findings list from the first three runs; reset closes Chrome Co-Authored-By: Claude Opus 5.5 --- cli/e2e/layer3/FINDINGS.md | 34 ++++++++++++++++++++++++++++++++++ cli/e2e/layer3/cells.json | 4 ++-- 2 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 cli/e2e/layer3/FINDINGS.md diff --git a/cli/e2e/layer3/FINDINGS.md b/cli/e2e/layer3/FINDINGS.md new file mode 100644 index 0000000..824b2bf --- /dev/null +++ b/cli/e2e/layer3/FINDINGS.md @@ -0,0 +1,34 @@ +# Layer 3 findings + +Problems found while building and running layer 3, before the judge files issues on its own. Each entry says where it came from. Recordings live in `~/.box-layer3/runs/` on the machine that ran them, not in this repo. + +## Onboarding (product) + +| # | Finding | Target | Source | +|---|---|---|---| +| 1 | Claude Code's auto mode blocks `npx @simular-ai/box connect` as "Code from External". Claude Code falls back to the manual `claude mcp add` path in install.md, which works, so the main path fails under Claude Code's default mode. | install.md, cli | run `2026-11-05T06-23-49-887Z-claude-code-landing`, steps 5 and 6 | +| 2 | After adding box, the user is told to run `/mcp` and authenticate. `/mcp` in the same session shows "No MCP servers configured". Claude Code needs a restart first, and nothing says so. | install.md | same run, steps 7 to 9 | +| 3 | On box.simular.ai/sign-in, closing the Google sign-in popup leaves both sign-in buttons spinning until the page is reloaded. | portal | same run, steps 14 to 16 | +| 2a | Seen again with box fully working: the agent's reply says "Type /mcp here" as step 1 and "Start a new chat" as step 3. The driver had to guess the restart, so the run failed on a workaround although box connected and the task succeeded. | install.md | run `2026-11-05T06-36-07-346Z-claude-code-landing`, steps 8 to 13 | +| 13 | install.md and the agent's reply say "click **Approve**", but the consent page's button says **Allow**. | install.md | same run, steps 8 and 16 | +| 14 | After `/exit` and relaunch, Claude Code asked to trust `/home/user` again, although it had been trusted minutes earlier. Possibly upstream Claude Code behaviour, not yet confirmed. | client-docs | same run, step 12; also run `06-23-49`, step 9 | +| 4 | Fetching the install URL, Claude Code's first probe showed a red "Exit code 127" next to the HTTP 200 headers. It recovered, but a new user sees an error on the first step. | install.md | run `2026-11-05T06-20-57-248Z-claude-code-landing`, step 5 | + +## box platform + +| # | Finding | Target | Source | +|---|---|---|---| +| 5 | No vault or secret filling. box has no way to store a password or secret and fill it into a page without the agent seeing the value. Sign-in for tests relies on a Chrome profile signed in once by hand. | box | box docs (`/llms-full.txt`) and box's MCP tool list, 2026-11-05 | +| 6 | Copy and paste don't work in the screen view at box.simular.ai/box. | portal | reported by hand while signing the box's Chrome in, 2026-11-05 | + +## Harness + +| # | Finding | Status | +|---|---|---| +| 7 | Claude Code on the box loads the signed-in claude.ai account's connectors, including box, so a "clean" cell never installs anything. | Fixed: the clean cell sets `ENABLE_CLAUDEAI_MCP_SERVERS=false`. The connector path is now its own cell. | +| 8 | The driver typed ahead into a restarting Claude Code, and the Enter landed on the trust prompt, whose default is "No, exit". | Fixed: driver rule against typing ahead. | +| 9 | OAuth cells need a browser identity. The box's Chrome had no Google or GitHub account signed in, so sign-in needed a password. | Fixed by hand: Chrome on the box is signed in to the test account. | +| 9a | A box OAuth tab left open in Chrome from signing in by hand was still there when the next run started. | Fixed: reset closes Chrome. The profile, and its sign-in, stays. | +| 10 | No `funnel.json` yet. The judge decides from screenshots alone. | Open | +| 11 | Runs leave their OAuth grant on the test account. | Open | +| 12 | The judge proposes issues but nothing files them. | Open | diff --git a/cli/e2e/layer3/cells.json b/cli/e2e/layer3/cells.json index e1e13d6..5b0f65d 100644 --- a/cli/e2e/layer3/cells.json +++ b/cli/e2e/layer3/cells.json @@ -6,7 +6,7 @@ "auth": "oauth", "environment": "clean", "reset": [ - "pkill -x claude >/dev/null 2>&1; pkill -x xfce4-terminal >/dev/null 2>&1; true", + "pkill -x claude >/dev/null 2>&1; pkill -x xfce4-terminal >/dev/null 2>&1; pkill -x chrome >/dev/null 2>&1; true", "npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1 || sudo npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1", "for s in user local project; do claude mcp remove box -s $s >/dev/null 2>&1; done; true", "grep -q '^export ENABLE_CLAUDEAI_MCP_SERVERS=false$' ~/.bashrc || echo 'export ENABLE_CLAUDEAI_MCP_SERVERS=false' >> ~/.bashrc", @@ -23,7 +23,7 @@ "auth": "claude.ai-connector", "environment": "claude.ai account already has the box connector", "reset": [ - "pkill -x claude >/dev/null 2>&1; pkill -x xfce4-terminal >/dev/null 2>&1; true", + "pkill -x claude >/dev/null 2>&1; pkill -x xfce4-terminal >/dev/null 2>&1; pkill -x chrome >/dev/null 2>&1; true", "npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1 || sudo npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1", "for s in user local project; do claude mcp remove box -s $s >/dev/null 2>&1; done; true", "sed -i '/^export ENABLE_CLAUDEAI_MCP_SERVERS=false$/d' ~/.bashrc", From 9c0699e66ec6348d67bdbcc027f777c4a56ff2fa Mon Sep 17 00:00:00 2001 From: lihautan Date: Thu, 5 Nov 2026 14:52:44 +0800 Subject: [PATCH 6/9] Layer 3: link findings to their box-portal issues Co-Authored-By: Claude Opus 5.5 --- cli/e2e/layer3/FINDINGS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cli/e2e/layer3/FINDINGS.md b/cli/e2e/layer3/FINDINGS.md index 824b2bf..1207d02 100644 --- a/cli/e2e/layer3/FINDINGS.md +++ b/cli/e2e/layer3/FINDINGS.md @@ -14,6 +14,8 @@ Problems found while building and running layer 3, before the judge files issues | 14 | After `/exit` and relaunch, Claude Code asked to trust `/home/user` again, although it had been trusted minutes earlier. Possibly upstream Claude Code behaviour, not yet confirmed. | client-docs | same run, step 12; also run `06-23-49`, step 9 | | 4 | Fetching the install URL, Claude Code's first probe showed a red "Exit code 127" next to the HTTP 200 headers. It recovered, but a new user sees an error on the first step. | install.md | run `2026-11-05T06-20-57-248Z-claude-code-landing`, step 5 | +Filed in simular-ai/box-portal (private): #1 as box-portal#146, #2 and #2a as box-portal#147, #3 as box-portal#148, #4 as box-portal#149, #13 as box-portal#150. #14 is not filed until it is confirmed. + ## box platform | # | Finding | Target | Source | @@ -21,6 +23,8 @@ Problems found while building and running layer 3, before the judge files issues | 5 | No vault or secret filling. box has no way to store a password or secret and fill it into a page without the agent seeing the value. Sign-in for tests relies on a Chrome profile signed in once by hand. | box | box docs (`/llms-full.txt`) and box's MCP tool list, 2026-11-05 | | 6 | Copy and paste don't work in the screen view at box.simular.ai/box. | portal | reported by hand while signing the box's Chrome in, 2026-11-05 | +#5 is covered by box-portal#72 (sign in to sites in the box without leaving the app). #6 is not filed yet. + ## Harness | # | Finding | Status | From 8789eed07ed953ff71e6af8d64626787a662818b Mon Sep 17 00:00:00 2001 From: lihautan Date: Thu, 5 Nov 2026 15:02:55 +0800 Subject: [PATCH 7/9] Layer 3: funnel.json from Grants, grant cleanup, and filing with gh grants.mjs calls cocoonbox Grants over gRPC-web with the box key. A run's grants are the ones its client created inside the run window: they become funnel.json (approval and first token use) and are revoked after the run. A run is blocked when the client already holds a live grant, since consent would renew it. file.mjs files the judge's improvements with gh: product targets to box-portal with screenshots on a layer3-evidence branch, harness targets to this public repo as text only, emails masked everywhere, a model dedupe against open issues, at most three new issues per run, and filed.json so a run files once. Co-Authored-By: Claude Opus 5.5 --- cli/e2e/layer3/DESIGN.md | 7 +- cli/e2e/layer3/FINDINGS.md | 12 +-- cli/e2e/layer3/file.mjs | 157 +++++++++++++++++++++++++++++++++ cli/e2e/layer3/funnel.mjs | 28 ++++++ cli/e2e/layer3/grants.mjs | 127 ++++++++++++++++++++++++++ cli/e2e/layer3/judge.md | 2 +- cli/e2e/layer3/run.mjs | 25 +++++- cli/test/layer3-file.test.js | 8 ++ cli/test/layer3-funnel.test.js | 32 +++++++ 9 files changed, 385 insertions(+), 13 deletions(-) create mode 100644 cli/e2e/layer3/file.mjs create mode 100644 cli/e2e/layer3/funnel.mjs create mode 100644 cli/e2e/layer3/grants.mjs create mode 100644 cli/test/layer3-file.test.js create mode 100644 cli/test/layer3-funnel.test.js diff --git a/cli/e2e/layer3/DESIGN.md b/cli/e2e/layer3/DESIGN.md index 241364a..c351043 100644 --- a/cli/e2e/layer3/DESIGN.md +++ b/cli/e2e/layer3/DESIGN.md @@ -147,9 +147,8 @@ After that, in order: ## Open questions -1. **One box or two.** The machine under test and the box under test both belong to the test account. If an account can have only one box, the client's first tool call acts on the same box the driver is using. Do we allow a second box for the test account, or accept a first task that only reads (for example, a screenshot)? +1. **One box or two.** Settled for now: the client under test reaches the same box the driver is using (run 1 described its own terminal). A read-only first task, a screenshot, is safe on one box. 2. **Where the driver runs.** A local Claude Code session for the first run. For nightly runs, a GitHub workflow or the Mac mini. A GitHub workflow needs a box key or OAuth token as a secret. 3. **Pass rate.** One run is a sample. How many runs per cell per night, and what pass rate counts as green? -4. **Grant cleanup.** Step 7 needs a way to revoke the run's grant. There is no API for it from the harness yet, so runs leave grants behind on the test account. -5. **Funnel.** `funnel.json` is not collected yet. The judge decides from screenshots until box api exposes the funnel per run. -6. **Sign-in.** Does Google sign-in accept a computer-use driver on a real Chrome window, or does the profile need signing in by hand once and saving into the snapshot? +4. **Which tool was called.** `funnel.json` comes from `Grants.List`: approval time and first token use. box api does not expose token exchange or `tools/call` per grant yet, so the screenshots still decide whether the task ran. +5. **Sign-in.** Settled for now: the box's Chrome is signed in to the test account once by hand, and the consent page then needs one click. box has no way to fill a web password without the agent seeing it. diff --git a/cli/e2e/layer3/FINDINGS.md b/cli/e2e/layer3/FINDINGS.md index 1207d02..921e494 100644 --- a/cli/e2e/layer3/FINDINGS.md +++ b/cli/e2e/layer3/FINDINGS.md @@ -14,13 +14,15 @@ Problems found while building and running layer 3, before the judge files issues | 14 | After `/exit` and relaunch, Claude Code asked to trust `/home/user` again, although it had been trusted minutes earlier. Possibly upstream Claude Code behaviour, not yet confirmed. | client-docs | same run, step 12; also run `06-23-49`, step 9 | | 4 | Fetching the install URL, Claude Code's first probe showed a red "Exit code 127" next to the HTTP 200 headers. It recovered, but a new user sees an error on the first step. | install.md | run `2026-11-05T06-20-57-248Z-claude-code-landing`, step 5 | -Filed in simular-ai/box-portal (private): #1 as box-portal#146, #2 and #2a as box-portal#147, #3 as box-portal#148, #4 as box-portal#149, #13 as box-portal#150. #14 is not filed until it is confirmed. +Filed automatically by run `2026-11-05T06-58-25-137Z-claude-code-landing`: box-portal#152 (`box connect` should print the restart-first order; same root cause as #147, fixed in the CLI) and box-portal#153 (the consent page shows no confirmation after Allow, so a second click lands on a dead localhost callback). The same run commented "seen again" on box-portal#147. + +Filed by hand in simular-ai/box-portal (private): #1 as box-portal#146, #2 and #2a as box-portal#147, #3 as box-portal#148, #4 as box-portal#149, #13 as box-portal#150. #14 is not filed until it is confirmed. ## box platform | # | Finding | Target | Source | |---|---|---|---| -| 5 | No vault or secret filling. box has no way to store a password or secret and fill it into a page without the agent seeing the value. Sign-in for tests relies on a Chrome profile signed in once by hand. | box | box docs (`/llms-full.txt`) and box's MCP tool list, 2026-11-05 | +| 5 | No password filling for web pages. box has a secrets vault for **API keys** (box-portal `src/secrets/`, cocoonbox `Keys`): a box holds a stand-in in an env var and its egress swaps in the real value on the key's hosts only. It cannot fill a password into a sign-in page without the agent seeing it. Sign-in for tests relies on a Chrome profile signed in once by hand. | box | box-portal source and `keys.proto`, 2026-11-05 | | 6 | Copy and paste don't work in the screen view at box.simular.ai/box. | portal | reported by hand while signing the box's Chrome in, 2026-11-05 | #5 is covered by box-portal#72 (sign in to sites in the box without leaving the app). #6 is not filed yet. @@ -33,6 +35,6 @@ Filed in simular-ai/box-portal (private): #1 as box-portal#146, #2 and #2a as bo | 8 | The driver typed ahead into a restarting Claude Code, and the Enter landed on the trust prompt, whose default is "No, exit". | Fixed: driver rule against typing ahead. | | 9 | OAuth cells need a browser identity. The box's Chrome had no Google or GitHub account signed in, so sign-in needed a password. | Fixed by hand: Chrome on the box is signed in to the test account. | | 9a | A box OAuth tab left open in Chrome from signing in by hand was still there when the next run started. | Fixed: reset closes Chrome. The profile, and its sign-in, stays. | -| 10 | No `funnel.json` yet. The judge decides from screenshots alone. | Open | -| 11 | Runs leave their OAuth grant on the test account. | Open | -| 12 | The judge proposes issues but nothing files them. | Open | +| 10 | No `funnel.json`. The judge decided from screenshots alone. | Fixed: `funnel.json` from cocoonbox `Grants.List` shows approval and first token use. Which tool was called still needs box api. | +| 11 | Runs left their OAuth grant on the test account. | Fixed: the run revokes the grants its client created inside the run window. A run is blocked if the client already holds a live grant, because consent would renew it. | +| 12 | The judge proposed issues but nothing filed them. | Fixed: `file.mjs` files with `gh`, dedupes against open issues, caps new issues at three per run. | diff --git a/cli/e2e/layer3/file.mjs b/cli/e2e/layer3/file.mjs new file mode 100644 index 0000000..8d85067 --- /dev/null +++ b/cli/e2e/layer3/file.mjs @@ -0,0 +1,157 @@ +// File a judged run's improvements as GitHub issues with the local `gh`, once per run. +// node e2e/layer3/file.mjs [--dry-run] +// Product targets go to box-portal (private) with screenshots; `harness` goes to box-plugin (public) +// as text only; `client-docs` is listed, never filed. A repeat of an open issue becomes a comment. +import fs from 'node:fs' +import path from 'node:path' +import spawn from 'cross-spawn' + +const PRIVATE_REPO = process.env.LAYER3_ISSUES_REPO || 'simular-ai/box-portal' +const PUBLIC_REPO = process.env.LAYER3_HARNESS_REPO || 'simular-ai/box-plugin' +const EVIDENCE_BRANCH = 'layer3-evidence' +const MAX_NEW = 3 +const ROUTES = { 'install.md': PRIVATE_REPO, portal: PRIVATE_REPO, cli: PRIVATE_REPO, harness: PUBLIC_REPO } +const LABELS = { 'install.md': 'documentation', portal: 'bug', cli: 'bug', harness: 'enhancement' } +const FOOTER = '🤖 Filed by the layer 3 onboarding runs (simular-ai/box-plugin#11) with [Claude Code](https://claude.com/claude-code)' + +function run(cmd, args, { input, allowFail = false } = {}) { + const res = spawn.sync(cmd, args, { input, encoding: 'utf8', maxBuffer: 1 << 26 }) + if (res.error) throw res.error + if (res.status !== 0 && !allowFail) throw new Error(`${cmd} ${args.slice(0, 3).join(' ')} failed: ${res.stderr || res.stdout}`) + return res +} +const gh = (args, opts) => run('gh', args, opts) +const readJson = (dir, name) => JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8')) +const readSteps = (dir) => fs.readFileSync(path.join(dir, 'steps.jsonl'), 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l)) + +/** Ask a fresh model which open issue, if any, each improvement repeats. */ +function matchOpenIssues(improvements, openIssues) { + if (!openIssues.length) return improvements.map(() => null) + const schema = { type: 'object', additionalProperties: false, required: ['matches'], properties: { matches: { type: 'array', items: { type: ['integer', 'null'] } } } } + const prompt = [ + 'For each proposed improvement, return the number of the open GitHub issue that describes the same problem, or null if none does. Same problem means a maintainer fixing that issue would also fix the improvement. Answer with one entry per improvement, in order.', + '', + 'Improvements:', + ...improvements.map((imp, i) => `${i}. [${imp.target}] ${imp.title}\n ${imp.detail}`), + '', + 'Open issues:', + ...openIssues.map((iss) => `#${iss.number} ${iss.title}\n ${(iss.body || '').replace(/\s+/g, ' ').slice(0, 400)}`), + ].join('\n') + const res = run('claude', ['-p', prompt, '--tools', '', '--strict-mcp-config', '--output-format', 'json', '--json-schema', JSON.stringify(schema), '--no-session-persistence']) + const out = JSON.parse(res.stdout) + const { matches } = out.structured_output ?? JSON.parse(out.result) + const known = new Set(openIssues.map((i) => i.number)) + return improvements.map((_, i) => (known.has(matches[i]) ? matches[i] : null)) +} + +function ensureEvidenceBranch() { + const ref = gh(['api', `repos/${PRIVATE_REPO}/git/ref/heads/${EVIDENCE_BRANCH}`], { allowFail: true }) + if (ref.status === 0) return + const base = JSON.parse(gh(['api', `repos/${PRIVATE_REPO}`]).stdout).default_branch + const sha = JSON.parse(gh(['api', `repos/${PRIVATE_REPO}/git/ref/heads/${base}`]).stdout).object.sha + gh(['api', '-X', 'POST', `repos/${PRIVATE_REPO}/git/refs`, '-f', `ref=refs/heads/${EVIDENCE_BRANCH}`, '-f', `sha=${sha}`]) +} + +/** Upload one screenshot to the evidence branch and return a link that renders for repo readers. */ +function uploadScreenshot(runDir, runId, file) { + const dest = `layer3/${runId}/${file}` + const body = JSON.stringify({ message: `layer3 evidence: ${runId} ${file}`, branch: EVIDENCE_BRANCH, content: fs.readFileSync(path.join(runDir, file)).toString('base64') }) + const res = gh(['api', '-X', 'PUT', `repos/${PRIVATE_REPO}/contents/${dest}`, '--input', '-'], { input: body, allowFail: true }) + if (res.status !== 0 && !/sha/i.test(res.stdout + res.stderr)) throw new Error(`upload ${dest}: ${res.stderr || res.stdout}`) + return `https://github.com/${PRIVATE_REPO}/blob/${EVIDENCE_BRANCH}/${dest}?raw=true` +} + +function evidence(runDir, run, steps, cited, { withScreenshots }) { + const lines = [] + for (const i of [...new Set(cited)].sort((a, b) => a - b)) { + const step = steps[i] + if (!step) continue + lines.push(`**Step ${i}.** ${step.note || '(no note)'}`) + if (withScreenshots) for (const file of step.screenshots) lines.push(`![step ${i}](${uploadScreenshot(runDir, run.runId, file)})`) + } + return lines.join('\n\n') +} + +/** Mask email addresses: recordings show the test account's screen. */ +export const redact = (text) => text.replace(/[\w.+-]+@[\w-]+(\.[\w-]+)+/g, '[email]') + +function issueBody(imp, judge, run, evidenceText, { isPrivate }) { + const reasons = isPrivate ? judge.reasons.filter((r) => r.steps.some((s) => imp.steps.includes(s))).map((r) => `- ${r.text}`) : [] + return [ + imp.detail, + '', + '## Where it showed up', + '', + `Run \`${run.runId}\`: cell \`${run.cell.name}\` (${run.cell.client}, ${run.clientVersion || 'version unknown'}, entry ${run.cell.entry}, auth ${run.cell.auth}). Verdict **${judge.verdict}**. Steps ${imp.steps.join(', ')}.`, + ...(reasons.length ? ['', 'Judge:', '', ...reasons] : []), + '', + '## Evidence', + '', + evidenceText || (isPrivate ? '_No screenshots attached._' : '_This repo is public, so the recording stays private. Ask in #box for the run folder._'), + '', + FOOTER, + ].join('\n') +} + +function seenAgainBody(imp, judge, run, evidenceText) { + return [`Seen again in run \`${run.runId}\` (${run.cell.name}, ${run.clientVersion || 'version unknown'}, verdict **${judge.verdict}**): ${imp.title}`, '', evidenceText, '', FOOTER].join('\n') +} + +export function fileRun(runDir, { dryRun = false } = {}) { + const filedPath = path.join(runDir, 'filed.json') + if (fs.existsSync(filedPath)) return JSON.parse(fs.readFileSync(filedPath, 'utf8')) + const run = readJson(runDir, 'run.json') + const judge = readJson(runDir, 'judge.json') + const steps = fs.existsSync(path.join(runDir, 'steps.jsonl')) ? readSteps(runDir) : [] + const results = [] + if (judge.verdict === 'blocked') { + results.push(...judge.improvements.filter((i) => i.target === 'harness').map((imp) => ({ imp, action: 'skipped', why: 'blocked runs file only harness issues, and only by hand' }))) + } + const fileable = judge.verdict === 'blocked' ? [] : judge.improvements + const byRepo = new Map() + for (const imp of fileable) { + const repo = ROUTES[imp.target] + if (!repo) { + results.push({ imp, action: 'listed', why: `${imp.target} is for the client's maker` }) + continue + } + if (!byRepo.has(repo)) byRepo.set(repo, []) + byRepo.get(repo).push(imp) + } + + let created = 0 + for (const [repo, imps] of byRepo) { + const open = JSON.parse(gh(['issue', 'list', '--repo', repo, '--state', 'open', '--limit', '200', '--json', 'number,title,body']).stdout) + const matches = matchOpenIssues(imps, open) + const isPrivate = repo === PRIVATE_REPO + if (isPrivate && !dryRun && imps.some((imp) => imp.steps.length)) ensureEvidenceBranch() + imps.forEach((imp, i) => { + const ev = isPrivate ? evidence(runDir, run, steps, imp.steps, { withScreenshots: !dryRun }) : '' + if (matches[i]) { + if (!dryRun) gh(['issue', 'comment', String(matches[i]), '--repo', repo, '--body-file', '-'], { input: redact(seenAgainBody(imp, judge, run, ev)) }) + results.push({ imp, action: 'commented', repo, issue: matches[i] }) + } else if (created >= MAX_NEW) { + results.push({ imp, action: 'skipped', why: `cap of ${MAX_NEW} new issues per run` }) + } else { + created++ + let url = null + if (!dryRun) { + url = gh(['issue', 'create', '--repo', repo, '--title', redact(imp.title), '--label', LABELS[imp.target], '--body-file', '-'], { input: redact(issueBody(imp, judge, run, ev, { isPrivate })) }).stdout.trim() + } + results.push({ imp, action: 'created', repo, url }) + } + }) + } + + const summary = results.map(({ imp, ...r }) => ({ target: imp.target, title: imp.title, ...r })) + if (!dryRun) fs.writeFileSync(filedPath, JSON.stringify(summary, null, 2) + '\n') + return summary +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const [dir, flag] = process.argv.slice(2) + if (!dir) throw new Error('usage: node e2e/layer3/file.mjs [--dry-run]') + for (const r of fileRun(path.resolve(dir), { dryRun: flag === '--dry-run' })) { + console.log(`${r.action.padEnd(9)} [${r.target}] ${r.title}${r.issue ? ` -> ${r.repo}#${r.issue}` : ''}${r.url ? ` -> ${r.url}` : ''}${r.why ? ` (${r.why})` : ''}`) + } +} diff --git a/cli/e2e/layer3/funnel.mjs b/cli/e2e/layer3/funnel.mjs new file mode 100644 index 0000000..2ac0fa9 --- /dev/null +++ b/cli/e2e/layer3/funnel.mjs @@ -0,0 +1,28 @@ +// Server-side evidence for a run, from the test account's OAuth grants, and the run's cleanup. +// A grant belongs to the run when it was created inside the run's window by the cell's client, so a +// grant the account held before the run (another app, a claude.ai connector) is never touched. + +/** Clients by the name their OAuth metadata gives box's consent page. */ +const CLIENT_NAMES = { 'claude-code': ['Claude Code'] } + +export function runGrants(grants, { client, startedAtMs, endedAtMs }) { + const names = CLIENT_NAMES[client] + return grants.filter((g) => g.createdAtMs >= startedAtMs && g.createdAtMs <= endedAtMs && (!names || names.includes(g.clientName))) +} + +/** Live grants the cell's client already holds. Consent renews such a grant instead of creating one. */ +export function liveGrantsOf(grants, client, nowMs) { + const names = CLIENT_NAMES[client] || [] + return grants.filter((g) => names.includes(g.clientName) && (!g.expiresAtMs || g.expiresAtMs > nowMs)) +} + +export function funnelFrom(grants) { + const at = (ms) => (ms ? new Date(ms).toISOString() : null) + return { + source: 'cocoonbox Grants.List for the test account', + note: 'approvedAt is the consent (Approve). firstUsedAt is the first call made with the grant\'s token, to minute precision; it is not specific to tools/call. Token exchange and tools/call are not exposed yet, so the screenshots still decide whether the task ran.', + approved: grants.length > 0, + used: grants.some((g) => g.usedAtMs > 0), + grants: grants.map((g) => ({ id: g.id, clientName: g.clientName, clientHost: g.clientHost, approvedAt: at(g.createdAtMs), firstUsedAt: at(g.usedAtMs), maxBoxes: g.maxBoxes, boxesCreated: g.boxIds.length })), + } +} diff --git a/cli/e2e/layer3/grants.mjs b/cli/e2e/layer3/grants.mjs new file mode 100644 index 0000000..3c8fbc9 --- /dev/null +++ b/cli/e2e/layer3/grants.mjs @@ -0,0 +1,127 @@ +// cocoonbox Grants over gRPC-web with a personal key: list the caller's OAuth grants, revoke one. +// Hand-rolled protobuf for the two messages used here, so the e2e needs no generated code. +// node e2e/layer3/grants.mjs list +export const GRANTS_URL = process.env.BOX_GRANTS_URL || 'https://relay.simular.cloud/grpc-web' +const SERVICE = 'cocoonbox.control.v1.Grants' + +function varint(n) { + const out = [] + let v = BigInt(n) + do { + let b = Number(v & 0x7fn) + v >>= 7n + if (v) b |= 0x80 + out.push(b) + } while (v) + return out +} + +function encodeString(field, s) { + const bytes = Buffer.from(s, 'utf8') + return Buffer.from([...varint((field << 3) | 2), ...varint(bytes.length), ...bytes]) +} + +/** Decode one protobuf message into { field: [values] }; length-delimited values stay Buffers. */ +function decode(buf) { + const fields = {} + let i = 0 + const readVarint = () => { + let shift = 0n + let v = 0n + for (;;) { + const b = buf[i++] + v |= BigInt(b & 0x7f) << shift + if (!(b & 0x80)) return v + shift += 7n + } + } + while (i < buf.length) { + const tag = Number(readVarint()) + const field = tag >> 3 + const wire = tag & 7 + let value + if (wire === 0) value = readVarint() + else if (wire === 2) { + const len = Number(readVarint()) + value = buf.subarray(i, i + len) + i += len + } else if (wire === 1) { + value = buf.subarray(i, i + 8) + i += 8 + } else if (wire === 5) { + value = buf.subarray(i, i + 4) + i += 4 + } else throw new Error(`unsupported wire type ${wire}`) + ;(fields[field] ??= []).push(value) + } + return fields +} + +const str = (f, n) => (f[n]?.[0] ? f[n][0].toString('utf8') : '') +const num = (f, n) => Number(f[n]?.[0] ?? 0n) +const strs = (f, n) => (f[n] || []).map((b) => b.toString('utf8')) + +function parseGrant(buf) { + const f = decode(buf) + return { + id: str(f, 1), + clientId: str(f, 2), + clientName: str(f, 3), + clientHost: str(f, 4), + maxBoxes: num(f, 5), + maxRunning: num(f, 6), + boxIds: strs(f, 7), + chosenBoxIds: strs(f, 8), + createdAtMs: num(f, 9), + expiresAtMs: num(f, 10), + usedAtMs: num(f, 11), + } +} + +async function call(key, method, body) { + const frame = Buffer.concat([Buffer.from([0, ...[24, 16, 8, 0].map((s) => (body.length >>> s) & 0xff)]), body]) + const res = await fetch(`${GRANTS_URL}/${SERVICE}/${method}`, { + method: 'POST', + headers: { 'content-type': 'application/grpc-web+proto', 'x-grpc-web': '1', authorization: `Bearer ${key}` }, + body: frame, + signal: AbortSignal.timeout(30000), + }) + const buf = Buffer.from(await res.arrayBuffer()) + let message = null + let trailers = '' + for (let i = 0; i + 5 <= buf.length; ) { + const flag = buf[i] + const len = buf.readUInt32BE(i + 1) + const payload = buf.subarray(i + 5, i + 5 + len) + if (flag & 0x80) trailers += payload.toString('utf8') + else message = payload + i += 5 + len + } + const status = Number(res.headers.get('grpc-status') ?? /grpc-status:\s*(\d+)/i.exec(trailers)?.[1] ?? (res.ok ? 0 : -1)) + if (status !== 0) { + const msg = decodeURIComponent(res.headers.get('grpc-message') || /grpc-message:\s*(.*)/i.exec(trailers)?.[1]?.trim() || `HTTP ${res.status}`) + throw new Error(`Grants.${method}: grpc-status ${status}: ${msg}`) + } + return message ?? Buffer.alloc(0) +} + +export async function listGrants(key) { + const f = decode(await call(key, 'List', Buffer.alloc(0))) + return (f[1] || []).map(parseGrant) +} + +export async function revokeGrant(key, grantId) { + await call(key, 'Revoke', encodeString(1, grantId)) +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const { default: path } = await import('node:path') + const { default: fs } = await import('node:fs') + const env = path.join(path.dirname(new URL(import.meta.url).pathname), '.env') + if (fs.existsSync(env)) process.loadEnvFile(env) + if (!process.env.BOX_TOKEN) throw new Error('set BOX_TOKEN') + for (const g of await listGrants(process.env.BOX_TOKEN)) { + const at = (ms) => (ms ? new Date(ms).toISOString() : '-') + console.log(`${g.id} ${g.clientName.padEnd(24)} created ${at(g.createdAtMs)} used ${at(g.usedAtMs)} boxes ${g.boxIds.length}`) + } +} diff --git a/cli/e2e/layer3/judge.md b/cli/e2e/layer3/judge.md index ca02007..41e4798 100644 --- a/cli/e2e/layer3/judge.md +++ b/cli/e2e/layer3/judge.md @@ -5,7 +5,7 @@ The run folder is `{{runDir}}`. Read, in this order: 1. `run.json` for the cell, the client version, what the driver was handed, and the driver's own claim. 2. `steps.jsonl`, one line per driver action, with the driver's note, the tool input, a result summary and the screenshot file. 3. The screenshots the steps point to. Look at them. The driver's notes can be wrong. -4. `funnel.json` if it exists, for what the box server saw. If it is missing, judge from the screenshots alone and say so in a reason. +4. `funnel.json` if it exists, for what the box server saw: whether the user approved a grant for the client during the run, and whether that grant's token was used. It cannot show which tool was called, so a screenshot must still show the task's result. If the funnel says no grant was approved but the screenshots show box working, say so: the client may have used a connection that existed before the run. If the file is missing, judge from the screenshots alone and say so in a reason. Do not trust the driver's claim. Verify every part of it against the screenshots. diff --git a/cli/e2e/layer3/run.mjs b/cli/e2e/layer3/run.mjs index 04350a9..ff47253 100644 --- a/cli/e2e/layer3/run.mjs +++ b/cli/e2e/layer3/run.mjs @@ -2,6 +2,7 @@ // node e2e/layer3/run.mjs reset, drive, extract, judge // BOX_TOKEN comes from the environment or e2e/layer3/.env (see .env.example). // node e2e/layer3/run.mjs --judge judge an existing run again +// --no-file skip filing issues with gh // Needs a signed-in `claude` CLI on this machine. Recordings hold the test account's screen, so they // are written outside the repo (LAYER3_RUNS_DIR, default ~/.box-layer3/runs) and never committed. import fs from 'node:fs' @@ -12,6 +13,9 @@ import spawn from 'cross-spawn' import { DEFAULT_MCP_URL } from '../../src/constants.js' import { connectMcp } from '../mcp-client.mjs' import { extractRun } from './extract.mjs' +import { fileRun } from './file.mjs' +import { funnelFrom, liveGrantsOf, runGrants } from './funnel.mjs' +import { listGrants, revokeGrant } from './grants.mjs' const HERE = path.dirname(fileURLToPath(import.meta.url)) if (fs.existsSync(path.join(HERE, '.env'))) process.loadEnvFile(path.join(HERE, '.env')) @@ -55,6 +59,8 @@ async function drive(cellName) { const toolNames = (await mcp.listTools()).map((t) => t.name) const missing = [...DRIVER_TOOLS, 'shell'].filter((t) => !toolNames.includes(t)) if (missing.length) return finish({ blocked: `box MCP server lacks tools: ${missing.join(', ')}` }) + const held = liveGrantsOf(await listGrants(key), cell.client, Date.now()) + if (held.length) return finish({ blocked: `the test account already holds a live ${cell.client} grant (${held.map((g) => g.id).join(', ')}). Approving again would renew it instead of creating one, so the run could not tell its own consent apart. Revoke it at https://box.simular.ai/keys, then run again.` }) const log = [] for (const command of [...cell.reset, cell.precheck]) { @@ -88,7 +94,17 @@ async function drive(cellName) { if (res.stderr) fs.writeFileSync(path.join(runDir, 'driver.stderr'), res.stderr.split(key).join('***')) const { steps, final } = extractRun(runDir) - return finish({ driver: { exit: res.status, model: process.env.LAYER3_DRIVER_MODEL || 'default', steps: steps.length, ...final } }) + run.driver = { exit: res.status, model: process.env.LAYER3_DRIVER_MODEL || 'default', steps: steps.length, ...final } + + const mine = runGrants(await listGrants(key), { client: cell.client, startedAtMs: Date.parse(run.startedAt), endedAtMs: Date.now() }) + writeJson(runDir, 'funnel.json', funnelFrom(mine)) + run.funnel = 'funnel.json' + run.revoked = [] + for (const g of mine) { + await revokeGrant(key, g.id) + run.revoked.push(g.id) + } + return finish({}) } function judge(runDir) { @@ -117,9 +133,12 @@ function judge(runDir) { } const argv = process.argv.slice(2) -const runDir = argv[0] === '--judge' ? path.resolve(argv[1]) : await drive(argv[0] || 'claude-code-landing') +const noFile = argv.includes('--no-file') +const args = argv.filter((a) => a !== '--no-file') +const runDir = args[0] === '--judge' ? path.resolve(args[1]) : await drive(args[0] || 'claude-code-landing') const verdict = judge(runDir) console.log(`${verdict.verdict.toUpperCase()} ${runDir}`) for (const r of verdict.reasons) console.log(` - ${r.text} (steps ${r.steps.join(', ') || 'none'})`) -for (const imp of verdict.improvements) console.log(` * [${imp.target}] ${imp.title}`) +if (noFile) for (const imp of verdict.improvements) console.log(` * [${imp.target}] ${imp.title}`) +else for (const r of fileRun(runDir)) console.log(` * ${r.action} [${r.target}] ${r.title}${r.issue ? ` -> ${r.repo}#${r.issue}` : ''}${r.url ? ` -> ${r.url}` : ''}${r.why ? ` (${r.why})` : ''}`) process.exitCode = verdict.verdict === 'pass' ? 0 : verdict.verdict === 'blocked' ? 2 : 1 diff --git a/cli/test/layer3-file.test.js b/cli/test/layer3-file.test.js new file mode 100644 index 0000000..985922f --- /dev/null +++ b/cli/test/layer3-file.test.js @@ -0,0 +1,8 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { redact } from '../e2e/layer3/file.mjs' + +test('redact masks email addresses and leaves other text alone', () => { + assert.equal(redact('Signed in as someone.name+box@example.co.uk, then clicked Allow'), 'Signed in as [email], then clicked Allow') + assert.equal(redact('box.simular.ai/sign-in'), 'box.simular.ai/sign-in') +}) diff --git a/cli/test/layer3-funnel.test.js b/cli/test/layer3-funnel.test.js new file mode 100644 index 0000000..0eeb5b0 --- /dev/null +++ b/cli/test/layer3-funnel.test.js @@ -0,0 +1,32 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { funnelFrom, liveGrantsOf, runGrants } from '../e2e/layer3/funnel.mjs' + +const grant = (id, clientName, createdAtMs, usedAtMs = 0) => ({ id, clientName, clientHost: '', createdAtMs, usedAtMs, maxBoxes: 2, boxIds: [] }) +const window = { client: 'claude-code', startedAtMs: 1000, endedAtMs: 2000 } + +test('only grants the cell client created inside the run window belong to the run', () => { + const grants = [ + grant('oag_before', 'Claude Code', 500, 1500), + grant('oag_connector', 'Claude', 1200, 1300), + grant('oag_run', 'Claude Code', 1500, 1600), + grant('oag_after', 'Claude Code', 2500), + ] + assert.deepEqual(runGrants(grants, window).map((g) => g.id), ['oag_run']) +}) + +test('a live grant the cell client already holds is found, an expired one or another client is not', () => { + const grants = [{ ...grant('oag_live', 'Claude Code', 100), expiresAtMs: 9000 }, { ...grant('oag_old', 'Claude Code', 100), expiresAtMs: 500 }, { ...grant('oag_web', 'Claude', 100), expiresAtMs: 9000 }] + assert.deepEqual(liveGrantsOf(grants, 'claude-code', 1000).map((g) => g.id), ['oag_live']) +}) + +test('the funnel reports approval and first use from the run grants', () => { + const none = funnelFrom([]) + assert.equal(none.approved, false) + assert.equal(none.used, false) + + const f = funnelFrom([grant('oag_run', 'Claude Code', 1500, 1600)]) + assert.equal(f.approved, true) + assert.equal(f.used, true) + assert.equal(f.grants[0].approvedAt, new Date(1500).toISOString()) +}) From 654aebd63f277136d08b9161767c7194054b5595 Mon Sep 17 00:00:00 2001 From: lihautan Date: Thu, 5 Nov 2026 15:13:36 +0800 Subject: [PATCH 8/9] Layer 3: plain wording for funnel.json's note Co-Authored-By: Claude Opus 5.5 --- cli/e2e/layer3/funnel.mjs | 4 ++-- cli/e2e/layer3/judge.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/e2e/layer3/funnel.mjs b/cli/e2e/layer3/funnel.mjs index 2ac0fa9..a303b23 100644 --- a/cli/e2e/layer3/funnel.mjs +++ b/cli/e2e/layer3/funnel.mjs @@ -19,8 +19,8 @@ export function liveGrantsOf(grants, client, nowMs) { export function funnelFrom(grants) { const at = (ms) => (ms ? new Date(ms).toISOString() : null) return { - source: 'cocoonbox Grants.List for the test account', - note: 'approvedAt is the consent (Approve). firstUsedAt is the first call made with the grant\'s token, to minute precision; it is not specific to tools/call. Token exchange and tools/call are not exposed yet, so the screenshots still decide whether the task ran.', + source: "box's list of apps connected to the test account", + note: 'approved means someone clicked Allow on the box sign-in page during this run. used means the app then sent at least one request to box. box does not record which tools the app called, so only the screenshots show whether the task ran.', approved: grants.length > 0, used: grants.some((g) => g.usedAtMs > 0), grants: grants.map((g) => ({ id: g.id, clientName: g.clientName, clientHost: g.clientHost, approvedAt: at(g.createdAtMs), firstUsedAt: at(g.usedAtMs), maxBoxes: g.maxBoxes, boxesCreated: g.boxIds.length })), diff --git a/cli/e2e/layer3/judge.md b/cli/e2e/layer3/judge.md index 41e4798..4aaa857 100644 --- a/cli/e2e/layer3/judge.md +++ b/cli/e2e/layer3/judge.md @@ -5,7 +5,7 @@ The run folder is `{{runDir}}`. Read, in this order: 1. `run.json` for the cell, the client version, what the driver was handed, and the driver's own claim. 2. `steps.jsonl`, one line per driver action, with the driver's note, the tool input, a result summary and the screenshot file. 3. The screenshots the steps point to. Look at them. The driver's notes can be wrong. -4. `funnel.json` if it exists, for what the box server saw: whether the user approved a grant for the client during the run, and whether that grant's token was used. It cannot show which tool was called, so a screenshot must still show the task's result. If the funnel says no grant was approved but the screenshots show box working, say so: the client may have used a connection that existed before the run. If the file is missing, judge from the screenshots alone and say so in a reason. +4. `funnel.json` if it exists. It is what box's server saw. `approved` means someone clicked Allow on the box sign-in page during the run. `used` means the app then sent at least one request to box. The file does not show which tools the app called, so a screenshot must still show the task's result. If `approved` is false but the screenshots show box working, say so in a reason, because the app may have used a connection made before the run. If the file is missing, judge from the screenshots alone and say so in a reason. Do not trust the driver's claim. Verify every part of it against the screenshots. From c4e1404395cef2d3ffed206d355b15b9283afb85 Mon Sep 17 00:00:00 2001 From: lihautan Date: Thu, 5 Nov 2026 15:45:05 +0800 Subject: [PATCH 9/9] Layer 3: name cells by app/setup/alreadySetUp, document the dimensions A cell's name is now its three values joined with /, so it can't disagree with its fields. cells.mjs holds the allowed values with plain meanings and rejects unknown ones; the app entry also holds the name the app shows on box's sign-in page, which the funnel uses. DESIGN.md gains a Cells section (dimensions, every setup value, what a cell holds) and a funnel section that matches funnel.json. Co-Authored-By: Claude Opus 5.5 --- cli/e2e/layer3/DESIGN.md | 115 +++++++++++++++++++++++++-------- cli/e2e/layer3/cells.json | 72 +++++++++++---------- cli/e2e/layer3/cells.mjs | 45 +++++++++++++ cli/e2e/layer3/file.mjs | 4 +- cli/e2e/layer3/funnel.mjs | 20 +++--- cli/e2e/layer3/run.mjs | 23 +++---- cli/test/layer3-cells.test.js | 22 +++++++ cli/test/layer3-funnel.test.js | 8 +-- 8 files changed, 221 insertions(+), 88 deletions(-) create mode 100644 cli/e2e/layer3/cells.mjs create mode 100644 cli/test/layer3-cells.test.js diff --git a/cli/e2e/layer3/DESIGN.md b/cli/e2e/layer3/DESIGN.md index c351043..7c04e78 100644 --- a/cli/e2e/layer3/DESIGN.md +++ b/cli/e2e/layer3/DESIGN.md @@ -20,10 +20,80 @@ A run must not depend on steps we wrote down in advance. Scripted steps break wh Later versions add a Mac mini as the machine under test for macOS builds, and Parallels VMs on it for Windows and Linux. The roles and the run folder below stay the same. Only the driver's tool set changes. +## Cells + +A cell is one test case: one way a user might set up box, starting from one state. Cells live in `cells.json`. `cells.mjs` holds the allowed values for each dimension and rejects a cell that uses any other value. + +### Dimensions + +Each cell picks one value for each of these three: + +| Field | Question it answers | +|---|---| +| `app` | Which app is the user setting box up in? | +| `setup` | What does the user do first? | +| `alreadySetUp` | What is already there before the run starts? | + +A cell's name is the three values joined with `/`, so the name reads as a sentence: `claude-code-cli/paste-landing-line/nothing` is "Claude Code CLI, the user pastes the landing line, nothing set up yet". Each cell also has a `title` with that sentence written out, which run output and filed issues show. + +Two more dimensions have only one value so far, so they are not fields yet. The machine is always a Linux box. The account is always the test account. + +#### `app` + +| Value | App | +|---|---| +| `claude-code-cli` | Claude Code, the command-line app | + +Each app in `cells.mjs` also records how it signs in to box and the name it shows on box's sign-in page. Sign-in is not a separate dimension, because it follows from the app. Most apps click Allow in the browser (OAuth). Hermes, OpenClaw and dsh paste a personal key instead. A cell names its sign-in only for an app that supports both. + +#### `setup` + +| Value | What the user does first | Apps it applies to | +|---|---|---| +| `paste-landing-line` | Pastes `Set up box from https://box.simular.ai/install` into their agent, and the agent does the rest | Claude Code, Codex, Cursor, Gemini CLI, OpenCode, any coding agent. claude.ai and ChatGPT chat reply with the steps instead of installing | +| `click-open-in-button` | Clicks "Open in Claude / Codex / Cursor" on the box site, which opens the app with the setup prompt filled in | Claude Code (`claude://`), Codex (`codex://`), Cursor (`cursor.com/link`) | +| `click-add-to-claude-link` | Clicks the one-tap "Add to Claude" link, which opens claude.ai connector settings with box filled in | claude.ai, the Claude desktop app | +| `click-install-in-vscode` | Clicks the VS Code install link, then Install in VS Code | VS Code | +| `run-npx-connect` | Runs `npx @simular-ai/box connect` in their own terminal | Claude Code, Codex, Cursor, OpenCode, Gemini CLI, Hermes, OpenClaw, VS Code | +| `add-connector-in-settings` | Adds box by hand in the app's connector settings, following install.md | claude.ai, the Claude desktop app, ChatGPT | +| `edit-app-config` | Follows install.md's manual steps for their app, such as `claude mcp add`, Codex's TOML file or Cursor's JSON file | Claude Code, Codex, Cursor, Gemini CLI, dsh, any MCP app | + +`setup` is what the user does first, not every step that follows. In runs 3 and 5, Claude Code was blocked from running `npx` and fell back to `claude mcp add`. That run is still `paste-landing-line`. The fallback is something the judge reports from the recording. + +#### `alreadySetUp` + +| Value | What is there before the run | +|---|---| +| `nothing` | box is not set up anywhere: no MCP entry, no claude.ai connector, no live grant | +| `claude-ai-connector` | The user's claude.ai account already has the box by Simular connector, and the app loads it | + +Values to add next, from issue #11: `second-codex-on-path`, `old-box-entry`, `another-server-named-box`, `two-google-accounts`, `expired-access`. + +### What a cell holds + +Besides its three values and its title, a cell has: + +- `reset`. Shell commands the harness runs on the box to reach the cell's starting state, such as installing the latest app or turning claude.ai connectors off. +- `precheck`. A command that must print `OK`. If it doesn't, the run stops as `blocked` instead of testing the wrong thing. +- `precheckHelp`. What a person has to fix by hand when the precheck fails. +- `handover`. The user's situation, as the driver hears it. This is all the driver knows about how to start. +- `task`. The one thing the user wants to do with box once it works. + +The steps between `handover` and `task` are not written down on purpose. The driver works them out from the screen on each run, so an upstream change shows up as the driver getting stuck. + +### Cells today + +| Name | Title | +|---|---| +| `claude-code-cli/paste-landing-line/nothing` | Claude Code CLI: a new user pastes the landing line, with nothing set up yet | +| `claude-code-cli/paste-landing-line/claude-ai-connector` | Claude Code CLI: the user pastes the landing line, but their claude.ai account already has the box connector | + +To add a cell, add an entry to `cells.json`. A new `app` value also needs an entry in `cells.mjs` with the name the app shows on box's sign-in page, so the run can find its grant. + ## How a run works -1. **Reset.** Restore the machine under test from a known snapshot. Apply the cell's seeds (for example, a second `codex` on PATH, or two Google accounts signed in to Chrome). -2. **Hand over.** Give the driver only what a real user gets for this entry point: the landing line `Set up box from https://box.simular.ai/install`, an "Open in" link, or the install.md URL. The driver also gets the rules below. +1. **Reset.** Run the cell's `reset` commands on the box, then its `precheck`. +2. **Hand over.** Give the driver the cell's `handover` and `task`, and the rules below. 3. **Act.** The driver works through the onboarding on the box's screen until it gets a reply to one real task through box, or decides it is stuck. 4. **Collect.** The harness saves the run folder, including the server-side records for the run. 5. **Judge.** A fresh judge agent reads the run folder and writes a verdict and improvements. @@ -54,8 +124,10 @@ Runs are written to `~/.box-layer3/runs/` (or `LAYER3_RUNS_DIR`), outside the re ``` runs// - run.json cell, machine, snapshot id, client and version, driver model, - account label, start and end time, the driver's own claim + run.json the cell (name, title, values), the app's version, driver model, + account label, start and end time, the driver's own claim, + and the grants the run revoked + reset.log the reset and precheck commands and what they printed steps.jsonl one line per tool call: index, time, tool, arguments, result summary, screenshot file, driver note screenshots/ 000.png, 001.png, ... @@ -68,15 +140,15 @@ The run folder is the contract between the driver and the judge. Either side can ### Server-side funnel -`funnel.json` lists which onboarding steps the box service saw for the run: +`funnel.json` is what box's server saw for the run. It comes from box's list of apps connected to the test account (cocoonbox `Grants.List`). + +- `approved` is true when someone clicked Allow on box's sign-in page during the run, for the cell's app. +- `used` is true when the app then sent at least one request to box. +- `grants` lists the grants the run created, with when each was allowed and first used. -1. consent page served and approved (`/oauth/authorize`) -2. token exchange -3. `initialize` on `/mcp` -4. `tools/list` -5. first `tools/call` +A grant counts as the run's own when the cell's app created it during the run. A grant from before the run, such as a claude.ai connector, never counts. After the run, the harness revokes the run's own grants. A run stops as `blocked` when the cell's app already holds a live grant, because clicking Allow again renews that grant instead of creating one. -Each entry records the user and grant. Steps 1, 3, 4 and 5 are in box api logs today. Step 2, and any 401 responses, are not logged yet. That logging is a follow-up in box-portal. +`used` does not mean a tool ran. Connecting counts as a request. box does not record which tools an app called, so the screenshots still show whether the task ran. ## Judge @@ -101,7 +173,7 @@ The judge is a fresh agent. It gets only the run folder, never the driver's live Verdicts: -- **pass.** `funnel.json` shows a first `tools/call` as the intended account. The driver used no workarounds. The user saw an example prompt after connecting. +- **pass.** A screenshot shows the app's reply to the task through box. The driver used no workarounds. The user saw an example prompt or a clear next step after connecting. - **fail.** The onboarding or its docs failed a user. Only this verdict files product issues. - **blocked.** The harness failed: the box didn't wake, the snapshot didn't restore, or the model hit a limit. It files nothing about the product, so harness problems don't fill the tracker. @@ -127,22 +199,13 @@ Until then: - Free-tier cells (caps, `max_boxes`) wait for the dedicated account, unless the current account is on the free tier. - The box's Chrome profile is signed in only to the test account, so no other account appears in screenshots. -## First run - -One cell, driven by hand once to test the recording and the judge before building a runner: - -- Entry point: the landing line pasted into Claude Code CLI on the box -- Client: Claude Code CLI, latest version -- Auth: MCP OAuth -- Environment: clean snapshot, with the preinstalled `codex` and `claude` removed - -The run counts if the judge can reach a verdict from the folder alone, and every reason it gives points at a step and a screenshot. +## Next cells -After that, in order: +In order: -1. Codex CLI with a second `codex` on PATH. A stock box already has `codex` and `claude` in `/usr/local/bin`, so this seed is the snapshot as it is today. -2. Two Google accounts signed in to Chrome, asserting the grant user. -3. ChatGPT desktop (Linux preview). +1. `codex-cli/paste-landing-line/second-codex-on-path`. A stock box already has `codex` and `claude` in `/usr/local/bin`, so this starting state is the box as it is today. +2. `claude-code-cli/paste-landing-line/two-google-accounts`, checking that the grant goes to the intended account. +3. ChatGPT desktop (Linux preview) with `click-open-in-button`. 4. A nightly schedule over the cells, with an alert when a scheduled run posts no result. ## Open questions diff --git a/cli/e2e/layer3/cells.json b/cli/e2e/layer3/cells.json index 5b0f65d..53c27a0 100644 --- a/cli/e2e/layer3/cells.json +++ b/cli/e2e/layer3/cells.json @@ -1,37 +1,39 @@ { - "//": "Layer 3 cells. Each is one onboarding a new user might do. reset runs on the box over its shell tool before the driver starts; precheck must print OK or the run is blocked. Claude Code on the box loads the signed-in claude.ai account's connectors, so a clean cell turns them off with ENABLE_CLAUDEAI_MCP_SERVERS=false in ~/.bashrc.", - "claude-code-landing": { - "entry": "landing-line", - "client": "claude-code", - "auth": "oauth", - "environment": "clean", - "reset": [ - "pkill -x claude >/dev/null 2>&1; pkill -x xfce4-terminal >/dev/null 2>&1; pkill -x chrome >/dev/null 2>&1; true", - "npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1 || sudo npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1", - "for s in user local project; do claude mcp remove box -s $s >/dev/null 2>&1; done; true", - "grep -q '^export ENABLE_CLAUDEAI_MCP_SERVERS=false$' ~/.bashrc || echo 'export ENABLE_CLAUDEAI_MCP_SERVERS=false' >> ~/.bashrc", - "claude --version" - ], - "precheck": "cd ~ && test -s ~/.claude/.credentials.json && grep -q '^export ENABLE_CLAUDEAI_MCP_SERVERS=false$' ~/.bashrc && ! ENABLE_CLAUDEAI_MCP_SERVERS=false claude mcp list 2>/dev/null | grep -q 'mcp.box.simular.ai' && echo OK", - "precheckHelp": "Sign Claude Code in on the box once by hand (open a terminal on the box, run claude, finish /login). The run keeps that sign-in, turns off claude.ai connectors, and removes any local box MCP entry.", - "handover": "You use Claude Code in a terminal on this computer. You found box by Simular and the landing page told you to paste this into your agent:\n\n Set up box from https://box.simular.ai/install", - "task": "Once box is set up, ask Claude Code to use box to take a screenshot of the box and describe what is on it." - }, - "claude-code-landing-connector": { - "entry": "landing-line", - "client": "claude-code", - "auth": "claude.ai-connector", - "environment": "claude.ai account already has the box connector", - "reset": [ - "pkill -x claude >/dev/null 2>&1; pkill -x xfce4-terminal >/dev/null 2>&1; pkill -x chrome >/dev/null 2>&1; true", - "npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1 || sudo npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1", - "for s in user local project; do claude mcp remove box -s $s >/dev/null 2>&1; done; true", - "sed -i '/^export ENABLE_CLAUDEAI_MCP_SERVERS=false$/d' ~/.bashrc", - "claude --version" - ], - "precheck": "cd ~ && test -s ~/.claude/.credentials.json && claude mcp list 2>/dev/null | grep -q '^claude.ai .*mcp.box.simular.ai.*Connected' && echo OK", - "precheckHelp": "Sign Claude Code in on the box with a claude.ai account that has the box by Simular connector.", - "handover": "You use Claude Code in a terminal on this computer. You found box by Simular and the landing page told you to paste this into your agent:\n\n Set up box from https://box.simular.ai/install", - "task": "Once box is set up, ask Claude Code to use box to take a screenshot of the box and describe what is on it." - } + "//": "Layer 3 cells. A cell's name is app/setup/alreadySetUp; the allowed values and what they mean are in cells.mjs. reset runs on the box over its shell tool before the driver starts, and precheck must print OK or the run is blocked. Claude Code on the box loads the signed-in claude.ai account's connectors, so a cell with nothing set up turns them off with ENABLE_CLAUDEAI_MCP_SERVERS=false in ~/.bashrc.", + "cells": [ + { + "app": "claude-code-cli", + "setup": "paste-landing-line", + "alreadySetUp": "nothing", + "title": "Claude Code CLI: a new user pastes the landing line, with nothing set up yet", + "reset": [ + "pkill -x claude >/dev/null 2>&1; pkill -x xfce4-terminal >/dev/null 2>&1; pkill -x chrome >/dev/null 2>&1; true", + "npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1 || sudo npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1", + "for s in user local project; do claude mcp remove box -s $s >/dev/null 2>&1; done; true", + "grep -q '^export ENABLE_CLAUDEAI_MCP_SERVERS=false$' ~/.bashrc || echo 'export ENABLE_CLAUDEAI_MCP_SERVERS=false' >> ~/.bashrc", + "claude --version" + ], + "precheck": "cd ~ && test -s ~/.claude/.credentials.json && grep -q '^export ENABLE_CLAUDEAI_MCP_SERVERS=false$' ~/.bashrc && ! ENABLE_CLAUDEAI_MCP_SERVERS=false claude mcp list 2>/dev/null | grep -q 'mcp.box.simular.ai' && echo OK", + "precheckHelp": "Sign Claude Code in on the box once by hand (open a terminal on the box, run claude, finish /login). The run keeps that sign-in, turns off claude.ai connectors, and removes any local box MCP entry.", + "handover": "You use Claude Code in a terminal on this computer. You found box by Simular and the landing page told you to paste this into your agent:\n\n Set up box from https://box.simular.ai/install", + "task": "Once box is set up, ask Claude Code to use box to take a screenshot of the box and describe what is on it." + }, + { + "app": "claude-code-cli", + "setup": "paste-landing-line", + "alreadySetUp": "claude-ai-connector", + "title": "Claude Code CLI: the user pastes the landing line, but their claude.ai account already has the box connector", + "reset": [ + "pkill -x claude >/dev/null 2>&1; pkill -x xfce4-terminal >/dev/null 2>&1; pkill -x chrome >/dev/null 2>&1; true", + "npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1 || sudo npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1", + "for s in user local project; do claude mcp remove box -s $s >/dev/null 2>&1; done; true", + "sed -i '/^export ENABLE_CLAUDEAI_MCP_SERVERS=false$/d' ~/.bashrc", + "claude --version" + ], + "precheck": "cd ~ && test -s ~/.claude/.credentials.json && claude mcp list 2>/dev/null | grep -q '^claude.ai .*mcp.box.simular.ai.*Connected' && echo OK", + "precheckHelp": "Sign Claude Code in on the box with a claude.ai account that has the box by Simular connector.", + "handover": "You use Claude Code in a terminal on this computer. You found box by Simular and the landing page told you to paste this into your agent:\n\n Set up box from https://box.simular.ai/install", + "task": "Once box is set up, ask Claude Code to use box to take a screenshot of the box and describe what is on it." + } + ] } diff --git a/cli/e2e/layer3/cells.mjs b/cli/e2e/layer3/cells.mjs new file mode 100644 index 0000000..be0f956 --- /dev/null +++ b/cli/e2e/layer3/cells.mjs @@ -0,0 +1,45 @@ +// The dimensions a cell picks from, and the cells in cells.json. A cell's name is app/setup/alreadySetUp. +import fs from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +const HERE = path.dirname(fileURLToPath(import.meta.url)) + +/** Which app the user sets box up in. names: what the app calls itself on box's sign-in page. */ +export const APPS = { + 'claude-code-cli': { label: 'Claude Code CLI', signIn: 'click-allow-in-browser', names: ['Claude Code'] }, +} + +/** What the user does first. */ +export const SETUPS = { + 'paste-landing-line': 'Pastes "Set up box from https://box.simular.ai/install" into their agent, and the agent does the rest.', + 'click-open-in-button': 'Clicks "Open in Claude / Codex / Cursor" on the box site, which opens the app with the setup prompt filled in.', + 'click-add-to-claude-link': 'Clicks the one-tap "Add to Claude" link, which opens claude.ai connector settings with box filled in.', + 'click-install-in-vscode': 'Clicks the VS Code install link, then Install in VS Code.', + 'run-npx-connect': 'Runs npx @simular-ai/box connect in their own terminal.', + 'add-connector-in-settings': "Adds box by hand in the app's connector settings, following install.md.", + 'edit-app-config': "Follows install.md's manual steps for their app, such as claude mcp add or Codex's TOML file.", +} + +/** What is already there before the run starts. */ +export const ALREADY_SET_UP = { + nothing: 'box is not set up anywhere: no MCP entry, no claude.ai connector, no live grant.', + 'claude-ai-connector': "The user's claude.ai account already has the box by Simular connector, and the app loads it.", +} + +export const cellName = (cell) => `${cell.app}/${cell.setup}/${cell.alreadySetUp}` + +/** Load and check cells.json: every value must be one of the dimensions above, and names unique. */ +export function loadCells(file = path.join(HERE, 'cells.json')) { + const { cells } = JSON.parse(fs.readFileSync(file, 'utf8')) + const byName = new Map() + for (const cell of cells) { + for (const [field, allowed] of [['app', APPS], ['setup', SETUPS], ['alreadySetUp', ALREADY_SET_UP]]) { + if (!(cell[field] in allowed)) throw new Error(`cell "${cell.title}": ${field} "${cell[field]}" is not one of ${Object.keys(allowed).join(', ')}`) + } + const name = cellName(cell) + if (byName.has(name)) throw new Error(`two cells are named ${name}`) + byName.set(name, { name, ...cell }) + } + return byName +} diff --git a/cli/e2e/layer3/file.mjs b/cli/e2e/layer3/file.mjs index 8d85067..2f2cfd0 100644 --- a/cli/e2e/layer3/file.mjs +++ b/cli/e2e/layer3/file.mjs @@ -82,7 +82,7 @@ function issueBody(imp, judge, run, evidenceText, { isPrivate }) { '', '## Where it showed up', '', - `Run \`${run.runId}\`: cell \`${run.cell.name}\` (${run.cell.client}, ${run.clientVersion || 'version unknown'}, entry ${run.cell.entry}, auth ${run.cell.auth}). Verdict **${judge.verdict}**. Steps ${imp.steps.join(', ')}.`, + `${run.cell.title}. Cell \`${run.cell.name}\`, ${run.clientVersion || 'version unknown'}, run \`${run.runId}\`. Verdict **${judge.verdict}**. Steps ${imp.steps.join(', ')}.`, ...(reasons.length ? ['', 'Judge:', '', ...reasons] : []), '', '## Evidence', @@ -94,7 +94,7 @@ function issueBody(imp, judge, run, evidenceText, { isPrivate }) { } function seenAgainBody(imp, judge, run, evidenceText) { - return [`Seen again in run \`${run.runId}\` (${run.cell.name}, ${run.clientVersion || 'version unknown'}, verdict **${judge.verdict}**): ${imp.title}`, '', evidenceText, '', FOOTER].join('\n') + return [`Seen again in run \`${run.runId}\`. ${run.cell.title} (cell \`${run.cell.name}\`, ${run.clientVersion || 'version unknown'}, verdict **${judge.verdict}**). ${imp.title}`, '', evidenceText, '', FOOTER].join('\n') } export function fileRun(runDir, { dryRun = false } = {}) { diff --git a/cli/e2e/layer3/funnel.mjs b/cli/e2e/layer3/funnel.mjs index a303b23..135a291 100644 --- a/cli/e2e/layer3/funnel.mjs +++ b/cli/e2e/layer3/funnel.mjs @@ -1,18 +1,18 @@ // Server-side evidence for a run, from the test account's OAuth grants, and the run's cleanup. -// A grant belongs to the run when it was created inside the run's window by the cell's client, so a -// grant the account held before the run (another app, a claude.ai connector) is never touched. +// A grant belongs to the run when the cell's app created it inside the run's window, so a grant the +// account held before the run (another app, a claude.ai connector) is never touched. +import { APPS } from './cells.mjs' -/** Clients by the name their OAuth metadata gives box's consent page. */ -const CLIENT_NAMES = { 'claude-code': ['Claude Code'] } +const namesOf = (app) => APPS[app]?.names || [] -export function runGrants(grants, { client, startedAtMs, endedAtMs }) { - const names = CLIENT_NAMES[client] - return grants.filter((g) => g.createdAtMs >= startedAtMs && g.createdAtMs <= endedAtMs && (!names || names.includes(g.clientName))) +export function runGrants(grants, { app, startedAtMs, endedAtMs }) { + const names = namesOf(app) + return grants.filter((g) => g.createdAtMs >= startedAtMs && g.createdAtMs <= endedAtMs && names.includes(g.clientName)) } -/** Live grants the cell's client already holds. Consent renews such a grant instead of creating one. */ -export function liveGrantsOf(grants, client, nowMs) { - const names = CLIENT_NAMES[client] || [] +/** Live grants the cell's app already holds. Allowing again renews such a grant instead of creating one. */ +export function liveGrantsOf(grants, app, nowMs) { + const names = namesOf(app) return grants.filter((g) => names.includes(g.clientName) && (!g.expiresAtMs || g.expiresAtMs > nowMs)) } diff --git a/cli/e2e/layer3/run.mjs b/cli/e2e/layer3/run.mjs index ff47253..f1593f0 100644 --- a/cli/e2e/layer3/run.mjs +++ b/cli/e2e/layer3/run.mjs @@ -1,5 +1,5 @@ // Layer 3: an agent plays a new user onboarding box on a box, then a separate judge reviews the recording. -// node e2e/layer3/run.mjs reset, drive, extract, judge +// node e2e/layer3/run.mjs reset, drive, extract, judge, file; cell is app/setup/alreadySetUp // BOX_TOKEN comes from the environment or e2e/layer3/.env (see .env.example). // node e2e/layer3/run.mjs --judge judge an existing run again // --no-file skip filing issues with gh @@ -12,6 +12,7 @@ import { fileURLToPath } from 'node:url' import spawn from 'cross-spawn' import { DEFAULT_MCP_URL } from '../../src/constants.js' import { connectMcp } from '../mcp-client.mjs' +import { loadCells } from './cells.mjs' import { extractRun } from './extract.mjs' import { fileRun } from './file.mjs' import { funnelFrom, liveGrantsOf, runGrants } from './funnel.mjs' @@ -38,17 +39,17 @@ function textOf(result) { } async function drive(cellName) { - const cells = JSON.parse(read('cells.json')) - const cell = cells[cellName] - if (!cell) throw new Error(`unknown cell "${cellName}". Cells: ${Object.keys(cells).filter((k) => k !== '//').join(', ')}`) + const cells = loadCells() + const cell = cells.get(cellName) + if (!cell) throw new Error(`unknown cell "${cellName}". Cells:\n ${[...cells.keys()].join('\n ')}`) const key = process.env.BOX_TOKEN if (!key) throw new Error(`set BOX_TOKEN in ${path.join(HERE, '.env')} to a box key for the test account (https://box.simular.ai/keys)`) const url = process.env.BOX_MCP_URL || DEFAULT_MCP_URL - const runId = `${new Date().toISOString().replace(/[:.]/g, '-')}-${cellName}` + const runId = `${new Date().toISOString().replace(/[:.]/g, '-')}-${cellName.replaceAll('/', '--')}` const runDir = path.join(process.env.LAYER3_RUNS_DIR || path.join(os.homedir(), '.box-layer3', 'runs'), runId) fs.mkdirSync(runDir, { recursive: true }) - const run = { runId, cell: { name: cellName, ...cell }, mcpUrl: url, account: process.env.LAYER3_ACCOUNT || 'test-account', startedAt: new Date().toISOString(), funnel: 'unavailable' } + const run = { runId, cell, mcpUrl: url, account: process.env.LAYER3_ACCOUNT || 'test-account', startedAt: new Date().toISOString(), funnel: 'unavailable' } const finish = (extra) => { Object.assign(run, extra, { endedAt: new Date().toISOString() }) writeJson(runDir, 'run.json', run) @@ -59,8 +60,8 @@ async function drive(cellName) { const toolNames = (await mcp.listTools()).map((t) => t.name) const missing = [...DRIVER_TOOLS, 'shell'].filter((t) => !toolNames.includes(t)) if (missing.length) return finish({ blocked: `box MCP server lacks tools: ${missing.join(', ')}` }) - const held = liveGrantsOf(await listGrants(key), cell.client, Date.now()) - if (held.length) return finish({ blocked: `the test account already holds a live ${cell.client} grant (${held.map((g) => g.id).join(', ')}). Approving again would renew it instead of creating one, so the run could not tell its own consent apart. Revoke it at https://box.simular.ai/keys, then run again.` }) + const held = liveGrantsOf(await listGrants(key), cell.app, Date.now()) + if (held.length) return finish({ blocked: `the test account already holds a live ${cell.app} grant (${held.map((g) => g.id).join(', ')}). Approving again would renew it instead of creating one, so the run could not tell its own consent apart. Revoke it at https://box.simular.ai/keys, then run again.` }) const log = [] for (const command of [...cell.reset, cell.precheck]) { @@ -87,7 +88,7 @@ async function drive(cellName) { '--no-session-persistence', ...(process.env.LAYER3_DRIVER_MODEL ? ['--model', process.env.LAYER3_DRIVER_MODEL] : []), ] - console.log(`driving ${cellName} on ${url} (recording to ${runDir})`) + console.log(`driving ${cell.title}\n cell ${cellName} on ${url} (recording to ${runDir})`) const transcript = path.join(runDir, 'transcript.jsonl') const res = claude(driverArgs, { cwd: runDir, env: { ...process.env, BOX_TOKEN: key }, stdoutFile: transcript }) fs.writeFileSync(transcript, fs.readFileSync(transcript, 'utf8').split(key).join('***')) @@ -96,7 +97,7 @@ async function drive(cellName) { const { steps, final } = extractRun(runDir) run.driver = { exit: res.status, model: process.env.LAYER3_DRIVER_MODEL || 'default', steps: steps.length, ...final } - const mine = runGrants(await listGrants(key), { client: cell.client, startedAtMs: Date.parse(run.startedAt), endedAtMs: Date.now() }) + const mine = runGrants(await listGrants(key), { app: cell.app, startedAtMs: Date.parse(run.startedAt), endedAtMs: Date.now() }) writeJson(runDir, 'funnel.json', funnelFrom(mine)) run.funnel = 'funnel.json' run.revoked = [] @@ -135,7 +136,7 @@ function judge(runDir) { const argv = process.argv.slice(2) const noFile = argv.includes('--no-file') const args = argv.filter((a) => a !== '--no-file') -const runDir = args[0] === '--judge' ? path.resolve(args[1]) : await drive(args[0] || 'claude-code-landing') +const runDir = args[0] === '--judge' ? path.resolve(args[1]) : await drive(args[0] || 'claude-code-cli/paste-landing-line/nothing') const verdict = judge(runDir) console.log(`${verdict.verdict.toUpperCase()} ${runDir}`) for (const r of verdict.reasons) console.log(` - ${r.text} (steps ${r.steps.join(', ') || 'none'})`) diff --git a/cli/test/layer3-cells.test.js b/cli/test/layer3-cells.test.js new file mode 100644 index 0000000..40e8005 --- /dev/null +++ b/cli/test/layer3-cells.test.js @@ -0,0 +1,22 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import test from 'node:test' +import { loadCells } from '../e2e/layer3/cells.mjs' + +test('every cell in cells.json uses known values and has what a run needs', () => { + const cells = loadCells() + assert.ok(cells.size > 0) + for (const [name, cell] of cells) { + assert.match(name, /^[a-z0-9-]+\/[a-z0-9-]+\/[a-z0-9-]+$/) + for (const field of ['title', 'precheck', 'precheckHelp', 'handover', 'task']) assert.ok(cell[field], `${name}: ${field}`) + assert.ok(cell.reset.length, `${name}: reset`) + } +}) + +test('a cell with an unknown value is rejected by name', () => { + const file = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'cells-')), 'cells.json') + fs.writeFileSync(file, JSON.stringify({ cells: [{ app: 'claude-code-cli', setup: 'paste-the-line', alreadySetUp: 'nothing', title: 'x' }] })) + assert.throws(() => loadCells(file), /setup "paste-the-line" is not one of/) +}) diff --git a/cli/test/layer3-funnel.test.js b/cli/test/layer3-funnel.test.js index 0eeb5b0..955db01 100644 --- a/cli/test/layer3-funnel.test.js +++ b/cli/test/layer3-funnel.test.js @@ -3,9 +3,9 @@ import test from 'node:test' import { funnelFrom, liveGrantsOf, runGrants } from '../e2e/layer3/funnel.mjs' const grant = (id, clientName, createdAtMs, usedAtMs = 0) => ({ id, clientName, clientHost: '', createdAtMs, usedAtMs, maxBoxes: 2, boxIds: [] }) -const window = { client: 'claude-code', startedAtMs: 1000, endedAtMs: 2000 } +const window = { app: 'claude-code-cli', startedAtMs: 1000, endedAtMs: 2000 } -test('only grants the cell client created inside the run window belong to the run', () => { +test('only grants the cell app created inside the run window belong to the run', () => { const grants = [ grant('oag_before', 'Claude Code', 500, 1500), grant('oag_connector', 'Claude', 1200, 1300), @@ -15,9 +15,9 @@ test('only grants the cell client created inside the run window belong to the ru assert.deepEqual(runGrants(grants, window).map((g) => g.id), ['oag_run']) }) -test('a live grant the cell client already holds is found, an expired one or another client is not', () => { +test('a live grant the cell app already holds is found, an expired one or another client is not', () => { const grants = [{ ...grant('oag_live', 'Claude Code', 100), expiresAtMs: 9000 }, { ...grant('oag_old', 'Claude Code', 100), expiresAtMs: 500 }, { ...grant('oag_web', 'Claude', 100), expiresAtMs: 9000 }] - assert.deepEqual(liveGrantsOf(grants, 'claude-code', 1000).map((g) => g.id), ['oag_live']) + assert.deepEqual(liveGrantsOf(grants, 'claude-code-cli', 1000).map((g) => g.id), ['oag_live']) }) test('the funnel reports approval and first use from the run grants', () => {