diff --git a/.github/workflows/publish-cli.yml b/.github/workflows/publish-cli.yml index 34dc8d3..5bb2063 100644 --- a/.github/workflows/publish-cli.yml +++ b/.github/workflows/publish-cli.yml @@ -26,6 +26,7 @@ name: Publish CLI # The repo is public, so npm attaches a provenance attestation. # # TODO(layer 3): nightly run against the latest client versions plus a real agent task on a box. +# Design: cli/e2e/layer3/DESIGN.md (simular-ai/box-plugin#11). on: pull_request: diff --git a/cli/.gitignore b/cli/.gitignore index 6b1d0bf..d35f03d 100644 --- a/cli/.gitignore +++ b/cli/.gitignore @@ -1 +1,2 @@ LICENSE +.env diff --git a/cli/e2e/layer3/.env.example b/cli/e2e/layer3/.env.example new file mode 100644 index 0000000..add119f --- /dev/null +++ b/cli/e2e/layer3/.env.example @@ -0,0 +1,3 @@ +# Copy to .env and fill in. .env is gitignored. +# A box key for the test account, from https://box.simular.ai/keys +BOX_TOKEN= diff --git a/cli/e2e/layer3/DESIGN.md b/cli/e2e/layer3/DESIGN.md new file mode 100644 index 0000000..7c04e78 --- /dev/null +++ b/cli/e2e/layer3/DESIGN.md @@ -0,0 +1,217 @@ +# Layer 3: agent-driven onboarding runs + +Status: first cell built (`run.mjs`, `cells.json`, `driver.md`, `judge.md`), not yet run live. Tracks [simular-ai/box-plugin#11](https://github.com/simular-ai/box-plugin/issues/11). + +## Goal + +Find out when a new user can no longer get from "I want box" to a first working tool call, before a user tells us. The main cause we want to catch is upstream change: Claude Code, Codex, Cursor, Hermes and the desktop apps ship often, and any release can move a path, a button or an auth flow. Our own regressions are a secondary target. Layers 1 and 2 already cover the parts that don't change. + +A run must not depend on steps we wrote down in advance. Scripted steps break when a client's UI moves, so they test the script, not the onboarding. Instead an agent plays the new user and works the steps out each time from what a user would see. + +## Roles + +| Role | What it is | First version | +|---|---|---| +| Driver | An agent acting as the new user. It sees the screen, clicks and types. | Claude Code with the box MCP server | +| Machine under test | The "user's computer". The client gets installed here. | A box (Ubuntu desktop), reset from a snapshot per run | +| Client under test | The app the user is setting box up in | Claude Code CLI, then Codex CLI, Cursor, ChatGPT desktop (Linux) | +| Box under test | The box the client reaches over MCP after sign-in | The test account's box on prod | +| Judge | A separate agent that reads the recording afterwards | Claude Code, fresh session per run | + +Later versions add a Mac mini as the machine under test for macOS builds, and Parallels VMs on it for Windows and Linux. The roles and the run folder below stay the same. Only the driver's tool set changes. + +## Cells + +A cell is one test case: one way a user might set up box, starting from one state. Cells live in `cells.json`. `cells.mjs` holds the allowed values for each dimension and rejects a cell that uses any other value. + +### Dimensions + +Each cell picks one value for each of these three: + +| Field | Question it answers | +|---|---| +| `app` | Which app is the user setting box up in? | +| `setup` | What does the user do first? | +| `alreadySetUp` | What is already there before the run starts? | + +A cell's name is the three values joined with `/`, so the name reads as a sentence: `claude-code-cli/paste-landing-line/nothing` is "Claude Code CLI, the user pastes the landing line, nothing set up yet". Each cell also has a `title` with that sentence written out, which run output and filed issues show. + +Two more dimensions have only one value so far, so they are not fields yet. The machine is always a Linux box. The account is always the test account. + +#### `app` + +| Value | App | +|---|---| +| `claude-code-cli` | Claude Code, the command-line app | + +Each app in `cells.mjs` also records how it signs in to box and the name it shows on box's sign-in page. Sign-in is not a separate dimension, because it follows from the app. Most apps click Allow in the browser (OAuth). Hermes, OpenClaw and dsh paste a personal key instead. A cell names its sign-in only for an app that supports both. + +#### `setup` + +| Value | What the user does first | Apps it applies to | +|---|---|---| +| `paste-landing-line` | Pastes `Set up box from https://box.simular.ai/install` into their agent, and the agent does the rest | Claude Code, Codex, Cursor, Gemini CLI, OpenCode, any coding agent. claude.ai and ChatGPT chat reply with the steps instead of installing | +| `click-open-in-button` | Clicks "Open in Claude / Codex / Cursor" on the box site, which opens the app with the setup prompt filled in | Claude Code (`claude://`), Codex (`codex://`), Cursor (`cursor.com/link`) | +| `click-add-to-claude-link` | Clicks the one-tap "Add to Claude" link, which opens claude.ai connector settings with box filled in | claude.ai, the Claude desktop app | +| `click-install-in-vscode` | Clicks the VS Code install link, then Install in VS Code | VS Code | +| `run-npx-connect` | Runs `npx @simular-ai/box connect` in their own terminal | Claude Code, Codex, Cursor, OpenCode, Gemini CLI, Hermes, OpenClaw, VS Code | +| `add-connector-in-settings` | Adds box by hand in the app's connector settings, following install.md | claude.ai, the Claude desktop app, ChatGPT | +| `edit-app-config` | Follows install.md's manual steps for their app, such as `claude mcp add`, Codex's TOML file or Cursor's JSON file | Claude Code, Codex, Cursor, Gemini CLI, dsh, any MCP app | + +`setup` is what the user does first, not every step that follows. In runs 3 and 5, Claude Code was blocked from running `npx` and fell back to `claude mcp add`. That run is still `paste-landing-line`. The fallback is something the judge reports from the recording. + +#### `alreadySetUp` + +| Value | What is there before the run | +|---|---| +| `nothing` | box is not set up anywhere: no MCP entry, no claude.ai connector, no live grant | +| `claude-ai-connector` | The user's claude.ai account already has the box by Simular connector, and the app loads it | + +Values to add next, from issue #11: `second-codex-on-path`, `old-box-entry`, `another-server-named-box`, `two-google-accounts`, `expired-access`. + +### What a cell holds + +Besides its three values and its title, a cell has: + +- `reset`. Shell commands the harness runs on the box to reach the cell's starting state, such as installing the latest app or turning claude.ai connectors off. +- `precheck`. A command that must print `OK`. If it doesn't, the run stops as `blocked` instead of testing the wrong thing. +- `precheckHelp`. What a person has to fix by hand when the precheck fails. +- `handover`. The user's situation, as the driver hears it. This is all the driver knows about how to start. +- `task`. The one thing the user wants to do with box once it works. + +The steps between `handover` and `task` are not written down on purpose. The driver works them out from the screen on each run, so an upstream change shows up as the driver getting stuck. + +### Cells today + +| Name | Title | +|---|---| +| `claude-code-cli/paste-landing-line/nothing` | Claude Code CLI: a new user pastes the landing line, with nothing set up yet | +| `claude-code-cli/paste-landing-line/claude-ai-connector` | Claude Code CLI: the user pastes the landing line, but their claude.ai account already has the box connector | + +To add a cell, add an entry to `cells.json`. A new `app` value also needs an entry in `cells.mjs` with the name the app shows on box's sign-in page, so the run can find its grant. + +## How a run works + +1. **Reset.** Run the cell's `reset` commands on the box, then its `precheck`. +2. **Hand over.** Give the driver the cell's `handover` and `task`, and the rules below. +3. **Act.** The driver works through the onboarding on the box's screen until it gets a reply to one real task through box, or decides it is stuck. +4. **Collect.** The harness saves the run folder, including the server-side records for the run. +5. **Judge.** A fresh judge agent reads the run folder and writes a verdict and improvements. +6. **File.** Improvements become issues or draft PRs (see Filing). +7. **Clean up.** Revoke the run's grant and delete anything the run created on the test account. + +### Driver rules + +The driver must act as a user would. It controls the box through `screenshot`, `computer_act` and `computer_state`. It types into the box's own terminal window when a user would use a terminal (for example, `npx @simular-ai/box connect`), so the step is on screen and in the recording. + +The driver must not: + +- use `shell`, `write_file`, `edit_file` or `browser_*` during the user flow. These tools are for steps 1 and 7 only, run by the harness. +- edit config files by hand, read our source code, or use any URL that the user-facing pages don't show. +- retry the same failing step more than three times. + +If the driver breaks a rule to get past a problem, it must say so in a step note. The judge counts every such step as a workaround. + +## Recording + +The recording must not depend on the driver remembering to take notes. The driver is a headless `claude -p` session with `--output-format stream-json`, which writes every tool call, its arguments, its result and every screenshot to `transcript.jsonl`. `extract.mjs` turns that into `steps.jsonl` and `screenshots/`. The driver's notes ("I clicked Login because...") are the text it writes before each call. + +The driver rules are enforced by the CLI, not only by the prompt. `--tools ""` removes every built-in tool, `--strict-mcp-config` loads only the box server, `--allowedTools` allows only `screenshot`, `computer_act` and `computer_state`, and `--disallowedTools` denies every other box tool. A denied attempt still shows in the transcript, so the judge sees it. + +Runs are written to `~/.box-layer3/runs/` (or `LAYER3_RUNS_DIR`), outside the repo, because they show the test account's screen. + +### Run folder + +``` +runs// + run.json the cell (name, title, values), the app's version, driver model, + account label, start and end time, the driver's own claim, + and the grants the run revoked + reset.log the reset and precheck commands and what they printed + steps.jsonl one line per tool call: index, time, tool, arguments, result + summary, screenshot file, driver note + screenshots/ 000.png, 001.png, ... + transcript.jsonl the driver's full Claude Code transcript + funnel.json server-side records for this run's user and time window + judge.json written by the judge +``` + +The run folder is the contract between the driver and the judge. Either side can change as long as the folder keeps this shape. + +### Server-side funnel + +`funnel.json` is what box's server saw for the run. It comes from box's list of apps connected to the test account (cocoonbox `Grants.List`). + +- `approved` is true when someone clicked Allow on box's sign-in page during the run, for the cell's app. +- `used` is true when the app then sent at least one request to box. +- `grants` lists the grants the run created, with when each was allowed and first used. + +A grant counts as the run's own when the cell's app created it during the run. A grant from before the run, such as a claude.ai connector, never counts. After the run, the harness revokes the run's own grants. A run stops as `blocked` when the cell's app already holds a live grant, because clicking Allow again renews that grant instead of creating one. + +`used` does not mean a tool ran. Connecting counts as a request. box does not record which tools an app called, so the screenshots still show whether the task ran. + +## Judge + +The judge is a fresh agent. It gets only the run folder, never the driver's live session, so the driver's claim can't sway it. It writes `judge.json`: + +```json +{ + "verdict": "pass | fail | blocked", + "reasons": [{ "text": "...", "steps": [12, 13] }], + "workarounds": [{ "text": "...", "steps": [20] }], + "improvements": [ + { + "target": "install.md | cli | portal | client-docs | harness", + "title": "...", + "detail": "...", + "steps": [8, 9], + "proposed": "issue | draft-pr" + } + ] +} +``` + +Verdicts: + +- **pass.** A screenshot shows the app's reply to the task through box. The driver used no workarounds. The user saw an example prompt or a clear next step after connecting. +- **fail.** The onboarding or its docs failed a user. Only this verdict files product issues. +- **blocked.** The harness failed: the box didn't wake, the snapshot didn't restore, or the model hit a limit. It files nothing about the product, so harness problems don't fill the tracker. + +Any workaround makes the run a fail, even when the funnel completed. An agent that hand-edits a config still connects, but a real user would not. + +## Filing + +Recordings show the test account's email, the Google account chooser and sometimes tokens. This repo is public, so evidence never goes here. + +- Issues go to `simular-ai/box-portal` (private) with screenshots and the relevant transcript lines. Run folders live in private storage, and issues link to them. +- Before filing, the judge searches open issues for the same failure. If it finds one, it adds a comment ("seen again in run X") instead of opening a new issue. +- At most three new issues per run. +- PRs only for doc and copy fixes, for example install.md. They open as drafts and never merge without review. +- `harness` improvements (driver prompt, seeds, judge rubric) go to this repo, with no screenshots. + +## Test account + +A team member's account is the test account for now. It moves to `dev@simular.ai` later. The account is one config value, so the switch is one line. + +Until then: + +- Cleanup (step 7) matters more, because runs create grants on a real account. +- Free-tier cells (caps, `max_boxes`) wait for the dedicated account, unless the current account is on the free tier. +- The box's Chrome profile is signed in only to the test account, so no other account appears in screenshots. + +## Next cells + +In order: + +1. `codex-cli/paste-landing-line/second-codex-on-path`. A stock box already has `codex` and `claude` in `/usr/local/bin`, so this starting state is the box as it is today. +2. `claude-code-cli/paste-landing-line/two-google-accounts`, checking that the grant goes to the intended account. +3. ChatGPT desktop (Linux preview) with `click-open-in-button`. +4. A nightly schedule over the cells, with an alert when a scheduled run posts no result. + +## Open questions + +1. **One box or two.** Settled for now: the client under test reaches the same box the driver is using (run 1 described its own terminal). A read-only first task, a screenshot, is safe on one box. +2. **Where the driver runs.** A local Claude Code session for the first run. For nightly runs, a GitHub workflow or the Mac mini. A GitHub workflow needs a box key or OAuth token as a secret. +3. **Pass rate.** One run is a sample. How many runs per cell per night, and what pass rate counts as green? +4. **Which tool was called.** `funnel.json` comes from `Grants.List`: approval time and first token use. box api does not expose token exchange or `tools/call` per grant yet, so the screenshots still decide whether the task ran. +5. **Sign-in.** Settled for now: the box's Chrome is signed in to the test account once by hand, and the consent page then needs one click. box has no way to fill a web password without the agent seeing it. diff --git a/cli/e2e/layer3/FINDINGS.md b/cli/e2e/layer3/FINDINGS.md new file mode 100644 index 0000000..921e494 --- /dev/null +++ b/cli/e2e/layer3/FINDINGS.md @@ -0,0 +1,40 @@ +# Layer 3 findings + +Problems found while building and running layer 3, before the judge files issues on its own. Each entry says where it came from. Recordings live in `~/.box-layer3/runs/` on the machine that ran them, not in this repo. + +## Onboarding (product) + +| # | Finding | Target | Source | +|---|---|---|---| +| 1 | Claude Code's auto mode blocks `npx @simular-ai/box connect` as "Code from External". Claude Code falls back to the manual `claude mcp add` path in install.md, which works, so the main path fails under Claude Code's default mode. | install.md, cli | run `2026-11-05T06-23-49-887Z-claude-code-landing`, steps 5 and 6 | +| 2 | After adding box, the user is told to run `/mcp` and authenticate. `/mcp` in the same session shows "No MCP servers configured". Claude Code needs a restart first, and nothing says so. | install.md | same run, steps 7 to 9 | +| 3 | On box.simular.ai/sign-in, closing the Google sign-in popup leaves both sign-in buttons spinning until the page is reloaded. | portal | same run, steps 14 to 16 | +| 2a | Seen again with box fully working: the agent's reply says "Type /mcp here" as step 1 and "Start a new chat" as step 3. The driver had to guess the restart, so the run failed on a workaround although box connected and the task succeeded. | install.md | run `2026-11-05T06-36-07-346Z-claude-code-landing`, steps 8 to 13 | +| 13 | install.md and the agent's reply say "click **Approve**", but the consent page's button says **Allow**. | install.md | same run, steps 8 and 16 | +| 14 | After `/exit` and relaunch, Claude Code asked to trust `/home/user` again, although it had been trusted minutes earlier. Possibly upstream Claude Code behaviour, not yet confirmed. | client-docs | same run, step 12; also run `06-23-49`, step 9 | +| 4 | Fetching the install URL, Claude Code's first probe showed a red "Exit code 127" next to the HTTP 200 headers. It recovered, but a new user sees an error on the first step. | install.md | run `2026-11-05T06-20-57-248Z-claude-code-landing`, step 5 | + +Filed automatically by run `2026-11-05T06-58-25-137Z-claude-code-landing`: box-portal#152 (`box connect` should print the restart-first order; same root cause as #147, fixed in the CLI) and box-portal#153 (the consent page shows no confirmation after Allow, so a second click lands on a dead localhost callback). The same run commented "seen again" on box-portal#147. + +Filed by hand in simular-ai/box-portal (private): #1 as box-portal#146, #2 and #2a as box-portal#147, #3 as box-portal#148, #4 as box-portal#149, #13 as box-portal#150. #14 is not filed until it is confirmed. + +## box platform + +| # | Finding | Target | Source | +|---|---|---|---| +| 5 | No password filling for web pages. box has a secrets vault for **API keys** (box-portal `src/secrets/`, cocoonbox `Keys`): a box holds a stand-in in an env var and its egress swaps in the real value on the key's hosts only. It cannot fill a password into a sign-in page without the agent seeing it. Sign-in for tests relies on a Chrome profile signed in once by hand. | box | box-portal source and `keys.proto`, 2026-11-05 | +| 6 | Copy and paste don't work in the screen view at box.simular.ai/box. | portal | reported by hand while signing the box's Chrome in, 2026-11-05 | + +#5 is covered by box-portal#72 (sign in to sites in the box without leaving the app). #6 is not filed yet. + +## Harness + +| # | Finding | Status | +|---|---|---| +| 7 | Claude Code on the box loads the signed-in claude.ai account's connectors, including box, so a "clean" cell never installs anything. | Fixed: the clean cell sets `ENABLE_CLAUDEAI_MCP_SERVERS=false`. The connector path is now its own cell. | +| 8 | The driver typed ahead into a restarting Claude Code, and the Enter landed on the trust prompt, whose default is "No, exit". | Fixed: driver rule against typing ahead. | +| 9 | OAuth cells need a browser identity. The box's Chrome had no Google or GitHub account signed in, so sign-in needed a password. | Fixed by hand: Chrome on the box is signed in to the test account. | +| 9a | A box OAuth tab left open in Chrome from signing in by hand was still there when the next run started. | Fixed: reset closes Chrome. The profile, and its sign-in, stays. | +| 10 | No `funnel.json`. The judge decided from screenshots alone. | Fixed: `funnel.json` from cocoonbox `Grants.List` shows approval and first token use. Which tool was called still needs box api. | +| 11 | Runs left their OAuth grant on the test account. | Fixed: the run revokes the grants its client created inside the run window. A run is blocked if the client already holds a live grant, because consent would renew it. | +| 12 | The judge proposed issues but nothing filed them. | Fixed: `file.mjs` files with `gh`, dedupes against open issues, caps new issues at three per run. | diff --git a/cli/e2e/layer3/cells.json b/cli/e2e/layer3/cells.json new file mode 100644 index 0000000..53c27a0 --- /dev/null +++ b/cli/e2e/layer3/cells.json @@ -0,0 +1,39 @@ +{ + "//": "Layer 3 cells. A cell's name is app/setup/alreadySetUp; the allowed values and what they mean are in cells.mjs. reset runs on the box over its shell tool before the driver starts, and precheck must print OK or the run is blocked. Claude Code on the box loads the signed-in claude.ai account's connectors, so a cell with nothing set up turns them off with ENABLE_CLAUDEAI_MCP_SERVERS=false in ~/.bashrc.", + "cells": [ + { + "app": "claude-code-cli", + "setup": "paste-landing-line", + "alreadySetUp": "nothing", + "title": "Claude Code CLI: a new user pastes the landing line, with nothing set up yet", + "reset": [ + "pkill -x claude >/dev/null 2>&1; pkill -x xfce4-terminal >/dev/null 2>&1; pkill -x chrome >/dev/null 2>&1; true", + "npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1 || sudo npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1", + "for s in user local project; do claude mcp remove box -s $s >/dev/null 2>&1; done; true", + "grep -q '^export ENABLE_CLAUDEAI_MCP_SERVERS=false$' ~/.bashrc || echo 'export ENABLE_CLAUDEAI_MCP_SERVERS=false' >> ~/.bashrc", + "claude --version" + ], + "precheck": "cd ~ && test -s ~/.claude/.credentials.json && grep -q '^export ENABLE_CLAUDEAI_MCP_SERVERS=false$' ~/.bashrc && ! ENABLE_CLAUDEAI_MCP_SERVERS=false claude mcp list 2>/dev/null | grep -q 'mcp.box.simular.ai' && echo OK", + "precheckHelp": "Sign Claude Code in on the box once by hand (open a terminal on the box, run claude, finish /login). The run keeps that sign-in, turns off claude.ai connectors, and removes any local box MCP entry.", + "handover": "You use Claude Code in a terminal on this computer. You found box by Simular and the landing page told you to paste this into your agent:\n\n Set up box from https://box.simular.ai/install", + "task": "Once box is set up, ask Claude Code to use box to take a screenshot of the box and describe what is on it." + }, + { + "app": "claude-code-cli", + "setup": "paste-landing-line", + "alreadySetUp": "claude-ai-connector", + "title": "Claude Code CLI: the user pastes the landing line, but their claude.ai account already has the box connector", + "reset": [ + "pkill -x claude >/dev/null 2>&1; pkill -x xfce4-terminal >/dev/null 2>&1; pkill -x chrome >/dev/null 2>&1; true", + "npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1 || sudo npm install -g @anthropic-ai/claude-code@latest >/dev/null 2>&1", + "for s in user local project; do claude mcp remove box -s $s >/dev/null 2>&1; done; true", + "sed -i '/^export ENABLE_CLAUDEAI_MCP_SERVERS=false$/d' ~/.bashrc", + "claude --version" + ], + "precheck": "cd ~ && test -s ~/.claude/.credentials.json && claude mcp list 2>/dev/null | grep -q '^claude.ai .*mcp.box.simular.ai.*Connected' && echo OK", + "precheckHelp": "Sign Claude Code in on the box with a claude.ai account that has the box by Simular connector.", + "handover": "You use Claude Code in a terminal on this computer. You found box by Simular and the landing page told you to paste this into your agent:\n\n Set up box from https://box.simular.ai/install", + "task": "Once box is set up, ask Claude Code to use box to take a screenshot of the box and describe what is on it." + } + ] +} diff --git a/cli/e2e/layer3/cells.mjs b/cli/e2e/layer3/cells.mjs new file mode 100644 index 0000000..be0f956 --- /dev/null +++ b/cli/e2e/layer3/cells.mjs @@ -0,0 +1,45 @@ +// The dimensions a cell picks from, and the cells in cells.json. A cell's name is app/setup/alreadySetUp. +import fs from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +const HERE = path.dirname(fileURLToPath(import.meta.url)) + +/** Which app the user sets box up in. names: what the app calls itself on box's sign-in page. */ +export const APPS = { + 'claude-code-cli': { label: 'Claude Code CLI', signIn: 'click-allow-in-browser', names: ['Claude Code'] }, +} + +/** What the user does first. */ +export const SETUPS = { + 'paste-landing-line': 'Pastes "Set up box from https://box.simular.ai/install" into their agent, and the agent does the rest.', + 'click-open-in-button': 'Clicks "Open in Claude / Codex / Cursor" on the box site, which opens the app with the setup prompt filled in.', + 'click-add-to-claude-link': 'Clicks the one-tap "Add to Claude" link, which opens claude.ai connector settings with box filled in.', + 'click-install-in-vscode': 'Clicks the VS Code install link, then Install in VS Code.', + 'run-npx-connect': 'Runs npx @simular-ai/box connect in their own terminal.', + 'add-connector-in-settings': "Adds box by hand in the app's connector settings, following install.md.", + 'edit-app-config': "Follows install.md's manual steps for their app, such as claude mcp add or Codex's TOML file.", +} + +/** What is already there before the run starts. */ +export const ALREADY_SET_UP = { + nothing: 'box is not set up anywhere: no MCP entry, no claude.ai connector, no live grant.', + 'claude-ai-connector': "The user's claude.ai account already has the box by Simular connector, and the app loads it.", +} + +export const cellName = (cell) => `${cell.app}/${cell.setup}/${cell.alreadySetUp}` + +/** Load and check cells.json: every value must be one of the dimensions above, and names unique. */ +export function loadCells(file = path.join(HERE, 'cells.json')) { + const { cells } = JSON.parse(fs.readFileSync(file, 'utf8')) + const byName = new Map() + for (const cell of cells) { + for (const [field, allowed] of [['app', APPS], ['setup', SETUPS], ['alreadySetUp', ALREADY_SET_UP]]) { + if (!(cell[field] in allowed)) throw new Error(`cell "${cell.title}": ${field} "${cell[field]}" is not one of ${Object.keys(allowed).join(', ')}`) + } + const name = cellName(cell) + if (byName.has(name)) throw new Error(`two cells are named ${name}`) + byName.set(name, { name, ...cell }) + } + return byName +} diff --git a/cli/e2e/layer3/driver.md b/cli/e2e/layer3/driver.md new file mode 100644 index 0000000..193b99f --- /dev/null +++ b/cli/e2e/layer3/driver.md @@ -0,0 +1,32 @@ +You are testing an onboarding flow by acting as a new user on a Linux desktop. You control the desktop only through its screen: take screenshots, click, type and press keys. You have no shell and no file access, on purpose. A real user has only the screen, and this test exists to find out whether the screen is enough. + +## Your situation + +{{handover}} + +## Your goal + +{{task}} + +## How to act + +- Do what a reasonable new user would do with what is on the screen and what the pages and apps tell you. Read instructions as you meet them. Do not use anything you know about how box works inside. +- When a user would use a terminal, open the terminal window on the desktop and type there. +- When a browser page asks you to sign in or approve access, do it if the account is already signed in or offered for one click. Never type a password. If a password is needed, stop and report it. +- Before each action, say in one short sentence what you see and why you are doing it. These notes are the record a reviewer reads later. +- Do not type ahead into a program that is still starting. Take a screenshot first, because a prompt you didn't expect may take the keys. +- If the same step fails three times, stop. + +## What not to do + +- Do not hand-edit config files or settings files, even if you could open them in an editor on screen. +- Do not guess URLs that no page or app showed you. +- If you do anything a new user would not, say so plainly in your note with the word WORKAROUND. + +## When you finish + +End your final message with exactly one line: + + RESULT: done +or + RESULT: stuck: diff --git a/cli/e2e/layer3/extract.mjs b/cli/e2e/layer3/extract.mjs new file mode 100644 index 0000000..3031683 --- /dev/null +++ b/cli/e2e/layer3/extract.mjs @@ -0,0 +1,63 @@ +// Turn a Claude Code stream-json transcript into the run folder's steps.jsonl and screenshots/. +import fs from 'node:fs' +import path from 'node:path' + +const EXT = { 'image/png': 'png', 'image/jpeg': 'jpg', 'image/webp': 'webp', 'image/gif': 'gif' } +const blocks = (m) => (Array.isArray(m?.message?.content) ? m.message.content : []) + +function summarize(content) { + if (typeof content === 'string') return content.slice(0, 2000) + return (content || []).filter((b) => b.type === 'text').map((b) => b.text).join('\n').slice(0, 2000) +} + +/** Parse stream-json lines into { steps, images, final }. Pure: no file access. */ +export function parseTranscript(lines) { + const steps = [] + const byId = new Map() + const images = [] + let note = '' + let final = null + for (const line of lines) { + if (!line.trim()) continue + const msg = JSON.parse(line) + if (msg.type === 'assistant') { + for (const b of blocks(msg)) { + if (b.type === 'text') note = note ? `${note}\n${b.text}` : b.text + if (b.type === 'tool_use') { + const step = { i: steps.length, tool: b.name, input: b.input, note: note.trim(), result: null, isError: false, screenshots: [] } + steps.push(step) + byId.set(b.id, step) + note = '' + } + } + } else if (msg.type === 'user') { + for (const b of blocks(msg)) { + if (b.type !== 'tool_result') continue + const step = byId.get(b.tool_use_id) + if (!step) continue + step.result = summarize(b.content) + step.isError = Boolean(b.is_error) + for (const c of Array.isArray(b.content) ? b.content : []) { + if (c.type !== 'image' || c.source?.type !== 'base64') continue + const file = `screenshots/${String(step.i).padStart(3, '0')}${step.screenshots.length ? `-${step.screenshots.length}` : ''}.${EXT[c.source.media_type] || 'png'}` + step.screenshots.push(file) + images.push({ file, data: c.source.data }) + } + } + } else if (msg.type === 'result') { + final = { text: msg.result ?? '', subtype: msg.subtype, turns: msg.num_turns, costUsd: msg.total_cost_usd, durationMs: msg.duration_ms } + } + } + const claim = /RESULT:\s*(done|stuck(?::.*)?)\s*$/im.exec(final?.text || '')?.[1] ?? null + return { steps, images, final: final && { ...final, claim, trailingNote: note.trim() || undefined } } +} + +/** Write steps.jsonl and screenshots/ into runDir from its transcript.jsonl. */ +export function extractRun(runDir) { + const lines = fs.readFileSync(path.join(runDir, 'transcript.jsonl'), 'utf8').split('\n') + const parsed = parseTranscript(lines) + fs.mkdirSync(path.join(runDir, 'screenshots'), { recursive: true }) + for (const img of parsed.images) fs.writeFileSync(path.join(runDir, img.file), Buffer.from(img.data, 'base64')) + fs.writeFileSync(path.join(runDir, 'steps.jsonl'), parsed.steps.map((s) => JSON.stringify(s)).join('\n') + (parsed.steps.length ? '\n' : '')) + return parsed +} diff --git a/cli/e2e/layer3/file.mjs b/cli/e2e/layer3/file.mjs new file mode 100644 index 0000000..2f2cfd0 --- /dev/null +++ b/cli/e2e/layer3/file.mjs @@ -0,0 +1,157 @@ +// File a judged run's improvements as GitHub issues with the local `gh`, once per run. +// node e2e/layer3/file.mjs [--dry-run] +// Product targets go to box-portal (private) with screenshots; `harness` goes to box-plugin (public) +// as text only; `client-docs` is listed, never filed. A repeat of an open issue becomes a comment. +import fs from 'node:fs' +import path from 'node:path' +import spawn from 'cross-spawn' + +const PRIVATE_REPO = process.env.LAYER3_ISSUES_REPO || 'simular-ai/box-portal' +const PUBLIC_REPO = process.env.LAYER3_HARNESS_REPO || 'simular-ai/box-plugin' +const EVIDENCE_BRANCH = 'layer3-evidence' +const MAX_NEW = 3 +const ROUTES = { 'install.md': PRIVATE_REPO, portal: PRIVATE_REPO, cli: PRIVATE_REPO, harness: PUBLIC_REPO } +const LABELS = { 'install.md': 'documentation', portal: 'bug', cli: 'bug', harness: 'enhancement' } +const FOOTER = '🤖 Filed by the layer 3 onboarding runs (simular-ai/box-plugin#11) with [Claude Code](https://claude.com/claude-code)' + +function run(cmd, args, { input, allowFail = false } = {}) { + const res = spawn.sync(cmd, args, { input, encoding: 'utf8', maxBuffer: 1 << 26 }) + if (res.error) throw res.error + if (res.status !== 0 && !allowFail) throw new Error(`${cmd} ${args.slice(0, 3).join(' ')} failed: ${res.stderr || res.stdout}`) + return res +} +const gh = (args, opts) => run('gh', args, opts) +const readJson = (dir, name) => JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8')) +const readSteps = (dir) => fs.readFileSync(path.join(dir, 'steps.jsonl'), 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l)) + +/** Ask a fresh model which open issue, if any, each improvement repeats. */ +function matchOpenIssues(improvements, openIssues) { + if (!openIssues.length) return improvements.map(() => null) + const schema = { type: 'object', additionalProperties: false, required: ['matches'], properties: { matches: { type: 'array', items: { type: ['integer', 'null'] } } } } + const prompt = [ + 'For each proposed improvement, return the number of the open GitHub issue that describes the same problem, or null if none does. Same problem means a maintainer fixing that issue would also fix the improvement. Answer with one entry per improvement, in order.', + '', + 'Improvements:', + ...improvements.map((imp, i) => `${i}. [${imp.target}] ${imp.title}\n ${imp.detail}`), + '', + 'Open issues:', + ...openIssues.map((iss) => `#${iss.number} ${iss.title}\n ${(iss.body || '').replace(/\s+/g, ' ').slice(0, 400)}`), + ].join('\n') + const res = run('claude', ['-p', prompt, '--tools', '', '--strict-mcp-config', '--output-format', 'json', '--json-schema', JSON.stringify(schema), '--no-session-persistence']) + const out = JSON.parse(res.stdout) + const { matches } = out.structured_output ?? JSON.parse(out.result) + const known = new Set(openIssues.map((i) => i.number)) + return improvements.map((_, i) => (known.has(matches[i]) ? matches[i] : null)) +} + +function ensureEvidenceBranch() { + const ref = gh(['api', `repos/${PRIVATE_REPO}/git/ref/heads/${EVIDENCE_BRANCH}`], { allowFail: true }) + if (ref.status === 0) return + const base = JSON.parse(gh(['api', `repos/${PRIVATE_REPO}`]).stdout).default_branch + const sha = JSON.parse(gh(['api', `repos/${PRIVATE_REPO}/git/ref/heads/${base}`]).stdout).object.sha + gh(['api', '-X', 'POST', `repos/${PRIVATE_REPO}/git/refs`, '-f', `ref=refs/heads/${EVIDENCE_BRANCH}`, '-f', `sha=${sha}`]) +} + +/** Upload one screenshot to the evidence branch and return a link that renders for repo readers. */ +function uploadScreenshot(runDir, runId, file) { + const dest = `layer3/${runId}/${file}` + const body = JSON.stringify({ message: `layer3 evidence: ${runId} ${file}`, branch: EVIDENCE_BRANCH, content: fs.readFileSync(path.join(runDir, file)).toString('base64') }) + const res = gh(['api', '-X', 'PUT', `repos/${PRIVATE_REPO}/contents/${dest}`, '--input', '-'], { input: body, allowFail: true }) + if (res.status !== 0 && !/sha/i.test(res.stdout + res.stderr)) throw new Error(`upload ${dest}: ${res.stderr || res.stdout}`) + return `https://github.com/${PRIVATE_REPO}/blob/${EVIDENCE_BRANCH}/${dest}?raw=true` +} + +function evidence(runDir, run, steps, cited, { withScreenshots }) { + const lines = [] + for (const i of [...new Set(cited)].sort((a, b) => a - b)) { + const step = steps[i] + if (!step) continue + lines.push(`**Step ${i}.** ${step.note || '(no note)'}`) + if (withScreenshots) for (const file of step.screenshots) lines.push(`![step ${i}](${uploadScreenshot(runDir, run.runId, file)})`) + } + return lines.join('\n\n') +} + +/** Mask email addresses: recordings show the test account's screen. */ +export const redact = (text) => text.replace(/[\w.+-]+@[\w-]+(\.[\w-]+)+/g, '[email]') + +function issueBody(imp, judge, run, evidenceText, { isPrivate }) { + const reasons = isPrivate ? judge.reasons.filter((r) => r.steps.some((s) => imp.steps.includes(s))).map((r) => `- ${r.text}`) : [] + return [ + imp.detail, + '', + '## Where it showed up', + '', + `${run.cell.title}. Cell \`${run.cell.name}\`, ${run.clientVersion || 'version unknown'}, run \`${run.runId}\`. Verdict **${judge.verdict}**. Steps ${imp.steps.join(', ')}.`, + ...(reasons.length ? ['', 'Judge:', '', ...reasons] : []), + '', + '## Evidence', + '', + evidenceText || (isPrivate ? '_No screenshots attached._' : '_This repo is public, so the recording stays private. Ask in #box for the run folder._'), + '', + FOOTER, + ].join('\n') +} + +function seenAgainBody(imp, judge, run, evidenceText) { + return [`Seen again in run \`${run.runId}\`. ${run.cell.title} (cell \`${run.cell.name}\`, ${run.clientVersion || 'version unknown'}, verdict **${judge.verdict}**). ${imp.title}`, '', evidenceText, '', FOOTER].join('\n') +} + +export function fileRun(runDir, { dryRun = false } = {}) { + const filedPath = path.join(runDir, 'filed.json') + if (fs.existsSync(filedPath)) return JSON.parse(fs.readFileSync(filedPath, 'utf8')) + const run = readJson(runDir, 'run.json') + const judge = readJson(runDir, 'judge.json') + const steps = fs.existsSync(path.join(runDir, 'steps.jsonl')) ? readSteps(runDir) : [] + const results = [] + if (judge.verdict === 'blocked') { + results.push(...judge.improvements.filter((i) => i.target === 'harness').map((imp) => ({ imp, action: 'skipped', why: 'blocked runs file only harness issues, and only by hand' }))) + } + const fileable = judge.verdict === 'blocked' ? [] : judge.improvements + const byRepo = new Map() + for (const imp of fileable) { + const repo = ROUTES[imp.target] + if (!repo) { + results.push({ imp, action: 'listed', why: `${imp.target} is for the client's maker` }) + continue + } + if (!byRepo.has(repo)) byRepo.set(repo, []) + byRepo.get(repo).push(imp) + } + + let created = 0 + for (const [repo, imps] of byRepo) { + const open = JSON.parse(gh(['issue', 'list', '--repo', repo, '--state', 'open', '--limit', '200', '--json', 'number,title,body']).stdout) + const matches = matchOpenIssues(imps, open) + const isPrivate = repo === PRIVATE_REPO + if (isPrivate && !dryRun && imps.some((imp) => imp.steps.length)) ensureEvidenceBranch() + imps.forEach((imp, i) => { + const ev = isPrivate ? evidence(runDir, run, steps, imp.steps, { withScreenshots: !dryRun }) : '' + if (matches[i]) { + if (!dryRun) gh(['issue', 'comment', String(matches[i]), '--repo', repo, '--body-file', '-'], { input: redact(seenAgainBody(imp, judge, run, ev)) }) + results.push({ imp, action: 'commented', repo, issue: matches[i] }) + } else if (created >= MAX_NEW) { + results.push({ imp, action: 'skipped', why: `cap of ${MAX_NEW} new issues per run` }) + } else { + created++ + let url = null + if (!dryRun) { + url = gh(['issue', 'create', '--repo', repo, '--title', redact(imp.title), '--label', LABELS[imp.target], '--body-file', '-'], { input: redact(issueBody(imp, judge, run, ev, { isPrivate })) }).stdout.trim() + } + results.push({ imp, action: 'created', repo, url }) + } + }) + } + + const summary = results.map(({ imp, ...r }) => ({ target: imp.target, title: imp.title, ...r })) + if (!dryRun) fs.writeFileSync(filedPath, JSON.stringify(summary, null, 2) + '\n') + return summary +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const [dir, flag] = process.argv.slice(2) + if (!dir) throw new Error('usage: node e2e/layer3/file.mjs [--dry-run]') + for (const r of fileRun(path.resolve(dir), { dryRun: flag === '--dry-run' })) { + console.log(`${r.action.padEnd(9)} [${r.target}] ${r.title}${r.issue ? ` -> ${r.repo}#${r.issue}` : ''}${r.url ? ` -> ${r.url}` : ''}${r.why ? ` (${r.why})` : ''}`) + } +} diff --git a/cli/e2e/layer3/funnel.mjs b/cli/e2e/layer3/funnel.mjs new file mode 100644 index 0000000..135a291 --- /dev/null +++ b/cli/e2e/layer3/funnel.mjs @@ -0,0 +1,28 @@ +// Server-side evidence for a run, from the test account's OAuth grants, and the run's cleanup. +// A grant belongs to the run when the cell's app created it inside the run's window, so a grant the +// account held before the run (another app, a claude.ai connector) is never touched. +import { APPS } from './cells.mjs' + +const namesOf = (app) => APPS[app]?.names || [] + +export function runGrants(grants, { app, startedAtMs, endedAtMs }) { + const names = namesOf(app) + return grants.filter((g) => g.createdAtMs >= startedAtMs && g.createdAtMs <= endedAtMs && names.includes(g.clientName)) +} + +/** Live grants the cell's app already holds. Allowing again renews such a grant instead of creating one. */ +export function liveGrantsOf(grants, app, nowMs) { + const names = namesOf(app) + return grants.filter((g) => names.includes(g.clientName) && (!g.expiresAtMs || g.expiresAtMs > nowMs)) +} + +export function funnelFrom(grants) { + const at = (ms) => (ms ? new Date(ms).toISOString() : null) + return { + source: "box's list of apps connected to the test account", + note: 'approved means someone clicked Allow on the box sign-in page during this run. used means the app then sent at least one request to box. box does not record which tools the app called, so only the screenshots show whether the task ran.', + approved: grants.length > 0, + used: grants.some((g) => g.usedAtMs > 0), + grants: grants.map((g) => ({ id: g.id, clientName: g.clientName, clientHost: g.clientHost, approvedAt: at(g.createdAtMs), firstUsedAt: at(g.usedAtMs), maxBoxes: g.maxBoxes, boxesCreated: g.boxIds.length })), + } +} diff --git a/cli/e2e/layer3/grants.mjs b/cli/e2e/layer3/grants.mjs new file mode 100644 index 0000000..3c8fbc9 --- /dev/null +++ b/cli/e2e/layer3/grants.mjs @@ -0,0 +1,127 @@ +// cocoonbox Grants over gRPC-web with a personal key: list the caller's OAuth grants, revoke one. +// Hand-rolled protobuf for the two messages used here, so the e2e needs no generated code. +// node e2e/layer3/grants.mjs list +export const GRANTS_URL = process.env.BOX_GRANTS_URL || 'https://relay.simular.cloud/grpc-web' +const SERVICE = 'cocoonbox.control.v1.Grants' + +function varint(n) { + const out = [] + let v = BigInt(n) + do { + let b = Number(v & 0x7fn) + v >>= 7n + if (v) b |= 0x80 + out.push(b) + } while (v) + return out +} + +function encodeString(field, s) { + const bytes = Buffer.from(s, 'utf8') + return Buffer.from([...varint((field << 3) | 2), ...varint(bytes.length), ...bytes]) +} + +/** Decode one protobuf message into { field: [values] }; length-delimited values stay Buffers. */ +function decode(buf) { + const fields = {} + let i = 0 + const readVarint = () => { + let shift = 0n + let v = 0n + for (;;) { + const b = buf[i++] + v |= BigInt(b & 0x7f) << shift + if (!(b & 0x80)) return v + shift += 7n + } + } + while (i < buf.length) { + const tag = Number(readVarint()) + const field = tag >> 3 + const wire = tag & 7 + let value + if (wire === 0) value = readVarint() + else if (wire === 2) { + const len = Number(readVarint()) + value = buf.subarray(i, i + len) + i += len + } else if (wire === 1) { + value = buf.subarray(i, i + 8) + i += 8 + } else if (wire === 5) { + value = buf.subarray(i, i + 4) + i += 4 + } else throw new Error(`unsupported wire type ${wire}`) + ;(fields[field] ??= []).push(value) + } + return fields +} + +const str = (f, n) => (f[n]?.[0] ? f[n][0].toString('utf8') : '') +const num = (f, n) => Number(f[n]?.[0] ?? 0n) +const strs = (f, n) => (f[n] || []).map((b) => b.toString('utf8')) + +function parseGrant(buf) { + const f = decode(buf) + return { + id: str(f, 1), + clientId: str(f, 2), + clientName: str(f, 3), + clientHost: str(f, 4), + maxBoxes: num(f, 5), + maxRunning: num(f, 6), + boxIds: strs(f, 7), + chosenBoxIds: strs(f, 8), + createdAtMs: num(f, 9), + expiresAtMs: num(f, 10), + usedAtMs: num(f, 11), + } +} + +async function call(key, method, body) { + const frame = Buffer.concat([Buffer.from([0, ...[24, 16, 8, 0].map((s) => (body.length >>> s) & 0xff)]), body]) + const res = await fetch(`${GRANTS_URL}/${SERVICE}/${method}`, { + method: 'POST', + headers: { 'content-type': 'application/grpc-web+proto', 'x-grpc-web': '1', authorization: `Bearer ${key}` }, + body: frame, + signal: AbortSignal.timeout(30000), + }) + const buf = Buffer.from(await res.arrayBuffer()) + let message = null + let trailers = '' + for (let i = 0; i + 5 <= buf.length; ) { + const flag = buf[i] + const len = buf.readUInt32BE(i + 1) + const payload = buf.subarray(i + 5, i + 5 + len) + if (flag & 0x80) trailers += payload.toString('utf8') + else message = payload + i += 5 + len + } + const status = Number(res.headers.get('grpc-status') ?? /grpc-status:\s*(\d+)/i.exec(trailers)?.[1] ?? (res.ok ? 0 : -1)) + if (status !== 0) { + const msg = decodeURIComponent(res.headers.get('grpc-message') || /grpc-message:\s*(.*)/i.exec(trailers)?.[1]?.trim() || `HTTP ${res.status}`) + throw new Error(`Grants.${method}: grpc-status ${status}: ${msg}`) + } + return message ?? Buffer.alloc(0) +} + +export async function listGrants(key) { + const f = decode(await call(key, 'List', Buffer.alloc(0))) + return (f[1] || []).map(parseGrant) +} + +export async function revokeGrant(key, grantId) { + await call(key, 'Revoke', encodeString(1, grantId)) +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const { default: path } = await import('node:path') + const { default: fs } = await import('node:fs') + const env = path.join(path.dirname(new URL(import.meta.url).pathname), '.env') + if (fs.existsSync(env)) process.loadEnvFile(env) + if (!process.env.BOX_TOKEN) throw new Error('set BOX_TOKEN') + for (const g of await listGrants(process.env.BOX_TOKEN)) { + const at = (ms) => (ms ? new Date(ms).toISOString() : '-') + console.log(`${g.id} ${g.clientName.padEnd(24)} created ${at(g.createdAtMs)} used ${at(g.usedAtMs)} boxes ${g.boxIds.length}`) + } +} diff --git a/cli/e2e/layer3/judge.md b/cli/e2e/layer3/judge.md new file mode 100644 index 0000000..4aaa857 --- /dev/null +++ b/cli/e2e/layer3/judge.md @@ -0,0 +1,30 @@ +You review one recorded onboarding run. An agent (the driver) acted as a new user on a Linux desktop and tried to set up box by Simular in an AI client, then use it for one task. You decide whether a real new user would have succeeded, and what should change. + +The run folder is `{{runDir}}`. Read, in this order: + +1. `run.json` for the cell, the client version, what the driver was handed, and the driver's own claim. +2. `steps.jsonl`, one line per driver action, with the driver's note, the tool input, a result summary and the screenshot file. +3. The screenshots the steps point to. Look at them. The driver's notes can be wrong. +4. `funnel.json` if it exists. It is what box's server saw. `approved` means someone clicked Allow on the box sign-in page during the run. `used` means the app then sent at least one request to box. The file does not show which tools the app called, so a screenshot must still show the task's result. If `approved` is false but the screenshots show box working, say so in a reason, because the app may have used a connection made before the run. If the file is missing, judge from the screenshots alone and say so in a reason. + +Do not trust the driver's claim. Verify every part of it against the screenshots. + +## Verdict + +- `pass` when all hold: a screenshot shows the client's reply to the task using box (for example, a description of the box's screen that came from a box tool); the driver needed no workaround; after connecting, the user was shown at least one example prompt or a clear next step. +- `fail` when the onboarding, its docs or a client let the user down: an error, a dead end, a confusing instruction the driver had to guess past, a missing next step, or any workaround. +- `blocked` when the test setup failed, not the product: the desktop never responded, the driver ran out of turns or budget before reaching the client, a precondition was missing. Explain which. + +A workaround is any step where the driver did something a new user would not: edited a config file, guessed an unlisted URL, used knowledge no page showed. Any workaround makes the run `fail`, even if box worked at the end. + +## Improvements + +For each problem, propose one improvement a maintainer can act on. Name the target: + +- `install.md` for the setup instructions the agent reads (box-portal) +- `portal` for box.simular.ai pages, the consent page, sign-in +- `cli` for `@simular-ai/box` +- `client-docs` for something only the client's maker can fix +- `harness` for this test itself: the driver prompt, the cell, the judge + +Every reason and improvement must cite step indexes. Prefer a few sharp improvements over many vague ones. If nothing should change, return no improvements. diff --git a/cli/e2e/layer3/judge.schema.json b/cli/e2e/layer3/judge.schema.json new file mode 100644 index 0000000..88a041b --- /dev/null +++ b/cli/e2e/layer3/judge.schema.json @@ -0,0 +1,36 @@ +{ + "type": "object", + "additionalProperties": false, + "required": ["verdict", "reasons", "workarounds", "improvements"], + "properties": { + "verdict": { "enum": ["pass", "fail", "blocked"] }, + "reasons": { "type": "array", "items": { "$ref": "#/$defs/cited" } }, + "workarounds": { "type": "array", "items": { "$ref": "#/$defs/cited" } }, + "improvements": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["target", "title", "detail", "steps", "proposed"], + "properties": { + "target": { "enum": ["install.md", "portal", "cli", "client-docs", "harness"] }, + "title": { "type": "string" }, + "detail": { "type": "string" }, + "steps": { "type": "array", "items": { "type": "integer" } }, + "proposed": { "enum": ["issue", "draft-pr"] } + } + } + } + }, + "$defs": { + "cited": { + "type": "object", + "additionalProperties": false, + "required": ["text", "steps"], + "properties": { + "text": { "type": "string" }, + "steps": { "type": "array", "items": { "type": "integer" } } + } + } + } +} diff --git a/cli/e2e/layer3/run.mjs b/cli/e2e/layer3/run.mjs new file mode 100644 index 0000000..f1593f0 --- /dev/null +++ b/cli/e2e/layer3/run.mjs @@ -0,0 +1,145 @@ +// Layer 3: an agent plays a new user onboarding box on a box, then a separate judge reviews the recording. +// node e2e/layer3/run.mjs reset, drive, extract, judge, file; cell is app/setup/alreadySetUp +// BOX_TOKEN comes from the environment or e2e/layer3/.env (see .env.example). +// node e2e/layer3/run.mjs --judge judge an existing run again +// --no-file skip filing issues with gh +// Needs a signed-in `claude` CLI on this machine. Recordings hold the test account's screen, so they +// are written outside the repo (LAYER3_RUNS_DIR, default ~/.box-layer3/runs) and never committed. +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import spawn from 'cross-spawn' +import { DEFAULT_MCP_URL } from '../../src/constants.js' +import { connectMcp } from '../mcp-client.mjs' +import { loadCells } from './cells.mjs' +import { extractRun } from './extract.mjs' +import { fileRun } from './file.mjs' +import { funnelFrom, liveGrantsOf, runGrants } from './funnel.mjs' +import { listGrants, revokeGrant } from './grants.mjs' + +const HERE = path.dirname(fileURLToPath(import.meta.url)) +if (fs.existsSync(path.join(HERE, '.env'))) process.loadEnvFile(path.join(HERE, '.env')) +const SERVER = 'box' +const DRIVER_TOOLS = ['screenshot', 'computer_act', 'computer_state'] +const read = (f) => fs.readFileSync(path.join(HERE, f), 'utf8') +const fill = (tpl, vars) => tpl.replace(/\{\{(\w+)\}\}/g, (_, k) => vars[k] ?? '') +const writeJson = (dir, name, obj) => fs.writeFileSync(path.join(dir, name), JSON.stringify(obj, null, 2) + '\n') + +function claude(args, { cwd, env, stdoutFile }) { + const out = stdoutFile ? fs.openSync(stdoutFile, 'w') : 'pipe' + const res = spawn.sync('claude', args, { cwd, env, stdio: ['ignore', out, 'pipe'], encoding: 'utf8', maxBuffer: 1 << 28 }) + if (stdoutFile) fs.closeSync(out) + if (res.error) throw res.error + return res +} + +function textOf(result) { + return (result?.content || []).filter((c) => c.type === 'text').map((c) => c.text).join('\n') +} + +async function drive(cellName) { + const cells = loadCells() + const cell = cells.get(cellName) + if (!cell) throw new Error(`unknown cell "${cellName}". Cells:\n ${[...cells.keys()].join('\n ')}`) + const key = process.env.BOX_TOKEN + if (!key) throw new Error(`set BOX_TOKEN in ${path.join(HERE, '.env')} to a box key for the test account (https://box.simular.ai/keys)`) + const url = process.env.BOX_MCP_URL || DEFAULT_MCP_URL + + const runId = `${new Date().toISOString().replace(/[:.]/g, '-')}-${cellName.replaceAll('/', '--')}` + const runDir = path.join(process.env.LAYER3_RUNS_DIR || path.join(os.homedir(), '.box-layer3', 'runs'), runId) + fs.mkdirSync(runDir, { recursive: true }) + const run = { runId, cell, mcpUrl: url, account: process.env.LAYER3_ACCOUNT || 'test-account', startedAt: new Date().toISOString(), funnel: 'unavailable' } + const finish = (extra) => { + Object.assign(run, extra, { endedAt: new Date().toISOString() }) + writeJson(runDir, 'run.json', run) + return runDir + } + + const mcp = await connectMcp(url, key, { clientName: 'box-layer3-harness' }) + const toolNames = (await mcp.listTools()).map((t) => t.name) + const missing = [...DRIVER_TOOLS, 'shell'].filter((t) => !toolNames.includes(t)) + if (missing.length) return finish({ blocked: `box MCP server lacks tools: ${missing.join(', ')}` }) + const held = liveGrantsOf(await listGrants(key), cell.app, Date.now()) + if (held.length) return finish({ blocked: `the test account already holds a live ${cell.app} grant (${held.map((g) => g.id).join(', ')}). Approving again would renew it instead of creating one, so the run could not tell its own consent apart. Revoke it at https://box.simular.ai/keys, then run again.` }) + + const log = [] + for (const command of [...cell.reset, cell.precheck]) { + const res = await mcp.callTool('shell', { command, timeout_s: 300 }) + log.push(`$ ${command}\n${textOf(res)}`) + } + fs.writeFileSync(path.join(runDir, 'reset.log'), log.join('\n\n') + '\n') + run.clientVersion = log.at(-2)?.split('\n')[1]?.trim() + if (!/\bOK\b/.test(log.at(-1))) return finish({ blocked: `precheck failed. ${cell.precheckHelp}` }) + + writeJson(runDir, 'mcp.json', { mcpServers: { [SERVER]: { type: 'http', url, headers: { Authorization: 'Bearer ${BOX_TOKEN}' } } } }) + const allowed = DRIVER_TOOLS.map((t) => `mcp__${SERVER}__${t}`) + const denied = toolNames.filter((t) => !DRIVER_TOOLS.includes(t)).map((t) => `mcp__${SERVER}__${t}`) + const prompt = fill(read('driver.md'), cell) + const driverArgs = [ + '-p', prompt, + '--mcp-config', path.join(runDir, 'mcp.json'), '--strict-mcp-config', + '--tools', '', + '--allowedTools', allowed.join(','), + '--disallowedTools', denied.join(','), + '--output-format', 'stream-json', '--verbose', + '--max-turns', process.env.LAYER3_MAX_TURNS || '150', + '--max-budget-usd', process.env.LAYER3_MAX_USD || '10', + '--no-session-persistence', + ...(process.env.LAYER3_DRIVER_MODEL ? ['--model', process.env.LAYER3_DRIVER_MODEL] : []), + ] + console.log(`driving ${cell.title}\n cell ${cellName} on ${url} (recording to ${runDir})`) + const transcript = path.join(runDir, 'transcript.jsonl') + const res = claude(driverArgs, { cwd: runDir, env: { ...process.env, BOX_TOKEN: key }, stdoutFile: transcript }) + fs.writeFileSync(transcript, fs.readFileSync(transcript, 'utf8').split(key).join('***')) + if (res.stderr) fs.writeFileSync(path.join(runDir, 'driver.stderr'), res.stderr.split(key).join('***')) + + const { steps, final } = extractRun(runDir) + run.driver = { exit: res.status, model: process.env.LAYER3_DRIVER_MODEL || 'default', steps: steps.length, ...final } + + const mine = runGrants(await listGrants(key), { app: cell.app, startedAtMs: Date.parse(run.startedAt), endedAtMs: Date.now() }) + writeJson(runDir, 'funnel.json', funnelFrom(mine)) + run.funnel = 'funnel.json' + run.revoked = [] + for (const g of mine) { + await revokeGrant(key, g.id) + run.revoked.push(g.id) + } + return finish({}) +} + +function judge(runDir) { + const run = JSON.parse(fs.readFileSync(path.join(runDir, 'run.json'), 'utf8')) + if (run.blocked) { + const verdict = { verdict: 'blocked', reasons: [{ text: run.blocked, steps: [] }], workarounds: [], improvements: [] } + writeJson(runDir, 'judge.json', verdict) + return verdict + } + const res = claude([ + '-p', fill(read('judge.md'), { runDir }), + '--tools', 'Read,Glob,Grep', + '--allowedTools', 'Read,Glob,Grep', + '--add-dir', runDir, + '--strict-mcp-config', + '--output-format', 'json', + '--json-schema', read('judge.schema.json'), + '--no-session-persistence', + ...(process.env.LAYER3_JUDGE_MODEL ? ['--model', process.env.LAYER3_JUDGE_MODEL] : []), + ], { cwd: runDir, env: process.env }) + if (res.status !== 0) throw new Error(`judge exited ${res.status}: ${res.stderr || res.stdout}`) + const out = JSON.parse(res.stdout) + const verdict = out.structured_output ?? JSON.parse(out.result) + writeJson(runDir, 'judge.json', verdict) + return verdict +} + +const argv = process.argv.slice(2) +const noFile = argv.includes('--no-file') +const args = argv.filter((a) => a !== '--no-file') +const runDir = args[0] === '--judge' ? path.resolve(args[1]) : await drive(args[0] || 'claude-code-cli/paste-landing-line/nothing') +const verdict = judge(runDir) +console.log(`${verdict.verdict.toUpperCase()} ${runDir}`) +for (const r of verdict.reasons) console.log(` - ${r.text} (steps ${r.steps.join(', ') || 'none'})`) +if (noFile) for (const imp of verdict.improvements) console.log(` * [${imp.target}] ${imp.title}`) +else for (const r of fileRun(runDir)) console.log(` * ${r.action} [${r.target}] ${r.title}${r.issue ? ` -> ${r.repo}#${r.issue}` : ''}${r.url ? ` -> ${r.url}` : ''}${r.why ? ` (${r.why})` : ''}`) +process.exitCode = verdict.verdict === 'pass' ? 0 : verdict.verdict === 'blocked' ? 2 : 1 diff --git a/cli/e2e/mcp-client.mjs b/cli/e2e/mcp-client.mjs new file mode 100644 index 0000000..760efa9 --- /dev/null +++ b/cli/e2e/mcp-client.mjs @@ -0,0 +1,41 @@ +// Minimal MCP client over streamable HTTP with a bearer key. +export async function connectMcp(url, key, { clientName = 'box-ci' } = {}) { + let session + let nextId = 1 + async function rpc(method, params, { notify = false, timeoutMs = 60000 } = {}) { + const id = notify ? undefined : nextId++ + const res = await fetch(url, { + method: 'POST', + headers: { + 'content-type': 'application/json', + accept: 'application/json, text/event-stream', + authorization: `Bearer ${key}`, + ...(session ? { 'mcp-session-id': session } : {}), + ...(method !== 'initialize' ? { 'mcp-protocol-version': '2025-06-18' } : {}), + }, + body: JSON.stringify({ jsonrpc: '2.0', method, ...(params ? { params } : {}), ...(id !== undefined ? { id } : {}) }), + signal: AbortSignal.timeout(timeoutMs), + }) + if (res.status === 401) throw new Error('401: the key was rejected') + if (!res.ok && res.status !== 202) throw new Error(`${method}: HTTP ${res.status}`) + session = res.headers.get('mcp-session-id') || session + if (id === undefined) return null + const type = res.headers.get('content-type') || '' + const text = await res.text() + const messages = type.includes('text/event-stream') + ? text.split('\n').filter((l) => l.startsWith('data:')).map((l) => JSON.parse(l.slice(5))) + : [JSON.parse(text)] + const reply = messages.find((m) => m.id === id) + if (!reply) throw new Error(`${method}: no reply`) + if (reply.error) throw new Error(`${method}: ${reply.error.message}`) + return reply.result + } + + const init = await rpc('initialize', { protocolVersion: '2025-06-18', capabilities: {}, clientInfo: { name: clientName, version: '0' } }) + await rpc('notifications/initialized', undefined, { notify: true }) + return { + serverInfo: init.serverInfo, + listTools: async () => (await rpc('tools/list', {})).tools, + callTool: (name, args, { timeoutMs = 180000 } = {}) => rpc('tools/call', { name, arguments: args }, { timeoutMs }), + } +} diff --git a/cli/e2e/mcp-tools.mjs b/cli/e2e/mcp-tools.mjs index 732e9bc..34001b7 100644 --- a/cli/e2e/mcp-tools.mjs +++ b/cli/e2e/mcp-tools.mjs @@ -1,41 +1,13 @@ -// Minimal MCP client: initialize + tools/list over streamable HTTP with a bearer key. +// Minimal MCP check: initialize + tools/list over streamable HTTP with a bearer key. // Lists tools only; never calls one (calling would wake the box or act on it). // BOX_MCP_URL=... BOX_TOKEN=cb_... node e2e/mcp-tools.mjs +import { connectMcp } from './mcp-client.mjs' + const url = process.env.BOX_MCP_URL const key = process.env.BOX_TOKEN if (!url || !key) throw new Error('set BOX_MCP_URL and BOX_TOKEN') -let session -async function rpc(method, params, id) { - const res = await fetch(url, { - method: 'POST', - headers: { - 'content-type': 'application/json', - accept: 'application/json, text/event-stream', - authorization: `Bearer ${key}`, - ...(session ? { 'mcp-session-id': session } : {}), - ...(method !== 'initialize' ? { 'mcp-protocol-version': '2025-06-18' } : {}), - }, - body: JSON.stringify({ jsonrpc: '2.0', method, ...(params ? { params } : {}), ...(id !== undefined ? { id } : {}) }), - signal: AbortSignal.timeout(60000), - }) - if (res.status === 401) throw new Error('401: the key was rejected') - if (!res.ok && res.status !== 202) throw new Error(`${method}: HTTP ${res.status}`) - session = res.headers.get('mcp-session-id') || session - if (id === undefined) return null - const type = res.headers.get('content-type') || '' - const text = await res.text() - const messages = type.includes('text/event-stream') - ? text.split('\n').filter((l) => l.startsWith('data:')).map((l) => JSON.parse(l.slice(5))) - : [JSON.parse(text)] - const reply = messages.find((m) => m.id === id) - if (!reply) throw new Error(`${method}: no reply`) - if (reply.error) throw new Error(`${method}: ${reply.error.message}`) - return reply.result -} - -const init = await rpc('initialize', { protocolVersion: '2025-06-18', capabilities: {}, clientInfo: { name: 'box-ci', version: '0' } }, 1) -await rpc('notifications/initialized') -const { tools } = await rpc('tools/list', {}, 2) -console.log(`server ${init.serverInfo?.name} ${init.serverInfo?.version}: ${tools.length} tools: ${tools.map((t) => t.name).join(', ')}`) +const mcp = await connectMcp(url, key) +const tools = await mcp.listTools() +console.log(`server ${mcp.serverInfo?.name} ${mcp.serverInfo?.version}: ${tools.length} tools: ${tools.map((t) => t.name).join(', ')}`) if (!tools.length) throw new Error('no tools listed') diff --git a/cli/test/layer3-cells.test.js b/cli/test/layer3-cells.test.js new file mode 100644 index 0000000..40e8005 --- /dev/null +++ b/cli/test/layer3-cells.test.js @@ -0,0 +1,22 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import test from 'node:test' +import { loadCells } from '../e2e/layer3/cells.mjs' + +test('every cell in cells.json uses known values and has what a run needs', () => { + const cells = loadCells() + assert.ok(cells.size > 0) + for (const [name, cell] of cells) { + assert.match(name, /^[a-z0-9-]+\/[a-z0-9-]+\/[a-z0-9-]+$/) + for (const field of ['title', 'precheck', 'precheckHelp', 'handover', 'task']) assert.ok(cell[field], `${name}: ${field}`) + assert.ok(cell.reset.length, `${name}: reset`) + } +}) + +test('a cell with an unknown value is rejected by name', () => { + const file = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'cells-')), 'cells.json') + fs.writeFileSync(file, JSON.stringify({ cells: [{ app: 'claude-code-cli', setup: 'paste-the-line', alreadySetUp: 'nothing', title: 'x' }] })) + assert.throws(() => loadCells(file), /setup "paste-the-line" is not one of/) +}) diff --git a/cli/test/layer3-extract.test.js b/cli/test/layer3-extract.test.js new file mode 100644 index 0000000..bf75a9d --- /dev/null +++ b/cli/test/layer3-extract.test.js @@ -0,0 +1,34 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import test from 'node:test' +import { extractRun } from '../e2e/layer3/extract.mjs' + +const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==' +const line = (o) => JSON.stringify(o) +const transcript = [ + line({ type: 'system', subtype: 'init' }), + line({ type: 'assistant', message: { content: [{ type: 'text', text: 'The desktop is empty. I will open a terminal.' }, { type: 'tool_use', id: 't1', name: 'mcp__box__screenshot', input: {} }] } }), + line({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 't1', content: [{ type: 'text', text: '1280x800' }, { type: 'image', source: { type: 'base64', media_type: 'image/png', data: PNG } }] }] } }), + line({ type: 'assistant', message: { content: [{ type: 'tool_use', id: 't2', name: 'mcp__box__computer_act', input: { action: 'type', text: 'claude' } }] } }), + line({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 't2', is_error: true, content: 'denied' }] } }), + line({ type: 'result', subtype: 'success', result: 'Connected.\nRESULT: stuck: the consent page never loaded', num_turns: 3, total_cost_usd: 0.1 }), +].join('\n') + +test('extractRun writes one step per tool call, its screenshots, and the driver claim', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'layer3-')) + fs.writeFileSync(path.join(dir, 'transcript.jsonl'), transcript) + + const { final } = extractRun(dir) + const steps = fs.readFileSync(path.join(dir, 'steps.jsonl'), 'utf8').trim().split('\n').map((l) => JSON.parse(l)) + + assert.equal(steps.length, 2) + assert.equal(steps[0].note, 'The desktop is empty. I will open a terminal.') + assert.deepEqual(steps[0].screenshots, ['screenshots/000.png']) + assert.equal(steps[0].result, '1280x800') + assert.ok(fs.readFileSync(path.join(dir, 'screenshots/000.png')).subarray(1, 4).equals(Buffer.from('PNG'))) + assert.equal(steps[1].isError, true) + assert.equal(steps[1].result, 'denied') + assert.equal(final.claim, 'stuck: the consent page never loaded') +}) diff --git a/cli/test/layer3-file.test.js b/cli/test/layer3-file.test.js new file mode 100644 index 0000000..985922f --- /dev/null +++ b/cli/test/layer3-file.test.js @@ -0,0 +1,8 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { redact } from '../e2e/layer3/file.mjs' + +test('redact masks email addresses and leaves other text alone', () => { + assert.equal(redact('Signed in as someone.name+box@example.co.uk, then clicked Allow'), 'Signed in as [email], then clicked Allow') + assert.equal(redact('box.simular.ai/sign-in'), 'box.simular.ai/sign-in') +}) diff --git a/cli/test/layer3-funnel.test.js b/cli/test/layer3-funnel.test.js new file mode 100644 index 0000000..955db01 --- /dev/null +++ b/cli/test/layer3-funnel.test.js @@ -0,0 +1,32 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { funnelFrom, liveGrantsOf, runGrants } from '../e2e/layer3/funnel.mjs' + +const grant = (id, clientName, createdAtMs, usedAtMs = 0) => ({ id, clientName, clientHost: '', createdAtMs, usedAtMs, maxBoxes: 2, boxIds: [] }) +const window = { app: 'claude-code-cli', startedAtMs: 1000, endedAtMs: 2000 } + +test('only grants the cell app created inside the run window belong to the run', () => { + const grants = [ + grant('oag_before', 'Claude Code', 500, 1500), + grant('oag_connector', 'Claude', 1200, 1300), + grant('oag_run', 'Claude Code', 1500, 1600), + grant('oag_after', 'Claude Code', 2500), + ] + assert.deepEqual(runGrants(grants, window).map((g) => g.id), ['oag_run']) +}) + +test('a live grant the cell app already holds is found, an expired one or another client is not', () => { + const grants = [{ ...grant('oag_live', 'Claude Code', 100), expiresAtMs: 9000 }, { ...grant('oag_old', 'Claude Code', 100), expiresAtMs: 500 }, { ...grant('oag_web', 'Claude', 100), expiresAtMs: 9000 }] + assert.deepEqual(liveGrantsOf(grants, 'claude-code-cli', 1000).map((g) => g.id), ['oag_live']) +}) + +test('the funnel reports approval and first use from the run grants', () => { + const none = funnelFrom([]) + assert.equal(none.approved, false) + assert.equal(none.used, false) + + const f = funnelFrom([grant('oag_run', 'Claude Code', 1500, 1600)]) + assert.equal(f.approved, true) + assert.equal(f.used, true) + assert.equal(f.grants[0].approvedAt, new Date(1500).toISOString()) +})