Scene AI: a JS app in a flow exports run(), one feeding a field get()

cloud/apps/auth-web/evals/cases/scenes.ts

@@ -261,6 +261,22 @@ export const cases: EvalCase[] = [
     seedSlug: "weather",
     tags: ["modify", "fields"],
   },
+  {
+    // A button flow is run(), not get(): the chess puzzle's toggle app
+    // exported get() and the press did nothing (2026-10-17).
+    checks: [
+      { type: "delivered", tool: "modify_scene" },
+      { type: "lint_clean" },
+      { type: "preserves_node_ids", minFraction: 0.8 },
+      { type: "preserves_fields" },
+      { type: "flow_sets_state", event: "button" },
+      { type: "render_ok" },
+    ],
+    id: "modify-weather-button-unit",
+    prompt: "When someone presses a button on the frame, switch between Celsius and Fahrenheit and redraw.",
+    seedSlug: "weather",
+    tags: ["modify", "events"],
+  },
   {
     checks: [
       { type: "delivered", tool: "modify_scene" },

cloud/apps/auth-web/evals/lib/checks.test.ts

@@ -80,3 +80,51 @@ describe("code_nodes_min", () => {
     expect(result.passed).toBe(false);
   });
 });
+
+describe("flow_sets_state", () => {
+  const buttonScene = (nodes: unknown[], apps: Record<string, unknown> = {}) => ({
+    apps,
+    edges: [
+      { id: "b1", source: "button", sourceHandle: "next", target: "act", targetHandle: "prev", type: "appNodeEdge" },
+      { id: "b2", source: "act", sourceHandle: "next", target: "redraw", targetHandle: "prev", type: "appNodeEdge" },
+    ],
+    nodes: [
+      event,
+      { data: { keyword: "button" }, id: "button", type: "event" },
+      ...nodes,
+      { data: { config: {}, keyword: "render" }, id: "redraw", type: "dispatch" },
+    ],
+  });
+  const toggleApp = (source: string) => ({ toggle: { sources: { "app.ts": source, "config.json": "{}" } } });
+  const toggleNode = { data: { config: {}, keyword: "toggle" }, id: "act", type: "app" };
+
+  it("passes for a run() app that sets state and for logic/setAsState", () => {
+    const viaRun = run({ event: "button", type: "flow_sets_state" }, [
+      buttonScene([toggleNode], toggleApp("export function run(app) { frameos.setState('unit', 'f') }")),
+    ]);
+    expect(viaRun.passed).toBe(true);
+    expect(viaRun.detail).toContain("scene app toggle (act)");
+    const viaApp = run({ event: "button", type: "flow_sets_state" }, [
+      buttonScene([{ data: { config: { stateKey: "unit" }, keyword: "logic/setAsState" }, id: "act", type: "app" }]),
+    ]);
+    expect(viaApp.passed).toBe(true);
+  });
+
+  it("fails for the get()-only app the frame never calls in a flow", () => {
+    const result = run({ event: "button", type: "flow_sets_state" }, [
+      buttonScene([toggleNode], toggleApp("export function get(app) { frameos.setState('unit', 'f'); return null }")),
+    ]);
+    expect(result.passed).toBe(false);
+    expect(result.detail).toContain("toggle (act, no run())");
+  });
+
+  it("fails without a button event node or without a state change", () => {
+    expect(run({ event: "button", type: "flow_sets_state" }, [{ nodes: [event, renderImage] }]).detail).toBe(
+      'no "button" event node',
+    );
+    const redrawOnly = run({ event: "button", type: "flow_sets_state" }, [
+      buttonScene([{ data: { config: {}, keyword: "render" }, id: "act", type: "dispatch" }]),
+    ]);
+    expect(redrawOnly.passed).toBe(false);
+  });
+});

cloud/apps/auth-web/evals/lib/checks.ts

@@ -1,7 +1,7 @@
 // Deterministic checks over what the agent delivered. Each returns a
 // CheckResult with a one-line detail so a failing eval explains itself in
 // the report without opening the JSON.
-import type { LintResult } from "../../src/lib/ai/scene-lint";
+import { moduleExports, type LintResult } from "../../src/lib/ai/scene-lint";
 import type { Check, CheckResult, JudgeVerdict, RenderSummary } from "./types";
 
 type JsonObject = Record<string, unknown>;
@@ -109,6 +109,62 @@ function svgRoute(scenes: unknown[]): string | null {
   return codeNode ? `code node ${String(codeNode.id)}` : null;
 }
 
+// What the flows an event starts do to state, walking next -> prev (and
+// field/<slot> -> prev) edges from its event nodes. A scene-local JS app in a
+// flow counts only when its run() is there to call: the frame calls nothing
+// else in a flow, so a get()-only app there is listed as inert.
+function flowStateWriters(scenes: unknown[], event: string): { found: boolean; writers: string[]; inert: string[] } {
+  const writers: string[] = [];
+  const inert: string[] = [];
+  let found = false;
+  for (const scene of scenes) {
+    const entry = obj(scene);
+    const apps = obj(entry?.apps) ?? {};
+    const nodes = new Map(nodesOf([scene]).map((node) => [String(node.id), node]));
+    const edges = Array.isArray(entry?.edges) ? entry.edges.map(obj).filter((edge): edge is JsonObject => Boolean(edge)) : [];
+    const next = new Map<string, string[]>();
+    for (const edge of edges) {
+      if (edge.targetHandle === "prev") {
+        next.set(String(edge.source), [...(next.get(String(edge.source)) ?? []), String(edge.target)]);
+      }
+    }
+    const stack = [...nodes.values()]
+      .filter((node) => node.type === "event" && obj(node.data)?.keyword === event)
+      .map((node) => String(node.id));
+    found ||= stack.length > 0;
+    const seen = new Set<string>();
+    while (stack.length > 0) {
+      const id = stack.pop()!;
+      if (seen.has(id)) {
+        continue;
+      }
+      seen.add(id);
+      stack.push(...(next.get(id) ?? []));
+      const node = nodes.get(id);
+      const keyword = obj(node?.data)?.keyword;
+      if (!node || node.type === "event" || typeof keyword !== "string") {
+        continue;
+      }
+      if ((node.type === "app" && keyword === "logic/setAsState") || (node.type === "dispatch" && keyword === "setSceneState")) {
+        writers.push(`${keyword} (${id})`);
+        continue;
+      }
+      const sources = obj(obj(apps[keyword])?.sources);
+      const main = Object.entries(sources ?? {}).find(([file]) => /^app\.(ts|tsx|js|jsx)$/.test(file))?.[1];
+      if (node.type !== "app" || typeof main !== "string") {
+        continue;
+      }
+      const exported = moduleExports(main);
+      if (exported && !exported.has("run")) {
+        inert.push(`${keyword} (${id}, no run())`);
+      } else if (/frameos\.setState\s*\(/.test(main)) {
+        writers.push(`scene app ${keyword} (${id})`);
+      }
+    }
+  }
+  return { found, inert, writers };
+}
+
 function regex(pattern: string, flags?: string): RegExp {
   return new RegExp(pattern, flags ?? "i");
 }
@@ -213,6 +269,18 @@ export function runCheck(check: Check, input: CheckInput): CheckResult {
         ? pass(`uses ${alternative}`)
         : fail(`no SVG drawn: no render/svg or jsSvg app, no frameos.svg(...) or <svg in a JS app or code node (has: ${[...new Set(used)].join(", ") || "none"})`);
     }
+    case "flow_sets_state": {
+      const { found, inert, writers } = flowStateWriters(scenes, check.event);
+      if (!found) {
+        return fail(`no "${check.event}" event node`);
+      }
+      if (inert.length > 0) {
+        return fail(`the ${check.event} flow runs apps that do nothing there: ${inert.join(", ")}`);
+      }
+      return writers.length > 0
+        ? pass(`the ${check.event} flow sets state via ${writers.join(", ")}`)
+        : fail(`the ${check.event} flow changes no state (no logic/setAsState, setSceneState dispatch or run() calling frameos.setState)`);
+    }
     case "scene_count": {
       const count = scenes.length;
       if (check.min !== undefined && count < check.min) {

cloud/apps/auth-web/evals/lib/types.ts

@@ -23,6 +23,12 @@ export type Check =
   | { type: "code_nodes_min"; min: number }
   /** The scene draws with SVG, by any route; `orApps` lists apps that satisfy the request just as well. */
   | { type: "draws_svg"; orApps?: string[] }
+  /**
+   * An `event` node with this keyword starts a flow that changes state (logic/setAsState, a
+   * setSceneState dispatch, or a scene-local JS app whose run() calls frameos.setState), and no
+   * JS app in that flow lacks the run() the frame calls there.
+   */
+  | { type: "flow_sets_state"; event: string }
   | { type: "scene_count"; min?: number; max?: number }
   | { type: "json_matches"; pattern: string; flags?: string }
   | { type: "json_not_matches"; pattern: string; flags?: string }