From fc241b4860ec5797b78057f323ab120a1b6610e3 Mon Sep 17 00:00:00 2001 From: trvon Date: Tue, 13 Oct 2026 23:10:42 -0600 Subject: [PATCH 1/6] perf: bundle startup and profile fuzz workers Reduce the internal module graph without deferring RPC, tool registration, or recovery. Keep host packages external and index/API runtime modules shared. Add receipt-aware paired startup measurements and bounded property-worker CPU profiles with campaign provenance, failure retention, source identity, and explicit unknown counters. --- benchmarks/fuzz-profile-report.ts | 114 ++++++++++++ benchmarks/fuzz-profile-setup.ts | 69 ++++++++ benchmarks/profile-fuzz.ts | 276 ++++++++++++++++++++++++++++++ benchmarks/profile-startup.ts | 183 ++++++++++++++++++++ benchmarks/startup-report.ts | 84 +++++++++ docs/TESTING.md | 30 +++- package-lock.json | 1 + package.json | 5 +- scripts/bundle.mjs | 26 +++ scripts/package-smoke.mjs | 36 ++++ test/fuzz-profiling.test.ts | 215 +++++++++++++++++++++++ test/startup-profiling.test.ts | 99 +++++++++++ tsconfig.dev.json | 2 +- vitest.config.ts | 1 + 14 files changed, 1138 insertions(+), 3 deletions(-) create mode 100644 benchmarks/fuzz-profile-report.ts create mode 100644 benchmarks/fuzz-profile-setup.ts create mode 100644 benchmarks/profile-fuzz.ts create mode 100644 benchmarks/profile-startup.ts create mode 100644 benchmarks/startup-report.ts create mode 100644 scripts/bundle.mjs create mode 100644 test/fuzz-profiling.test.ts create mode 100644 test/startup-profiling.test.ts diff --git a/benchmarks/fuzz-profile-report.ts b/benchmarks/fuzz-profile-report.ts new file mode 100644 index 0000000..162632b --- /dev/null +++ b/benchmarks/fuzz-profile-report.ts @@ -0,0 +1,114 @@ +import { createHash } from "node:crypto"; +import { readdirSync, readFileSync, realpathSync, statSync } from "node:fs"; +import { join, resolve } from "node:path"; + +export const MAX_ARTIFACT_BYTES = 64 * 1024 * 1024; +export function profileText(value: unknown, maximumBytes = MAX_ARTIFACT_BYTES): string { + const raw = JSON.stringify(value); + if (typeof raw !== "string" || Buffer.byteLength(raw) > maximumBytes) throw new Error(`CPU profile exceeds ${maximumBytes} bytes or is absent`); + return raw; +} +export const sha256 = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex"); +const object = (value: unknown): Record => value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +const count = (value: unknown): number | null => typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null; +const flag = (value: unknown): boolean | null => typeof value === "boolean" ? value : null; +const canonicalPath = (path: string) => { + try { return realpathSync(path); } catch { return resolve(path); } +}; + +export function readArtifact(path: string) { + const size = statSync(path).size; + if (size > MAX_ARTIFACT_BYTES) throw new Error(`Artifact exceeds ${MAX_ARTIFACT_BYTES} bytes: ${path}`); + const raw = readFileSync(path); + try { return { path, sizeBytes: raw.length, sha256: sha256(raw), value: JSON.parse(raw.toString("utf8")) as unknown, parseError: null }; } + catch (error) { return { path, sizeBytes: raw.length, sha256: sha256(raw), value: null, parseError: String(error) }; } +} + +export function summarizeCampaign(value: unknown, suite: string) { + const rows = Array.isArray(value) ? value : [value]; + if (!rows.length || rows.some(row => row === null || typeof row !== "object" || Array.isArray(row))) throw new Error("Invalid campaign report"); + return rows.map(value => { + const row = object(value); + const counters = object(row.counters); + return { + name: typeof row.name === "string" ? row.name : suite, + actualRuns: count(row.numRuns), + completed: count(row.completed ?? row.completedObservations), + checks: count(row.executions ?? counters.executions), + shrinks: count(row.numShrinks), + actions: count(row.commands ?? counters.commands), + actionCounts: row.coverage ?? row.counts ?? counters.operations ?? null, + failed: flag(row.failed), + interrupted: flag(row.interrupted), + counterexamplePath: typeof row.counterexamplePath === "string" ? row.counterexamplePath : null, + }; + }); +} +export type CampaignSummary = ReturnType; + +export function summarizeVitest(value: unknown, selectedTests?: readonly string[]) { + const row = object(value); + if (typeof row.success !== "boolean" || !Array.isArray(row.testResults)) throw new Error("Invalid Vitest JSON report"); + const assertions = row.testResults.flatMap(result => { + const rows = object(result).assertionResults; + return Array.isArray(rows) ? rows.map(object) : []; + }); + // Vitest retains pattern-excluded tests as pending. Do not confuse deliberate + // selection with a selected property that was skipped or never executed. + const filtered = assertions.filter(row => selectedTests && typeof row.fullName === "string" && !selectedTests.includes(row.fullName) && (row.status === "pending" || row.status === "skipped")); + const selected = assertions.filter(row => !filtered.includes(row)); + return { + success: row.success, + passed: selected.filter(row => row.status === "passed").length, + failed: selected.filter(row => row.status === "failed").length, + skipped: selected.filter(row => row.status !== "passed" && row.status !== "failed").length, + properties: selected.map(row => ({ name: row.fullName, status: row.status })), + filtered: filtered.map(row => ({ name: row.fullName, status: row.status })), + }; +} +export type TestSummary = ReturnType; + +export function collectProfiles(dir: string, expectedSource: string) { + const profiles: Array<{ path: string; sha256: string; sizeBytes: number; metadataSHA256: string; metadata: Record }> = []; + const errors: string[] = []; + let entries: string[]; + try { entries = readdirSync(dir); } + catch (error) { return { profiles, errors: [`Worker profile directory: ${String(error)}`] }; } + const names = entries.filter(name => name.endsWith(".cpuprofile")); + if (names.length > 64) return { profiles, errors: ["Worker profile count exceeds 64"] }; + for (const name of names) { + try { + const profile = readArtifact(join(dir, name)); + const meta = readArtifact(join(dir, name.replace(/\.cpuprofile$/, ".meta.json"))); + const metadata = object(meta.value); + if (metadata.sha256 !== profile.sha256 || metadata.sizeBytes !== profile.sizeBytes) throw new Error(`Worker profile checksum/size mismatch: ${name}`); + if (metadata.role !== "property-worker" || metadata.complete !== true || typeof metadata.sourceFile !== "string" || canonicalPath(metadata.sourceFile) !== canonicalPath(expectedSource) || (count(metadata.pid) ?? 0) < 1 || typeof metadata.node !== "string" || typeof metadata.wallTimeMs !== "number" || !Number.isFinite(metadata.wallTimeMs) || metadata.wallTimeMs < 0 || !metadata.resources) throw new Error(`Incomplete or wrong worker profile metadata: ${name}`); + const resources = object(metadata.resources); + if (count(resources.userCPUTimeMicros) === null || count(resources.systemCPUTimeMicros) === null || count(resources.maxRSSKiB) === null) throw new Error(`Incomplete worker resource metadata: ${name}`); + const data = object(profile.value); + if (!Array.isArray(data.nodes) || !data.nodes.length || !Array.isArray(data.samples) || !data.samples.length) throw new Error(`Worker CPU profile has no samples: ${name}`); + profiles.push({ path: profile.path, sha256: profile.sha256, sizeBytes: profile.sizeBytes, metadataSHA256: meta.sha256, metadata }); + } catch (error) { errors.push(error instanceof Error ? error.message : String(error)); } + } + for (const name of entries.filter(name => name.startsWith("worker-") && name.endsWith(".meta.json"))) { + if (!entries.includes(name.replace(/\.meta\.json$/, ".cpuprofile"))) errors.push(`Incomplete worker profile: ${name}`); + } + return { profiles, errors }; +} + +export interface ChildStatus { + exitCode: number | null; + signal: string | null; + error: string | null; + timedOut: boolean; +} + +export function decideOutcome(evidence: { child: ChildStatus; campaign: CampaignSummary; tests: TestSummary | null; profileCount: number; errors: string[] }) { + const { child, campaign, tests, profileCount, errors } = evidence; + if (campaign.some(row => row.failed === true)) return "FAIL"; + if (child.timedOut || child.signal || campaign.some(row => row.interrupted === true)) return "INTERRUPTED"; + if (tests?.failed || (child.exitCode !== null && child.exitCode !== 0)) return "FAIL"; + if (errors.length || child.error || tests?.success === false) return "FAIL"; + if (child.exitCode === null || !tests || tests.passed === 0 || tests.skipped > 0 || !campaign.length || campaign.some(row => (row.actualRuns ?? row.completed ?? 0) < 1) || profileCount === 0) return "INCOMPLETE"; + return "PASS"; +} diff --git a/benchmarks/fuzz-profile-setup.ts b/benchmarks/fuzz-profile-setup.ts new file mode 100644 index 0000000..0c7a102 --- /dev/null +++ b/benchmarks/fuzz-profile-setup.ts @@ -0,0 +1,69 @@ +import { afterAll, beforeAll, expect } from "vitest"; +import { profileText } from "./fuzz-profile-report.js"; + +// The config does not load this module unless profiling was explicitly requested. +const directory = process.env.PI_LOOP_FUZZ_PROFILE_DIR; +const clock = process.hrtime.bigint.bind(process.hrtime); +const usage = process.resourceUsage.bind(process); +let session: import("node:inspector").Session | undefined; +let started: bigint | undefined; +let initialUsage: NodeJS.ResourceUsage | undefined; +let sourceFile: string | undefined; +let setupError: unknown; + +function post(method: string, params: Record = {}): Promise> { + return new Promise((resolve, reject) => { + if (!session) { reject(new Error("Worker inspector session is absent")); return; } + session.post(method, params, (error, result) => error ? reject(error) : resolve(result as Record)); + }); +} + +if (directory) { + beforeAll(async () => { + try { + const { Session } = await import("node:inspector"); + sourceFile = expect.getState().testPath; + if (!sourceFile) throw new Error("Worker profile requires a test source file"); + session = new Session(); + session.connect(); + await post("Profiler.enable"); + await post("Profiler.setSamplingInterval", { interval: 1000 }); + initialUsage = usage(); + started = clock(); + await post("Profiler.start"); + } catch (error) { setupError = error; throw error; } + }); + + afterAll(async () => { + let metadataPath: string | undefined; + try { + const [{ writeFileSync }, { join }, { createHash, randomUUID }] = await Promise.all([import("node:fs"), import("node:path"), import("node:crypto")]); + const base = join(directory, `worker-${process.pid}-${randomUUID()}`); + metadataPath = `${base}.meta.json`; + if (setupError || !started || !initialUsage) throw setupError ?? new Error("Worker profiler did not start"); + const { profile } = await post("Profiler.stop"); + const finished = clock(); + const resources = usage(); + const raw = profileText(profile); + writeFileSync(`${base}.cpuprofile`, raw, { flag: "wx" }); + writeFileSync(metadataPath, JSON.stringify({ + schemaVersion: 1, role: "property-worker", complete: true, + sourceFile, pid: process.pid, node: process.version, + wallTimeMs: Number(finished - started) / 1e6, + resources: { userCPUTimeMicros: resources.userCPUTime - initialUsage.userCPUTime, systemCPUTimeMicros: resources.systemCPUTime - initialUsage.systemCPUTime, maxRSSKiB: resources.maxRSS }, + resourceCoverage: "CPU delta during suite hooks; RSS is process lifetime high-water mark. Coordinator and native descendants excluded.", + sha256: createHash("sha256").update(raw).digest("hex"), sizeBytes: Buffer.byteLength(raw), + }, null, 2), { flag: "wx" }); + } catch (error) { + // Hook errors remain secondary to assertions in Vitest's raw log/report. + console.error("Worker CPU profiling/reporting failed:", error); + if (metadataPath) { + try { + const { writeFileSync } = await import("node:fs"); + writeFileSync(metadataPath, JSON.stringify({ role: "property-worker", complete: false, sourceFile, pid: process.pid, error: String(error) }), { flag: "wx" }); + } catch { /* The parent also treats absent metadata/profile as incomplete. */ } + } + throw error; + } finally { session?.disconnect(); } + }); +} diff --git a/benchmarks/profile-fuzz.ts b/benchmarks/profile-fuzz.ts new file mode 100644 index 0000000..4b799c6 --- /dev/null +++ b/benchmarks/profile-fuzz.ts @@ -0,0 +1,276 @@ +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import { type CampaignSummary, type ChildStatus, collectProfiles, decideOutcome, readArtifact, sha256, summarizeCampaign, summarizeVitest, type TestSummary } from "./fuzz-profile-report.js"; + +export const SUITES = { + "monitor-runtime": { + reportEnv: "MONITOR_FUZZ_REPORT", maximumRuns: 1000, + properties: [ + "generic monitor runtime model preserves authoritative outcomes through fault-injected cancellation, output, timers and shutdown", + "generic monitor runtime model isolates generated callback consumers from authoritative outcomes and later receipts", + ], + }, + "monitor-inactivity": { + reportEnv: "MONITOR_INACTIVITY_REPORT", maximumRuns: 1000, + properties: ["models generated inactivity renewal and timeout followed by zero close without success inference"], + }, + "monitor-native": { + reportEnv: "NATIVE_MONITOR_REPORT", maximumRuns: 64, + properties: ["models bounded generated POSIX producer patterns without trusting their exit markers"], + }, + "workflow-restart": { + reportEnv: "PI_LOOP_RESTART_FUZZ_REPORT", maximumRuns: 1000, + properties: ["file-backed workflow restart sequence properties preserves ownership/CAS on reload and permits eligible recovery without resetting work"], + }, +} as const; +export type Suite = keyof typeof SUITES; +const NATIVE_CONTROLS = [ + "preserves the primary native trace and attempts remaining cleanup after a cleanup failure", + "keeps native pre-gate TERM exit-zero output distinct from manual cancellation", + "records normal native gate-body completion separately from cancellation", +]; +export const GLOBAL_MAXIMUM_RUNS = 1000; +export const MAX_OUTPUT_BYTES = 1024 * 1024; + +function positiveInteger(name: string, value: string): number { + if (!/^[0-9]+$/.test(value) || !Number.isSafeInteger(Number(value)) || Number(value) < 1) throw new Error(`${name} must be a positive safe integer`); + return Number(value); +} + +export function parseProfileArgs(args: string[]) { + const { values } = parseArgs({ args, strict: true, allowPositionals: false, options: { + suite: { type: "string", default: "monitor-runtime" }, runs: { type: "string", default: "100" }, + seed: { type: "string" }, path: { type: "string" }, test: { type: "string" }, + out: { type: "string", default: ".artifacts/profiles/fuzz" }, "timeout-ms": { type: "string", default: "600000" }, + } }); + if (!values.suite || !Object.hasOwn(SUITES, values.suite)) throw new Error(`Unknown suite: ${values.suite}`); + const suite = values.suite as Suite; + const requestedRuns = positiveInteger("--runs", values.runs ?? "100"); + const seed = positiveInteger("--seed", values.seed ?? "130363"); + const timeoutMs = positiveInteger("--timeout-ms", values["timeout-ms"] ?? "600000"); + if (timeoutMs > 600000) throw new Error("--timeout-ms must not exceed 600000"); + if (!values.out) throw new Error("--out must not be empty"); + const pattern = values.test === undefined ? undefined : new RegExp(values.test); + if (values.test === "") throw new Error("--test must not be empty"); + const selectedProperties = SUITES[suite].properties.filter(name => !pattern || pattern.test(name)); + if (!selectedProperties.length) throw new Error("--test must select a known property in the selected suite"); + const selectedTests = [...SUITES[suite].properties, ...(suite === "monitor-native" ? NATIVE_CONTROLS : [])].filter(name => !pattern || pattern.test(name)); + if (values.path !== undefined) { + if (values.seed === undefined) throw new Error("--path requires an explicit --seed"); + if (values.test === undefined) throw new Error("--path requires --test"); + if (!/^[0-9]+(?::[0-9]+)*$/.test(values.path)) throw new Error("--path must be a nonempty fast-check replay path"); + if (selectedProperties.length !== 1 || selectedTests.length !== 1) throw new Error("--path must select exactly one property and no control tests"); + } + const maximumRuns = SUITES[suite].maximumRuns; + const runs = Math.min(requestedRuns, GLOBAL_MAXIMUM_RUNS, maximumRuns); + return { suite, requestedRuns, runs, maximumRuns, globalMaximumRuns: GLOBAL_MAXIMUM_RUNS, clipped: requestedRuns > runs, seed, seedExplicit: values.seed !== undefined, path: values.path, replay: values.path !== undefined, test: values.test, selectedProperties, selectedTests, out: values.out, timeoutMs }; +} +export type ProfileOptions = ReturnType; + +export function buildChildPlan(options: ProfileOptions, cwd: string, dir: string, inherited: NodeJS.ProcessEnv = process.env) { + const env = { ...inherited }; + for (const key of Object.keys(env)) { + if (key.startsWith("FC_") || key.startsWith("PI_LOOP_FUZZ_") || Object.values(SUITES).some(suite => suite.reportEnv === key) || key === "NODE_OPTIONS") delete env[key]; + } + Object.assign(env, { FC_NUM_RUNS: String(options.runs), FC_SEED: String(options.seed), PI_LOOP: "off", PI_LOOP_SCOPE: "memory", TZ: "UTC", PI_LOOP_FUZZ_PROFILE_DIR: dir, [SUITES[options.suite].reportEnv]: join(dir, "campaign.json") }); + if (options.path !== undefined) env.FC_PATH = options.path; + const file = `test/property/${options.suite}.property.test.ts`; + return { + binary: process.execPath, + args: [join(cwd, "node_modules/vitest/vitest.mjs"), "run", file, "--maxWorkers=1", "--no-file-parallelism", "--reporter=default", "--reporter=json", `--outputFile.json=${join(dir, "vitest.json")}`, ...(options.test === undefined ? [] : ["--testNamePattern", options.test])], + file, env, cwd, + }; +} +export type ChildPlan = ReturnType; + +export interface ChildResult extends ChildStatus { + wallTimeMs: number; + stdout: string; + stderr: string; + stdoutBytes: number; + stderrBytes: number; + stdoutTruncated: boolean; + stderrTruncated: boolean; + stdoutSHA256: string; + stderrSHA256: string; + resourceCoverage: string; +} + +// Retain bounded prefixes while hashing the entire observed streams. +export function outputCollector(limit = MAX_OUTPUT_BYTES) { + const hash = createHash("sha256"); + let bytes = 0; + let retained = 0; + const chunks: Buffer[] = []; + return { + add(chunk: Buffer) { + hash.update(chunk); + bytes += chunk.length; + const part = chunk.subarray(0, Math.max(0, limit - retained)); + if (part.length) chunks.push(part); + retained += part.length; + }, + finish() { + let text = Buffer.concat(chunks).toString("utf8"); + // A clipped multibyte sequence must not make the saved UTF-8 prefix exceed the cap. + while (Buffer.byteLength(text) > limit) text = text.slice(0, -1); + return { text, bytes, truncated: bytes > limit, sha256: hash.digest("hex") }; + }, + }; +} + +export function runChild(plan: ChildPlan, timeoutMs: number, spawnChild: typeof spawn = spawn): Promise { + return new Promise(resolveResult => { + const started = process.hrtime.bigint(); + const stdout = outputCollector(); + const stderr = outputCollector(); + let child: ReturnType; + let error: string | null = null; + let timedOut = false; + let done = false; + let timeout: ReturnType | undefined; + let escalation: ReturnType | undefined; + let deadline: ReturnType | undefined; + const finish = (exitCode: number | null, signal: string | null) => { + if (done) return; + done = true; + clearTimeout(timeout); clearTimeout(escalation); clearTimeout(deadline); + const out = stdout.finish(); + const err = stderr.finish(); + resolveResult({ exitCode, signal, error, timedOut, wallTimeMs: Number(process.hrtime.bigint() - started) / 1e6, + stdout: out.text, stderr: err.text, stdoutBytes: out.bytes, stderrBytes: err.bytes, + stdoutTruncated: out.truncated, stderrTruncated: err.truncated, stdoutSHA256: out.sha256, stderrSHA256: err.sha256, + resourceCoverage: "Worker metadata only; coordinator CPU/RSS unknown. Native descendant CPU/RSS and detached-process termination unobserved.", + }); + }; + try { + child = spawnChild(plan.binary, plan.args, { cwd: plan.cwd, env: plan.env, shell: false, detached: process.platform !== "win32", stdio: ["ignore", "pipe", "pipe"] }); + } catch (cause) { error = String(cause); finish(null, null); return; } + child.stdout?.on("data", (chunk: Buffer) => { if (!done) stdout.add(chunk); }); + child.stderr?.on("data", (chunk: Buffer) => { if (!done) stderr.add(chunk); }); + child.on("error", cause => { error = cause.message; }); + child.on("close", (code, signal) => finish(code, signal)); + const terminate = (signal: NodeJS.Signals) => { + try { + if (process.platform !== "win32" && child.pid) process.kill(-child.pid, signal); + else child.kill(signal); + } catch (cause) { error ??= `Termination: ${String(cause)}`; } + }; + timeout = setTimeout(() => { + timedOut = true; + terminate("SIGTERM"); + escalation = setTimeout(() => terminate("SIGKILL"), 1000); + deadline = setTimeout(() => { + error ??= "Child close unobserved after bounded termination"; + child.stdout?.destroy(); child.stderr?.destroy(); child.unref(); + finish(null, null); + }, 2000); + }, timeoutMs); + }); +} + +// Hash the actual logic, not just the property fixture or a possibly dirty revision label. +function sourceManifest(cwd: string) { + const files: Array<{ file: string; sha256: string }> = []; + let bytes = 0; + const visit = (dir: string) => { + for (const entry of readdirSync(join(cwd, dir), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) { + const file = join(dir, entry.name); + if (entry.isDirectory()) visit(file); + else if (entry.isFile() && entry.name.endsWith(".ts")) { + const raw = readFileSync(join(cwd, file)); + bytes += raw.length; + if (files.length >= 1000 || bytes > 16 * 1024 * 1024) throw new Error("Source manifest exceeds bounds"); + files.push({ file, sha256: sha256(raw) }); + } + } + }; + try { visit("src"); return { complete: true, sha256: sha256(JSON.stringify(files)), files }; } + catch (error) { return { complete: false, sha256: null, files, error: String(error) }; } +} + +interface Dependencies { + runChild?: (plan: ChildPlan, timeoutMs: number) => Promise; + writeSummary?: (path: string, content: string) => void; +} + +export async function runProfile(options: ProfileOptions, cwd = process.cwd(), dependencies: Dependencies = {}) { + const root = resolve(cwd, options.out); + mkdirSync(root, { recursive: true }); + const artifactDir = mkdtempSync(join(root, `${options.suite}-`)); + const plan = buildChildPlan(options, cwd, artifactDir); + const errors: string[] = []; + const startedAt = new Date().toISOString(); + const child = await (dependencies.runChild ?? runChild)(plan, options.timeoutMs); + // Save the primary evidence before attempting any secondary parsing/reporting. + for (const stream of ["stdout", "stderr"] as const) { + try { writeFileSync(join(artifactDir, `${stream}.log`), child[stream], { flag: "wx" }); } + catch (error) { errors.push(`Saving ${stream}: ${String(error)}`); } + } + let campaign: CampaignSummary = []; + let tests: TestSummary | null = null; + let campaignArtifact: Omit, "value"> | null = null; + let testArtifact: Omit, "value"> | null = null; + try { + const { value, ...artifact } = readArtifact(join(artifactDir, "campaign.json")); + campaignArtifact = artifact; + if (artifact.parseError) throw new Error(artifact.parseError); + campaign = summarizeCampaign(value, options.suite); + if (campaign.length !== options.selectedProperties.length) errors.push("Campaign report does not cover exactly the selected properties"); + if (campaign.some(row => row.actualRuns === null && row.completed === null)) errors.push("Campaign has no observed completion/run evidence"); + } catch (error) { errors.push(`Campaign report: ${String(error)}`); } + try { + const { value, ...artifact } = readArtifact(join(artifactDir, "vitest.json")); + testArtifact = artifact; + if (artifact.parseError) throw new Error(artifact.parseError); + tests = summarizeVitest(value, options.selectedTests); + for (const property of options.selectedProperties) { + if (!tests.properties.some(row => row.name === property)) errors.push(`Vitest did not observe selected property: ${property}`); + } + if (options.replay && (tests.properties.length !== 1 || tests.properties[0]?.name !== options.selectedProperties[0])) errors.push("Replay selection was not unambiguous in the actual worker"); + if (tests.properties.length !== options.selectedTests.length) errors.push("Vitest selection differs from the allowlisted test inventory"); + } catch (error) { errors.push(`Vitest report: ${String(error)}`); } + const { profiles, errors: profileErrors } = collectProfiles(artifactDir, join(cwd, plan.file)); + errors.push(...profileErrors); + const sourceHashes: Record = {}; + for (const file of [plan.file, "test/property/config.ts", "vitest.config.ts", "benchmarks/fuzz-profile-setup.ts", "package.json", "package-lock.json"]) { + try { sourceHashes[file] = sha256(readFileSync(join(cwd, file))); } + catch { sourceHashes[file] = null; errors.push(`Required source identity unavailable: ${file}`); } + } + const manifest = sourceManifest(cwd); + if (!manifest.complete) errors.push(`Source manifest incomplete: ${manifest.error}`); + const result = { + schemaVersion: 1, kind: "fuzz-worker-cpu-profile", artifactDir, startedAt, + runtime: { node: process.version, execPath: process.execPath, platform: process.platform, arch: process.arch }, + sourceHashes, sourceManifest: manifest, options, command: { binary: plan.binary, args: plan.args, cwd: plan.cwd, shell: false }, + controlledEnvironmentKeys: ["FC_NUM_RUNS", "FC_SEED", ...(options.replay ? ["FC_PATH"] : []), "PI_LOOP", "PI_LOOP_SCOPE", "TZ", "PI_LOOP_FUZZ_PROFILE_DIR", SUITES[options.suite].reportEnv], + child: { exitCode: child.exitCode, signal: child.signal, error: child.error, timedOut: child.timedOut, wallTimeMs: child.wallTimeMs, resourceCoverage: child.resourceCoverage }, + output: { + maximumRetainedBytesPerStream: MAX_OUTPUT_BYTES, + stdout: { path: join(artifactDir, "stdout.log"), observedBytes: child.stdoutBytes, truncated: child.stdoutTruncated, observedSHA256: child.stdoutSHA256, retainedSHA256: sha256(child.stdout) }, + stderr: { path: join(artifactDir, "stderr.log"), observedBytes: child.stderrBytes, truncated: child.stderrTruncated, observedSHA256: child.stderrSHA256, retainedSHA256: sha256(child.stderr) }, + }, + campaignArtifact, testArtifact, campaign, tests, profiles, profileCount: profiles.length, + profileStatus: child.timedOut || child.signal || child.exitCode === null ? "unknown/incomplete" : profiles.length ? "worker-profile-present" : "missing", errors, + coverage: "Property worker suite hooks only; not coordinator, startup or native descendant attribution. Unknown counters are null. Abnormal exit may leave incomplete profiles/reports. No retries.", + outcome: decideOutcome({ child, campaign, tests, profileCount: profiles.length, errors }), + }; + try { (dependencies.writeSummary ?? ((path, content) => writeFileSync(path, content, { flag: "wx" })))(join(artifactDir, "summary.json"), `${JSON.stringify(result, null, 2)}\n`); } + catch (error) { errors.push(`Summary reporting: ${String(error)}`); result.outcome = "FAIL"; } + return result; +} + +if (process.argv[1] && realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url))) { + try { + const result = await runProfile(parseProfileArgs(process.argv.slice(2))); + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); + process.exitCode = result.outcome === "PASS" ? 0 : 1; + } catch (error) { + process.stderr.write(`Fuzz profiling failed: ${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + } +} diff --git a/benchmarks/profile-startup.ts b/benchmarks/profile-startup.ts new file mode 100644 index 0000000..3e9f39e --- /dev/null +++ b/benchmarks/profile-startup.ts @@ -0,0 +1,183 @@ +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { performance } from "node:perf_hooks"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { build } from "esbuild"; +import { pairedStartupSummary, preserveStartupCleanup, type StartupSample, startupOptions, summarizeDurations } from "./startup-report.js"; + +const sha256 = (data: string | Buffer) => createHash("sha256").update(data).digest("hex"); +function readJson(path: string, decode: (value: unknown) => T): T { + try { return decode(JSON.parse(readFileSync(path, "utf8"))); } + catch (cause) { throw new Error(`Cannot read profiling JSON: ${path}`, { cause }); } +} +function object(value: unknown): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Expected JSON object"); + return value as Record; +} +function readVersion(path: string): string { + return readJson(path, value => { + const version = object(value).version; + if (typeof version !== "string" || !version) throw new Error("Missing package version"); + return version; + }); +} +interface StartupMetrics { node: string; exec: string; cpuUserMicros: number; cpuSystemMicros: number; maxRssKiB: number; rssBytes: number } +function readMetrics(path: string): StartupMetrics { + return readJson(path, value => { + const data = object(value); + const resources = object(data.resources); + const memory = object(data.memory); + if (typeof data.node !== "string" || typeof data.exec !== "string") throw new Error("Missing observed child identity"); + const number = (value: unknown) => { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new Error("Invalid resource measurement"); + return value; + }; + return { node: data.node, exec: data.exec, cpuUserMicros: number(resources.userCPUTime), cpuSystemMicros: number(resources.systemCPUTime), maxRssKiB: number(resources.maxRSS), rssBytes: number(memory.rss) }; + }); +} + +export async function staticModuleInventory(entry: string) { + const workingDirectory = dirname(resolve(entry)); + const result = await build({ absWorkingDir: workingDirectory, entryPoints: [resolve(entry)], bundle: true, platform: "node", format: "esm", packages: "external", write: false, metafile: true, logLevel: "silent" }); + const inputs = result.metafile!.inputs; + const visited = new Set(); + const files: Array<{ path: string; bytes: number; sha256: string }> = []; + const external = new Set(); + const dynamic: string[] = []; + const visit = (key: string) => { + if (visited.has(key)) return; + visited.add(key); + const input = inputs[key]; + if (!input) throw new Error(`Missing module inventory input: ${key}`); + const path = resolve(workingDirectory, key); + const data = readFileSync(path); + files.push({ path, bytes: data.length, sha256: sha256(data) }); + for (const dependency of input.imports) { + if (dependency.external) external.add(dependency.path); + else if (dependency.kind === "import-statement") visit(dependency.path); + else dynamic.push(`${key}: ${dependency.kind} ${dependency.path}`); + } + }; + const entryKey = Object.keys(inputs).find(key => resolve(workingDirectory, key) === resolve(entry)); + if (!entryKey) throw new Error("Entry absent from module inventory"); + visit(entryKey); + return { kind: "static relative ESM graph, not observed runtime loading", modules: files.length, bytes: files.reduce((n, f) => n + f.bytes, 0), files, external: [...external].sort(), dynamic }; +} + +export async function runStartupProfile(args: string[]) { + const options = startupOptions(args); + const root = fileURLToPath(new URL("..", import.meta.url)); + const cli = resolve(options.cli ?? join(root, "node_modules/@earendil-works/pi-coding-agent/dist/cli.js")); + const baseline = resolve(options.baseline ?? join(root, "dist/index.js")); + const candidate = options.candidate ? resolve(options.candidate) : undefined; + for (const path of [cli, baseline, ...(candidate ? [candidate] : [])]) if (!existsSync(path)) throw new Error(`Missing startup input: ${path}`); + const graph = { baseline: await staticModuleInventory(baseline), ...(candidate ? { candidate: await staticModuleInventory(candidate) } : {}) }; + const sdkVersion = readVersion(join(dirname(cli), "../package.json")); + const packageVersion = readVersion(join(root, "package.json")); + const outputRoot = resolve(options.out); + mkdirSync(outputRoot, { recursive: true }); + const output = mkdtempSync(join(outputRoot, "run-")); + const workspace = mkdtempSync(join(tmpdir(), "pi-loop-startup-")); + const noop = join(workspace, "noop.mjs"); + const probe = join(workspace, "metrics.mjs"); + const profiler = join(workspace, "profiler.mjs"); + const targets: Record = { noop, baseline, ...(candidate ? { candidate } : {}) }; + const wrappers = Object.fromEntries(Object.keys(targets).map(name => [name, join(workspace, `${name}-wrapper.mjs`)])); + const common = [cli, "--no-extensions", "--no-mcp", "--list-models"]; + const cases: Record = { none: common, ...Object.fromEntries(Object.keys(targets).map(name => [name, [...common, "-e", wrappers[name]!]])) }; + const env: NodeJS.ProcessEnv = { ...process.env, PI_CODING_AGENT_DIR: join(workspace, "config"), PI_OFFLINE: "1", PI_SKIP_VERSION_CHECK: "1", PI_TELEMETRY: "0", PI_LOOP: "off", PI_LOOP_SCOPE: "memory" }; + for (const key of ["PI_SESSION_ID", "PI_SESSION_FILE", "PI_PROVIDER", "PI_MODEL", "PI_REASONING_LEVEL", "NODE_OPTIONS", "PI_LOOP_STARTUP_RECEIPT", "PI_LOOP_STARTUP_PROFILE"]) delete env[key as keyof typeof env]; + const metadata = { + schemaVersion: 1, node: process.version, platform: process.platform, arch: process.arch, + sdk: sdkVersion, + packageVersion, + sourceRevision: spawnSync("git", ["rev-parse", "HEAD"], { cwd: root, encoding: "utf8" }).stdout?.trim() || null, + mode: "CLI list-models; fresh processes with warm OS caches; not TUI first render or first-request latency", + options, graph, loaderEvidence: "Owned wrapper records only successful factory return/resolution; absent/invalid receipt rejects the run. Shared wrapper I/O adds instrumentation, not uninstrumented CLI/TUI performance.", maximumProfileBytes: 64 * 1024 * 1024, + requestedTimingChildren: (options.rounds + options.warmups) * Object.keys(cases).length, + samples: [] as Array, + profileSamples: [] as Array, + profiles: [] as Array<{ name: string; path: string; sha256: string; bytes: number }>, complete: false, + summaries: {} as Record>, paired: null as ReturnType | null, + failure: null as string | null, + }; + const save = () => writeFileSync(join(output, "report.json"), `${JSON.stringify(metadata, null, 2)}\n`); + const execute = (name: string, round: number, profile = false) => { + const label = `${profile ? "profile" : "timing"}-${round}-${name}`; + const metricsPath = join(output, `${label}.metrics.json`); + const profileDir = join(output, "cpu"); + const profilePath = join(profileDir, `${label}.cpuprofile`); + const receiptPath = join(output, `${label}.loaded.json`); + if (profile) mkdirSync(profileDir, { recursive: true }); + const childArgs = ["--import", probe, ...(profile ? ["--import", profiler] : []), ...cases[name]!]; + const start = performance.now(); + const child = spawnSync(process.execPath, childArgs, { cwd: workspace, env: { ...env, PI_LOOP_STARTUP_METRICS: metricsPath, PI_LOOP_STARTUP_RECEIPT: receiptPath, PI_LOOP_STARTUP_PROFILE: profilePath }, encoding: "utf8", timeout: 30_000, maxBuffer: 1024 * 1024 }); + const wallMs = performance.now() - start; + const stdout = child.stdout ?? ""; + const stderr = child.stderr ?? ""; + const extensionError = /failed to load|error loading extension/i.test(stderr); + const sample = { round, warmup: round < options.warmups, name, wallMs, exitCode: child.status, signal: child.signal, ...(child.error ? { error: child.error.message } : extensionError ? { error: "Extension loader reported failure" } : {}), stdoutHash: sha256(stdout), stderrHash: sha256(stderr), metrics: null as StartupMetrics | null }; + (profile ? metadata.profileSamples : metadata.samples).push(sample); + // Persist physical observations before secondary output/metrics parsing can fail. + save(); + writeFileSync(join(output, `${label}.stdout.log`), stdout); + writeFileSync(join(output, `${label}.stderr.log`), stderr); + sample.metrics = existsSync(metricsPath) ? readMetrics(metricsPath) : null; + save(); + if (name !== "none") readJson(receiptPath, value => { + const row = object(value); + if (row.target !== pathToFileURL(targets[name]!).href || row.phase !== "factory-returned" || row.node !== process.version) throw new Error(`${label}: extension load receipt invalid`); + return row; + }); + if (child.status !== 0 || child.signal || child.error || extensionError || !sample.metrics) throw new Error(`${label} incomplete: ${JSON.stringify(sample)}`); + if (profile) { + const path = profilePath; + const data = readFileSync(path); + readJson(path, value => { + const nodes = object(value).nodes; + if (!Array.isArray(nodes) || !nodes.length) throw new Error(`Empty CPU profile: ${label}`); + return { nodeCount: nodes.length }; + }); + metadata.profiles.push({ name, path, sha256: sha256(data), bytes: statSync(path).size }); + save(); + } + }; + let operationError: unknown; + try { + writeFileSync(noop, "export default function () {}\n"); + writeFileSync(probe, `import {writeFileSync} from 'node:fs'; process.once('exit',()=>writeFileSync(process.env.PI_LOOP_STARTUP_METRICS,JSON.stringify({node:process.version,exec:process.execPath,resources:process.resourceUsage(),memory:process.memoryUsage()})));\n`); + writeFileSync(profiler, `import {Session} from 'node:inspector'; import {writeFileSync} from 'node:fs'; const s=new Session(); s.connect(); const post=m=>new Promise((r,j)=>s.post(m,(e,v)=>e?j(e):r(v))); await post('Profiler.enable'); await post('Profiler.start'); process.once('exit',()=>s.post('Profiler.stop',(e,v)=>{if(e)throw e; const raw=JSON.stringify(v.profile); if(Buffer.byteLength(raw)>${metadata.maximumProfileBytes}) throw new Error('Startup CPU profile exceeds byte limit'); writeFileSync(process.env.PI_LOOP_STARTUP_PROFILE,raw,{flag:'wx'}); s.disconnect();}));\n`); + for (const [name, target] of Object.entries(targets)) { + const href = pathToFileURL(target).href; + writeFileSync(wrappers[name]!, `import factory from ${JSON.stringify(href)}; import {writeFileSync} from 'node:fs'; export default function(pi){if(typeof factory!=='function')throw new Error('No extension factory'); const value=factory(pi); const receipt=()=>writeFileSync(process.env.PI_LOOP_STARTUP_RECEIPT,JSON.stringify({target:${JSON.stringify(href)},phase:'factory-returned',node:process.version}),{flag:'wx'}); if(value&&typeof value.then==='function')return Promise.resolve(value).then(result=>{receipt();return result;}); receipt();return value;}\n`); + } + save(); + for (let round = 0; round < options.rounds + options.warmups; round++) { + const names = Object.keys(cases); + if (round % 2) names.reverse(); + for (const name of names) execute(name, round); + console.info(JSON.stringify({ progress: { current: round + 1, total: options.rounds + options.warmups, message: "startup timing round" } })); + } + for (const name of Object.keys(cases)) metadata.summaries[name] = summarizeDurations(metadata.samples.filter(s => !s.warmup && s.name === name).map(s => s.wallMs)); + if (candidate) metadata.paired = pairedStartupSummary(metadata.samples); + // CPU instrumentation changes timings; these observations never enter the paired statistics. + if (options.profiles) for (const name of Object.keys(cases)) execute(name, -1, true); + metadata.complete = true; + save(); + console.info(JSON.stringify({ output, complete: true, summaries: metadata.summaries, paired: metadata.paired })); + } catch (error) { + metadata.failure = error instanceof Error ? error.stack ?? error.message : String(error); + operationError = error; + try { save(); } + catch (reportError) { operationError = new AggregateError([error, reportError], "Startup failure and report write failure"); } + throw operationError; + } finally { + preserveStartupCleanup(operationError, () => rmSync(workspace, { recursive: true, force: true })); + } + return { output, metadata }; +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await runStartupProfile(process.argv.slice(2)); diff --git a/benchmarks/startup-report.ts b/benchmarks/startup-report.ts new file mode 100644 index 0000000..316672b --- /dev/null +++ b/benchmarks/startup-report.ts @@ -0,0 +1,84 @@ +import { parseArgs } from "node:util"; + +export function preserveStartupCleanup(primary: unknown, cleanup: () => void): void { + try { cleanup(); } + catch (error) { + if (primary !== undefined) throw new AggregateError([primary, error], "Startup failure and cleanup failure"); + throw error; + } +} + +export interface StartupSample { + round: number; + warmup: boolean; + name: string; + wallMs: number; + exitCode: number | null; + signal: string | null; + error?: string; +} + +function integer(value: string, name: string, minimum: number, maximum: number): number { + if (!/^\d+$/.test(value)) throw new Error(`${name} must be an integer`); + const result = Number(value); + if (!Number.isSafeInteger(result) || result < minimum || result > maximum) { + throw new Error(`${name} must be between ${minimum} and ${maximum}`); + } + return result; +} + +export function startupOptions(args: string[]) { + const { values } = parseArgs({ args, strict: true, options: { + rounds: { type: "string", default: "15" }, warmups: { type: "string", default: "3" }, + baseline: { type: "string" }, candidate: { type: "string" }, cli: { type: "string" }, + out: { type: "string", default: ".artifacts/profiles/startup" }, profiles: { type: "boolean", default: false }, + } }); + return { ...values, rounds: integer(values.rounds, "rounds", 3, 100), warmups: integer(values.warmups, "warmups", 0, 10) }; +} + +function percentile(sorted: number[], fraction: number): number { + const index = (sorted.length - 1) * fraction; + const lower = Math.floor(index); + return sorted[lower]! + (sorted[Math.ceil(index)]! - sorted[lower]!) * (index - lower); +} + +export function summarizeDurations(values: number[]) { + if (!values.length || values.some(value => !Number.isFinite(value) || value < 0)) { + throw new Error("Durations must contain finite nonnegative observations"); + } + const sorted = [...values].sort((a, b) => a - b); + return { n: sorted.length, medianMs: percentile(sorted, 0.5), minMs: sorted[0]!, maxMs: sorted.at(-1)!, p25Ms: percentile(sorted, 0.25), p75Ms: percentile(sorted, 0.75) }; +} + +export function pairedStartupSummary(samples: StartupSample[]) { + const groups = new Map>(); + for (const sample of samples) { + if (sample.warmup || !["baseline", "candidate"].includes(sample.name)) continue; + if (sample.exitCode !== 0 || sample.signal || sample.error) throw new Error("Startup child failed; timing comparison incomplete"); + summarizeDurations([sample.wallMs]); + let group = groups.get(sample.round); + if (!group) { group = new Map(); groups.set(sample.round, group); } + if (group.has(sample.name)) throw new Error("Startup round has duplicate observations"); + group.set(sample.name, sample); + } + if (!groups.size) throw new Error("No complete startup pairs"); + const pairs = [...groups].map(([round, group]) => { + const baseline = group.get("baseline"); + const candidate = group.get("candidate"); + if (!baseline || !candidate) throw new Error("Missing startup pair"); + return { round, baselineMs: baseline.wallMs, candidateMs: candidate.wallMs, improvementMs: baseline.wallMs - candidate.wallMs }; + }); + const differences = pairs.map(pair => pair.improvementMs).sort((a, b) => a - b); + // Fixed bootstrap seed makes report regeneration reproducible, not timings deterministic. + let state = 104729; + const medians: number[] = []; + for (let iteration = 0; iteration < 2000; iteration++) { + const resample = differences.map(() => { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0; + return differences[state % differences.length]!; + }).sort((a, b) => a - b); + medians.push(percentile(resample, 0.5)); + } + medians.sort((a, b) => a - b); + return { completePairs: pairs.length, pairs, medianImprovementMs: percentile(differences, 0.5), bootstrap95Ms: [percentile(medians, 0.025), percentile(medians, 0.975)], bootstrapSeed: 104729, bootstrapResamples: 2000 }; +} diff --git a/docs/TESTING.md b/docs/TESTING.md index b84ac35..11534ae 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -39,6 +39,8 @@ Lint currently reports two established optional-chain warnings; new warnings are | Live E2E | `npm run test:e2e` | Pi RPC/model conformance | | Benchmarks | `npm run bench` | Fixed core workloads | | CPU profile | `npm run profile:core` | V8 profile plus checksummed metadata | +| Startup profile | `npm run profile:startup` | Fresh-process CLI timing and separate CPU samples | +| Fuzz profile | `npm run profile:fuzz` | Allowlisted property-worker CPU capture and campaign evidence | Unit tests use Vitest, fake timers for schedules, in-memory stores for pure behavior, temporary files for persistence/restart, and one shared Pi event-bus mock. @@ -180,7 +182,33 @@ npm run bench:compare npm run profile:core ``` -`npm run bench` prints a table of throughput, mean latency, and margin of error. `bench:baseline` writes those results to `.artifacts/benchmarks/baseline.json`, and `bench:compare` reports the throughput change of a fresh run against that baseline. Compare benchmarks only on the same machine, Node version, architecture, timezone, and power state. Profiles are written to `.artifacts/profiles/`; load the `.cpuprofile` in a V8-compatible viewer. Shared benchmark workloads live in `benchmarks/workloads.ts`. +Core workloads live in `benchmarks/workloads.ts`. `bench:baseline` writes `.artifacts/benchmarks/baseline.json`; `bench:compare` reports throughput changes. Load `.cpuprofile` files in a V8-compatible viewer. Compare only the same machine, Node version, architecture, timezone, and power state. + +### Startup + +Keep the prior compiled `dist` tree for comparison. Both builds must resolve the same host dependencies. + +```bash +npm run profile:startup -- --baseline /path/to/prior/dist/index.js --candidate dist/index.js --profiles +``` + +The default is 15 measured rounds after 3 warmups, serial fresh processes, alternating case order. `--rounds` accepts 3–100; `--warmups` accepts 0–10. `--cli` selects a compiled Pi CLI. `--out` selects the artifact root; default `.artifacts/profiles/startup`. Each run owns a unique directory. + +`report.json` retains raw wall time, physical child status, CPU/RSS, versions, static graph hashes, paired medians and a seeded bootstrap interval. Owned wrappers require affirmative factory-return receipts. CPU-profile children run separately; their timings do not enter comparisons. Profiles are capped at 64 MiB before write. Static graphs are not observed runtime-loading traces. CLI `--list-models` is not TUI first render or first-request latency. Do not run timing comparisons alongside fuzz campaigns or gates. + +### Fuzz workers + +```bash +npm run profile:fuzz -- --suite monitor-runtime --runs 100 --seed 130363 +npm run profile:fuzz -- --suite monitor-runtime --runs 100 --seed 130363 \ + --test fault-injected --path '5:4:4:4:7:7:7:7:7:7:7' +``` + +Suites: `monitor-runtime`, `monitor-inactivity`, `monitor-native`, `workflow-restart`. Runs and seeds are positive safe integers. The runner caps cases at 1,000; native cases retain their existing cap of 64. Replay requires an explicit seed and a pattern selecting exactly one property, with no control tests. Each invocation runs one serial Vitest child without a shell or retries. + +Artifacts default to `.artifacts/profiles/fuzz/-/`. `--out` changes the root. `--timeout-ms` defaults to and cannot exceed 600,000. Retained output is capped at 1 MiB per stream; hashes also cover observed discarded bytes. Worker profiles are capped at 64 MiB before write. + +`summary.json` links raw campaign/reporter files and checksums. It distinguishes requested, capped, and observed counts. Unreported counters remain `null`. Failed, interrupted, skipped, missing-profile and missing-source-identity runs do not pass. Primary assertion logs survive reporting failures. Profiling is opt-in through `PI_LOOP_FUZZ_PROFILE_DIR`, set by the runner; normal tests do not start an inspector. Capture covers suite hooks in the actual property worker, not startup, the coordinator, or native descendants. Worker RSS is a process-lifetime high-water mark. Native POSIX skips are not Windows passes. Timing and profiles do not certify correctness. ## Change evidence diff --git a/package-lock.json b/package-lock.json index 0948a96..1139b1e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "@earendil-works/pi-tui": "1.2.0", "@types/node": "^26.6.5", "@vitest/coverage-v8": "^5.0.4", + "esbuild": "0.28.2", "fast-check": "^4.10.2", "tsx": "^4.23.15", "typebox": "1.3.37", diff --git a/package.json b/package.json index 1379bd0..8ab532f 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ }, "scripts": { "clean": "node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\"", - "build": "npm run clean && tsc", + "build": "npm run clean && tsc && node scripts/bundle.mjs", "audit:all": "npm audit", "audit:production": "npm audit --omit=dev --audit-level=moderate", "prepublishOnly": "npm run audit:all && npm run lint && npm run typecheck && npm run test && npm run build && npm run test:package", @@ -76,6 +76,8 @@ "bench:baseline": "BENCH_BASELINE_OUT=.artifacts/benchmarks/baseline.json npm run bench", "bench:compare": "BENCH_BASELINE_IN=.artifacts/benchmarks/baseline.json npm run bench", "profile:core": "node -e \"const fs=require('node:fs');fs.mkdirSync('.artifacts/profiles',{recursive:true});fs.rmSync('.artifacts/profiles/core.cpuprofile',{force:true})\" && TZ=UTC node --import tsx --cpu-prof --cpu-prof-interval=1000 --cpu-prof-dir=.artifacts/profiles --cpu-prof-name=core.cpuprofile benchmarks/profile-core.ts", + "profile:startup": "node --import tsx benchmarks/profile-startup.ts", + "profile:fuzz": "TZ=UTC node --import tsx benchmarks/profile-fuzz.ts", "hooks:install": "git config core.hooksPath .githooks", "typecheck": "npm run typecheck:prod && tsc -p tsconfig.dev.json", "typecheck:prod": "tsc --noEmit", @@ -88,6 +90,7 @@ "@earendil-works/pi-tui": "1.2.0", "@types/node": "^26.6.5", "@vitest/coverage-v8": "^5.0.4", + "esbuild": "0.28.2", "fast-check": "^4.10.2", "tsx": "^4.23.15", "typebox": "1.3.37", diff --git a/scripts/bundle.mjs b/scripts/bundle.mjs new file mode 100644 index 0000000..936c78c --- /dev/null +++ b/scripts/bundle.mjs @@ -0,0 +1,26 @@ +import { readdirSync, rmSync } from "node:fs"; +import { join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { build } from "esbuild"; + +const root = fileURLToPath(new URL("..", import.meta.url)); +const dist = join(root, "dist"); +const result = await build({ + absWorkingDir: root, + entryPoints: { index: "src/index.ts", api: "src/api.ts" }, + outdir: "dist", chunkNames: "chunks/[name]-[hash]", + bundle: true, splitting: true, format: "esm", platform: "node", target: "es2022", + // Host-provided packages must retain the host's classes and registries. + packages: "external", keepNames: true, sourcemap: true, metafile: true, +}); +const emitted = new Set(Object.keys(result.metafile.outputs).map(path => resolve(root, path))); +function removeLegacyJavaScript(directory) { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isDirectory()) removeLegacyJavaScript(path); + else if ((path.endsWith(".js") || path.endsWith(".js.map")) && !emitted.has(path)) rmSync(path); + } +} +// tsc declarations still describe the public API; unsupported deep JS entries do not ship twice. +removeLegacyJavaScript(dist); +console.log(`bundle: ${[...emitted].filter(path => path.endsWith(".js")).map(path => relative(dist, path)).join(", ")}; peer packages external`); diff --git a/scripts/package-smoke.mjs b/scripts/package-smoke.mjs index 85f63f7..1ac46aa 100644 --- a/scripts/package-smoke.mjs +++ b/scripts/package-smoke.mjs @@ -1,5 +1,7 @@ import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; +import { EventEmitter } from "node:events"; +import { build } from "esbuild"; import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -9,6 +11,36 @@ const rootDirectory = fileURLToPath(new URL("..", import.meta.url)); const temporaryDirectory = mkdtempSync(join(tmpdir(), "pi-loop-package-")); const npmCommand = process.platform === "win32" ? "npm.cmd" : "npm"; +async function assertFactoryReadiness(factory, api, external) { + const previous = { loop: process.env.PI_LOOP, scope: process.env.PI_LOOP_SCOPE }; + process.env.PI_LOOP = "off"; + process.env.PI_LOOP_SCOPE = "memory"; + const bus = new EventEmitter(); + const hooks = new Map(); + const tools = new Map(); + const events = { emit: (name, data) => bus.emit(name, data), on: (name, handler) => { bus.on(name, handler); return () => bus.off(name, handler); } }; + if (external) events.on(api.TASKS_RPC.ping, request => queueMicrotask(() => events.emit(api.replyChannel(api.TASKS_RPC.ping, request.requestId), { success: true, data: { version: api.PROTOCOL_VERSION, provider: "package-control" } }))); + const pi = { events, on: (name, handler) => { hooks.set(name, [...(hooks.get(name) ?? []), handler]); }, registerTool: tool => tools.set(tool.name, tool), registerCommand() {}, sendMessage() {}, sendUserMessage() {} }; + const ctx = { cwd: temporaryDirectory, hasUI: false, isIdle: () => true, hasPendingMessages: () => false, sessionManager: { getSessionId: () => "package-control" }, ui: { setStatus() {}, setWidget() {}, notify() {} } }; + const dispatch = async name => { for (const handler of hooks.get(name) ?? []) await handler({}, ctx); }; + try { + factory(pi); + const ping = await api.rpcProbe(events, api.TASKS_RPC.ping, 1000); + assert.equal(ping?.version, api.PROTOCOL_VERSION, "factory-time current-protocol task RPC remains discoverable"); + await new Promise(resolve => setImmediate(resolve)); + await dispatch("session_start"); + assert.equal(tools.has("TaskCreate"), !external, "provider selection determines initial native task tools"); + const prefix = [...tools.values()].map(tool => ({ name: tool.name, parameters: tool.parameters, description: tool.description, promptGuidelines: tool.promptGuidelines })); + await dispatch("turn_start"); + assert.deepEqual([...tools.values()].map(tool => ({ name: tool.name, parameters: tool.parameters, description: tool.description, promptGuidelines: tool.promptGuidelines })), prefix, "first-turn schemas/guidelines do not change after session_start"); + for (const name of ["LoopCreate", "MonitorCreate", "WorkflowCreate", "WorkflowClaim", "WorkflowTransition"]) assert(tools.has(name), `${name} remains available`); + } finally { + await dispatch("session_shutdown"); + if (previous.loop === undefined) delete process.env.PI_LOOP; else process.env.PI_LOOP = previous.loop; + if (previous.scope === undefined) delete process.env.PI_LOOP_SCOPE; else process.env.PI_LOOP_SCOPE = previous.scope; + } +} + try { const rawReport = execFileSync(npmCommand, ["pack", "--json", "--ignore-scripts", "--pack-destination", temporaryDirectory], { cwd: rootDirectory, @@ -43,6 +75,10 @@ try { const packedApi = await import(pathToFileURL(join(temporaryDirectory, "package", "dist", "api.js")).href); assert.equal(typeof packedRoot.default, "function", "packed root must load as the Pi extension"); assert.equal(typeof packedApi.TaskStore, "function", "packed api must load TaskStore"); + await assertFactoryReadiness(packedRoot.default, packedApi, false); + await assertFactoryReadiness(packedRoot.default, packedApi, true); + const graph = await build({ absWorkingDir: join(temporaryDirectory, "package"), entryPoints: ["dist/index.js"], bundle: true, platform: "node", format: "esm", packages: "external", metafile: true, write: false, logLevel: "silent" }); + assert(Object.keys(graph.metafile.inputs).length <= 5, "compiled entry must not restore the eager many-file internal graph"); assert.equal(typeof packedApi.diagnoseWorkflowGraph, "function", "packed api must expose graph diagnostics"); assert.deepEqual(packedApi.diagnoseWorkflowGraph({ definition: { version: 1, initialState: "work", states: { work: { prompt: "Work" } } }, diff --git a/test/fuzz-profiling.test.ts b/test/fuzz-profiling.test.ts new file mode 100644 index 0000000..b4439db --- /dev/null +++ b/test/fuzz-profiling.test.ts @@ -0,0 +1,215 @@ +import type { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { EventEmitter } from "node:events"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { collectProfiles, decideOutcome, profileText, summarizeCampaign, summarizeVitest } from "../benchmarks/fuzz-profile-report.js"; +import { buildChildPlan, outputCollector, parseProfileArgs, runChild, runProfile } from "../benchmarks/profile-fuzz.js"; +import config from "../vitest.config.js"; + +// Feature expectations: no production defect/red-reproduction claim. +describe("opt-in fuzz profiling feature", () => { + it("bounds UTF-8 profile bytes before any artifact write", () => { + const value = { name: "é" }; + const expected = JSON.stringify(value); + expect(profileText(value, Buffer.byteLength(expected))).toBe(expected); + expect(() => profileText(value, Buffer.byteLength(expected) - 1)).toThrow(/exceeds/); + expect(() => profileText(undefined)).toThrow(/absent/); + }); + it("uses strict positive safe integers, bounded runs and explicit replay selection", () => { + expect(parseProfileArgs([])).toMatchObject({ suite: "monitor-runtime", requestedRuns: 100, runs: 100, seed: 130363, timeoutMs: 600000 }); + for (const bad of ["0", "-1", "1.2", "3x", "1e3", "Infinity", "9007199254740992", "", " 3"]) { + expect(() => parseProfileArgs(["--runs", bad])).toThrow(); + expect(() => parseProfileArgs(["--seed", bad])).toThrow(); + } + expect(() => parseProfileArgs(["--suite", "../../other"])).toThrow(); + expect(() => parseProfileArgs(["--unknown"])).toThrow(); + expect(() => parseProfileArgs(["--timeout-ms", "600001"])).toThrow(); + expect(() => parseProfileArgs(["--suite", "monitor-native", "--path", "0", "--seed", "3", "--test", ".*"])).toThrow(/one/); + expect(() => parseProfileArgs(["--path", "0", "--test", "fault-injected"])).toThrow(/seed/); + expect(() => parseProfileArgs(["--path", "0", "--seed", "3"])).toThrow(/test/); + expect(() => parseProfileArgs(["--path", "0", "--seed", "3", "--test", ".*"])).toThrow(/one/); + expect(() => parseProfileArgs(["--path", "", "--seed", "3", "--test", "fault-injected"])).toThrow(); + expect(() => parseProfileArgs(["--test", "["])).toThrow(); + expect(() => parseProfileArgs(["--test", "missing-property"])).toThrow(); + expect(parseProfileArgs(["--path", "0:1", "--seed", "3", "--test", "fault-injected"])).toMatchObject({ path: "0:1", replay: true, selectedProperties: [expect.stringContaining("fault-injected")] }); + }); + + it("reports global and native caps without claiming actual runs", () => { + expect(parseProfileArgs(["--runs", "1001"])).toMatchObject({ requestedRuns: 1001, runs: 1000, maximumRuns: 1000, clipped: true }); + for (const [requested, actual] of [[63, 63], [64, 64], [65, 64], [1001, 64]] as const) { + expect(parseProfileArgs(["--suite", "monitor-native", "--runs", String(requested)])).toMatchObject({ requestedRuns: requested, runs: actual, maximumRuns: 64, globalMaximumRuns: 1000, clipped: requested > actual }); + } + }); + + it("spawns one allowlisted file without shell and removes inherited replay/report settings", () => { + const options = parseProfileArgs(["--suite", "monitor-inactivity", "--runs", "3"]); + const inherited = { PATH: "/bin", FC_PATH: "stale", FC_SEED: "9", MONITOR_FUZZ_REPORT: "/user/file", NODE_OPTIONS: "--cpu-prof", PI_LOOP_FUZZ_PROFILE_DIR: "/user/profile", SECRET: "not metadata" }; + const plan = buildChildPlan(options, "/repo", "/owned", inherited); + expect(plan.binary).toBe(process.execPath); + expect(plan.args).toEqual(expect.arrayContaining(["/repo/node_modules/vitest/vitest.mjs", "test/property/monitor-inactivity.property.test.ts", "--maxWorkers=1", "--no-file-parallelism", "--reporter=default", "--reporter=json"])); + expect(plan.env).toMatchObject({ FC_NUM_RUNS: "3", FC_SEED: "130363", PI_LOOP: "off", PI_LOOP_SCOPE: "memory", MONITOR_INACTIVITY_REPORT: "/owned/campaign.json", PI_LOOP_FUZZ_PROFILE_DIR: "/owned" }); + expect(plan.env.FC_PATH).toBeUndefined(); + expect(plan.env.MONITOR_FUZZ_REPORT).toBeUndefined(); + expect(plan.env.NODE_OPTIONS).toBeUndefined(); + expect(inherited.FC_PATH).toBe("stale"); + expect(buildChildPlan(parseProfileArgs(["--path", "0:1", "--seed", "7", "--test", "fault-injected"]), "/repo", "/owned", {}).env.FC_PATH).toBe("0:1"); + }); + + it("links observed counters; unknown is not zero or requested", () => { + expect(summarizeCampaign({ numRuns: 2, numShrinks: 1, failed: true, counters: { executions: 5, commands: 9, operations: { stop: 2 } } }, "monitor-runtime")).toEqual([expect.objectContaining({ actualRuns: 2, checks: 5, shrinks: 1, actions: 9, interrupted: null, failed: true })]); + expect(summarizeCampaign({ completed: 3, coverage: { restart: 8 }, options: { numRuns: 100 } }, "workflow-restart")).toEqual([expect.objectContaining({ actualRuns: null, completed: 3, checks: null, shrinks: null, actions: null, actionCounts: { restart: 8 }, interrupted: null, failed: null })]); + expect(summarizeCampaign({}, "monitor-native")[0]).toMatchObject({ actualRuns: null, interrupted: null, failed: null }); + expect(summarizeCampaign({ numRuns: 1, failed: false, interrupted: true }, "monitor-native")[0]).toMatchObject({ failed: false, interrupted: true }); + expect(() => summarizeCampaign([], "monitor-runtime")).toThrow(); + }); + + it("never reports skips, missing artifacts, interruption, or abnormal child exit as PASS", () => { + const child = { exitCode: 0, signal: null, error: null, timedOut: false }; + const campaign = summarizeCampaign({ numRuns: 3, failed: false }, "monitor-runtime"); + const tests = summarizeVitest({ success: true, testResults: [{ assertionResults: [{ status: "passed", fullName: "property" }] }] }); + const good = { child, campaign, tests, profileCount: 1, errors: [] }; + expect(decideOutcome(good)).toBe("PASS"); + expect(decideOutcome({ ...good, profileCount: 0 })).toBe("INCOMPLETE"); + expect(decideOutcome({ ...good, campaign: [] })).toBe("INCOMPLETE"); + expect(decideOutcome({ ...good, campaign: summarizeCampaign({ numRuns: 0, failed: false }, "monitor-runtime") })).toBe("INCOMPLETE"); + expect(decideOutcome({ ...good, tests: null })).toBe("INCOMPLETE"); + expect(decideOutcome({ ...good, tests: { ...tests, skipped: 1 } })).toBe("INCOMPLETE"); + expect(decideOutcome({ ...good, campaign: summarizeCampaign({ interrupted: true, failed: false }, "monitor-native") })).toBe("INTERRUPTED"); + expect(decideOutcome({ ...good, child: { ...child, timedOut: true, signal: "SIGTERM" } })).toBe("INTERRUPTED"); + expect(decideOutcome({ ...good, child: { ...child, exitCode: 1 }, campaign: summarizeCampaign({ failed: false, interrupted: true }, "monitor-native"), tests: { ...tests, failed: 1 } })).toBe("INTERRUPTED"); + expect(decideOutcome({ ...good, child: { ...child, exitCode: 1 } })).toBe("FAIL"); + expect(decideOutcome({ ...good, errors: ["profile checksum mismatch"] })).toBe("FAIL"); + }); + + it("distinguishes pattern-excluded tests from skipped selected properties", () => { + const report = { success: true, testResults: [{ assertionResults: [{ fullName: "selected", status: "passed" }, { fullName: "excluded", status: "pending" }] }] }; + expect(summarizeVitest(report, ["selected"])).toMatchObject({ passed: 1, skipped: 0, filtered: [{ name: "excluded", status: "pending" }], properties: [{ name: "selected", status: "passed" }] }); + expect(summarizeVitest(report, ["selected", "excluded"]).skipped).toBe(1); + report.testResults[0]!.assertionResults[1]!.status = "failed"; + expect(summarizeVitest(report, ["selected"]).failed).toBe(1); + }); + + it("bounds retained output and hashes the entire observed stream even after truncation", () => { + const collector = outputCollector(4); + collector.add(Buffer.from("abc")); + collector.add(Buffer.from("def")); + expect(collector.finish()).toEqual({ text: "abcd", bytes: 6, truncated: true, sha256: createHash("sha256").update("abcdef").digest("hex") }); + const unicode = outputCollector(1); + unicode.add(Buffer.from("é")); + expect(Buffer.byteLength(unicode.finish().text)).toBeLessThanOrEqual(1); + }); + + it("terminates a timed-out command once, preserves output and bounds an unobserved close", async () => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + try { + const stdout = Object.assign(new EventEmitter(), { destroy: vi.fn() }); + const stderr = Object.assign(new EventEmitter(), { destroy: vi.fn() }); + const child = Object.assign(new EventEmitter(), { stdout, stderr, pid: undefined, kill: vi.fn(() => true), unref: vi.fn() }); + const spawnChild = vi.fn(() => child) as unknown as typeof spawn; + const plan = buildChildPlan(parseProfileArgs(["--runs", "3"]), "/repo", "/owned", {}); + const pending = runChild(plan, 5, spawnChild); + stdout.emit("data", Buffer.from("primary assertion")); + stderr.emit("data", Buffer.from("original error")); + await vi.advanceTimersByTimeAsync(2005); + expect(await pending).toMatchObject({ exitCode: null, signal: null, timedOut: true, error: "Child close unobserved after bounded termination", stdout: "primary assertion", stderr: "original error" }); + expect(spawnChild).toHaveBeenCalledTimes(1); + expect(child.kill.mock.calls).toEqual([["SIGTERM"], ["SIGKILL"]]); + expect(stdout.destroy).toHaveBeenCalledOnce(); + expect(stderr.destroy).toHaveBeenCalledOnce(); + expect(child.unref).toHaveBeenCalledOnce(); + expect(vi.getTimerCount()).toBe(0); + } finally { vi.useRealTimers(); } + }); + + it("requires checksummed worker profiles and never mutates user files", () => { + const dir = mkdtempSync(join(tmpdir(), "fuzz-profile-unit-")); + try { + const sourceFile = "/repo/test/property/monitor-runtime.property.test.ts"; + expect(collectProfiles(dir, sourceFile)).toMatchObject({ profiles: [], errors: [] }); + const data = JSON.stringify({ nodes: [{ id: 1, callFrame: { functionName: "campaign" } }], samples: [1] }); + writeFileSync(join(dir, "worker.cpuprofile"), data); + const metadata = { role: "property-worker", sourceFile, pid: 12, node: process.version, sha256: createHash("sha256").update(data).digest("hex"), sizeBytes: Buffer.byteLength(data), wallTimeMs: 12, resources: { userCPUTimeMicros: 5, systemCPUTimeMicros: 3, maxRSSKiB: 99 }, complete: true }; + writeFileSync(join(dir, "worker.meta.json"), JSON.stringify(metadata)); + expect(collectProfiles(dir, sourceFile)).toMatchObject({ profiles: [expect.objectContaining({ metadata })], errors: [] }); + writeFileSync(join(dir, "worker.meta.json"), JSON.stringify({ ...metadata, role: "coordinator" })); + expect(collectProfiles(dir, sourceFile).errors.join(" ")).toMatch(/worker/); + writeFileSync(join(dir, "worker.meta.json"), JSON.stringify({ ...metadata, sourceFile: "/wrong.test.ts" })); + expect(collectProfiles(dir, sourceFile).errors.join(" ")).toMatch(/worker/); + writeFileSync(join(dir, "worker.meta.json"), JSON.stringify({ ...metadata, sha256: "bad" })); + expect(collectProfiles(dir, sourceFile).errors.join(" ")).toMatch(/checksum/); + expect(readFileSync(join(dir, "worker.cpuprofile"), "utf8")).toBe(data); + } finally { rmSync(dir, { recursive: true, force: true }); } + }); + + it("preserves primary failed output and child status through reporting errors in unique owned runs", async () => { + const dir = mkdtempSync(join(tmpdir(), "fuzz-profile-cli-")); + try { + const options = parseProfileArgs(["--out", dir, "--runs", "3"]); + const runChild = async () => ({ exitCode: 1, signal: null, error: null, timedOut: false, wallTimeMs: 4, stdout: "original assertion trace", stderr: "primary failure", stdoutBytes: 24, stderrBytes: 15, stdoutTruncated: false, stderrTruncated: false, stdoutSHA256: "raw", stderrSHA256: "raw", resourceCoverage: "worker-only; coordinator CPU/RSS unknown" }); + const first = await runProfile(options, "/repo", { runChild, writeSummary: () => { throw new Error("reporting failed"); } }); + const second = await runProfile(options, "/repo", { runChild }); + expect(first.artifactDir).not.toBe(second.artifactDir); + expect(first.outcome).toBe("FAIL"); + expect(first.child.exitCode).toBe(1); + expect(first.errors.join(" ")).toContain("reporting failed"); + expect(readFileSync(join(first.artifactDir, "stdout.log"), "utf8")).toBe("original assertion trace"); + expect(readFileSync(join(first.artifactDir, "stderr.log"), "utf8")).toBe("primary failure"); + expect(JSON.stringify(first)).not.toContain("SECRET"); + } finally { rmSync(dir, { recursive: true, force: true }); } + }); + + it("links raw campaign and reporter hashes independently of their JSON validity", async () => { + const dir = mkdtempSync(join(tmpdir(), "fuzz-profile-links-")); + try { + const options = parseProfileArgs(["--suite", "monitor-inactivity", "--runs", "3", "--out", dir]); + const runChild = async (plan: ReturnType) => { + const owned = plan.env.PI_LOOP_FUZZ_PROFILE_DIR!; + writeFileSync(join(owned, "campaign.json"), "malformed report"); + writeFileSync(join(owned, "vitest.json"), JSON.stringify({ success: true, testResults: [{ assertionResults: [{ status: "passed", fullName: options.selectedProperties[0] }] }] })); + return { exitCode: 0, signal: null, error: null, timedOut: false, wallTimeMs: 4, stdout: "", stderr: "", stdoutBytes: 0, stderrBytes: 0, stdoutTruncated: false, stderrTruncated: false, stdoutSHA256: "raw", stderrSHA256: "raw", resourceCoverage: "unknown" }; + }; + const result = await runProfile(options, process.cwd(), { runChild }); + expect(result.outcome).not.toBe("PASS"); + expect(result.campaignArtifact).toMatchObject({ sha256: createHash("sha256").update("malformed report").digest("hex"), sizeBytes: 16, parseError: expect.any(String) }); + expect(result.testArtifact?.sha256).toBe(createHash("sha256").update(readFileSync(join(result.artifactDir, "vitest.json"))).digest("hex")); + expect(result.profileStatus).toBe("missing"); + } finally { rmSync(dir, { recursive: true, force: true }); } + }); + + it("does not certify profiling evidence when required source identity is unavailable", async () => { + const dir = mkdtempSync(join(tmpdir(), "fuzz-profile-identity-")); + try { + const options = parseProfileArgs(["--suite", "monitor-inactivity", "--runs", "3", "--out", dir]); + const runChild = async (plan: ReturnType) => { + const owned = plan.env.PI_LOOP_FUZZ_PROFILE_DIR!; + writeFileSync(join(owned, "campaign.json"), JSON.stringify({ numRuns: 3, failed: false })); + writeFileSync(join(owned, "vitest.json"), JSON.stringify({ success: true, testResults: [{ assertionResults: [{ status: "passed", fullName: options.selectedProperties[0] }] }] })); + const raw = JSON.stringify({ nodes: [{ id: 1 }], samples: [1] }); + writeFileSync(join(owned, "worker.cpuprofile"), raw); + writeFileSync(join(owned, "worker.meta.json"), JSON.stringify({ role: "property-worker", complete: true, sourceFile: join(dir, plan.file), pid: 1, node: process.version, wallTimeMs: 1, resources: { userCPUTimeMicros: 1, systemCPUTimeMicros: 1, maxRSSKiB: 1 }, sha256: createHash("sha256").update(raw).digest("hex"), sizeBytes: Buffer.byteLength(raw) })); + return { exitCode: 0, signal: null, error: null, timedOut: false, wallTimeMs: 1, stdout: "", stderr: "", stdoutBytes: 0, stderrBytes: 0, stdoutTruncated: false, stderrTruncated: false, stdoutSHA256: "raw", stderrSHA256: "raw", resourceCoverage: "control" }; + }; + const result = await runProfile(options, dir, { runChild }); + expect(result.child.exitCode).toBe(0); + expect(result.tests?.passed).toBe(1); + expect(result.profileCount).toBe(1); + expect(result.sourceManifest.complete).toBe(false); + expect(result.errors).toEqual(expect.arrayContaining([expect.stringContaining("Required source identity"), expect.stringContaining("Source manifest incomplete")])); + expect(result.outcome).toBe("FAIL"); + } finally { rmSync(dir, { recursive: true, force: true }); } + }); + + it("has no default setup profiling or environment/listener/resource mutations during planning", () => { + const environment = { ...process.env }; + const resources = process.getActiveResourcesInfo().sort(); + const listeners = process.eventNames().map(name => [name, process.listenerCount(name)]); + expect(config.test?.setupFiles).toBeUndefined(); + buildChildPlan(parseProfileArgs([]), "/repo", "/owned"); + expect(process.env).toEqual(environment); + expect(process.getActiveResourcesInfo().sort()).toEqual(resources); + expect(process.eventNames().map(name => [name, process.listenerCount(name)])).toEqual(listeners); + }); +}); diff --git a/test/startup-profiling.test.ts b/test/startup-profiling.test.ts new file mode 100644 index 0000000..18dab98 --- /dev/null +++ b/test/startup-profiling.test.ts @@ -0,0 +1,99 @@ +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { runStartupProfile } from "../benchmarks/profile-startup.js"; +import { pairedStartupSummary, preserveStartupCleanup, startupOptions, summarizeDurations } from "../benchmarks/startup-report.js"; + +describe("startup profiling expectations", () => { + it("requires affirmative factory receipts despite a zero-exit CLI swallowing load errors", async () => { + const dir = mkdtempSync(join(tmpdir(), "startup-profile-control-")); + try { + mkdirSync(join(dir, "bin")); + writeFileSync(join(dir, "package.json"), JSON.stringify({ version: "control-only" })); + const cli = join(dir, "bin/cli.mjs"); + writeFileSync(cli, `import {pathToFileURL} from 'node:url'; const i=process.argv.indexOf('-e'); try {if(i>=0)await(await import(pathToFileURL(process.argv[i+1]).href)).default({});} catch {}\n`); + const entry = join(dir, "entry.mjs"); + const args = (out: string) => ["--cli", cli, "--baseline", entry, "--rounds", "3", "--warmups", "0", "--out", join(dir, out)]; + writeFileSync(entry, "export default function() {}\n"); + const normal = await runStartupProfile(args("normal")); + expect(normal.metadata.complete).toBe(true); + expect(normal.metadata.samples).toHaveLength(9); + writeFileSync(entry, "export default function() { throw new Error('throwing factory control'); }\n"); + await expect(runStartupProfile(args("throwing"))).rejects.toThrow(); + const output = join(dir, "throwing", readdirSync(join(dir, "throwing"))[0]!, "report.json"); + const failed = JSON.parse(readFileSync(output, "utf8")); + expect(failed.complete).toBe(false); + expect(failed.samples).toHaveLength(3); + expect(failed.samples[2]).toMatchObject({ name: "baseline", exitCode: 0, signal: null }); + expect(failed.failure).toContain("loaded.json"); + } finally { rmSync(dir, { recursive: true, force: true }); } + }); + + it("retains physical child observations when metrics parsing fails", async () => { + const dir = mkdtempSync(join(tmpdir(), "startup-profile-metrics-")); + try { + mkdirSync(join(dir, "bin")); + writeFileSync(join(dir, "package.json"), JSON.stringify({ version: "control-only" })); + const cli = join(dir, "bin/cli.mjs"); + writeFileSync(cli, `import {writeFileSync} from 'node:fs'; process.once('exit',()=>writeFileSync(process.env.PI_LOOP_STARTUP_METRICS,'invalid metrics'));\n`); + const entry = join(dir, "entry.mjs"); + writeFileSync(entry, "export default function() {}\n"); + const root = join(dir, "out"); + await expect(runStartupProfile(["--cli", cli, "--baseline", entry, "--rounds", "3", "--warmups", "0", "--out", root])).rejects.toThrow(); + const report = JSON.parse(readFileSync(join(root, readdirSync(root)[0]!, "report.json"), "utf8")); + expect(report.complete).toBe(false); + expect(report.samples).toEqual([expect.objectContaining({ name: "none", exitCode: 0, signal: null, metrics: null })]); + expect(report.failure).toBeTruthy(); + } finally { rmSync(dir, { recursive: true, force: true }); } + }); + it("retains primary failure when owned-directory cleanup also fails", () => { + const primary = new Error("primary assertion"); + const cleanup = new Error("cleanup"); + expect(() => preserveStartupCleanup(primary, () => { throw cleanup; })).toThrow(AggregateError); + try { preserveStartupCleanup(primary, () => { throw cleanup; }); } + catch (error) { expect((error as AggregateError).errors).toEqual([primary, cleanup]); } + expect(() => preserveStartupCleanup(primary, () => {})).not.toThrow(); + expect(() => preserveStartupCleanup(undefined, () => { throw cleanup; })).toThrow(cleanup); + }); + it("validates bounded options without partially parsing numbers", () => { + expect(startupOptions([])).toMatchObject({ rounds: 15, warmups: 3, profiles: false }); + for (const value of ["0", "-1", "2x", "1.5", "101", "Infinity"]) { + expect(() => startupOptions(["--rounds", value])).toThrow(); + } + expect(() => startupOptions(["--warmups", "11"])).toThrow(); + expect(() => startupOptions(["--unknown"])).toThrow(); + expect(startupOptions(["--rounds", "3", "--warmups", "0", "--profiles"])).toMatchObject({ rounds: 3, warmups: 0, profiles: true }); + }); + + it("keeps all observations and computes even medians and quantiles", () => { + expect(summarizeDurations([4, 1, 3, 2])).toMatchObject({ n: 4, medianMs: 2.5, minMs: 1, maxMs: 4 }); + expect(() => summarizeDurations([])).toThrow(); + expect(() => summarizeDurations([Number.NaN])).toThrow(); + }); + + it("excludes warmup rounds and pairs observed results, not requested counts", () => { + const samples = [ + { round: 0, warmup: true, name: "baseline", wallMs: 999, exitCode: 0, signal: null }, + { round: 0, warmup: true, name: "candidate", wallMs: 1, exitCode: 0, signal: null }, + ...Array.from({ length: 5 }, (_, i) => [ + { round: i + 1, warmup: false, name: "baseline", wallMs: 120 + i, exitCode: 0, signal: null }, + { round: i + 1, warmup: false, name: "candidate", wallMs: 100 + i, exitCode: 0, signal: null }, + ]).flat(), + ]; + const result = pairedStartupSummary(samples); + expect(result).toMatchObject({ completePairs: 5, medianImprovementMs: 20, bootstrap95Ms: [20, 20] }); + expect(result.pairs.every(pair => pair.improvementMs === 20)).toBe(true); + expect(samples[0]?.wallMs).toBe(999); + }); + + it("rejects missing, duplicate or failed observations rather than declaring improvement", () => { + const baseline = { round: 1, warmup: false, name: "baseline", wallMs: 120, exitCode: 0, signal: null }; + const candidate = { ...baseline, name: "candidate", wallMs: 100 }; + expect(() => pairedStartupSummary([baseline])).toThrow(/pair/); + expect(() => pairedStartupSummary([baseline, baseline, candidate])).toThrow(/duplicate/); + expect(() => pairedStartupSummary([baseline, { ...candidate, exitCode: 1 }])).toThrow(/failed/); + expect(() => pairedStartupSummary([baseline, { ...candidate, signal: "SIGTERM" }])).toThrow(/failed/); + expect(() => pairedStartupSummary([baseline, { ...candidate, error: "spawn failed" }])).toThrow(/failed/); + }); +}); diff --git a/tsconfig.dev.json b/tsconfig.dev.json index ec8f7d5..36f7799 100644 --- a/tsconfig.dev.json +++ b/tsconfig.dev.json @@ -4,5 +4,5 @@ "noEmit": true, "rootDir": "." }, - "include": ["src/**/*.ts", "test/property/**/*.ts", "benchmarks/**/*.ts"] + "include": ["src/**/*.ts", "test/property/**/*.ts", "benchmarks/**/*.ts", "test/startup-profiling.test.ts", "test/fuzz-profiling.test.ts"] } diff --git a/vitest.config.ts b/vitest.config.ts index ecfe923..dfcb2cc 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { environment: "node", + ...(process.env.PI_LOOP_FUZZ_PROFILE_DIR ? { setupFiles: ["./benchmarks/fuzz-profile-setup.ts"] } : {}), include: ["test/**/*.test.{ts,mjs}"], exclude: ["node_modules", "dist"], // CI runners are slower than local; 15s gives real-child-process From 2404a1681e04b05f3a4af9740255db902eb6febf Mon Sep 17 00:00:00 2001 From: trvon Date: Tue, 13 Oct 2026 23:14:29 -0600 Subject: [PATCH 2/6] docs: use an observed fuzz replay example The mismatched minimized path executed zero cases, correctly reported as incomplete despite a passing test. Use the seed/path combination observed to execute 100 cases and retain a worker profile. --- docs/TESTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/TESTING.md b/docs/TESTING.md index 11534ae..08118fb 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -201,7 +201,7 @@ The default is 15 measured rounds after 3 warmups, serial fresh processes, alter ```bash npm run profile:fuzz -- --suite monitor-runtime --runs 100 --seed 130363 npm run profile:fuzz -- --suite monitor-runtime --runs 100 --seed 130363 \ - --test fault-injected --path '5:4:4:4:7:7:7:7:7:7:7' + --test fault-injected --path '0' ``` Suites: `monitor-runtime`, `monitor-inactivity`, `monitor-native`, `workflow-restart`. Runs and seeds are positive safe integers. The runner caps cases at 1,000; native cases retain their existing cap of 64. Replay requires an explicit seed and a pattern selecting exactly one property, with no control tests. Each invocation runs one serial Vitest child without a shell or retries. From fa9179f642667f63b2788907917b4914900c479e Mon Sep 17 00:00:00 2001 From: trvon Date: Wed, 14 Oct 2026 00:39:19 -0600 Subject: [PATCH 3/6] fix(profiling): use portable preload URLs --- benchmarks/profile-startup.ts | 2 +- test/fuzz-profiling.test.ts | 4 +-- test/startup-profiling.test.ts | 57 +++++++++++++++++++++++++++++++++- 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/benchmarks/profile-startup.ts b/benchmarks/profile-startup.ts index 3e9f39e..dfb32fe 100644 --- a/benchmarks/profile-startup.ts +++ b/benchmarks/profile-startup.ts @@ -112,7 +112,7 @@ export async function runStartupProfile(args: string[]) { const profilePath = join(profileDir, `${label}.cpuprofile`); const receiptPath = join(output, `${label}.loaded.json`); if (profile) mkdirSync(profileDir, { recursive: true }); - const childArgs = ["--import", probe, ...(profile ? ["--import", profiler] : []), ...cases[name]!]; + const childArgs = ["--import", pathToFileURL(probe).href, ...(profile ? ["--import", pathToFileURL(profiler).href] : []), ...cases[name]!]; const start = performance.now(); const child = spawnSync(process.execPath, childArgs, { cwd: workspace, env: { ...env, PI_LOOP_STARTUP_METRICS: metricsPath, PI_LOOP_STARTUP_RECEIPT: receiptPath, PI_LOOP_STARTUP_PROFILE: profilePath }, encoding: "utf8", timeout: 30_000, maxBuffer: 1024 * 1024 }); const wallMs = performance.now() - start; diff --git a/test/fuzz-profiling.test.ts b/test/fuzz-profiling.test.ts index b4439db..c449e06 100644 --- a/test/fuzz-profiling.test.ts +++ b/test/fuzz-profiling.test.ts @@ -49,8 +49,8 @@ describe("opt-in fuzz profiling feature", () => { const inherited = { PATH: "/bin", FC_PATH: "stale", FC_SEED: "9", MONITOR_FUZZ_REPORT: "/user/file", NODE_OPTIONS: "--cpu-prof", PI_LOOP_FUZZ_PROFILE_DIR: "/user/profile", SECRET: "not metadata" }; const plan = buildChildPlan(options, "/repo", "/owned", inherited); expect(plan.binary).toBe(process.execPath); - expect(plan.args).toEqual(expect.arrayContaining(["/repo/node_modules/vitest/vitest.mjs", "test/property/monitor-inactivity.property.test.ts", "--maxWorkers=1", "--no-file-parallelism", "--reporter=default", "--reporter=json"])); - expect(plan.env).toMatchObject({ FC_NUM_RUNS: "3", FC_SEED: "130363", PI_LOOP: "off", PI_LOOP_SCOPE: "memory", MONITOR_INACTIVITY_REPORT: "/owned/campaign.json", PI_LOOP_FUZZ_PROFILE_DIR: "/owned" }); + expect(plan.args).toEqual(expect.arrayContaining([join("/repo", "node_modules/vitest/vitest.mjs"), "test/property/monitor-inactivity.property.test.ts", "--maxWorkers=1", "--no-file-parallelism", "--reporter=default", "--reporter=json"])); + expect(plan.env).toMatchObject({ FC_NUM_RUNS: "3", FC_SEED: "130363", PI_LOOP: "off", PI_LOOP_SCOPE: "memory", MONITOR_INACTIVITY_REPORT: join("/owned", "campaign.json"), PI_LOOP_FUZZ_PROFILE_DIR: "/owned" }); expect(plan.env.FC_PATH).toBeUndefined(); expect(plan.env.MONITOR_FUZZ_REPORT).toBeUndefined(); expect(plan.env.NODE_OPTIONS).toBeUndefined(); diff --git a/test/startup-profiling.test.ts b/test/startup-profiling.test.ts index 18dab98..704e90f 100644 --- a/test/startup-profiling.test.ts +++ b/test/startup-profiling.test.ts @@ -1,11 +1,66 @@ +import { spawnSync } from "node:child_process"; import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { pathToFileURL } from "node:url"; +import { describe, expect, it, vi } from "vitest"; + import { runStartupProfile } from "../benchmarks/profile-startup.js"; import { pairedStartupSummary, preserveStartupCleanup, startupOptions, summarizeDurations } from "../benchmarks/startup-report.js"; +vi.mock("node:child_process", async () => { + const actual = await vi.importActual("node:child_process"); + return { ...actual, spawnSync: vi.fn(actual.spawnSync) }; +}); + describe("startup profiling expectations", () => { + it("rejects raw Windows drive preloads while an existing file URL loads", () => { + const dir = mkdtempSync(join(tmpdir(), "startup-loader-")); + try { + const preload = join(dir, "preload space é #.mjs"); + writeFileSync(preload, "process.stdout.write('owned preload loaded');\n"); + const rejected = spawnSync(process.execPath, ["--import", String.raw`C:\owned\preload.mjs`, "--eval", ""], { encoding: "utf8", timeout: 5000, maxBuffer: 1024 * 1024 }); + expect(rejected.status).toBe(1); + expect(rejected.signal).toBeNull(); + expect(rejected.stderr).toContain("ERR_UNSUPPORTED_ESM_URL_SCHEME"); + expect(rejected.stderr).toContain("c:"); + expect(rejected.stdout).not.toContain("owned preload loaded"); + const loaded = spawnSync(process.execPath, ["--import", pathToFileURL(preload).href, "--eval", ""], { encoding: "utf8", timeout: 5000, maxBuffer: 1024 * 1024 }); + expect(loaded.status).toBe(0); + expect(loaded.signal).toBeNull(); + expect(loaded.stderr).toBe(""); + expect(loaded.stdout).toBe("owned preload loaded"); + } finally { rmSync(dir, { recursive: true, force: true }); } + }); + + it("passes file URLs for every actual metrics and CPU preload", async () => { + const dir = mkdtempSync(join(tmpdir(), "startup-profile-argv-")); + try { + mkdirSync(join(dir, "bin")); + writeFileSync(join(dir, "package.json"), JSON.stringify({ version: "control-only" })); + const cli = join(dir, "bin/cli.mjs"); + writeFileSync(cli, `import {pathToFileURL} from 'node:url'; const i=process.argv.indexOf('-e'); if(i>=0)await(await import(pathToFileURL(process.argv[i+1]).href)).default({});\n`); + const entry = join(dir, "entry space é #.mjs"); + writeFileSync(entry, "export default function() {}\n"); + vi.mocked(spawnSync).mockClear(); + const result = await runStartupProfile(["--cli", cli, "--baseline", entry, "--rounds", "3", "--warmups", "0", "--profiles", "--out", join(dir, "out")]); + expect(result.metadata.complete).toBe(true); + expect(result.metadata.samples).toHaveLength(9); + expect(result.metadata.profileSamples).toHaveLength(3); + expect(result.metadata.profiles).toHaveLength(3); + const children = vi.mocked(spawnSync).mock.calls.filter(([binary]) => binary === process.execPath); + expect(children).toHaveLength(12); + let preloads = 0; + for (const [, args] of children) { + for (let i = 0; i < (args?.length ?? 0); i++) { + if (args?.[i] !== "--import") continue; + preloads++; + expect(args[i + 1]).toMatch(/^file:\/\//); + } + } + expect(preloads).toBe(15); + } finally { rmSync(dir, { recursive: true, force: true }); } + }); it("requires affirmative factory receipts despite a zero-exit CLI swallowing load errors", async () => { const dir = mkdtempSync(join(tmpdir(), "startup-profile-control-")); try { From b820312ae2149175df65a1e06ea2f51838e68c7e Mon Sep 17 00:00:00 2001 From: trvon Date: Wed, 14 Oct 2026 00:39:20 -0600 Subject: [PATCH 4/6] test(monitor): await observed idle completion --- test/index.test.ts | 43 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/test/index.test.ts b/test/index.test.ts index 55d660e..3842038 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -13,6 +13,20 @@ import { TriggerSystem } from "../src/trigger-system.js"; import type { LoopEntry, LoopFireOrigin } from "../src/types.js"; import { createCtx, createMockPi, flushAsync } from "./helpers/mock-pi.js"; +async function expectIdleMonitorWake(receipts: Pick, "emittedEvents" | "sentMessages">, prompt: string, exitCode: number): Promise { + // Child close and wake delivery can occur after an arbitrary sleep. + await vi.waitFor(() => { + const terminal = receipts.emittedEvents.filter(event => event.name === "monitor:finished" && event.payload.monitorId === "1"); + expect(terminal).toHaveLength(1); + expect(terminal[0].payload).toMatchObject({ monitorId: "1", status: exitCode === 0 ? "completed" : "error", exitCode }); + expect(receipts.sentMessages).toHaveLength(1); + expect(receipts.sentMessages[0].message.content).toContain(prompt); + }, { timeout: 5000, interval: 10 }); + await flushAsync(); + expect(receipts.sentMessages).toHaveLength(1); + expect(receipts.sentMessages[0].message.content).toContain(prompt); +} + function readJsonFile(path: string): any { try { return JSON.parse(readFileSync(path, "utf-8")); @@ -3065,7 +3079,7 @@ describe("monitor tool wrappers", () => { }, 10000); it("delivers monitor completion wake immediately in an idle session after agent_end", async () => { - const { pi, toolMap, extensionHandlers, sentMessages: sentCustomMessages } = createMockPi(); + const { pi, toolMap, extensionHandlers, sentMessages: sentCustomMessages, emittedEvents } = createMockPi(); extension(pi as any); await vi.advanceTimersByTimeAsync(6100); @@ -3095,16 +3109,14 @@ describe("monitor tool wrappers", () => { }); expect(result.content[0].text).toContain("Completion wake loop"); - // Wait for process to exit and wake to deliver - await new Promise(r => setTimeout(r, 500)); + await expectIdleMonitorWake({ emittedEvents, sentMessages: sentCustomMessages }, "Monitor completed during idle session", 0); - // It should deliver the wake immediately because the session is idle! expect(sentCustomMessages).toHaveLength(1); expect((sentCustomMessages[0].message as { content: string }).content).toContain("Monitor completed during idle session"); }, 10000); it("delivers monitor completion wake even when the command exits with non-zero status in an idle session", async () => { - const { pi, toolMap, extensionHandlers, sentMessages: sentCustomMessages } = createMockPi(); + const { pi, toolMap, extensionHandlers, sentMessages: sentCustomMessages, emittedEvents } = createMockPi(); extension(pi as any); await vi.advanceTimersByTimeAsync(6100); @@ -3132,13 +3144,32 @@ describe("monitor tool wrappers", () => { }); expect(result.content[0].text).toContain("Completion wake loop"); - await new Promise(r => setTimeout(r, 500)); + await expectIdleMonitorWake({ emittedEvents, sentMessages: sentCustomMessages }, "Monitor failed in idle session", 3); // Wake should still be delivered to notify about the failure! expect(sentCustomMessages).toHaveLength(1); expect((sentCustomMessages[0].message as { content: string }).content).toContain("Monitor failed in idle session"); }, 10000); + it("observes delayed nonzero completion before asserting its idle wake", async () => { + const { pi, toolMap, extensionHandlers, sentMessages, emittedEvents } = createMockPi(); + extension(pi as any); + await vi.advanceTimersByTimeAsync(6100); + vi.useRealTimers(); + const ctx = createCtx(); + for (const name of ["turn_start", "agent_start", "agent_end"]) { + for (const handler of extensionHandlers.get(name) ?? []) await handler(null, ctx); + } + const result = await toolMap.get("MonitorCreate")!.execute?.("1", { + command: "sleep 1; exit 3", + onDone: "Delayed nonzero idle wake", + }); + expect(result.content[0].text).toContain("Completion wake loop"); + await expectIdleMonitorWake({ emittedEvents, sentMessages }, "Delayed nonzero idle wake", 3); + expect(sentMessages).toHaveLength(1); + expect((sentMessages[0].message as { content: string }).content).toContain("Delayed nonzero idle wake"); + }, 10000); + it("delivers monitor completion wake when the command times out", async () => { const { pi, toolMap, extensionHandlers, sentMessages: sentCustomMessages } = createMockPi(); From 7d3412127c592b2a4f323ba2f78e46b26271da55 Mon Sep 17 00:00:00 2001 From: trvon Date: Wed, 14 Oct 2026 07:29:19 -0600 Subject: [PATCH 5/6] test(monitor): await custom completion receipt --- test/index.test.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/test/index.test.ts b/test/index.test.ts index 3842038..4e0c0a2 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -2778,8 +2778,11 @@ describe("monitor tool wrappers", () => { expect(result.content[0].text).toContain("not found"); }); - it("onDone monitor completion delivers a custom message wake", async () => { - const { pi, toolMap, sentMessages: sentCustomMessages } = createMockPi(); + it.each([ + { title: "onDone monitor completion delivers a custom message wake", command: "echo 'monitor done'" }, + { title: "onDone delayed completion preserves the custom wake outcome", command: "sleep 1; echo 'monitor done'" }, + ])("$title", async ({ command }) => { + const { pi, toolMap, emittedEvents, sentMessages: sentCustomMessages } = createMockPi(); extension(pi as any); await vi.advanceTimersByTimeAsync(6100); @@ -2789,12 +2792,12 @@ describe("monitor tool wrappers", () => { expect(monitorCreate?.execute).toBeDefined(); const result = await monitorCreate!.execute?.("1", { - command: "echo 'monitor done'", + command, onDone: "Monitor finished — report results", }); expect(result.content[0].text).toContain("Completion wake loop"); - await new Promise(r => setTimeout(r, 500)); + await expectIdleMonitorWake({ emittedEvents, sentMessages: sentCustomMessages }, "Monitor finished — report results", 0); expect(sentCustomMessages).toHaveLength(1); expect((sentCustomMessages[0].message as { content: string }).content).toContain( @@ -3017,7 +3020,7 @@ describe("monitor tool wrappers", () => { }, 10000); it("onDone monitor completion does not rely on monitor:done event dispatch", async () => { - const { pi, toolMap, sentMessages: sentCustomMessages } = createMockPi({ suppressMonitorDoneDispatch: true }); + const { pi, toolMap, emittedEvents, sentMessages: sentCustomMessages } = createMockPi({ suppressMonitorDoneDispatch: true }); extension(pi as any); await vi.advanceTimersByTimeAsync(6100); @@ -3032,7 +3035,7 @@ describe("monitor tool wrappers", () => { }); expect(result.content[0].text).toContain("Completion wake loop"); - await new Promise(r => setTimeout(r, 500)); + await expectIdleMonitorWake({ emittedEvents, sentMessages: sentCustomMessages }, "Monitor finished without event dispatch", 0); expect(sentCustomMessages).toHaveLength(1); expect((sentCustomMessages[0].message as { content: string }).content).toContain("Monitor finished without event dispatch"); From ab350bb128b53ae7ed9319022aeefee66eeddb9b Mon Sep 17 00:00:00 2001 From: Trevon Date: Wed, 14 Oct 2026 08:31:21 -0600 Subject: [PATCH 6/6] Potential fix for pull request finding 'CodeQL / Improper code sanitization' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- benchmarks/profile-startup.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/benchmarks/profile-startup.ts b/benchmarks/profile-startup.ts index dfb32fe..7f6322b 100644 --- a/benchmarks/profile-startup.ts +++ b/benchmarks/profile-startup.ts @@ -9,6 +9,21 @@ import { build } from "esbuild"; import { pairedStartupSummary, preserveStartupCleanup, type StartupSample, startupOptions, summarizeDurations } from "./startup-report.js"; const sha256 = (data: string | Buffer) => createHash("sha256").update(data).digest("hex"); +const unsafeCharMap: Record = { + "<": "\\u003C", + ">": "\\u003E", + "/": "\\u002F", + "\\": "\\\\", + "\b": "\\b", + "\f": "\\f", + "\n": "\\n", + "\r": "\\r", + "\t": "\\t", + "\0": "\\0", + "\u2028": "\\u2028", + "\u2029": "\\u2029", +}; +const escapeUnsafeChars = (value: string) => value.replace(/[<>\/\\\b\f\n\r\t\0\u2028\u2029]/g, (ch) => unsafeCharMap[ch]!); function readJson(path: string, decode: (value: unknown) => T): T { try { return decode(JSON.parse(readFileSync(path, "utf8"))); } catch (cause) { throw new Error(`Cannot read profiling JSON: ${path}`, { cause }); } @@ -152,7 +167,7 @@ export async function runStartupProfile(args: string[]) { writeFileSync(profiler, `import {Session} from 'node:inspector'; import {writeFileSync} from 'node:fs'; const s=new Session(); s.connect(); const post=m=>new Promise((r,j)=>s.post(m,(e,v)=>e?j(e):r(v))); await post('Profiler.enable'); await post('Profiler.start'); process.once('exit',()=>s.post('Profiler.stop',(e,v)=>{if(e)throw e; const raw=JSON.stringify(v.profile); if(Buffer.byteLength(raw)>${metadata.maximumProfileBytes}) throw new Error('Startup CPU profile exceeds byte limit'); writeFileSync(process.env.PI_LOOP_STARTUP_PROFILE,raw,{flag:'wx'}); s.disconnect();}));\n`); for (const [name, target] of Object.entries(targets)) { const href = pathToFileURL(target).href; - writeFileSync(wrappers[name]!, `import factory from ${JSON.stringify(href)}; import {writeFileSync} from 'node:fs'; export default function(pi){if(typeof factory!=='function')throw new Error('No extension factory'); const value=factory(pi); const receipt=()=>writeFileSync(process.env.PI_LOOP_STARTUP_RECEIPT,JSON.stringify({target:${JSON.stringify(href)},phase:'factory-returned',node:process.version}),{flag:'wx'}); if(value&&typeof value.then==='function')return Promise.resolve(value).then(result=>{receipt();return result;}); receipt();return value;}\n`); + writeFileSync(wrappers[name]!, `import factory from ${escapeUnsafeChars(JSON.stringify(href))}; import {writeFileSync} from 'node:fs'; export default function(pi){if(typeof factory!=='function')throw new Error('No extension factory'); const value=factory(pi); const receipt=()=>writeFileSync(process.env.PI_LOOP_STARTUP_RECEIPT,JSON.stringify({target:${escapeUnsafeChars(JSON.stringify(href))},phase:'factory-returned',node:process.version}),{flag:'wx'}); if(value&&typeof value.then==='function')return Promise.resolve(value).then(result=>{receipt();return result;}); receipt();return value;}\n`); } save(); for (let round = 0; round < options.rounds + options.warmups; round++) {