From 9e1110017bbbb6028c6e2e52e88f34a3cb083d2d Mon Sep 17 00:00:00 2001 From: Nick Nisi Date: Mon, 28 Sep 2026 16:04:47 -0500 Subject: [PATCH 1/6] fix(ruby): deliver credentials and require real application auth wiring --- .gitignore | 3 + src/integrations/ruby/index.spec.ts | 105 ++++++++++++++++++++++++ src/integrations/ruby/index.ts | 73 ++++++++++------- src/lib/completion-data.ts | 6 +- src/lib/run-with-core.ruby.spec.ts | 120 ++++++++++++++++++++++++++++ 5 files changed, 278 insertions(+), 29 deletions(-) create mode 100644 src/integrations/ruby/index.spec.ts create mode 100644 src/lib/run-with-core.ruby.spec.ts diff --git a/.gitignore b/.gitignore index 725c1b92..d9173d95 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,9 @@ src/generated/agent-sdk-manifest.ts *.sublime-* dist/ +# Isolated local preparation and verification artifacts +.artifacts/ + # Eval results tests/eval-results/ .next/ diff --git a/src/integrations/ruby/index.spec.ts b/src/integrations/ruby/index.spec.ts new file mode 100644 index 00000000..bfdc6bd5 --- /dev/null +++ b/src/integrations/ruby/index.spec.ts @@ -0,0 +1,105 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { run } from './index.js'; +import { initializeAgent, runAgent } from '../../lib/agent-interface.js'; +import { getOrAskForWorkOSCredentials } from '../../utils/ui-utils.js'; +import { autoConfigureWorkOSEnvironment } from '../../lib/workos-management.js'; +import type { InstallerOptions } from '../../utils/types.js'; + +vi.mock('../../lib/agent-interface.js', () => ({ initializeAgent: vi.fn(), runAgent: vi.fn() })); +vi.mock('../../utils/ui-utils.js', () => ({ getOrAskForWorkOSCredentials: vi.fn() })); +vi.mock('../../lib/workos-management.js', () => ({ autoConfigureWorkOSEnvironment: vi.fn() })); +vi.mock('../../lib/skills-assets.js', () => ({ getReference: vi.fn(async () => 'Pinned Ruby reference') })); +vi.mock('../../utils/analytics.js', () => ({ analytics: { capture: vi.fn(), shutdown: vi.fn() } })); + +let directory: string; +let options: InstallerOptions; +beforeEach(async () => { + vi.clearAllMocks(); + vi.stubGlobal( + 'fetch', + vi.fn(() => { + throw new Error('Unexpected network'); + }), + ); + directory = await mkdtemp(join(tmpdir(), 'ruby-integration-')); + options = { installDir: directory, debug: false, forceInstall: false, local: false, ci: true, skipAuth: true }; + await mkdir(join(directory, 'config')); + await writeFile(join(directory, 'config/puma.rb'), 'port ENV.fetch("PORT", 4100)'); + vi.mocked(getOrAskForWorkOSCredentials).mockResolvedValue({ + apiKey: 'sk_test_synthetic', + clientId: 'client_synthetic', + }); + vi.mocked(runAgent).mockResolvedValue({}); +}); +afterEach(async () => { + vi.unstubAllGlobals(); + await rm(directory, { recursive: true, force: true }); +}); + +describe('real Ruby integration with fake agent and credentials', () => { + it.each([undefined, 'http://app.fizzy.localhost:3006/workos/callback'])( + 'uses one callback and origin (%s)', + async (redirectUri) => { + const callback = redirectUri ?? 'http://localhost:4100/auth/callback'; + const origin = new URL(callback).origin; + const summary = await run({ ...options, redirectUri }); + const prompt = vi.mocked(runAgent).mock.calls[0][1]; + expect(prompt).toContain(`WORKOS_REDIRECT_URI=${callback}`); + expect(prompt).toContain(`${origin}/auth/login`); + expect(prompt).toContain(`${origin}/`); + expect(summary).toContain(callback); + expect(summary).toContain('not verified'); + expect(summary).not.toContain('What the agent did'); + expect(autoConfigureWorkOSEnvironment).not.toHaveBeenCalled(); + expect(fetch).not.toHaveBeenCalled(); + }, + ); + + it.each([false, true])( + 'writes selected credentials without putting secrets in the prompt (package.json: %s)', + async (hasPackage) => { + if (hasPackage) await writeFile(join(directory, 'package.json'), '{}'); + const file = hasPackage ? '.env.local' : '.env'; + await writeFile(join(directory, file), 'OTHER=preserved\n'); + await run(options); + const env = await readFile(join(directory, file), 'utf8'); + expect(env).toContain('WORKOS_API_KEY=sk_test_synthetic'); + expect(env).toContain('WORKOS_CLIENT_ID=client_synthetic'); + expect(env).toContain('WORKOS_REDIRECT_URI=http://localhost:4100/auth/callback'); + expect(env).toContain('OTHER=preserved'); + const prompt = vi.mocked(runAgent).mock.calls[0][1]; + expect(prompt).toContain(file); + expect(prompt).toContain('loaded before WorkOS initialization'); + expect(prompt).not.toContain('sk_test_synthetic'); + expect(await readFile(join(directory, '.gitignore'), 'utf8')).toContain(file); + expect(initializeAgent).toHaveBeenCalledWith(expect.objectContaining({ workingDirectory: directory }), options); + }, + ); + + it('requires real UI/session integration without choosing account policy', async () => { + await run(options); + const prompt = vi.mocked(runAgent).mock.calls[0][1]; + for (const requirement of [ + 'visible sign-in', + 'signed-in account', + 'repeat login', + 'existing authorization', + 'account-linking', + 'magic-link', + 'passkey', + 'SDK', + 'protected access', + 'not global provider-session revocation', + ]) { + expect(prompt).toContain(requirement); + } + }); + + it('does not report success on agent failure', async () => { + vi.mocked(runAgent).mockResolvedValue({ error: 'synthetic failure' }); + await expect(run(options)).rejects.toThrow('synthetic failure'); + }); +}); diff --git a/src/integrations/ruby/index.ts b/src/integrations/ruby/index.ts index 30c1919e..3bb94d99 100644 --- a/src/integrations/ruby/index.ts +++ b/src/integrations/ruby/index.ts @@ -7,7 +7,11 @@ import { analytics } from '../../utils/analytics.js'; import { INSTALLER_INTERACTION_EVENT_NAME } from '../../lib/constants.js'; import { initializeAgent, runAgent } from '../../lib/agent-interface.js'; import { getOrAskForWorkOSCredentials } from '../../utils/ui-utils.js'; -import { autoConfigureWorkOSEnvironment } from '../../lib/workos-management.js'; +import { basename } from 'node:path'; +import { resolveRedirectUri, getSignInPath } from '../../lib/port-detection.js'; +import { buildApplicationSetup } from '../../lib/authkit-application-setup.js'; +import { writeCredentialsEnv } from '../../lib/env-writer.js'; +import { resolveProjectEnvPath } from '../../lib/project-env.js'; import { getReference } from '../../lib/skills-assets.js'; import { buildSignInSection } from '../../lib/sign-in-route.js'; @@ -46,15 +50,14 @@ export const config: FrameworkConfig = { prompts: {}, ui: { - successMessage: 'WorkOS AuthKit integration complete', + successMessage: 'Ruby agent finished; integration verification pending', getOutroChanges: () => [ - 'Analyzed your Rails project structure', - 'Installed and configured the WorkOS Ruby SDK', - 'Created authentication controller with login, callback, and logout', - 'Added authentication routes to config/routes.rb', + 'Requested SDK configuration, visible auth controls, and application session integration', + 'Requested login, callback, and safe logout routes preserving existing authorization', ], getOutroNextSteps: () => [ - 'Start your Rails server with `rails server` to test authentication', + 'Review the diff and use the project’s documented launcher; a Gemfile does not verify startup behavior', + 'Verify visible auth controls, callback identity/account context, repeat login, and protected access after logout', 'Visit the WorkOS Dashboard to manage users and settings', ], }, @@ -79,23 +82,25 @@ export async function run(options: InstallerOptions): Promise { }); // Get WorkOS credentials - const { apiKey, clientId: _clientId } = await getOrAskForWorkOSCredentials( - options, - config.environment.requiresApiKey, - ); - - // Auto-configure WorkOS environment (redirect URI, CORS, homepage) if not already done - const callerHandledConfig = Boolean(options.apiKey || options.clientId); - if (!callerHandledConfig && apiKey) { - const port = 3000; // Rails default - await autoConfigureWorkOSEnvironment(apiKey, config.metadata.integration, port, { - homepageUrl: options.homepageUrl, - redirectUri: options.redirectUri, - }); - } + const { apiKey, clientId } = await getOrAskForWorkOSCredentials(options, config.environment.requiresApiKey); + + // The common installer owns URL provisioning after the agent, with a single + // sandbox target and read-back. Never perform legacy pre-agent URL writes here. + const redirectUri = resolveRedirectUri('ruby', options); + const setup = buildApplicationSetup({ + clientId, + redirectUri, + homepageUrl: options.homepageUrl, + signInPath: getSignInPath('ruby'), + }); + writeCredentialsEnv(options.installDir, { + WORKOS_API_KEY: apiKey, + WORKOS_CLIENT_ID: clientId, + WORKOS_REDIRECT_URI: redirectUri, + }); + const envFile = basename(resolveProjectEnvPath(options.installDir)); - // Build prompt for the agent - const redirectUri = options.redirectUri || 'http://localhost:3000/auth/callback'; + // Keep credentials out of the prompt/transcript; the agent can read the ignored file. const refContent = await getReference('workos-ruby'); const prompt = `You are integrating WorkOS AuthKit into this Ruby on Rails application. @@ -106,7 +111,9 @@ export async function run(options: InstallerOptions): Promise { ## Environment -The following environment variables are needed (create a .env file if one does not exist): +The installer wrote the selected credentials to the gitignored ${envFile}: +Ensure this file is loaded before WorkOS initialization using the project's existing environment-loading convention (Rails does not load dotenv files by itself). Preserve unrelated settings. Never print secrets or commit them. Verify variable presence without displaying values. +The variables are: - WORKOS_API_KEY - WORKOS_CLIENT_ID - WORKOS_REDIRECT_URI=${redirectUri} @@ -115,7 +122,17 @@ The following environment variables are needed (create a .env file if one does n ${refContent} -${buildSignInSection(config)}Report your progress using [STATUS] prefixes. +${buildSignInSection(config)}## Application integration requirements (take precedence over generic examples) + +- Use callback ${redirectUri}, Initiate login ${setup.initiateLoginUri}, and sign-out return destination ${setup.signOutUri}. CORS origin is ${new URL(redirectUri).origin}. The sign-out return destination is not the logout action. Do not guess a different host/port from Puma when an explicit callback is supplied. Do not write dashboard settings; the installer configures the selected environment after agent execution. +- Add visible sign-in controls while signed out and signed-in account/logout controls in the existing layouts/navigation. Wire real routes, not unused SDK examples. Preserve existing routes; if the required sign-in path conflicts, report the conflict rather than silently replacing it or choosing an unregistered alternative. +- Trace the app's real identity, session, account scope, and active membership/role checks. The callback must establish that existing authenticated context, not merely store an unrelated token or replace current_user. Preserve existing authorization and cross-account boundaries. +- Do not invent account-linking, identity/account auto-creation, membership or role assignment policy. Ask the user for the mapping policy if absent; leave that integration pending rather than using User.find_or_create_by(email:) or a hardcoded identity. Preserve magic-link and passkey behavior; do not silently replace the authentication system. +- Ensure repeat login does not duplicate identities, accounts, or memberships under the approved policy. +- Implement safe logout using the app's session termination/cookie clearing and the installed SDK's supported session logout behavior. Use CSRF protection for local session mutation. Verify protected access is denied afterward, including replay of the old app session. Local cookie deletion is not global provider-session revocation. If the SDK cannot end the upstream session, report that limitation instead of claiming logout is complete. +- Add local route/session tests with synthetic identities and stubbed network where possible. Distinguish source changes from checks actually run; source strings and a successful agent exit are not behavioral proof. Report commands/results, unavailable checks, and pending policy decisions. Hosted AuthKit/browser flows remain unverified until actually exercised. + +Report your progress using [STATUS] prefixes. Begin integration now.`; @@ -152,9 +169,11 @@ Begin integration now.`; const nextSteps = config.ui.getOutroNextSteps({}); const lines: string[] = [ - 'Successfully installed WorkOS AuthKit!', + 'Ruby agent finished. Application behavior and hosted AuthKit flows are not verified.', + `Requested callback: ${redirectUri}`, + 'Application URL registration is handled separately by the installer after this step.', '', - 'What the agent did:', + 'Instructions given to the agent (not verified changes):', ...changes.map((c) => `• ${c}`), '', 'Next steps:', diff --git a/src/lib/completion-data.ts b/src/lib/completion-data.ts index 1dc8a26d..a102efc4 100644 --- a/src/lib/completion-data.ts +++ b/src/lib/completion-data.ts @@ -63,11 +63,13 @@ export async function buildCompletionData(ctx: CompletionContext, deps: Completi const dev = await deps.resolveDevCommand(ctx.installDir); const devCommand = [dev.command, ...dev.args].join(' '); const port = deps.detectPort(ctx.integration as Integration, ctx.installDir); - const url = `http://localhost:${port}`; + const url = deps.applicationSetup ? new URL(deps.applicationSetup.redirectUri).origin : `http://localhost:${port}`; const files = ctx.changedFiles ?? []; const concrete = [ - `Run \`${devCommand}\` to start your dev server`, + ctx.integration === 'ruby' + ? `Use the project's documented launcher (inferred command: \`${devCommand}\`; startup not verified)` + : `Run \`${devCommand}\` to start your dev server`, `Open ${url} to test authentication`, ...(deps.signInSnippet ? [deps.signInSnippet] : []), ]; diff --git a/src/lib/run-with-core.ruby.spec.ts b/src/lib/run-with-core.ruby.spec.ts new file mode 100644 index 00000000..eeb14bd3 --- /dev/null +++ b/src/lib/run-with-core.ruby.spec.ts @@ -0,0 +1,120 @@ +import { afterEach, expect, it, vi } from 'vitest'; +import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { runWithCore } from './run-with-core.js'; +import { runAgent } from './agent-interface.js'; +import { configureAuthkitApplication } from './authkit-application-setup.js'; +import { setOutputMode } from '../utils/output.js'; + +vi.mock('./agent-interface.js', () => ({ initializeAgent: vi.fn(), runAgent: vi.fn(async () => ({})) })); +vi.mock('./skills-assets.js', () => ({ getReference: vi.fn(async () => 'Ruby reference') })); +vi.mock('./authkit-application-setup.js', async (original) => ({ + ...(await original()), + configureAuthkitApplication: vi.fn(async (setup) => ({ + ...setup, + callbackRegistered: true, + verified: false, + reason: 'Synthetic pending read-back', + })), +})); +vi.mock('./credentials.js', () => ({ getAccessToken: vi.fn(() => null), saveCredentials: vi.fn() })); +vi.mock('./config-store.js', () => ({ + getActiveEnvironment: vi.fn(() => null), + isUnclaimedEnvironment: vi.fn(() => false), +})); +vi.mock('../utils/debug.js', () => ({ + initLogFile: vi.fn(), + enableDebugLogs: vi.fn(), + logInfo: vi.fn(), + logError: vi.fn(), + debug: vi.fn(), +})); +vi.mock('../utils/analytics.js', () => ({ + analytics: { + setGatewayUrl: vi.fn(), + capture: vi.fn(), + configureAuthFromAvailableSources: vi.fn(), + sessionStart: vi.fn(), + shutdown: vi.fn(), + setTag: vi.fn(), + }, +})); +vi.mock('./post-install.js', () => ({ + detectChanges: vi.fn(() => ({ hasChanges: false, files: [] })), + stageAndCommit: vi.fn(), + pushBranch: vi.fn(), + createPullRequest: vi.fn(), +})); +vi.mock('../utils/git-utils.js', () => ({ + getCurrentBranch: vi.fn(() => 'synthetic'), + isProtectedBranch: vi.fn(() => false), + createBranch: vi.fn(), + branchExists: vi.fn(() => false), +})); +vi.mock('../utils/ui-utils.js', () => ({ + getPackageDotJson: vi.fn(), + isInGitRepo: vi.fn(() => false), + getUncommittedOrUntrackedFiles: vi.fn(() => []), + getOrAskForWorkOSCredentials: vi.fn(async (options) => ({ apiKey: options.apiKey, clientId: options.clientId })), +})); + +let directory: string; +afterEach(async () => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + setOutputMode('human'); + if (directory) await rm(directory, { recursive: true, force: true }); +}); + +it('runs the real Ruby installer before the common URL path and reports the explicit origin without claiming verification', async () => { + directory = await mkdtemp(join(tmpdir(), 'ruby-orchestration-')); + await mkdir(join(directory, 'config')); + await writeFile(join(directory, 'Gemfile'), 'gem "rails"'); + await writeFile(join(directory, 'config/puma.rb'), 'port 3000'); + vi.stubGlobal( + 'fetch', + vi.fn(() => { + throw new Error('Unexpected network'); + }), + ); + const output: string[] = []; + vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => { + output.push(String(chunk)); + return true; + }); + setOutputMode('json'); + await runWithCore({ + installDir: directory, + integration: 'ruby', + apiKey: 'sk_test_synthetic', + clientId: 'client_synthetic', + redirectUri: 'http://app.fizzy.localhost:3006/auth/callback', + ci: true, + debug: false, + local: false, + forceInstall: false, + skipAuth: true, + noBranch: true, + noCommit: true, + noGitCheck: true, + }); + expect(runAgent).toHaveBeenCalledOnce(); + expect(configureAuthkitApplication).toHaveBeenCalledOnce(); + expect(vi.mocked(runAgent).mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(configureAuthkitApplication).mock.invocationCallOrder[0], + ); + expect(configureAuthkitApplication).toHaveBeenCalledWith( + expect.objectContaining({ + redirectUri: 'http://app.fizzy.localhost:3006/auth/callback', + corsOrigin: 'http://app.fizzy.localhost:3006', + signOutUri: 'http://app.fizzy.localhost:3006/', + initiateLoginUri: 'http://app.fizzy.localhost:3006/auth/login', + }), + 'client_synthetic', + 'sk_test_synthetic', + ); + expect(output.join('')).toContain('Synthetic pending read-back'); + expect(output.join('')).toContain('startup not verified'); + expect(fetch).not.toHaveBeenCalled(); +}); From 15ff87e8752b5523dc27b02ff3a8e11d60d05bc1 Mon Sep 17 00:00:00 2001 From: Nick Nisi Date: Mon, 28 Sep 2026 16:04:47 -0500 Subject: [PATCH 2/6] test(evals): add pinned isolated Fizzy preparation and opt-in scenario --- tests/evals/__tests__/agent-executor.spec.ts | 25 ++- tests/evals/agent-executor.ts | 11 +- tests/evals/cli.ts | 1 + tests/evals/fixture-manager.ts | 16 +- tests/evals/fizzy-fixture.spec.ts | 184 +++++++++++++++++++ tests/evals/fizzy-fixture.ts | 138 ++++++++++++++ tests/evals/fizzy-selection.spec.ts | 31 ++++ tests/evals/fizzy.ts | 27 +++ tests/evals/graders/fizzy.grader.spec.ts | 59 ++++++ tests/evals/graders/fizzy.grader.ts | 46 +++++ tests/evals/parallel-runner.ts | 7 + tests/evals/runner.ts | 17 +- tests/fixtures/ruby/fizzy/fixture.json | 23 +++ 13 files changed, 577 insertions(+), 8 deletions(-) create mode 100644 tests/evals/fizzy-fixture.spec.ts create mode 100644 tests/evals/fizzy-fixture.ts create mode 100644 tests/evals/fizzy-selection.spec.ts create mode 100644 tests/evals/fizzy.ts create mode 100644 tests/evals/graders/fizzy.grader.spec.ts create mode 100644 tests/evals/graders/fizzy.grader.ts create mode 100644 tests/fixtures/ruby/fizzy/fixture.json diff --git a/tests/evals/__tests__/agent-executor.spec.ts b/tests/evals/__tests__/agent-executor.spec.ts index 08a67959..ad85101c 100644 --- a/tests/evals/__tests__/agent-executor.spec.ts +++ b/tests/evals/__tests__/agent-executor.spec.ts @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; -import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import { mkdtempSync, writeFileSync, readFileSync, rmSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; @@ -34,6 +34,10 @@ vi.mock('../../../src/lib/agent-interface.js', () => ({ runAgent: mockRunAgent, })); +vi.mock('../../../src/lib/agent-sdk-assets.js', () => ({ + ensureClaudeCodeExecutable: vi.fn(async () => '/synthetic/claude'), +})); + // Mock dependencies vi.mock('../env-loader.js', () => ({ loadCredentials: vi.fn(() => mockCredentials), @@ -112,6 +116,25 @@ describe('AgentExecutor', () => { expect(agentRunConfig.sdkEnv.ANTHROPIC_BASE_URL).toBeUndefined(); }); + it('uses isolated Ruby environment and explicit callback without inheriting host secrets', async () => { + mockRunAgent.mockResolvedValue({}); + const executor = new AgentExecutor(testDir, 'ruby', { + environment: { HOME: '/synthetic/home', PATH: '/synthetic/bin' }, + redirectUri: 'http://app.fizzy.localhost:3006/auth/callback', + }); + await executor.run({ enabled: false, maxRetries: 0 }); + const [config, prompt] = mockRunAgent.mock.calls[0]; + expect(config.sdkEnv.HOME).toBe('/synthetic/home'); + expect(config.sdkEnv.PATH).toBe('/synthetic/bin'); + expect(config.sdkEnv).not.toHaveProperty('WORKOS_API_KEY'); + expect(prompt).toContain('configured in .env:'); + expect(prompt).not.toContain('configured in .env.local'); + expect(prompt).toContain('WORKOS_REDIRECT_URI=http://app.fizzy.localhost:3006/auth/callback'); + expect(readFileSync(join(testDir, '.env'), 'utf8')).toContain( + 'WORKOS_REDIRECT_URI=http://app.fizzy.localhost:3006/auth/callback', + ); + }); + it('passes RetryConfig when correction is enabled', async () => { mockRunAgent.mockResolvedValue({ retryCount: 0 }); diff --git a/tests/evals/agent-executor.ts b/tests/evals/agent-executor.ts index d0f4d52d..73658041 100644 --- a/tests/evals/agent-executor.ts +++ b/tests/evals/agent-executor.ts @@ -34,6 +34,9 @@ export interface AgentRetryConfig { export interface AgentExecutorOptions { verbose?: boolean; scenarioName?: string; + /** Allowlisted environment for isolated real-app fixtures. */ + environment?: NodeJS.ProcessEnv; + redirectUri?: string; } // Skill name mapping for each framework @@ -106,6 +109,7 @@ export class AgentExecutor { const envVars = { WORKOS_API_KEY: this.credentials.workosApiKey, WORKOS_CLIENT_ID: this.credentials.workosClientId, + ...(this.options.redirectUri ? { WORKOS_REDIRECT_URI: this.options.redirectUri } : {}), }; if (JS_FRAMEWORKS.includes(this.framework)) { @@ -118,7 +122,7 @@ export class AgentExecutor { const prompt = this.buildPrompt(skillName); const sdkEnv: Record = { - ...process.env, + ...(this.options.environment ?? process.env), ANTHROPIC_API_KEY: this.credentials.anthropicApiKey, ANTHROPIC_BASE_URL: undefined, ANTHROPIC_AUTH_TOKEN: undefined, @@ -202,9 +206,12 @@ export class AgentExecutor { - Working directory: ${this.workDir} ## Environment -The following environment variables have been configured in .env.local: +The following environment variables have been configured in ${JS_FRAMEWORKS.includes(this.framework) ? '.env.local' : '.env'}: - WORKOS_API_KEY - WORKOS_CLIENT_ID +${this.options.redirectUri ? `- WORKOS_REDIRECT_URI=${this.options.redirectUri}\n` : ''} +Ensure the app loads this file before SDK initialization. Never print or commit credentials. +For an existing authentication system, preserve identity/session/account boundaries and existing login methods. Do not invent account linking, auto-creation or membership/role assignment: ask for approved policy and leave unsupported behavior pending. Require visible login/account/logout controls, idempotent repeat login and protected access denied after logout. A source match is not behavioral evidence. ## Your Task Use the \`${skillName}\` skill to integrate WorkOS AuthKit into this application. diff --git a/tests/evals/cli.ts b/tests/evals/cli.ts index 5a71af8f..54679f5c 100644 --- a/tests/evals/cli.ts +++ b/tests/evals/cli.ts @@ -39,6 +39,7 @@ const FRAMEWORKS = [ 'elixir', ]; const STATES = [ + 'fizzy', // Opt-in only: also requires --framework=ruby and explicit approval. 'example', 'example-auth0', 'partial-install', diff --git a/tests/evals/fixture-manager.ts b/tests/evals/fixture-manager.ts index 080e2bcf..88bcc79f 100644 --- a/tests/evals/fixture-manager.ts +++ b/tests/evals/fixture-manager.ts @@ -1,8 +1,9 @@ -import { cp, rm, mkdtemp } from 'node:fs/promises'; +import { cp, rm, mkdtemp, mkdir } from 'node:fs/promises'; import { existsSync, readdirSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { execFileNoThrow } from '../../src/utils/exec-file.js'; +import { bootstrapFizzy, prepareFizzyFixture } from './fizzy-fixture.js'; export interface FixtureOptions { keepOnFail?: boolean; @@ -21,6 +22,19 @@ export class FixtureManager { } async setup(): Promise { + if (this.framework === 'ruby' && this.state === 'fizzy') { + if (process.env.FIZZY_APPROVED_RUN !== '1' || !process.env.FIZZY_ARCHIVE) { + throw new Error( + 'Fizzy eval requires explicit spending/policy approval (FIZZY_APPROVED_RUN=1) and FIZZY_ARCHIVE. Use tests/evals/fizzy.ts for offline preparation.', + ); + } + const parent = join(process.cwd(), '.artifacts/fizzy-evals'); + await mkdir(parent, { recursive: true }); + this.tempDir = await mkdtemp(join(parent, 'attempt-')); + const app = await prepareFizzyFixture(this.tempDir, process.env.FIZZY_ARCHIVE); + await bootstrapFizzy(this.tempDir); + return app; + } // Create temp directory with random suffix for parallel safety const suffix = Math.random().toString(36).substring(2, 8); this.tempDir = await mkdtemp(join(tmpdir(), `eval-${this.framework}-${this.state}-${suffix}-`)); diff --git a/tests/evals/fizzy-fixture.spec.ts b/tests/evals/fizzy-fixture.spec.ts new file mode 100644 index 00000000..7f083d7c --- /dev/null +++ b/tests/evals/fizzy-fixture.spec.ts @@ -0,0 +1,184 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { createHash } from 'node:crypto'; +import { + bootstrapFizzy, + FIZZY_FIXTURE, + fizzyEnvironment, + preflightFizzy, + prepareFizzyFixture, +} from './fizzy-fixture.js'; +import { FixtureManager } from './fixture-manager.js'; +import { execFileNoThrow } from '../../src/utils/exec-file.js'; + +vi.mock('../../src/utils/exec-file.js', () => ({ execFileNoThrow: vi.fn() })); +vi.mock('../fixtures/ruby/fizzy/fixture.json', async (original) => { + const actual = await original(); + const { createHash } = await import('node:crypto'); + return { + default: { ...actual.default, archiveSha256: createHash('sha256').update('synthetic archive').digest('hex') }, + }; +}); +// These must never be reached by offline preparation. +vi.mock('./env-loader.js', () => ({ + loadCredentials: () => { + throw new Error('Credential access forbidden'); + }, +})); +vi.mock('./agent-executor.js', () => ({ + AgentExecutor: class { + constructor() { + throw new Error('Paid executor forbidden'); + } + }, +})); + +let root: string; +let target: string; +let archive: string; +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'fizzy-offline-')); + target = join(root, 'fixture'); + archive = join(root, 'source.tar.gz'); + await mkdir(target); + await writeFile(archive, 'synthetic archive'); + vi.stubGlobal( + 'fetch', + vi.fn(() => { + throw new Error('Network forbidden'); + }), + ); + vi.mocked(execFileNoThrow) + .mockReset() + .mockImplementation(async (executable, args, options) => { + if (executable === 'tar') { + const app = options!.cwd!; + await writeFile(join(app, '.ruby-version'), '3.4.8\n'); + await writeFile(join(app, 'LICENSE.md'), "O'Saasy synthetic notice"); + await mkdir(join(app, 'storage')); + await mkdir(join(app, 'tmp')); + } + const stdout = + executable === 'ruby' + ? 'ruby 3.4.8 (synthetic)' + : executable === 'bundle' && args[0] === '--version' + ? 'Bundler version 4.0.18' + : args[0] === 'rev-parse' + ? 'synthetic-baseline' + : ''; + return { status: 0, stdout, stderr: '' }; + }); +}); +afterEach(async () => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + await rm(root, { recursive: true, force: true }); +}); + +describe('isolated pinned preparation', () => { + it('records immutable source, license, empty identity baseline and no patches', async () => { + const app = await prepareFizzyFixture(target, archive); + expect(app).toBe(join(target, 'app')); + const record = JSON.parse(await readFile(join(target, 'artifacts/fixture.json'), 'utf8')); + expect(record).toMatchObject({ + commit: '477c943e0506f109e5bc83ae9dadbe519732c045', + license: "O'Saasy", + ruby: '3.4.8', + patches: [], + baselineCommit: 'synthetic-baseline', + bootstrapped: false, + }); + expect(record.baselineIdentity).toContain('undecided'); + expect(execFileNoThrow).not.toHaveBeenCalledWith('bundle', expect.anything(), expect.anything()); + expect(fetch).not.toHaveBeenCalled(); + }); + + it('fails closed on modified source and existing output without executing anything', async () => { + await writeFile(archive, 'mutable main download'); + await expect(prepareFizzyFixture(target, archive)).rejects.toThrow('SHA-256'); + await writeFile(join(target, 'keep'), 'existing project'); + await expect(prepareFizzyFixture(target, archive)).rejects.toThrow('empty isolated'); + expect(execFileNoThrow).not.toHaveBeenCalled(); + }); + + it('isolates inherited credentials, home, dependency caches and database configuration', () => { + for (const key of [ + 'WORKOS_API_KEY', + 'ANTHROPIC_API_KEY', + 'DATABASE_URL', + 'SAAS', + 'BUNDLE_GEMFILE', + 'RUBYOPT', + 'GIT_CONFIG_COUNT', + ]) + vi.stubEnv(key, 'must-not-inherit'); + const env = fizzyEnvironment(target); + expect(env.HOME).toBe(join(target, 'home')); + expect(env.BUNDLE_PATH).toBe(join(target, 'dependencies')); + expect(env.BUNDLE_GEMFILE).toBe(join(target, 'app/Gemfile')); + expect(env.RAILS_ENV).toBe('test'); + expect(Object.values(env)).not.toContain('must-not-inherit'); + expect(env).not.toHaveProperty('SAAS'); + }); + + it('preflight only probes runtime versions and never claims acceptance', async () => { + vi.mocked(execFileNoThrow).mockResolvedValue({ status: 0, stdout: 'ruby 4.0.7', stderr: '' }); + const result = await preflightFizzy(target); + expect(result.runtimeAvailable).toBe(false); + expect(result.acceptance).toBe('unverified'); + expect(execFileNoThrow).toHaveBeenCalledTimes(2); + expect(fetch).not.toHaveBeenCalled(); + }); + + it('bootstraps only fresh test schema, never upstream scripts or seeds/reset', async () => { + await prepareFizzyFixture(target, archive); + await bootstrapFizzy(target); + expect(execFileNoThrow).toHaveBeenCalledWith( + 'bundle', + ['exec', 'rails', 'db:prepare'], + expect.objectContaining({ env: expect.objectContaining({ RAILS_ENV: 'test' }) }), + ); + const calls = JSON.stringify(vi.mocked(execFileNoThrow).mock.calls); + for (const forbidden of ['bin/setup', 'db:reset', 'db:seed', 'Gemfile.saas']) + expect(calls).not.toContain(forbidden); + await expect(bootstrapFizzy(target)).rejects.toThrow('fresh pinned fixture'); + }); + + it.each(['storage/test.sqlite3', 'tmp/saas.txt'])('refuses bootstrap with %s', async (file) => { + await prepareFizzyFixture(target, archive); + await writeFile(join(target, 'app', file), 'preserve'); + await expect(bootstrapFizzy(target)).rejects.toThrow('existing database or SaaS'); + }); + + it('requires explicit opt-in in FixtureManager before any execution', async () => { + vi.stubEnv('FIZZY_APPROVED_RUN', ''); + const manager = new FixtureManager('ruby', 'fizzy'); + await expect(manager.setup()).rejects.toThrow('explicit spending/policy approval'); + await manager.cleanup(); + expect(manager.getTempDir()).toBeNull(); + expect(execFileNoThrow).not.toHaveBeenCalled(); + }); + + it('cleans up a failed manager preparation without touching its input archive', async () => { + vi.stubEnv('FIZZY_APPROVED_RUN', '1'); + vi.stubEnv('FIZZY_ARCHIVE', archive); + await writeFile(archive, 'wrong pin'); + const manager = new FixtureManager('ruby', 'fizzy'); + await expect(manager.setup()).rejects.toThrow('SHA-256'); + const attempt = manager.getTempDir()!; + expect(attempt).toContain('.artifacts/fizzy-evals/attempt-'); + await manager.cleanup(); + await expect(readFile(join(attempt, 'source.tar.gz'))).rejects.toMatchObject({ code: 'ENOENT' }); + expect(await readFile(archive, 'utf8')).toBe('wrong pin'); + expect(manager.getTempDir()).toBeNull(); + }); + + it('descriptor pin matches the inspected public archive checksum', async () => { + const real = JSON.parse(await readFile(join(process.cwd(), 'tests/fixtures/ruby/fizzy/fixture.json'), 'utf8')); + expect(real.archiveUrl).toContain(real.commit); + expect(real.archiveSha256).toBe('4cfc52d62d082f304a946dcf02d6097886100f1430eb502daf503e9f2439a628'); + expect(FIZZY_FIXTURE.archiveSha256).toBe(createHash('sha256').update('synthetic archive').digest('hex')); + }); +}); diff --git a/tests/evals/fizzy-fixture.ts b/tests/evals/fizzy-fixture.ts new file mode 100644 index 00000000..3ba0a682 --- /dev/null +++ b/tests/evals/fizzy-fixture.ts @@ -0,0 +1,138 @@ +import { createHash } from 'node:crypto'; +import { mkdir, readFile, readdir, writeFile } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; +import { execFileNoThrow } from '../../src/utils/exec-file.js'; +import fixture from '../fixtures/ruby/fizzy/fixture.json'; + +export { fixture as FIZZY_FIXTURE }; + +/** Deliberately allowlisted: no developer credentials, SaaS flags, database URLs or home config. */ +export function fizzyEnvironment(root: string): Record { + return { + PATH: process.env.PATH ?? '/usr/bin:/bin', + HOME: join(root, 'home'), + XDG_CONFIG_HOME: join(root, 'home/config'), + XDG_CACHE_HOME: join(root, 'cache'), + TMPDIR: join(root, 'tmp'), + BUNDLE_USER_HOME: join(root, 'home/bundle'), + BUNDLE_APP_CONFIG: join(root, 'bundle-config'), + BUNDLE_PATH: join(root, 'dependencies'), + BUNDLE_GEMFILE: join(root, 'app/Gemfile'), + BUNDLE_FROZEN: 'true', + GEM_HOME: join(root, 'gems'), + MISE_DATA_DIR: join(root, 'mise/data'), + MISE_CONFIG_DIR: join(root, 'mise/config'), + MISE_CACHE_DIR: join(root, 'mise/cache'), + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + RAILS_ENV: 'test', + DATABASE_ADAPTER: 'sqlite', + DISABLE_SPRING: '1', + CI: '1', + PORT: '3006', + SECRET_KEY_BASE: 'synthetic-fixture-only-not-for-deployment'.repeat(3), + }; +} + +async function command(root: string, executable: string, args: string[], cwd = join(root, 'app')) { + const result = await execFileNoThrow(executable, args, { cwd, env: fizzyEnvironment(root), timeout: 600_000 }); + if (result.status !== 0) throw new Error(`${executable} ${args.join(' ')} failed: ${result.stderr}`); + return result.stdout.trim(); +} + +/** No network and no scripts from the downloaded app. Only accepts the immutable, hashed archive. */ +export async function prepareFizzyFixture(root: string, archive: string): Promise { + root = resolve(root); + if ((await readdir(root)).length) throw new Error('Fizzy preparation requires an empty isolated directory'); + const bytes = await readFile(archive); + if (createHash('sha256').update(bytes).digest('hex') !== fixture.archiveSha256) { + throw new Error('Fizzy archive SHA-256 mismatch'); + } + for (const dir of ['app', 'home', 'home/config', 'cache', 'tmp', 'artifacts', 'gems']) { + await mkdir(join(root, dir), { recursive: true }); + } + // Extract the bytes we verified, not a caller-owned path that could change between hash and extraction. + await writeFile(join(root, 'source.tar.gz'), bytes); + await command(root, 'tar', ['-xzf', join(root, 'source.tar.gz'), '--strip-components=1']); + const app = join(root, 'app'); + if ((await readFile(join(app, '.ruby-version'), 'utf8')).trim() !== fixture.ruby) { + throw new Error('Unexpected Fizzy Ruby version'); + } + if (!(await readFile(join(app, fixture.licenseFile), 'utf8')).includes("O'Saasy")) { + throw new Error('Fizzy license notice missing'); + } + await command(root, 'git', ['init']); + await command(root, 'git', ['add', '-A']); + await command(root, 'git', [ + '-c', + 'user.name=Fixture Baseline', + '-c', + 'user.email=fixture@example.invalid', + '-c', + 'core.hooksPath=/dev/null', + 'commit', + '--no-gpg-sign', + '-m', + `Fizzy baseline ${fixture.commit}`, + ]); + const baselineCommit = await command(root, 'git', ['rev-parse', 'HEAD']); + await writeFile( + join(root, 'artifacts/fixture.json'), + JSON.stringify({ ...fixture, baselineCommit, prepared: true, bootstrapped: false }, null, 2), + ); + return app; +} + +/** Probe tool versions only; never construct AgentExecutor or read eval credentials. */ +export async function preflightFizzy(root: string) { + const checks: { name: string; available: boolean; detail: string }[] = []; + for (const [executable, args, expected] of [ + ['ruby', ['--version'], `ruby ${fixture.ruby} `], + ['bundle', ['--version'], fixture.bundler], + ] as const) { + const result = await execFileNoThrow(executable, [...args], { + cwd: root, + env: fizzyEnvironment(root), + timeout: 10_000, + }); + checks.push({ + name: executable, + available: result.status === 0 && result.stdout.includes(expected), + detail: result.stdout.trim() || result.stderr.trim(), + }); + } + return { + fixture: fixture.commit, + checks, + runtimeAvailable: checks.every((check) => check.available), + acceptance: 'unverified', + blockers: [ + 'Account-linking/membership/creation/coexistence policy requires approval', + 'No route/session/browser or hosted AuthKit evidence collected', + ], + }; +} + +/** Opt-in dependency install/test schema only, never upstream bin/setup, db:reset, or SaaS. */ +export async function bootstrapFizzy(root: string): Promise { + root = resolve(root); + const metadata = JSON.parse(await readFile(join(root, 'artifacts/fixture.json'), 'utf8')); + if (metadata.commit !== fixture.commit || metadata.bootstrapped) throw new Error('Not a fresh pinned fixture'); + const app = join(root, 'app'); + const tracked = await command(root, 'git', ['status', '--porcelain', '--untracked-files=all']); + if (tracked) throw new Error('Bootstrap requires an unchanged baseline'); + for (const directory of ['storage', 'tmp']) { + const entries = await readdir(join(app, directory), { recursive: true }); + if (entries.some((entry) => /(?:\.sqlite3(?:-|$)|saas\.txt$)/.test(entry))) { + throw new Error('Refusing an existing database or SaaS marker'); + } + } + if (!(await preflightFizzy(root)).runtimeAvailable) throw new Error('Fizzy runtime prerequisites unavailable'); + await command(root, 'bundle', ['install']); + await command(root, 'bundle', ['exec', 'rails', 'db:prepare']); + await writeFile( + join(root, 'artifacts/fixture.json'), + JSON.stringify({ ...metadata, bootstrapped: true, databaseEnvironment: 'test' }, null, 2), + ); +} diff --git a/tests/evals/fizzy-selection.spec.ts b/tests/evals/fizzy-selection.spec.ts new file mode 100644 index 00000000..e5aa1898 --- /dev/null +++ b/tests/evals/fizzy-selection.spec.ts @@ -0,0 +1,31 @@ +import { expect, it, vi } from 'vitest'; +import { selectScenarios } from './runner.js'; +import { parseArgs } from './cli.js'; + +vi.mock('./env-loader.js', () => ({ + loadCredentials: () => { + throw new Error('No credential loading during selection'); + }, +})); +vi.mock('./agent-executor.js', () => ({ + AgentExecutor: class { + constructor() { + throw new Error('No paid executor during selection'); + } + }, +})); + +it('never expands the default or Ruby paid sweep with Fizzy', () => { + for (const options of [{}, { framework: ['ruby'] }, { state: 'fizzy' }]) { + expect(selectScenarios(options).some((scenario) => scenario.state === 'fizzy')).toBe(false); + } +}); +it('selects exactly one real-world scenario with explicit flags', () => { + const options = parseArgs(['--framework=ruby', '--state=fizzy', '--retry=0', '--sequential', '--no-correction']); + expect(options.retry).toBe(0); + expect(options.sequential).toBe(true); + expect(options.noCorrection).toBe(true); + expect(selectScenarios(options)).toEqual([ + expect.objectContaining({ framework: 'ruby', state: 'fizzy', optIn: true }), + ]); +}); diff --git a/tests/evals/fizzy.ts b/tests/evals/fizzy.ts new file mode 100644 index 00000000..a0be6f3b --- /dev/null +++ b/tests/evals/fizzy.ts @@ -0,0 +1,27 @@ +#!/usr/bin/env bun +// Deliberately independent of runner.ts, env-loader.ts and AgentExecutor. +import { mkdir, writeFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; +import { bootstrapFizzy, FIZZY_FIXTURE, preflightFizzy, prepareFizzyFixture } from './fizzy-fixture.js'; + +const [action, output, archive] = process.argv.slice(2); +if (!output || !['download', 'prepare', 'preflight', 'bootstrap'].includes(action)) { + throw new Error( + 'Usage: bun tests/evals/fizzy.ts download | prepare | preflight | bootstrap ', + ); +} +if (action === 'download') { + const response = await fetch(FIZZY_FIXTURE.archiveUrl); + if (!response.ok) throw new Error(`Source download failed: ${response.status}`); + await writeFile(resolve(output), new Uint8Array(await response.arrayBuffer()), { flag: 'wx' }); +} else if (action === 'prepare') { + if (!archive) throw new Error('prepare requires a local pinned archive'); + await mkdir(resolve(output)); // Refuse all existing directories, not just nonempty ones. + console.log(await prepareFizzyFixture(resolve(output), resolve(archive))); +} else if (action === 'preflight') { + const result = await preflightFizzy(resolve(output)); + console.log(JSON.stringify(result, null, 2)); + if (!result.runtimeAvailable) process.exitCode = 1; +} else { + await bootstrapFizzy(resolve(output)); +} diff --git a/tests/evals/graders/fizzy.grader.spec.ts b/tests/evals/graders/fizzy.grader.spec.ts new file mode 100644 index 00000000..492cf849 --- /dev/null +++ b/tests/evals/graders/fizzy.grader.spec.ts @@ -0,0 +1,59 @@ +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { FizzyGrader, FIZZY_ACCEPTANCE } from './fizzy.grader.js'; + +let directory: string; +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'fizzy-grader-')); + vi.stubGlobal( + 'fetch', + vi.fn(() => { + throw new Error('Network forbidden'); + }), + ); + for (const path of ['app/controllers', 'app/views/layouts', 'config']) + await mkdir(join(directory, path), { recursive: true }); +}); +afterEach(async () => { + vi.unstubAllGlobals(); + await rm(directory, { recursive: true, force: true }); +}); + +it.each([ + ['unused SDK strings', '# authorization_url authenticate_with_code sealed_session'], + ['hardcoded identity', 'def callback; authenticate_with_code; Current.identity = Identity.first; end'], + [ + 'duplicate provisioning', + 'def callback; authenticate_with_code; Identity.create!; Account.create!; User.create!; end', + ], + ['logout leaving access', 'def logout; redirect_to root_path; end # authenticate_with_code'], + [ + 'plausible source without runtime evidence', + 'def callback; result = authenticate_with_code; start_new_session_for(mapped_identity(result)); end', + ], +])('never treats %s as proven acceptance', async (_label, source) => { + await writeFile(join(directory, 'Gemfile'), 'gem "workos"'); + await writeFile(join(directory, 'app/controllers/auth_controller.rb'), source); + await writeFile(join(directory, 'config/routes.rb'), 'get "/auth/login", to: "auth#login"'); + await writeFile( + join(directory, 'app/views/layouts/application.html.erb'), + 'Sign in', + ); + const result = await new FizzyGrader(directory).grade(); + expect(result.passed).toBe(false); + for (const name of FIZZY_ACCEPTANCE) + expect(result.checks.find((check) => check.name === name)).toMatchObject({ + passed: false, + message: expect.stringContaining('UNVERIFIED'), + }); + expect(result.checks.some((check) => check.name.startsWith('Static only:') && check.passed)).toBe(true); + expect(fetch).not.toHaveBeenCalled(); +}); + +it('reports missing UI/routes/source as absent, not successful behavior', async () => { + const result = await new FizzyGrader(directory).grade(); + expect(result.passed).toBe(false); + expect(result.checks.every((check) => !check.passed)).toBe(true); +}); diff --git a/tests/evals/graders/fizzy.grader.ts b/tests/evals/graders/fizzy.grader.ts new file mode 100644 index 00000000..0ba356e1 --- /dev/null +++ b/tests/evals/graders/fizzy.grader.ts @@ -0,0 +1,46 @@ +import { FileGrader } from './file-grader.js'; +import type { Grader, GradeResult } from '../types.js'; + +export const FIZZY_ACCEPTANCE = [ + 'Visible signed-out login and signed-in account/logout controls', + 'Callback establishes approved identity/session/account association', + 'Repeat login does not duplicate identities/accounts/memberships', + 'Protected access denied after logout, including old-session replay', + 'Targeted callback/CORS/sign-out/Initiate login read-back preserves unrelated settings', + 'Existing account boundaries, roles, magic links/passkeys and routes preserved', +] as const; + +/** Source observations only. Never turn a grep match into Rails/hosted acceptance. */ +export class FizzyGrader implements Grader { + constructor(private workDir: string) {} + + async grade(): Promise { + const files = new FileGrader(this.workDir); + const checks = [ + ...(await files.checkFileContains('Gemfile', ['workos'])), + await files.checkFileWithPattern( + 'app/controllers/**/*.rb', + [/authenticate_with_code/], + 'Callback source candidate', + ), + await files.checkFileWithPattern('config/routes.rb', [/auth/], 'Auth route source candidate'), + await files.checkFileWithPattern( + 'app/views/**/*', + [/logout|sign.out/i, /login|sign.in/i], + 'Auth UI source candidate', + ), + ].map((check) => ({ ...check, name: `Static only: ${check.name}` })); + return { + passed: false, + checks: [ + ...checks, + ...FIZZY_ACCEPTANCE.map((name) => ({ + name, + passed: false, + message: + 'UNVERIFIED: requires recorded route/session/browser or hosted evidence; this source grader cannot prove acceptance. Account policy must be approved first.', + })), + ], + }; + } +} diff --git a/tests/evals/parallel-runner.ts b/tests/evals/parallel-runner.ts index 2da38472..fa5a53f6 100644 --- a/tests/evals/parallel-runner.ts +++ b/tests/evals/parallel-runner.ts @@ -1,6 +1,7 @@ import pLimit from 'p-limit'; import type { EvalResult, Grader, GradeCheck, ToolCall } from './types.js'; import { FixtureManager } from './fixture-manager.js'; +import { FIZZY_FIXTURE, fizzyEnvironment } from './fizzy-fixture.js'; import { AgentExecutor } from './agent-executor.js'; import { detectConcurrency } from './concurrency.js'; import { evalEvents } from './events.js'; @@ -104,6 +105,12 @@ export class ParallelRunner { const executor = new AgentExecutor(workDir, scenario.framework, { verbose: this.options.verbose, scenarioName, + ...(scenario.framework === 'ruby' && scenario.state === 'fizzy' + ? { + environment: { ...fizzyEnvironment(fixtureManager.getTempDir()!), BUNDLE_FROZEN: 'false' }, + redirectUri: FIZZY_FIXTURE.redirectUri, + } + : {}), }); const agentResult = await executor.run( this.options.noCorrection ? { enabled: false, maxRetries: 0 } : undefined, diff --git a/tests/evals/runner.ts b/tests/evals/runner.ts index 98bd6243..fec2b314 100644 --- a/tests/evals/runner.ts +++ b/tests/evals/runner.ts @@ -8,6 +8,7 @@ import { SvelteKitGrader } from './graders/sveltekit.grader.js'; import { NodeGrader } from './graders/node.grader.js'; import { PythonGrader } from './graders/python.grader.js'; import { RubyGrader } from './graders/ruby.grader.js'; +import { FizzyGrader } from './graders/fizzy.grader.js'; import { GoGrader } from './graders/go.grader.js'; import { PhpGrader } from './graders/php.grader.js'; import { PhpLaravelGrader } from './graders/php-laravel.grader.js'; @@ -26,6 +27,7 @@ interface Scenario { framework: string; state: string; grader: new (workDir: string) => Grader; + optIn?: boolean; } const SCENARIOS: Scenario[] = [ @@ -80,6 +82,7 @@ const SCENARIOS: Scenario[] = [ { framework: 'python', state: 'example-auth0', grader: PythonGrader }, { framework: 'python', state: 'partial-install', grader: PythonGrader }, { framework: 'python', state: 'conflicting-auth', grader: PythonGrader }, + { framework: 'ruby', state: 'fizzy', grader: FizzyGrader, optIn: true }, { framework: 'ruby', state: 'example', grader: RubyGrader }, { framework: 'ruby', state: 'example-auth0', grader: RubyGrader }, { framework: 'ruby', state: 'partial-install', grader: RubyGrader }, @@ -119,6 +122,15 @@ export interface ExtendedEvalOptions extends EvalOptions { quality?: boolean; } +export function selectScenarios(options: EvalOptions): Scenario[] { + return SCENARIOS.filter( + (s) => + (!s.optIn || (options.state === s.state && options.framework?.includes(s.framework))) && + (!options.framework || options.framework.includes(s.framework)) && + (!options.state || s.state === options.state), + ); +} + export async function runEvals(options: ExtendedEvalOptions): Promise { // Capture version metadata at start const versionMeta = await captureVersionMetadata(); @@ -127,10 +139,7 @@ export async function runEvals(options: ExtendedEvalOptions): Promise - (!options.framework || options.framework.includes(s.framework)) && (!options.state || s.state === options.state), - ); + const scenarios = selectScenarios(options); const maxAttempts = (options.retry ?? 2) + 1; diff --git a/tests/fixtures/ruby/fizzy/fixture.json b/tests/fixtures/ruby/fizzy/fixture.json new file mode 100644 index 00000000..169c58e1 --- /dev/null +++ b/tests/fixtures/ruby/fizzy/fixture.json @@ -0,0 +1,23 @@ +{ + "repository": "https://github.com/basecamp/fizzy", + "commit": "477c943e0506f109e5bc83ae9dadbe519732c045", + "archiveUrl": "https://codeload.github.com/basecamp/fizzy/tar.gz/477c943e0506f109e5bc83ae9dadbe519732c045", + "archiveSha256": "4cfc52d62d082f304a946dcf02d6097886100f1430eb502daf503e9f2439a628", + "license": "O'Saasy", + "licenseFile": "LICENSE.md", + "ruby": "3.4.8", + "bundler": "4.0.18", + "origin": "http://app.fizzy.localhost:3006", + "redirectUri": "http://app.fizzy.localhost:3006/auth/callback", + "launcher": "bin/dev", + "isolatedLauncher": "bundle exec rails server -b 127.0.0.1 -p 3006", + "patches": [], + "seeds": "None. Test database only; no development seed/reset. Synthetic identities require approved account policy.", + "baselineIdentity": "None; account-linking, membership/role assignment and creation policy undecided.", + "prerequisites": [ + "Ruby 3.4.8 and Bundler 4.0.18 on a toolchain PATH, not auto-installing shims", + "Native build tools, SQLite and libvips; see pinned Brewfile/.mise.toml for platform prerequisites", + "Public gem and pinned Git dependency access for explicit bootstrap (never Gemfile.saas)", + "Browser/driver and local app.fizzy.localhost resolution for later behavioral checks" + ] +} From 29b560967272923cc4cc30f390167ecdee4d64ff Mon Sep 17 00:00:00 2001 From: Nick Nisi Date: Mon, 28 Sep 2026 16:04:47 -0500 Subject: [PATCH 3/6] test(ruby): document Fizzy acceptance gates and verify offline evidence --- src/integrations/ruby/index.ts | 1 + src/lib/run-with-core.ruby.spec.ts | 110 +++++++++++---------- tests/evals/agent-executor.ts | 3 +- tests/evals/fizzy-fixture.spec.ts | 13 +++ tests/evals/fizzy-fixture.ts | 23 ++++- tests/evals/fizzy.ts | 2 +- tests/fixtures/ruby/fizzy/README.md | 146 ++++++++++++++++++++++++++++ 7 files changed, 242 insertions(+), 56 deletions(-) create mode 100644 tests/fixtures/ruby/fizzy/README.md diff --git a/src/integrations/ruby/index.ts b/src/integrations/ruby/index.ts index 3bb94d99..0a38320a 100644 --- a/src/integrations/ruby/index.ts +++ b/src/integrations/ruby/index.ts @@ -170,6 +170,7 @@ Begin integration now.`; const lines: string[] = [ 'Ruby agent finished. Application behavior and hosted AuthKit flows are not verified.', + `Credentials and callback written to ${envFile}; runtime loading is not verified.`, `Requested callback: ${redirectUri}`, 'Application URL registration is handled separately by the installer after this step.', '', diff --git a/src/lib/run-with-core.ruby.spec.ts b/src/lib/run-with-core.ruby.spec.ts index eeb14bd3..6727f6e3 100644 --- a/src/lib/run-with-core.ruby.spec.ts +++ b/src/lib/run-with-core.ruby.spec.ts @@ -67,54 +67,62 @@ afterEach(async () => { if (directory) await rm(directory, { recursive: true, force: true }); }); -it('runs the real Ruby installer before the common URL path and reports the explicit origin without claiming verification', async () => { - directory = await mkdtemp(join(tmpdir(), 'ruby-orchestration-')); - await mkdir(join(directory, 'config')); - await writeFile(join(directory, 'Gemfile'), 'gem "rails"'); - await writeFile(join(directory, 'config/puma.rb'), 'port 3000'); - vi.stubGlobal( - 'fetch', - vi.fn(() => { - throw new Error('Unexpected network'); - }), - ); - const output: string[] = []; - vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => { - output.push(String(chunk)); - return true; - }); - setOutputMode('json'); - await runWithCore({ - installDir: directory, - integration: 'ruby', - apiKey: 'sk_test_synthetic', - clientId: 'client_synthetic', - redirectUri: 'http://app.fizzy.localhost:3006/auth/callback', - ci: true, - debug: false, - local: false, - forceInstall: false, - skipAuth: true, - noBranch: true, - noCommit: true, - noGitCheck: true, - }); - expect(runAgent).toHaveBeenCalledOnce(); - expect(configureAuthkitApplication).toHaveBeenCalledOnce(); - expect(vi.mocked(runAgent).mock.invocationCallOrder[0]).toBeLessThan( - vi.mocked(configureAuthkitApplication).mock.invocationCallOrder[0], - ); - expect(configureAuthkitApplication).toHaveBeenCalledWith( - expect.objectContaining({ - redirectUri: 'http://app.fizzy.localhost:3006/auth/callback', - corsOrigin: 'http://app.fizzy.localhost:3006', - signOutUri: 'http://app.fizzy.localhost:3006/', - initiateLoginUri: 'http://app.fizzy.localhost:3006/auth/login', - }), - 'client_synthetic', - 'sk_test_synthetic', - ); - expect(output.join('')).toContain('Synthetic pending read-back'); - expect(output.join('')).toContain('startup not verified'); - expect(fetch).not.toHaveBeenCalled(); -}); +it.each([undefined, 'http://app.fizzy.localhost:3006/auth/callback'])( + 'runs Ruby before common URL setup and consistently reports its origin (%s)', + async (redirectUri) => { + vi.clearAllMocks(); + const callback = redirectUri ?? 'http://localhost:4100/auth/callback'; + const origin = new URL(callback).origin; + directory = await mkdtemp(join(tmpdir(), 'ruby-orchestration-')); + await mkdir(join(directory, 'config')); + await writeFile(join(directory, 'Gemfile'), 'gem "rails"'); + await writeFile(join(directory, 'config/puma.rb'), 'port 4100'); + vi.stubGlobal( + 'fetch', + vi.fn(() => { + throw new Error('Unexpected network'); + }), + ); + const output: string[] = []; + vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => { + output.push(String(chunk)); + return true; + }); + setOutputMode('json'); + await runWithCore({ + installDir: directory, + integration: 'ruby', + apiKey: 'sk_test_synthetic', + clientId: 'client_synthetic', + redirectUri, + ci: true, + debug: false, + local: false, + forceInstall: false, + skipAuth: true, + noBranch: true, + noCommit: true, + noGitCheck: true, + }); + expect(runAgent).toHaveBeenCalledOnce(); + expect(configureAuthkitApplication).toHaveBeenCalledOnce(); + expect(vi.mocked(runAgent).mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(configureAuthkitApplication).mock.invocationCallOrder[0], + ); + expect(configureAuthkitApplication).toHaveBeenCalledWith( + expect.objectContaining({ + redirectUri: callback, + corsOrigin: origin, + signOutUri: `${origin}/`, + initiateLoginUri: `${origin}/auth/login`, + }), + 'client_synthetic', + 'sk_test_synthetic', + ); + expect(vi.mocked(runAgent).mock.calls[0][1]).toContain(`WORKOS_REDIRECT_URI=${callback}`); + expect(output.join('')).toContain(`Open ${origin} to test authentication`); + expect(output.join('')).toContain('Synthetic pending read-back'); + expect(output.join('')).toContain('startup not verified'); + expect(fetch).not.toHaveBeenCalled(); + }, +); diff --git a/tests/evals/agent-executor.ts b/tests/evals/agent-executor.ts index 73658041..c26fa29f 100644 --- a/tests/evals/agent-executor.ts +++ b/tests/evals/agent-executor.ts @@ -210,8 +210,7 @@ The following environment variables have been configured in ${JS_FRAMEWORKS.incl - WORKOS_API_KEY - WORKOS_CLIENT_ID ${this.options.redirectUri ? `- WORKOS_REDIRECT_URI=${this.options.redirectUri}\n` : ''} -Ensure the app loads this file before SDK initialization. Never print or commit credentials. -For an existing authentication system, preserve identity/session/account boundaries and existing login methods. Do not invent account linking, auto-creation or membership/role assignment: ask for approved policy and leave unsupported behavior pending. Require visible login/account/logout controls, idempotent repeat login and protected access denied after logout. A source match is not behavioral evidence. +${this.framework === 'ruby' ? 'Ensure the app loads this file before SDK initialization. Never print or commit credentials.\nFor an existing authentication system, preserve identity/session/account boundaries and existing login methods. Do not invent account linking, auto-creation or membership/role assignment: ask for approved policy and leave unsupported behavior pending. Require visible login/account/logout controls, idempotent repeat login and protected access denied after logout. A source match is not behavioral evidence.' : ''} ## Your Task Use the \`${skillName}\` skill to integrate WorkOS AuthKit into this application. diff --git a/tests/evals/fizzy-fixture.spec.ts b/tests/evals/fizzy-fixture.spec.ts index 7f083d7c..11c987f0 100644 --- a/tests/evals/fizzy-fixture.spec.ts +++ b/tests/evals/fizzy-fixture.spec.ts @@ -127,11 +127,24 @@ describe('isolated pinned preparation', () => { vi.mocked(execFileNoThrow).mockResolvedValue({ status: 0, stdout: 'ruby 4.0.7', stderr: '' }); const result = await preflightFizzy(target); expect(result.runtimeAvailable).toBe(false); + expect(result.prepared).toBe(false); expect(result.acceptance).toBe('unverified'); expect(execFileNoThrow).toHaveBeenCalledTimes(2); expect(fetch).not.toHaveBeenCalled(); }); + it('preflight verifies the prepared source and fails closed if the archive changes', async () => { + await prepareFizzyFixture(target, archive); + expect(await preflightFizzy(target)).toMatchObject({ + prepared: true, + runtimeAvailable: true, + acceptance: 'unverified', + }); + await writeFile(join(target, 'source.tar.gz'), 'changed'); + expect(await preflightFizzy(target)).toMatchObject({ prepared: false }); + await expect(bootstrapFizzy(target)).rejects.toThrow('source/runtime prerequisites unavailable'); + }); + it('bootstraps only fresh test schema, never upstream scripts or seeds/reset', async () => { await prepareFizzyFixture(target, archive); await bootstrapFizzy(target); diff --git a/tests/evals/fizzy-fixture.ts b/tests/evals/fizzy-fixture.ts index 3ba0a682..bf986426 100644 --- a/tests/evals/fizzy-fixture.ts +++ b/tests/evals/fizzy-fixture.ts @@ -102,10 +102,27 @@ export async function preflightFizzy(root: string) { detail: result.stdout.trim() || result.stderr.trim(), }); } + const runtimeAvailable = checks.every((check) => check.available); + let prepared = false; + try { + const metadata = JSON.parse(await readFile(join(root, 'artifacts/fixture.json'), 'utf8')); + const archive = await readFile(join(root, 'source.tar.gz')); + prepared = + metadata.commit === fixture.commit && + createHash('sha256').update(archive).digest('hex') === fixture.archiveSha256; + } catch { + // Missing preparation is an unavailable prerequisite, not a successful check. + } + checks.push({ + name: 'pinned source archive', + available: prepared, + detail: prepared ? fixture.commit : 'Missing or mismatched prepared archive/metadata', + }); return { fixture: fixture.commit, checks, - runtimeAvailable: checks.every((check) => check.available), + prepared, + runtimeAvailable, acceptance: 'unverified', blockers: [ 'Account-linking/membership/creation/coexistence policy requires approval', @@ -128,7 +145,9 @@ export async function bootstrapFizzy(root: string): Promise { throw new Error('Refusing an existing database or SaaS marker'); } } - if (!(await preflightFizzy(root)).runtimeAvailable) throw new Error('Fizzy runtime prerequisites unavailable'); + const preflight = await preflightFizzy(root); + if (!preflight.prepared || !preflight.runtimeAvailable) + throw new Error('Fizzy source/runtime prerequisites unavailable'); await command(root, 'bundle', ['install']); await command(root, 'bundle', ['exec', 'rails', 'db:prepare']); await writeFile( diff --git a/tests/evals/fizzy.ts b/tests/evals/fizzy.ts index a0be6f3b..bc32ee65 100644 --- a/tests/evals/fizzy.ts +++ b/tests/evals/fizzy.ts @@ -21,7 +21,7 @@ if (action === 'download') { } else if (action === 'preflight') { const result = await preflightFizzy(resolve(output)); console.log(JSON.stringify(result, null, 2)); - if (!result.runtimeAvailable) process.exitCode = 1; + if (!result.runtimeAvailable || !result.prepared) process.exitCode = 1; } else { await bootstrapFizzy(resolve(output)); } diff --git a/tests/fixtures/ruby/fizzy/README.md b/tests/fixtures/ruby/fizzy/README.md new file mode 100644 index 00000000..f7b390b0 --- /dev/null +++ b/tests/fixtures/ruby/fizzy/README.md @@ -0,0 +1,146 @@ +# AUTH-6736 — Fizzy acceptance fixture (incomplete) + +**This is preparation and offline regression evidence, not a successful Fizzy installation.** No paid agent, OAuth login, dashboard write, provisioning, hosted AuthKit flow, or Rails/browser session test was executed in this job. Account policy remains undecided, as confirmed by Riker. + +## Source and local evidence + +- CLI starting HEAD: `2f199267d93fa5b966e677b77923663ccfee1919` (0.23.0). +- Inspected merged PR250's final code at `586b2fa1b88a9631accd4b4d69f28468a4f08760`, plus current HEAD. Its single-target sandbox/read-back application setup remains authoritative. Relevant all-ref history was inspected; no recorded Fizzy rerun or linked ticket PR was established. This is not a claim that no unlinked work exists. +- Source: , commit `477c943e0506f109e5bc83ae9dadbe519732c045`. +- Downloaded codeload archive SHA-256: `4cfc52d62d082f304a946dcf02d6097886100f1430eb502daf503e9f2439a628`. Preparation fails closed on another checksum; do not automatically refresh it if GitHub archive packaging changes. +- License: **O'Saasy**, copyright © 2025, 37signals LLC; not MIT or unrestricted hosting permission. The complete `LICENSE.md` stays in the extracted fixture. No app source is vendored here; descriptor and harness code only. No hosting/deployment work. +- Actual prepared checkout: `.artifacts/fizzy-prepared/app`; clean local baseline commit `06cbd18ca38292be7c47d176cb8e6f5928736796`. Future baseline hashes can differ due to commit timestamps; the upstream archive pin/hash is authoritative. Each preparation records its local baseline in `artifacts/fixture.json`. +- Source patches: none. Seeds: none. Baseline identity: none. No existing database was read or reset. +- Bun: **1.4.2**. Required Ruby: **3.4.8**; Bundler: **4.0.18**. Actual local preflight found Ruby **4.0.7**, Bundler **4.0.20**: unavailable for this pinned app. No dependency/bootstrap command was run. +- Locked Rails: `8.2.0.alpha`, Git revision `3df2cbea2027026a29edb92cbb7e336a63e35444`; the remaining public Git/gem dependency revisions/checksums stay in upstream `Gemfile.lock`. Bootstrap uses frozen resolution, not `bundle update`. +- Bundled `@workos/skills`: **0.7.3**, Ruby reference loaded through `skills-assets.ts`. Agent SDK: **0.3.211**. Configured model: **claude-opus-4-5-20251101** (not executed). WorkOS Ruby gem version is **not yet established**; a future run must retain its resolved lockfile/version and API documentation snapshot. + +Ignored local evidence is under `.artifacts/`: `rails-red.log` (5 failing regressions before fix), `rails-green.log`, `orchestration.log`, `fixture-tests.log`, `fizzy-prepare.log`, `fizzy-preflight.json`, `check.log`, and `build.log`. These artifacts are local, not published. + +## Why the development port is 3006 + +The pinned `docs/development.md` recommends `bin/setup`, then `bin/dev`, at `http://app.fizzy.localhost:3006`. Inspection showed: + +- `config/puma.rb` defaults to `ENV.fetch("PORT", 3000)`. +- `bin/dev` explicitly exports `PORT=3006`. +- Root `Procfile.dev` runs `bin/rails server -b 0.0.0.0 -p ${PORT:-3006}`. + +Thus 3006 is **source-confirmed launcher configuration, not an observed running server**. Always supply the verified explicit callback `http://app.fizzy.localhost:3006/auth/callback` for this fixture. Generic CLI detection remains unchanged and does not special-case Fizzy. + +`bin/setup` was read, **not executed**: it can install/upgrade system packages and mise, trust config, configure Git hooks, install tools, prepare/seed/reset databases, and select SaaS dependencies. `bin/dev` can install Foreman or opt into Tailscale/1Password/SaaS behavior. The harness bypasses both. No `--push`, `--tailscale`, `SAAS`, `tmp/saas.txt`, `Gemfile.saas`, MySQL, or private dependencies are allowed. + +## Offline preparation (no credentials or model) + +From the CLI worktree root: + +```sh +mkdir -p .artifacts +# Public source network only; optional if the archive is already available. +bun tests/evals/fizzy.ts download .artifacts/fizzy-source.tar.gz +# New output path required. Never point this at an existing repository. +bun tests/evals/fizzy.ts prepare .artifacts/fizzy-prepared-new .artifacts/fizzy-source.tar.gz +bun tests/evals/fizzy.ts preflight .artifacts/fizzy-prepared-new +``` + +`prepare` verifies the archive before extracting, retains license notices, and makes an unsigned local baseline with a synthetic Git author and no global Git config/hooks. It does not execute project scripts, install dependencies, or import the eval executor/credential loader. `preflight` only checks tool versions and prepared archive/metadata; missing prerequisites exit nonzero and acceptance stays `unverified`. + +After installing the exact runtime and native prerequisites **in an approved disposable toolchain**, explicit dependency bootstrap is: + +```sh +bun tests/evals/fizzy.ts bootstrap .artifacts/fizzy-prepared-new +``` + +This runs `bundle install` and `bundle exec rails db:prepare` with `RAILS_ENV=test`, SQLite, isolated HOME/config/cache/gems/Bundler/temp paths and no inherited credentials/database/SaaS configuration. It refuses a dirty baseline, existing SQLite database, SaaS marker, or repeated successful bootstrap. It does not run development seeds or `db:reset`. A failed attempt should be discarded and prepared afresh, not repaired by resetting a database. Use a PATH containing actual approved tool binaries, not auto-installing shims. Native libraries/toolchain remain external prerequisites, not something this script installs. + +Cleanup: retain redacted evidence first, then remove **only the newly created artifact root**. Existing `FixtureManager.cleanup()` owns removal for eval attempts, including failed preparation; unit tests cover this. No automatic cleanup of arbitrary user-supplied paths is provided. + +## CLI changes and offline checks + +The custom Ruby integration now writes the selected credential pair and resolved callback using existing env-file/backup/gitignore helpers; no secrets are embedded in its prompt. It tells the agent to load that file before SDK initialization. Rails does not load dotenv files automatically. The file choice follows the existing helper (`.env`, or `.env.local` if package.json is present), and both branches are tested. + +The integration uses `resolveRedirectUri()` rather than hardcoded port 3000. Its legacy pre-agent REST setup was removed; common `runWithCore` remains the sole URL provisioning path. Its prompt uses the same callback origin for Initiate login, sign-out return and CORS, requires visible UI and real app session/account integration, and forbids inventing account policy or silently replacing authentication. Completion distinguishes instructions from observed credential writes and unverified behavior; common completion uses the configured callback origin and labels the Ruby dev command inferred, not verified. + +Offline tests run the **real Ruby integration** with fake credentials/agent/network, plus the **real installer state machine** through Ruby and the common post-agent URL setup with a mocked backend. Existing production/sandbox, target selection, preservation, and read-back regression suites pass unchanged. New fixture tests mock command execution, use synthetic archive data, and cannot load credentials, download source or run models. Test setup replaces both native keyring and macOS security backends; keyring-isolation regressions are included in the full check. + +`FizzyGrader` separates static source candidates from all six mandatory acceptance checks. It intentionally returns `passed: false` until a real behavioral/hosted acceptance mechanism exists. Unused SDK strings, missing routes/UI, hardcoded identities, duplicate provisioning, logout that leaves access, and even plausible source cannot become proven acceptance. Positive controls prove only that static observations are collected. The existing Sinatra `RubyGrader` remains unchanged; its `server.rb` syntax bonus is not used for Fizzy. + +## Final local validation + +Using Bun 1.4.2, `bun run test && bun run typecheck && bun run lint && bun run format:check` passed: **174 test files, 3,231 tests**, TypeScript, Oxlint and formatting all green. Existing Node `fs.rmdir` deprecation warnings were non-failing. `bun run build` also passed and produced `dist/workos`; the binary was not run against credentials or an app. + +The real preparation command succeeded. The real offline preflight exited 1 as expected: pinned source verified, runtime prerequisites unavailable, acceptance unverified. Unit tests include positive and negative preflight/bootstrap/source-check controls without executing Ruby, Bundler, network or models. + +## Evidence matrix + +| Requirement | Offline evidence | Rails/browser evidence | Hosted evidence | +| ---------------------------------------------------------- | ---------------------------------------------------------------------------------- | ------------------------- | --------------------------- | +| Signed-out login and signed-in account/logout UI | Prompt regression; static candidates explicitly not proof | Unavailable | Unverified | +| Callback creates intended identity/session/account context | Policy-preserving instructions; hardcoded-identity negative control | Blocked by policy/runtime | Unverified | +| Repeat login does not duplicate provisioning | Prompt regression; duplicate-provisioning negative control | Blocked by policy/runtime | Unverified | +| Logout denies protected access, old cookie replay fails | Instructions distinguish local invalidation from upstream logout; negative control | Unavailable | Unverified | +| Correct target/URLs; unrelated settings preserved | Real orchestration with fake backend; existing sandbox/target/read-back tests | App routes unverified | No live dashboard read-back | +| Existing account boundaries/roles/routes/login methods | No app auth code changed; explicit preservation requirement | Blocked by policy/runtime | Unverified | + +No static grader is an account-isolation test, and a sandbox configuration read-back is not proof of session or UI behavior. + +## Product decision gate — ask Nick + +Fizzy's `Identity` owns sessions, users, accounts, magic links and passkeys; normalized `email_address` is not itself an approved WorkOS mapping policy. `User` belongs to an account, optionally an identity, and has role/active membership semantics. `Session` belongs to an identity. The authentication concern resumes signed session cookies, supports bearer access, establishes account context before authentication, and destroys the app session/clears its cookie on logout. + +Before implementing or accepting callback/account behavior, obtain explicit answers: + +1. Map WorkOS subject to which existing Identity key? Is email linking allowed, under which verification and conflict rules? What happens on email change or multiple candidate identities? +2. May an unknown identity/account be created? If so, when, by whom, and with what retry-safe uniqueness constraints? +3. Which account may be selected? How do multiple accounts, invitations, inactive users/accounts, roles and memberships constrain access? Does any WorkOS organization map to a Fizzy account? +4. Must magic-link/passkey and bearer-token entry points coexist? Which logout scopes are intended: current app session, other app sessions, WorkOS session, independently issued access tokens? Do not infer global revocation. + +Pending answers block synthetic callback/account assertions and any claim of real acceptance; they do not block the preparation and CLI fixes delivered here. Do not implement `User.find_or_create_by(email:)` just to make an eval green. + +## Future approved run — NOT executed here + +**Explicit approval is required before spending, auth/login, resource creation or dashboard writes.** No dollar ceiling, sandbox access, or account policy has been approved. Have the operator approve the policy above, exact target client/environment, protected-page/account fixtures, logout scope, runtime/browser setup, cost ceiling and time bound first. + +1. Use a disposable OS user/container/VM and a new fixture root. HOME alone does not isolate the system keychain. Use `--insecure-storage` with its private HOME for approved CLI reads/writes; never run credential-clearing/diagnostic commands against the host keychain. Keep all DBs, browser profiles, dependency caches and credentials under that disposable root. Deny unneeded outbound services. Do not copy Nick's profile or repositories. +2. Pin this branch's final CLI commit (`git rev-parse HEAD`) and compiled binary SHA-256, Bun 1.4.2, the fixture descriptor/archive SHA, skills 0.7.3, Agent SDK 0.3.211 and model `claude-opus-4-5-20251101`. Save these with `Gemfile.lock` before/after, resolved WorkOS Ruby gem version and documentation snapshots. The bundled reference fetches mutable upstream README/docs; this remaining reproducibility gap must be recorded, not described as pinned SDK behavior. +3. Prepare/bootstrap as above. Add a reviewed `AUTHKIT_ACCOUNT_POLICY.md` and synthetic test seed/assertion code to this disposable app only after policy approval; record their hashes/diff and baseline row counts. No production identities/data. Baseline here deliberately has no identity. +4. Operator supplies sandbox API key/client ID in the app's ignored, mode-0600 env file and direct model credentials through the disposable process environment, **not arguments, transcripts or committed files**. An approved dashboard session in the isolated storage is needed for full sign-out/Initiate login read-back; API-key-only setup can legitimately remain partial. Confirm key/client/session target alignment and capture redacted before-settings, including unrelated entries. Do not create resources just to bypass missing access. +5. Run the production installer from the disposable root with a **30-minute process-group wall-time limit**, one attempt, no outer retries. The Ruby custom agent currently has zero self-correction retries. Enforce a separately approved provider spend cap; wall time is not a cost cap. Equivalent CLI arguments (with the isolated environment/toolchain already applied) are: + + ```sh + /absolute/path/to/pinned/workos install \ + --install-dir "$ROOT/app" \ + --redirect-uri http://app.fizzy.localhost:3006/auth/callback \ + --no-branch --no-commit --no-git-check --direct --json --insecure-storage + ``` + + Use the disposable job supervisor to retain stdout/stderr/exit code and terminate the entire descendant process group at the bound. Omit homepage override to test preservation, or explicitly approve one. No PR/push flags. Do not call the install complete when the agent exits. + +6. With network stubs first, run the approved Rails request/session tests and synthetic fixture seed in **test** environment. Run the equivalent loopback launcher `bundle exec rails server -b 127.0.0.1 -p 3006` with `PORT=3006`, isolated environment and the app's verified env loader. Do not use the generic `rails server` completion hint as startup evidence. Confirm local hostname resolution/Host handling and actual origin before using hosted redirects. +7. Then, only with approved live access, drive the real hosted login/callback/logout and external Initiate login flow. Capture signed-out/signed-in screenshots, route responses, before/after Identity/Account/User counts and IDs, current account/role, old-cookie replay denial, cross-account denial and existing magic-link/passkey/routes regression results. Use synthetic identities across at least two accounts with active/inactive and role-boundary cases agreed in step 3. Record SDK logout response/destination without storing tokens. Repeat login must retain approved associations without duplicate provisioning. +8. Read back callback/CORS/sign-out/Initiate login in the selected sandbox and diff against the before-settings. Retain unrelated callbacks/origins/logout URIs/homepage/initiate-login values unless an explicit approved change was required. Verify routes in browser; dashboard values alone are insufficient. + +Store sanitized run metadata, command results, source diff, lockfiles, test results, browser screenshots and redacted network/settings evidence in `$ROOT/artifacts//`. Raw env files, cookies, tokens, session dumps and keychain files must not be attached. Record unavailable checks as unavailable. Human review of the six requirements is still required; no evidence-import schema/checker currently upgrades the source grader to pass. + +### Optional existing skills-eval selector (separate approval) + +The existing runner recognizes **only the explicit pair** `--framework=ruby --state=fizzy`; neither default sweeps nor Ruby sweeps include it. It requires a verified local `FIZZY_ARCHIVE` and `FIZZY_APPROVED_RUN=1` before bootstrap or executor construction. The flag records operator intent; it is not itself policy approval or acceptance evidence. + +After separate approval, with isolated process HOME/config and credential setup per `tests/evals/README.md` (root `.env.local` must contain only the approved eval credentials): + +```sh +FIZZY_APPROVED_RUN=1 FIZZY_ARCHIVE=/absolute/path/to/pinned-source.tar.gz \ + bun run eval --framework=ruby --state=fizzy --retry=0 --sequential --no-correction --keep +``` + +This means one scenario, one attempt, concurrency one, zero correction retries, no quality grader. Apply the same 30-minute process-group limit. Each attempt uses `.artifacts/fizzy-evals/attempt-*`; existing eval log/results artifacts remain under `tests/eval-results/`. The agent may update Gemfile/lockfile to install WorkOS after frozen baseline bootstrap; retain that diff. The source grader will report unverified acceptance, not pass. + +**This is a skills-agent eval, not production `runWithCore`: it does not execute the CLI's post-agent URL provisioning.** It cannot replace step 5 or live dashboard proof. Do not run it merely to produce another unsupported success claim. + +Expected initial spend is **one production agent session**; optional skills evaluation adds **one separately approved session**. No dollar estimate is justified without token assumptions and current provider rates. Use `sum(inputTokens × inputRate + outputTokens × outputRate + cacheReadTokens × cacheReadRate + cacheWriteTokens × cacheWriteRate)`, with rates normalized to per-token units, plus any explicitly approved service costs. Confirm model availability/rates and a dollar ceiling first. No invented rate or approved ceiling is implied here. + +## Handoffs and overlaps + +- No SDK or separate skills repository was edited. Upstream Ruby skill handoff: its generic Rails example stores a user in session and clears locally on logout; it lacks an app-specific identity/account policy, visible Rails UI and demonstrated SDK session logout. Its method examples defer to fetched SDK docs. Correct/version those upstream after verifying the actual gem API; do not claim the CLI prompt alone fixed the skill. +- Changed shared file `src/lib/completion-data.ts` only for origin-aware reporting and a Ruby inferred-startup caveat. `run-with-core.ts`, `authkit-application-setup.ts`, `sign-in-route.ts`, auth recovery, spinner behavior, commit/PR logic, SPA guidance and dev-command detection were not changed. +- Existing unmerged Unauthorized-recovery/spinner history was observed but not cherry-picked or depended on. Shared eval executor/runner files changed for explicit Fizzy isolation/selection; coordinate overlapping eval work before publication. +- AUTH-6736 remains incomplete until approved policy and retained real-app acceptance evidence satisfy all six checks. From 5b0953dff54737085850ad3a6353225f9e269318 Mon Sep 17 00:00:00 2001 From: Nick Nisi Date: Tue, 29 Sep 2026 14:02:02 -0500 Subject: [PATCH 4/6] fix(evals): cap Fizzy at one agent attempt without extra model calls --- tests/evals/fizzy-execution.spec.ts | 188 ++++++++++++++++++++++++++++ tests/evals/parallel-runner.ts | 15 ++- tests/evals/runner.ts | 12 +- tests/fixtures/ruby/fizzy/README.md | 2 +- 4 files changed, 207 insertions(+), 10 deletions(-) create mode 100644 tests/evals/fizzy-execution.spec.ts diff --git a/tests/evals/fizzy-execution.spec.ts b/tests/evals/fizzy-execution.spec.ts new file mode 100644 index 00000000..0f6aa4e8 --- /dev/null +++ b/tests/evals/fizzy-execution.spec.ts @@ -0,0 +1,188 @@ +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { mkdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import { ParallelRunner } from './parallel-runner.js'; +import { runEvals } from './runner.js'; +import { FixtureManager } from './fixture-manager.js'; +import { prepareFizzyFixture, bootstrapFizzy } from './fizzy-fixture.js'; +import { AgentExecutor } from './agent-executor.js'; +import { loadCredentials } from './env-loader.js'; +import { QualityGrader } from './graders/quality-grader.js'; +import { collectKeyFiles } from './graders/collect-key-files.js'; +import { FizzyGrader } from './graders/fizzy.grader.js'; + +const { agentRun, qualityGrade } = vi.hoisted(() => ({ agentRun: vi.fn(), qualityGrade: vi.fn() })); +vi.mock('./agent-executor.js', () => ({ + AgentExecutor: vi.fn( + class { + run = agentRun; + }, + ), +})); +vi.mock('./fizzy-fixture.js', async (original) => ({ + ...(await original()), + prepareFizzyFixture: vi.fn(async (root: string) => { + const app = join(root, 'app'); + await mkdir(app); + return app; + }), + bootstrapFizzy: vi.fn(async () => {}), +})); +vi.mock('./env-loader.js', () => ({ loadCredentials: vi.fn(() => ({ anthropicApiKey: 'synthetic-only' })) })); +vi.mock('./graders/quality-grader.js', () => ({ + QualityGrader: vi.fn( + class { + grade = qualityGrade; + }, + ), +})); +vi.mock('./graders/collect-key-files.js', () => ({ + collectKeyFiles: vi.fn(async () => new Map([['app.rb', 'synthetic source']])), +})); +vi.mock('./versioning.js', () => ({ + captureVersionMetadata: vi.fn(async () => ({ + skillVersions: {}, + cliVersion: 'synthetic', + modelVersion: 'not-executed', + })), +})); +vi.mock('./history.js', () => ({ saveResults: vi.fn(async () => 'synthetic-results') })); +vi.mock('./log-writer.js', () => ({ + LogWriter: class { + getFilePath() { + return 'synthetic-log'; + } + cleanup() {} + }, +})); + +const fizzy = { framework: 'ruby', state: 'fizzy', grader: FizzyGrader }; +beforeEach(() => { + vi.clearAllMocks(); + vi.stubEnv('FIZZY_APPROVED_RUN', '1'); + vi.stubEnv('FIZZY_ARCHIVE', '/synthetic/pinned-archive'); + vi.stubGlobal( + 'fetch', + vi.fn(() => { + throw new Error('Network forbidden'); + }), + ); + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(process, 'on').mockReturnValue(process); + agentRun.mockResolvedValue({ + success: true, + output: 'Synthetic agent finished', + toolCalls: [], + correctionAttempts: 0, + selfCorrected: false, + }); + qualityGrade.mockResolvedValue(null); +}); +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); +}); + +function expectSingleAttempt() { + expect(prepareFizzyFixture).toHaveBeenCalledOnce(); + expect(bootstrapFizzy).toHaveBeenCalledOnce(); + expect(AgentExecutor).toHaveBeenCalledOnce(); + expect(agentRun).toHaveBeenCalledExactlyOnceWith({ enabled: false, maxRetries: 0 }); + expect(vi.mocked(bootstrapFizzy).mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(AgentExecutor).mock.invocationCallOrder[0], + ); + expect(loadCredentials).not.toHaveBeenCalled(); + expect(QualityGrader).not.toHaveBeenCalled(); + expect(qualityGrade).not.toHaveBeenCalled(); + expect(collectKeyFiles).not.toHaveBeenCalled(); + expect(fetch).not.toHaveBeenCalled(); +} + +it.each([undefined, 0, 5, -1, NaN])('caps runEvals with retry=%s and quality/correction overrides', async (retry) => { + const results = await runEvals({ + framework: ['ruby'], + state: 'fizzy', + retry, + noCorrection: false, + quality: true, + noFail: true, + }); + expect(results[0]).toMatchObject({ scenario: 'ruby/fizzy', passed: false, attempts: 1 }); + expect(results[0].checks?.some((check) => check.message?.includes('UNVERIFIED'))).toBe(true); + expectSingleAttempt(); +}); + +it.each([1, 3, 9, 0, -1, NaN, Infinity, 1.5])('caps direct ParallelRunner maxAttempts=%s', async (maxAttempts) => { + const results = await new ParallelRunner([fizzy], { maxAttempts, concurrency: 1, noCorrection: false }).run(); + expect(results[0]).toMatchObject({ passed: false, attempts: 1 }); + expectSingleAttempt(); +}); + +it.each(['FIZZY_APPROVED_RUN', 'FIZZY_ARCHIVE'])( + 'refuses before preparation/bootstrap/agent/quality when %s is missing', + async (key) => { + vi.stubEnv(key, ''); + const results = await runEvals({ framework: ['ruby'], state: 'fizzy', quality: true, retry: 4, noFail: true }); + expect(results[0]).toMatchObject({ + passed: false, + attempts: 1, + error: expect.stringContaining('explicit spending/policy approval'), + }); + expect(prepareFizzyFixture).not.toHaveBeenCalled(); + expect(bootstrapFizzy).not.toHaveBeenCalled(); + expect(AgentExecutor).not.toHaveBeenCalled(); + expect(loadCredentials).not.toHaveBeenCalled(); + expect(QualityGrader).not.toHaveBeenCalled(); + }, +); + +it('does not construct an agent or retry when bootstrap preflight refuses', async () => { + vi.mocked(bootstrapFizzy).mockRejectedValueOnce(new Error('Fizzy source/runtime prerequisites unavailable')); + const results = await new ParallelRunner([fizzy], { maxAttempts: 3, concurrency: 1 }).run(); + expect(results[0]).toMatchObject({ + passed: false, + attempts: 1, + error: expect.stringContaining('prerequisites unavailable'), + }); + expect(prepareFizzyFixture).toHaveBeenCalledOnce(); + expect(bootstrapFizzy).toHaveBeenCalledOnce(); + expect(AgentExecutor).not.toHaveBeenCalled(); +}); + +it.each([false, true])('preserves retries/correction for other scenarios (noCorrection=%s)', async (noCorrection) => { + vi.spyOn(FixtureManager.prototype, 'setup').mockResolvedValue('/synthetic/app'); + const grader = class { + async grade() { + return { passed: false, checks: [] }; + } + }; + const results = await new ParallelRunner([{ framework: 'ruby', state: 'example', grader }], { + maxAttempts: 3, + concurrency: 1, + noCorrection, + }).run(); + expect(results[0]).toMatchObject({ passed: false, attempts: 3 }); + expect(agentRun).toHaveBeenCalledTimes(3); + for (const [config] of agentRun.mock.calls) + expect(config).toEqual(noCorrection ? { enabled: false, maxRetries: 0 } : undefined); +}); + +it('never collects Fizzy quality inputs even with an overridden passing grader', async () => { + const grader = class { + async grade() { + return { passed: true, checks: [] }; + } + }; + await new ParallelRunner([{ ...fizzy, grader }], { maxAttempts: 3, concurrency: 1 }).run(); + expectSingleAttempt(); +}); + +it.each(['ruby/fizzy', 'ruby/example'])('quality eligibility is enforced again for result %s', async (scenario) => { + vi.spyOn(ParallelRunner.prototype, 'run').mockResolvedValue([ + { scenario, passed: true, duration: 1, keyFiles: new Map([['app.rb', 'synthetic source']]) }, + ]); + await runEvals({ framework: ['ruby'], state: scenario.split('/')[1], quality: true, noFail: true }); + expect(qualityGrade).toHaveBeenCalledTimes(scenario === 'ruby/fizzy' ? 0 : 1); + expect(loadCredentials).toHaveBeenCalledTimes(scenario === 'ruby/fizzy' ? 0 : 1); +}); diff --git a/tests/evals/parallel-runner.ts b/tests/evals/parallel-runner.ts index fa5a53f6..ac654bdf 100644 --- a/tests/evals/parallel-runner.ts +++ b/tests/evals/parallel-runner.ts @@ -68,13 +68,17 @@ export class ParallelRunner { private async runScenario(scenario: Scenario): Promise { const scenarioName = `${scenario.framework}/${scenario.state}`; + const isFizzy = scenarioName === 'ruby/fizzy'; + // This source-only grader deliberately cannot pass acceptance. Never turn + // that pending result into additional paid attempts, regardless of options. + const maxAttempts = isFizzy ? 1 : this.options.maxAttempts; console.log(`Starting: ${scenarioName}`); let lastResult: EvalResult | null = null; let lastToolCalls: ToolCall[] = []; let attempt = 0; - while (attempt < this.options.maxAttempts && !this.isShuttingDown) { + while (attempt < maxAttempts && !this.isShuttingDown) { attempt++; // Emit start/retry event @@ -87,7 +91,7 @@ export class ParallelRunner { if (attempt === 1) { evalEvents.emitScenarioStart(eventPayload); } else { - console.log(`[${scenarioName}] Retry attempt ${attempt}/${this.options.maxAttempts}...`); + console.log(`[${scenarioName}] Retry attempt ${attempt}/${maxAttempts}...`); evalEvents.emitScenarioRetry(eventPayload); } @@ -105,7 +109,7 @@ export class ParallelRunner { const executor = new AgentExecutor(workDir, scenario.framework, { verbose: this.options.verbose, scenarioName, - ...(scenario.framework === 'ruby' && scenario.state === 'fizzy' + ...(isFizzy ? { environment: { ...fizzyEnvironment(fixtureManager.getTempDir()!), BUNDLE_FROZEN: 'false' }, redirectUri: FIZZY_FIXTURE.redirectUri, @@ -113,7 +117,7 @@ export class ParallelRunner { : {}), }); const agentResult = await executor.run( - this.options.noCorrection ? { enabled: false, maxRetries: 0 } : undefined, + isFizzy || this.options.noCorrection ? { enabled: false, maxRetries: 0 } : undefined, ); lastToolCalls = agentResult.toolCalls; @@ -121,7 +125,8 @@ export class ParallelRunner { const gradeResult = await grader.grade(); // Collect key files for quality grading (only on pass to avoid wasted effort) - const keyFiles = gradeResult.passed ? await collectKeyFiles(workDir, scenario.framework) : undefined; + const keyFiles = + !isFizzy && gradeResult.passed ? await collectKeyFiles(workDir, scenario.framework) : undefined; lastResult = { scenario: scenarioName, diff --git a/tests/evals/runner.ts b/tests/evals/runner.ts index fec2b314..432a8705 100644 --- a/tests/evals/runner.ts +++ b/tests/evals/runner.ts @@ -167,15 +167,19 @@ export async function runEvals(options: ExtendedEvalOptions): Promise result.scenario !== 'ruby/fizzy' && result.passed && result.keyFiles && result.keyFiles.size > 0, + ); + if (options.quality && qualityCandidates.length > 0) { const credentials = loadCredentials(); const qualityGrader = new QualityGrader(credentials.anthropicApiKey); console.log('\nRunning quality grading on passing scenarios...'); - for (const result of results) { - if (result.passed && result.keyFiles && result.keyFiles.size > 0) { + for (const result of qualityCandidates) { + if (result.keyFiles) { const framework = result.scenario.split('/')[0]; // Build metadata from result diff --git a/tests/fixtures/ruby/fizzy/README.md b/tests/fixtures/ruby/fizzy/README.md index f7b390b0..d665ff44 100644 --- a/tests/fixtures/ruby/fizzy/README.md +++ b/tests/fixtures/ruby/fizzy/README.md @@ -132,7 +132,7 @@ FIZZY_APPROVED_RUN=1 FIZZY_ARCHIVE=/absolute/path/to/pinned-source.tar.gz \ bun run eval --framework=ruby --state=fizzy --retry=0 --sequential --no-correction --keep ``` -This means one scenario, one attempt, concurrency one, zero correction retries, no quality grader. Apply the same 30-minute process-group limit. Each attempt uses `.artifacts/fizzy-evals/attempt-*`; existing eval log/results artifacts remain under `tests/eval-results/`. The agent may update Gemfile/lockfile to install WorkOS after frozen baseline bootstrap; retain that diff. The source grader will report unverified acceptance, not pass. +This means one scenario, one attempt, concurrency one, zero correction retries, no quality grader. The runner enforces the Fizzy single-attempt cap and disables self-correction even when retry/correction options are omitted, overridden, or invalid; direct `ParallelRunner` calls cannot bypass it. Fizzy is excluded from model-quality grading even with `--quality` or a passing grader override. Authorization and bootstrap preflight must succeed before constructing an agent. A further invocation requires separate authorization, not automatic retries. Other scenarios retain their existing retry/correction/quality behavior. Apply the same 30-minute process-group limit. Each attempt uses `.artifacts/fizzy-evals/attempt-*`; existing eval log/results artifacts remain under `tests/eval-results/`. The agent may update Gemfile/lockfile to install WorkOS after frozen baseline bootstrap; retain that diff. The source grader will report unverified acceptance, not pass. **This is a skills-agent eval, not production `runWithCore`: it does not execute the CLI's post-agent URL provisioning.** It cannot replace step 5 or live dashboard proof. Do not run it merely to produce another unsupported success claim. From 206795fcfb72fa7cfaa7c35f0e38b0fca1a9f607 Mon Sep 17 00:00:00 2001 From: Nick Nisi Date: Tue, 29 Sep 2026 14:02:02 -0500 Subject: [PATCH 5/6] fix(evals): protect non-JS credentials before writing env files --- src/lib/env-writer.ts | 16 +- .../__tests__/agent-executor.secrets.spec.ts | 176 ++++++++++++++++++ tests/evals/__tests__/agent-executor.spec.ts | 7 +- tests/evals/agent-executor.ts | 45 +++-- tests/fixtures/ruby/fizzy/README.md | 6 +- 5 files changed, 222 insertions(+), 28 deletions(-) create mode 100644 tests/evals/__tests__/agent-executor.secrets.spec.ts diff --git a/src/lib/env-writer.ts b/src/lib/env-writer.ts index a7c67a38..c65256e1 100644 --- a/src/lib/env-writer.ts +++ b/src/lib/env-writer.ts @@ -8,7 +8,7 @@ const ENV_COVERING_PATTERNS = ['.env', '.env*']; /** * Ensure the given filename is in .gitignore. * Creates .gitignore if it doesn't exist. - * No-ops if one of `coveringPatterns` is already present. + * No-ops if a recognized covering pattern appears after any negations. */ function ensureGitignore(installDir: string, filename: string, coveringPatterns: string[]): void { const gitignorePath = join(installDir, '.gitignore'); @@ -21,9 +21,14 @@ function ensureGitignore(installDir: string, filename: string, coveringPatterns: const content = readFileSync(gitignorePath, 'utf-8'); const lines = content.split('\n').map((line) => line.trim()); - if (lines.some((line) => coveringPatterns.includes(line))) { - return; + // A later negation can expose a previously covered secret. Conservatively + // append an explicit rule after negations rather than attempting to parse globs. + let covered = false; + for (const line of lines) { + if (line.startsWith('!')) covered = false; + else if (coveringPatterns.includes(line)) covered = true; } + if (covered) return; const separator = content.endsWith('\n') ? '' : '\n'; writeFileSync(gitignorePath, `${content}${separator}${filename}\n`); @@ -206,6 +211,11 @@ export function writeCredentialsEnv(installDir: string, envVars: Partial): void { const envPath = join(installDir, '.env'); backupEnvFile(installDir, envPath); diff --git a/tests/evals/__tests__/agent-executor.secrets.spec.ts b/tests/evals/__tests__/agent-executor.secrets.spec.ts new file mode 100644 index 00000000..6628c472 --- /dev/null +++ b/tests/evals/__tests__/agent-executor.secrets.spec.ts @@ -0,0 +1,176 @@ +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { mkdir, mkdtemp, readFile, rm, stat, symlink, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { AgentExecutor } from '../agent-executor.js'; +import { runAgent } from '../../../src/lib/agent-interface.js'; +import { execFileNoThrow } from '../../../src/utils/exec-file.js'; +import { collectKeyFiles } from '../graders/collect-key-files.js'; +import type { SDKMessage } from '@anthropic-ai/claude-agent-sdk'; + +const secrets = vi.hoisted(() => ({ + workosApiKey: 'sk_test_SYNTHETIC_EVAL_SECRET', + workosClientId: 'client_SYNTHETIC', + anthropicApiKey: 'sk-ant-SYNTHETIC_EVAL_SECRET', +})); +vi.mock('../env-loader.js', () => ({ loadCredentials: vi.fn(() => secrets) })); +vi.mock('../../../src/lib/agent-interface.js', () => ({ runAgent: vi.fn(async () => ({})) })); +vi.mock('../../../src/lib/agent-sdk-assets.js', () => ({ + ensureClaudeCodeExecutable: vi.fn(async () => '/synthetic/claude'), +})); +vi.mock('../../../src/lib/validation/quick-checks.js', () => ({ quickCheckValidateAndFormat: vi.fn() })); + +let root: string; +let app: string; +let environment: NodeJS.ProcessEnv; +async function git(...args: string[]) { + const result = await execFileNoThrow('git', args, { cwd: app, env: environment }); + expect(result.status, result.stderr).toBe(0); + return result.stdout; +} + +beforeEach(async () => { + vi.clearAllMocks(); + vi.mocked(runAgent).mockReset().mockResolvedValue({}); + await mkdir('.artifacts', { recursive: true }); + root = await mkdtemp(join(process.cwd(), '.artifacts/eval-secret-test-')); + app = join(root, 'app'); + await mkdir(app); + await mkdir(join(root, 'home')); + environment = { + PATH: process.env.PATH, + HOME: join(root, 'home'), + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_AUTHOR_NAME: 'Synthetic fixture', + GIT_AUTHOR_EMAIL: 'fixture@example.invalid', + GIT_COMMITTER_NAME: 'Synthetic fixture', + GIT_COMMITTER_EMAIL: 'fixture@example.invalid', + }; + vi.stubGlobal( + 'fetch', + vi.fn(() => { + throw new Error('Network forbidden'); + }), + ); + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + vi.spyOn(console, 'error').mockImplementation(() => {}); + await git('init'); + await writeFile(join(app, 'Gemfile'), 'source "https://rubygems.org"\ngem "rails"\n'); + await git('add', '-A'); + await git('-c', 'core.hooksPath=/dev/null', 'commit', '--no-gpg-sign', '-m', 'synthetic baseline'); +}); +afterEach(async () => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + await rm(root, { recursive: true, force: true }); +}); + +it.each(['ruby', 'python', 'go', 'php', 'php-laravel', 'kotlin', 'dotnet', 'elixir'])( + 'protects %s .env from status, diff, staging, output and collected artifacts', + async (framework) => { + const ignore = '# Preserve unrelated rules\n/vendor\n/cache'; + await writeFile(join(app, '.gitignore'), ignore); + // A non-JS fixture can also have frontend tooling; it still needs .env, not .env.local. + await writeFile(join(app, 'package.json'), '{"private":true}\n'); + await git('add', '-A'); + await git('-c', 'core.hooksPath=/dev/null', 'commit', '--no-gpg-sign', '-m', 'fixture ignores'); + vi.mocked(runAgent).mockImplementation(async (...args) => { + // Exercise the leak path: ordinary agent Git inspection/staging enters its transcript. + const status = await git('status', '--porcelain', '--untracked-files=all'); + await git('add', '-A'); + const diff = await git('diff', '--cached'); + args[6]?.({ + type: 'assistant', + message: { content: [{ type: 'text', text: `${status}\n${diff}` }] }, + } as SDKMessage); + return {}; + }); + const result = await new AgentExecutor(app, framework, { environment, verbose: true }).run({ + enabled: false, + maxRetries: 0, + }); + const env = await readFile(join(app, '.env'), 'utf8'); + expect(env).toContain(`WORKOS_API_KEY=${secrets.workosApiKey}`); + expect(env).toContain(`WORKOS_CLIENT_ID=${secrets.workosClientId}`); + if (process.platform !== 'win32') expect((await stat(join(app, '.env'))).mode & 0o777).toBe(0o600); + expect(await readFile(join(app, '.gitignore'), 'utf8')).toContain(ignore); + expect(await git('check-ignore', '.env')).toBe('.env\n'); + expect(await git('status', '--porcelain', '--untracked-files=all')).not.toMatch(/(?:\?\?| M|A ) \.env\n/); + await git('add', '-A'); + expect(await git('ls-files', '--', '.env')).toBe(''); + const diff = await git('diff', 'HEAD'); + const stagedDiff = await git('diff', '--cached'); + const files = await collectKeyFiles(app, framework); + const artifact = JSON.stringify({ result, keyFiles: Object.fromEntries(files), diff, stagedDiff }); + await writeFile(join(root, 'result.json'), artifact); + const prompt = vi.mocked(runAgent).mock.calls[0][1]; + const logs = JSON.stringify([ + vi.mocked(console.log).mock.calls, + vi.mocked(console.warn).mock.calls, + vi.mocked(console.error).mock.calls, + ]); + // Assert raw content, not redacted/filtered copies. Secrets are present only in the protected env file. + for (const secret of Object.values(secrets)) { + expect(prompt).not.toContain(secret); + expect(logs).not.toContain(secret); + expect(await readFile(join(root, 'result.json'), 'utf8')).not.toContain(secret); + } + expect(fetch).not.toHaveBeenCalled(); + }, +); + +it('protects an existing .env and its backup despite earlier ignore rules followed by negations', async () => { + const original = '# existing config\nOTHER=keep\nWORKOS_API_KEY=sk_test_SYNTHETIC_OLD\n'; + const ignore = '.env*\n!.env\n!.env.bak\n'; + await writeFile(join(app, '.gitignore'), ignore); + await writeFile(join(app, '.env'), original, { mode: 0o644 }); + await new AgentExecutor(app, 'ruby', { environment }).run(); + expect(await readFile(join(app, '.env.bak'), 'utf8')).toBe(original); + expect(await readFile(join(app, '.env'), 'utf8')).toContain('OTHER=keep'); + expect(await readFile(join(app, '.gitignore'), 'utf8')).toContain(ignore); + for (const path of ['.env', '.env.bak']) { + if (process.platform !== 'win32') expect((await stat(join(app, path))).mode & 0o777).toBe(0o600); + expect(await git('check-ignore', path)).toBe(`${path}\n`); + } + await git('add', '-A'); + expect(await git('ls-files', '--', '.env', '.env.bak')).toBe(''); + expect(await git('diff', '--cached')).not.toContain('sk_test_SYNTHETIC'); +}); + +it('fails before writing credentials or starting an agent when ignore protection cannot be installed', async () => { + await mkdir(join(app, '.gitignore')); + await expect(new AgentExecutor(app, 'ruby', { environment }).run()).rejects.toThrow(); + await expect(readFile(join(app, '.env'))).rejects.toMatchObject({ code: 'ENOENT' }); + expect(runAgent).not.toHaveBeenCalled(); +}); + +it.skipIf(process.platform === 'win32')( + 'refuses credential symlinks without reading or changing their targets', + async () => { + const target = join(root, 'unrelated.env'); + await writeFile(target, 'UNRELATED=untouched\n', { mode: 0o644 }); + await symlink(target, join(app, '.env')); + await expect(new AgentExecutor(app, 'ruby', { environment }).run()).rejects.toThrow('non-regular'); + expect(await readFile(target, 'utf8')).toBe('UNRELATED=untouched\n'); + expect((await stat(target)).mode & 0o777).toBe(0o644); + expect(runAgent).not.toHaveBeenCalled(); + }, +); + +it('refuses credentials when Git protection cannot be checked', async () => { + await rm(join(app, '.git'), { recursive: true, force: true }); + // Prevent Git from walking up into the CLI worktree containing the test artifact. + environment.GIT_CEILING_DIRECTORIES = root; + await expect(new AgentExecutor(app, 'ruby', { environment }).run()).rejects.toThrow('Git fixture'); + await expect(readFile(join(app, '.env'))).rejects.toMatchObject({ code: 'ENOENT' }); + expect(runAgent).not.toHaveBeenCalled(); +}); + +it.each(['.env', '.env.bak'])('refuses a tracked %s rather than relying on ineffective ignore rules', async (path) => { + await writeFile(join(app, path), 'INNOCUOUS_TEMPLATE=1\n'); + await git('add', path); + await expect(new AgentExecutor(app, 'ruby', { environment }).run()).rejects.toThrow('tracked'); + expect(await readFile(join(app, path), 'utf8')).toBe('INNOCUOUS_TEMPLATE=1\n'); + expect(runAgent).not.toHaveBeenCalled(); +}); diff --git a/tests/evals/__tests__/agent-executor.spec.ts b/tests/evals/__tests__/agent-executor.spec.ts index ad85101c..1171ceb9 100644 --- a/tests/evals/__tests__/agent-executor.spec.ts +++ b/tests/evals/__tests__/agent-executor.spec.ts @@ -43,12 +43,13 @@ vi.mock('../env-loader.js', () => ({ loadCredentials: vi.fn(() => mockCredentials), })); -vi.mock('../../../src/lib/env-writer.js', () => ({ +vi.mock('../../../src/lib/env-writer.js', async (original) => ({ + ...(await original()), writeEnvLocal: vi.fn(), })); -vi.mock('../../../src/utils/env-parser.js', () => ({ - parseEnvFile: vi.fn(() => ({})), +vi.mock('../../../src/utils/exec-file.js', () => ({ + execFileNoThrow: vi.fn(async () => ({ status: 0, stdout: '', stderr: '' })), })); vi.mock('../../../src/lib/settings.js', () => ({ diff --git a/tests/evals/agent-executor.ts b/tests/evals/agent-executor.ts index c26fa29f..23d29438 100644 --- a/tests/evals/agent-executor.ts +++ b/tests/evals/agent-executor.ts @@ -1,8 +1,8 @@ -import { writeFileSync, existsSync, readFileSync } from 'node:fs'; +import { chmod, lstat } from 'node:fs/promises'; import { join } from 'node:path'; import { loadCredentials } from './env-loader.js'; -import { writeEnvLocal } from '../../src/lib/env-writer.js'; -import { parseEnvFile } from '../../src/utils/env-parser.js'; +import { writeEnvFile, writeEnvLocal } from '../../src/lib/env-writer.js'; +import { execFileNoThrow } from '../../src/utils/exec-file.js'; import { getConfig } from '../../src/lib/settings.js'; import { LatencyTracker } from './latency-tracker.js'; import { quickCheckValidateAndFormat } from '../../src/lib/validation/quick-checks.js'; @@ -62,23 +62,6 @@ const SKILL_NAMES: Record = { /** Frameworks that use package.json / .env.local */ const JS_FRAMEWORKS = ['nextjs', 'react', 'react-router', 'tanstack-start', 'vanilla-js', 'sveltekit', 'node']; -/** - * Write a .env file (for non-JS frameworks). - * Merges with existing .env if present. - */ -function writeEnvFile(workDir: string, envVars: Record): void { - const envPath = join(workDir, '.env'); - let existing: Record = {}; - if (existsSync(envPath)) { - existing = parseEnvFile(readFileSync(envPath, 'utf-8')); - } - const merged = { ...existing, ...envVars }; - const content = Object.entries(merged) - .map(([key, value]) => `${key}=${value}`) - .join('\n'); - writeFileSync(envPath, content + '\n'); -} - export class AgentExecutor { private options: AgentExecutorOptions; private credentials: ReturnType; @@ -115,6 +98,28 @@ export class AgentExecutor { if (JS_FRAMEWORKS.includes(this.framework)) { writeEnvLocal(this.workDir, envVars); } else { + // Git ignores cannot protect tracked files. Refuse rather than silently + // untracking a fixture or allowing secrets into its ordinary diff/artifacts. + const tracked = await execFileNoThrow('git', ['ls-files', '--', '.env', '.env.bak'], { + cwd: this.workDir, + env: this.options.environment, + }); + if (tracked.status !== 0 || tracked.stdout.trim()) { + throw new Error('Refusing eval credentials: .env/.env.bak must be untracked in a Git fixture.'); + } + for (const name of ['.env', '.env.bak']) { + const path = join(this.workDir, name); + const info = await lstat(path).catch((error: NodeJS.ErrnoException) => { + if (error.code !== 'ENOENT') throw error; + return undefined; + }); + if (info) { + if (!info.isFile()) throw new Error(`Refusing non-regular eval credential file: ${name}`); + // Tighten existing files before writing or backing up any credentials. + await chmod(path, 0o600); + } + } + // Shared writer installs ignores BEFORE .env/backup writes; new files are 0600. writeEnvFile(this.workDir, envVars); } diff --git a/tests/fixtures/ruby/fizzy/README.md b/tests/fixtures/ruby/fizzy/README.md index d665ff44..cb66b3c0 100644 --- a/tests/fixtures/ruby/fizzy/README.md +++ b/tests/fixtures/ruby/fizzy/README.md @@ -15,7 +15,7 @@ - Locked Rails: `8.2.0.alpha`, Git revision `3df2cbea2027026a29edb92cbb7e336a63e35444`; the remaining public Git/gem dependency revisions/checksums stay in upstream `Gemfile.lock`. Bootstrap uses frozen resolution, not `bundle update`. - Bundled `@workos/skills`: **0.7.3**, Ruby reference loaded through `skills-assets.ts`. Agent SDK: **0.3.211**. Configured model: **claude-opus-4-5-20251101** (not executed). WorkOS Ruby gem version is **not yet established**; a future run must retain its resolved lockfile/version and API documentation snapshot. -Ignored local evidence is under `.artifacts/`: `rails-red.log` (5 failing regressions before fix), `rails-green.log`, `orchestration.log`, `fixture-tests.log`, `fizzy-prepare.log`, `fizzy-preflight.json`, `check.log`, and `build.log`. These artifacts are local, not published. +Ignored local evidence is under `.artifacts/`: `rails-red.log` (5 failing regressions before fix), `rails-green.log`, `orchestration.log`, `fixture-tests.log`, `fizzy-prepare.log`, `fizzy-preflight.json`, `check.log`, and `build.log`. Review fixes additionally retain `fizzy-cap-red.log`/`fizzy-cap-green.log`, `eval-secrets-red.log`/`eval-secrets-green.log`, `review-check.log`, and `review-build.log`. All review reproductions use fake agents and synthetic credentials only. These artifacts are local, not published. ## Why the development port is 3006 @@ -62,11 +62,13 @@ The integration uses `resolveRedirectUri()` rather than hardcoded port 3000. Its Offline tests run the **real Ruby integration** with fake credentials/agent/network, plus the **real installer state machine** through Ruby and the common post-agent URL setup with a mocked backend. Existing production/sandbox, target selection, preservation, and read-back regression suites pass unchanged. New fixture tests mock command execution, use synthetic archive data, and cannot load credentials, download source or run models. Test setup replaces both native keyring and macOS security backends; keyring-isolation regressions are included in the full check. +The eval executor now uses the shared protected writer for non-JS `.env` files (including Ruby projects with frontend package.json files). It refuses tracked `.env`/`.env.bak` files or unavailable Git protection, rejects non-regular credential paths, tightens existing file permissions before writing/backing up, and creates new files as 0600. Ignore rules are installed before secret writes, preserve unrelated rules, and override later negations that would expose the files. Offline temporary-Git-repo tests inspect ordinary status/diff/staging, raw fake-agent Git transcripts, collected key files and serialized result artifacts using synthetic credentials only. This proves the tested Git leak path is closed, not that arbitrary future model output is automatically redacted. + `FizzyGrader` separates static source candidates from all six mandatory acceptance checks. It intentionally returns `passed: false` until a real behavioral/hosted acceptance mechanism exists. Unused SDK strings, missing routes/UI, hardcoded identities, duplicate provisioning, logout that leaves access, and even plausible source cannot become proven acceptance. Positive controls prove only that static observations are collected. The existing Sinatra `RubyGrader` remains unchanged; its `server.rb` syntax bonus is not used for Fizzy. ## Final local validation -Using Bun 1.4.2, `bun run test && bun run typecheck && bun run lint && bun run format:check` passed: **174 test files, 3,231 tests**, TypeScript, Oxlint and formatting all green. Existing Node `fs.rmdir` deprecation warnings were non-failing. `bun run build` also passed and produced `dist/workos`; the binary was not run against credentials or an app. +Using Bun 1.4.2, `bun run test && bun run typecheck && bun run lint && bun run format:check` passed after review fixes: **176 test files, 3,266 tests**, TypeScript, Oxlint and formatting all green. Existing Node `fs.rmdir` deprecation warnings were non-failing. `bun run build` also passed and produced `dist/workos`; the binary was not run against credentials or an app. The real preparation command succeeded. The real offline preflight exited 1 as expected: pinned source verified, runtime prerequisites unavailable, acceptance unverified. Unit tests include positive and negative preflight/bootstrap/source-check controls without executing Ruby, Bundler, network or models. From 08d286858962930f087ef12e5941ef688202caa6 Mon Sep 17 00:00:00 2001 From: Nick Nisi Date: Tue, 29 Sep 2026 14:02:02 -0500 Subject: [PATCH 6/6] test: isolate destructive skills extraction fixtures The extraction race specs deleted the version/UID-keyed temp cache used by parallel doctor --fix tests. A deletion between materialization and discoverSkills made real refreshWorkOSSkills return null, failing the sibling-protection assertion. Reproduced the exact null failure with a filesystem scheduling barrier in isolated archives of base 2f19926 and AUTH-6733 d8f5a4e (3/3 each). The same barrier passes 3/3 with this fixture isolation; an unchanged baseline still fails. Ordinary paired runs passed 12/12 each, confirming the timing sensitivity rather than relying on a retry until green. Mock only this spec's tmpdir to a unique directory, clean it afterward, and assert it differs from the real shared temp directory. Keep real materialization, allowlist and sibling-write protection assertions unchanged. No production or installer branch behavior changes. (cherry picked from commit c9fd8b6654a206c5c7fc8cc820f15923c5e88c8a) --- src/lib/skills-assets.spec.ts | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/lib/skills-assets.spec.ts b/src/lib/skills-assets.spec.ts index b270ca96..fa31425a 100644 --- a/src/lib/skills-assets.spec.ts +++ b/src/lib/skills-assets.spec.ts @@ -1,9 +1,21 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, utimesSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { describe, expect, it, vi } from 'vitest'; +import { afterAll, describe, expect, it, vi } from 'vitest'; import { BUNDLED_SKILLS_VERSION, getReference, getSkillsDir } from './skills-assets.js'; +// All imports of node:os in this worker see the same private temp root, even +// after resetModules(). Other specs and CLI processes keep their own cache. +vi.mock('node:os', async (importOriginal) => { + const actual = await importOriginal(); + const { mkdtempSync } = await import('node:fs'); + const { join } = await import('node:path'); + const root = mkdtempSync(join(actual.tmpdir(), 'workos-skills-spec-')); + return { ...actual, tmpdir: () => root }; +}); + +afterAll(() => rmSync(tmpdir(), { recursive: true, force: true })); + function extractionSuffix(): string { return process.platform === 'win32' ? '' : `-${process.getuid?.() ?? 0}`; } @@ -25,7 +37,12 @@ function tempFilesUnder(root: string): string[] { describe('embedded skills assets', () => { it('materializes the complete plugin tree to a real directory', async () => { + const { tmpdir: sharedTmpdir } = await vi.importActual('node:os'); + // This spec deletes extraction roots: never share them with other workers + // (e.g. doctor --fix copying these assets) or real CLI invocations. + expect(tmpdir()).not.toBe(sharedTmpdir()); const skillsDir = getSkillsDir(); + expect(skillsDir.startsWith(join(tmpdir(), 'workos-skills-'))).toBe(true); expect(skillsDir).toContain(`workos-skills-${BUNDLED_SKILLS_VERSION}`); expect(existsSync(join(skillsDir, 'workos', 'SKILL.md'))).toBe(true); expect(existsSync(join(skillsDir, 'workos-widgets', 'SKILL.md'))).toBe(true); @@ -68,8 +85,7 @@ describe('embedded skills assets', () => { }); describe('materializeFile concurrent extraction race', () => { - // The extraction root is version-keyed and shared with the other tests and - // real CLI runs on this machine; each test wipes it first so materializeFile's + // Wipe only this spec's private extraction root so materializeFile's // identical-target pre-check can't short-circuit before the mocked rename runs. const extractionRoot = join(tmpdir(), `workos-skills-${BUNDLED_SKILLS_VERSION}${extractionSuffix()}`);