Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions workers/skill-harness/runner/orchestrate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,12 @@ type RunnerResult = ProofResult extends infer Result ? Result extends { elapsedM

const unavailableReport: Report = { _tag: 'Unavailable', reason: 'The runner did not receive a Harness usage report.', warnings: [] }

/** Same paths and contents, in any order. */
function sameFiles(a: Success['files'], b: Success['files']): boolean {
const byPath = new Map(b.map(file => [file.path, file.content]))
return a.length === b.length && a.every(file => byPath.get(file.path) === file.content)
}

export function aggregateReports(reports: readonly (NativeReport | undefined)[]): Report {
if (reports.length === 0 || reports.some(report => !report))
return unavailableReport
Expand All @@ -21,6 +27,8 @@ export async function runGeneration(options: {
generate: (findings: Success['review']['findings']) => Promise<Outcome<Candidate>>
review: (candidate: Candidate) => Promise<Outcome<Success['review']>>
readFiles: (candidate: Candidate) => Promise<Success['files']>
/** The current Skill of an update run. */
baseline?: Success['files']
}): Promise<RunnerResult> {
let findings: Success['review']['findings'] = []
let previousCandidate: Candidate | undefined
Expand All @@ -32,6 +40,12 @@ export async function runGeneration(options: {
if (generation._tag === 'Err')
return { _tag: 'Err', code: 'GENERATION_FAILED', detail: JSON.stringify(generation.error).slice(0, 8000), generation: aggregateReports(generations), reviewReport: reviews.length ? aggregateReports(reviews) : undefined, candidateFiles: previousCandidate ? await options.readFiles(previousCandidate) : undefined }
previousCandidate = generation.value
// update-package-skill leaves the copy byte-identical when the release touched nothing it claims.
if (options.baseline && repairAttempts === 0) {
const candidateFiles = await options.readFiles(generation.value)
if (sameFiles(candidateFiles, options.baseline))
return { _tag: 'Ok', files: candidateFiles, generation: aggregateReports(generations), reviewReport: { _tag: 'Unavailable', reason: 'No review: the update left the Skill unchanged.', warnings: [] }, review: { summary: 'No change needed.', findings: [] }, sourceAttempts: generation.value.sourceAttempts, repairAttempts }
}
const checked = await options.review(generation.value)
reviews.push(checked.report)
if (checked._tag === 'Err')
Expand Down
5 changes: 3 additions & 2 deletions workers/skill-harness/runner/run.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -50,12 +50,12 @@ async function execute() {
'The network gateway allows npm registry and GitHub codeload only. Other destinations are unavailable.',
'Do not retry a denied destination. Use prepared source and installed package types as evidence.',
'If an example cannot run here, record it as untested. Never claim it passed.',
'This task updates an existing Skill. Keep unsupported claims out and avoid unrelated framework tutorials.',
'This task updates an existing Skill for a new release. Test only the lines the release changed; the earlier version\'s tests cover the rest.',
'Batch related source reads into one tool call. Read each source file once unless evidence requires another read.',
'Use one final command to check frontmatter, paths, and examples. Do not repeat counts, greps, or confirmation reads.',
'After writing and checking the output, finish immediately. A separate independent review follows generation.',
stage === 'generation'
? 'Write SKILL.md to the output directory by model turn 20, then verify and edit it there. A deadline then still leaves a candidate. Aim to finish within 35 model turns. Reserve the remaining run budget for independent review and any repair.'
? 'Copy the current Skill to the output directory first, so a deadline still leaves a candidate. Aim to finish within 25 model turns. Reserve the remaining run budget for independent review and any repair.'
: 'Review source claims in batches. Aim to finish within 20 model turns. Return concrete findings when checks finish.',
...(findings.length ? ['Correct these review findings against the exact package source. Preserve other supported guidance.', JSON.stringify(findings)] : []),
].join('\n\n'))
Expand Down Expand Up @@ -95,6 +95,7 @@ async function execute() {
path: file.path,
content: await readFile(join(candidate.outputDir, file.path), 'utf8'),
}))),
baseline: input.currentSkill,
})
}

Expand Down
13 changes: 13 additions & 0 deletions workers/skill-harness/test/orchestrate.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,19 @@ describe('generation and review', () => {
expect(review).toHaveBeenCalledTimes(2)
})

it('skips review when an update leaves the Skill byte-identical, so Skillgen opens nothing', async () => {
const review = vi.fn()
const result = await runGeneration({ generate: async () => generated, review, readFiles: async () => files, baseline: [{ path: 'SKILL.md', content: 'checked Skill' }] })
expect(review).not.toHaveBeenCalled()
expect(result).toMatchObject({ _tag: 'Ok', files, review: { summary: 'No change needed.', findings: [] }, reviewReport: { _tag: 'Unavailable' }, repairAttempts: 0 })
})

it('reviews an update that changed the Skill', async () => {
const review = vi.fn(async () => clean)
await runGeneration({ generate: async () => generated, review, readFiles: async () => files, baseline: [{ path: 'SKILL.md', content: 'older Skill' }] })
expect(review).toHaveBeenCalledOnce()
})

it('returns clean output without spending a repair attempt', async () => {
const generate = vi.fn(async () => generated)
const result = await runGeneration({ generate, review: async () => clean, readFiles: async () => files })
Expand Down
Loading