From deffc82477766ea733cf3d03720fae3f71601d6a Mon Sep 17 00:00:00 2001 From: Brad Hilton Date: Mon, 28 Sep 2026 09:42:16 +0000 Subject: [PATCH 1/6] Reuse exact TrainerRank GPU validation on draft promotion --- .github/workflows/trainer-rank-gpu.yml | 38 ++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/.github/workflows/trainer-rank-gpu.yml b/.github/workflows/trainer-rank-gpu.yml index bc1faebfd..f3a7a263d 100644 --- a/.github/workflows/trainer-rank-gpu.yml +++ b/.github/workflows/trainer-rank-gpu.yml @@ -18,12 +18,22 @@ jobs: outputs: required: ${{ steps.changes.outputs.required }} head_sha: ${{ steps.changes.outputs.head_sha }} + reused: ${{ steps.reuse.outputs.cache-hit }} steps: - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} fetch-depth: 0 + - id: reuse + name: Find exact prior GPU validation + if: github.event_name == 'pull_request' && github.event.action == 'ready_for_review' + uses: actions/cache/restore@v4 + with: + path: ${{ runner.temp }}/trainer-rank-gpu-success + key: trainer-rank-gpu-success-v1-${{ github.event.pull_request.base.sha }}-${{ github.event.pull_request.head.sha }} + lookup-only: true + - id: changes name: Classify changed files env: @@ -65,7 +75,7 @@ jobs: validate: name: Run on 2x H200 needs: classify - if: needs.classify.outputs.required == 'true' + if: needs.classify.outputs.required == 'true' && needs.classify.outputs.reused != 'true' runs-on: ubuntu-latest timeout-minutes: 45 environment: trainer-rank-gpu-validation @@ -150,6 +160,23 @@ jobs: "${sky_venv}/bin/python" scripts/ci/trainer-rank-gpu.py \ "${RUNNER_TEMP}/trainer-rank-result" + - name: Record successful validation + if: github.event_name == 'pull_request' + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ needs.classify.outputs.head_sha }} + run: | + mkdir -p "${RUNNER_TEMP}/trainer-rank-gpu-success" + printf '%s %s\n' "${BASE_SHA}" "${HEAD_SHA}" \ + > "${RUNNER_TEMP}/trainer-rank-gpu-success/passed" + + - name: Publish exact successful validation + if: github.event_name == 'pull_request' + uses: actions/cache/save@v4 + with: + path: ${{ runner.temp }}/trainer-rank-gpu-success + key: trainer-rank-gpu-success-v1-${{ github.event.pull_request.base.sha }}-${{ github.event.pull_request.head.sha }} + - name: Preserve remote result and diagnostic logs if: always() uses: actions/upload-artifact@v4 @@ -167,17 +194,22 @@ jobs: - env: CLASSIFY_RESULT: ${{ needs.classify.result }} REQUIRED: ${{ needs.classify.outputs.required }} + REUSED: ${{ needs.classify.outputs.reused }} RESULT: ${{ needs.validate.result }} run: | if [ "${CLASSIFY_RESULT}" != "success" ]; then echo "TrainerRank GPU classification failed." >&2 exit 1 fi - if [ "${REQUIRED}" = "true" ] && [ "${RESULT}" != "success" ]; then + if [ "${REQUIRED}" = "true" ] \ + && [ "${REUSED}" != "true" ] \ + && [ "${RESULT}" != "success" ]; then echo "TrainerRank GPU validation required but result was ${RESULT}." >&2 exit 1 fi - if [ "${REQUIRED}" = "true" ]; then + if [ "${REUSED}" = "true" ]; then + echo "TrainerRank GPU validation reused for this exact base and head." + elif [ "${REQUIRED}" = "true" ]; then echo "TrainerRank GPU validation passed." else echo "TrainerRank GPU validation was not required." From c35e2674ad379279c39a12c274b5bf9f4d529035 Mon Sep 17 00:00:00 2001 From: Brad Hilton Date: Mon, 28 Sep 2026 09:47:11 +0000 Subject: [PATCH 2/6] Authenticate reused GPU validations --- .github/workflows/trainer-rank-gpu.yml | 60 ++++++++++++++++---------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/.github/workflows/trainer-rank-gpu.yml b/.github/workflows/trainer-rank-gpu.yml index f3a7a263d..a19670fe7 100644 --- a/.github/workflows/trainer-rank-gpu.yml +++ b/.github/workflows/trainer-rank-gpu.yml @@ -6,6 +6,7 @@ on: workflow_dispatch: permissions: + actions: read contents: read concurrency: @@ -18,7 +19,7 @@ jobs: outputs: required: ${{ steps.changes.outputs.required }} head_sha: ${{ steps.changes.outputs.head_sha }} - reused: ${{ steps.reuse.outputs.cache-hit }} + reused: ${{ steps.reuse.outputs.reused }} steps: - uses: actions/checkout@v4 with: @@ -28,11 +29,43 @@ jobs: - id: reuse name: Find exact prior GPU validation if: github.event_name == 'pull_request' && github.event.action == 'ready_for_review' - uses: actions/cache/restore@v4 + continue-on-error: true + uses: actions/github-script@v7 with: - path: ${{ runner.temp }}/trainer-rank-gpu-success - key: trainer-rank-gpu-success-v1-${{ github.event.pull_request.base.sha }}-${{ github.event.pull_request.head.sha }} - lookup-only: true + script: | + const pull = context.payload.pull_request; + const runs = await github.rest.actions.listWorkflowRuns({ + ...context.repo, + workflow_id: 'trainer-rank-gpu.yml', + event: 'pull_request', + head_sha: pull.head.sha, + status: 'success', + per_page: 100, + }); + const candidates = runs.data.workflow_runs.filter(run => + run.id !== context.runId && + run.path === '.github/workflows/trainer-rank-gpu.yml' && + run.pull_requests.some(previous => + previous.number === pull.number && + previous.head.sha === pull.head.sha && + previous.base.sha === pull.base.sha + ) + ).slice(0, 10); + for (const run of candidates) { + const jobs = await github.rest.actions.listJobsForWorkflowRun({ + ...context.repo, + run_id: run.id, + filter: 'latest', + per_page: 100, + }); + if (jobs.data.jobs.some(job => + job.name === 'Run on 2x H200' && job.conclusion === 'success' + )) { + core.setOutput('reused', 'true'); + core.info(`Reusing successful workflow run ${run.id}.`); + return; + } + } - id: changes name: Classify changed files @@ -160,23 +193,6 @@ jobs: "${sky_venv}/bin/python" scripts/ci/trainer-rank-gpu.py \ "${RUNNER_TEMP}/trainer-rank-result" - - name: Record successful validation - if: github.event_name == 'pull_request' - env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} - HEAD_SHA: ${{ needs.classify.outputs.head_sha }} - run: | - mkdir -p "${RUNNER_TEMP}/trainer-rank-gpu-success" - printf '%s %s\n' "${BASE_SHA}" "${HEAD_SHA}" \ - > "${RUNNER_TEMP}/trainer-rank-gpu-success/passed" - - - name: Publish exact successful validation - if: github.event_name == 'pull_request' - uses: actions/cache/save@v4 - with: - path: ${{ runner.temp }}/trainer-rank-gpu-success - key: trainer-rank-gpu-success-v1-${{ github.event.pull_request.base.sha }}-${{ github.event.pull_request.head.sha }} - - name: Preserve remote result and diagnostic logs if: always() uses: actions/upload-artifact@v4 From 1505e03fd4e403e838763af290a73249763e2e40 Mon Sep 17 00:00:00 2001 From: Brad Hilton Date: Mon, 28 Sep 2026 09:50:56 +0000 Subject: [PATCH 3/6] Bind reused validation to its base artifact --- .github/workflows/trainer-rank-gpu.yml | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/trainer-rank-gpu.yml b/.github/workflows/trainer-rank-gpu.yml index a19670fe7..8ca48213b 100644 --- a/.github/workflows/trainer-rank-gpu.yml +++ b/.github/workflows/trainer-rank-gpu.yml @@ -44,12 +44,9 @@ jobs: }); const candidates = runs.data.workflow_runs.filter(run => run.id !== context.runId && + run.head_sha === pull.head.sha && run.path === '.github/workflows/trainer-rank-gpu.yml' && - run.pull_requests.some(previous => - previous.number === pull.number && - previous.head.sha === pull.head.sha && - previous.base.sha === pull.base.sha - ) + run.pull_requests.some(previous => previous.number === pull.number) ).slice(0, 10); for (const run of candidates) { const jobs = await github.rest.actions.listJobsForWorkflowRun({ @@ -58,9 +55,19 @@ jobs: filter: 'latest', per_page: 100, }); - if (jobs.data.jobs.some(job => + const artifacts = await github.rest.actions.listWorkflowRunArtifacts({ + ...context.repo, + run_id: run.id, + per_page: 100, + }); + const passed = jobs.data.jobs.some(job => job.name === 'Run on 2x H200' && job.conclusion === 'success' - )) { + ); + const base = artifacts.data.artifacts.some(artifact => + !artifact.expired && artifact.name === + `trainer-rank-result-${run.id}-${run.run_attempt}-${pull.base.sha}` + ); + if (passed && base) { core.setOutput('reused', 'true'); core.info(`Reusing successful workflow run ${run.id}.`); return; @@ -197,7 +204,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: trainer-rank-result-${{ github.run_id }}-${{ github.run_attempt }} + name: trainer-rank-result-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.event.pull_request.base.sha || 'manual' }} path: ${{ runner.temp }}/trainer-rank-result if-no-files-found: warn From 3a236bd29f9a122189f304cd966f79603ed09a8e Mon Sep 17 00:00:00 2001 From: Brad Hilton Date: Mon, 28 Sep 2026 09:55:08 +0000 Subject: [PATCH 4/6] Report the reused GPU run --- .github/workflows/trainer-rank-gpu.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/trainer-rank-gpu.yml b/.github/workflows/trainer-rank-gpu.yml index 8ca48213b..44ecea5a8 100644 --- a/.github/workflows/trainer-rank-gpu.yml +++ b/.github/workflows/trainer-rank-gpu.yml @@ -20,6 +20,7 @@ jobs: required: ${{ steps.changes.outputs.required }} head_sha: ${{ steps.changes.outputs.head_sha }} reused: ${{ steps.reuse.outputs.reused }} + reused_run: ${{ steps.reuse.outputs.run_id }} steps: - uses: actions/checkout@v4 with: @@ -69,6 +70,7 @@ jobs: ); if (passed && base) { core.setOutput('reused', 'true'); + core.setOutput('run_id', String(run.id)); core.info(`Reusing successful workflow run ${run.id}.`); return; } @@ -218,6 +220,7 @@ jobs: CLASSIFY_RESULT: ${{ needs.classify.result }} REQUIRED: ${{ needs.classify.outputs.required }} REUSED: ${{ needs.classify.outputs.reused }} + REUSED_RUN: ${{ needs.classify.outputs.reused_run }} RESULT: ${{ needs.validate.result }} run: | if [ "${CLASSIFY_RESULT}" != "success" ]; then @@ -231,7 +234,8 @@ jobs: exit 1 fi if [ "${REUSED}" = "true" ]; then - echo "TrainerRank GPU validation reused for this exact base and head." + echo "TrainerRank GPU validation reused from exact run ${REUSED_RUN}." \ + | tee -a "${GITHUB_STEP_SUMMARY}" elif [ "${REQUIRED}" = "true" ]; then echo "TrainerRank GPU validation passed." else From 4e7e8b7925a76bbaa7a71805139c0bbdf6a5fa2a Mon Sep 17 00:00:00 2001 From: Brad Hilton Date: Mon, 28 Sep 2026 10:02:46 +0000 Subject: [PATCH 5/6] Harden prior GPU validation lookup --- .github/workflows/trainer-rank-gpu.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/trainer-rank-gpu.yml b/.github/workflows/trainer-rank-gpu.yml index 44ecea5a8..e7f074e0a 100644 --- a/.github/workflows/trainer-rank-gpu.yml +++ b/.github/workflows/trainer-rank-gpu.yml @@ -6,7 +6,6 @@ on: workflow_dispatch: permissions: - actions: read contents: read concurrency: @@ -16,6 +15,9 @@ concurrency: jobs: classify: runs-on: ubuntu-latest + permissions: + actions: read + contents: read outputs: required: ${{ steps.changes.outputs.required }} head_sha: ${{ steps.changes.outputs.head_sha }} @@ -31,6 +33,7 @@ jobs: name: Find exact prior GPU validation if: github.event_name == 'pull_request' && github.event.action == 'ready_for_review' continue-on-error: true + timeout-minutes: 2 uses: actions/github-script@v7 with: script: | @@ -47,7 +50,7 @@ jobs: run.id !== context.runId && run.head_sha === pull.head.sha && run.path === '.github/workflows/trainer-rank-gpu.yml' && - run.pull_requests.some(previous => previous.number === pull.number) + (run.pull_requests ?? []).some(previous => previous.number === pull.number) ).slice(0, 10); for (const run of candidates) { const jobs = await github.rest.actions.listJobsForWorkflowRun({ @@ -234,6 +237,7 @@ jobs: exit 1 fi if [ "${REUSED}" = "true" ]; then + test -n "${REUSED_RUN}" echo "TrainerRank GPU validation reused from exact run ${REUSED_RUN}." \ | tee -a "${GITHUB_STEP_SUMMARY}" elif [ "${REQUIRED}" = "true" ]; then From 1318ddc75c5a580b19ae2e1c2f9f99ea12edacab Mon Sep 17 00:00:00 2001 From: Brad Hilton Date: Mon, 28 Sep 2026 10:06:14 +0000 Subject: [PATCH 6/6] Make reused-run audit check strict --- .github/workflows/trainer-rank-gpu.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/trainer-rank-gpu.yml b/.github/workflows/trainer-rank-gpu.yml index e7f074e0a..0d208ad19 100644 --- a/.github/workflows/trainer-rank-gpu.yml +++ b/.github/workflows/trainer-rank-gpu.yml @@ -226,6 +226,7 @@ jobs: REUSED_RUN: ${{ needs.classify.outputs.reused_run }} RESULT: ${{ needs.validate.result }} run: | + set -euo pipefail if [ "${CLASSIFY_RESULT}" != "success" ]; then echo "TrainerRank GPU classification failed." >&2 exit 1