diff --git a/.config/nextest.toml b/.config/nextest.toml index 7bd3cea9..a76b60ea 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -5,3 +5,22 @@ slow-timeout = "30s" status-level = "slow" final-status-level = "slow" + +# The macOS gate on pull requests into main runs only these tests (#893); +# the full suite runs on macOS on pushes to main, nightly and with the +# `test:macos` label. See docs/ci-performance.md. The filter must cover the +# tests of every macOS-specific code path, so add new ones here: +# - `#[cfg(target_os = "macos")]` process identity (src/logs/process_identity.rs), +# read by the logs unit tests and used to signal by `thirdshift stop`; +# - the non-Linux skills directory handle (src/skills/directory.rs); +# - the `cfg!(target_os = "linux")` branches in child Run handling +# (src/child_run.rs), started from the executable's path by a Spec run. +[profile.macos-gate] +default-filter = """ +(binary_id(thirdshift::bin/thirdshift) & test(/^(logs|skills|child_run)::/)) +| binary_id(thirdshift::stop) +| (binary_id(thirdshift::spec_run) & ( + test(=a_linear_spec_lands_each_ticket_in_order_then_opens_a_ready_spec_pr) + | test(=a_signal_to_the_spec_run_alone_fails_the_ticket_run_then_ends_the_spec_run_as_interrupted) +)) +""" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac9ab5c8..0e0a2fef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,17 @@ name: CI on: push: branches: [main] + # `labeled` lets the `test:macos` label start the full macOS suite on a + # PR's current head. Any label starts a whole new run, which replaces the + # older one under the concurrency group below. pull_request: + types: [opened, synchronize, reopened, labeled] + # Nightly, on main: the full macOS suite catches what the PR gate's + # filterset misses. GitHub tells the user who last changed this cron when + # it fails; see docs/ci-performance.md for what to do then. + schedule: + - cron: "17 6 * * *" + workflow_dispatch: # PR numbers distinguish fork branches. Unique non-PR groups also preserve # pending main runs, which GitHub would otherwise replace. @@ -69,7 +79,52 @@ jobs: # which the tests build from tests/fakes.rs with rustc. - run: cargo test + # Which macOS work this event gets (#893, docs/ci-performance.md): the full + # suite on pushes to main, nightly, by hand or for a PR into main labelled + # `test:macos`; the gate for any other PR into main; none for a PR into + # another base, such as a Spec branch. + macos-plan: + runs-on: ubuntu-24.04 + outputs: + suite: ${{ steps.plan.outputs.suite }} + steps: + - id: plan + env: + EVENT: ${{ github.event_name }} + BASE: ${{ github.base_ref }} + LABELLED: ${{ contains(github.event.pull_request.labels.*.name, 'test:macos') }} + run: | + if [ "$EVENT" != pull_request ]; then suite=full + elif [ "$BASE" != main ]; then suite=none + elif [ "$LABELLED" = true ]; then suite=full + else suite=gate + fi + echo "macOS suite: $suite" + echo "suite=$suite" >> "$GITHUB_OUTPUT" + + # One macOS job: clippy for the macOS target and the tests of the + # macOS-specific code, which the macos-gate profile in .config/nextest.toml + # selects. + test-macos-gate: + needs: macos-plan + if: ${{ needs.macos-plan.outputs.suite == 'gate' }} + runs-on: macos-26 + steps: + - uses: actions/checkout@v7 + - name: Install the Rust version and components rust-toolchain.toml names + run: | + rustup toolchain install + rustc --version + - uses: Swatinem/rust-cache@v2 + with: + shared-key: test-macos + - run: cargo clippy --all-targets -- -D warnings + - uses: taiki-e/install-action@nextest + - run: cargo nextest run --profile macos-gate + test-macos-shard: + needs: macos-plan + if: ${{ needs.macos-plan.outputs.suite == 'full' }} runs-on: macos-26 strategy: fail-fast: false @@ -94,13 +149,24 @@ jobs: # Hash partitions cover the same suite once across all four runners. - run: cargo nextest run --partition hash:${{ matrix.shard }}/${{ strategy.job-total }} - # Keep one stable check name and reject failed, cancelled or skipped shards. + # Keep one stable check name on every run. It requires what the plan + # asked for: every shard passed, or the gate passed, or, with no macOS work, + # both were skipped. A failed, cancelled or wrongly skipped job fails it. test-macos: - needs: test-macos-shard + needs: [macos-plan, test-macos-gate, test-macos-shard] if: ${{ always() }} runs-on: ubuntu-24.04 steps: - - name: Require every macOS shard to pass + - name: Require the macOS work the plan asked for to pass env: - RESULT: ${{ needs.test-macos-shard.result }} - run: test "$RESULT" = success + SUITE: ${{ needs.macos-plan.outputs.suite }} + GATE: ${{ needs.test-macos-gate.result }} + SHARDS: ${{ needs.test-macos-shard.result }} + run: | + echo "plan: ${SUITE:-none made}, gate: $GATE, shards: $SHARDS" + case "$SUITE" in + full) test "$SHARDS" = success ;; + gate) test "$GATE" = success ;; + none) test "$GATE" = skipped && test "$SHARDS" = skipped ;; + *) exit 1 ;; + esac diff --git a/docs/ci-performance.md b/docs/ci-performance.md index e78543e0..e6da2b50 100644 --- a/docs/ci-performance.md +++ b/docs/ci-performance.md @@ -28,8 +28,66 @@ runs and does not require Release on PRs. Thus there is no current required Release check stranded by the path filter. If protection/rulesets later require a Release check on every PR, replace the trigger filter with an always-created gate before enabling that requirement. CI's stable `test-macos` aggregate stays -unfiltered, with `always()` and an explicit success-only test rejecting failed, -cancelled or skipped shards. +unfiltered: it runs with `always()` on every run and passes only on the macOS +work its event requires, as below. + +### Which runs test on macOS + +Issue [#893](https://github.com/JacobStephens2/thirdshift/issues/893) moved the +full macOS suite off pull requests. A `macos-plan` job in CI decides, per event: + +| Event | macOS work | `test-macos` passes when | +| --- | --- | --- | +| Push to `main` | Full 4-shard suite | every shard passed | +| Nightly `schedule`, on `main` | Full 4-shard suite | every shard passed | +| `workflow_dispatch` | Full 4-shard suite on the chosen ref | every shard passed | +| PR into `main` | One gate job | the gate passed | +| PR into `main` labelled `test:macos` | Full 4-shard suite, no gate | every shard passed | +| PR into any other base (Spec branch, Base fix base, anything else) | None | both were skipped | + +A failed or cancelled job, or a skipped one the event requires, fails +`test-macos`. The Linux checks `test`, `site-test` and `console-test` run on +every pull request as before. + +The gate is one `macos-26` job: `cargo clippy --all-targets -- -D warnings`, +then `cargo nextest run --profile macos-gate`. That profile's `default-filter` +in `.config/nextest.toml` selects the tests of the macOS-specific code: the +`#[cfg(target_os = "macos")]` process identity, the non-Linux skills directory +handle, and the platform branches in child Run handling. New macOS-specific +code adds its tests there. + +Adding `test:macos` to an open PR into `main` starts the full suite on its +current head: CI also runs on the `labeled` activity type. Any label starts a +whole new run, Linux included, and the concurrency group cancels the older run, +so no older result stands for the head. A label stays on a PR, so its later +pushes run the full suite too. + +**Why.** The research in [macos-ci-speed.md](research/macos-ci-speed.md) found +the macOS queue comes from this repository's own load on GitHub's 5-job macOS +cap for personal accounts: 3.8 of 5 slots busy on average, and a median 33 +minutes from job creation to the last shard, against 11 minutes for Linux. PRs +into `main` used 47% of macOS runner time, PRs into Spec branches 29% and pushes +to `main` 23%. Every Ticket PR in a Spec run waited about an hour on macOS +before it could Self-merge into its Spec branch, and the same code met macOS +again on the Spec PR. The gate takes one slot for a few minutes where the full +suite takes four for about 18. This is uv's pattern: macOS tests only on main +without a label to opt in. + +**Trade-offs.** + +- A macOS-only regression from a Spec's Ticket shows up at the Spec PR. The + gate on the Spec PR catches it if it is in the gate's filterset; otherwise the + push to `main` after the Spec PR merges, or the nightly run, does. +- A macOS-only regression outside the gate's filterset shows up on `main` or + overnight, not on its PR. A PR that touches platform-sensitive code can ask + for the full suite with `test:macos`. + +**When the nightly run is red.** GitHub sends a scheduled workflow's failure +notification to the user who last changed its `schedule` cron +([GitHub: notifications for workflow runs](https://docs.github.com/en/actions/concepts/workflows-and-actions/notifications-for-workflow-runs)), +so the Day shift hears of it there, and sees it as the red run in the CI +workflow's history on `main`. Fix forward on `main`: open the fixing PR and label +it `test:macos`, so its own run shows the full suite green before it merges. ## Generated Release ownership