ci(site): run the explorer's browser tests

.github/workflows/ci.yml

@@ -272,9 +272,89 @@ jobs:
             exit 1
           fi
 
+  # The explorer in site/ driven in a real browser: the form, the JSON view, the examples, share links, saved models, cancelling a long count, phone width and the colour schemes, which the unit tests cannot see. It runs only when what the site is built from changed, and decides that itself rather than through a path filter on the workflow or a separate job: a filtered workflow leaves a required check missing, where this job always reports, passing at once when nothing relevant changed, so Required Checks can depend on it.
+  browser:
+    name: Browser
+    needs: determine-runner
+    runs-on: ${{ fromJson(needs.determine-runner.outputs.selected-runner) }}
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      # The files the site, its tests, the library it is built from and this workflow live in. A dispatch always runs. The base is the pull request's base or the commit a push replaces; one that cannot be fetched (a force-pushed branch, a first push) runs the tests, since not knowing is no reason to skip them. The diff is of two trees, so one commit of history each is enough.
+      - name: Decide whether the browser tests are needed
+        id: changes
+        env:
+          EVENT: ${{ github.event_name }}
+          BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }}
+          REPOSITORY: ${{ github.repository }}
+        run: |
+          if [ "$EVENT" = workflow_dispatch ] || [ -z "$BASE_SHA" ] || [ "$BASE_SHA" = 0000000000000000000000000000000000000000 ]; then
+            echo "relevant=true" >> "$GITHUB_OUTPUT"
+            echo "The browser tests run: there is no base to compare with."
+            exit 0
+          fi
+          if ! git fetch --no-tags --depth=1 "https://github.com/${REPOSITORY}.git" "$BASE_SHA"; then
+            echo "relevant=true" >> "$GITHUB_OUTPUT"
+            echo "The browser tests run: the base commit could not be fetched."
+            exit 0
+          fi
+          if git diff --name-only "$BASE_SHA" HEAD | grep -Eq '^(site/|src/|e2e/|playwright\.config\.ts$|package\.json$|pnpm-lock\.yaml$|\.github/workflows/ci\.yml$)'; then
+            echo "relevant=true" >> "$GITHUB_OUTPUT"
+            echo "The browser tests run: the site, the library, the tests or the dependencies changed."
+          else
+            echo "relevant=false" >> "$GITHUB_OUTPUT"
+            echo "The browser tests are skipped: nothing they depend on changed."
+          fi
+      - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
+        if: steps.changes.outputs.relevant == 'true'
+      - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
+        if: steps.changes.outputs.relevant == 'true'
+        with:
+          node-version: '22'
+          cache: pnpm
+      - run: pnpm install --frozen-lockfile
+        if: steps.changes.outputs.relevant == 'true'
+      - name: Resolve Playwright version
+        id: playwright
+        if: steps.changes.outputs.relevant == 'true'
+        run: echo "version=$(pnpm exec playwright --version | awk '{print $2}')" >> "$GITHUB_OUTPUT"
+      # Browser binaries are versioned by Playwright, so its version, with the platform and architecture the binary is for, is the whole key.
+      - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        id: browsers
+        if: steps.changes.outputs.relevant == 'true'
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-${{ runner.os }}-${{ runner.arch }}-${{ steps.playwright.outputs.version }}
+      # Only the headless shell, which is what a headless run uses: the full Chromium that a plain `install chromium` adds is another download the tests never open.
+      - name: Install Chromium's headless shell
+        if: steps.changes.outputs.relevant == 'true' && steps.browsers.outputs.cache-hit != 'true'
+        run: pnpm exec playwright install --only-shell chromium
+      # The cache holds the browser download only; the operating system libraries it needs are installed by apt and are not in it. They need root, which a runner pod need not have: without it the image's own libraries are used, and Chromium names any that is missing when it starts.
+      - name: Install the operating system libraries Chromium needs
+        if: steps.changes.outputs.relevant == 'true'
+        run: |
+          if [ "$(id -u)" -eq 0 ] || sudo -n true 2>/dev/null; then
+            pnpm exec playwright install-deps chromium
+          else
+            echo "Neither root nor passwordless sudo: the runner image's libraries are used."
+          fi
+      - name: Run the browser tests
+        if: steps.changes.outputs.relevant == 'true'
+        run: pnpm test:e2e
+      # Traces of the tests that failed, which are kept only for a failure.
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        if: failure() && steps.changes.outputs.relevant == 'true'
+        with:
+          name: browser-test-results
+          path: test-results
+          retention-days: 7
+          if-no-files-found: ignore
+
   required-checks:
     name: Required Checks
-    needs: [determine-runner, commitlint, lint, typecheck, package, actionlint, zizmor]
+    needs: [determine-runner, commitlint, lint, typecheck, package, actionlint, zizmor, browser]
     # Always runs, even if a dependency failed, so it reports that failure itself rather than being skipped: a required check that is silently absent blocks a PR indefinitely instead of failing it visibly.
     if: always()
     runs-on: ${{ fromJson(needs.determine-runner.outputs.selected-runner) }}

.gitignore

@@ -6,6 +6,8 @@ coverage/
 .eslintcache
 .turbo/
 reports/
+test-results/
+playwright-report/
 
 # The instruction files are tracked symlinks to README.md; a global ignore may otherwise exclude them.
 !CLAUDE.md

README.md

@@ -680,8 +680,9 @@ Node 22 or later, pnpm. ESM only.
 | `pnpm test` | Vitest, tests colocated as `*.unit.test.ts` |
 | `pnpm build` | tsdown into `dist`, then `schemas/combinatrix.schema.json` from `scripts/generate-json-schema.ts` |
 | `pnpm site:build` / `site:dev` / `site:preview` | Build, serve or preview the explorer in `site/`, which imports the library from `src/` |
+| `pnpm test:e2e` | Playwright tests in `e2e/` that drive the built explorer in headless Chromium, starting and stopping its own preview server; run `pnpm exec playwright install --only-shell chromium` once first |
 
-CI also runs `pnpm exec publint` and `pnpm exec attw --pack --profile esm-only` on the build, and installs the packed tarball into a scratch project on each supported Node line to exercise the public API and to compile a TypeScript consumer against it with no DOM library and no type packages, so the published types, and trilean's, depend on neither. trilean's tree types are inferred from zod schemas, and zod's own declarations name the `URL` global, so a consumer that checks library declarations (`skipLibCheck` off) needs the DOM library or a runtime's types for zod's sake; that diagnostic is the one the check allows.
+CI runs the browser tests in `e2e/` as their own job whenever the site, the library source, the tests, the dependencies or the workflow itself change, and the job passes at once when none did, so the required check is never missing. It also runs `pnpm exec publint` and `pnpm exec attw --pack --profile esm-only` on the build, and installs the packed tarball into a scratch project on each supported Node line to exercise the public API and to compile a TypeScript consumer against it with no DOM library and no type packages, so the published types, and trilean's, depend on neither. trilean's tree types are inferred from zod schemas, and zod's own declarations name the `URL` global, so a consumer that checks library declarations (`skipLibCheck` off) needs the DOM library or a runtime's types for zod's sake; that diagnostic is the one the check allows.
 
 ## Releases
 

e2e/explorer.ts

@@ -0,0 +1,175 @@
+import type { Locator, Page } from '@playwright/test';
+import { expect } from '@playwright/test';
+
+import type { Json, JsonObject, JsonStep } from '../site/src/json';
+import { getIn, isJsonObject, parseJson } from '../site/src/json';
+
+/** The most a check of the model, or an operation on it, may take to show its result, in milliseconds. It bounds a hang: the page's own work is counted in combinations, and the models here are small. */
+export const RESULT_TIMEOUT_MS = 20_000;
+
+/** The model of the page's first example, which the page starts on: two rules' worth of structure over three dimensions. */
+export const STARTING_EXAMPLE = 'A rule: bun is not offered on Windows';
+
+/** How much of an unnamed control's markup is put in a failure message, so the control can be found. */
+const REPORTED_HTML_LENGTH = 80;
+
+/** A text node of a rule's tree, as the form writes it. */
+export function textLiteral(value: string): JsonObject {
+  return { kind: 'textLiteral', value };
+}
+
+/** A reference to a dimension, as the form writes it. */
+export function reference(key: string): JsonObject {
+  return { kind: 'reference', key };
+}
+
+/** The comparison of a dimension with a text. */
+export function textEquals(key: string, value: string): JsonObject {
+  return { kind: 'textCompare', op: 'equals', left: reference(key), right: textLiteral(value) };
+}
+
+/** Runs an action on each item, one after the other: the page is one stateful thing, so the steps of a test cannot overlap. */
+export async function inOrder<T>(items: readonly T[], action: (item: T) => Promise<void>): Promise<void> {
+  await items.reduce(async (previous, item) => {
+    await previous;
+    await action(item);
+  }, Promise.resolve());
+}
+
+/** The addresses of the requests a page makes outside the origin of the base address, as the page makes them: the explorer is meant to make none besides the page's own files, the optional gist fetch aside. The list is filled as the page runs. */
+export function trackExternalRequests(page: Page, baseURL: string | undefined): string[] {
+  const origin = new URL(String(baseURL)).host;
+  const outside: string[] = [];
+  page.on('request', (request) => {
+    const { protocol, host } = new URL(request.url());
+    if (protocol !== 'data:' && protocol !== 'blob:' && host !== origin) outside.push(request.url());
+  });
+
+  return outside;
+}
+
+/** What the tests know of the explorer's page: its controls by the field id the form gives each, and the few things every test starts with. */
+export class Explorer {
+  readonly page: Page;
+
+  constructor(page: Page) {
+    this.page = page;
+  }
+
+  /** Opens the page, at a path under its base address, and waits for the model it starts on to be checked. */
+  static async open(page: Page, path = ''): Promise<Explorer> {
+    const explorer = new Explorer(page);
+    await page.goto(path);
+    await explorer.ready();
+
+    return explorer;
+  }
+
+  /** The control with a field id: every control of the form has one, the same on every draw. */
+  field(id: string): Locator {
+    return this.page.locator(`[data-field=${JSON.stringify(id)}]`);
+  }
+
+  /** The element with an id. */
+  id(id: string): Locator {
+    return this.page.locator(`#${id}`);
+  }
+
+  /** Waits until the model in the editor has been checked and found valid. */
+  async ready(): Promise<void> {
+    await expect(this.page.locator('#validation .valid')).toBeVisible({ timeout: RESULT_TIMEOUT_MS });
+  }
+
+  /** The Advanced view, opened if it is closed. */
+  async advanced(): Promise<Locator> {
+    const open = await this.id('advanced').evaluate((element) => element instanceof HTMLDetailsElement && element.open);
+    if (!open) await this.page.locator('#advanced summary').click();
+
+    return this.id('model');
+  }
+
+  /** Replaces the model's JSON text, as typing it would, and waits for it to be checked and found valid unless the caller expects it not to be. */
+  async setJson(model: Json, options: { readonly expectValid?: boolean } = {}): Promise<void> {
+    const editor = await this.advanced();
+    await editor.fill(typeof model === 'string' ? model : JSON.stringify(model));
+    if (options.expectValid !== false) await this.ready();
+  }
+
+  /** Replaces the JSON text with text that need not be JSON. */
+  async setText(text: string): Promise<void> {
+    const editor = await this.advanced();
+    await editor.fill(text);
+  }
+
+  /** The model the JSON view holds, parsed. */
+  async json(): Promise<JsonObject> {
+    const editor = await this.advanced();
+    const parsed = parseJson(await editor.inputValue());
+    if (!isJsonObject(parsed)) throw new Error('the JSON view does not hold an object');
+
+    return parsed;
+  }
+
+  /** The value at a path of the model the JSON view holds. */
+  async at(path: readonly JsonStep[]): Promise<Json | undefined> {
+    return getIn(await this.json(), path);
+  }
+
+  /** Loads an example from the menu, and waits for it to be valid. */
+  async loadExample(name: string): Promise<void> {
+    await this.id('example').selectOption({ label: name });
+    await this.ready();
+  }
+
+  /** The text of an operation's result. */
+  result(id: 'count' | 'list' | 'diagnose' | 'covering' | 'combination'): Locator {
+    return this.id(`${id}-result`);
+  }
+
+  /** Runs an operation by its button. */
+  async run(button: 'count' | 'list' | 'diagnose' | 'covering' | 'at' | 'explain'): Promise<void> {
+    await this.id(button).click();
+  }
+
+  /** Sets a number field. */
+  async setNumber(id: 'list-limit' | 'strength' | 'index', value: number): Promise<void> {
+    await this.id(id).fill(String(value));
+  }
+
+  /** The cards of the form's dimensions, or of its rules. */
+  cards(part: 'dimensions' | 'rules'): Locator {
+    return this.page.locator(`#${part}-heading`).locator('..').locator('li.card');
+  }
+
+  /** The accessible names of the controls of the form, which must be given to each. */
+  async unnamedControls(): Promise<string[]> {
+    return this.page.evaluate(() =>
+      [...document.querySelectorAll<HTMLElement>('#form button, #form input, #form select, #form textarea')]
+        .filter((element) => {
+          const labelled = element instanceof HTMLInputElement || element instanceof HTMLSelectElement || element instanceof HTMLTextAreaElement ? (element.labels?.length ?? 0) > 0 : false;
+
+          return !labelled && element.getAttribute('aria-label') === null && element.textContent.trim() === '';
+        })
+        .map((element) => element.dataset['field'] ?? element.outerHTML.slice(0, REPORTED_HTML_LENGTH)),
+    );
+  }
+
+  /** The field id of the control that has focus, or the empty text when none has one. */
+  async focused(): Promise<string> {
+    return this.page.evaluate(() => document.activeElement?.getAttribute('data-field') ?? '');
+  }
+
+  /** Presses Tab until the control with a field id has focus, as a keyboard-only user would. It gives up after a full turn of the page's focusable controls, which is as many presses as there are controls. */
+  async tabTo(id: string): Promise<void> {
+    const controls = await this.page.locator('button, input, select, textarea, summary, a[href]').count();
+    await this.tabToWithin(id, controls);
+  }
+
+  private async tabToWithin(id: string, presses: number): Promise<void> {
+    if ((await this.focused()) === id) return;
+    if (presses === 0) throw new Error(`Tab never reached ${id}; focus is on ${await this.focused()}`);
+
+    await this.page.keyboard.press('Tab');
+    await this.tabToWithin(id, presses - 1);
+  }
+}