feat(api): document REST client and automate spec updates

.github/workflows/openapi-spec-updates.yml

@@ -0,0 +1,96 @@
+name: OpenAPI spec updates
+
+on:
+  schedule:
+    - cron: "0 8 * * 1" # Mondays at 08:00 UTC
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: openapi-spec-updates
+  cancel-in-progress: false
+
+jobs:
+  update:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Setup Python environment
+        uses: ./.github/actions/setup-python-env
+
+      - name: Update pinned specification
+        id: update
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mise exec -- python py/scripts/update-openapi-spec.py \
+            --summary-file "$RUNNER_TEMP/openapi-update.md" >> "$GITHUB_OUTPUT"
+
+      - name: Regenerate client and public reference
+        id: generate
+        if: steps.update.outputs.changed == 'true'
+        continue-on-error: true
+        run: mise exec -- make -C py generate-api-client
+
+      - name: Run codegen tests
+        id: codegen
+        if: steps.update.outputs.changed == 'true'
+        continue-on-error: true
+        run: mise exec -- make -C py test-api-codegen
+
+      - name: Run REST client runtime tests
+        id: runtime
+        if: steps.update.outputs.changed == 'true'
+        continue-on-error: true
+        run: mise exec -- make -C py test-core
+
+      - name: Run public API type tests
+        id: types
+        if: steps.update.outputs.changed == 'true'
+        continue-on-error: true
+        run: mise exec -- uv run --project ./py nox -f ./py/noxfile.py -s test_types
+
+      - name: Add validation results to pull request body
+        if: steps.update.outputs.changed == 'true'
+        env:
+          GENERATE_OUTCOME: ${{ steps.generate.outcome }}
+          CODEGEN_OUTCOME: ${{ steps.codegen.outcome }}
+          RUNTIME_OUTCOME: ${{ steps.runtime.outcome }}
+          TYPES_OUTCOME: ${{ steps.types.outcome }}
+        run: |
+          {
+            echo
+            echo "### Workflow results"
+            echo
+            echo "| Check | Result |"
+            echo "| --- | --- |"
+            echo "| Regeneration | \`$GENERATE_OUTCOME\` |"
+            echo "| Codegen tests | \`$CODEGEN_OUTCOME\` |"
+            echo "| Runtime tests | \`$RUNTIME_OUTCOME\` |"
+            echo "| Type tests | \`$TYPES_OUTCOME\` |"
+          } >> "$RUNNER_TEMP/openapi-update.md"
+
+      - name: Create pull request
+        if: steps.update.outputs.changed == 'true'
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
+        with:
+          commit-message: "chore(api): update pinned OpenAPI spec"
+          branch: auto/update-openapi-spec
+          title: "chore(api): update pinned OpenAPI spec"
+          body-path: ${{ runner.temp }}/openapi-update.md
+          delete-branch: true
+
+      - name: Fail when automated validation did not pass
+        if: >-
+          steps.update.outputs.changed == 'true' &&
+          (steps.generate.outcome != 'success' ||
+          steps.codegen.outcome != 'success' ||
+          steps.runtime.outcome != 'success' ||
+          steps.types.outcome != 'success')
+        run: |
+          echo "The update PR was created, but at least one automated validation failed."
+          exit 1

openapi/README.md

@@ -15,7 +15,8 @@ make generate-api-client
 make check-api-client-codegen
 ```
 
-The check regenerates in a temporary directory and reports drift without changing the worktree.
+The check regenerates in a temporary directory and reports drift without changing the worktree. Generation also synchronizes the reviewed resource, method, and type inventories in the [public REST API client README](../py/src/braintrust/api/README.md).
+
 The reviewed generated surface includes these tags:
 
 - core resources: Projects, Experiments, Datasets, Prompts, and Functions;
@@ -70,6 +71,6 @@ BRAINTRUST_OPENAPI_ROOT=../../braintrust-openapi make fetch-openapi-spec
 ```
 
 The checkout must be at the commit pinned in `config.json`, and its spec must match the pinned hash.
-To update the snapshot, update the commit and hash in `config.json`, fetch, regenerate, and review both
-the upstream spec diff and generated-source diff. Validation and generation apply only to selected tags
-and their transitively reachable schemas.
+To update the snapshot manually, update the commit and hash in `config.json`, fetch, regenerate, and review both the upstream spec diff and generated-source diff. Validation and generation apply only to selected tags and their transitively reachable schemas.
+
+The scheduled and manually dispatchable [OpenAPI spec updates workflow](../.github/workflows/openapi-spec-updates.yml) checks the latest upstream commit that changed the spec. When the pin changes, it updates the snapshot, regenerates the client and public reference, runs codegen, runtime, and type tests, and opens or updates a review PR containing operation/schema summaries and a link to the upstream diff. The workflow never auto-merges its PR. If generation or validation fails, it still opens the update PR with the failure status and then fails the workflow so the new API shape can be reviewed explicitly.

py/Makefile

@@ -41,9 +41,11 @@ fetch-openapi-spec:
 
 generate-api-client:
 	uv run --no-default-groups --group api-codegen python scripts/generate-api-client.py
+	$(PYTHON) scripts/generate-api-docs.py
 
 check-api-client-codegen:
 	uv run --no-default-groups --group api-codegen python scripts/generate-api-client.py --check
+	$(PYTHON) scripts/generate-api-docs.py --check
 
 test-api-codegen:
 	uv run nox -s test_api_codegen
@@ -91,8 +93,8 @@ help:
 	@echo "  build               - Build Python package"
 	@echo "  check-stale-cassettes - Detect orphaned cassette version directories"
 	@echo "  fetch-openapi-spec  - Fetch the hash-verified pinned OpenAPI spec"
-	@echo "  generate-api-client - Generate private REST API models from the pinned spec"
-	@echo "  check-api-client-codegen - Check committed REST API models for drift"
+	@echo "  generate-api-client - Generate the REST API client and public reference from the pinned spec"
+	@echo "  check-api-client-codegen - Check the committed REST API client and public reference for drift"
 	@echo "  test-api-codegen    - Run OpenAPI validator and generator tests"
 	@echo "  sync-pytest-pin     - Sync [dependency-groups].test pytest pin from matrix"
 	@echo "  clean               - Remove build artifacts"

py/scripts/generate-api-docs.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+"""Update the generated reference sections in the public REST API README."""
+
+import argparse
+import ast
+import sys
+from pathlib import Path
+from typing import Iterable
+
+from openapi_codegen import (
+    CONFIG_PATH,
+    SPEC_PATH,
+    collect_generated_operations,
+    generated_resource_name,
+    load_config,
+    read_and_verify_spec,
+)
+
+
+API_ROOT = Path(__file__).resolve().parents[1] / "src" / "braintrust" / "api"
+README_PATH = API_ROOT / "README.md"
+RESOURCE_START = "<!-- BEGIN GENERATED RESOURCE REFERENCE -->"
+RESOURCE_END = "<!-- END GENERATED RESOURCE REFERENCE -->"
+API_EXPORT_START = "<!-- BEGIN GENERATED API EXPORTS -->"
+API_EXPORT_END = "<!-- END GENERATED API EXPORTS -->"
+TYPE_START = "<!-- BEGIN GENERATED REST TYPES -->"
+TYPE_END = "<!-- END GENERATED REST TYPES -->"
+
+
+def _literal_all(path: Path) -> list[str]:
+    tree = ast.parse(path.read_text(encoding="utf-8"))
+    assignment = next(
+        node
+        for node in tree.body
+        if isinstance(node, ast.Assign)
+        and any(isinstance(target, ast.Name) and target.id == "__all__" for target in node.targets)
+    )
+    value = ast.literal_eval(assignment.value)
+    if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
+        raise ValueError(f"{path}.__all__ must be a literal list of strings")
+    return value
+
+
+def _render_resource_reference() -> str:
+    config = load_config(CONFIG_PATH)
+    spec = read_and_verify_spec(config, SPEC_PATH)
+    operations, _ = collect_generated_operations(spec, config)
+    operations_by_tag = {tag: [] for tag in config["endpoint_generator"]["generated_tags"]}
+    for operation in operations:
+        operations_by_tag[operation.tag].append(operation)
+
+    rows = ["| Client property | Methods |", "| --- | --- |"]
+    for tag, tag_operations in operations_by_tag.items():
+        methods = "<br>".join(
+            f"`{operation.constant_name.lower()}` — `{operation.method} {operation.path}`"
+            for operation in tag_operations
+        )
+        rows.append(f"| `client.{generated_resource_name(tag)}` | {methods} |")
+    return "\n".join(rows)
+
+
+def _render_name_table(names: Iterable[str], columns: int = 3) -> str:
+    values = [f"`{name}`" for name in names]
+    rows = ["| " + " | ".join(["Name"] * columns) + " |", "| " + " | ".join(["---"] * columns) + " |"]
+    for index in range(0, len(values), columns):
+        row = values[index : index + columns]
+        row.extend([""] * (columns - len(row)))
+        rows.append("| " + " | ".join(row) + " |")
+    return "\n".join(rows)
+
+
+def _replace_section(content: str, start: str, end: str, replacement: str) -> str:
+    if content.count(start) != 1 or content.count(end) != 1:
+        raise ValueError(f"{README_PATH} must contain exactly one {start!r} and {end!r} marker")
+    prefix, remainder = content.split(start, 1)
+    _, suffix = remainder.split(end, 1)
+    return f"{prefix}{start}\n{replacement}\n{end}{suffix}"
+
+
+def render_readme(content: str) -> str:
+    content = _replace_section(content, RESOURCE_START, RESOURCE_END, _render_resource_reference())
+    content = _replace_section(
+        content,
+        API_EXPORT_START,
+        API_EXPORT_END,
+        _render_name_table(_literal_all(API_ROOT / "__init__.py")),
+    )
+    return _replace_section(
+        content,
+        TYPE_START,
+        TYPE_END,
+        _render_name_table(_literal_all(API_ROOT / "types" / "__init__.py")),
+    )
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--check", action="store_true", help="Report README drift without changing the file.")
+    args = parser.parse_args()
+
+    current = README_PATH.read_text(encoding="utf-8")
+    rendered = render_readme(current)
+    if current == rendered:
+        print(f"Public REST API documentation is current: {README_PATH}")
+        return 0
+    if args.check:
+        print(
+            "Public REST API documentation drift detected. Run `cd py && make generate-api-client`.",
+            file=sys.stderr,
+        )
+        return 1
+    README_PATH.write_text(rendered, encoding="utf-8")
+    print(f"Updated public REST API documentation: {README_PATH}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

py/scripts/openapi_codegen.py

@@ -433,7 +433,8 @@ def _partition_model_source(
 
     common_names = {name for name, tags in owners.items() if len(tags) > 1}
     module_for_name = {
-        name: "common" if name in common_names else _snake_case(next(iter(tags))) for name, tags in owners.items()
+        name: "common" if name in common_names else generated_resource_name(next(iter(tags)))
+        for name, tags in owners.items()
     }
 
     def source_for(node: ast.stmt) -> str:
@@ -467,7 +468,7 @@ def source_for(node: ast.stmt) -> str:
 def generate_tree(output_root: Path, config: Mapping[str, Any], spec: Mapping[str, Any]) -> ValidationReport:
     validate_config(config)
     report = validate_spec(spec, config)
-    operations, inline_models = _collect_generated_operations(spec, config)
+    operations, inline_models = collect_generated_operations(spec, config)
     selected_spec = _slice_model_spec(spec, {operation.operation_id for operation in operations})
     model_spec = _with_inline_models(_extract_colliding_inline_models(selected_spec), inline_models)
     output_root.mkdir(parents=True, exist_ok=True)
@@ -840,7 +841,7 @@ def _operation_retry_mode(method: str, operation_id: str, safe_reads: Set[str],
     return "NONE"
 
 
-def _collect_generated_operations(
+def collect_generated_operations(
     spec: Mapping[str, Any], config: Mapping[str, Any]
 ) -> Tuple[List[GeneratedOperation], List[Tuple[str, Mapping[str, Any]]]]:
     endpoint = _endpoint_config(config)
@@ -1002,7 +1003,7 @@ def _generate_resources(
 
     generated_paths = []
     for tag, tag_operations in sorted(by_tag.items()):
-        resource_path = root / f"{_snake_case(tag)}.py"
+        resource_path = root / f"{generated_resource_name(tag)}.py"
         _write_generated_file(resource_path, _resource_module_source(tag, tag_operations, model_modules), config)
         generated_paths.append(resource_path)
     return generated_paths
@@ -1143,6 +1144,12 @@ def _python_argument_name(value: str) -> str:
     return result
 
 
+def generated_resource_name(tag: str) -> str:
+    """Return the Python client property and module name for an OpenAPI tag."""
+
+    return _snake_case(tag)
+
+
 def _snake_case(value: str) -> str:
     value = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", value)
     return re.sub(r"[^A-Za-z0-9]+", "_", value).strip("_").lower()