From 18a59b7cb6994ebe3f0408b2de0b40b030a44591 Mon Sep 17 00:00:00 2001 From: yuecideng Date: Sat, 17 Oct 2026 04:07:58 +0000 Subject: [PATCH 1/3] fix(docs): preserve published versions when deploying queued artifacts --- .github/workflows/docs-pages.yml | 102 ++++++---- docs/scripts/merge_published_site.py | 46 ++++- docs/scripts/prepare_docs_deployment.py | 205 +++++++++++++++++++++ tests/ci/test_workflows.py | 31 ++++ tests/docs/test_merge_published_site.py | 17 ++ tests/docs/test_prepare_docs_deployment.py | 205 +++++++++++++++++++++ 6 files changed, 566 insertions(+), 40 deletions(-) create mode 100644 docs/scripts/prepare_docs_deployment.py create mode 100644 tests/docs/test_prepare_docs_deployment.py diff --git a/.github/workflows/docs-pages.yml b/.github/workflows/docs-pages.yml index cf3e5fa96..88e8693bb 100644 --- a/.github/workflows/docs-pages.yml +++ b/.github/workflows/docs-pages.yml @@ -12,9 +12,17 @@ on: required: false type: string artifact_run_id: - description: "Existing CI/CD Pipeline run ID to deploy from instead of rebuilding" + description: "Existing workflow run ID to deploy from instead of rebuilding" required: false type: string + artifact_name: + description: "Site artifact to recover (set ref to the version it rebuilt)" + required: false + default: docs-site + type: choice + options: + - docs-site + - docs-site-manual release: types: [published] @@ -25,8 +33,8 @@ permissions: id-token: write concurrency: - # The release run waits for its dispatched deployment, so it must not hold - # the deployment concurrency group while the child workflow is queued. + # Release dispatches must not hold the deployment group while queuing a child. + # All actual deployments share this lock through merge, publish and verification. group: ${{ github.event_name == 'release' && format('github-pages-release-{0}', github.run_id) || 'github-pages' }} cancel-in-progress: false @@ -41,10 +49,9 @@ jobs: github.event.action == 'published' && startsWith(github.event.release.tag_name, 'v') runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 5 permissions: actions: write - checks: read contents: read steps: - name: Dispatch release docs build @@ -62,12 +69,9 @@ jobs: echo "Could not determine dispatched workflow run ID from: ${dispatched_url}" >&2 exit 1 fi - echo "Waiting for dispatched docs workflow ${dispatched_run_id}." - gh run watch "${dispatched_run_id}" \ - --repo "${GITHUB_REPOSITORY}" \ - --compact \ - --exit-status \ - --interval 10 + echo "Release docs queued: ${dispatched_url}" + echo "### Release documentation" >> "${GITHUB_STEP_SUMMARY}" + echo "Build and deployment status: [${RELEASE_TAG}](${dispatched_url})" >> "${GITHUB_STEP_SUMMARY}" inspect-docs-artifact: if: > @@ -112,11 +116,32 @@ jobs: - name: Download docs site artifact from CI/CD Pipeline run uses: actions/download-artifact@v4 with: - name: docs-site + name: ${{ inputs.artifact_name || 'docs-site' }} run-id: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.id || inputs.artifact_run_id }} github-token: ${{ secrets.GITHUB_TOKEN }} path: ./docs-site + # Deployment tooling comes from this workflow revision, not the release tag. + - uses: actions/checkout@v4 + with: + ref: ${{ github.workflow_sha }} + path: deployment-tools + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install deployment dependencies + run: python -m pip install --index-url https://pypi.org/simple packaging + + - name: Preserve currently published versions + env: + GH_TOKEN: ${{ github.token }} + SITE_URL: https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }} + BUILT_VERSION: ${{ github.event_name == 'workflow_run' && 'main' || startsWith(inputs.ref, 'v') && inputs.ref || 'main' }} + run: | + python deployment-tools/docs/scripts/prepare_docs_deployment.py prepare \ + --build-dir ./docs-site --site-url "${SITE_URL}" \ + --version "${BUILT_VERSION}" --repository "${GITHUB_REPOSITORY}" + - name: Stamp Pages artifact env: DEPLOYMENT_ID: ${{ github.run_id }}-${{ github.run_attempt }} @@ -126,6 +151,7 @@ jobs: uses: actions/upload-pages-artifact@v3 with: name: github-pages-${{ github.run_attempt }} + retention-days: 90 path: ./docs-site - name: Deploy GitHub Pages @@ -139,18 +165,9 @@ jobs: EXPECTED_DEPLOYMENT_ID: ${{ github.run_id }}-${{ github.run_attempt }} PAGE_URL: ${{ steps.deployment.outputs.page_url }} run: | - for attempt in {1..30}; do - marker_url="${PAGE_URL%/}/pages-deployment-id.txt?run=${EXPECTED_DEPLOYMENT_ID}&attempt=${attempt}" - deployed_id="$(curl --location --connect-timeout 5 --max-time 10 --fail --silent --show-error "${marker_url}" || true)" - if [[ "${deployed_id}" == "${EXPECTED_DEPLOYMENT_ID}" ]]; then - echo "Verified Pages deployment ${EXPECTED_DEPLOYMENT_ID}." - exit 0 - fi - echo "Waiting for Pages deployment ${EXPECTED_DEPLOYMENT_ID} (attempt ${attempt}/30)..." - sleep 10 - done - echo "Pages did not serve deployment ${EXPECTED_DEPLOYMENT_ID}." >&2 - exit 1 + python deployment-tools/docs/scripts/prepare_docs_deployment.py verify \ + --build-dir ./docs-site --site-url "${PAGE_URL}" \ + --deployment-id "${EXPECTED_DEPLOYMENT_ID}" build-and-deploy: if: github.event_name == 'workflow_dispatch' && inputs.artifact_run_id == '' @@ -269,6 +286,27 @@ jobs: name: docs-site-manual path: ./docs-site + # Deployment tooling comes from this workflow revision, not the release tag. + - uses: actions/checkout@v4 + with: + ref: ${{ github.workflow_sha }} + path: deployment-tools + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install deployment dependencies + run: python -m pip install --index-url https://pypi.org/simple packaging + + - name: Preserve currently published versions + env: + GH_TOKEN: ${{ github.token }} + SITE_URL: https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }} + BUILT_VERSION: ${{ github.event_name == 'workflow_run' && 'main' || startsWith(inputs.ref, 'v') && inputs.ref || 'main' }} + run: | + python deployment-tools/docs/scripts/prepare_docs_deployment.py prepare \ + --build-dir ./docs-site --site-url "${SITE_URL}" \ + --version "${BUILT_VERSION}" --repository "${GITHUB_REPOSITORY}" + - name: Stamp Pages artifact env: DEPLOYMENT_ID: ${{ github.run_id }}-${{ github.run_attempt }} @@ -278,6 +316,7 @@ jobs: uses: actions/upload-pages-artifact@v3 with: name: github-pages-${{ github.run_attempt }} + retention-days: 90 path: ./docs-site - name: Deploy GitHub Pages @@ -291,15 +330,6 @@ jobs: EXPECTED_DEPLOYMENT_ID: ${{ github.run_id }}-${{ github.run_attempt }} PAGE_URL: ${{ steps.deployment.outputs.page_url }} run: | - for attempt in {1..30}; do - marker_url="${PAGE_URL%/}/pages-deployment-id.txt?run=${EXPECTED_DEPLOYMENT_ID}&attempt=${attempt}" - deployed_id="$(curl --location --connect-timeout 5 --max-time 10 --fail --silent --show-error "${marker_url}" || true)" - if [[ "${deployed_id}" == "${EXPECTED_DEPLOYMENT_ID}" ]]; then - echo "Verified Pages deployment ${EXPECTED_DEPLOYMENT_ID}." - exit 0 - fi - echo "Waiting for Pages deployment ${EXPECTED_DEPLOYMENT_ID} (attempt ${attempt}/30)..." - sleep 10 - done - echo "Pages did not serve deployment ${EXPECTED_DEPLOYMENT_ID}." >&2 - exit 1 + python deployment-tools/docs/scripts/prepare_docs_deployment.py verify \ + --build-dir ./docs-site --site-url "${PAGE_URL}" \ + --deployment-id "${EXPECTED_DEPLOYMENT_ID}" diff --git a/docs/scripts/merge_published_site.py b/docs/scripts/merge_published_site.py index 05bf031ff..8ef45e5ba 100644 --- a/docs/scripts/merge_published_site.py +++ b/docs/scripts/merge_published_site.py @@ -29,6 +29,7 @@ import shutil import subprocess import sys +import time from pathlib import Path from typing import Any from urllib.error import HTTPError, URLError @@ -48,24 +49,35 @@ def load_versions_manifest( *, site_base_url: str | None = None, published_root: Path | None = None, + required: bool = False, ) -> dict[str, Any] | None: """Load ``versions.json`` from a local tree or the live site URL.""" if published_root is not None: manifest_path = published_root / "versions.json" if not manifest_path.is_file(): + if required: + raise FileNotFoundError(manifest_path) return None return json.loads(manifest_path.read_text(encoding="utf-8")) if not site_base_url: return None - manifest_url = f"{site_base_url.rstrip('/')}/versions.json" + manifest_url = f"{site_base_url.rstrip('/')}/versions.json?fresh={time.time_ns()}" try: with urlopen(manifest_url, timeout=30) as response: if response.status != 200: + if required: + raise RuntimeError( + f"Cannot read published manifest: {response.status}" + ) return None return json.loads(response.read().decode("utf-8")) except (HTTPError, URLError, TimeoutError, json.JSONDecodeError) as exc: + if required: + raise RuntimeError( + f"Cannot read published manifest at {manifest_url}" + ) from exc print(f"No published manifest at {manifest_url}: {exc}", file=sys.stderr) return None @@ -165,7 +177,9 @@ def flatten_nested_version_dirs(build_dir: Path) -> list[tuple[Path, Path | None return changes -def _download_version_wget(site_base_url: str, version: str, dest: Path) -> None: +def _download_version_wget( + site_base_url: str, version: str, dest: Path, *, required: bool = False +) -> None: """Download one version subtree with wget (available in CI containers).""" url = f"{site_base_url.rstrip('/')}/{version}/" dest.parent.mkdir(parents=True, exist_ok=True) @@ -177,6 +191,8 @@ def _download_version_wget(site_base_url: str, version: str, dest: Path) -> None [ "wget", "-q", + "--timeout=30", + "--tries=2", "-r", "-l", "50", @@ -189,6 +205,10 @@ def _download_version_wget(site_base_url: str, version: str, dest: Path) -> None check=False, ) if result.returncode != 0: + if required: + raise RuntimeError( + f"Incomplete download of {url} (exit {result.returncode})" + ) print(f"wget failed for {url} (exit {result.returncode})", file=sys.stderr) # wget may create dest.parent// or preserve extra URL path @@ -222,6 +242,8 @@ def merge_published_site( site_base_url: str | None = None, published_root: Path | None = None, skip_versions: frozenset[str] | None = None, + refresh_versions: frozenset[str] = frozenset(), + required: bool = False, ) -> list[str]: """Copy missing version dirs from published site into ``build_dir``. @@ -230,6 +252,8 @@ def merge_published_site( site_base_url: Live Pages base, e.g. ``https://org.github.io/Repo``. published_root: Local published tree for tests (``versions.json`` + dirs). skip_versions: Version names to leave for a fresh build (e.g. ``main``). + refresh_versions: Existing versions to replace with their published tree. + required: Fail if the manifest or a published version homepage is unavailable. Returns: Names of versions merged from the published site. @@ -246,6 +270,7 @@ def merge_published_site( manifest = load_versions_manifest( site_base_url=site_base_url, published_root=published_root, + required=required, ) if not manifest: print("No published versions manifest; skipping merge.") @@ -254,14 +279,23 @@ def merge_published_site( merged: list[str] = [] for entry in manifest.get("versions", []): name = entry.get("name") + if name and (Path(name).name != name or name in {".", ".."}): + raise ValueError(f"Invalid published version: {name!r}") if not name or name in skip: continue - if (build_dir / name).is_dir(): + exists = ( + (build_dir / name / "index.html").is_file() + if required + else (build_dir / name).is_dir() + ) + if exists and name not in refresh_versions: continue if published_root is not None: src = published_root / name if not src.is_dir(): + if required: + raise FileNotFoundError(src) print( f"Published root missing directory {name}; skip.", file=sys.stderr ) @@ -271,7 +305,9 @@ def merge_published_site( merged.append(name) elif site_base_url: print(f"Downloading published version: {name}") - _download_version_wget(site_base_url, name, build_dir / name) + _download_version_wget( + site_base_url, name, build_dir / name, required=required + ) if (build_dir / name).is_dir(): merged.append(name) else: @@ -279,6 +315,8 @@ def merge_published_site( "Neither published_root nor site_base_url set; cannot merge.", file=sys.stderr, ) + if required and not (build_dir / name / "index.html").is_file(): + raise RuntimeError(f"Published version {name} has no downloaded homepage") final_changes = normalize_artifact_paths(build_dir) if final_changes: diff --git a/docs/scripts/prepare_docs_deployment.py b/docs/scripts/prepare_docs_deployment.py new file mode 100644 index 000000000..5c5020cc4 --- /dev/null +++ b/docs/scripts/prepare_docs_deployment.py @@ -0,0 +1,205 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +"""Prepare and verify a Pages replacement without losing published versions.""" + +from __future__ import annotations + +import argparse +import json +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +import time +from pathlib import Path +from urllib.request import urlopen + +from packaging.version import Version + +from merge_published_site import load_versions_manifest, merge_published_site + +__all__ = ["prepare_deployment", "verify_deployment"] + + +def _read_url(site_url: str, path: str) -> bytes: + url = f"{site_url.rstrip('/')}/{path}?fresh={time.time_ns()}" + with urlopen(url, timeout=10) as response: + return response.read() + + +def _published_snapshot( + site_url: str, repository: str, destination: Path +) -> Path | None: + """Prefer the exact deployed artifact; use the live-site mirror if expired.""" + marker = _read_url(site_url, "pages-deployment-id.txt").decode().strip() + if not re.fullmatch(r"[0-9]+-[0-9]+", marker): + raise ValueError(f"Invalid published deployment marker: {marker!r}") + run_id, attempt = marker.split("-") + result = subprocess.run( + [ + "gh", + "run", + "download", + run_id, + "--repo", + repository, + "--name", + f"github-pages-{attempt}", + "--dir", + str(destination), + ], + capture_output=True, + text=True, + check=False, + timeout=180, + ) + if result.returncode: + print("Published Pages artifact unavailable; mirroring the live site.") + return None + with tarfile.open(destination / "artifact.tar") as archive: + archive.extractall(destination / "site", filter="data") + snapshot = destination / "site" + if (snapshot / "pages-deployment-id.txt").read_text().strip() != marker: + raise RuntimeError("Published artifact does not match the live deployment") + return snapshot + + +def prepare_deployment( + build_dir: Path, + version: str, + *, + site_url: str | None = None, + published_root: Path | None = None, + max_versions: int = 5, +) -> None: + """Merge the current site immediately before replacing it with an artifact. + + Args: + build_dir: Downloaded candidate site. + version: Only version rebuilt by this candidate (main or a release tag). + site_url: Live Pages URL, used when no exact published snapshot is supplied. + published_root: Exact previously deployed site, or a local test fixture. + max_versions: Number of newest stable versions to retain. + """ + if version != "main": + if not version.startswith("v") or Path(version).name != version: + raise ValueError(f"Unsupported documentation version: {version!r}") + Version(version) + if max_versions < 1: + raise ValueError("max_versions must be positive") + if not (build_dir / version / "index.html").is_file(): + raise FileNotFoundError(f"Candidate is missing {version}/index.html") + manifest = load_versions_manifest( + site_base_url=site_url, published_root=published_root, required=True + ) + if not isinstance(manifest, dict) or not isinstance(manifest.get("versions"), list): + raise ValueError("Invalid published versions manifest") + names = {entry["name"] for entry in manifest["versions"]} + if manifest.get("latest") not in names: + raise ValueError("Published latest version is missing from its manifest") + # Only the rebuilt version belongs to this candidate. Cached versions must + # never replace content that has been deployed since the build started. + merge_published_site( + build_dir, + site_base_url=site_url, + published_root=published_root, + skip_versions=frozenset({version}), + refresh_versions=frozenset(names), + required=True, + ) + # Use the existing PEP 440 ordering, including post releases, before pruning. + generator = Path(__file__).with_name("generate_versions_json.py") + subprocess.run( + [sys.executable, str(generator), "--build-dir", str(build_dir)], check=True + ) + candidate = json.loads((build_dir / "versions.json").read_text()) + tags = [entry["name"] for entry in candidate["versions"] if entry["type"] == "tag"] + removed = set(tags[max_versions:]) + if version in removed: + raise ValueError(f"{version} falls outside the newest {max_versions} releases") + for name in removed: + shutil.rmtree(build_dir / name) + for name in (names | {version}) - removed: + if not (build_dir / name / "index.html").is_file(): + raise FileNotFoundError(f"Prepared site is missing {name}/index.html") + subprocess.run( + [sys.executable, str(generator), "--build-dir", str(build_dir)], check=True + ) + + +def verify_deployment(build_dir: Path, site_url: str, deployment_id: str) -> None: + """Check the live marker, complete version manifest and every version homepage. + + Args: + build_dir: Prepared site containing the expected versions.json. + site_url: Deployed Pages URL. + deployment_id: Expected workflow run ID and attempt. + """ + marker = _read_url(site_url, "pages-deployment-id.txt").decode().strip() + if marker != deployment_id: + raise RuntimeError(f"Pages serves {marker}, expected {deployment_id}") + expected = json.loads((build_dir / "versions.json").read_text()) + actual = json.loads(_read_url(site_url, "versions.json")) + if actual != expected: + raise RuntimeError("Published versions manifest differs from the prepared site") + for entry in expected["versions"]: + _read_url(site_url, f"{entry['name']}/index.html") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["prepare", "verify"]) + parser.add_argument("--build-dir", type=Path, required=True) + parser.add_argument("--site-url", required=True) + parser.add_argument("--version", default="main") + parser.add_argument("--repository") + parser.add_argument("--deployment-id") + args = parser.parse_args() + if args.command == "prepare": + with tempfile.TemporaryDirectory() as directory: + snapshot = None + if args.repository: + snapshot = _published_snapshot( + args.site_url, args.repository, Path(directory) + ) + prepare_deployment( + args.build_dir, + args.version, + site_url=args.site_url, + published_root=snapshot, + ) + else: + if not args.deployment_id: + parser.error("verify requires --deployment-id") + for attempt in range(30): + try: + verify_deployment(args.build_dir, args.site_url, args.deployment_id) + print( + f"Verified Pages deployment {args.deployment_id} and all version homepages." + ) + return + except (OSError, ValueError, RuntimeError) as exc: + if attempt == 29: + raise + print(f"Waiting for Pages ({attempt + 1}/30): {exc}", flush=True) + time.sleep(10) + + +if __name__ == "__main__": + main() diff --git a/tests/ci/test_workflows.py b/tests/ci/test_workflows.py index 752d64ff3..10477c067 100644 --- a/tests/ci/test_workflows.py +++ b/tests/ci/test_workflows.py @@ -65,6 +65,37 @@ def test_manual_docs_build_trusts_checkout_before_architecture_generation() -> N ) +@pytest.mark.parametrize("job_name", ["deploy-from-artifact", "deploy-manual-build"]) +def test_pages_paths_merge_under_deployment_lock_and_verify_versions( + job_name: str, +) -> None: + workflow = _load_workflow("docs-pages.yml") + assert "'github-pages'" in workflow["concurrency"]["group"] + assert workflow["concurrency"]["cancel-in-progress"] == "false" + steps = workflow["jobs"][job_name]["steps"] + names = [step.get("name", "") for step in steps] + prepare = names.index("Preserve currently published versions") + upload = names.index("Upload Pages artifact") + publish = names.index("Deploy GitHub Pages") + verify = names.index("Verify deployed artifact") + assert prepare < upload < publish < verify + assert "prepare_docs_deployment.py prepare" in steps[prepare]["run"] + assert "prepare_docs_deployment.py verify" in steps[verify]["run"] + assert steps[upload]["with"]["retention-days"] == "90" + checkout = next( + step for step in steps if step.get("uses", "").startswith("actions/checkout@") + ) + assert checkout["with"]["ref"] == "${{ github.workflow_sha }}" + + +def test_release_dispatch_reports_child_without_timing_out_while_queued() -> None: + steps = _load_workflow("docs-pages.yml")["jobs"]["redispatch-release"]["steps"] + command = steps[0]["run"] + assert "gh workflow run docs-pages.yml" in command + assert "gh run watch" not in command + assert "GITHUB_STEP_SUMMARY" in command + + def test_test_package_install_uses_explicit_public_index_and_private_sources() -> None: steps = _load_workflow("main.yml")["jobs"]["test"]["steps"] install = next(step for step in steps if step.get("name") == "Install test package") diff --git a/tests/docs/test_merge_published_site.py b/tests/docs/test_merge_published_site.py index 464fb3467..832428924 100644 --- a/tests/docs/test_merge_published_site.py +++ b/tests/docs/test_merge_published_site.py @@ -305,3 +305,20 @@ def fake_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess[str "copy" ) assert not (build_dir / "EmbodiChain").exists() + + +def test_required_download_rejects_partial_tree( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def failed_download( + *args: object, **kwargs: object + ) -> subprocess.CompletedProcess[str]: + (tmp_path / "v0.3.1").mkdir() + (tmp_path / "v0.3.1" / "index.html").write_text("partial") + return subprocess.CompletedProcess(args=[], returncode=8) + + monkeypatch.setattr(subprocess, "run", failed_download) + with pytest.raises(RuntimeError, match="Incomplete download"): + download_version_wget( + "https://example.org", "v0.3.1", tmp_path / "v0.3.1", required=True + ) diff --git a/tests/docs/test_prepare_docs_deployment.py b/tests/docs/test_prepare_docs_deployment.py new file mode 100644 index 000000000..b5200e4db --- /dev/null +++ b/tests/docs/test_prepare_docs_deployment.py @@ -0,0 +1,205 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +from __future__ import annotations + +import importlib.util +import json +import subprocess +import tarfile +from pathlib import Path +from urllib.error import HTTPError + +import pytest + +from .test_merge_published_site import _write_published_site + +_SCRIPT = ( + Path(__file__).resolve().parents[2] / "docs/scripts/prepare_docs_deployment.py" +) +_spec = importlib.util.spec_from_file_location("prepare_docs_deployment", _SCRIPT) +assert _spec is not None and _spec.loader is not None +_deploy = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_deploy) + + +@pytest.mark.parametrize("version", ["main", "v0.3.1"]) +def test_queued_artifact_preserves_versions_published_after_build( + tmp_path: Path, version: str +) -> None: + candidate, published = tmp_path / "candidate", tmp_path / "published" + _write_published_site(candidate, ["v0.3.0", version], latest="v0.3.0") + _write_published_site(published, ["v0.3.0", "v0.3.1", "main"], latest="v0.3.1") + (candidate / version / "index.html").write_text("freshly built") + (candidate / "v0.3.0" / "index.html").write_text("stale cached release") + if version != "main": + (candidate / "main").mkdir() + (candidate / "main" / "index.html").write_text("stale cached main") + + _deploy.prepare_deployment(candidate, version, published_root=published) + + manifest = json.loads((candidate / "versions.json").read_text()) + assert manifest["latest"] == "v0.3.1" + assert {entry["name"] for entry in manifest["versions"]} == { + "v0.3.0", + "v0.3.1", + "main", + } + assert (candidate / version / "index.html").read_text() == "freshly built" + for name in {"v0.3.0", "v0.3.1", "main"} - {version}: + assert (candidate / name / "index.html").read_bytes() == ( + published / name / "index.html" + ).read_bytes() + assert "./v0.3.1/index.html" in (candidate / "index.html").read_text() + + +@pytest.mark.parametrize("missing", ["manifest", "homepage", "target"]) +def test_incomplete_sites_stop_preparation(tmp_path: Path, missing: str) -> None: + candidate, published = tmp_path / "candidate", tmp_path / "published" + _write_published_site(candidate, ["main"], latest="main") + _write_published_site(published, ["main", "v0.3.1"], latest="v0.3.1") + if missing == "manifest": + (published / "versions.json").unlink() + elif missing == "homepage": + (published / "v0.3.1" / "index.html").unlink() + else: + (candidate / "main" / "index.html").unlink() + with pytest.raises((FileNotFoundError, RuntimeError)): + _deploy.prepare_deployment(candidate, "main", published_root=published) + + +def test_retention_applies_after_merging_live_releases(tmp_path: Path) -> None: + candidate, published = tmp_path / "candidate", tmp_path / "published" + tags = [f"v0.3.{i}" for i in range(6)] + _write_published_site(candidate, ["main"], latest="main") + _write_published_site(published, [*tags, "main"], latest=tags[-1]) + _deploy.prepare_deployment(candidate, "main", published_root=published) + manifest = json.loads((candidate / "versions.json").read_text()) + assert [entry["name"] for entry in manifest["versions"]] == [ + *reversed(tags[1:]), + "main", + ] + assert not (candidate / tags[0]).exists() + + +def test_old_release_outside_retention_is_rejected(tmp_path: Path) -> None: + candidate, published = tmp_path / "candidate", tmp_path / "published" + _write_published_site(candidate, ["v0.1.0"], latest="v0.1.0") + _write_published_site(published, ["v0.3.0", "main"], latest="v0.3.0") + with pytest.raises(ValueError, match="outside"): + _deploy.prepare_deployment( + candidate, "v0.1.0", published_root=published, max_versions=1 + ) + + +@pytest.mark.parametrize("failure", [None, "marker", "manifest", "homepage"]) +def test_verification_checks_versions_and_homepages( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: str | None +) -> None: + _write_published_site(tmp_path, ["main", "v0.3.1"], latest="v0.3.1") + requests = [] + + def read_url(site: str, path: str) -> bytes: + requests.append(path) + if path == "pages-deployment-id.txt": + return b"old" if failure == "marker" else b"123-1" + if path == "versions.json" and failure == "manifest": + return b'{"latest": "main", "versions": []}' + if path == "v0.3.1/index.html" and failure == "homepage": + raise HTTPError(site + path, 404, "Not found", None, None) + return (tmp_path / path).read_bytes() + + monkeypatch.setattr(_deploy, "_read_url", read_url) + if failure: + with pytest.raises((RuntimeError, HTTPError)): + _deploy.verify_deployment(tmp_path, "https://example.org", "123-1") + else: + _deploy.verify_deployment(tmp_path, "https://example.org", "123-1") + assert "main/index.html" in requests + assert "v0.3.1/index.html" in requests + + +def test_deployed_snapshot_uses_exact_pages_artifact( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + site, downloaded = tmp_path / "site", tmp_path / "downloaded" + _write_published_site(site, ["main", "v0.3.1"], latest="v0.3.1") + (site / "pages-deployment-id.txt").write_text("123-2") + downloaded.mkdir() + with tarfile.open(downloaded / "artifact.tar", "w") as archive: + for path in site.rglob("*"): + if path.is_file(): + archive.add(path, arcname=path.relative_to(site)) + monkeypatch.setattr(_deploy, "_read_url", lambda *args: b"123-2") + + def download( + command: list[str], **kwargs: object + ) -> subprocess.CompletedProcess[str]: + assert command[:4] == ["gh", "run", "download", "123"] + assert "github-pages-2" in command + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(subprocess, "run", download) + snapshot = _deploy._published_snapshot( + "https://example.org", "owner/repo", downloaded + ) + assert snapshot is not None + assert (snapshot / "v0.3.1" / "index.html").read_bytes() == ( + site / "v0.3.1" / "index.html" + ).read_bytes() + + +def test_expired_artifact_falls_back_to_live_site( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(_deploy, "_read_url", lambda *args: b"123-2") + monkeypatch.setattr( + subprocess, "run", lambda *args, **kwargs: subprocess.CompletedProcess([], 1) + ) + assert ( + _deploy._published_snapshot("https://example.org", "owner/repo", tmp_path) + is None + ) + + +@pytest.mark.parametrize("version", ["v0.3.1.post1", "v0.4.0rc1"]) +def test_pep440_release_tags_are_supported(tmp_path: Path, version: str) -> None: + candidate, published = tmp_path / "candidate", tmp_path / "published" + _write_published_site(candidate, [version], latest=version) + _write_published_site(published, ["main"], latest="main") + _deploy.prepare_deployment(candidate, version, published_root=published) + assert json.loads((candidate / "versions.json").read_text())["latest"] == version + + +@pytest.mark.parametrize("version", ["../main", "v../main", "feature/docs", "version"]) +def test_invalid_target_cannot_select_other_paths(tmp_path: Path, version: str) -> None: + with pytest.raises(ValueError): + _deploy.prepare_deployment(tmp_path, version, published_root=tmp_path) + + +def test_required_manifest_read_failure_is_not_silently_skipped( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import merge_published_site + + _write_published_site(tmp_path, ["main"], latest="main") + + def unavailable(*args: object, **kwargs: object) -> None: + raise HTTPError("https://example.org", 503, "Unavailable", None, None) + + monkeypatch.setattr(merge_published_site, "urlopen", unavailable) + with pytest.raises(RuntimeError, match="Cannot read published manifest"): + _deploy.prepare_deployment(tmp_path, "main", site_url="https://example.org") From de3c99da93c560f6c53a47aba660211bdafd8b49 Mon Sep 17 00:00:00 2001 From: yuecideng Date: Sat, 17 Oct 2026 04:19:55 +0000 Subject: [PATCH 2/3] fix(docs): require complete published artifacts before deployment --- docs/scripts/prepare_docs_deployment.py | 34 +++++++++---------- tests/docs/test_prepare_docs_deployment.py | 38 +++++++++++++++++++--- 2 files changed, 48 insertions(+), 24 deletions(-) diff --git a/docs/scripts/prepare_docs_deployment.py b/docs/scripts/prepare_docs_deployment.py index 5c5020cc4..512ba1d43 100644 --- a/docs/scripts/prepare_docs_deployment.py +++ b/docs/scripts/prepare_docs_deployment.py @@ -43,10 +43,8 @@ def _read_url(site_url: str, path: str) -> bytes: return response.read() -def _published_snapshot( - site_url: str, repository: str, destination: Path -) -> Path | None: - """Prefer the exact deployed artifact; use the live-site mirror if expired.""" +def _published_snapshot(site_url: str, repository: str, destination: Path) -> Path: + """Require the exact deployed artifact, including JavaScript-loaded resources.""" marker = _read_url(site_url, "pages-deployment-id.txt").decode().strip() if not re.fullmatch(r"[0-9]+-[0-9]+", marker): raise ValueError(f"Invalid published deployment marker: {marker!r}") @@ -70,8 +68,10 @@ def _published_snapshot( timeout=180, ) if result.returncode: - print("Published Pages artifact unavailable; mirroring the live site.") - return None + raise RuntimeError( + f"Published Pages artifact {run_id}/{attempt} is unavailable. " + "Refusing to replace the site without a complete published snapshot." + ) with tarfile.open(destination / "artifact.tar") as archive: archive.extractall(destination / "site", filter="data") snapshot = destination / "site" @@ -84,8 +84,7 @@ def prepare_deployment( build_dir: Path, version: str, *, - site_url: str | None = None, - published_root: Path | None = None, + published_root: Path, max_versions: int = 5, ) -> None: """Merge the current site immediately before replacing it with an artifact. @@ -93,7 +92,6 @@ def prepare_deployment( Args: build_dir: Downloaded candidate site. version: Only version rebuilt by this candidate (main or a release tag). - site_url: Live Pages URL, used when no exact published snapshot is supplied. published_root: Exact previously deployed site, or a local test fixture. max_versions: Number of newest stable versions to retain. """ @@ -105,9 +103,7 @@ def prepare_deployment( raise ValueError("max_versions must be positive") if not (build_dir / version / "index.html").is_file(): raise FileNotFoundError(f"Candidate is missing {version}/index.html") - manifest = load_versions_manifest( - site_base_url=site_url, published_root=published_root, required=True - ) + manifest = load_versions_manifest(published_root=published_root, required=True) if not isinstance(manifest, dict) or not isinstance(manifest.get("versions"), list): raise ValueError("Invalid published versions manifest") names = {entry["name"] for entry in manifest["versions"]} @@ -117,7 +113,6 @@ def prepare_deployment( # never replace content that has been deployed since the build started. merge_published_site( build_dir, - site_base_url=site_url, published_root=published_root, skip_versions=frozenset({version}), refresh_versions=frozenset(names), @@ -172,16 +167,17 @@ def main() -> None: parser.add_argument("--deployment-id") args = parser.parse_args() if args.command == "prepare": + if not args.repository: + parser.error( + "prepare requires --repository for the complete published artifact" + ) with tempfile.TemporaryDirectory() as directory: - snapshot = None - if args.repository: - snapshot = _published_snapshot( - args.site_url, args.repository, Path(directory) - ) + snapshot = _published_snapshot( + args.site_url, args.repository, Path(directory) + ) prepare_deployment( args.build_dir, args.version, - site_url=args.site_url, published_root=snapshot, ) else: diff --git a/tests/docs/test_prepare_docs_deployment.py b/tests/docs/test_prepare_docs_deployment.py index b5200e4db..2b1f0a138 100644 --- a/tests/docs/test_prepare_docs_deployment.py +++ b/tests/docs/test_prepare_docs_deployment.py @@ -19,6 +19,7 @@ import importlib.util import json import subprocess +import sys import tarfile from pathlib import Path from urllib.error import HTTPError @@ -43,6 +44,11 @@ def test_queued_artifact_preserves_versions_published_after_build( candidate, published = tmp_path / "candidate", tmp_path / "published" _write_published_site(candidate, ["v0.3.0", version], latest="v0.3.0") _write_published_site(published, ["v0.3.0", "v0.3.1", "main"], latest="v0.3.1") + runtime_data = "_static/architecture/architecture.json" + for name in ("v0.3.0", "v0.3.1", "main"): + data = published / name / runtime_data + data.parent.mkdir(parents=True) + data.write_text('{"nodes": ["runtime-loaded"]}') (candidate / version / "index.html").write_text("freshly built") (candidate / "v0.3.0" / "index.html").write_text("stale cached release") if version != "main": @@ -63,6 +69,9 @@ def test_queued_artifact_preserves_versions_published_after_build( assert (candidate / name / "index.html").read_bytes() == ( published / name / "index.html" ).read_bytes() + assert (candidate / name / runtime_data).read_bytes() == ( + published / name / runtime_data + ).read_bytes() assert "./v0.3.1/index.html" in (candidate / "index.html").read_text() @@ -162,17 +171,34 @@ def download( ).read_bytes() -def test_expired_artifact_falls_back_to_live_site( +def test_expired_artifact_aborts_before_mutating_candidate( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(_deploy, "_read_url", lambda *args: b"123-2") monkeypatch.setattr( subprocess, "run", lambda *args, **kwargs: subprocess.CompletedProcess([], 1) ) - assert ( - _deploy._published_snapshot("https://example.org", "owner/repo", tmp_path) - is None + _write_published_site(tmp_path, ["main"], latest="main") + runtime_data = tmp_path / "main/_static/architecture/architecture.json" + runtime_data.parent.mkdir(parents=True) + runtime_data.write_text("cached runtime data") + monkeypatch.setattr( + sys, + "argv", + [ + str(_SCRIPT), + "prepare", + "--build-dir", + str(tmp_path), + "--site-url", + "https://example.org", + "--repository", + "owner/repo", + ], ) + with pytest.raises(RuntimeError, match="complete published snapshot"): + _deploy.main() + assert runtime_data.read_text() == "cached runtime data" @pytest.mark.parametrize("version", ["v0.3.1.post1", "v0.4.0rc1"]) @@ -202,4 +228,6 @@ def unavailable(*args: object, **kwargs: object) -> None: monkeypatch.setattr(merge_published_site, "urlopen", unavailable) with pytest.raises(RuntimeError, match="Cannot read published manifest"): - _deploy.prepare_deployment(tmp_path, "main", site_url="https://example.org") + merge_published_site.load_versions_manifest( + site_base_url="https://example.org", required=True + ) From 5b89ebaa0110efd10be381374886d5a8558e44c5 Mon Sep 17 00:00:00 2001 From: yuecideng Date: Sat, 17 Oct 2026 04:40:08 +0000 Subject: [PATCH 3/3] fix(ci): install Node for workflow contract tests in simulator image --- .github/workflows/main.yml | 9 +++++++++ tests/ci/test_workflows.py | 22 ++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b087d3809..478b6b525 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -346,6 +346,12 @@ jobs: container: *container_template steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + # Workflow contract tests execute JavaScript; the simulator image has no node. + - uses: actions/setup-node@v4 + with: + node-version: '22' + - name: Preserve Node path for login shells + run: echo "CI_NODE_BIN_DIR=$(dirname "$(command -v node)")" >> "$GITHUB_ENV" - uses: ./.github/actions/activate-conda - name: Download test plan uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 @@ -390,6 +396,9 @@ jobs: shell: bash -l {0} run: | source activate py311 + # The image's login profile can reset the PATH set by setup-node. + export PATH="${CI_NODE_BIN_DIR}:${PATH}" + node --version # Run GPU processes before the long DexSim lane. Native Vulkan/CUDA # teardown can briefly retain allocations after the sim subprocess; # this order prevents the GPU lane from inheriting that pressure. diff --git a/tests/ci/test_workflows.py b/tests/ci/test_workflows.py index 10477c067..6639ebade 100644 --- a/tests/ci/test_workflows.py +++ b/tests/ci/test_workflows.py @@ -110,6 +110,28 @@ def test_test_package_install_uses_explicit_public_index_and_private_sources() - assert "--extra-index-url https://download.blender.org/pypi/" in package_install +def test_simulator_test_job_installs_node_before_workflow_contract_tests() -> None: + steps = _load_workflow("main.yml")["jobs"]["test"]["steps"] + setup_index = next( + i + for i, step in enumerate(steps) + if step.get("uses", "").startswith("actions/setup-node@") + ) + tests_index = next( + i for i, step in enumerate(steps) if step.get("name") == "Run selected tests" + ) + assert steps[setup_index]["with"]["node-version"] == "22" + assert setup_index < tests_index + capture_index = next( + i + for i, step in enumerate(steps) + if step.get("name") == "Preserve Node path for login shells" + ) + assert setup_index < capture_index < tests_index + assert "GITHUB_ENV" in steps[capture_index]["run"] + assert 'export PATH="${CI_NODE_BIN_DIR}:${PATH}"' in steps[tests_index]["run"] + + @pytest.mark.parametrize( "invalid_contents", [