Migration/step 08

.github/workflows/ci.yml

@@ -27,6 +27,43 @@ jobs:
       - name: Run shellcheck
         run: shellcheck scripts/*.sh
 
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+
+      # markdown-link-check runs in its own job below. The frontends' lint-staged
+      # hooks only check staged files; test-node.yml runs the same checks.
+      - name: Run pre-commit
+        uses: pre-commit/action@v3.0.1
+        env:
+          SKIP: markdown-link-check-docs,lint-staged-query-tool,lint-staged-annotation-tool
+
+  markdown-link-check:
+    name: Docs link check (non-blocking)
+    runs-on: ubuntu-latest
+    # Already fails upstream on an external link (see migration/baseline.md)
+    continue-on-error: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+
+      - name: Run markdown-link-check
+        uses: pre-commit/action@v3.0.1
+        with:
+          extra_args: markdown-link-check-docs --all-files
+
   changes:
     runs-on: ubuntu-latest
     permissions:

.pre-commit-config.yaml

@@ -0,0 +1,253 @@
+# Root pre-commit configuration for the Neurobagel monorepo.
+#
+# Merged from the upstream configs of api, federation-api, bagel-cli and the
+# docs, keeping each repository's hook versions and arguments. Hooks run from
+# the repository root, so every hook is scoped to its folder with `files:` and
+# is given that folder's own config file explicitly: tools such as flake8,
+# isort, codespell and markdownlint otherwise look for config files in the
+# current directory only. Each hook has an alias `<hook>-<folder>` for SKIP=.
+#
+# The frontends keep their lint-staged configs; the local hooks at the end run
+# lint-staged in the tool folder on staged files (this replaces husky).
+
+# `pre-commit install` sets up both hook types. Hooks run at commit time unless
+# they set `stages:` (the frontends' post-merge hooks below).
+default_install_hook_types: [pre-commit, post-merge]
+default_stages: [pre-commit]
+
+ci:
+  autofix_prs: false
+  skip: [docker-compose-check-api]
+
+repos:
+  # ------------------------------------------------- Python tools and docs
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 26.10.1
+    hooks:
+      - id: black
+        alias: black-api
+        files: ^tools/api/
+        args: [--config=tools/api/pyproject.toml]
+      - id: black
+        alias: black-federation-api
+        files: ^tools/federation-api/
+        args: [--config=tools/federation-api/pyproject.toml]
+      - id: black
+        alias: black-bagel-cli
+        files: ^tools/bagel-cli/
+        args: [--config=tools/bagel-cli/pyproject.toml]
+
+  - repo: https://github.com/PyCQA/flake8
+    rev: 7.4.1
+    hooks:
+      - id: flake8
+        alias: flake8-api
+        files: ^tools/api/
+        language_version: python3
+        # TODO: replace this with ruff to bring all config into pyproject.toml
+        args: [--config=tools/api/.flake8]
+      - id: flake8
+        alias: flake8-federation-api
+        files: ^tools/federation-api/
+        language_version: python3
+        args:
+          - --extend-ignore=E501
+          - --extend-select=B950
+          - --per-file-ignores=tools/federation-api/app/api/models.py:F722
+      - id: flake8
+        alias: flake8-bagel-cli
+        files: ^tools/bagel-cli/
+        language_version: python3
+        args: [--config=tools/bagel-cli/.flake8]
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      - id: check-yaml
+        alias: check-yaml-api
+        files: ^tools/api/
+      - id: check-toml
+        alias: check-toml-api
+        files: ^tools/api/
+      - id: pretty-format-json
+        alias: pretty-format-json-api
+        files: ^tools/api/
+        args: ["--autofix", "--no-sort-keys"]
+      - id: check-yaml
+        alias: check-yaml-federation-api
+        files: ^tools/federation-api/
+      - id: check-yaml
+        alias: check-yaml-bagel-cli
+        files: ^tools/bagel-cli/
+      - id: check-toml
+        alias: check-toml-bagel-cli
+        files: ^tools/bagel-cli/
+      # docs
+      - id: check-ast
+        alias: check-ast-docs
+        files: ^docs/
+      - id: check-case-conflict
+        alias: check-case-conflict-docs
+        files: ^docs/
+      - id: check-json
+        alias: check-json-docs
+        files: ^docs/
+      - id: check-merge-conflict
+        alias: check-merge-conflict-docs
+        files: ^docs/
+      - id: check-yaml
+        alias: check-yaml-docs
+        files: ^docs/
+        exclude: ^docs/mkdocs\.yml$
+      - id: end-of-file-fixer
+        alias: end-of-file-fixer-docs
+        files: ^docs/
+      - id: mixed-line-ending
+        alias: mixed-line-ending-docs
+        files: ^docs/
+        args: [--fix=lf]
+      - id: trailing-whitespace
+        alias: trailing-whitespace-docs
+        files: ^docs/
+
+  - repo: https://github.com/pycqa/isort
+    rev: 9.0.2
+    hooks:
+      # src_paths default to the settings file's folder, so each tool's package
+      # (app, bagel) stays first-party, as when upstream ran isort in the tool folder
+      - id: isort
+        alias: isort-api
+        files: ^tools/api/
+        args: [--settings-path=tools/api/pyproject.toml]
+      - id: isort
+        alias: isort-federation-api
+        files: ^tools/federation-api/
+        args: [--settings-path=tools/federation-api/pyproject.toml]
+      - id: isort
+        alias: isort-bagel-cli
+        files: ^tools/bagel-cli/
+        args: [--settings-path=tools/bagel-cli/pyproject.toml]
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.4.3
+    hooks:
+      - id: codespell
+        alias: codespell-api
+        files: ^tools/api/
+        additional_dependencies: [tomli]
+        args: [--config=tools/api/.codespellrc, --toml=tools/api/pyproject.toml]
+      - id: codespell
+        alias: codespell-federation-api
+        files: ^tools/federation-api/
+      - id: codespell
+        alias: codespell-bagel-cli
+        files: ^tools/bagel-cli/
+        additional_dependencies: [tomli]
+        args: [--toml=tools/bagel-cli/pyproject.toml]
+
+  - repo: https://github.com/iamthefij/docker-pre-commit
+    rev: v3.0.1
+    hooks:
+      - id: docker-compose-check
+        alias: docker-compose-check-api
+        files: ^tools/api/(.*/)?(docker-)?compose\.ya?ml$
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v2.4.0
+    hooks:
+      - id: mypy
+        alias: mypy-api
+        files: ^tools/api/
+        additional_dependencies: [pydantic]
+        args: [--config-file=tools/api/pyproject.toml]
+      - id: mypy
+        alias: mypy-bagel-cli
+        files: ^tools/bagel-cli/
+        args: [--config-file=tools/bagel-cli/pyproject.toml]
+
+  # federation-api validated its own GitHub workflows; in the monorepo those are the root workflows
+  - repo: https://github.com/python-jsonschema/check-jsonschema
+    rev: 0.38.2
+    hooks:
+      - id: check-github-workflows
+        files: ^\.github/workflows/
+
+  # --------------------------------------------------------------- docs
+  - repo: https://github.com/tcort/markdown-link-check
+    rev: v3.14.2
+    hooks:
+      - id: markdown-link-check
+        alias: markdown-link-check-docs
+        files: ^docs/
+        args: [--quiet, --config=docs/md_link_check_config.json]
+
+  # Check formatting of CSS, JSON (prettier: https://prettier.io/)
+  - repo: https://github.com/rbubley/mirrors-prettier
+    rev: v3.8.1
+    hooks:
+      - id: prettier
+        alias: prettier-docs
+        files: ^docs/
+        types_or: [css, json]
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.4.1
+    hooks:
+      - id: codespell
+        alias: codespell-docs
+        files: ^docs/
+        args: [--config=docs/.codespellrc]
+
+  - repo: https://github.com/igorshubovych/markdownlint-cli
+    rev: v0.47.0
+    hooks:
+      - id: markdownlint
+        alias: markdownlint-docs
+        files: ^docs/
+        args: [-c, docs/.markdownlint.jsonc]
+
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 26.1.0
+    hooks:
+      - id: black
+        alias: black-docs
+        files: ^docs/
+        args: [--safe]
+
+  # ---------------------------------------------------------- frontends
+  # Same checks as the tools' lint-staged configs (typecheck, eslint, prettier),
+  # run by lint-staged on staged files. Needs `npm ci` in the tool folder.
+  - repo: local
+    hooks:
+      - id: lint-staged-query-tool
+        name: lint-staged (query-tool)
+        language: system
+        files: ^tools/query-tool/
+        pass_filenames: false
+        entry: bash -c 'cd tools/query-tool && if [ ! -d node_modules ]; then echo "Run npm ci in tools/query-tool first"; exit 1; fi && npx --no-install lint-staged'
+      - id: lint-staged-annotation-tool
+        name: lint-staged (annotation-tool)
+        language: system
+        files: ^tools/annotation-tool/
+        pass_filenames: false
+        entry: bash -c 'cd tools/annotation-tool && if [ ! -d node_modules ]; then echo "Run npm ci in tools/annotation-tool first"; exit 1; fi && npx --no-install lint-staged'
+
+  # After a merge or pull (git sets ORIG_HEAD to the previous HEAD), reinstall a
+  # frontend's dependencies if its package.json or package-lock.json changed.
+  # Replaces the frontends' husky post-merge hooks.
+  - repo: local
+    hooks:
+      - id: npm-ci-on-merge-query-tool
+        name: npm ci after merge (query-tool)
+        language: system
+        stages: [post-merge]
+        always_run: true
+        pass_filenames: false
+        entry: bash -c 'git diff --quiet ORIG_HEAD HEAD -- tools/query-tool/package.json tools/query-tool/package-lock.json || (cd tools/query-tool && npm ci)'
+      - id: npm-ci-on-merge-annotation-tool
+        name: npm ci after merge (annotation-tool)
+        language: system
+        stages: [post-merge]
+        always_run: true
+        pass_filenames: false
+        entry: bash -c 'git diff --quiet ORIG_HEAD HEAD -- tools/annotation-tool/package.json tools/annotation-tool/package-lock.json || (cd tools/annotation-tool && npm ci)'

README.md

@@ -33,11 +33,29 @@ just check-unchanged  # check that imported tools' source and tests are unchange
 
 `just lint` needs [actionlint](https://github.com/rhysd/actionlint) and [shellcheck](https://www.shellcheck.net/).
 
+## Commit hooks
+
+One [pre-commit](https://pre-commit.com/) configuration at the root, `.pre-commit-config.yaml`, holds every tool's hooks, each scoped to its folder and using the tool's own config files. Install it once:
+
+```bash
+pip install pre-commit   # or: uv tool install pre-commit
+pre-commit install
+```
+
+This installs two git hooks: `pre-commit` runs the checks on every commit, and `post-merge` runs `npm ci` in a frontend after a merge or pull that changed its `package.json` or `package-lock.json`, so local `node_modules` stay in sync.
+
+The frontends' hooks run their `lint-staged` configs and need `npm ci` in `tools/query-tool/` and `tools/annotation-tool/` (they fail with that instruction otherwise). To run all hooks on all files:
+
+```bash
+(cd tools/query-tool && npm ci) && (cd tools/annotation-tool && npm ci)
+SKIP=markdown-link-check-docs pre-commit run --all-files
+```
+
 ## Repository-wide workflows
 
 | Workflow | What it does |
 | --- | --- |
-| `ci.yml` | Lints workflows and scripts, detects which tools a change touches, runs those tools' tests, and checks that imported source and tests are unchanged |
+| `ci.yml` | Runs the pre-commit hooks (and the docs link check, non-blocking), lints workflows and scripts, detects which tools a change touches, runs those tools' tests, and checks that imported source and tests are unchanged |
 | `docs.yml` | Builds the docs on pull requests and deploys them to GitHub Pages on merge |
 | `linkspector.yml` | Checks the links in api's `README.md` (configured by `tools/api/.linkspector.yml`) on changes to api's Markdown |
 
@@ -61,3 +79,4 @@ just check-unchanged  # check that imported tools' source and tests are unchange
 | R2 (Step 7b) | api, federation-api and query-tool images are built from the repository root (`docker build -f tools/<tool>/Dockerfile .`, or `just build <tool>`), with an allowlist `tools/<tool>/Dockerfile.dockerignore` per image; query-tool's Dockerfile copies `examples/` directly; the temporary `scripts/materialize-context.sh` is removed | Docker does not follow symlinks out of a tool folder; one build context for all images, with each image's files listed explicitly | api, federation-api, query-tool | [api](tools/api/README.md#docker-image), [federation-api](tools/federation-api/README.md#docker-image), [query-tool](tools/query-tool/README.md#docker-image) |
 | R3 (Step 7c) | The bagel-cli image installs one pre-built wheel (`tools/bagel-cli/dist/*.whl`), the same file that is published to PyPI, instead of running `pip install bagel` from PyPI; `Dockerfile.nightly` is removed and nightly and release images share one Dockerfile | The image always contains exactly the released (or nightly) artifact, and the build no longer depends on PyPI having the new release yet | bagel-cli | [bagel-cli README: Docker image](tools/bagel-cli/README.md#docker-image) |
 | Step 7d | api's link check runs from the root workflow `linkspector.yml`, in `tools/api`; the other upstream repository-automation workflows are not carried over (`handle_external_pr`, `add_iss2project`, `stale`, `scorecard`, query-tool's example-diff workflows) or are replaced later | GitHub only runs root workflows | api | [Repository-wide workflows](#repository-wide-workflows) |
+| Step 8 | One root `.pre-commit-config.yaml` replaces the per-tool configs of api, federation-api, bagel-cli and docs, and the frontends' husky hooks; CI runs it on every pull request | pre-commit only reads a config at the repository root, and husky only installs hooks there | all | [Commit hooks](#commit-hooks) |