Fix race conditions when running tests in separate worktrees

.github/copilot-instructions.md

@@ -57,6 +57,16 @@ tail -f ~/.pgrx/17.log
 ./scripts/pg-stop.sh
 ```
 
+Local runners require Python 3 and automatically coordinate across worktrees
+using a shared PostgreSQL file lock. Use the scripts above rather than raw
+`cargo pgrx test`; a waiting message is normal when another run owns the lock.
+For low-level commands, use `python3 scripts/with-pg-lock.py <command> [args...]`.
+The wrapper supports non-interactive commands only, including `bash -c`.
+Never delete the lock file to bypass a waiter. `--keep` does not reserve the
+server after the outer command exits; group non-interactive diagnostics into the
+[same locked command](../docs/TESTING.md#concurrent-local-runs) when needed.
+Build/check-only commands do not need the lock.
+
 ## Critical Patterns
 
 ### Schema-Qualify Extension-Owned SQL

.github/workflows/ci.yml

@@ -61,6 +61,9 @@ jobs:
       - name: Test upgrade lifecycle harness
         run: python3 -m unittest discover -s scripts -p test_upgrade_lifecycle.py
 
+      - name: Test local PostgreSQL locking
+        run: python3 -m unittest discover -s tests -p test_pg_lock.py
+
   make_install_smoke:
     name: Source Install Checks
     runs-on: ubuntu-latest
@@ -76,6 +79,9 @@ jobs:
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
+      - name: Test local PostgreSQL locking
+        run: python3 -m unittest discover -s tests -p test_pg_lock.py
+
       - name: Install Rust stable
         run: |
           rustup toolchain install stable --profile minimal
@@ -297,7 +303,7 @@ jobs:
         run: scripts/pgspot-gate.sh /tmp/pg_durable-install.sql
 
       - name: Run unit tests
-        run: cargo pgrx test pg${{ matrix.pg_version }}
+        run: ./scripts/test-unit.sh --pg-version ${{ matrix.pg_version }}
 
       - name: Run E2E tests (all startup policies)
         id: e2e_tests

CONTRIBUTING.md

@@ -46,6 +46,19 @@ tail -f ~/.pgrx/17.log
 
 If you cannot run the targeted check locally, say so in the PR and CI will still run the full suite.
 
+### Concurrent worktrees
+
+Local test scripts require Python 3 and automatically queue for exclusive access
+to the shared PostgreSQL installation. Use these scripts rather than invoking
+`cargo pgrx test` directly. Builds and checks that do not install the extension
+can still run in parallel.
+
+`--keep` leaves the server running but releases the lock when the command exits;
+a queued run may then stop or reset it. See
+[concurrent local runs](docs/TESTING.md#concurrent-local-runs) for wrapping
+non-interactive commands and diagnostics. Interactive investigations require
+coordination with other worktrees.
+
 ### Full pre-PR checks
 
 Before opening a pull request, run the checks relevant to your change:

Makefile

@@ -10,7 +10,7 @@ EXTRA_FEATURES ?=
 ACR_REGISTRY ?= myregistry.azurecr.io
 ACR_IMAGE ?= pg_durable
 
-.PHONY: all build package install uninstall test test-unit test-e2e test-regress pg-clean docker-build docker-push pg-install META.json pgxn-zip install-pgrx pgrx-init
+.PHONY: all build package install uninstall test test-unit test-e2e test-regress _test-regress pg-clean docker-build docker-push pg-install META.json pgxn-zip install-pgrx pgrx-init
 
 # Default target
 all: package
@@ -52,7 +52,7 @@ test:
 
 # Run only pgrx unit tests
 test-unit:
-	./scripts/test.sh --unit
+	./scripts/test-unit.sh --pg-version $(subst pg,,$(PG_VERSION))
 
 # Run only E2E tests (Docker-based)
 test-e2e:
@@ -70,6 +70,7 @@ docker-push: docker-build
 
 # Run local development server
 run:
+	@echo "Warning: interactive development is not protected by the test lock; coordinate with other worktrees." >&2
 	cargo pgrx run pg17
 
 # Clean build artifacts (renamed to avoid PGXS conflict)
@@ -80,15 +81,19 @@ pg-clean:
 
 # Install extension locally (renamed to avoid PGXS conflict)
 pg-install:
-	cargo pgrx install
+	python3 ./scripts/with-pg-lock.py cargo pgrx install
 
 # Run pg_regress tests (convenience target)
 # Override version: make test-regress PG_VERSION=pg18
 test-regress:
+	python3 ./scripts/with-pg-lock.py $(MAKE) _test-regress
+
+# Keep reset, installation, and installcheck in one critical section.
+_test-regress:
 	@echo "Resetting PostgreSQL..."
 	./scripts/pg-reset.sh $(subst pg,,$(PG_VERSION))
 	@echo "Starting PostgreSQL with PGDATABASE=contrib_regression..."
-	PGDATABASE=contrib_regression ./scripts/pg-start.sh --pg-version $(subst pg,,$(PG_VERSION))
+	PGDATABASE=contrib_regression ./scripts/pg-start.sh --build --pg-version $(subst pg,,$(PG_VERSION))
 	@echo "Running pg_regress tests..."
 	PGHOST=$(HOME)/.pgrx PGUSER=postgres PG_CONFIG=$$(cargo pgrx info pg-config $(PG_VERSION)) $(MAKE) -e installcheck
 
@@ -106,7 +111,7 @@ help:
 	@echo "  installcheck  - Run pg_regress tests (requires PostgreSQL running, via PGXS)"
 	@echo "  docker-build  - Build Docker image"
 	@echo "  docker-push   - Build and push to ACR"
-	@echo "  run           - Start local pgrx dev server"
+	@echo "  run           - Start an interactive pgrx dev session (not locked)"
 	@echo "  pg-clean      - Clean build artifacts"
 	@echo "  install       - Install prebuilt artifacts for the selected PostgreSQL"
 	@echo "  uninstall     - Remove installed artifacts for the selected PostgreSQL"

README.md

@@ -308,12 +308,16 @@ GRANT pg_durable_user TO app_backend, etl_service;
 All pull requests must pass the following checks before merging:
 
 1. **Format Check** — `cargo fmt --check`
-2. **Clippy & Tests** — `cargo clippy`, unit tests (`cargo pgrx test pg17`), pg_regress tests, and E2E tests
+2. **Clippy & Tests** — `cargo clippy`, unit tests (`./scripts/test-unit.sh`), pg_regress tests, and E2E tests
 
 The CI workflow is defined in [.github/workflows/ci.yml](.github/workflows/ci.yml). It uses pgrx to download and manage PostgreSQL.
 
 ## Testing
 
+Local test runners require Python 3 and automatically queue across worktrees
+sharing PostgreSQL. See [concurrent local runs](docs/TESTING.md#concurrent-local-runs)
+for queueing behavior, cancellation, and `--keep` limitations.
+
 pg_durable has two test suites:
 
 ### pg_regress Tests (Standard PostgreSQL Regression Tests)

docs/E2E_TESTING.md

@@ -168,6 +168,12 @@ SELECT 'TEST PASSED: my_test_name' AS result;
 
 ## Debugging Failed Tests
 
+`--keep` leaves PostgreSQL running but does not reserve it after the runner exits.
+Another worktree's queued tests may stop or reset it. Use a
+[locked non-interactive command](TESTING.md#concurrent-local-runs) for checks and
+diagnostics that need exclusive access. Coordinate with other worktrees before
+investigating interactively.
+
 ### View PostgreSQL logs
 
 ```bash

docs/TESTING.md

@@ -14,6 +14,61 @@ This guide covers all testing scenarios for pg_durable.
 | Stop PostgreSQL | `./scripts/pg-stop.sh` |
 | Deploy to ACR | `./scripts/deploy-acr.sh` |
 
+## Concurrent local runs
+
+Local test runners require Python 3 (standard library only) and the system `ps`
+command, available on Linux and macOS. They automatically
+take an exclusive file lock at `~/.pgrx/pg_durable-test.lock` before using the
+shared PostgreSQL installation. You can start tests from several worktrees at
+once: competing commands print a waiting message and queue instead of replacing
+each other's extension binaries or resetting each other's databases.
+
+The lock covers installation, tests, and cleanup. Unit, local E2E, upgrade,
+coverage, worker lifecycle, and aggregate runners use it, as do `make test-unit`,
+`make test-regress`, `make pg-install`, and the local start/stop/reset
+scripts. Nested runners share the outer command's lock. All PostgreSQL versions
+share one lock because some diagnostic files are also shared across versions.
+Build/check-only commands and Docker tests do not take this local lock.
+
+This is cooperative locking: every participating worktree must contain the
+updated runners. Direct `cargo pgrx` commands, `make installcheck`, and manual
+database access do not automatically coordinate. Wrap such commands when they
+use the shared local installation:
+
+```bash
+python3 scripts/with-pg-lock.py cargo pgrx test pg17
+```
+
+The wrapper supports **non-interactive commands only**, including `bash -c`.
+Group checks and non-interactive diagnostics into one command to keep the lock
+between them:
+
+```bash
+python3 scripts/with-pg-lock.py bash -e -c \
+  './scripts/test-unit.sh; ./scripts/test-e2e-local.sh'
+```
+
+`--keep` still exits after the tests and leaves PostgreSQL running, but it does
+**not** reserve that server. A queued command may immediately stop or reset it.
+Coordinate with other worktrees before investigating interactively. `make run`
+starts an unprotected interactive session; interactive shells and database
+clients are not supported inside the wrapper.
+
+The supervisor forwards SIGHUP, SIGINT, and SIGTERM to the command's process
+group and holds the lock until every live process in that group exits, including
+cleanup after the command itself exits. Work that ignores cancellation keeps the
+lock until it finishes; there is no automatic forced-kill timeout. Use these
+signals rather than killing only the supervisor with SIGKILL, which bypasses
+cleanup.
+
+PostgreSQL started by `pg_ctl` creates a separate session, so a kept server does
+not retain the lock. Other work that deliberately leaves the supervised process
+group is outside this protection: do not detach tasks that use the shared
+installation. If process monitoring fails, the supervisor reports the error and
+keeps the lock until monitoring recovers and confirms that the work has stopped.
+The empty lock file stays on disk; do not delete it to unblock a waiter, since
+that would let two commands lock different files at the same path.
+
 ## Test Suites Overview
 
 pg_durable has **three test suites**:
@@ -57,7 +112,7 @@ Runs Rust unit tests using `cargo pgrx test`. These test individual functions in
 Standard PostgreSQL regression tests for core DSL functionality. These tests are fast, deterministic, and use PostgreSQL's industry-standard testing framework.
 
 ```bash
-# Recommended: reset the dedicated local cluster, install, and run all tests
+# Recommended: lock the local cluster, reset, rebuild/install, and run all tests
 make test-regress
 
 # Run against an already-running disposable test cluster
@@ -148,6 +203,12 @@ alongside the handwritten E2E suite. Existing CI runner checks and live E2E jobs
 cover all families without any generator or new CI job.
 
 **Investigation mode (`--keep`):**
+
+The lock is released when the runner exits. When sharing PostgreSQL with other
+worktrees, coordinate any interactive investigation before starting another run.
+Non-interactive diagnostics can be included in the
+[same locked command](#concurrent-local-runs) as the tests.
+
 ```bash
 # Run tests, keep server running
 ./scripts/test-e2e-local.sh --keep 04_parallel

scripts/pg-common.sh

@@ -4,6 +4,29 @@
 
 PG_DURABLE_TEST_HTTP_DOMAINS='*.blob.core.windows.net,*.blob.storage.azure.net,*.queue.core.windows.net,*.table.core.windows.net,*.file.core.windows.net,*.azurewebsites.net,*.azurecontainerapps.io,*.azurecontainer.io,*.azure-api.net,*.documents.azure.com,*.servicebus.windows.net,*.openai.azure.com,*.cognitiveservices.azure.com,*.services.ai.azure.com,*.vault.azure.net,*.redis.cache.windows.net,*.database.windows.net,*.kusto.windows.net,*.azurefd.net,*.azureedge.net,*.azure-devices.net,*.trafficmanager.net,*.cloudapp.azure.com,api.github.com,httpbingo.org'
 
+# Save arguments before parsing so lock acquisition can re-exec the same runner.
+PG_DURABLE_LOCK_ARGS=("$@")
+
+lock_local_postgres() {
+    if [ "${_PG_DURABLE_LOCK_EXEC_PID:-}" = "$$" ]; then
+        unset _PG_DURABLE_LOCK_EXEC_PID
+        # Bash defers these traps until the foreground command finishes. Keep
+        # the lock while a build/test child is still using the installation.
+        trap 'exit 129' HUP
+        trap 'exit 130' INT
+        trap 'exit 143' TERM
+        return
+    fi
+
+    # Bash 3.2 treats an empty array as unset under nounset.
+    exec python3 "$(dirname "${BASH_SOURCE[0]}")/with-pg-lock.py" \
+        bash "$1" ${PG_DURABLE_LOCK_ARGS[@]+"${PG_DURABLE_LOCK_ARGS[@]}"}
+}
+
+warn_unreserved_postgres() {
+    echo "Warning: PostgreSQL is not reserved after the outer command exits; a queued run may stop or reset it." >&2
+}
+
 resolve_pgrx_environment() {
     local pg_major="$1"
 

scripts/pg-reset.sh

@@ -15,6 +15,10 @@ PG_MAJOR="${1:-17}"
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+. "$SCRIPT_DIR/pg-common.sh"
+lock_local_postgres "$SCRIPT_DIR/pg-reset.sh"
+
 DATA_DIR="$HOME/.pgrx/data-$PG_MAJOR"
 
 cd "$PROJECT_DIR"
@@ -31,4 +35,3 @@ rm -f "$HOME/.pgrx/${PG_MAJOR}.log"
 echo -e "\033[0;32mPostgreSQL data and logs wiped.\033[0m"
 echo ""
 echo "Run ./scripts/pg-start.sh to start fresh."
-

scripts/pg-start.sh

@@ -12,6 +12,12 @@
 
 set -e
 
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+# shellcheck source=./pg-common.sh
+. "$SCRIPT_DIR/pg-common.sh"
+
 PG_MAJOR="${PG_MAJOR:-17}"
 BUILD_MODE="auto"
 
@@ -52,12 +58,7 @@ while [[ $# -gt 0 ]]; do
     esac
 done
 
-SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
-
-# shellcheck source=./pg-common.sh
-. "$SCRIPT_DIR/pg-common.sh"
-
+lock_local_postgres "$SCRIPT_DIR/pg-start.sh"
 resolve_pgrx_environment "$PG_MAJOR"
 
 cd "$PROJECT_DIR"
@@ -76,7 +77,7 @@ fi
 
 if [ "$BUILD_MODE" != "skip" ]; then
     echo -e "\033[0;33mBuilding and installing extension...\033[0m"
-    cargo pgrx install --pg-config "$PG_CONFIG" 2>&1 | grep -v "^warning:" || true
+    cargo pgrx install --pg-config "$PG_CONFIG"
 fi
 
 echo -e "\033[0;33mPreparing PostgreSQL data directory...\033[0m"
@@ -99,4 +100,4 @@ echo "  tail -f ~/.pgrx/${PG_MAJOR}.log"
 echo ""
 echo -e "\033[0;36mStop:\033[0m"
 echo "  ./scripts/pg-stop.sh"
-
+warn_unreserved_postgres

scripts/pg-stop.sh

@@ -14,6 +14,9 @@
 
 set -e
 
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+. "$SCRIPT_DIR/pg-common.sh"
+
 STOP_LOCAL=false
 STOP_DOCKER=false
 PG_MAJOR="${PG_MAJOR:-17}"
@@ -58,6 +61,8 @@ NC='\033[0m'
 
 # Stop local pgrx PostgreSQL
 if [ "$STOP_LOCAL" = true ]; then
+    lock_local_postgres "$SCRIPT_DIR/pg-stop.sh"
+
     PGRX_HOME="$HOME/.pgrx"
     PG_CTL=$(ls $PGRX_HOME/$PG_MAJOR.*/pgrx-install/bin/pg_ctl 2>/dev/null | head -1)
     DATA_DIR="$PGRX_HOME/data-$PG_MAJOR"
@@ -96,4 +101,3 @@ if [ "$STOP_DOCKER" = true ]; then
         echo "Docker container is not running"
     fi
 fi
-

scripts/test-all-local.sh

@@ -22,6 +22,8 @@ set -e
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
 
+. "$SCRIPT_DIR/pg-common.sh"
+
 cd "$PROJECT_DIR"
 
 # Defaults
@@ -52,6 +54,8 @@ while [[ $# -gt 0 ]]; do
     esac
 done
 
+lock_local_postgres "$SCRIPT_DIR/test-all-local.sh"
+
 # Colors
 RED='\033[0;31m'
 GREEN='\033[0;32m'

scripts/test-coverage.sh

@@ -35,6 +35,8 @@ set -e
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
 
+. "$SCRIPT_DIR/pg-common.sh"
+
 # Defaults
 HTML=false
 WITH_UNIT=false
@@ -70,6 +72,8 @@ while [[ $# -gt 0 ]]; do
     esac
 done
 
+lock_local_postgres "$SCRIPT_DIR/test-coverage.sh"
+
 # Colors
 RED='\033[0;31m'
 GREEN='\033[0;32m'