fix(ci): validate cached dbt e2e environments; pin the uv venv to the system Python

.github/workflows/ci.yml

@@ -598,24 +598,34 @@ jobs:
   # altimate_change end
 
   # ---------------------------------------------------------------------------
-  # dbt-tools E2E — slow (~3 min), only on push to main.
+  # dbt-tools E2E — slow (~4 min): on push to main, and on pull requests that
+  # touch packages/dbt-tools (the `dbt-tools` change filter above). Without the
+  # PR path a fix to this job's own environment setup could only be proven
+  # after merging.
   # Tests dbt CLI fallbacks against real dbt versions (1.8, 1.10, 1.11) and
   # real Python environments (venv, uv, system).
   # ---------------------------------------------------------------------------
   dbt-tools-e2e:
     name: "dbt-tools E2E"
     needs: changes
-    if: github.event_name == 'push'
+    if: github.event_name == 'push' || needs.changes.outputs.dbt-tools == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 10
+    # Runs pull-request code (setup scripts, tests), so it gets the same
+    # read-only token scope and no persisted credentials as tracker-leaks.
+    permissions:
+      contents: read
     steps:
       - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          persist-credentials: false
 
       - uses: oven-sh/setup-bun@ecf28ddc73e819eb6fa29df6b34ef8921c743461 # v2
         with:
           bun-version: "1.3.14"
 
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        id: python
         with:
           python-version: "3.11"
 
@@ -629,21 +639,34 @@ jobs:
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: packages/dbt-tools/test/.dbt-venvs
-          key: dbt-venvs-${{ runner.os }}-1.8-1.10-1.11
+          # Both caches hold venvs linked to the setup-python interpreter, which
+          # is not cached. Keying on its exact version starts a fresh cache when
+          # the runner's 3.11 patch release moves, instead of restoring venvs
+          # that no longer run.
+          key: dbt-venvs-${{ runner.os }}-py${{ steps.python.outputs.python-version }}-1.8-1.10-1.11
 
       - name: Cache Python env scenarios
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: packages/dbt-tools/test/.dbt-resolve-envs
-          key: dbt-resolve-envs-${{ runner.os }}-v1
+          # v2: the v1 cache held a uv venv linked to a uv-managed Python that
+          # fresh runners lack (bin/dbt present, exec ENOENT). setup-resolve.sh
+          # now validates cached envs and pins uv to the system interpreter.
+          key: dbt-resolve-envs-${{ runner.os }}-py${{ steps.python.outputs.python-version }}-v2
 
       - name: Set up dbt versions
         run: ./test/e2e/setup-versions.sh 1.8 1.10 1.11
         working-directory: packages/dbt-tools
+        env:
+          DBT_E2E_PYTHON: ${{ steps.python.outputs.python-path }}
 
       - name: Set up Python env scenarios
         run: ./test/e2e/setup-resolve.sh venv uv system
         working-directory: packages/dbt-tools
+        env:
+          # Build the scenario venvs on the interpreter setup-python installed,
+          # not the image's /usr/bin/python3; setup-resolve.sh honours this.
+          DBT_E2E_PYTHON: ${{ steps.python.outputs.python-path }}
 
       - name: Run dbt-tools E2E tests
         run: bun run test:e2e

packages/dbt-tools/test/e2e/dbt-versions.test.ts

@@ -12,7 +12,9 @@
  * If no dbt is available at all, the entire suite is skipped.
  *
  * Environment variables:
- *   DBT_E2E_VERSIONS  — comma-separated list of versions to test (e.g. "1.8,1.9")
+ *   DBT_E2E_VERSIONS  — comma-separated list of versions to test (e.g. "1.8,1.9");
+ *                       a listed version without a working dbt of that version
+ *                       fails the run
  *   DBT_E2E_SKIP      — set to "1" to skip e2e tests entirely
  */
 
@@ -29,6 +31,7 @@ import { tmpdir } from "os"
 const FIXTURE_DIR = resolve(import.meta.dir, "../fixture")
 const VENVS_DIR = resolve(import.meta.dir, "../.dbt-venvs")
 const SKIP = process.env.DBT_E2E_SKIP === "1"
+const REQUESTED = process.env.DBT_E2E_VERSIONS?.split(",").map((v) => v.trim())
 
 /** Timeout for dbt commands (seed + build can be slow on first run) */
 const DBT_TIMEOUT = 120_000
@@ -51,13 +54,12 @@ interface DbtVersion {
 }
 
 function discoverVersions(): DbtVersion[] {
-  const filterVersions = process.env.DBT_E2E_VERSIONS?.split(",").map((v) => v.trim())
   const versions: DbtVersion[] = []
 
   // Check venvs
   if (existsSync(VENVS_DIR)) {
     for (const entry of readdirSync(VENVS_DIR)) {
-      if (filterVersions && !filterVersions.includes(entry)) continue
+      if (REQUESTED && !REQUESTED.includes(entry)) continue
       const dbtPath = join(VENVS_DIR, entry, "bin", "dbt")
       const pythonPath = join(VENVS_DIR, entry, "bin", "python")
       if (!existsSync(dbtPath)) continue
@@ -98,6 +100,16 @@ if (!HAS_DBT && !SKIP) {
   )
 }
 
+// Asking for versions (CI does) makes a missing one a failure, not a skip: a
+// stale venv cache once skipped every test below on every run without a sign.
+describe.skipIf(SKIP || !REQUESTED)("requested dbt versions", () => {
+  test("every version in DBT_E2E_VERSIONS has a dbt that runs and reports it", () => {
+    // Compare what each dbt reports (1.11.0-b3 → 1.11), not its directory name.
+    const reported = VERSIONS.map((v) => v.full.split(".").slice(0, 2).join("."))
+    expect(reported.sort()).toEqual([...(REQUESTED ?? [])].sort())
+  })
+})
+
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------
@@ -205,10 +217,15 @@ describe.skipIf(!HAS_DBT)("altimate-dbt e2e", () => {
     describe(`dbt ${version.label} (${version.full})`, () => {
       let workDir: string
 
-      beforeAll(() => {
+      beforeAll(async () => {
         console.log(`\n→ Setting up dbt ${version.full} project...`)
         workDir = setupProject(version)
         console.log(`  Project: ${workDir}`)
+        // dbt-cli.ts caches the dbt it resolves on first use. configure()
+        // clears that cache and points it at this version's venv; without it
+        // every version after the first runs the first version's dbt.
+        const { configure } = await import("../../src/dbt-cli")
+        configure({ pythonPath: version.pythonPath, projectRoot: workDir })
       }, DBT_TIMEOUT * 2)
 
       afterAll(() => {

packages/dbt-tools/test/e2e/setup-resolve.sh

@@ -33,8 +33,39 @@ ok() { echo "  ✓ $1"; }
 skip() { echo "  ⊘ $1 — skipped ($2)"; }
 fail() { echo "  ✗ $1 — $2"; }
 
+# A restored cache can hold an environment whose interpreter no longer exists:
+# CI caches only this directory, and a venv's `bin/python` is a symlink (or a
+# launcher shebang) into an interpreter outside it. uv in particular links to
+# its own managed Python under ~/.local/share/uv, which a fresh runner lacks,
+# so `bin/dbt` is present on disk yet fails with ENOENT when executed. The
+# `.done` marker alone therefore proves nothing; a cached environment counts
+# only if its dbt actually runs.
+with_timeout() {
+  # GNU timeout on Linux; Homebrew coreutils installs it as gtimeout on macOS.
+  # Without either, the check still runs, just unbounded.
+  if has timeout; then timeout "$TIMEOUT" "$@"
+  elif has gtimeout; then gtimeout "$TIMEOUT" "$@"
+  else "$@"; fi
+}
+env_ok() {
+  [ -x "$1" ] && with_timeout "$1" --version >/dev/null 2>&1
+}
+# Usage: cached_or_rebuild <scenario> <dbt binary>; returns 0 when the cached
+# environment is usable (caller returns), 1 when it must be rebuilt.
+cached_or_rebuild() {
+  local dir="$ENVS_DIR/$1"
+  [ -f "$dir/.done" ] || return 1
+  if env_ok "$2"; then ok "$1 (cached)"; return 0; fi
+  echo "  ↻ $1 cache is stale (dbt does not run) — rebuilding..."
+  return 1
+}
+
 # Find a real (non-shim) python3 for venv creation
 find_real_python() {
+  # An explicit interpreter wins. CI sets DBT_E2E_PYTHON to the interpreter
+  # actions/setup-python installed, so the venvs build on the Python the
+  # workflow chose rather than whatever the runner image happens to ship.
+  if [ -n "${DBT_E2E_PYTHON:-}" ] && [ -x "$DBT_E2E_PYTHON" ]; then echo "$DBT_E2E_PYTHON"; return; fi
   # Try pyenv's actual python first
   if has pyenv; then
     local p
@@ -55,7 +86,7 @@ echo "Using Python: $REAL_PYTHON ($($REAL_PYTHON --version 2>&1))"
 
 setup_venv() {
   local dir="$ENVS_DIR/venv"
-  if [ -f "$dir/.done" ]; then ok "venv (cached)"; return; fi
+  cached_or_rebuild venv "$dir/bin/dbt" && return
   rm -rf "$dir"
   echo "  → Setting up venv..."
   "$REAL_PYTHON" -m venv "$dir"
@@ -68,12 +99,14 @@ setup_venv() {
 setup_uv() {
   local dir="$ENVS_DIR/uv"
   if ! has uv; then skip "uv" "uv not installed"; return; fi
-  if [ -f "$dir/.done" ]; then ok "uv (cached)"; return; fi
+  cached_or_rebuild uv "$dir/.venv/bin/dbt" && return
   rm -rf "$dir"
   echo "  → Setting up uv..."
   mkdir -p "$dir"
-  # uv project mode: create .venv in dir
-  uv venv "$dir/.venv" --quiet
+  # uv project mode: create .venv in dir. Pin the interpreter to the system
+  # Python so the venv survives a cache restore on a fresh runner; left to
+  # itself uv links a managed Python that lives outside the cached directory.
+  uv venv "$dir/.venv" --python "$REAL_PYTHON" --quiet
   uv pip install --quiet --python "$dir/.venv/bin/python" "$DBT_SPEC"
   touch "$dir/.done"
   ok "uv ($("$dir/.venv/bin/dbt" --version 2>&1 | grep -oE 'installed:\s+\S+' | head -1))"
@@ -82,7 +115,7 @@ setup_uv() {
 setup_pipx() {
   local dir="$ENVS_DIR/pipx"
   if ! has pipx; then skip "pipx" "pipx not installed"; return; fi
-  if [ -f "$dir/.done" ]; then ok "pipx (cached)"; return; fi
+  cached_or_rebuild pipx "$dir/bin/dbt" && return
   rm -rf "$dir"
   echo "  → Setting up pipx..."
   mkdir -p "$dir/bin" "$dir/venvs"
@@ -100,7 +133,7 @@ setup_pipx() {
 setup_conda() {
   local dir="$ENVS_DIR/conda"
   if ! has conda; then skip "conda" "conda not installed"; return; fi
-  if [ -f "$dir/.done" ]; then ok "conda (cached)"; return; fi
+  cached_or_rebuild conda "$dir/bin/dbt" && return
   rm -rf "$dir"
   echo "  → Setting up conda..."
   conda create -y -p "$dir" python=3.11 --quiet 2>/dev/null
@@ -117,7 +150,7 @@ setup_conda() {
 setup_poetry() {
   local dir="$ENVS_DIR/poetry"
   if ! has poetry; then skip "poetry" "poetry not installed"; return; fi
-  if [ -f "$dir/.done" ]; then ok "poetry (cached)"; return; fi
+  cached_or_rebuild poetry "$dir/.venv/bin/dbt" && return
   rm -rf "$dir"
   echo "  → Setting up poetry (in-project venv)..."
   mkdir -p "$dir"
@@ -153,7 +186,7 @@ setup_pyenv_venv() {
   # Simulates a pyenv user who creates a venv with their pyenv-managed python
   local dir="$ENVS_DIR/pyenv-venv"
   if ! has pyenv; then skip "pyenv-venv" "pyenv not installed"; return; fi
-  if [ -f "$dir/.done" ]; then ok "pyenv-venv (cached)"; return; fi
+  cached_or_rebuild pyenv-venv "$dir/bin/dbt" && return
   rm -rf "$dir"
   echo "  → Setting up pyenv + venv..."
   local pyenv_python

packages/dbt-tools/test/e2e/setup-versions.sh

@@ -47,6 +47,19 @@ get_pip_flags() {
   esac
 }
 
+# CI passes the interpreter actions/setup-python installed; locally the first
+# python3 on PATH is used.
+PYTHON="${DBT_E2E_PYTHON:-python3}"
+
+# Prints e.g. `core=1.8.7`, or nothing when `dbt --version` fails. A restored
+# cache can hold a venv whose bin/python links to an interpreter that no longer
+# exists, so bin/dbt being on disk proves nothing.
+installed_version() {
+  local out
+  out=$("$1" --version 2>&1) || return 0
+  echo "$out" | grep -oE 'installed: [0-9.a-z]+' | head -1 | sed 's/installed: /core=/' || true
+}
+
 mkdir -p "$VENVS_DIR"
 
 for ver in "${VERSIONS[@]}"; do
@@ -60,13 +73,17 @@ for ver in "${VERSIONS[@]}"; do
   fi
 
   if [ -f "$venv_dir/bin/dbt" ]; then
-    existing=$("$venv_dir/bin/dbt" --version 2>&1 | grep -oE 'installed: [0-9.a-z]+' | head -1 | sed 's/installed: /core=/' | head -1 || echo "unknown")
-    echo "✓ dbt $ver already installed ($existing) at $venv_dir"
-    continue
+    existing=$(installed_version "$venv_dir/bin/dbt")
+    if [[ "$existing" == "core=$ver."* ]]; then
+      echo "✓ dbt $ver already installed ($existing) at $venv_dir"
+      continue
+    fi
+    echo "↻ dbt $ver cache is stale (found ${existing:-no dbt that runs}) — rebuilding..."
+    rm -rf "$venv_dir"
   fi
 
   echo "→ Installing dbt $ver..."
-  python3 -m venv "$venv_dir"
+  "$PYTHON" -m venv "$venv_dir"
   "$venv_dir/bin/pip" install --quiet --upgrade pip
   # shellcheck disable=SC2086
   "$venv_dir/bin/pip" install --quiet $extra_flags $install_spec