diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cf965faa5d..e927811c37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -598,24 +598,34 @@ jobs: # altimate_change end # --------------------------------------------------------------------------- - # dbt-tools E2E — slow (~3 min), only on push to main. + # dbt-tools E2E — slow (~4 min): on push to main, and on pull requests that + # touch packages/dbt-tools (the `dbt-tools` change filter above). Without the + # PR path a fix to this job's own environment setup could only be proven + # after merging. # Tests dbt CLI fallbacks against real dbt versions (1.8, 1.10, 1.11) and # real Python environments (venv, uv, system). # --------------------------------------------------------------------------- dbt-tools-e2e: name: "dbt-tools E2E" needs: changes - if: github.event_name == 'push' + if: github.event_name == 'push' || needs.changes.outputs.dbt-tools == 'true' runs-on: ubuntu-latest timeout-minutes: 10 + # Runs pull-request code (setup scripts, tests), so it gets the same + # read-only token scope and no persisted credentials as tracker-leaks. + permissions: + contents: read steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false - uses: oven-sh/setup-bun@ecf28ddc73e819eb6fa29df6b34ef8921c743461 # v2 with: bun-version: "1.3.14" - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + id: python with: python-version: "3.11" @@ -629,21 +639,34 @@ jobs: uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: packages/dbt-tools/test/.dbt-venvs - key: dbt-venvs-${{ runner.os }}-1.8-1.10-1.11 + # Both caches hold venvs linked to the setup-python interpreter, which + # is not cached. Keying on its exact version starts a fresh cache when + # the runner's 3.11 patch release moves, instead of restoring venvs + # that no longer run. + key: dbt-venvs-${{ runner.os }}-py${{ steps.python.outputs.python-version }}-1.8-1.10-1.11 - name: Cache Python env scenarios uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: packages/dbt-tools/test/.dbt-resolve-envs - key: dbt-resolve-envs-${{ runner.os }}-v1 + # v2: the v1 cache held a uv venv linked to a uv-managed Python that + # fresh runners lack (bin/dbt present, exec ENOENT). setup-resolve.sh + # now validates cached envs and pins uv to the system interpreter. + key: dbt-resolve-envs-${{ runner.os }}-py${{ steps.python.outputs.python-version }}-v2 - name: Set up dbt versions run: ./test/e2e/setup-versions.sh 1.8 1.10 1.11 working-directory: packages/dbt-tools + env: + DBT_E2E_PYTHON: ${{ steps.python.outputs.python-path }} - name: Set up Python env scenarios run: ./test/e2e/setup-resolve.sh venv uv system working-directory: packages/dbt-tools + env: + # Build the scenario venvs on the interpreter setup-python installed, + # not the image's /usr/bin/python3; setup-resolve.sh honours this. + DBT_E2E_PYTHON: ${{ steps.python.outputs.python-path }} - name: Run dbt-tools E2E tests run: bun run test:e2e diff --git a/packages/dbt-tools/test/e2e/dbt-versions.test.ts b/packages/dbt-tools/test/e2e/dbt-versions.test.ts index 17ea69f3fe..a90586f872 100644 --- a/packages/dbt-tools/test/e2e/dbt-versions.test.ts +++ b/packages/dbt-tools/test/e2e/dbt-versions.test.ts @@ -12,7 +12,9 @@ * If no dbt is available at all, the entire suite is skipped. * * Environment variables: - * DBT_E2E_VERSIONS — comma-separated list of versions to test (e.g. "1.8,1.9") + * DBT_E2E_VERSIONS — comma-separated list of versions to test (e.g. "1.8,1.9"); + * a listed version without a working dbt of that version + * fails the run * DBT_E2E_SKIP — set to "1" to skip e2e tests entirely */ @@ -29,6 +31,7 @@ import { tmpdir } from "os" const FIXTURE_DIR = resolve(import.meta.dir, "../fixture") const VENVS_DIR = resolve(import.meta.dir, "../.dbt-venvs") const SKIP = process.env.DBT_E2E_SKIP === "1" +const REQUESTED = process.env.DBT_E2E_VERSIONS?.split(",").map((v) => v.trim()) /** Timeout for dbt commands (seed + build can be slow on first run) */ const DBT_TIMEOUT = 120_000 @@ -51,13 +54,12 @@ interface DbtVersion { } function discoverVersions(): DbtVersion[] { - const filterVersions = process.env.DBT_E2E_VERSIONS?.split(",").map((v) => v.trim()) const versions: DbtVersion[] = [] // Check venvs if (existsSync(VENVS_DIR)) { for (const entry of readdirSync(VENVS_DIR)) { - if (filterVersions && !filterVersions.includes(entry)) continue + if (REQUESTED && !REQUESTED.includes(entry)) continue const dbtPath = join(VENVS_DIR, entry, "bin", "dbt") const pythonPath = join(VENVS_DIR, entry, "bin", "python") if (!existsSync(dbtPath)) continue @@ -98,6 +100,16 @@ if (!HAS_DBT && !SKIP) { ) } +// Asking for versions (CI does) makes a missing one a failure, not a skip: a +// stale venv cache once skipped every test below on every run without a sign. +describe.skipIf(SKIP || !REQUESTED)("requested dbt versions", () => { + test("every version in DBT_E2E_VERSIONS has a dbt that runs and reports it", () => { + // Compare what each dbt reports (1.11.0-b3 → 1.11), not its directory name. + const reported = VERSIONS.map((v) => v.full.split(".").slice(0, 2).join(".")) + expect(reported.sort()).toEqual([...(REQUESTED ?? [])].sort()) + }) +}) + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- @@ -205,10 +217,15 @@ describe.skipIf(!HAS_DBT)("altimate-dbt e2e", () => { describe(`dbt ${version.label} (${version.full})`, () => { let workDir: string - beforeAll(() => { + beforeAll(async () => { console.log(`\n→ Setting up dbt ${version.full} project...`) workDir = setupProject(version) console.log(` Project: ${workDir}`) + // dbt-cli.ts caches the dbt it resolves on first use. configure() + // clears that cache and points it at this version's venv; without it + // every version after the first runs the first version's dbt. + const { configure } = await import("../../src/dbt-cli") + configure({ pythonPath: version.pythonPath, projectRoot: workDir }) }, DBT_TIMEOUT * 2) afterAll(() => { diff --git a/packages/dbt-tools/test/e2e/setup-resolve.sh b/packages/dbt-tools/test/e2e/setup-resolve.sh index 111c6eff24..d00c8d288a 100755 --- a/packages/dbt-tools/test/e2e/setup-resolve.sh +++ b/packages/dbt-tools/test/e2e/setup-resolve.sh @@ -33,8 +33,39 @@ ok() { echo " ✓ $1"; } skip() { echo " ⊘ $1 — skipped ($2)"; } fail() { echo " ✗ $1 — $2"; } +# A restored cache can hold an environment whose interpreter no longer exists: +# CI caches only this directory, and a venv's `bin/python` is a symlink (or a +# launcher shebang) into an interpreter outside it. uv in particular links to +# its own managed Python under ~/.local/share/uv, which a fresh runner lacks, +# so `bin/dbt` is present on disk yet fails with ENOENT when executed. The +# `.done` marker alone therefore proves nothing; a cached environment counts +# only if its dbt actually runs. +with_timeout() { + # GNU timeout on Linux; Homebrew coreutils installs it as gtimeout on macOS. + # Without either, the check still runs, just unbounded. + if has timeout; then timeout "$TIMEOUT" "$@" + elif has gtimeout; then gtimeout "$TIMEOUT" "$@" + else "$@"; fi +} +env_ok() { + [ -x "$1" ] && with_timeout "$1" --version >/dev/null 2>&1 +} +# Usage: cached_or_rebuild ; returns 0 when the cached +# environment is usable (caller returns), 1 when it must be rebuilt. +cached_or_rebuild() { + local dir="$ENVS_DIR/$1" + [ -f "$dir/.done" ] || return 1 + if env_ok "$2"; then ok "$1 (cached)"; return 0; fi + echo " ↻ $1 cache is stale (dbt does not run) — rebuilding..." + return 1 +} + # Find a real (non-shim) python3 for venv creation find_real_python() { + # An explicit interpreter wins. CI sets DBT_E2E_PYTHON to the interpreter + # actions/setup-python installed, so the venvs build on the Python the + # workflow chose rather than whatever the runner image happens to ship. + if [ -n "${DBT_E2E_PYTHON:-}" ] && [ -x "$DBT_E2E_PYTHON" ]; then echo "$DBT_E2E_PYTHON"; return; fi # Try pyenv's actual python first if has pyenv; then local p @@ -55,7 +86,7 @@ echo "Using Python: $REAL_PYTHON ($($REAL_PYTHON --version 2>&1))" setup_venv() { local dir="$ENVS_DIR/venv" - if [ -f "$dir/.done" ]; then ok "venv (cached)"; return; fi + cached_or_rebuild venv "$dir/bin/dbt" && return rm -rf "$dir" echo " → Setting up venv..." "$REAL_PYTHON" -m venv "$dir" @@ -68,12 +99,14 @@ setup_venv() { setup_uv() { local dir="$ENVS_DIR/uv" if ! has uv; then skip "uv" "uv not installed"; return; fi - if [ -f "$dir/.done" ]; then ok "uv (cached)"; return; fi + cached_or_rebuild uv "$dir/.venv/bin/dbt" && return rm -rf "$dir" echo " → Setting up uv..." mkdir -p "$dir" - # uv project mode: create .venv in dir - uv venv "$dir/.venv" --quiet + # uv project mode: create .venv in dir. Pin the interpreter to the system + # Python so the venv survives a cache restore on a fresh runner; left to + # itself uv links a managed Python that lives outside the cached directory. + uv venv "$dir/.venv" --python "$REAL_PYTHON" --quiet uv pip install --quiet --python "$dir/.venv/bin/python" "$DBT_SPEC" touch "$dir/.done" ok "uv ($("$dir/.venv/bin/dbt" --version 2>&1 | grep -oE 'installed:\s+\S+' | head -1))" @@ -82,7 +115,7 @@ setup_uv() { setup_pipx() { local dir="$ENVS_DIR/pipx" if ! has pipx; then skip "pipx" "pipx not installed"; return; fi - if [ -f "$dir/.done" ]; then ok "pipx (cached)"; return; fi + cached_or_rebuild pipx "$dir/bin/dbt" && return rm -rf "$dir" echo " → Setting up pipx..." mkdir -p "$dir/bin" "$dir/venvs" @@ -100,7 +133,7 @@ setup_pipx() { setup_conda() { local dir="$ENVS_DIR/conda" if ! has conda; then skip "conda" "conda not installed"; return; fi - if [ -f "$dir/.done" ]; then ok "conda (cached)"; return; fi + cached_or_rebuild conda "$dir/bin/dbt" && return rm -rf "$dir" echo " → Setting up conda..." conda create -y -p "$dir" python=3.11 --quiet 2>/dev/null @@ -117,7 +150,7 @@ setup_conda() { setup_poetry() { local dir="$ENVS_DIR/poetry" if ! has poetry; then skip "poetry" "poetry not installed"; return; fi - if [ -f "$dir/.done" ]; then ok "poetry (cached)"; return; fi + cached_or_rebuild poetry "$dir/.venv/bin/dbt" && return rm -rf "$dir" echo " → Setting up poetry (in-project venv)..." mkdir -p "$dir" @@ -153,7 +186,7 @@ setup_pyenv_venv() { # Simulates a pyenv user who creates a venv with their pyenv-managed python local dir="$ENVS_DIR/pyenv-venv" if ! has pyenv; then skip "pyenv-venv" "pyenv not installed"; return; fi - if [ -f "$dir/.done" ]; then ok "pyenv-venv (cached)"; return; fi + cached_or_rebuild pyenv-venv "$dir/bin/dbt" && return rm -rf "$dir" echo " → Setting up pyenv + venv..." local pyenv_python diff --git a/packages/dbt-tools/test/e2e/setup-versions.sh b/packages/dbt-tools/test/e2e/setup-versions.sh index 34612ed0a2..b069ae7b33 100755 --- a/packages/dbt-tools/test/e2e/setup-versions.sh +++ b/packages/dbt-tools/test/e2e/setup-versions.sh @@ -47,6 +47,19 @@ get_pip_flags() { esac } +# CI passes the interpreter actions/setup-python installed; locally the first +# python3 on PATH is used. +PYTHON="${DBT_E2E_PYTHON:-python3}" + +# Prints e.g. `core=1.8.7`, or nothing when `dbt --version` fails. A restored +# cache can hold a venv whose bin/python links to an interpreter that no longer +# exists, so bin/dbt being on disk proves nothing. +installed_version() { + local out + out=$("$1" --version 2>&1) || return 0 + echo "$out" | grep -oE 'installed: [0-9.a-z]+' | head -1 | sed 's/installed: /core=/' || true +} + mkdir -p "$VENVS_DIR" for ver in "${VERSIONS[@]}"; do @@ -60,13 +73,17 @@ for ver in "${VERSIONS[@]}"; do fi if [ -f "$venv_dir/bin/dbt" ]; then - existing=$("$venv_dir/bin/dbt" --version 2>&1 | grep -oE 'installed: [0-9.a-z]+' | head -1 | sed 's/installed: /core=/' | head -1 || echo "unknown") - echo "✓ dbt $ver already installed ($existing) at $venv_dir" - continue + existing=$(installed_version "$venv_dir/bin/dbt") + if [[ "$existing" == "core=$ver."* ]]; then + echo "✓ dbt $ver already installed ($existing) at $venv_dir" + continue + fi + echo "↻ dbt $ver cache is stale (found ${existing:-no dbt that runs}) — rebuilding..." + rm -rf "$venv_dir" fi echo "→ Installing dbt $ver..." - python3 -m venv "$venv_dir" + "$PYTHON" -m venv "$venv_dir" "$venv_dir/bin/pip" install --quiet --upgrade pip # shellcheck disable=SC2086 "$venv_dir/bin/pip" install --quiet $extra_flags $install_spec