feat(differential): add reproducible glmGamPoi dispersion-trend fits

.github/workflows/ci.yml

@@ -16,6 +16,138 @@ concurrency:
   cancel-in-progress: ${{ github.event_name != 'push' }}
 
 jobs:
+  # Restore the required relevance contexts removed by the baseline sync (#178).
+  # A failed comparison fails its gate; a docs-only diff can skip expensive jobs.
+  gate:
+    name: Code relevance
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    outputs:
+      relevant: ${{ steps.push.outputs.relevant || steps.pr.outputs.relevant }}
+    steps:
+      # A push only starts this workflow on main (the branch filter above), so
+      # the gate has nothing to decide: every push is relevant.
+      - name: A push is already branch-filtered
+        id: push
+        if: github.event_name == 'push'
+        run: echo 'relevant=true' >> "$GITHUB_OUTPUT"
+
+      # Full history, because the three-dot diff names the merge base of the
+      # PR's base tip and its merge commit, and a stacked PR's merge base sits
+      # further back still.
+      - name: Checkout the change
+        if: github.event_name == 'pull_request'
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4.4.0
+        with:
+          persist-credentials: false   # nothing here pushes
+          fetch-depth: 0
+
+      # --no-renames, so a file renamed out of a listed path is reported as a
+      # deletion under its original name and still counts. -z with a
+      # NUL-delimited read, so a pathname Git would quote or that contains a
+      # newline arrives raw and matches the pattern. The base SHA travels
+      # through env, not template interpolation into the script.
+      - name: Does the change touch anything the suite exercises?
+        id: pr
+        if: github.event_name == 'pull_request'
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          relevant=false
+          # Every pattern is quoted, so the glob survives array assignment and
+          # fires at `case` time, against the diff -- an unquoted `src/*` would
+          # expand against the checkout the runner happens to be standing in.
+          patterns=('src/*' 'test/*' 'scripts/*' 'frontend/*' 'config/*'
+                    'bench/*' 'data/*' 'proofs/*' 'R/*' 'renv/*' 'Project.toml'
+                    'Manifest.toml' 'renv.lock' '.Rprofile' 'install.sh' 'install.jl'
+                    'start.sh' 'docs/occupancy/*' 'docs/occupancy-2026-10.adoc'
+                    '.github/workflows/*')
+          # The diff goes to a file first because a process substitution hides
+          # git diff's exit status: an unavailable comparison SHA (a force-push
+          # on the base branch, for instance) or a repository error would read
+          # as "no changes", and the gate would post relevant=false -- skipping
+          # the Julia suite and the proof checker, whose `skipped` conclusion
+          # satisfies a required check -- so a relevant PR would pass silently.
+          # A failed diff must fail the gate itself: that is the one failure
+          # the required-check design cannot afford to skip. As in the
+          # occupancy gate, which this mirrors.
+          changed="$RUNNER_TEMP/ci-changed-files"
+          if ! git diff --no-renames --name-only -z "$BASE_SHA...HEAD" > "$changed"; then
+            echo "ERROR: git diff $BASE_SHA...HEAD failed" >&2
+            exit 1
+          fi
+          while IFS= read -r -d '' file; do
+            for pattern in "${patterns[@]}"; do
+              case "$file" in
+                $pattern) relevant=true; break 2 ;;
+              esac
+            done
+          done < "$changed"
+          echo "relevant=$relevant" >> "$GITHUB_OUTPUT"
+
+  occupancy_gate:
+    name: Occupancy relevance
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    outputs:
+      relevant: ${{ steps.push.outputs.relevant || steps.pr.outputs.relevant }}
+    steps:
+      # All pushes to main are relevant; PRs use the path classification below.
+      - name: A push to main is relevant
+        id: push
+        if: github.event_name == 'push'
+        run: echo 'relevant=true' >> "$GITHUB_OUTPUT"
+
+      # Full history, because the three-dot diff names the merge base of the
+      # PR's base tip and its merge commit -- a shallow checkout does not have
+      # both, and a stacked PR's merge base sits further back still.
+      - name: Checkout the change
+        if: github.event_name == 'pull_request'
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4.4.0
+        with:
+          persist-credentials: false   # nothing here pushes
+          fetch-depth: 0
+
+      - name: Does the change touch the occupancy reading?
+        id: pr
+        if: github.event_name == 'pull_request'
+        env:
+          # zizmor flags template expressions interpolated into run scripts
+          # (template-injection), and the gate script is easier to run locally
+          # with the value in an environment variable in any case. The base SHA
+          # is GitHub-supplied, but the indirection costs nothing.
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          relevant=false
+          patterns=('src/core/memory_probe.jl' 'src/core/provenance.jl' 'src/pipeline/tools.jl'
+                    'src/server/routes/pipeline.jl' 'test/occupancy_disagreement_test.jl'
+                    'test/unit/test_occupancy_grade.jl' 'scripts/check-occupancy-grades.jl'
+                    'scripts/lint-occupancy-readings.sh' 'docs/occupancy/*'
+                    'docs/occupancy-2026-10.adoc' 'scripts/check-occupancy.sh'
+                    'scripts/fetch-siblings.sh' 'scripts/install-ci-julia.sh'
+                    'Project.toml' 'Manifest.toml' 'renv.lock' 'config/defaults/tool_versions.yml'
+                    '.github/workflows/*')
+          # The diff goes to a file first because a process substitution hides
+          # git diff's exit status: an unavailable comparison SHA or a repository
+          # error would read as "no changes", and the gate would post
+          # relevant=false -- skipping the two occupancy jobs, whose `skipped`
+          # conclusion satisfies a required check -- so a relevant PR would
+          # pass silently. A failed diff must fail the gate itself: that is
+          # the one failure the required-check design cannot afford to skip.
+          changed="$RUNNER_TEMP/occupancy-changed-files"
+          if ! git diff --no-renames --name-only -z "$BASE_SHA...HEAD" > "$changed"; then
+            echo "ERROR: git diff $BASE_SHA...HEAD failed" >&2
+            exit 1
+          fi
+          while IFS= read -r -d '' file; do
+            for pattern in "${patterns[@]}"; do
+              case "$file" in
+                $pattern) relevant=true; break 2 ;;
+              esac
+            done
+          done < "$changed"
+          echo "relevant=$relevant" >> "$GITHUB_OUTPUT"
+
   test:
     # The name is a fixed string, it carries no version, and the job has NO matrix.
     # Both are required, and the first without the second is a trap that has already
@@ -40,6 +172,8 @@ jobs:
     # silently. If a matrix is ever genuinely wanted here, the required context must move
     # to a matrix-free aggregator job first.
     name: Julia tests
+    needs: gate
+    if: needs.gate.outputs.relevant == 'true'
     # Explicit, not ubuntu-latest: the R pin below names an apt package revision built
     # for 24.04, and a runner that silently rolled to the next LTS would take that pin
     # with it. `runs-on` cannot read `env`, so this is a literal.
@@ -58,17 +192,16 @@ jobs:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4.4.0
 
       - name: Set up Julia
-        uses: julia-actions/setup-julia@4c0cb0fce8556fdb04a90347310e5db8b1f98fb9  # v2.7.0
-        with:
-          # Pinned rather than tracking latest stable, because a pinned Manifest.toml
-          # resolved by an unpinned Julia is not a reproducible build. This must equal
-          # julia_version in Manifest.toml; it cannot be read from the pin file, since
-          # nothing can be read before Julia exists. test/unit/test_install_pins.jl
-          # fails if the three ever disagree.
-          version: "1.12.5"
+        run: bash scripts/install-ci-julia.sh
 
       - name: Cache Julia packages
-        uses: julia-actions/cache@d10a6fd8f31b12404a54613ebad242900567f2b9  # v2.1.0
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.julia/artifacts
+            ~/.julia/packages
+            ~/.julia/registries
+          key: julia-${{ runner.os }}-${{ hashFiles('Manifest.toml', 'config/defaults/tool_versions.yml') }}
 
       # Every external version CI installs is read from the committed pin file, so CI
       # and a developer's machine cannot drift apart. A temporary environment is used
@@ -156,7 +289,7 @@ jobs:
           sudo chmod 777 "$RENV_PATHS_CACHE"
           sudo env R_CLI_DYNAMIC=false TERM=dumb Rscript --no-init-file -e 'install.packages(c("renv", "BiocManager"), repos="https://cloud.r-project.org", lib=.Library)'
           sudo env RENV_PATHS_CACHE="$RENV_PATHS_CACHE" R_CLI_DYNAMIC=false TERM=dumb Rscript --no-init-file -e 'renv::restore(project=".", library=.Library, prompt=FALSE)'
-          Rscript --no-init-file -e 'for (pkg in c("dada2","Biostrings","ShortRead","vegan","MASS","zCompositions")) if (!require(pkg,character.only=TRUE,quietly=TRUE)) stop(pkg, " failed to install")'
+          Rscript --no-init-file -e 'for (pkg in c("dada2","Biostrings","ShortRead","vegan","MASS","zCompositions","glmGamPoi")) if (!require(pkg,character.only=TRUE,quietly=TRUE)) stop(pkg, " failed to install")'
 
       - name: Check DADA2 CSV assembly
         run: Rscript --vanilla test/unit/test_dada2_tables.R
@@ -250,20 +383,22 @@ jobs:
           julia --project=. -e '
             using RCall
             R"cat(.libPaths(), sep=\"\n\")"
-            for pkg in ["dada2", "vegan", "tibble", "MASS", "zCompositions"]
+            for pkg in ["dada2", "vegan", "tibble", "MASS", "zCompositions", "glmGamPoi"]
               R"if (!require($pkg, character.only=TRUE, quietly=TRUE)) stop($pkg, \" not found\")"
             end
           '
 
       - name: Run tests
         env:
+          OCCUPANCY_RESULTS: ${{ runner.temp }}/occupancy-results.yml
           CI_SKIP_TAXONOMY: "1"
           R_LIBS_SITE: /usr/local/lib/R/site-library:/usr/lib/R/site-library:/usr/lib/R/library
         # A failure also posts annotations, which the checks API serves without the full
         # log. Annotations are capped at 4096 bytes, so each failing assertion gets its own
         # (up to 8), followed by the summary rows that report a failure or an error.
         shell: bash
         run: |
+          command -v vsearch # The full suite must run the occupancy experiment, not skip it.
           set +e
           julia --project=. -t 2 --code-coverage=user --compiled-modules=no test/runtests.jl --integration --server 2>&1 | tee /tmp/julia-tests.log
           status="${PIPESTATUS[0]}"
@@ -292,8 +427,13 @@ jobs:
           fi
           exit "$status"
 
+      - name: Validate measured occupancy grades
+        run: |
+          julia --project=. scripts/check-occupancy-grades.jl "$RUNNER_TEMP/occupancy-results.yml"
+          bash scripts/lint-occupancy-readings.sh
+
       - name: Process coverage
-        uses: julia-actions/julia-processcoverage@03114f09f119417c3242a9fb6e0b722676aedf38  # v1.2.2
+        run: julia scripts/process-ci-coverage.jl
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f  # v6.0.2
@@ -314,6 +454,8 @@ jobs:
   # reads them.
   agda:
     name: Agda proofs
+    needs: gate
+    if: needs.gate.outputs.relevant == 'true'
     runs-on: ubuntu-24.04
     container: debian:trixie-20250929-slim@sha256:1caf1c703c8f7e15dcf2e7769b35000c764e6f50e4d7401c355fb0248f3ddfdb
     timeout-minutes: 30
@@ -346,7 +488,9 @@ jobs:
           agda MetaManifold/All.agda
 
   occupancy:
-    name: Occupancy proof gate
+    name: Occupancy typed peak
+    needs: occupancy_gate
+    if: needs.occupancy_gate.outputs.relevant == 'true'
     runs-on: ubuntu-24.04
     container: debian:trixie-20250929-slim@sha256:1caf1c703c8f7e15dcf2e7769b35000c764e6f50e4d7401c355fb0248f3ddfdb
     timeout-minutes: 30
@@ -356,8 +500,27 @@ jobs:
         run: |
           apt-get update -qq
           apt-get install -y --no-install-recommends ca-certificates curl git agda-bin=2.6.4.3-1+b2 agda-stdlib=2.1-4
-      - uses: julia-actions/setup-julia@4c0cb0fce8556fdb04a90347310e5db8b1f98fb9
-        with:
-          version: "1.12.5"
+      - name: Set up Julia
+        run: bash scripts/install-ci-julia.sh
       - name: Check Session IR and occupancy proofs
         run: bash scripts/check-occupancy.sh
+
+  # The full suite already runs the actual two-thread vsearch experiment. This
+  # required context reports that result, including the fresh grade validation,
+  # instead of paying for a second Julia/R installation and repeating the run.
+  disagreement:
+    name: Occupancy disagreement test
+    needs: [occupancy_gate, test]
+    if: always() && needs.occupancy_gate.result == 'success' && needs.occupancy_gate.outputs.relevant == 'true'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    steps:
+      - name: Require the executed experiment and grade validation to pass
+        env:
+          TEST_RESULT: ${{ needs.test.result }}
+        run: |
+          if [ "$TEST_RESULT" != success ]; then
+            echo "ERROR: full suite, occupancy experiment or measured-grade validation did not pass ($TEST_RESULT)" >&2
+            exit 1
+          fi
+          echo 'The full Julia suite passed, including the two-thread vsearch experiment and fresh empirical grade validation.' >> "$GITHUB_STEP_SUMMARY"

R/_renv_dependencies.R

@@ -6,4 +6,5 @@ if (FALSE) {
     # Retain while installer and CI explicitly verify this package.
     library(tibble)
     library(MASS)
+    library(glmGamPoi)
 }

config/defaults/pipeline.yml

@@ -358,6 +358,14 @@ analysis:
     # rle is refused when no taxon is present in every sample.
     # Used by nb_glm only; clr_welch ignores it, since log-ratios carry no library size.
     offset: tss
+    # nb_glm only: where each taxon's negative-binomial theta comes from. ml
+    # (the default) takes the maximum-likelihood estimate MASS::glm.nb fits per
+    # taxon. glmGamPoi takes the shrunken dispersion glmGamPoi::glm_gp fits
+    # across all taxa in one call -- sharing information across taxa, which
+    # steadies low-count taxa -- and fits each taxon through
+    # MASS::negative.binomial(theta = 1/dispersion), so the Wald test is
+    # unchanged. Requires the glmGamPoi R package.
+    dispersion: ml
     # Fraction of samples, 0 to 1, in which a taxon must have reads to be
     # tested. Taxa below it are listed as filtered, without a p-value. For
     # clr_welch it is applied before zero replacement, so on a sparse table it also

frontend/src/api/types.ts

@@ -439,6 +439,8 @@ export interface DifferentialResult {
     min_prevalence: number
     /** nb_glm only. */
     offset?: 'tss' | 'rle'
+    /** nb_glm only: where each taxon's theta comes from. */
+    dispersion?: 'ml' | 'glmGamPoi'
     /** clr_welch only. */
     replacement_delta?: number
   }

frontend/src/components/PipelineStages.test.ts

@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+import { expect, test } from 'bun:test'
+import { CONFIG_TYPES } from './PipelineStages'
+
+test('dispersion uses the enum editor with both supported models', () => {
+  expect(CONFIG_TYPES['analysis.differential.dispersion']).toEqual({
+    kind: 'enum', options: ['ml', 'glmGamPoi'],
+  })
+})

frontend/src/components/PipelineStages.tsx

@@ -152,6 +152,7 @@ export const CONFIG_DESCRIPTIONS: Record<string, string> = {
   'analysis.beta.depth':          'Reads per sample for rarefy or srs; 0 uses the smallest non-empty sample.',
   'analysis.differential.method': 'Model fitted to each taxon: nb_glm (negative-binomial GLM on read counts; effect is a log2 fold change) or clr_welch (Welch\'s t-test on centred log-ratios over the taxa passing min_prevalence, after zCompositions::cmultRepl zero replacement; effect is a CLR difference, not a fold change).',
   'analysis.differential.offset': 'Library-size offset in each negative-binomial model: tss (total reads per sample) or rle (median-of-ratios over taxa present in every sample; refused when there are none). Used by nb_glm only.',
+  'analysis.differential.dispersion': 'nb_glm only: ml estimates theta separately with MASS::glm.nb; glmGamPoi uses a cross-taxon dispersion trend in fixed-scale negative-binomial Wald fits.',
   'analysis.differential.min_prevalence': 'Fraction of samples, 0 to 1, in which a taxon must have reads to be tested; taxa below it are listed as filtered.',
   'analysis.differential.replacement_delta': 'clr_welch only: cmultRepl\'s frac, between 0 and 1; an imputed proportion above its taxon\'s smallest observed proportion is lowered to this fraction of it. Replaced zeros are not measurements.',
   'analysis.alpha.show_points':   'Show individual samples overlaid on alpha comparison boxplots.',
@@ -252,6 +253,7 @@ export const CONFIG_TYPES: Record<string, ConfigType> = {
   'analysis.beta.depth':          { kind: 'int' },
   'analysis.differential.method': { kind: 'enum', options: ['nb_glm', 'clr_welch'] },
   'analysis.differential.offset': { kind: 'enum', options: ['tss', 'rle'] },
+  'analysis.differential.dispersion': { kind: 'enum', options: ['ml', 'glmGamPoi'] },
   'analysis.differential.min_prevalence': { kind: 'float', min: 0, max: 1, step: 0.05 },
   'analysis.differential.replacement_delta': { kind: 'float', min: 0.01, max: 0.99, step: 0.05 },
   'analysis.alpha.show_points':   { kind: 'boolean' },