Add tomographic and 3x2pt covariance realspace measurement

papers/bmodes/config/config.yaml

@@ -144,4 +144,4 @@ glass_mocks:
 
 # External tools
 tools:
-  cosmocov_executable: "/n23data1/n06data/lgoh/scratch/UNIONS/CosmoCov/covs/cov"
+  onecov_executable: "/home/guerrini/OneCovariance/covariance.py"

papers/cosmo_val/config/config.yaml

@@ -149,4 +149,4 @@ glass_mocks:
   seed_range: [1, 350]
 
 tools:
-  cosmocov_executable: "/n23data1/n06data/lgoh/scratch/UNIONS/CosmoCov/covs/cov"
+  onecov_executable: "/home/guerrini/OneCovariance/covariance.py"

scratch/lgoh/cov_config_test.yaml

@@ -0,0 +1,27 @@
+fiducial:
+  version: SP_v1.4.6
+  blind: A
+  min_sep: 1.0
+  max_sep: 100.0
+  nbins: 20
+  npatch: 30
+  mock_version: SP_v1.4.6
+  gaussian: g
+
+probe: wl
+glass_mocks:
+  version: "v0"
+  seed_range: [1, 350]
+
+covariance:
+  default_masked: True
+
+tools:
+  python_executable: /n23data1/n06data/lgoh/scratch/my_env_new/bin/python
+  onecov_executable: /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/covariance.py
+
+SP_v1.4.6:
+  cov_th:
+    sigma_e: 0.28, 0.28
+    n_e: 5.0, 5.0
+    A: 1000

src/sp_validation/tests/regression/test_tomo_covariance_rules_call_external_checkout.py

@@ -1,4 +1,4 @@
-"""Exercise real covariance rules without catalogues or CosmoCov.
+"""Exercise real covariance rules without catalogues or OneCovariance.
 
 These tests pass on develop: it uses checkout-local scripts and launch-relative
 outputs. They guard against the external paths introduced on the tomography
@@ -13,7 +13,6 @@
 from pathlib import Path
 from types import SimpleNamespace
 
-import numpy as np
 import pytest
 
 import sp_validation
@@ -82,7 +81,6 @@ def covariance_rules(tmp_path):
     snakefile.write_text(
         f"""
 from pathlib import Path
-from snakemake.io import apply_wildcards
 import json
 COSMO_INFERENCE = Path({str(tmp_path / "cosmo_inference")!r})
 COSMO_VAL = Path({str(tmp_path / "cosmo_val")!r})
@@ -92,14 +90,12 @@ def covariance_rules(tmp_path):
 PLANCK18 = dict.fromkeys(
     ["Omega_m", "Omega_v", "sigma_8", "n_s", "h", "Omega_b"], 0.0
 )
-BLOCK_PAIRS = []
 DEFAULT_MASK_SUFFIX = ""
 FIDUCIAL = dict.fromkeys([
     "mock_version", "version", "blind", "min_sep", "max_sep", "nbins",
     "min_sep_int", "max_sep_int", "nbins_int",
 ], "synthetic")
 config = {{
-    "tools": {{"cosmocov_executable": "unused"}},
     "glass_mocks": {{"seed_range": [1, 1]}},
 }}
 def fiducial_binning_suffix():
@@ -108,6 +104,11 @@ def covariance_path(*args, **kwargs):
     return "unused-fiducial-input"
 def covariance_dir(*args, **kwargs):
     return "unused-covariance-directory"
+def covariance_base(*args, **kwargs):
+    return "unused-covariance-base"
+def cov_output(suffix):
+    return str(COSMO_INFERENCE / ("covariance_{{version}}_{{gaussian}}_minsep={{min_sep}}"
+        "_maxsep={{max_sep}}_nbins={{nbins}}{{mask_suffix}}{{tomo_suffix}}" + suffix))
 def redshift_path(*args, **kwargs):
     return "unused-redshift-input"
 def build_redshift_path(*args, **kwargs):
@@ -117,78 +118,28 @@ def cv_cov_tau(*args, **kwargs):
 def cv_tau_stats(*args, **kwargs):
     return "unused-tau-statistics"
 wildcard_constraints:
-    mask_suffix="(?:_masked)?"
+    mask_suffix="(?:_masked)?",
+    tomo_suffix="(?:_tomo)?"
 include: {str(source)!r}
 manifest = {{}}
 for name in [
-    "covariance_process", "generate_glass_mock_rhotau_samples",
-    "covariance_glass_mock",
+    "generate_glass_mock_rhotau_samples", "covariance_glass_mock",
 ]:
     registered = workflow.get_rule(name)
     manifest[name] = {{
         "script": registered.script, "shellcmd": registered.shellcmd,
         "basedir": str(registered.basedir),
         "output": list(map(str, registered.output)),
     }}
-synthetic_wildcards = dict(
-    version="synthetic", blind="A", gaussian="ng", min_sep="1", max_sep="2",
-    nbins="1", mask_suffix="",
-)
-manifest["covariance_process"]["synthetic_matrix"] = apply_wildcards(
-    workflow.get_rule("covariance_process").output[0], synthetic_wildcards
-)
 Path("rules.json").write_text(json.dumps(manifest, indent=2))
 """
     )
-    # Upper-triangle dump: columns 8 and 9 contain Gaussian and non-Gaussian
-    # terms. Their sum is the positive-definite matrix [[3, .25], [.25, 6]].
-    raw = np.array(
-        [
-            [0, 0, 0, 0, 0, 0, 0, 0, 2, 1],
-            [0, 1, 0, 0, 0, 0, 0, 0, 0.25, 0],
-            [1, 1, 0, 0, 0, 0, 0, 0, 4, 2],
-        ]
-    )
     parsed = _run_snakemake(case, "--list-rules")
     assert parsed.returncode == 0, parsed.stdout + parsed.stderr
     case.manifest = json.loads((tmp_path / "rules.json").read_text())
-    case.matrix = Path(case.manifest["covariance_process"]["synthetic_matrix"])
-    raw_path = Path(str(case.matrix).replace("_processed.txt", ".txt"))
-    raw_path.parent.mkdir(parents=True)
-    np.savetxt(raw_path, raw)
     return case
 
 
-@pytest.mark.slow
-def test_covariance_postprocessing_runs_from_launched_checkout(covariance_rules):
-    """Process a synthetic CosmoCov dump without relying on another checkout.
-
-    The three input rows give G+NG = [[3, .25], [.25, 6]], reflecting the supplied
-    off-diagonal element, and G = [[2, .25], [.25, 4]]. Snakemake executes the
-    actual rule with existing raw input, so no survey data or expensive
-    covariance calculation is needed. A missing external entry point is a
-    workflow defect. Develop passes because its processor is checkout-local;
-    this guards against the tomography merge reintroducing an external script.
-    """
-    case = covariance_rules
-    result = _run_snakemake(case, "--printshellcmds", str(case.matrix))
-    log = result.stdout + result.stderr
-    diagnosis = next(
-        (line for line in log.splitlines() if "can't open file" in line), log[-2500:]
-    )
-    assert result.returncode == 0, (
-        f"covariance_process returned {result.returncode}; expected exit 0 and "
-        f"matrix [[3.0, 0.25], [0.25, 6.0]]; {diagnosis}"
-    )
-    np.testing.assert_allclose(
-        np.loadtxt(case.matrix), [[3, 0.25], [0.25, 6]], rtol=0, atol=0
-    )
-    gaussian = Path(str(case.matrix).replace("_processed.txt", "_processed_g.txt"))
-    np.testing.assert_allclose(
-        np.loadtxt(gaussian), [[2, 0.25], [0.25, 4]], rtol=0, atol=0
-    )
-
-
 def test_tau_sampling_selects_script_in_launched_checkout(covariance_rules):
     """The real tau-sampling rule must select code from the launched checkout.
 

workflow/README.md

@@ -87,7 +87,7 @@ itself — no rule writes its own `apptainer exec` call. The image name comes
 from the `container:` directive in `workflow/Snakefile` (or a rule's own
 override, e.g. the image-sims `SIF`).
 
-A few rules shell out to a host toolchain (CosmoCov, ImageMagick) and keep
+A few rules shell out to a host toolchain (OneCovariance, ImageMagick) and keep
 `container: None`; each says why in its own docstring.
 
 The slurm executor submits with `--export=ALL`, so every job starts with the
@@ -146,10 +146,10 @@ halves of one commit, split.
 The catalogue config is the launched checkout's `cosmo_val/cat_config.yaml`.
 `COSMO_VAL` defaults to the launched checkout's `cosmo_val/output`, so writing
 into another checkout's products means naming it; `COSMO_INFERENCE` defaults to
-the shared candide tree, which holds the CosmoCov covariances and only its owner
+the shared candide tree, which holds the covariances and only its owner
 can write. Anyone else launches with `COSMO_INFERENCE=<tree>` of their own,
 holding a link to the shared tree's `data/mask/` (the one input the covariance
-rules take from it); the CosmoCov chain then runs there:
+rules take from it); the OneCovariance rules then run there:
 
 ```bash
 mkdir -p <tree>/data

workflow/common.py

@@ -84,7 +84,6 @@ def _load_checkout_module(name):
 # The catalogue config of the launched checkout: the one file both the host
 # (CATALOG_CONFIG, loaded in configure) and every job read catalogues from.
 CAT_CONFIG = str(REPO_ROOT / "cosmo_val" / "cat_config.yaml")
-BLOCK_PAIRS = [("++", "1"), ("--", "2"), ("+-", "3")]
 
 # Fiducial cosmology: Planck 2018 (astropy Planck18, Table 2 + BAO)
 # Source of truth: cs_util.cosmo.PLANCK18
@@ -98,12 +97,16 @@ def _load_checkout_module(name):
 WILDCARD_CONSTRAINTS = {
     "version": r"SP_v[\d.]+(_[ABC])?(_w_iv)?(_ecut\d+)?(_leak_corr)?",
     "nbins": r"\d+",
+    # Without this, npatch greedily swallows a trailing {tomo_suffix}.
+    "npatch": r"\d+",
     "min_sep": r"[0-9.]+",
     "max_sep": r"[0-9.]+",
     "gaussian": r"(g|ng)",
     "block_pm": r"(\+\+|--|\+-)",
     "block_i": r"[123]",
     "mask_suffix": r"(_masked)?",
+    # Empty means non-tomographic, so existing paths are unchanged.
+    "tomo_suffix": r"(_tomo)?",
     "mock_id": r"\d{5}",
     "nside": r"\d+",
 }
@@ -226,39 +229,70 @@ def fiducial_binning_suffix(fiducial=None):
     )
 
 
+# Covariance naming, one scheme for every OneCovariance product. Rule outputs
+# use the template with wildcards left in braces; covariance_base() formats it,
+# so the two stay in sync by construction. An empty {tomo_suffix} is the
+# non-tomographic covariance.
+COV_BASE_TEMPLATE = (
+    "covariance_{version}_{gaussian}"
+    "_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}"
+    "{mask_suffix}{tomo_suffix}"
+)
+
+
+def cov_output(suffix):
+    """Wildcard-bearing covariance path: ``<BASE>/<BASE>{suffix}``."""
+    return str(
+        COSMO_INFERENCE
+        / f"data/covariance/{COV_BASE_TEMPLATE}/{COV_BASE_TEMPLATE}{suffix}"
+    )
+
+
 def covariance_base(
     version,
     gaussian="ng",
     min_sep=None,
     max_sep=None,
     nbins=None,
     mask_suffix=None,
+    tomo_suffix=None,
     fiducial=None,
     default_mask_suffix=None,
 ):
-    """Construct covariance base name."""
+    """Construct covariance base name (concrete values, no wildcards)."""
     fiducial = fiducial or FIDUCIAL
+    gaussian = gaussian if gaussian is not None else fiducial["gaussian"]
     min_sep = min_sep if min_sep is not None else fiducial["min_sep"]
     max_sep = max_sep if max_sep is not None else fiducial["max_sep"]
     nbins = nbins if nbins is not None else fiducial["nbins"]
-    mask_suffix = (
-        mask_suffix
-        if mask_suffix is not None
-        else (
+    if mask_suffix is None:
+        mask_suffix = (
             DEFAULT_MASK_SUFFIX if default_mask_suffix is None else default_mask_suffix
         )
-    )
-    return (
-        f"covariance_{version}_{gaussian}"
-        f"_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}"
+    return COV_BASE_TEMPLATE.format(
+        version=version,
+        gaussian=gaussian,
+        min_sep=min_sep,
+        max_sep=max_sep,
+        nbins=nbins,
+        mask_suffix=mask_suffix,
+        tomo_suffix=tomo_suffix or "",
     )
 
 
 def covariance_dir(
-    version, gaussian="ng", min_sep=None, max_sep=None, nbins=None, mask_suffix=None
+    version,
+    gaussian="ng",
+    min_sep=None,
+    max_sep=None,
+    nbins=None,
+    mask_suffix=None,
+    tomo_suffix=None,
 ):
     """Construct covariance directory path."""
-    base = covariance_base(version, gaussian, min_sep, max_sep, nbins, mask_suffix)
+    base = covariance_base(
+        version, gaussian, min_sep, max_sep, nbins, mask_suffix, tomo_suffix
+    )
     return str(COSMO_INFERENCE / f"data/covariance/{base}")
 
 
@@ -269,10 +303,13 @@ def covariance_path(
     max_sep=None,
     nbins=None,
     mask_suffix=None,
-    suffix="_processed.txt",
+    tomo_suffix=None,
+    suffix=".dat",
 ):
-    """Construct covariance file path."""
-    base = covariance_base(version, gaussian, min_sep, max_sep, nbins, mask_suffix)
+    """Construct covariance file path; by default the OneCovariance matrix."""
+    base = covariance_base(
+        version, gaussian, min_sep, max_sep, nbins, mask_suffix, tomo_suffix
+    )
     return str(COSMO_INFERENCE / f"data/covariance/{base}/{base}{suffix}")
 
 

workflow/rules/cosmo_val.smk

@@ -10,15 +10,15 @@
 #                   ├─ integration part ─┬──→ pure_eb (part, npz, figures)
 #                   │                    └──→ cosebis (part, npz, figures)
 #                   └─ reporting part ──→ 2pcf plot, ratio_xi_sys_xi
-#   CosmoCov ξ±, integration grid ──→ pure_eb, cosebis (their covariances)
+#   OneCov ξ±, integration grid ──→ pure_eb, cosebis (their covariances)
 #   catalogue ──→ pseudo_cl (part) ──┬─→ pseudo-Cl figures
-#   CosmoCov ξ±, reporting grid ─────┤
+#   OneCov ξ±, reporting grid ───────┤
 #   rho/tau (part + FITS) ───────────┼─→ summarize_bmodes (reads the products)
 #                                    └─→ assemble_sacc ──→ {version}.sacc
 #
 # The B-mode rules and the ξ± figures are ingests: they work from the parts and
 # the covariance inputs, never from a catalogue, so a blinded part keeps
-# everything downstream blinded. The analytic covariances (CosmoCov ξ±,
+# everything downstream blinded. The analytic covariances (OneCovariance ξ±,
 # NaMaster pseudo-Cℓ) are what assembly puts in the terminal file, replacing
 # the estimates a part was born with.
 #
@@ -80,7 +80,7 @@ def cv_pure_eb_figures(version):
 
 
 def _grid_cov(version, grid, gaussian):
-    """CosmoCov-processed ξ± covariance on a named grid's own binning."""
+    """OneCovariance ξ± covariance on a named grid's own binning."""
     binning = XI_GRIDS[grid]
     return covariance_path(
         version,
@@ -93,7 +93,7 @@ def _grid_cov(version, grid, gaussian):
 
 
 def cv_xi_cov_integration(version):
-    """CosmoCov gaussian ξ± covariance on the integration grid.
+    """OneCovariance Gaussian ξ± covariance on the integration grid.
 
     The one analytic covariance both B-mode statistics take: the pure-E/B Monte
     Carlo draws from it and COSEBIs carry it through their kernel. Gaussian
@@ -150,7 +150,7 @@ def cv_pseudo_cl_cov(version):
 
 
 def cv_xi_cov(version):
-    """CosmoCov ξ± covariance on the reporting grid, the terminal file's."""
+    """OneCovariance ξ± covariance on the reporting grid, the terminal file's."""
     return _grid_cov(version, "reporting", "ng")
 
 
@@ -405,7 +405,7 @@ rule cv_pure_eb:
 rule cv_cosebis:
     """COSEBIs E/B decomposition for one version, from its integration-grid part.
 
-    The modes come from the part; the covariance is the CosmoCov ξ± covariance
+    The modes come from the part; the covariance is the OneCovariance ξ± covariance
     on the same grid through the same kernel as the modes.
     """
     input: