diff --git a/papers/bmodes/config/config.yaml b/papers/bmodes/config/config.yaml index 1964f8ef..5accc706 100644 --- a/papers/bmodes/config/config.yaml +++ b/papers/bmodes/config/config.yaml @@ -144,4 +144,4 @@ glass_mocks: # External tools tools: - cosmocov_executable: "/n23data1/n06data/lgoh/scratch/UNIONS/CosmoCov/covs/cov" + onecov_executable: "/home/guerrini/OneCovariance/covariance.py" diff --git a/papers/cosmo_val/config/config.yaml b/papers/cosmo_val/config/config.yaml index 26ade8e2..44d66f5e 100644 --- a/papers/cosmo_val/config/config.yaml +++ b/papers/cosmo_val/config/config.yaml @@ -149,4 +149,4 @@ glass_mocks: seed_range: [1, 350] tools: - cosmocov_executable: "/n23data1/n06data/lgoh/scratch/UNIONS/CosmoCov/covs/cov" + onecov_executable: "/home/guerrini/OneCovariance/covariance.py" diff --git a/scratch/lgoh/cov_config_test.yaml b/scratch/lgoh/cov_config_test.yaml new file mode 100644 index 00000000..a5eade51 --- /dev/null +++ b/scratch/lgoh/cov_config_test.yaml @@ -0,0 +1,27 @@ +fiducial: + version: SP_v1.4.6 + blind: A + min_sep: 1.0 + max_sep: 100.0 + nbins: 20 + npatch: 30 + mock_version: SP_v1.4.6 + gaussian: g + +probe: wl +glass_mocks: + version: "v0" + seed_range: [1, 350] + +covariance: + default_masked: True + +tools: + python_executable: /n23data1/n06data/lgoh/scratch/my_env_new/bin/python + onecov_executable: /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/covariance.py + +SP_v1.4.6: + cov_th: + sigma_e: 0.28, 0.28 + n_e: 5.0, 5.0 + A: 1000 \ No newline at end of file diff --git a/src/sp_validation/tests/regression/test_tomo_covariance_rules_call_external_checkout.py b/src/sp_validation/tests/regression/test_tomo_covariance_rules_call_external_checkout.py index c5079874..db95dd16 100644 --- a/src/sp_validation/tests/regression/test_tomo_covariance_rules_call_external_checkout.py +++ b/src/sp_validation/tests/regression/test_tomo_covariance_rules_call_external_checkout.py @@ -1,4 +1,4 @@ -"""Exercise real covariance rules without catalogues or CosmoCov. +"""Exercise real covariance rules without catalogues or OneCovariance. These tests pass on develop: it uses checkout-local scripts and launch-relative outputs. They guard against the external paths introduced on the tomography @@ -13,7 +13,6 @@ from pathlib import Path from types import SimpleNamespace -import numpy as np import pytest import sp_validation @@ -82,7 +81,6 @@ def covariance_rules(tmp_path): snakefile.write_text( f""" from pathlib import Path -from snakemake.io import apply_wildcards import json COSMO_INFERENCE = Path({str(tmp_path / "cosmo_inference")!r}) COSMO_VAL = Path({str(tmp_path / "cosmo_val")!r}) @@ -92,14 +90,12 @@ def covariance_rules(tmp_path): PLANCK18 = dict.fromkeys( ["Omega_m", "Omega_v", "sigma_8", "n_s", "h", "Omega_b"], 0.0 ) -BLOCK_PAIRS = [] DEFAULT_MASK_SUFFIX = "" FIDUCIAL = dict.fromkeys([ "mock_version", "version", "blind", "min_sep", "max_sep", "nbins", "min_sep_int", "max_sep_int", "nbins_int", ], "synthetic") config = {{ - "tools": {{"cosmocov_executable": "unused"}}, "glass_mocks": {{"seed_range": [1, 1]}}, }} def fiducial_binning_suffix(): @@ -108,6 +104,11 @@ def covariance_path(*args, **kwargs): return "unused-fiducial-input" def covariance_dir(*args, **kwargs): return "unused-covariance-directory" +def covariance_base(*args, **kwargs): + return "unused-covariance-base" +def cov_output(suffix): + return str(COSMO_INFERENCE / ("covariance_{{version}}_{{gaussian}}_minsep={{min_sep}}" + "_maxsep={{max_sep}}_nbins={{nbins}}{{mask_suffix}}{{tomo_suffix}}" + suffix)) def redshift_path(*args, **kwargs): return "unused-redshift-input" def build_redshift_path(*args, **kwargs): @@ -117,12 +118,12 @@ def cv_cov_tau(*args, **kwargs): def cv_tau_stats(*args, **kwargs): return "unused-tau-statistics" wildcard_constraints: - mask_suffix="(?:_masked)?" + mask_suffix="(?:_masked)?", + tomo_suffix="(?:_tomo)?" include: {str(source)!r} manifest = {{}} for name in [ - "covariance_process", "generate_glass_mock_rhotau_samples", - "covariance_glass_mock", + "generate_glass_mock_rhotau_samples", "covariance_glass_mock", ]: registered = workflow.get_rule(name) manifest[name] = {{ @@ -130,65 +131,15 @@ def cv_tau_stats(*args, **kwargs): "basedir": str(registered.basedir), "output": list(map(str, registered.output)), }} -synthetic_wildcards = dict( - version="synthetic", blind="A", gaussian="ng", min_sep="1", max_sep="2", - nbins="1", mask_suffix="", -) -manifest["covariance_process"]["synthetic_matrix"] = apply_wildcards( - workflow.get_rule("covariance_process").output[0], synthetic_wildcards -) Path("rules.json").write_text(json.dumps(manifest, indent=2)) """ ) - # Upper-triangle dump: columns 8 and 9 contain Gaussian and non-Gaussian - # terms. Their sum is the positive-definite matrix [[3, .25], [.25, 6]]. - raw = np.array( - [ - [0, 0, 0, 0, 0, 0, 0, 0, 2, 1], - [0, 1, 0, 0, 0, 0, 0, 0, 0.25, 0], - [1, 1, 0, 0, 0, 0, 0, 0, 4, 2], - ] - ) parsed = _run_snakemake(case, "--list-rules") assert parsed.returncode == 0, parsed.stdout + parsed.stderr case.manifest = json.loads((tmp_path / "rules.json").read_text()) - case.matrix = Path(case.manifest["covariance_process"]["synthetic_matrix"]) - raw_path = Path(str(case.matrix).replace("_processed.txt", ".txt")) - raw_path.parent.mkdir(parents=True) - np.savetxt(raw_path, raw) return case -@pytest.mark.slow -def test_covariance_postprocessing_runs_from_launched_checkout(covariance_rules): - """Process a synthetic CosmoCov dump without relying on another checkout. - - The three input rows give G+NG = [[3, .25], [.25, 6]], reflecting the supplied - off-diagonal element, and G = [[2, .25], [.25, 4]]. Snakemake executes the - actual rule with existing raw input, so no survey data or expensive - covariance calculation is needed. A missing external entry point is a - workflow defect. Develop passes because its processor is checkout-local; - this guards against the tomography merge reintroducing an external script. - """ - case = covariance_rules - result = _run_snakemake(case, "--printshellcmds", str(case.matrix)) - log = result.stdout + result.stderr - diagnosis = next( - (line for line in log.splitlines() if "can't open file" in line), log[-2500:] - ) - assert result.returncode == 0, ( - f"covariance_process returned {result.returncode}; expected exit 0 and " - f"matrix [[3.0, 0.25], [0.25, 6.0]]; {diagnosis}" - ) - np.testing.assert_allclose( - np.loadtxt(case.matrix), [[3, 0.25], [0.25, 6]], rtol=0, atol=0 - ) - gaussian = Path(str(case.matrix).replace("_processed.txt", "_processed_g.txt")) - np.testing.assert_allclose( - np.loadtxt(gaussian), [[2, 0.25], [0.25, 4]], rtol=0, atol=0 - ) - - def test_tau_sampling_selects_script_in_launched_checkout(covariance_rules): """The real tau-sampling rule must select code from the launched checkout. diff --git a/workflow/README.md b/workflow/README.md index 8af65cab..e3e15c6c 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -87,7 +87,7 @@ itself — no rule writes its own `apptainer exec` call. The image name comes from the `container:` directive in `workflow/Snakefile` (or a rule's own override, e.g. the image-sims `SIF`). -A few rules shell out to a host toolchain (CosmoCov, ImageMagick) and keep +A few rules shell out to a host toolchain (OneCovariance, ImageMagick) and keep `container: None`; each says why in its own docstring. The slurm executor submits with `--export=ALL`, so every job starts with the @@ -146,10 +146,10 @@ halves of one commit, split. The catalogue config is the launched checkout's `cosmo_val/cat_config.yaml`. `COSMO_VAL` defaults to the launched checkout's `cosmo_val/output`, so writing into another checkout's products means naming it; `COSMO_INFERENCE` defaults to -the shared candide tree, which holds the CosmoCov covariances and only its owner +the shared candide tree, which holds the covariances and only its owner can write. Anyone else launches with `COSMO_INFERENCE=` of their own, holding a link to the shared tree's `data/mask/` (the one input the covariance -rules take from it); the CosmoCov chain then runs there: +rules take from it); the OneCovariance rules then run there: ```bash mkdir -p /data diff --git a/workflow/common.py b/workflow/common.py index 9290a83e..c7d4116e 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -84,7 +84,6 @@ def _load_checkout_module(name): # The catalogue config of the launched checkout: the one file both the host # (CATALOG_CONFIG, loaded in configure) and every job read catalogues from. CAT_CONFIG = str(REPO_ROOT / "cosmo_val" / "cat_config.yaml") -BLOCK_PAIRS = [("++", "1"), ("--", "2"), ("+-", "3")] # Fiducial cosmology: Planck 2018 (astropy Planck18, Table 2 + BAO) # Source of truth: cs_util.cosmo.PLANCK18 @@ -98,12 +97,16 @@ def _load_checkout_module(name): WILDCARD_CONSTRAINTS = { "version": r"SP_v[\d.]+(_[ABC])?(_w_iv)?(_ecut\d+)?(_leak_corr)?", "nbins": r"\d+", + # Without this, npatch greedily swallows a trailing {tomo_suffix}. + "npatch": r"\d+", "min_sep": r"[0-9.]+", "max_sep": r"[0-9.]+", "gaussian": r"(g|ng)", "block_pm": r"(\+\+|--|\+-)", "block_i": r"[123]", "mask_suffix": r"(_masked)?", + # Empty means non-tomographic, so existing paths are unchanged. + "tomo_suffix": r"(_tomo)?", "mock_id": r"\d{5}", "nside": r"\d+", } @@ -226,6 +229,25 @@ def fiducial_binning_suffix(fiducial=None): ) +# Covariance naming, one scheme for every OneCovariance product. Rule outputs +# use the template with wildcards left in braces; covariance_base() formats it, +# so the two stay in sync by construction. An empty {tomo_suffix} is the +# non-tomographic covariance. +COV_BASE_TEMPLATE = ( + "covariance_{version}_{gaussian}" + "_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}" + "{mask_suffix}{tomo_suffix}" +) + + +def cov_output(suffix): + """Wildcard-bearing covariance path: ``/{suffix}``.""" + return str( + COSMO_INFERENCE + / f"data/covariance/{COV_BASE_TEMPLATE}/{COV_BASE_TEMPLATE}{suffix}" + ) + + def covariance_base( version, gaussian="ng", @@ -233,32 +255,44 @@ def covariance_base( max_sep=None, nbins=None, mask_suffix=None, + tomo_suffix=None, fiducial=None, default_mask_suffix=None, ): - """Construct covariance base name.""" + """Construct covariance base name (concrete values, no wildcards).""" fiducial = fiducial or FIDUCIAL + gaussian = gaussian if gaussian is not None else fiducial["gaussian"] min_sep = min_sep if min_sep is not None else fiducial["min_sep"] max_sep = max_sep if max_sep is not None else fiducial["max_sep"] nbins = nbins if nbins is not None else fiducial["nbins"] - mask_suffix = ( - mask_suffix - if mask_suffix is not None - else ( + if mask_suffix is None: + mask_suffix = ( DEFAULT_MASK_SUFFIX if default_mask_suffix is None else default_mask_suffix ) - ) - return ( - f"covariance_{version}_{gaussian}" - f"_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}" + return COV_BASE_TEMPLATE.format( + version=version, + gaussian=gaussian, + min_sep=min_sep, + max_sep=max_sep, + nbins=nbins, + mask_suffix=mask_suffix, + tomo_suffix=tomo_suffix or "", ) def covariance_dir( - version, gaussian="ng", min_sep=None, max_sep=None, nbins=None, mask_suffix=None + version, + gaussian="ng", + min_sep=None, + max_sep=None, + nbins=None, + mask_suffix=None, + tomo_suffix=None, ): """Construct covariance directory path.""" - base = covariance_base(version, gaussian, min_sep, max_sep, nbins, mask_suffix) + base = covariance_base( + version, gaussian, min_sep, max_sep, nbins, mask_suffix, tomo_suffix + ) return str(COSMO_INFERENCE / f"data/covariance/{base}") @@ -269,10 +303,13 @@ def covariance_path( max_sep=None, nbins=None, mask_suffix=None, - suffix="_processed.txt", + tomo_suffix=None, + suffix=".dat", ): - """Construct covariance file path.""" - base = covariance_base(version, gaussian, min_sep, max_sep, nbins, mask_suffix) + """Construct covariance file path; by default the OneCovariance matrix.""" + base = covariance_base( + version, gaussian, min_sep, max_sep, nbins, mask_suffix, tomo_suffix + ) return str(COSMO_INFERENCE / f"data/covariance/{base}/{base}{suffix}") diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 81d305d1..0c83b667 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -10,15 +10,15 @@ # ├─ integration part ─┬──→ pure_eb (part, npz, figures) # │ └──→ cosebis (part, npz, figures) # └─ reporting part ──→ 2pcf plot, ratio_xi_sys_xi -# CosmoCov ξ±, integration grid ──→ pure_eb, cosebis (their covariances) +# OneCov ξ±, integration grid ──→ pure_eb, cosebis (their covariances) # catalogue ──→ pseudo_cl (part) ──┬─→ pseudo-Cl figures -# CosmoCov ξ±, reporting grid ─────┤ +# OneCov ξ±, reporting grid ───────┤ # rho/tau (part + FITS) ───────────┼─→ summarize_bmodes (reads the products) # └─→ assemble_sacc ──→ {version}.sacc # # The B-mode rules and the ξ± figures are ingests: they work from the parts and # the covariance inputs, never from a catalogue, so a blinded part keeps -# everything downstream blinded. The analytic covariances (CosmoCov ξ±, +# everything downstream blinded. The analytic covariances (OneCovariance ξ±, # NaMaster pseudo-Cℓ) are what assembly puts in the terminal file, replacing # the estimates a part was born with. # @@ -80,7 +80,7 @@ def cv_pure_eb_figures(version): def _grid_cov(version, grid, gaussian): - """CosmoCov-processed ξ± covariance on a named grid's own binning.""" + """OneCovariance ξ± covariance on a named grid's own binning.""" binning = XI_GRIDS[grid] return covariance_path( version, @@ -93,7 +93,7 @@ def _grid_cov(version, grid, gaussian): def cv_xi_cov_integration(version): - """CosmoCov gaussian ξ± covariance on the integration grid. + """OneCovariance Gaussian ξ± covariance on the integration grid. The one analytic covariance both B-mode statistics take: the pure-E/B Monte Carlo draws from it and COSEBIs carry it through their kernel. Gaussian @@ -150,7 +150,7 @@ def cv_pseudo_cl_cov(version): def cv_xi_cov(version): - """CosmoCov ξ± covariance on the reporting grid, the terminal file's.""" + """OneCovariance ξ± covariance on the reporting grid, the terminal file's.""" return _grid_cov(version, "reporting", "ng") @@ -405,7 +405,7 @@ rule cv_pure_eb: rule cv_cosebis: """COSEBIs E/B decomposition for one version, from its integration-grid part. - The modes come from the part; the covariance is the CosmoCov ξ± covariance + The modes come from the part; the covariance is the OneCovariance ξ± covariance on the same grid through the same kernel as the modes. """ input: diff --git a/workflow/rules/covariance.smk b/workflow/rules/covariance.smk index c028ec99..1191b841 100644 --- a/workflow/rules/covariance.smk +++ b/workflow/rules/covariance.smk @@ -1,13 +1,5 @@ -# BLOCK_PAIRS, PLANCK18, COSMOLOGY_PARAMS defined in Snakefile - - -def get_cat_params(version): - """Covariance parameters (area, n_e, sigma_e) of ``version``'s catalogue entry.""" - cov_th = catalogue_entry(version)["cov_th"] - return cov_th["A"], cov_th["n_e"], cov_th["sigma_e"] - - -# covariance_dir(), covariance_base(), covariance_path() defined in Snakefile +# PLANCK18, COSMOLOGY_PARAMS defined in Snakefile +# cov_output(), covariance_dir(), covariance_base(), covariance_path() defined in Snakefile # DEFAULT_MASK_SUFFIX defined in Snakefile # Footprint mask power spectra (nside=4096, from comprehensive catalog with spatial cuts only) @@ -27,6 +19,41 @@ def get_mask_cls_path(version): return MASK_CLS_FILES["footprint_starhalo" if starhalo else "footprint"] +# --------------------------------------------------------------------------- +# OneCovariance: real-space ξ± covariance for cosmic shear, optionally +# tomographic. Named by covariance_base() in common.py. +# --------------------------------------------------------------------------- + +def get_onecov_cat_params(version): + """OneCovariance survey parameters (sigma_e, n_e, area) of ``version``'s + catalogue entry. Tomographic values are comma-separated strings, one entry + per bin.""" + cov_th = catalogue_entry(version)["cov_th"] + return cov_th["sigma_e"], cov_th["n_e"], cov_th["A"] + + +def _onecov_args(w): + """covariance_base() arguments of one job, from its wildcards.""" + return ( + w.version, w.gaussian, w.min_sep, w.max_sep, w.nbins, + w.mask_suffix, w.tomo_suffix, + ) + + +def _onecov_tool(key, default=None): + """``config["tools"][key]``, read when a job runs rather than at parse time.""" + def lookup(w): + value = config.get("tools", {}).get(key, default) + if value is None: + raise ValueError(f"Set tools.{key} in the workflow config to run OneCovariance.") + return value + return lookup + + +# Number of cores to use to execute OneCovariance +ONECOV_THREADS = 8 + + rule cosmology_params: """Generate cosmology parameters JSON from sp_validation. @@ -52,144 +79,6 @@ with open('{output}', 'w') as f: """ -rule covariance_ini: - input: - nz_file=lambda w: redshift_path(w.version), - mask=lambda w: [] if w.mask_suffix != "_masked" else [get_mask_cls_path(w.version)], - output: - str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}.ini") - params: - outdir=lambda w: covariance_dir( - w.version, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix - ), - ng_value=lambda wildcards: "1" if wildcards.gaussian == "ng" else "0", - omega_m=PLANCK18["Omega_m"], - omega_v=PLANCK18["Omega_v"], - sigma_8=PLANCK18["sigma_8"], - n_s=PLANCK18["n_s"], - h=PLANCK18["h"], - omega_b=PLANCK18["Omega_b"], - area=lambda w: get_cat_params(w.version)[0], - n_e=lambda w: get_cat_params(w.version)[1], - sigma_e_param=lambda w: get_cat_params(w.version)[2], - mask=lambda w: get_mask_cls_path(w.version) if w.mask_suffix == "_masked" else "", - threads: 1 - shell: - """ - mkdir -p {params.outdir} - - cat > {output} << 'EOF' -# -# Cosmological parameters -# -Omega_m : {params.omega_m} -Omega_v : {params.omega_v} -sigma_8 : {params.sigma_8} -n_spec : {params.n_s} -w0 : -1 -wa : 0 -omb : {params.omega_b} -h0 : {params.h} - - -# Survey and galaxy parameters -# -# area in degrees -# n_gal,lens_n_gal in gals/arcmin^2 - -area : {params.area} -sourcephotoz : multihisto -lensphotoz : multihisto -source_tomobins : 1 -lens_tomobins : 1 -sigma_e : {params.sigma_e_param} -source_n_gal : {params.n_e} -lens_n_gal : {params.n_e} - - -shear_REDSHIFT_FILE : {input.nz_file} -clustering_REDSHIFT_FILE : {input.nz_file} -c_footprint_file : {params.mask} - - -# IA parameters -IA : 1 -A_ia : 0.0 -eta_ia : 0.0 - - -# Covariance parameters -# -# tmin,tmax in arcminutes -tmin : {wildcards.min_sep} -tmax : {wildcards.max_sep} -ntheta : {wildcards.nbins} -ng : {params.ng_value} -cng : {params.ng_value} - - -outdir : ./ -filename : cov_tmp -ss : true -ls : false -ll : false -EOF - """ - - -rule covariance_cosmocov: - """Run the host-compiled CosmoCov binary. - - `container: None` on purpose: CosmoCov is a host-compiled Fortran/C binary - loaded through environment-modules (`module load gcc intelpython openmpi`), - not a Python entry point the container ships. - """ - input: - rules.covariance_ini.output, - output: - str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/cov_tmp_ssss_{block_pm}_cov_Ntheta{nbins}_Ntomo1_{block_i}") - params: - block_i="{block_i}", - outdir=lambda w: covariance_dir( - w.version, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix - ), - ini_path=lambda w: covariance_path( - w.version, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix, - suffix=".ini" - ), - cosmocov=config["tools"]["cosmocov_executable"], - container: - None - threads: 1 - shell: - """ - module unload gcc || true - module load gcc - module unload intelpython || true - module load intelpython/3-2024.1.0 - module load openmpi - - cd {params.outdir} - {params.cosmocov} {params.block_i} {params.ini_path} - """ - - -rule covariance_cat: - input: - cov_block=lambda w: [ - f"{covariance_dir(w.version, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix)}" - f"/cov_tmp_ssss_{pm}_cov_Ntheta{w.nbins}_Ntomo1_{idx}" - for pm, idx in BLOCK_PAIRS - ], - threads: 1 - output: - str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}.txt") - shell: - """ - cat {input} > {output} - """ - - rule covariance_glass_mock: input: xi=expand( @@ -230,28 +119,187 @@ rule generate_glass_mock_rhotau_samples: "../scripts/generate_glass_mock_rhotau_samples.py" -rule covariance_process: - """Post-process a raw CosmoCov matrix into the analysis-ready form.""" +rule covariance_ini: input: - str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}.txt") + nz_file=lambda w: redshift_path(w.version), + mask=lambda w: [] if w.mask_suffix != "_masked" else [get_mask_cls_path(w.version)], output: - matrix=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed.txt"), - gaussian=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed_g.txt"), - plot=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed_plot.pdf") + cov_output(".ini"), + params: + outdir=lambda w: covariance_dir(*_onecov_args(w)), + out_filename=lambda w: covariance_base(*_onecov_args(w)) + ".dat", + out_plot_filename=lambda w: covariance_base(*_onecov_args(w)) + "_corr_plot.pdf", + ng_value=lambda w: "1" if w.gaussian == "ng" else "0", + omega_m=PLANCK18["Omega_m"], + omega_v=PLANCK18["Omega_v"], + sigma_8=PLANCK18["sigma_8"], + n_s=PLANCK18["n_s"], + h=PLANCK18["h"], + omega_b=PLANCK18["Omega_b"], + hmcode_logT_AGN=7.75, + sigma_e_param=lambda w: get_onecov_cat_params(w.version)[0], + n_e=lambda w: get_onecov_cat_params(w.version)[1], + area=lambda w: get_onecov_cat_params(w.version)[2], + # OneCovariance takes directory and file separately. + mask_dir=lambda w: MASK_CLS_BASE if w.mask_suffix == "_masked" else "", + mask_file=lambda w: ( + os.path.basename(get_mask_cls_path(w.version)) + if w.mask_suffix == "_masked" + else "" + ), + nz_dir=lambda w: os.path.dirname(redshift_path(w.version)), + nz_file=lambda w: os.path.basename(redshift_path(w.version)), + num_cores=ONECOV_THREADS, threads: 1 - script: - "../scripts/cosmocov_process.py" + shell: + """ + mkdir -p {params.outdir} + cat > "{output}" << 'EOF' +[covariance terms] +gauss = True +split_gauss = False +nongauss = {params.ng_value} +ssc = {params.ng_value} +sn_only = False + +[observables] +cosmic_shear = True +est_shear = xi_pm +clustering = False +ggl = False + +[output settings] +directory = {params.outdir} +file = {params.out_filename} +style = matrix +save_configs = False +corrmatrix_plot = {params.out_plot_filename} + +[covTHETAspace settings] +theta_min = {wildcards.min_sep} +theta_max = {wildcards.max_sep} +theta_bins = {wildcards.nbins} +theta_type = log +xi_pp = True +xi_mm = True +theta_accuracy = 1e-3 +integration_intervals = 400 + +[survey specs] +mask_directory = {params.mask_dir} +mask_file_lensing = {params.mask_file} +ellipticity_dispersion = {params.sigma_e_param} +survey_area_lensing_in_deg2 = {params.area} +n_eff_lensing = {params.n_e} + +[redshift] +zlens_directory = {params.nz_dir} +zlens_file = {params.nz_file} +value_loc_in_lensbin = mid + +[cosmo] +sigma8 = {params.sigma_8} +h = {params.h} +omega_m = {params.omega_m} +omega_b = {params.omega_b} +omega_de = {params.omega_v} +w0 = -1.0 +wa = 0.0 +ns = {params.n_s} +neff = 3.046 +m_nu = 0.06 +tcmb0 = 2.725 + +[IA] +A_IA = 0.0 +eta_IA = 0.0 +z_pivot_IA = 0.3 + +[powspec evaluation] +non_linear_model = mead2020_feedback +HMCode_logT_AGN = {params.hmcode_logT_AGN} +log10k_bins = 300 +log10k_min = -3.46 +log10k_max = 3.15 + +[hod] +model_mor_cen = double_powerlaw +model_mor_sat = double_powerlaw +dpow_logm0_cen = 10.51 +dpow_logm1_cen = 11.38 +dpow_a_cen = 7.096 +dpow_b_cen = 0.2 +dpow_norm_cen = 1.0 +dpow_norm_sat = 0.56 +model_scatter_cen = lognormal +model_scatter_sat = modschechter +logn_sigma_c_cen = 0.35 +modsch_logmref_sat = 13.0 +modsch_alpha_s_sat = -0.858 +modsch_b_sat = -0.024, 1.149 + +[halomodel evaluation] +m_bins = 900 +log10m_min = 6 +log10m_max = 18 +hmf_model = Tinker10 +mdef_model = SOMean +mdef_params = overdensity, 200 +disable_mass_conversion = True +delta_c = 1.686 +transfer_model = CAMB +small_k_damping_for1h = damped + +[misc] +num_cores = {params.num_cores} -def fiducial_covariance_outputs(mask_suffix=""): - """Return processed covariance files for the fiducial version.""" +EOF + """ + + +rule covariance_run: + """Run OneCovariance on its generated .ini. + + `container: None` on purpose: OneCovariance runs from a host checkout under + environment-modules, set by tools.onecov_executable. + """ + input: + rules.covariance_ini.output, + output: + cov_output(".dat"), + params: + outdir=lambda w: covariance_dir(*_onecov_args(w)), + onecov=_onecov_tool("onecov_executable"), + python_executable=_onecov_tool("python_executable", "python"), + container: + None + threads: ONECOV_THREADS + shell: + """ + module unload gcc || true + module load gcc + module unload intelpython || true + module load intelpython/3-2024.1.0 + module load openmpi + + cd {params.outdir} + {params.python_executable} {params.onecov} {input} + """ + + +def fiducial_covariance_outputs(mask_suffix="", tomo_suffix=""): + """Return the covariance matrices for the fiducial version: non-Gaussian on + the reporting grid, Gaussian on the integration grid.""" ng_path = covariance_path( FIDUCIAL["version"], "ng", - FIDUCIAL["min_sep"], FIDUCIAL["max_sep"], FIDUCIAL["nbins"], mask_suffix + FIDUCIAL["min_sep"], FIDUCIAL["max_sep"], FIDUCIAL["nbins"], + mask_suffix, tomo_suffix, ) g_path = covariance_path( FIDUCIAL["version"], "g", - FIDUCIAL["min_sep_int"], FIDUCIAL["max_sep_int"], FIDUCIAL["nbins_int"], mask_suffix + FIDUCIAL["min_sep_int"], FIDUCIAL["max_sep_int"], FIDUCIAL["nbins_int"], + mask_suffix, tomo_suffix, ) return [ng_path, g_path] @@ -271,11 +319,14 @@ rule covariance_unmasked: fiducial_covariance_outputs(mask_suffix="") -ruleorder: covariance_ini > covariance_cosmocov > covariance_cat > covariance_process > covariance +rule covariance_tomo: + input: + fiducial_covariance_outputs(mask_suffix=DEFAULT_MASK_SUFFIX, tomo_suffix="_tomo") + + +ruleorder: covariance_ini > covariance_run > covariance localrules: cosmology_params, covariance_ini, - covariance_cat, - covariance_process, generate_glass_mock_rhotau_samples, diff --git a/workflow/rules/glass_mock.smk b/workflow/rules/glass_mock.smk index c5ec046a..ed77e80b 100644 --- a/workflow/rules/glass_mock.smk +++ b/workflow/rules/glass_mock.smk @@ -111,7 +111,7 @@ rule mock_cosebis_scatter: rule mock_cosebis_bias_test: """Gather: 100-mock COSEBIS bias test figure + evidence. - Collects per-mock COSEBIS from scatter jobs, propagates CosmoCov ξ± + Collects per-mock COSEBIS from scatter jobs, propagates OneCovariance ξ± covariance to COSEBIS space, tests mean B_n = 0 at σ/√N precision. """ input: diff --git a/workflow/rules/masks.smk b/workflow/rules/masks.smk index 6bc7f657..913e5bf6 100644 --- a/workflow/rules/masks.smk +++ b/workflow/rules/masks.smk @@ -8,7 +8,7 @@ MASK_CLS_FILES in covariance.smk. See covariance.md spec for details. This file contains rules for: - Processing HealSparse masks to individual nside values (parallelizable) - Calculating effective survey areas -- Computing mask power spectra for CosmoCov integration +- Computing mask power spectra for the OneCovariance survey mask - Generating comparison plots and analysis """ from datetime import datetime @@ -18,7 +18,6 @@ from datetime import datetime # Paths from config and Snakefile SOURCE_MASK_FILE = config["pixel_mask"]["source_file"] MASK_OUTPUT_DIR = "output/masks" -COSMOCOV_MASK_DIR = str(COSMO_INFERENCE / "data/masks") # Get target nside values from config (no fallbacks) target_nsides = config["pixel_mask"]["target_nsides"] @@ -92,7 +91,7 @@ rule combine_area_summaries: rule mask_power_spectrum: """Calculate angular power spectrum for a single mask. - Loads downgraded HEALPix mask, computes C_ell, exports for CosmoCov. + Loads downgraded HEALPix mask, computes C_ell, exports for OneCovariance. """ input: mask=f"{MASK_OUTPUT_DIR}/mask_nside{{nside}}.fits" @@ -185,9 +184,9 @@ rule effective_area_comparison: report_content.extend([ "", - "## CosmoCov Integration", + "## OneCovariance Integration", "", - "The following power spectrum files are ready for CosmoCov integration:", + "The following power spectrum files are ready for OneCovariance:", "" ]) @@ -199,7 +198,7 @@ rule effective_area_comparison: report_content.extend([ "", - "To use these in CosmoCov, set the `c_footprint_file` parameter to point to the appropriate power spectrum file.", + "To use one in OneCovariance, put it in `mask_directory` and name it in `mask_file_lensing`.", "", "## Recommendations", "" @@ -240,62 +239,9 @@ rule masks_full_analysis: f"{MASK_OUTPUT_DIR}/area_comparison_report.md" -# Integration with existing covariance workflow -rule prepare_mask_for_cosmocov: - """ - Prepare mask power spectra for integration with CosmoCov covariance calculations. - - This rule creates copies of mask power spectra in the CosmoCov data directory. - """ - input: - power_spectra=expand(f"{MASK_OUTPUT_DIR}/power_spectra/cl_mask_nside{{nside}}.txt", nside=target_nsides), - report=f"{MASK_OUTPUT_DIR}/area_comparison_report.md" - output: - cosmocov_ready=f"{MASK_OUTPUT_DIR}/cosmocov_integration_ready.txt" - params: - cosmocov_mask_dir=COSMOCOV_MASK_DIR - threads: 1 - resources: - mem_mb=500, - runtime=2 - run: - import os - import shutil - from pathlib import Path - - # Create CosmoCov mask directory if it doesn't exist - cosmocov_dir = Path(params.cosmocov_mask_dir) - cosmocov_dir.mkdir(parents=True, exist_ok=True) - - # Copy power spectrum files to CosmoCov directory - copied_files = [] - for ps_file in input.power_spectra: - src_path = Path(ps_file) - dst_path = cosmocov_dir / src_path.name - - shutil.copy2(src_path, dst_path) - copied_files.append(str(dst_path)) - print(f"Copied {src_path.name} to CosmoCov directory") - - # Create integration status file - with open(output.cosmocov_ready, 'w') as f: - f.write("UNIONS Mask Power Spectra - CosmoCov Integration Ready\n") - f.write(f"Generated: {datetime.now().strftime('%Y-%m-%d')}\n") - f.write(f"Source: {MASK_OUTPUT_DIR}\n") - f.write(f"Destination: {params.cosmocov_mask_dir}\n") - f.write("\nCopied files:\n") - for file_path in copied_files: - f.write(f"- {file_path}\n") - f.write("\nTo use in CosmoCov:\n") - f.write("Set c_footprint_file parameter to the appropriate power spectrum file path.\n") - - print(f"CosmoCov integration prepared: {output.cosmocov_ready}") - - # Fast rules that don't need cluster resources localrules: combine_area_summaries, effective_area_comparison, - prepare_mask_for_cosmocov, masks_only, masks_full_analysis, diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index d756bce2..71666503 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -7,7 +7,7 @@ rebuilt into one Sacc with a single ``BlockDiagonalCovariance``. Every part must carry a covariance block. ξ± reporting and pseudo-Cℓ take -theirs from the analytic inputs — the CosmoCov ``.txt`` (``--xi-cov``) and the +theirs from the analytic inputs — the OneCovariance ξ± matrix (``--xi-cov``) and the NaMaster covariance FITS (``--pseudo-cl-cov``) — which replace any estimate the part was born with; the pseudo-Cℓ cross-spectrum blocks (EE↔BB, …) are dropped, matching what the B-mode PTE reads today. @@ -178,7 +178,9 @@ def _from_cli(argv=None): ap.add_argument( f"--{name.replace('_', '-')}", default=None, help=f"{name} part" ) - ap.add_argument("--xi-cov", default=None, help="CosmoCov ξ covariance .txt") + ap.add_argument( + "--xi-cov", default=None, help="OneCovariance ξ± covariance matrix (.dat)" + ) ap.add_argument( "--pseudo-cl-cov", default=None, help="NaMaster pseudo-Cℓ covariance FITS" ) diff --git a/workflow/scripts/cosmocov_process.py b/workflow/scripts/cosmocov_process.py deleted file mode 100644 index 6e6c1723..00000000 --- a/workflow/scripts/cosmocov_process.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Assemble a raw CosmoCov block dump into an analysis-ready covariance matrix. - -CosmoCov writes one row per (i, j) element with the Gaussian term in column 8 -and the non-Gaussian term in column 9; this rebuilds the symmetric matrices, -checks positive-definiteness, and plots the correlation matrix. - -Run through Snakemake's ``script:`` directive, which injects ``snakemake`` as a -module global before this file executes. -""" - -import sys - -import matplotlib - -matplotlib.use("Agg") - -import matplotlib.pyplot as plt # noqa: E402 -import numpy as np # noqa: E402 - - -def get_cov(filename): - """Return (gaussian, non-gaussian, ndata) from a CosmoCov element list.""" - - data = np.loadtxt(filename) - ndata = int(np.max(data[:, 0])) + 1 - - cov_g = np.zeros((ndata, ndata)) - cov_ng = np.zeros((ndata, ndata)) - for i in range(data.shape[0]): - row, col = int(data[i, 0]), int(data[i, 1]) - cov_g[row, col] = cov_g[col, row] = data[i, 8] - cov_ng[row, col] = cov_ng[col, row] = data[i, 9] - - return cov_g, cov_ng, ndata - - -def plot_correlation(cov, ndata, plot_path): - """Save the correlation matrix with xi+/xi- block annotations.""" - - diag = np.sqrt(np.diag(cov)) - correlation = cov / np.outer(diag, diag) - - fig, ax = plt.subplots() - extent = (0, ndata, ndata, 0) - image = ax.imshow(correlation, cmap="seismic", vmin=-1, vmax=1, extent=extent) - - ax.axvline(x=ndata // 2, color="black", linewidth=1.0) - ax.axhline(y=ndata // 2, color="black", linewidth=1.0) - - fig.colorbar(image, orientation="vertical") - - ax.text(ndata // 4, ndata + 5, r"$\xi_+^{ij}(\theta)$", fontsize=12) - ax.text(3 * (ndata // 4), ndata + 5, r"$\xi_-^{ij}(\theta)$", fontsize=12) - ax.text(-9, ndata // 4, r"$\xi_+^{ij}(\theta)$", fontsize=12) - ax.text(-9, 3 * (ndata // 4), r"$\xi_-^{ij}(\theta)$", fontsize=12) - - fig.savefig(plot_path, dpi=300) - plt.close(fig) - - -cov_g, cov_ng, ndata = get_cov(snakemake.input[0]) # noqa: F821 -print(f"Dimension of cov: {ndata}x{ndata}") - -cov = cov_g + cov_ng - -eigenvalues = np.linalg.eigvalsh(cov) -print(f"min+max eigenvalues cov: {eigenvalues.min():e}, {eigenvalues.max():e}") -if eigenvalues.min() <= 0.0: - sys.exit("non-positive eigenvalue encountered! Covariance invalid!") - -np.savetxt(snakemake.output.matrix, cov) # noqa: F821 -np.savetxt(snakemake.output.gaussian, cov_g) # noqa: F821 -plot_correlation(cov, ndata, snakemake.output.plot) # noqa: F821 diff --git a/workflow/scripts/cv_cosebis.py b/workflow/scripts/cv_cosebis.py index ee9d8aef..5f69d888 100644 --- a/workflow/scripts/cv_cosebis.py +++ b/workflow/scripts/cv_cosebis.py @@ -1,6 +1,6 @@ """Rule cv_cosebis: COSEBIs E/B decomposition for one version. -A consumer of the integration-grid ξ± part and the CosmoCov ξ± covariance on +A consumer of the integration-grid ξ± part and the OneCovariance ξ± covariance on the same grid — nothing here touches a catalogue. The covariance goes through the same linear kernel as the modes to give the COSEBIs covariance; it is analytic, so no Hartlap debiasing applies. The COSEBIs part is a derivation of diff --git a/workflow/scripts/cv_summarize_bmodes.py b/workflow/scripts/cv_summarize_bmodes.py index 3f9d9e82..6ab3aec4 100644 --- a/workflow/scripts/cv_summarize_bmodes.py +++ b/workflow/scripts/cv_summarize_bmodes.py @@ -43,7 +43,7 @@ # this table wants. cosebis = np.load(snakemake.input["cosebis"][i]) row["COSEBIS"] = float(cosebis["pte_B"]) - cov_methods.add("COSEBIs: analytic (CosmoCov ξ± through the COSEBIs kernel)") + cov_methods.add("COSEBIs: analytic (OneCovariance ξ± through the COSEBIs kernel)") if p["include_pseudo_cl"]: from astropy.io import fits diff --git a/workflow/scripts/mock_cosebis_bias_test.py b/workflow/scripts/mock_cosebis_bias_test.py index e739491c..8141feed 100644 --- a/workflow/scripts/mock_cosebis_bias_test.py +++ b/workflow/scripts/mock_cosebis_bias_test.py @@ -1,6 +1,6 @@ """Gather: mock COSEBIS bias test from pre-computed per-mock results. -Loads per-mock COSEBIS B_n from scatter jobs, propagates CosmoCov ξ± +Loads per-mock COSEBIS B_n from scatter jobs, propagates OneCovariance ξ± covariance to COSEBIS space, tests mean B_n = 0 at σ/√N precision. Two-panel figure: diff --git a/workflow/tests/conftest.py b/workflow/tests/conftest.py index 202e75f3..af4e4258 100644 --- a/workflow/tests/conftest.py +++ b/workflow/tests/conftest.py @@ -11,7 +11,7 @@ The ``toy`` fixture is a disposable checkout: copies of ``workflow/`` and ``papers/cosmo_val/``, this checkout's ``src/`` symlinked in, a blinded and an -undeclared catalogue, stand-ins for the processed CosmoCov covariances (their +undeclared catalogue, stand-ins for the OneCovariance covariances (their inputs live on candide), and both output roots in tmp. The host never opens a blind, so there is no blind record. """ @@ -103,7 +103,7 @@ class Toy: env: dict config: dict common: object - covariances: dict # (version, "g" | "ng") -> the processed CosmoCov file + covariances: dict # (version, "g" | "ng") -> the OneCovariance matrix def snakemake(self, *args, config=(), cwd=None, env=None, timeout=300): """Run the host Snakemake in the toy's paper directory, or in ``cwd``. @@ -209,7 +209,7 @@ def toy(tmp_path_factory): ) common = _load_module(root / "workflow" / "common.py", "toy_common", env) - # The processed CosmoCov covariances the cosmo_val rules read, in place. + # The OneCovariance covariances the cosmo_val rules read, in place. grids = common.xi_grids(config, config["fiducial"]) mask = "_masked" if config["covariance"].get("default_masked") else "" covariances = {} diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index 7b1e4e3d..38de61f9 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -27,7 +27,7 @@ def grids(toy): def test_assemble_resolves(toy): """Each terminal file gathers every part and the analytic covariances. - The ξ± block takes the CosmoCov covariance on the reporting grid, and the + The ξ± block takes the OneCovariance covariance on the reporting grid, and the harmonic block is the part on the fiducial harmonic binning with the NaMaster covariance of that same binning. """