From a9ca979699d65cd2a869ebeaad2808ef9ce0aff5 Mon Sep 17 00:00:00 2001 From: Eleanor Frajka-Williams Date: Thu, 5 Nov 2026 06:43:54 +0100 Subject: [PATCH 1/2] Check QC overrides in the mooring description; CLI and docs sweep --- CHANGELOG.md | 11 +- CONTRIBUTING.md | 6 +- README.md | 4 +- docs/source/cli_reference.rst | 121 ++++++++++++-- docs/source/clock_offset.rst | 2 +- docs/source/directory_structure.rst | 7 +- docs/source/file_naming_conventions.rst | 10 +- docs/source/index.rst | 2 +- docs/source/legacy.rst | 13 +- docs/source/methods/auto_qc.rst | 35 ++-- .../methods/nortek_coordinate_transform.rst | 2 +- docs/source/methods/oceansites_export.rst | 37 ++++- docs/source/methods/time_gridding.rst | 2 +- docs/source/methods/trimming.rst | 4 +- docs/source/methods/vertical_gridding.rst | 4 +- docs/source/migration.rst | 2 +- docs/source/oceanSITES_manual.rst | 2 +- docs/source/oceanarray.rst | 5 +- docs/source/processing_framework.rst | 16 +- docs/source/project_structure.md | 29 ++-- docs/source/quickstart.rst | 17 ++ docs/source/roadmap.rst | 155 ++++++++---------- docs/source/style_guide.md | 72 +++----- docs/source/troubleshooting.rst | 21 +-- docs/source/yaml_configuration.rst | 90 ++++++++-- oceanarray/__main__.py | 6 + oceanarray/cli/__init__.py | 5 +- oceanarray/cli/_common.py | 70 ++++---- oceanarray/cli/_deprecate.py | 40 +++++ oceanarray/cli/animate.py | 15 +- oceanarray/cli/export.py | 38 ++++- oceanarray/cli/init.py | 6 + oceanarray/cli/list.py | 8 +- oceanarray/cli/plot.py | 37 ++++- oceanarray/cli/process.py | 60 +++++-- oceanarray/cli/report.py | 33 +++- oceanarray/cli/run.py | 25 ++- oceanarray/cli/stamp.py | 26 ++- oceanarray/cli/validate.py | 65 ++++++-- oceanarray/config/parameters.py | 2 +- oceanarray/mooring.py | 123 +++++++++++++- oceanarray/pipeline/coordinate.py | 2 +- oceanarray/pipeline/export.py | 22 ++- tests/integration/test_cli_paths.py | 30 +++- tests/integration/test_stamp.py | 18 +- tests/unit/test_export.py | 28 ++++ tests/unit/test_mooring.py | 92 +++++++++++ 47 files changed, 1078 insertions(+), 342 deletions(-) create mode 100644 oceanarray/__main__.py create mode 100644 oceanarray/cli/_deprecate.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e532296a..e3d11152 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ All notable changes to oceanarray are documented here. ### Added -- `oceanarray.export(mooring, *, mooring_dir=None, out_dir=None, array=None, grid=True, overwrite=False)` and `oceanarray export MOORING [--array YAML] [-o DIR] [--no-grid] [--overwrite]` — write a mooring's stage-3 files and grid as OceanSITES 1.4 files (default `/oceansites/`); instruments that cannot be exported are listed and left out. The data-format page (`oceanarray_format`) now describes these files, with the earlier departures from OceanSITES (`N_LEVELS`, `contributor_*`, compact times) dropped. +- `oceanarray.export(mooring, *, mooring_dir=None, out_dir=None, array=None, grid=True, overwrite=False, dry_run=False)` and `oceanarray export MOORING [--array YAML] [-o DIR] [--no-grid] [--force] [-n]` — write a mooring's stage-3 files and grid as OceanSITES 1.4 files (default `/oceansites/`); instruments that cannot be exported are listed and left out. The data-format page (`oceanarray_format`) now describes these files, with the earlier departures from OceanSITES (`N_LEVELS`, `contributor_*`, compact times) dropped. - `oceanarray.pipeline.oceansites.to_oceansites(ds, mooring, *, array=None, stack=None, source_file=None)` — one instrument's stage-3 dataset as an OceanSITES 1.4 deployment data file: variables on `(TIME, DEPTH)` with `DEPTH` the nominal instrument depth, `_QC` (reference table 2, the same flags), a data mode per variable (`DM_indicator`; `D` only with the calibration-dip correction recorded) and the §2.2 globals. `oceansites_filename` gives `OS____--m.nc`. An ADCP is one file with its bins on `DEPTH` (nominal transducer depth ± range, by orientation; refused without one), `PRES` the pressure at each bin and `TRANSDUCER_PRES(TIME)` the measured transducer pressure and a failed or suspect `seabed_qc` merged into the velocity flags. A grid becomes a gridded file (`GRD`, §4.2): `OS_-_-_GRD_-DP.nc`, `DEPTH` the pressure levels converted with `gsw.z_from_p`, `PRES` the levels, and `history` naming the stage files (pass `stack=`); only a provisional (`P`) grid is exported. The mooring description gains `platform_code` and `deployment_code` (default: the month of `deployment_time`); `site_code` comes from the array description and is left out when absent. The OceanSITES reader also reads `CSPD`, `CDIR`, `DOX2` and `TURB`. - `oceanarray.pipeline.read_mooring.read_mooring(mooring, *, stage=None, serials=None, allow_missing=False, mooring_dir=None, raw_dir=None)` — reads every instrument of one mooring into a dict of datasets keyed by serial. A mooring's identity is its `name:` field (every file is `__stage.nc`); `mooring` may be a dict, a `*.mooring.yaml` path, or a directory holding one. `stage=None` finds and reads each raw file (from `raw_dir`) as stage 1 does — a missing `filename` found by the recovery-file naming convention, the Nortek header (also in a legacy `nortek/` folder), the raw header's calibration metadata — and returns it unmarked; entries marked `skip` are left out; `stage=1|2|3` opens the stage files (from `mooring_dir`) and checks the marker; on every path the description's facts are stamped (per-instrument: serial, instrument, hab, clamp_id, sample_interval_seconds, file_type, source_file; per-mooring: mooring_name, latitude/longitude + source, deployment/recovery window + source, waterdepth, density_reference). A dataset whose `serial` contradicts the description, or whose `mooring_name` differs from `name:` by more than case, raises `MooringError`; a case-only difference is corrected with a `history` line. Position and window keep their placeholders when absent, loudly (`location_source`/`deployment_window_source` = `"none"`, a WARNING). A listed instrument with no file at the requested stage raises unless `allow_missing=True`. - `oceanarray.pipeline.write_mooring.write_mooring(datasets, mooring_dir, *, overwrite=False)` — writes a dict of per-instrument datasets to `//__stage.nc`, taking the name, instrument, serial and stage from each dataset's attributes; refuses a dataset whose `serial` attribute does not match its key. @@ -17,7 +17,7 @@ All notable changes to oceanarray are documented here. - `oceanarray.pipeline._stage` — the `processing_stage` marker: `stamp(ds, stage, *, source=)` writes the marker, a `history` line and `source_file`; `require(datasets, stage, next_stage)` raises `StageError` when a dataset is at the wrong stage, naming the stage to run first. A file written before the marker existed is tolerated for one release (a missing marker on read is a WARNING). - `oceanarray.mooring` — the single seam for reading a mooring description: `load` (path, name + `proc_dir`, or dict; validates and raises `MooringError` on an invalid config), `check` (never raises; adds an unknown-top-level-key warning), `template` and `save`, plus `oceanarray.MooringError`. Every stage, report and CLI path now receives the loaded dict instead of calling `yaml.safe_load` itself; a test (`test_config_sites`) fails if a new mooring-config `safe_load` appears outside the loader. `scripts/survey_mooring_keys.py` lists the top-level keys on a processed-data root that are not yet in `mooring.KNOWN_KEYS`. - `oceanarray.pipeline.stage1.stage1(datasets, mooring)`, `oceanarray.pipeline.stage2.stage2(datasets, mooring)`, `oceanarray.pipeline.stage3.stage3(datasets, mooring, *, caldip_dir=None)`, `oceanarray.pipeline.stack.stack(datasets, mooring, *, dt_seconds=60)` and `oceanarray.pipeline.grid.grid(ds_stack, *, pmin, pmax, dp)` — the processing steps as pure functions over datasets, with no file I/O. `MooringProcessor`, `Stage2Processor`, `Stage3Processor`, `MooringStacker` and `MooringGridder` now read, call these, and write; their outputs are unchanged in value. `oceanarray.pipeline.raw_files` holds stage 1's raw-file finding (`resolve_raw_file`, `guess_instrument_filename`) and reading (`read_raw_file`). -- `oceanarray stamp FILE... --stage N` and `oceanarray.pipeline.stamp.stamp_file(path, stage)` — write the `processing_stage` marker into a stage file written before it existed (`history`: "declared via stamp"). Refuses a stage that contradicts the filename or an existing marker; leaves an already-marked file untouched. +- `oceanarray stamp FILE... --declare N` and `oceanarray.pipeline.stamp.stamp_file(path, stage)` — write the `processing_stage` marker into a stage file written before it existed (`history`: "declared via stamp"). Refuses a stage that contradicts the filename or an existing marker; leaves an already-marked file untouched. - `oceanarray.process(mooring, *, stages=(1, 2, 3), serials=None, force=False, keep_going=True, mooring_dir=None, raw_dir=None, caldip_dir=None) -> list[Path]` — runs stages 1–3 for one mooring from its `*.mooring.yaml` path (or a directory holding one, or a loaded description with `mooring_dir=`), writing each stage's files and a log per stage, and returns every stage file of the stages run — written, or kept because it was already there (`force=True` rewrites; stage 1 also rewrites a file older than its raw file); each stage prints how many it wrote and kept. By default it carries on past a failed instrument (see `keep_going` below); `keep_going=False` raises `MooringError` naming the instruments that failed, after the stage has written the rest, and runs nothing after it. Stacking and gridding are not part of it. The previous form, `process(name, stage, proc_dir=…)` (all five stages, returns a bool), still works for one release (see *Deprecated*). - `oceanarray.run(mooring, *, stages=None, from_stage=None, serials=None, force=False, keep_going=True, mooring_dir=None, raw_dir=None, caldip_dir=None, dt_seconds=None, pmin=None, pmax=None, dp=None, stack_stage=3, report=False) -> list[Path]` — stages 1–3 (through `process`), then the stack (`_stack.nc`) and the grid (`_grid.nc`), rebuilt on every call; returns the stage files, then the stack and grid. `stages` lists the stages to run (default all); because the stack is rebuilt from the stage-3 files, they must run from the first one given through stage 3, so the stack never takes files older than the ones just written — `stages=(2, 3)` (or `from_stage=2`) re-runs stages 2 and 3 from the stage-1 files without `raw_dir`, while `stages=2` or `(1, 3)` is refused before anything runs. To run one stage on its own, use `process`. `stack_stage=2` (CLI `--stack-stage 2`) is a quick look at sea: it runs through stage 2 only and stacks and grids the stage-2 files (no QC flags, no interpolated pressure, no magnetic declination; every level's `source_stage` is 2). By default it carries on past failures (see `keep_going` below). With `keep_going=False` any failure raises, an instrument with no stage-3 file stops the stack, and a stack that left an instrument out is written but not gridded. The stack and grid settings come from the call, else from new optional `stack:` (`dt_seconds`) and `grid:` (`pmin`, `pmax`, `dp`) blocks in the mooring description, else 60 s and 200–1000 dbar every 20 dbar; the merged settings are checked before anything runs, and `mooring.check` refuses a malformed block. When a strict run stops after it has rewritten stage files, or its new stack cannot be gridded, the stack and grid that no longer match are renamed `_stack.stale.nc` / `_grid.stale.nc` (a line says so; one stale copy per product, replaced by the next) — whatever the error that stopped it. - `keep_going` on `process` and `run`, **on by default** — as much as possible on a first pass, troubleshoot later: a failed instrument is logged and left out and the next stage runs on what has files; a stage-3 failure is logged, not raised; `run` stacks each instrument's stage-3 file, else its stage-2 file. What was taken from stage 2 is recorded in the stack's new per-level `source_stage` variable and in a `stage2_instruments` attribute (the grid inherits it); instruments with neither, or that failed in this run (their older files are not used), are named in `instruments_omitted`; after a stage-3 failure every instrument is stacked from stage 2; what failed is recorded in a `keep_going_failures` attribute on the stack and grid; an instrument that failed in one stage is left out of the later stages of the same run (its older files are neither rebuilt from nor used as another instrument's pressure source); a stack that cannot be gridded (e.g. no pressure left) is kept, with a warning and no grid. @@ -40,6 +40,11 @@ All notable changes to oceanarray are documented here. ### Changed +- The command-line help says, for every subcommand, what it reads and writes, with an `Examples:` block; flags follow the family conventions. Renamed (old spellings accepted until 0.7.0 with a note on stderr): `process --report` → `--summary`, `plot --downsample` → `--downsample-seconds`, `export --overwrite` → `--force`, `stamp --stage` → `--declare`. All deprecated spellings now share one mechanism and message ("… is deprecated and removed in 0.7.0; use …"). +- `oceanarray validate` takes a mooring the way the other commands do (its `*.mooring.yaml`, its folder, or a name with `--proc-dir`) and gains `--strict` (exit 1 on warnings too; the default still exits 0 when there are only warnings). +- A usage error (a removed flag, a missing `--raw-dir` or `--proc-dir`, an option that does not combine) exits 2 with an `ERROR:` line on stderr; a processing failure still exits 1. A mooring name with `--proc-dir` whose folder does not exist is an error naming the folder. +- `python -m oceanarray` runs the CLI; `-v` is accepted after the subcommand as well as before it; `export -n/--dry-run` lists the files it would write. +- `process --stage` help marks `stack` and `grid` as deprecated (use `run`), and their tuning flags are hidden from `process --help`. - Private helpers follow one naming table (CONTRIBUTING, "Private helpers"): one verb per job, figure helpers `_draw_` / `_make__b64` with no `_fig` suffix, 85 renamed; their docstrings are one line, with the reasons the code cannot show kept as comments (≈ −1,400 lines). The three private `_safe_serial` wrappers are gone (`paths.safe_serial`). Four dead helpers deleted, the duplicate `_times_to_float` and the three module `_log`s collapsed (into `oceanarray.logger.log_message`), and four one-line wrappers inlined. No public name or behaviour changes; report `source:` lines name the new helpers. - `oceanarray process --dry-run` lists the file every stage would write (new, keep or rewrite) and, for stage 3, which instruments get interpolated pressure; it works by path and needs no `--raw-dir` (before: stage 3 only, by name under `--proc-dir`). The README, the demo notebook and the method pages use the verbs. - The report page switches lose their short forms (`-i`, `-s`, `-g`, `-A`; accepted with a note until 0.7.0): the short flags are `-o` (always an output directory), `-n` (`--dry-run`) and `-v`. @@ -76,6 +81,8 @@ Every old name below still works, warns (`DeprecationWarning`, naming its replac ### Breaking changes +- A mooring description whose QC override blocks (`qc_ranges`, `qc_spike`, `tilt_qc`, `qc_flat_line`, at mooring or instrument level) hold an unknown field, a span that is not two increasing numbers, a threshold that is not a number, or a new variable without a `fail_span` is now refused by `load` and reported as an ERROR by `oceanarray validate`; before, the mistake was dropped and QC ran with the defaults. A variable outside the registry is a WARNING. `qc_spike`, `tilt_qc` and `qc_flat_line` are no longer reported as unknown top-level keys. +- `current_direction` is written with `standard_name = "sea_water_velocity_to_direction"`; CF deprecated `direction_of_sea_water_velocity`. Files written before keep the old name. - Modules moved to the package layout (no aliases: nothing outside oceanarray imported them). Old → new: | old | new | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a1368e95..4ab1683b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -81,7 +81,7 @@ For all other bugs: ### Scope -oceanarray **processes moored-array observations**: raw instrument files from a mooring (MicroCATs, RBRs, Nortek and RDI current meters and ADCPs, SBE 16/56, Seapoint, …) → CF-NetCDF at each stage — stage 1 (raw → NetCDF), stage 2 (deployment window, clock), stage 3 (QC, velocity rotation, calibration-dip offsets applied) — then a stack on a common time axis and a grid on pressure levels, each step documented in an HTML report. The OceanSITES export of the stacked and gridded products is the next in-scope step. +oceanarray **processes moored-array observations**: raw instrument files from a mooring (MicroCATs, RBRs, Nortek and RDI current meters and ADCPs, SBE 16/56, Seapoint, …) → CF-NetCDF at each stage — stage 1 (raw → NetCDF), stage 2 (deployment window, clock), stage 3 (QC, velocity rotation; calibration-dip offsets once the apply step lands — the interface is in place) — then a stack on a common time axis and a grid on pressure levels, each step documented in an HTML report, and an OceanSITES 1.4 export of the stage-3 files and the grid. Exporting the stack is the next in-scope step. **Fixed — not open to variation:** the output contract. Variable names, CF standard names and the canonical unit spellings come from one registry (`config/parameters.py`) and are the same in every file the package writes; QC flags follow the QARTOD/OceanSITES vocabulary; every processing parameter is recorded in the output's attributes so a file can be reprocessed from itself. Contributions that change these change every user's files, and are discussed first, as an issue. @@ -91,7 +91,7 @@ oceanarray **processes moored-array observations**: raw instrument files from a - **Corrections to individual instruments.** How calibration-dip offsets and other per-instrument corrections (drift, pressure-sensor offset) are applied in stage 3 — constant, linear in time, pre/post weighted — is a method choice; the *derivation* of caldip offsets stays in `caldip`, oceanarray applies its output. - **QC tests.** New QARTOD-style tests or thresholds, as long as the flags land in the existing `_qc` companions with their thresholds recorded. - **Diagnostic figures and report panels.** New figures are welcome, and the report is meant to let a user choose their own selection of panels per page (a config-driven profile); contributions that make that selection easier are in scope. -- **Readers.** Instrument readers live in `seasenselib`; oceanarray's stage 1 dispatches on the file type (`config/parameters.INSTRUMENT_FILE_TYPES`) and keeps only a small set of its own readers (`readers/`: OceanSITES, RODB and the old Nortek CSV). A reader for an instrument `seasenselib` does not cover is welcome — but write it in `seasenselib` first and add the file type here; a reader that only oceanarray would ever use can live in `readers/`. (Where exactly that line is drawn is not written down yet — ask.) +- **Readers.** Instrument readers live in `seasenselib`; oceanarray's stage 1 dispatches on the file type (`config/parameters.INSTRUMENT_FILE_TYPES`) and keeps only a small set of its own readers and helpers (`readers/`: its own netCDF, OceanSITES, RODB, and the Nortek and Sea-Bird header parsers). A reader for an instrument `seasenselib` does not cover is welcome — but write it in `seasenselib` first and add the file type here; a reader that only oceanarray would ever use can live in `readers/`. (Where exactly that line is drawn is not written down yet — ask.) **Out of scope, because a sibling owns it:** shipboard CTD processing (`ctdcast`); deriving calibration-dip offsets (`caldip`); glider data (`glidertest`, `seagliderOG1`); transport and AMOC estimates from the gridded products (`AMOCatlas`); general-purpose plotting. Also out: features that need cruise-specific raw data in the repository, heavy new dependencies, or that break the CF-NetCDF contract. If you are unsure whether an idea fits, open an issue describing it rather than deciding alone. @@ -150,7 +150,7 @@ Co-authored-by: Name - **Type annotations**: all public function arguments and return values. - **Lint**: `ruff check .` must be clean. No bare `except:`. No suppressed warnings without a comment explaining why. - **Units**: every registered variable carries the registry's canonical spelling (`config/parameters.VARIABLES[var]["units"]`), and that spelling is identical in every file the package writes — except N², kept as `s-2` or `rad2 s-2` as it arrives (`config/unit_tables.UNIT_SPELLINGS_KEPT`). The pipeline rewrites known spellings through the shared units tables (`config/unit_tables.TABLES`) and refuses unknown ones. A new spelling is added **with evidence** — a reader that emits it — never from memory: to `config/unit_tables.UNIT_ALIAS_EXTRAS` if only oceanarray's readers emit it, or upstream in ctdcast's vendored `units_tables.py` if it belongs to the family. A mass↔volume conversion is never a scalar alias. Use `gsw` for all seawater property calculations. -- **Plot style**: discrete colorbars (`BoundaryNorm`, ≤ 20 levels). Report figures are styled at one choke point — `reports/_encode.render_b64` applies the vendored `report.mplstyle` and `reports/_slots.render` draws each figure at its slot width — so a report plotter does not call `plt.style.use` itself. +- **Plot style**: discrete colorbars (`BoundaryNorm`, ≤ 20 levels). Report figures are styled at one choke point — `reports/_vendored/figure_png.render_b64` applies the vendored `report.mplstyle` and `reports/_slots.render` draws each figure at its slot width — so a report plotter does not call `plt.style.use` itself. - **Private helpers**: a leading underscore keeps a helper out of the public API; it may be shared inside its own subpackage (`reports/`, `cli/`, `pipeline/`). Its docstring is **one line** plus a typed signature; a reason the code cannot show (a units or flag-table detail, a precedence order, an edge case) goes in a `#` comment at the line it explains. The name says what comes back, with one verb per job: | verb | returns | diff --git a/README.md b/README.md index f84e8e83..065ce547 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Python tools for processing moored oceanographic array observations from raw instrument files to quality-controlled, CF-compliant NetCDF. -It covers a single mooring's instruments — stages 1–3, stacking, and pressure gridding, each documented in an HTML report. Deriving calibration-dip offsets is handled by `caldip`, and shipboard CTD processing by `ctdcast`. +It covers a single mooring's instruments — stages 1–3, stacking, and pressure gridding, each documented in an HTML report, and an OceanSITES 1.4 export. One command, `oceanarray run MOORING --raw-dir RAW`, takes a mooring from raw files to the gridded product and its report. Deriving calibration-dip offsets is handled by `caldip`, and shipboard CTD processing by `ctdcast`. **Documentation**: https://ocean-uhh.github.io/oceanarray/ @@ -38,6 +38,7 @@ oceanarray init dsG3_1_2026 --proc-dir /data/proc # edit /data/proc/dsG3_1_2026/dsG3_1_2026.mooring.yaml oceanarray validate /data/proc/dsG3_1_2026/dsG3_1_2026.mooring.yaml oceanarray run /data/proc/dsG3_1_2026 --raw-dir /data/raw # stages 1-3, stack, grid, report +oceanarray export /data/proc/dsG3_1_2026 # OceanSITES 1.4 files (needs platform_code) ``` See the [documentation](https://ocean-uhh.github.io/oceanarray/) for the recommended first-processing workflow, YAML reference, and CLI reference. @@ -50,6 +51,7 @@ import oceanarray as oa yaml_path = "/data/proc/dsG3_1_2026/dsG3_1_2026.mooring.yaml" oa.process(yaml_path, raw_dir="/data/raw") # stages 1-3 oa.run(yaml_path, from_stage=3, report=True) # re-run stage 3, stack, grid, report +oa.export(yaml_path) # OceanSITES 1.4 files in /oceansites/ ``` See the [API reference](https://ocean-uhh.github.io/oceanarray/oceanarray.html) for the full interface. diff --git a/docs/source/cli_reference.rst b/docs/source/cli_reference.rst index cd2085b0..36ad7755 100644 --- a/docs/source/cli_reference.rst +++ b/docs/source/cli_reference.rst @@ -4,13 +4,16 @@ CLI Reference =================== -The typical workflow is: validate the YAML, run ``oceanarray process`` stage by -stage (including the ``stack`` and ``grid`` stages), inspect with -``oceanarray report``, then use ``oceanarray run`` to reproduce the full pipeline -in one command. +The typical workflow is: ``oceanarray init`` a mooring description and fill it +in, ``oceanarray validate`` it, run ``oceanarray process`` stage by stage and look +at the stage files with ``oceanarray report``, then ``oceanarray run`` for the +whole pipeline (stages 1–3, stack, grid, report) in one command, and +``oceanarray export`` to write OceanSITES files. ``python -m oceanarray`` works +as well as ``oceanarray``. -``process``, ``run``, ``report`` and ``animate`` take the mooring as the first -positional argument, in either of two forms: +``validate``, ``process``, ``run``, ``report``, ``export``, ``plot`` and +``animate`` take the mooring as the first positional argument, in either of two +forms: - the path to its ``*.mooring.yaml``, or to the folder holding it — the folder can have any name, and every file is named from the description's ``name:``; @@ -22,14 +25,19 @@ positional argument, in either of two forms: oceanarray run dsG3_1_2026 --proc-dir /data/proc --raw-dir /data/raw # the same ``--only SERIAL ...`` restricts a command to those instruments (``--serial`` is the -old spelling, accepted until 0.7.0). ``-v`` also prints the processing stages' -debug messages. +old spelling, accepted until 0.7.0). ``-v`` (before or after the command name) +also prints the processing stages' debug messages. Without ``--force``, the stages keep a file that already exists (stage 1 also rewrites one older than its raw file) and the report pages keep one newer than its inputs; add ``--force`` to rewrite. ``oceanarray run`` rebuilds the stack and grid every time. +**Exit status**: 0 when the command did its work (``process`` and ``run`` also +when instruments were left out, unless ``--strict``); 1 when it failed, or +``validate`` found an ERROR; 2 for a usage error (an unknown flag, or a missing one +such as ``--raw-dir`` for stage 1). + The flag lists below are generated from the argument parser (:func:`oceanarray.cli.build_parser`), so they cannot drift from the code. @@ -38,10 +46,11 @@ The flag lists below are generated from the argument parser ``oceanarray validate`` ------------------------ -Check one or more YAML configuration files for missing required fields, -unrecognised values, and structural errors. Does not require ``--raw-dir`` -or ``--proc-dir``. Run this before any processing to catch typos in -``file_type``, missing ``hab``, or malformed serial numbers. +Check one or more mooring descriptions for missing fields, unrecognised values and +structural errors, including the QC override blocks and the OceanSITES codes (see +:doc:`yaml_configuration`). Run it before any processing to catch typos in +``file_type``, a missing ``hab``, a malformed serial number or a misspelt QC +threshold. .. argparse:: :module: oceanarray.cli @@ -49,6 +58,7 @@ or ``--proc-dir``. Run this before any processing to catch typos in :prog: oceanarray :path: validate :nodescription: + :noepilog: **Example** @@ -56,8 +66,8 @@ or ``--proc-dir``. Run this before any processing to catch typos in oceanarray validate /data/proc/dsG3_1_2026/dsG3_1_2026.mooring.yaml -Prints errors and warnings to the console. Exits with a non-zero status -if any errors are found. +Prints every ERROR and WARNING. Exits 1 if any description has an ERROR, or with +``--strict`` a WARNING; 0 otherwise. ---- @@ -76,12 +86,13 @@ refused; a file already carrying the marker is left alone. :prog: oceanarray :path: stamp :nodescription: + :noepilog: **Example** .. code-block:: bash - oceanarray stamp /data/proc/dsG3_1_2026/*/dsG3_1_2026_*_stage3.nc --stage 3 + oceanarray stamp /data/proc/dsG3_1_2026/*/dsG3_1_2026_*_stage3.nc --declare 3 Exits with a non-zero status if any file was refused. @@ -101,6 +112,7 @@ once, before ``oceanarray validate``. :prog: oceanarray :path: init :nodescription: + :noepilog: **Example** @@ -133,6 +145,7 @@ The recommended sequence for a new mooring is: :prog: oceanarray :path: process :nodescription: + :noepilog: **Stage descriptions** @@ -142,10 +155,14 @@ The recommended sequence for a new mooring is: - **Stage 2**: trims to ``deployment_time`` / ``recovery_time`` from the YAML and applies clock drift correction. Writes ``{mooring}_{serial}_stage2.nc``. -- **Stage 3**: applies QARTOD gross-range QC flags, derives salinity, +- **Stage 3**: runs the QC tests (gross range, spike, flat line, tilt; see + :doc:`methods/auto_qc`), derives salinity, interpolates pressure for instruments without a pressure port, rotates Aquadopp velocities to earth coordinates, and applies magnetic declination correction. Writes ``{mooring}_{serial}_stage3.nc``. + +``oceanarray run`` then builds the two mooring-level products: + - **stack**: interpolates all instruments onto a common time axis and writes ``{mooring}_stack.nc``. Instruments sampling faster than ``--dt-seconds`` are subsampled by nearest-neighbour; slower instruments are interpolated linearly. @@ -204,6 +221,7 @@ description of what each report page contains. :prog: oceanarray :path: report :nodescription: + :noepilog: **Output created** @@ -258,6 +276,7 @@ and a quick report inspection). :prog: oceanarray :path: run :nodescription: + :noepilog: **Output created** @@ -273,6 +292,37 @@ reports. ---- +``oceanarray export`` +--------------------- + +Write a mooring's stage-3 files and grid as OceanSITES 1.4 files (see +:doc:`methods/oceansites_export`): one deployment file per instrument and one +gridded file. The mooring description needs ``platform_code``; ``--array`` adds +the array's ``site_code`` and ``project``. An instrument that cannot be exported +is listed and left out; the command exits 1 only when nothing was written. + +.. argparse:: + :module: oceanarray.cli + :func: build_parser + :prog: oceanarray + :path: export + :nodescription: + :noepilog: + +**Output created** + +``OS____--m.nc`` per instrument and +``OS_-_-_GRD_-DP.nc`` for the grid, in +``{proc_dir}/{mooring}/oceansites/`` (or ``-o DIR``). + +**Example** + +.. code-block:: bash + + oceanarray export dsG3_1_2026 --proc-dir /data/proc --array /data/proc/MIXSED2026.array.yaml + +---- + ``oceanarray list`` -------------------- @@ -289,6 +339,7 @@ Does not require a mooring name, ``--raw-dir``, or ``--proc-dir``. :prog: oceanarray :path: list :nodescription: + :noepilog: **Example** @@ -316,6 +367,7 @@ Show an interactive overview plot of processed data, or save it to a file. :prog: oceanarray :path: plot :nodescription: + :noepilog: **Example** @@ -339,6 +391,7 @@ be installed for GIF output. :prog: oceanarray :path: animate :nodescription: + :noepilog: **Example** @@ -351,6 +404,42 @@ be installed for GIF output. Deprecated flags ---------------- +Renamed in 0.6.0 +~~~~~~~~~~~~~~~~ + +The old spellings still work until 0.7.0, with a warning naming the new one. + +.. list-table:: + :header-rows: 1 + + * - Command + - Old + - New + * - ``process``, ``run``, ``report``, ``animate`` + - ``--serial`` + - ``--only`` + * - ``run``, ``process`` + - ``--dt`` + - ``--dt-seconds`` + * - ``report`` + - ``-i``, ``-s``, ``-g``, ``-A`` + - ``--instruments``, ``--stack``, ``--grid``, ``--all`` + * - ``process`` + - ``--report`` + - ``--summary`` + * - ``stamp`` + - ``--stage`` + - ``--declare`` + * - ``export`` + - ``--overwrite`` + - ``--force`` + * - ``plot`` + - ``--downsample`` + - ``--downsample-seconds`` + * - ``plot`` + - ``--var_y``, ``--var_color`` + - ``--var-y``, ``--var-color`` + ``--basedir DIR`` (removed) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/clock_offset.rst b/docs/source/clock_offset.rst index 1be5624a..de0c34c0 100644 --- a/docs/source/clock_offset.rst +++ b/docs/source/clock_offset.rst @@ -43,7 +43,7 @@ This approach is sensitive to smaller timing errors (seconds to minutes) and wor Implementation -------------- -The analysis is implemented in the ``oceanarray.clock_offset`` module with the following key functions: +The analysis is implemented in the ``oceanarray.pipeline.clock.offsets`` module with the following key functions: Core Functions ~~~~~~~~~~~~~~ diff --git a/docs/source/directory_structure.rst b/docs/source/directory_structure.rst index 7646920c..c63b44eb 100644 --- a/docs/source/directory_structure.rst +++ b/docs/source/directory_structure.rst @@ -90,12 +90,14 @@ grid) and ``oceanarray report``. │ └── ... ├── {mooring}_stack.nc ← all instruments on a common time grid ├── {mooring}_grid.nc ← vertically interpolated onto a pressure grid + ├── oceansites/ ← oceanarray export: OS_*.nc files ├── processing_logs/ │ └── {mooring}_{timestamp}_{stage}.mooring.log └── report/ ├── {mooring}_report.html ← mooring summary report ├── {mooring}_stack_report.html ├── {mooring}_grid_report.html + ├── {mooring}_report.pdf ← report --pdf: all pages as one A4 PDF └── instrument/ └── {mooring}_{serial}_report.html @@ -110,8 +112,9 @@ that mooring:: {proc_dir}/{mooring_name}/{mooring_name}.mooring.yaml The ``{mooring_name}`` component is the value of the ``name`` field at the -top of the YAML file (e.g. ``name: dsG3_1_2026``). The directory name, -the YAML filename, and the ``name`` field must all match exactly. +top of the YAML file (e.g. ``name: dsG3_1_2026``). Keep the directory name, +the YAML filename and the ``name`` field identical: every output file is named from +``name``, and ``oceanarray validate`` warns when the directory differs from it. Oceanarray looks for the YAML in this location when you supply ``--proc-dir``. You do not need to place a copy in the raw directory. diff --git a/docs/source/file_naming_conventions.rst b/docs/source/file_naming_conventions.rst index 629bdc00..c065fe9a 100644 --- a/docs/source/file_naming_conventions.rst +++ b/docs/source/file_naming_conventions.rst @@ -104,14 +104,20 @@ serial number from the YAML. - HTML report for the gridded dataset * - ``{mooring}_{serial}_report.html`` - Per-instrument HTML report + * - ``{mooring}_report.pdf`` + - All the report pages as one A4 PDF (``oceanarray report --pdf``) + * - ``oceansites/OS_{platform}_{YYYYMM}_{mode}_{type}-{serial}-{depth}m.nc`` + - OceanSITES 1.4 deployment file per instrument (``oceanarray export``) + * - ``oceansites/OS_{platform}_{YYYYMM}_{mode}_GRD_{interval}-DP{step}.nc`` + - OceanSITES 1.4 gridded file from ``{mooring}_grid.nc`` The ``{serial}`` component uses the value of the ``serial`` field in the YAML instrument entry. If the serial contains a trailing ``*`` (used in some instrument identifiers), the ``*`` is stripped from the filename. The ``{mooring}`` component is the value of the ``name`` field at the top -of the mooring YAML, which must also match the directory name under -``{proc_dir}`` and the YAML filename itself:: +of the mooring YAML. Keep the directory name under ``{proc_dir}`` and the YAML +filename the same as it (``oceanarray validate`` warns when the directory differs):: {proc_dir}/{mooring}/{mooring}.mooring.yaml diff --git a/docs/source/index.rst b/docs/source/index.rst index a7807478..98bbb208 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -58,6 +58,7 @@ Contents Stack into one NetCDF Grid onto pressure + Export to OceanSITES .. toctree:: :maxdepth: 1 @@ -91,7 +92,6 @@ Contents roadmap style_guide - OceanSITES export Combine Deployments Multi-site Merging diff --git a/docs/source/legacy.rst b/docs/source/legacy.rst index 4e6f3b5e..e7b9d90a 100644 --- a/docs/source/legacy.rst +++ b/docs/source/legacy.rst @@ -41,8 +41,17 @@ active processing code: Its YAML inputs live beside it: ``OS1_variable_names.yaml``, ``OS1_variable_attributes.yaml`` and ``OS1_sensor_attributes.yaml``. -``rapid_interp.py`` (top-level) - RAPID-specific interpolation helpers — not part of the active pipeline. +``qc_rodb.py`` + RODB-era QC flaggers for datasets with OceanSITES-style uppercase variables + (``TEMP``, ``CNDC``, ``PSAL``). + +``caldip.py`` + The earlier in-package calibration-dip analysis — superseded by the separate + ``caldip`` tool, whose per-cast netCDF Stage 3 reads (see :doc:`methods/calibration`). + +The RAPID climatological-gradient gridding routine is not legacy: it lives in +``oceanarray/pipeline/gridders/rapid_interp.py`` and is waiting to be wired into +``grid`` as a selectable method. Demo notebooks for the legacy workflow live in ``notebooks/legacy/`` and are kept for reference only. diff --git a/docs/source/methods/auto_qc.rst b/docs/source/methods/auto_qc.rst index fab718c8..b6e5d216 100644 --- a/docs/source/methods/auto_qc.rst +++ b/docs/source/methods/auto_qc.rst @@ -85,14 +85,18 @@ Stage 3 applies the following tests, in order: 4 (bad); values outside ``[suspect_span[0], suspect_span[1]]`` are flagged 3 (suspect). Note: ``fail_span`` defines the *pass range* — values outside it fail. -2. **Spike test**: detects single-sample spikes larger than the ``qc_spike`` threshold. - Flagged 3 (suspect). +2. **Spike test**: detects single-sample spikes larger than the ``qc_spike`` + thresholds: above ``suspect_threshold`` flagged 3, above ``fail_threshold`` 4. -3. **Tilt QC** (Aquadopp only): velocity variables are flagged when the combined +3. **Flat-line test**: flags a stuck sensor, a value that does not change by more + than ``tolerance`` over ``suspect_n`` (flag 3) or ``fail_n`` (flag 4) samples + (``qc_flat_line``; by default on pressure). + +4. **Tilt QC** (Aquadopp only): velocity variables are flagged when the combined pitch/roll angle exceeds a threshold. ADCP instruments use ``error_velocity`` QC instead (not a tilt test). -4. **Pressure interpolation flag**: pressure values that were not measured but +5. **Pressure interpolation flag**: pressure values that were not measured but interpolated from neighbouring instruments are flagged 8 (interpolated). QC is applied **after** pressure interpolation so that interpolated pressures also @@ -258,8 +262,11 @@ exceeds the threshold: Configuration ------------- -QC ranges are configured in the ``qc_ranges`` block of the mooring YAML, under each -instrument entry. If absent, global defaults are used. +QC ranges are configured in the ``qc_ranges`` block of the mooring YAML, at mooring +level or under an instrument entry (which wins for its own variables). If absent, +the package defaults are used. The spike, flat-line and tilt tests take the +``qc_spike``, ``qc_flat_line`` and ``tilt_qc`` blocks; all four are checked when +the description is read (see :doc:`../yaml_configuration`, "Other QC blocks"). .. code-block:: yaml @@ -328,10 +335,18 @@ these attributes: ds["temperature_qc"].attrs == { "long_name": "quality flag for temperature", - "flag_values": [1, 3, 4, 8, 9], - "flag_meanings": "good_data potentially_correctable_bad_data bad_data interpolated_value missing_value", - "fail_span": [-2.0, 35.0], - "suspect_span": [-1.5, 32.0], + "standard_name": "sea_water_temperature status_flag", + "flag_values": [0, 1, 2, 3, 4, 7, 8, 9], # int8, OceanSITES reference table 2 + "flag_meanings": "unknown good_data probably_good_data " + "potentially_correctable_bad_data bad_data nominal_value interpolated_value " + "missing_value", + "conventions": "OceanSITES reference table 2", + "qc_gross_range_fail_min": -2.5, + "qc_gross_range_fail_max": 40.0, + "qc_gross_range_suspect_min": -2.0, + "qc_gross_range_suspect_max": 35.0, + "qc_spike_suspect_threshold": 2.0, + "qc_spike_fail_threshold": 6.0, } The applied thresholds are stored as attributes so the exact QC configuration can be diff --git a/docs/source/methods/nortek_coordinate_transform.rst b/docs/source/methods/nortek_coordinate_transform.rst index ee50760e..f81fc0c2 100644 --- a/docs/source/methods/nortek_coordinate_transform.rst +++ b/docs/source/methods/nortek_coordinate_transform.rst @@ -288,4 +288,4 @@ See also - :doc:`auto_qc` — QARTOD flags applied to velocity after tilt QC - :py:func:`oceanarray.pipeline.coordinate.xyz_to_enu` — vectorised ENU rotation - :py:func:`oceanarray.pipeline.coordinate.apply_beam_to_enu` — full BEAM/XYZ → ENU pipeline -- :py:func:`oceanarray.pipeline.stage1._parse_nortek_T_matrix_hdr` — header parsing +- :py:func:`oceanarray.readers.nortek._parse_nortek_T_matrix_hdr` — header parsing diff --git a/docs/source/methods/oceansites_export.rst b/docs/source/methods/oceansites_export.rst index bbc29cd5..0b5ce50c 100644 --- a/docs/source/methods/oceansites_export.rst +++ b/docs/source/methods/oceansites_export.rst @@ -1,16 +1,41 @@ OceanSITES export ================= -.. note:: - In development. This page describes what the export writes; the converter is being - built against it. - oceanarray writes its products as `OceanSITES 1.4 `_ files (see :doc:`../oceanSITES_manual`): ``format_version = "1.4"``, ``Conventions`` naming CF, OceanSITES-1.4 and ACDD. The files follow the format; they are not yet ready to submit to a GDAC, which needs a ``site_code`` approved by the OceanSITES Project Office. +Running the export +------------------ + +After stage 3 (and the grid, if one is wanted), from the command line: + +.. code-block:: bash + + oceanarray export /data/proc/dune2_1_2026 --array /data/proc/MIXSED2026.array.yaml + oceanarray export dune2_1_2026 --proc-dir /data/proc -n # list the file names only + +or from Python: + +.. code-block:: python + + import oceanarray as oa + + oa.export("/data/proc/dune2_1_2026/dune2_1_2026.mooring.yaml", + array="/data/proc/MIXSED2026.array.yaml") + +The files go to ``/oceansites/`` (``-o DIR`` / ``out_dir=`` to change it). +An existing file is kept unless ``--force`` (``overwrite=True``) is given. An instrument +that cannot be exported is listed and left out; the others are written. ``--no-grid`` +(``grid=False``) leaves the grid out. + +The mooring YAML needs ``platform_code``; ``deployment_code`` (``YYYYMM``) is optional and +taken from ``deployment_time`` when absent — ``oceanarray validate`` warns when the two +disagree. The array YAML (``--array``) supplies ``site_code`` and ``project``. See +:doc:`../yaml_configuration`. + Products -------- @@ -56,9 +81,7 @@ The grid stays on its pressure levels: ``PRES(DEPTH)`` holds the levels, and ``D each level converted with ``gsw.z_from_p`` at the mooring latitude, so nothing is re-interpolated. Its ``history`` names the grid and the stage files the stack was built from (manual §4.2.1). A gridded file carries no ``_QC`` variables (§4.2.1: only good -source data went into it). A grid records no per-input data modes, so only a -provisional (``P``) grid is exported; until a grid can record them, any other mode is -refused. +source data went into it). Data mode --------- diff --git a/docs/source/methods/time_gridding.rst b/docs/source/methods/time_gridding.rst index f38eab88..c66503e3 100644 --- a/docs/source/methods/time_gridding.rst +++ b/docs/source/methods/time_gridding.rst @@ -114,7 +114,7 @@ Output Dimensions ^^^^^^^^^^ -``(N_LEVELS, time)`` following the OceanSITES convention. ``N_LEVELS`` is the instrument +``(time, N_LEVELS)`` following the OceanSITES convention. ``N_LEVELS`` is the instrument index (deepest first); ``time`` is the common time axis at ``dt_seconds`` resolution. Coordinates diff --git a/docs/source/methods/trimming.rst b/docs/source/methods/trimming.rst index bb05e302..fce7c480 100644 --- a/docs/source/methods/trimming.rst +++ b/docs/source/methods/trimming.rst @@ -64,13 +64,13 @@ Run Stage 2 for all instruments: .. code-block:: bash - oceanarray process dsG3_1_2026 --raw-dir /data/raw --proc-dir /data/proc --stage 2 + oceanarray process dsG3_1_2026 --proc-dir /data/proc --stage 2 Rerun after adjusting deployment times (single instrument): .. code-block:: bash - oceanarray process dsG3_1_2026 --raw-dir /data/raw --proc-dir /data/proc --stage 2 --only 26261 --force + oceanarray process dsG3_1_2026 --proc-dir /data/proc --stage 2 --only 26261 --force 4. Output format ---------------- diff --git a/docs/source/methods/vertical_gridding.rst b/docs/source/methods/vertical_gridding.rst index 9ba7fea1..62f5c574 100644 --- a/docs/source/methods/vertical_gridding.rst +++ b/docs/source/methods/vertical_gridding.rst @@ -1,7 +1,7 @@ Grid: Vertical Pressure Grid ============================= -The ``grid`` stage linearly interpolates the stacked ``(N_LEVELS, time)`` +The ``grid`` stage linearly interpolates the stacked ``(time, N_LEVELS)`` dataset onto a regular pressure grid, producing ``(time, pressure)`` output suitable for quick T-S section plots and density diagnostics. @@ -86,7 +86,7 @@ Dimensions Variables gridded ^^^^^^^^^^^^^^^^^ -All ``(N_LEVELS, time)`` variables present in the stack file, except ``pressure`` itself, +All ``(time, N_LEVELS)`` variables present in the stack file, except ``pressure`` itself, are interpolated onto the pressure axis. This includes derived quantities such as ``sigma0`` or ``sigma2`` computed at the stack step. diff --git a/docs/source/migration.rst b/docs/source/migration.rst index 3f160929..856f9a11 100644 --- a/docs/source/migration.rst +++ b/docs/source/migration.rst @@ -39,7 +39,7 @@ See :doc:`migrating_0_6_0`. the stage that produced them, and reading a stage file checks it. A file written before 0.6 has no marker: it is still read, with a WARNING, and its stage inferred from its filename. 0.7.0 will refuse unmarked files; mark existing ones once with - ``oceanarray stamp --stage N`` (it writes only the marker, and refuses a + ``oceanarray stamp --declare N`` (it writes only the marker, and refuses a file whose name says another stage). - **A stage file made under a different mooring name is refused.** Reading a stage file whose ``mooring_name`` differs from the description's ``name:`` by more than diff --git a/docs/source/oceanSITES_manual.rst b/docs/source/oceanSITES_manual.rst index f41a85f3..dec844e2 100644 --- a/docs/source/oceanSITES_manual.rst +++ b/docs/source/oceanSITES_manual.rst @@ -1078,7 +1078,7 @@ For example, if sea temperature on a mooring is measured by a series of 5 Microc * - CAPH - air_pressure * - CDIR - - direction_of_sea_water_velocity + - direction_of_sea_water_velocity (CF has deprecated this name; oceanarray writes ``sea_water_velocity_to_direction``) * - CNDC - sea_water_electrical_conductivity * - CSPD diff --git a/docs/source/oceanarray.rst b/docs/source/oceanarray.rst index e2664107..c02ff35c 100644 --- a/docs/source/oceanarray.rst +++ b/docs/source/oceanarray.rst @@ -36,6 +36,9 @@ The verbs .. autofunction:: oceanarray.report :no-index: +.. autofunction:: oceanarray.export + :no-index: + On datasets ^^^^^^^^^^^ @@ -285,7 +288,7 @@ ADCP QC steps: bins in the seabed or surface sidelobe zone, and velocity checks. stamp ^^^^^ Mark stage files written before the ``processing_stage`` marker existed -(``oceanarray stamp FILE... --stage N``). +(``oceanarray stamp FILE... --declare N``). .. automodule:: oceanarray.pipeline.stamp :members: diff --git a/docs/source/processing_framework.rst b/docs/source/processing_framework.rst index cbd470f5..74c2a8ad 100644 --- a/docs/source/processing_framework.rst +++ b/docs/source/processing_framework.rst @@ -60,13 +60,14 @@ The instrument-level processing carries out the following steps: magnetic declination correction. (Stage 1 applies the prior BEAM→XYZ step using the instrument T matrix from the ``.hdr`` file.) 7. Tilt QC for Aquadopps: velocity variables flagged suspect or bad when pitch/roll - exceed configurable thresholds (default 20°/30°). For RDI ADCPs with four + exceed configurable thresholds (default 30°/50°; ``tilt_qc``). For RDI ADCPs with four beams, ``error_velocity`` is used for QC instead. Output: ``_stage3.nc``. Applied QC thresholds are stored as attributes on each ``*_qc`` variable so the exact configuration can be recovered from the file. -- **Stage 4 (planned):** Full export to OceanSITES format with rich metadata. +- **Export:** ``oceanarray export`` writes the stage-3 files and the grid as + OceanSITES 1.4 files; see :doc:`methods/oceansites_export`. .. note:: @@ -123,8 +124,9 @@ mooring are combined: .. note:: - The stack step may include optional low-pass filtering to remove tides before - subsampling. This is controlled by YAML parameters; see :doc:`yaml_configuration`. + The stack does no filtering: a fast instrument is subsampled (nearest sample in + time) and a slow one interpolated linearly. Low-pass filtering, if wanted, is done + on the stage-3 files before stacking. .. admonition:: RAPID Analogy :class: hint @@ -136,7 +138,7 @@ mooring are combined: **Further details:** -- :doc:`methods/time_gridding` — low-pass filtering and common time axis. +- :doc:`methods/time_gridding` — the common time axis. - :doc:`methods/vertical_gridding` — pressure-grid interpolation. - :doc:`methods/concatenation` — joining multiple deployments. @@ -194,8 +196,8 @@ Summary table - QC & rotation - Pressure interpolation; QARTOD QC flags; salinity; velocity rotation (optional caldip T/C/P correction at the front — planned) * - 4 - - OceanSITES export (planned) - - Full export to OceanSITES format with rich metadata + - OceanSITES export + - Stage-3 files and the grid as OceanSITES 1.4 files (``oceanarray export``) * - A - Stack - Resample all instruments onto a common time axis; stack with depth dimension diff --git a/docs/source/project_structure.md b/docs/source/project_structure.md index 30f6eef1..7df13a93 100644 --- a/docs/source/project_structure.md +++ b/docs/source/project_structure.md @@ -9,7 +9,7 @@ This document provides an overview of the oceanarray codebase structure and orga ``` oceanarray/ ├── oceanarray/ # Main Python package -│ ├── __init__.py # Public API: the verbs (read, stage1–stage3, process, stack, grid, run, write, report) +│ ├── __init__.py # Public API: the verbs (read, read_mooring, stage1–stage3, process, stack, grid, run, write, write_mooring, report, export) │ ├── _version.py # Package version (auto-generated by setuptools-scm) │ ├── _deprecated.py # The one table of deprecated names and their replacements │ ├── mooring.py # The *.mooring.yaml: load / check / template / save @@ -27,6 +27,12 @@ oceanarray/ │ │ ├── stage3.py # Stage 3: pressure interpolation + QC + ENU (*_stage3.nc) │ │ ├── stack.py # stack(): instruments → common time grid (*_stack.nc) │ │ ├── grid.py # grid(): stack → regular pressure grid (*_grid.nc) +│ │ ├── export.py # export(): stage-3 files and grid → OceanSITES 1.4 files +│ │ ├── oceansites.py # to_oceansites(): the conversion export uses +│ │ ├── read_mooring.py # read_mooring(): every instrument of a mooring, by stage +│ │ ├── write_mooring.py # write_mooring(): a dict of datasets → stage files +│ │ ├── raw_files.py # Finding and reading each instrument's raw file +│ │ ├── stamp.py # Marking pre-0.6 stage files with their stage │ │ ├── pressure.py # Pressure interpolation for stage 3 │ │ ├── coordinate.py # BEAM→XYZ→ENU transforms │ │ ├── units.py # Canonical units @@ -105,7 +111,7 @@ runs any of stages 1–3; `oceanarray.run` adds the stack and grid. ### Stage 1 — Standardisation (`pipeline/stage1.py`) - **Input**: raw instrument files (`.cnv`, `.rsk`, `.dat`, `.hex`, RDI raw) -- **Output**: `{proc_dir}/{mooring}/{mooring}_{serial}_stage1.nc` +- **Output**: `{proc_dir}/{mooring}/{instrument}/{mooring}_{serial}_stage1.nc` - Faithful to the raw data — no QC, no trimming. Stores the transformation matrix and coordinate system so later stages can rotate correctly. @@ -117,17 +123,17 @@ runs any of stages 1–3; `oceanarray.run` adds the stack and grid. ### Stage 3 — QC, rotation, derived variables (`pipeline/stage3.py`) - **Input**: `*_stage2.nc` - **Output**: `*_stage3.nc` -- Gross-range and tilt QC flags, BEAM→ENU rotation (Aquadopp), magnetic declination +- QC flags (gross range, spike, flat line, tilt), BEAM→ENU rotation (Aquadopp), magnetic declination correction, salinity, density. ### Stack — multi-instrument coordination (`pipeline/stack.py`) - **Input**: `*_stage3.nc` files for all instruments on a mooring -- **Output**: `{mooring}_stack.nc` — `(N_LEVELS, time)` Dataset +- **Output**: `{mooring}_stack.nc` — `(time, N_LEVELS)` Dataset - Aligns instruments onto a common time axis; HAB-ordered deepest-first (index 0). ### Grid — vertical gridding (`pipeline/grid.py`) - **Input**: `*_stack.nc` -- **Output**: `{mooring}_grid.nc` — `(N_LEVELS, time)` on uniform pressure levels +- **Output**: `{mooring}_grid.nc` — `(time, pressure)` on uniform pressure levels - Simple 1-D linear interpolation at each time step (preliminary; no objective mapping). --- @@ -147,18 +153,21 @@ Raw files *_stage3.nc (QC flagged, ENU velocities, salinity/density) │ ▼ oceanarray run (stack) -{mooring}_stack.nc (N_LEVELS × time) +{mooring}_stack.nc (time × N_LEVELS) │ ├──▶ oceanarray report → HTML reports │ ▼ oceanarray run (grid) -{mooring}_grid.nc (uniform pressure × time) +{mooring}_grid.nc (time × uniform pressure) │ - └──▶ oceanarray report → grid HTML report + ├──▶ oceanarray report → grid HTML report + │ + ▼ oceanarray export +OS_*.nc (OceanSITES 1.4: one per instrument, and the grid) ``` -The full pipeline runs in one command: `oceanarray process MOORING --stage 1 2 3 stack grid` -(or `oceanarray run MOORING`). +The full pipeline (stages 1–3, stack, grid, report) runs in one command: +`oceanarray run MOORING`. --- diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst index 2ad5e9f9..d15497cf 100644 --- a/docs/source/quickstart.rst +++ b/docs/source/quickstart.rst @@ -311,6 +311,22 @@ See :doc:`reports` for a description of what each report contains. ---- +Export to OceanSITES +-------------------- + +To share the products as OceanSITES 1.4 files, add ``platform_code`` (the station, +e.g. ``DSG3``) to the mooring YAML, then: + +.. code-block:: bash + + oceanarray export dsG3_1_2026 --proc-dir /data/cruise2026/proc -n # list the names + oceanarray export dsG3_1_2026 --proc-dir /data/cruise2026/proc + +Each instrument's stage-3 file and the grid are written to +``/data/cruise2026/proc/dsG3_1_2026/oceansites/``. See :doc:`methods/oceansites_export`. + +---- + From Python ----------- @@ -323,6 +339,7 @@ The same steps from Python: yaml_path = "/data/cruise2026/proc/dsG3_1_2026/dsG3_1_2026.mooring.yaml" oa.process(yaml_path, stages=1, raw_dir="/data/cruise2026/raw") # stage 1 oa.run(yaml_path, raw_dir="/data/cruise2026/raw", report=True) # everything + oa.export(yaml_path) # OceanSITES files See :doc:`oceanarray` for the full API and :doc:`migrating_0_6_0` when updating code written for 0.5. diff --git a/docs/source/roadmap.rst b/docs/source/roadmap.rst index b1588e28..dd612a1e 100644 --- a/docs/source/roadmap.rst +++ b/docs/source/roadmap.rst @@ -9,34 +9,32 @@ Development Roadmap Status Overview =============== -✅ **Implemented & Working** - -- Stage 1: Standardisation (``stage1.py``) — sbe-cnv, sbe-asc, sbe-ascii, nortek-ascii, - nortek-csv, rbr-rsk, rbr-dat, rbr-hex, rdi-raw (via dolfyn) -- Stage 2: Clock correction + deployment trim (``stage2.py``) -- Stage 3: QARTOD gross-range + spike tests, tilt QC, ADCP seabed/surface QC, - BEAM→ENU rotation (Aquadopp), magnetic declination correction, salinity + density - (``stage3.py``) -- Stack: all instruments → common time axis → ``(N_LEVELS, time)`` NC (``mooring_level.py``) -- Grid: stacked data → regular pressure levels (``mooring_level.py``) -- HTML reports — mooring summary, per-instrument pages, stack report, grid report - (``report/`` package: ``_mooring.py``, ``_instrument.py``, ``_stack.py``, ``_grid.py``) -- Clock offset analysis (``clock_offset.py``) -- Multi-instrument overview plots (``plotters/`` package) -- YAML validation (``validation.py``) -- Configurable logging system - -🟡 **Partially Implemented** - -- Caldip / calibration comparison: ``castB1_detailed_statistics.csv`` format confirmed - (columns: serial, instrument_type, bl_press, temp_diff/std, cond_diff/std, press_diff/std, - status, date, time_start/end, CTD and instrument values, N, label). - Report integration and YAML linkage to cast files still needed — see Priority 1, item 3. - -❌ **Not Yet Implemented** - -- Calibration correction from caldip casts (optional step at the front of Stage 3) -- Stage 4: OceanSITES format conversion +**Implemented** + +- Stage 1: raw file → CF-NetCDF (``pipeline/stage1.py``); formats read through + ``seasenselib`` (run ``seasenselib list`` for the current list), RDI through dolfyn +- Stage 2: deployment trim and linear clock-drift correction (``pipeline/stage2.py``) +- Stage 3: QARTOD gross-range, spike and flat-line tests, tilt QC, ADCP seabed/surface QC, + BEAM→ENU rotation (Aquadopp), magnetic declination correction, salinity and density + (``pipeline/stage3.py``, ``pipeline/qc/``) +- Stack: all instruments on a common time axis, ``(time, N_LEVELS)`` (``pipeline/stack.py``) +- Grid: the stack on regular pressure levels, ``(time, pressure)`` (``pipeline/grid.py``) +- OceanSITES 1.4 export of the stage-3 files and the grid (``pipeline/export.py``, + ``pipeline/oceansites.py``; ``oceanarray export``) +- HTML reports — mooring summary, per-instrument, stack and grid pages, and one A4 PDF + (``reports/``) +- Clock offset analysis (``pipeline/clock/offsets.py``) +- Multi-instrument overview plots (``plotters/``) +- Mooring description validation (``oceanarray validate``, ``mooring.check``) + +**Partly implemented** + +- Calibration correction from caldip casts: the interface and a null-action hook are in + ``pipeline/qc/calibration.py``; applying the offsets is not yet written. + +**Not yet implemented** + +- OceanSITES export of the stack file - Concatenation of multiple deployments at a single location - Multi-site merging for boundary profiles @@ -45,93 +43,81 @@ Status Overview Priority 1: Near-term ===================== -1. Stage 3: Additional QC tests --------------------------------- - -**Currently implemented** (via ``ioos_qc``): gross-range and spike tests on scalar -variables; tilt QC, ADCP seabed/surface QC, and ENU velocity QC for multi-dimensional -variables. - -**Planned additional tests:** - -- Flat-line / stuck-sensor test -- Rate-of-change test -- Climatological range check (season-aware, e.g. from World Ocean Atlas) -- Spike threshold scaling by sampling interval - -**Post-OdB refactor note**: as the number of QC functions in ``stage3.py`` grows, -consider splitting into a ``stage3/`` sub-package with a dedicated ``stage3/qc.py`` -(or ``stage3/qc_scalar.py`` + ``stage3/qc_adcp.py``). The public entry-point -``process_stage3(mooring_yaml, proc_dir)`` would remain in ``stage3/__init__.py``. -No change to the CLI or output format — purely an internal organisation change. - -2. Calibration correction (caldip) — front of Stage 3, not a separate stage +1. Calibration correction (caldip) — front of Stage 3, not a separate stage ----------------------------------------------------------------------------- -**Status**: planned — not yet implemented (``pipeline/qc/calibration.py`` is a stub). +**Status**: interface only — ``pipeline/qc/calibration.py`` defines the functions and +the stage-3 hook, which does nothing yet. **Purpose**: apply per-instrument temperature / conductivity / pressure corrections derived from caldip casts (pre/post-deployment CTD comparisons). -**Design (decided 2026-09-05):** not a separate stage. When a mooring is configured with -caldip casts, the per-instrument offsets are applied to the raw values at the **front of -Stage 3**, before pressure interpolation, so every downstream step sees corrected data. -There is one ``_stage3.nc`` — calibrated when caldip is configured, otherwise byte-identical -to today's output. Stack and grid are unchanged. +**Design:** not a separate stage. When a mooring is configured with caldip casts, the +per-instrument offsets are applied to the raw values at the **front of Stage 3**, before +pressure interpolation, so every downstream step sees corrected data. There is one +``_stage3.nc`` — calibrated when caldip is configured, otherwise byte-identical to the +uncorrected output. Stack and grid are unchanged. -**Input**: ``--caldip-dir`` gives the root directory only; the mooring YAML names the cast -for each end, so a mooring whose two dips were processed on different cruises resolves -correctly:: +**Input**: the separate ``caldip`` tool writes one netCDF per cast +(``{cast}_caldip.nc``). The caldip folder (``caldip_dir``) gives the root only; the +mooring YAML names the cast for each end, so a mooring whose two dips were processed on +different cruises resolves correctly:: caldip: deployment: {cruise: msm142_2026, cast: castM6} recovery: {cruise: , cast: } A mooring with a single dip is an ordinary case — the attributes record which end supplied -the constant correction. - -**Current state**: caldip produces ``castXX_detailed_statistics.csv`` (one per cast) with -``serial``, ``temp_diff``, ``cond_diff``, ``press_diff``, their ``_std``, ``*_status``, -``ctd_sensor_used``, window times, etc. These files exist but are not yet read. +the constant correction. The file contract is in :doc:`methods/calibration`. **Remaining steps:** -- Reader; per-variable stop selection (pressure nearest deployment depth, temperature deepest +- Per-variable stop selection (pressure nearest deployment depth, temperature deepest stop, conductivity a low-noise stop near deployment conductivity or a slope across stops). - Per-variable application threshold (pressure ``> 5 dbar`` and low std; temperature a minimum threshold; conductivity by noise/range or fit quality). - Pre/post-dip corrections applied as a linear time trend, with the linearity recorded as an assumption (a two-point fit cannot test it) — conductivity flagged as the case where a mid-deployment biofouling step is invisible to that fit. -- Sign/units guards: caldip ``*_diff`` is *instrument − CTD* (subtract to correct); ``cond_diff`` - is mS/cm — reconcile against the stored conductivity unit; record both instrument and CTD - values so the sign stays verifiable by eye. -- Provenance in the ``_stage3.nc`` attributes; ``ctd_slope_adjusted`` recorded ``UNK`` until - caldip emits it. -- A per-instrument report figure showing the conductivity offset-vs-slope choice; display of - applied corrections in the HTML report (later work). +- Sign/units guards: caldip differences are *instrument − CTD* (subtract to correct); + reconcile the conductivity unit against the stored one and refuse an unknown unit; + record both instrument and CTD values so the sign stays verifiable by eye. +- Provenance in the ``_stage3.nc`` attributes. +- Display of the applied corrections in the HTML report. + +2. Stage 3: further QC tests +----------------------------- + +**Implemented** (via ``ioos_qc``): gross-range, spike and flat-line tests on scalar +variables; tilt QC, ADCP seabed/surface QC, and ENU velocity QC for multi-dimensional +variables. See :doc:`methods/auto_qc`. + +**Candidates:** + +- Rate-of-change test +- Climatological range check (season-aware, e.g. from World Ocean Atlas) +- Spike threshold scaling by sampling interval +- Aquadopp tilt thresholds set from an analysis of how tilt degrades the velocities + (the 30°/50° defaults are placeholders) 3. Test coverage ----------------- -**Current state**: tests exist for ``stage1``, ``stage2``, ``stage3``, -``stack``, ``grid``, ``plotters``, ``readers``, ``writers``, ``instruments``, -``location``, and ``logger``. - -**Remaining gap**: ``test_report.py`` — end-to-end tests for the HTML report -generation pipeline. +Tests cover the stages, stack, grid, export, reports, plotters, readers, writer and the +mooring description. The stage-3 and stack orchestration paths are the thinnest; new +work there adds tests as it goes. --- Priority 2: Longer-term ======================== -4. Stage 4: OceanSITES conversion and deployment concatenation ---------------------------------------------------------------- +4. Gridding +----------- -Lower priority. The existing ``_stack.nc`` / ``_grid.nc`` outputs follow CF conventions -and can be converted to OceanSITES with a relatively thin wrapper once the earlier stages -are stable. +- Apply tilt to ADCP bin pressures (knocked-down bins currently sit at nominal depth). +- RAPID climatological-gradient gridding (``pipeline/gridders/rapid_interp.py``) as a + selectable method, once the climatology is built. 5. Multi-site merging for boundary profiles ------------------------------------------- @@ -152,12 +138,11 @@ Dependencies ============ - ``scipy``: Welch PSD for spectral figures in the grid report -- ``ioos_qc``: QARTOD gross-range and spike tests +- ``ioos_qc``: QARTOD gross-range, spike and flat-line tests - ``gsw`` (TEOS-10): seawater property calculations - ``xarray`` / ``netCDF4``: core data handling - ``jinja2``: HTML report generation - ``matplotlib``: all figures - ``ppigrf``: IGRF magnetic declination for BEAM→ENU rotation -- ``seasenselib``: raw instrument format readers (sbe-cnv, sbe-ascii, nortek-ascii, - nortek-csv, rbr-rsk, rbr-dat, rbr-hex, rdi-raw); pulls in ``mhkit[dolfyn]`` for +- ``seasenselib``: raw instrument format readers; pulls in ``mhkit[dolfyn]`` for the RDI reader diff --git a/docs/source/style_guide.md b/docs/source/style_guide.md index e9f1ee4e..9ca59731 100644 --- a/docs/source/style_guide.md +++ b/docs/source/style_guide.md @@ -1,14 +1,14 @@ # Project Style Guide -> 🎨 This guide explains the coding and formatting conventions used in the project, and how to keep things consistent when adding or editing code. +> This guide explains the coding and formatting conventions used in the project, and how to keep things consistent when adding or editing code. It combines Python code conventions, documentation standards, and metadata formatting guidelines. --- -## 🧑‍💻 Python Coding Conventions +## Python Coding Conventions -### ✅ Function Definitions +### Function Definitions - Use **type hints** for all parameters and return types. ```python def convert_units( @@ -19,11 +19,11 @@ def convert_units( ... ``` -### 🐍 Naming +### Naming - Functions and variables: `snake_case` - Constants (e.g., in xarray datasets): `ALL_CAPS` -### 📝 Docstrings +### Docstrings - Use **NumPy-style** docstrings - Required for all functions. - Sections include: `Parameters`, `Returns`, `Notes`, (optionally `Examples`, `References`) @@ -61,79 +61,59 @@ def convert_units_var( """ ``` -> 📚 NumPy-style docstrings render cleanly in Sphinx documentation (e.g. on Read the Docs). See [NumPy docstring style](https://numpydoc.readthedocs.io/en/latest/format.html). +> NumPy-style docstrings render cleanly in Sphinx documentation (e.g. on Read the Docs). See [NumPy docstring style](https://numpydoc.readthedocs.io/en/latest/format.html). > -> 💡 Alternatives like [Google-style](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) and [reST-style](https://www.sphinx-doc.org/en/master/usage/restructuredtext/domains.html#info-field-lists) docstrings are also valid with Sphinx if configured properly. +> Alternatives like [Google-style](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) and [reST-style](https://www.sphinx-doc.org/en/master/usage/restructuredtext/domains.html#info-field-lists) docstrings are also valid with Sphinx if configured properly. --- -## 🗂 Dataset & Metadata Conventions +## Dataset & Metadata Conventions -### 🧬 Variable Names +### Variable Names - Variable naming conventions are currently inconsistent across the codebase - We need to verify and align with [OceanSites variable naming recommendations](https://www.ocean-ops.org/oceansites/data/index.html) - Mix of `ALL_CAPS` (`TIME`, `DEPTH`) and lowercase (`temperature`, `pressure`) currently in use - Keep names short and unambiguous -### 🧾 Attributes +### Attributes - Follow [OceanSites format conventions](https://www.ocean-ops.org/oceansites/data/index.html) - Use `units`, `long_name`, `comment` - Avoid placing units in variable names — use attributes instead -### 📐 Consistency +### Consistency - Units are checked and converted using helper functions - Time is standardized using utility functions across the project --- -## 🔁 Automating Formatting +## Formatting and linting -The project uses tools like `black`, `ruff`, and `pytest` to enforce style, linting, and test consistency. These are integrated into the workflow using pre-commit hooks. +The project uses `ruff` for both linting and formatting, and `pytest` for tests. CI runs the same checks, so run them before pushing: -**Pre-commit tools used:** ```bash -black . # Format code with black -ruff check . # Run ruff linter -ruff check . --fix # Auto-fix issues where possible -pre-commit run --all-files # Run all pre-commit hooks -codespell # Check for spelling errors +ruff check . --fix # lint, fixing what it can +ruff format # format (ruff format --check to verify only) +pytest # tests ``` -You don't need to run them manually, but setting up pre-commit ensures your code follows project standards automatically. - --- -## 🛠 Optional: VSCode Setup for Auto-formatting +## Optional: VSCode setup -To automatically format code when you save a file, add this to your **Workspace Settings** (`.vscode/settings.json`): +To format on save with ruff, install the Ruff extension and add this to your **Workspace Settings** (`.vscode/settings.json`): ```json { - "python.formatting.provider": "black", - "editor.formatOnSave": true, - "editor.codeActionsOnSave": { - "source.organizeImports": true + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff", + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.fixAll": "explicit", + "source.organizeImports": "explicit" + } } } ``` -You can also add a VSCode task to format manually: -```text -{ - "label": "Format Code", - "type": "shell", - "command": "black . && ruff check . --fix", - ... -} -``` -- Then bind it to a keyboard shortcut (e.g. `Cmd+Shift+R`) or run it from the command palette. - ---- - -## 💡 Future Considerations - -- Add `mypy` for static type checking -- Use Sphinx for auto-generating documentation (already partially configured) - --- -> ✅ By following these conventions, your code will integrate smoothly with the rest of the project — clean, consistent, and ready for collaboration. +By following these conventions, your code will integrate smoothly with the rest of the project. diff --git a/docs/source/troubleshooting.rst b/docs/source/troubleshooting.rst index dd616cbc..bd89c09c 100644 --- a/docs/source/troubleshooting.rst +++ b/docs/source/troubleshooting.rst @@ -58,13 +58,8 @@ If you use **conda**: conda activate oceanarray -If you also use a local shell script to set ``DATA_BASE`` or similar -environment variables, source that too before running ``oceanarray`` -commands: - -.. code-block:: bash - - source shlocal/run_mooring.sh # example — path will differ +If a shell script of your own sets the data paths, source it in the new +terminal too before running ``oceanarray`` commands. ---- @@ -75,8 +70,9 @@ Runtime problems ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Check that the ``--proc-dir`` path exists and that the mooring YAML is -present at ``{proc_dir}/{mooring_name}/{mooring_name}.mooring.yaml``. -Run with ``--verbose`` or ``-v`` for more detail. +present at ``{proc_dir}/{mooring_name}/{mooring_name}.mooring.yaml`` (or give +the YAML's path directly). Run with ``-v`` (``--verbose``) for more detail, and +``oceanarray validate MOORING`` to check the description. Stage 1 fails with "no reader found" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -92,8 +88,8 @@ For Nortek Aquadopps the BEAM→ENU transformation requires a transformation matrix parsed from the ``.hdr`` file. If the ``.hdr`` file was not found at stage 1, ``coordinate_system`` is stored as ``"BEAM"`` and stage 3 skips the rotation. Verify that the ``.hdr`` file is in the same directory as the -``.aqd`` data file and that the ``header:`` field in the mooring YAML points -to it. +``.aqd`` data file, or that the ``header_file:`` field in the mooring YAML +points to it. ---- @@ -104,4 +100,5 @@ If none of the above solves your problem: * Open an issue at https://github.com/ocean-uhh/oceanarray/issues and include the full traceback and the output of ``oceanarray --version``. -* Check that your branch is up to date: ``git pull``. +* Check that you have the latest release: ``pip install --upgrade oceanarray`` + (or ``git pull`` in an editable install). diff --git a/docs/source/yaml_configuration.rst b/docs/source/yaml_configuration.rst index e50a62a1..de9c2491 100644 --- a/docs/source/yaml_configuration.rst +++ b/docs/source/yaml_configuration.rst @@ -68,19 +68,31 @@ mooring as a whole. - Required - Mooring identifier, e.g. ``dsG3_1_2026``. Used in all output file names. + * - ``platform_code`` + - string + - For export + - The OceanSITES platform (the station, e.g. ``DUNE2``), without ``_``. + Required by ``oceanarray export``; see :doc:`methods/oceansites_export`. + Quote it if it could be read as a number. + * - ``deployment_code`` + - string + - Optional + - The OceanSITES deployment code, ``YYYYMM`` with a month 01–12. Default: + the month of ``deployment_time``; ``validate`` warns if the two disagree. * - ``waterdepth`` - number - Required - Water depth at the anchor position in metres. * - ``deployment_time`` - ISO-8601 string - - Required + - Recommended - When the mooring was deployed (UTC). Used by stage 2 to trim the record. Format: ``"YYYY-MM-DDTHH:MM:SS"``, e.g. - ``"2026-05-07T17:05:00"`` means 7 May 2026 at 17:05 UTC. + ``"2026-05-07T17:05:00"`` means 7 May 2026 at 17:05 UTC. Without it + (a WARNING), stage 2 uses the window detected from the records. * - ``recovery_time`` - ISO-8601 string - - Required + - Recommended - When the mooring was recovered (UTC). Used by stage 2 to trim the record. Same format as ``deployment_time``. * - ``year`` @@ -139,8 +151,14 @@ mooring as a whole. * - ``qc_ranges`` - mapping - Optional - - Mooring-wide QC thresholds applied in stage 3 to all instruments - unless overridden at the instrument level. See `QC ranges`_ below. + - Mooring-wide gross-range (and tilt) thresholds applied in stage 3 to all + instruments unless overridden at the instrument level. See `QC ranges`_ + below. + * - ``qc_spike``, ``qc_flat_line``, ``tilt_qc`` + - mapping + - Optional + - Mooring-wide spike, flat-line and tilt thresholds; see `Other QC + blocks`_ below. * - ``stack`` - mapping - Optional @@ -296,11 +314,18 @@ Optional fields instrument. Set to ``4`` to mark pressure as bad/absent (e.g. the pressure sensor failed or the instrument type has no pressure sensor). - * - ``qc_ranges`` + * - ``qc_ranges``, ``qc_spike``, ``qc_flat_line``, ``tilt_qc`` - mapping - - Per-instrument QC overrides. Same structure as mooring-level - ``qc_ranges``; instrument-level values override mooring-level - values for the same variable. + - Per-instrument QC overrides. Same structure as the mooring-level + blocks; instrument-level values override mooring-level values for the + same variable. + * - ``data_mode`` + - string or mapping + - OceanSITES data mode after a manual review: one letter for all the + instrument's variables, or ``{variable: letter}``, with ``P`` + (provisional, the default) or ``D`` (delayed mode). ``D`` is refused by + the export unless the stage-3 file records the calibration-dip + correction. * - ``skip`` - boolean - Set to ``true`` to skip this instrument entirely during processing. @@ -346,11 +371,15 @@ Additional fields ``"Aquadopp"``. Used by ``moordiag`` for diagram labels and may be used by ``oceanarray`` as a legend label in plots. +``clamp_id`` + The clamp position on the mooring line. Used by ``moordiag``; oceanarray + records it on each instrument's files (the ``clamp_id`` attribute). + Fields used only by moordiag (ignored by oceanarray) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The following fields are read by the ``moordiag`` mooring diagram package -and are ignored by ``oceanarray``: ``image``, ``clamp_id``, ``position``, +and are ignored by ``oceanarray``: ``image``, ``position``, ``hardware_type``, ``category``, ``component_id``, ``length``, ``repeat``. They may safely remain in the YAML alongside the processing fields. @@ -785,6 +814,39 @@ degrees-decimal-minutes format, e.g. ``"65 29.84 N"`` and ``"009 30.12 W"``. The degrees component for longitude must be zero-padded to three digits (``009``, not ``9``). +Other QC blocks +~~~~~~~~~~~~~~~ + +The spike, flat-line and tilt tests take their own blocks, at mooring level or in +an instrument entry, in the units of the variable (see :doc:`methods/auto_qc` for +the defaults): + +.. code-block:: yaml + + qc_spike: # per variable + temperature: {suspect_threshold: 2.0, fail_threshold: 6.0} + qc_flat_line: # per variable; counts of samples + pressure: {suspect_n: 3, fail_n: 10, tolerance: 0.001} + tilt_qc: # not per variable; degrees + suspect_threshold: 30 + fail_threshold: 50 + +``tilt`` may also be given in ``qc_ranges`` as symmetric spans +(``suspect_span: [-20, 20]``); ``tilt_qc`` takes precedence. + +**The blocks are checked when the description is read.** These are ERRORs, so +``oceanarray validate`` reports them and processing refuses the description: + +- an unknown field (e.g. ``suspect_spn``); +- a span that is not two increasing numbers; +- a threshold that is not a number of 0 or more; +- a sample count that is not a whole number of 1 or more; +- a block that is not a mapping; +- a ``qc_ranges`` variable without a default and without a ``fail_span``. + +A variable that is not in the registry is only a WARNING (most often a typo): +its thresholds apply if a file has that variable. + ---- Validate your YAML @@ -798,15 +860,17 @@ common mistakes: oceanarray validate /path/to/proc/dsG3_1_2026/dsG3_1_2026.mooring.yaml The validator checks that required fields are present, that ``instrument`` -and ``file_type`` values are recognised, and that the YAML can be parsed +and ``file_type`` values are recognised, the QC blocks (`Other QC blocks`_), +``platform_code`` and ``deployment_code``, and that the YAML can be parsed without errors. It also emits warnings for inline instruments with commas -in the serial number so you can confirm the ordering. +in the serial number so you can confirm the ordering. It exits 1 on an ERROR; +add ``--strict`` to fail on warnings too. You can validate multiple files at once: .. code-block:: bash - oceanarray validate /path/to/proc/*/ *.mooring.yaml + oceanarray validate /path/to/proc/*/*.mooring.yaml ---- diff --git a/oceanarray/__main__.py b/oceanarray/__main__.py new file mode 100644 index 00000000..91120c85 --- /dev/null +++ b/oceanarray/__main__.py @@ -0,0 +1,6 @@ +"""Run the command line as ``python -m oceanarray``.""" + +from oceanarray.cli import main + +if __name__ == "__main__": + main() diff --git a/oceanarray/cli/__init__.py b/oceanarray/cli/__init__.py index f4ad122b..6f8cbe9c 100644 --- a/oceanarray/cli/__init__.py +++ b/oceanarray/cli/__init__.py @@ -48,7 +48,7 @@ def build_parser() -> argparse.ArgumentParser: """Build and return the top-level argument parser for the oceanarray CLI.""" epilog = ( - "Typical workflow (MOORING is the *.mooring.yaml, its folder, or a name with --proc-dir):\n" + "Typical workflow (MOORING is the .mooring.yaml, its folder, or a name with --proc-dir):\n" " oceanarray init dsG3_1_2026 --proc-dir /data/proc write a skeleton description\n" " oceanarray validate /data/proc/dsG3_1_2026/dsG3_1_2026.mooring.yaml\n" " oceanarray process MOORING --raw-dir /data/raw --stage 1 one stage on its own\n" @@ -116,6 +116,9 @@ def main() -> None: _skip_next = True elif _tok.startswith("--raw-dir="): _raw_dir_unknowns.append(_tok) + elif _tok in ("-v", "--verbose"): + # -v is accepted after the command name as well as before it. + args.verbose = True else: _other_unknowns.append(_tok) if _raw_dir_unknowns: diff --git a/oceanarray/cli/_common.py b/oceanarray/cli/_common.py index 4786d117..9cd15108 100644 --- a/oceanarray/cli/_common.py +++ b/oceanarray/cli/_common.py @@ -5,11 +5,19 @@ import argparse import sys from pathlib import Path +from typing import NoReturn from oceanarray import paths +from oceanarray.cli._deprecate import DeprecatedAlias from oceanarray.pipeline import resolve_stage +def _usage_error(message: str) -> NoReturn: + """Print *message* as an ERROR and exit 2, the exit status of a usage error.""" + print(f"ERROR: {message}", file=sys.stderr) + raise SystemExit(2) + + def _resolve_dirs( args: argparse.Namespace, *, @@ -21,8 +29,8 @@ def _resolve_dirs( proc_dir = getattr(args, "proc_dir", None) if basedir: - raise SystemExit( - "ERROR: --basedir has been removed. Use --raw-dir + --proc-dir " + _usage_error( + "--basedir has been removed. Use --raw-dir + --proc-dir " "(the mooring directory is now /, not " "/moor/proc/). See the migration guide: " f"docs/source/migration.rst or {paths.DOCS_URL}migration.html" @@ -30,9 +38,7 @@ def _resolve_dirs( if not proc_dir: if proc_required: - raise SystemExit( - "ERROR: --proc-dir is required (and --raw-dir for stage 1)." - ) + _usage_error("--proc-dir is required (and --raw-dir for stage 1).") return Path(raw_dir) if raw_dir else None, None return Path(raw_dir) if raw_dir else None, Path(proc_dir) @@ -43,7 +49,13 @@ def _mooring_location(args: argparse.Namespace) -> Path: if getattr(args, "proc_dir", None): proc_root = Path(args.proc_dir) paths.require_current_layout(proc_root, args.mooring) - return paths.mooring_proc_dir(proc_root, args.mooring) + folder = paths.mooring_proc_dir(proc_root, args.mooring) + if not folder.is_dir(): + raise SystemExit( + f"ERROR: no mooring folder {folder}: {args.mooring} is not a mooring name " + "under --proc-dir." + ) + return folder # Without --proc-dir the positional is a path: a *.mooring.yaml or the folder # holding one, under any name. given = Path(args.mooring) @@ -95,8 +107,8 @@ def _add_dir_args(p: argparse.ArgumentParser, raw_needed: bool = True) -> None: default=None, dest="raw_dir", metavar="DIR", - help="Cruise-level raw data directory (pipeline appends /{mooring}/). " - "Required for stage 1.", + help="The cruise's raw-data folder; a mooring's raw files are in " + "//. Needed only when stage 1 runs.", ) else: # Commands that read only processed NetCDF (plot, animate) take no --raw-dir. @@ -106,35 +118,11 @@ def _add_dir_args(p: argparse.ArgumentParser, raw_needed: bool = True) -> None: default=None, dest="proc_dir", metavar="DIR", - help="Cruise-level processed output directory (pipeline appends /{mooring}/).", + help="The cruise's processed-data folder; a mooring's files are in " + "//. With it, MOORING may be just the name.", ) -class _OldFlag(argparse.Action): - """Accept a renamed flag for one release, saying what it is called now.""" - - def __init__(self, *args: object, new: str, **kwargs: object) -> None: - """Remember the new spelling.""" - self.new = new - super().__init__(*args, **kwargs) - - def __call__( - self, - parser: argparse.ArgumentParser, # noqa: ARG002 — argparse.Action signature - namespace: argparse.Namespace, - values: object, - option_string: str | None = None, - ) -> None: - """Store the value and print the new spelling to stderr.""" - print( - f"oceanarray: {option_string} is now {self.new} ({option_string} is " - "removed in 0.7.0).", - file=sys.stderr, - ) - # nargs=0 is a renamed on/off switch; otherwise forward the parsed values. - setattr(namespace, self.dest, True if self.nargs == 0 else values) - - def _add_only(p: argparse.ArgumentParser, text: str) -> None: """Add ``--only SERIAL...`` (and the old ``--serial`` spelling, hidden).""" p.add_argument( @@ -144,16 +132,20 @@ def _add_only(p: argparse.ArgumentParser, text: str) -> None: "--serial", nargs="+", dest="serials", - action=_OldFlag, + action=DeprecatedAlias, new="--only", help=argparse.SUPPRESS, ) def _add_stack_grid_args( - p: argparse.ArgumentParser, *, from_description: bool = False + p: argparse.ArgumentParser, *, from_description: bool = False, hidden: bool = False ) -> None: - """Add the shared stack/grid tuning flags (--dt-seconds, --dp, --pmin, --pmax).""" + """Add the shared stack/grid tuning flags (--dt-seconds, --dp, --pmin, --pmax). + + *hidden* keeps them working but out of ``--help`` (``process``, where they only + serve the deprecated ``--stage stack|grid``). + """ from ..pipeline._run import DEFAULTS # The flags spell the Python keywords; the time step carries its unit. @@ -179,13 +171,13 @@ def _add_stack_grid_args( # keeps the fixed defaults. default=None if from_description else kind(default), metavar=metavar, - help=f"{text} (default: {where})", + help=argparse.SUPPRESS if hidden else f"{text} (default: {where})", ) p.add_argument( "--dt", type=int, dest="dt_seconds", - action=_OldFlag, + action=DeprecatedAlias, new="--dt-seconds", help=argparse.SUPPRESS, ) diff --git a/oceanarray/cli/_deprecate.py b/oceanarray/cli/_deprecate.py new file mode 100644 index 00000000..cd507183 --- /dev/null +++ b/oceanarray/cli/_deprecate.py @@ -0,0 +1,40 @@ +"""Renamed CLI flags: the old spelling works for one release, with a warning.""" + +from __future__ import annotations + +import argparse +import sys +import warnings + +#: The release that removes the old spellings. +REMOVED_IN = "0.7.0" + + +class DeprecatedAlias(argparse.Action): + """Accept a renamed flag, store its value on the new flag's destination, and warn. + + Use with ``new="--new-name"`` and ``help=argparse.SUPPRESS``, so the old spelling + still works but is not advertised. + """ + + def __init__(self, *args: object, new: str, **kwargs: object) -> None: + """Remember the new spelling.""" + self.new = new + super().__init__(*args, **kwargs) + + def __call__( + self, + parser: argparse.ArgumentParser, # noqa: ARG002 — argparse.Action signature + namespace: argparse.Namespace, + values: object, + option_string: str | None = None, + ) -> None: + """Store the value and warn that *option_string* is now :attr:`new`.""" + msg = ( + f"{option_string} is deprecated and removed in {REMOVED_IN}; " + f"use {self.new}." + ) + warnings.warn(msg, DeprecationWarning, stacklevel=2) + print(f"oceanarray: warning: {msg}", file=sys.stderr) + # nargs=0 is a renamed on/off switch; otherwise forward the parsed values. + setattr(namespace, self.dest, True if self.nargs == 0 else values) diff --git a/oceanarray/cli/animate.py b/oceanarray/cli/animate.py index 1f867824..a4029de8 100644 --- a/oceanarray/cli/animate.py +++ b/oceanarray/cli/animate.py @@ -152,7 +152,20 @@ def build_parser(sub: argparse._SubParsersAction) -> None: # type: ignore[type- """Add the ``animate`` subcommand to *sub*.""" p_animate = sub.add_parser( "animate", - help="Write an animated hodograph GIF for Aquadopp instrument(s).", + help="Write an animated hodograph GIF for each current meter.", + description=( + "Read each Aquadopp's stage-3 file and write a GIF of its velocity hodograph\n" + "building up over the deployment, one per instrument, beside its stage file\n" + "(or -o DIR; --output FILE.gif for a single instrument)." + ), + epilog=( + "Examples:\n" + " # every current meter on the mooring\n" + " oceanarray animate /data/proc/dsG3_1_2026\n" + " # one instrument, one file\n" + " oceanarray animate /data/proc/dsG3_1_2026 --only 9920 --output 9920.gif" + ), + formatter_class=argparse.RawDescriptionHelpFormatter, ) p_animate.add_argument("mooring", help=_MOORING_HELP) _add_dir_args(p_animate, raw_needed=False) diff --git a/oceanarray/cli/export.py b/oceanarray/cli/export.py index 3305ab69..2c44649e 100644 --- a/oceanarray/cli/export.py +++ b/oceanarray/cli/export.py @@ -6,6 +6,7 @@ from pathlib import Path from oceanarray.cli._common import _add_dir_args, _folder, _mooring_location +from oceanarray.cli._deprecate import DeprecatedAlias def cmd_export(args: argparse.Namespace) -> int: @@ -22,13 +23,15 @@ def cmd_export(args: argparse.Namespace) -> int: out_dir=args.output_dir, array=args.array, grid=not args.no_grid, - overwrite=args.overwrite, + overwrite=args.force, + dry_run=args.dry_run, ) except (MooringError, OceanSitesError, FileExistsError) as e: print(f"ERROR: {e}") return 1 + verb = "would write" if args.dry_run else "wrote" for path in written: - print(f"wrote {path}") + print(f"{verb} {path}") return 0 @@ -44,11 +47,20 @@ def build_parser(sub: argparse._SubParsersAction) -> None: # type: ignore[type- "platform_code. Instruments that cannot be exported are listed and left out.\n" "Exits 0 if any file was written, 1 if none." ), + epilog=( + "Examples:\n" + " # every instrument and the grid, into /oceansites/\n" + " oceanarray export /data/proc/dsG3_1_2026 --array /data/proc/MIXSED2026.array.yaml\n" + " # see the file names first\n" + " oceanarray export dsG3_1_2026 --proc-dir /data/proc -n\n" + " # instruments only, replacing earlier files\n" + " oceanarray export dsG3_1_2026 --proc-dir /data/proc --no-grid --force" + ), formatter_class=argparse.RawDescriptionHelpFormatter, ) p.add_argument( "mooring", - help="The *.mooring.yaml, its folder, or a mooring name with --proc-dir.", + help="The .mooring.yaml, its folder, or a mooring name with --proc-dir.", ) _add_dir_args(p, raw_needed=False) p.add_argument( @@ -56,7 +68,7 @@ def build_parser(sub: argparse._SubParsersAction) -> None: # type: ignore[type- type=Path, default=None, metavar="YAML", - help="The *.array.yaml; supplies site_code and project.", + help="The .array.yaml; supplies site_code and project.", ) p.add_argument( "-o", @@ -68,5 +80,21 @@ def build_parser(sub: argparse._SubParsersAction) -> None: # type: ignore[type- help="Where to write the files (default: /oceansites).", ) p.add_argument("--no-grid", action="store_true", help="Leave the grid out.") - p.add_argument("--overwrite", action="store_true", help="Replace files that exist.") + p.add_argument( + "--force", action="store_true", help="Replace OceanSITES files that exist." + ) + p.add_argument( + "--overwrite", + dest="force", + nargs=0, + action=DeprecatedAlias, + new="--force", + help=argparse.SUPPRESS, + ) + p.add_argument( + "-n", + "--dry-run", + action="store_true", + help="Convert and list the files that would be written; write nothing.", + ) p.set_defaults(func=cmd_export) diff --git a/oceanarray/cli/init.py b/oceanarray/cli/init.py index 6a0cd664..ef0c2ad4 100644 --- a/oceanarray/cli/init.py +++ b/oceanarray/cli/init.py @@ -34,6 +34,12 @@ def build_parser(sub: argparse._SubParsersAction) -> None: # type: ignore[type- "fields for all mandatory metadata and one example entry per instrument type.\n" "Edit the file to fill in real values and delete unused instrument blocks." ), + epilog=( + "Examples:\n" + " oceanarray init dsG3_1_2026 --proc-dir /data/proc\n" + " oceanarray validate /data/proc/dsG3_1_2026/dsG3_1_2026.mooring.yaml" + ), + formatter_class=argparse.RawDescriptionHelpFormatter, ) p_init.add_argument("mooring", help="Mooring name, e.g. dsG3_1_2026") p_init.add_argument( diff --git a/oceanarray/cli/list.py b/oceanarray/cli/list.py index 8a78dfe6..4d3955f1 100644 --- a/oceanarray/cli/list.py +++ b/oceanarray/cli/list.py @@ -48,7 +48,13 @@ def build_parser(sub: argparse._SubParsersAction) -> None: # type: ignore[type- """Add the ``list`` subcommand to *sub*.""" p_list = sub.add_parser( "list", - help="List allowed instrument types and file_type values for mooring YAML files.", + help="List the instrument types and file_type values a mooring YAML may use.", + description=( + "Print the values the 'instrument' and 'file_type' fields of a mooring\n" + "description accept. Reads and writes nothing." + ), + epilog="Examples:\n oceanarray list\n oceanarray list file-types", + formatter_class=argparse.RawDescriptionHelpFormatter, ) p_list.add_argument( "topic", diff --git a/oceanarray/cli/plot.py b/oceanarray/cli/plot.py index 0cc90156..85fb37e8 100644 --- a/oceanarray/cli/plot.py +++ b/oceanarray/cli/plot.py @@ -10,8 +10,8 @@ _add_dir_args, _folder, _mooring_location, - _OldFlag, ) +from oceanarray.cli._deprecate import DeprecatedAlias def cmd_plot(args: argparse.Namespace) -> int: @@ -19,7 +19,7 @@ def cmd_plot(args: argparse.Namespace) -> int: Reads the best available processed NetCDF (stage3 preferred, falling back to stage2) for all instruments and produces a scatter/line plot with a user-chosen - y-axis variable (default: pressure) and colour variable (default: temperature). + y-axis variable (default: temperature) and an optional colour variable. When ``--output`` is given the plot is saved to that filename in the proc directory (or ``--output-dir`` if specified). Without ``--output``, the plot @@ -57,7 +57,25 @@ def cmd_plot(args: argparse.Namespace) -> int: def build_parser(sub: argparse._SubParsersAction) -> None: # type: ignore[type-arg] """Add the ``plot`` subcommand to *sub*.""" - p_plot = sub.add_parser("plot", help="Plot multi-instrument mooring overview.") + p_plot = sub.add_parser( + "plot", + help="Plot one variable of every instrument on a mooring in one figure.", + description=( + "Read each instrument's stage-3 file (stage 2 where there is none) and plot\n" + "one variable against time for all of them, as lines or, with --var-color,\n" + "as points coloured by a second variable. Shows the figure (--show) or saves\n" + "it (--output / -o)." + ), + epilog=( + "Examples:\n" + " # temperature of every instrument, saved next to the stage files\n" + " oceanarray plot /data/proc/dsG3_1_2026 --output overview.png\n" + " # pressure coloured by temperature, on screen\n" + " oceanarray plot dsG3_1_2026 --proc-dir /data/proc --var-y pressure " + "--var-color temperature --show" + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) p_plot.add_argument("mooring", help=_MOORING_HELP) _add_dir_args(p_plot, raw_needed=False) p_plot.add_argument( @@ -76,7 +94,7 @@ def build_parser(sub: argparse._SubParsersAction) -> None: # type: ignore[type- p_plot.add_argument( old, dest=new[2:].replace("-", "_"), - action=_OldFlag, + action=DeprecatedAlias, new=new, help=argparse.SUPPRESS, ) @@ -93,12 +111,21 @@ def build_parser(sub: argparse._SubParsersAction) -> None: # type: ignore[type- help="Scatter marker size in points² (default 4). Only used in scatter mode.", ) p_plot.add_argument( - "--downsample", + "--downsample-seconds", type=int, default=None, + dest="downsample", metavar="SECONDS", help="Resample interval in seconds (default: 120)", ) + p_plot.add_argument( + "--downsample", + type=int, + dest="downsample", + action=DeprecatedAlias, + new="--downsample-seconds", + help=argparse.SUPPRESS, + ) p_plot.add_argument( "--output", default=None, diff --git a/oceanarray/cli/process.py b/oceanarray/cli/process.py index 36715f47..8d72c218 100644 --- a/oceanarray/cli/process.py +++ b/oceanarray/cli/process.py @@ -16,7 +16,9 @@ _mooring_location, _parse_stage_token, _raw_dir, + _usage_error, ) +from oceanarray.cli._deprecate import DeprecatedAlias from oceanarray.pipeline import STAGES @@ -54,11 +56,11 @@ def cmd_process(args: argparse.Namespace) -> int: ints = sorted({int(s) for s in stages}) return 0 if _print_stage_plan(args, location, ints, "Process") else 1 if 1 in stages and raw_dir is None: - raise SystemExit("ERROR: --raw-dir is required for stage 1.") + _usage_error("--raw-dir is required for stage 1.") if products and not args.keep_going: - raise SystemExit( - "ERROR: --strict does not apply to --stage stack/grid; use " + _usage_error( + "--strict does not apply to --stage stack/grid; use " "'oceanarray run MOORING --strict' to build the stack and grid." ) ok = True @@ -102,8 +104,8 @@ def _process_with_classes( from ..pipeline import _process_legacy if not getattr(args, "proc_dir", None): - raise SystemExit( - "ERROR: --stage stack/grid needs a mooring name and --proc-dir; or use " + _usage_error( + "--stage stack/grid needs a mooring name and --proc-dir; or use " "'oceanarray run PATH' for the stack and grid." ) if any(s in ("stack", "grid") for s in stages): @@ -151,13 +153,25 @@ def build_parser(sub: argparse._SubParsersAction) -> None: # type: ignore[type- """Add the ``process`` subcommand to *sub*.""" p_process = sub.add_parser( "process", - help="Run per-instrument processing stages 1-3 (raw→NetCDF, trim+clock, QC+ENU).", + help="Run the per-instrument stages 1-3 (raw files to QC'd netCDF).", description=( - "Run one or more per-instrument processing stages for a mooring.\n" - " Stage 1: read raw files → CF-NetCDF (_stage1.nc)\n" - " Stage 2: trim to deployment window + apply clock-drift correction (_stage2.nc)\n" - " Stage 3: QC flags, BEAM→ENU rotation, magnetic declination, derived vars (_stage3.nc)\n" - "Default when --stage is omitted: stages 1, 2 and 3." + "Run one or more per-instrument processing stages for a mooring. Reads the\n" + "mooring description and the raw files (stage 1) or the previous stage's files;\n" + "writes ///__stage.nc.\n" + " Stage 1: read raw files into CF-NetCDF\n" + " Stage 2: trim to the deployment window, correct clock drift\n" + " Stage 3: QC flags, BEAM to ENU rotation, magnetic declination, derived variables\n" + "Default when --stage is omitted: stages 1, 2 and 3. For the stack, grid and\n" + "report as well, use 'oceanarray run'." + ), + epilog=( + "Examples:\n" + " # all three stages from the raw files\n" + " oceanarray process dsG3_1_2026 --proc-dir /data/proc --raw-dir /data/raw\n" + " # re-run stage 3 for two instruments\n" + " oceanarray process dsG3_1_2026 --proc-dir /data/proc --stage 3 --only 14321 400118\n" + " # see what stage 2 would write\n" + " oceanarray process /data/proc/dsG3_1_2026 --stage 2 -n" ), formatter_class=argparse.RawDescriptionHelpFormatter, ) @@ -170,19 +184,28 @@ def build_parser(sub: argparse._SubParsersAction) -> None: # type: ignore[type- choices=[s.number if s.number is not None else s.name for s in STAGES], default=None, metavar="{1,2,3,stack,grid}", - help="Stage(s) to run (default: 1 2 3). " - "Use 'stack' and 'grid' for mooring-level steps.", + help="Stage(s) to run (default: 1 2 3). 'stack' and 'grid' are deprecated and " + "removed in 0.7.0: 'oceanarray run' builds them.", ) - _add_stack_grid_args(p_process) + _add_stack_grid_args(p_process, hidden=True) p_process.add_argument( "--plot", action="store_true", - help="Generate plots of processed microcat data after stage 2", + help="After stage 2, plot the MicroCAT records (temperature, conductivity, pressure)", ) p_process.add_argument( - "--report", + "--summary", action="store_true", - help="Print a summary of processed records per instrument and serial number", + dest="report", + help="Print the number of records per instrument and serial number", + ) + p_process.add_argument( + "--report", + nargs=0, + dest="report", + action=DeprecatedAlias, + new="--summary", + help=argparse.SUPPRESS, ) _add_only( p_process, @@ -198,7 +221,8 @@ def build_parser(sub: argparse._SubParsersAction) -> None: # type: ignore[type- p_process.add_argument( "--force", action="store_true", - help="Overwrite existing output files (both stages skip existing files by default)", + help="Rewrite stage files that already exist (default: keep them; stage 1 " + "rewrites only a file older than its raw file)", ) p_process.add_argument( "-n", diff --git a/oceanarray/cli/report.py b/oceanarray/cli/report.py index 3189ec7b..1490e74f 100644 --- a/oceanarray/cli/report.py +++ b/oceanarray/cli/report.py @@ -13,9 +13,9 @@ _add_shared_report_args, _folder, _mooring_location, - _OldFlag, _resolve_dirs, ) +from oceanarray.cli._deprecate import DeprecatedAlias def cmd_report(args: argparse.Namespace) -> int: @@ -204,7 +204,25 @@ def build_parser(sub: argparse._SubParsersAction) -> None: # type: ignore[type- """Add the ``report`` subcommand to *sub*.""" p_report = sub.add_parser( "report", - help="Generate a mooring recovery HTML report.", + help="Build a mooring's HTML report pages from its processed files.", + description=( + "Build the report for one mooring from the files already processed: the\n" + "summary page, and with --instruments / --stack / --grid (or --all) the\n" + "per-instrument, stack and grid pages. Reads the stage, stack and grid files;\n" + "writes HTML into /report/ (or -o / --report-dir), and with\n" + "--pdf one A4 PDF. Processes no data. With --array, builds the array index\n" + "from an .array.yaml instead." + ), + epilog=( + "Examples:\n" + " # the summary page\n" + " oceanarray report /data/proc/dsG3_1_2026\n" + " # every page, and the PDF\n" + " oceanarray report dsG3_1_2026 --proc-dir /data/proc --all --pdf\n" + " # the array index\n" + " oceanarray report /data/proc/MIXSED2026.array.yaml --array" + ), + formatter_class=argparse.RawDescriptionHelpFormatter, ) p_report.add_argument("mooring", help=_MOORING_HELP) _add_dir_args(p_report, raw_needed=True) @@ -291,8 +309,8 @@ def build_parser(sub: argparse._SubParsersAction) -> None: # type: ignore[type- action="store_true", dest="skip_existing", default=False, - help="Skip any output file that already exists, regardless of source mtime " - "(old behaviour; use --force to always regenerate).", + help="Skip any page that already exists, even if its files are newer " + "(--force always rebuilds).", ) # The short spellings went in 0.6 (family rule: only -o, -n and -v are short); # accepted until 0.7.0 with a note. @@ -303,6 +321,11 @@ def build_parser(sub: argparse._SubParsersAction) -> None: # type: ignore[type- ("-A", "all_reports", "--all"), ): p_report.add_argument( - short, dest=dest, nargs=0, action=_OldFlag, new=new, help=argparse.SUPPRESS + short, + dest=dest, + nargs=0, + action=DeprecatedAlias, + new=new, + help=argparse.SUPPRESS, ) p_report.set_defaults(func=cmd_report) diff --git a/oceanarray/cli/run.py b/oceanarray/cli/run.py index 10596742..c5812a19 100644 --- a/oceanarray/cli/run.py +++ b/oceanarray/cli/run.py @@ -13,6 +13,7 @@ _add_stack_grid_args, _mooring_location, _raw_dir, + _usage_error, ) @@ -47,8 +48,8 @@ def cmd_run(args: argparse.Namespace) -> int: if args.dry_run: return _run_dry_run(args, yaml_path, wanted) if wanted and wanted[0] == 1 and raw_dir is None: - raise SystemExit( - "ERROR: --raw-dir is required for stage 1 (or pass --from 2 to start " + _usage_error( + "--raw-dir is required for stage 1 (or pass --from 2 to start " "from the stage-1 files)." ) @@ -115,7 +116,25 @@ def build_parser(sub: argparse._SubParsersAction) -> None: # type: ignore[type- """Add the ``run`` subcommand to *sub*.""" p_run = sub.add_parser( "run", - help="Full pipeline: stages 1-3, stack, grid, and all reports in one command.", + help="Run stages 1-3, the stack, the grid and the report pages in one go.", + description=( + "Process a mooring from its raw files to its products and report: stages 1-3\n" + "for every instrument, then the stack (instruments on one time base), the grid\n" + "(time x pressure levels) and the HTML pages. Writes the stage files, \n" + "_stack.nc and _grid.nc in the mooring folder, and the pages in\n" + "/report/ (or -o / --report-dir). Instruments that fail are\n" + "left out and named in the products, unless --strict." + ), + epilog=( + "Examples:\n" + " # everything, from the raw files\n" + " oceanarray run dsG3_1_2026 --proc-dir /data/proc --raw-dir /data/raw\n" + " # from the stage-1 files on, regridded at 10 dbar\n" + " oceanarray run /data/proc/dsG3_1_2026 --from 2 --dp 10\n" + " # list what would be written\n" + " oceanarray run /data/proc/dsG3_1_2026 --from 2 -n" + ), + formatter_class=argparse.RawDescriptionHelpFormatter, ) p_run.add_argument("mooring", help=_MOORING_HELP) _add_dir_args(p_run, raw_needed=True) diff --git a/oceanarray/cli/stamp.py b/oceanarray/cli/stamp.py index 78c88768..5fab0fed 100644 --- a/oceanarray/cli/stamp.py +++ b/oceanarray/cli/stamp.py @@ -3,14 +3,20 @@ from __future__ import annotations import argparse +import sys from pathlib import Path +from oceanarray.cli._deprecate import DeprecatedAlias + def cmd_stamp(args: argparse.Namespace) -> int: """Write the processing_stage marker into stage files written before it existed.""" from ..pipeline._stage import StageError from ..pipeline.stamp import stamp_file + if args.stage is None: + print("oceanarray stamp: error: --declare N is required", file=sys.stderr) + return 2 failed = False for path in args.files: try: @@ -35,14 +41,30 @@ def build_parser(sub: argparse._SubParsersAction) -> None: # type: ignore[type- "warn. Refuses a file whose name or existing marker says another stage.\n" "Exits 0 if every file is marked, 1 if any was refused." ), + epilog=( + "Examples:\n" + " # mark one mooring's stage-1 files\n" + " oceanarray stamp /data/proc/dsG3_1_2026/*/*_stage1.nc --declare 1" + ), formatter_class=argparse.RawDescriptionHelpFormatter, ) p_stamp.add_argument("files", nargs="+", help="Stage file(s) to mark") + p_stamp.add_argument( + "--declare", + type=int, + choices=[1, 2, 3], + dest="stage", + default=None, + metavar="N", + help="The stage the files hold (1, 2 or 3); must match their filenames. Required.", + ) p_stamp.add_argument( "--stage", type=int, - required=True, choices=[1, 2, 3], - help="The stage the files hold (must match their filenames).", + dest="stage", + action=DeprecatedAlias, + new="--declare", + help=argparse.SUPPRESS, ) p_stamp.set_defaults(func=cmd_stamp) diff --git a/oceanarray/cli/validate.py b/oceanarray/cli/validate.py index 195c33f7..8de6fc4b 100644 --- a/oceanarray/cli/validate.py +++ b/oceanarray/cli/validate.py @@ -3,17 +3,34 @@ from __future__ import annotations import argparse +from pathlib import Path + +from oceanarray import paths +from oceanarray.cli._common import _add_dir_args + + +def _description_path(given: str, proc_dir: str | None) -> Path: + """Return the ``*.mooring.yaml`` for a path, a mooring folder, or a name with ``--proc-dir``.""" + path = Path(given) + if path.is_dir(): + found = sorted(path.glob("*.mooring.yaml")) + if len(found) == 1: + return found[0] + return path / f"{path.name}.mooring.yaml" + if not path.exists() and proc_dir: + return paths.mooring_proc_dir(Path(proc_dir), given) / f"{given}.mooring.yaml" + return path def cmd_validate(args: argparse.Namespace) -> int: - """Validate one or more mooring YAML files.""" + """Validate each mooring description; 1 on any error (or, with --strict, any warning).""" all_ok = True - for yaml_path in args.yaml: - ok = _print_report(yaml_path) - if not ok: - all_ok = False + for given in args.mooring: + ok = _print_report( + str(_description_path(given, args.proc_dir)), strict=args.strict + ) + all_ok = all_ok and ok print() - return 0 if all_ok else 1 @@ -21,21 +38,39 @@ def build_parser(sub: argparse._SubParsersAction) -> None: # type: ignore[type- """Add the ``validate`` subcommand to *sub*.""" p_validate = sub.add_parser( "validate", - help="Validate mooring YAML configuration file(s).", + help="Check mooring descriptions (.mooring.yaml) without processing anything.", description=( - "Parse and validate one or more .mooring.yaml files.\n" - "Checks instrument types, serial numbers, file_type values, HAB fields,\n" - "deployment dates, and inline instrument conventions.\n" - "Exits 0 if all files are valid, 1 if any file has errors or warnings." + "Read and check one or more mooring descriptions: instrument types, serial\n" + "numbers, file types, heights above bottom, deployment dates, inline\n" + "instruments, QC override blocks and the OceanSITES codes. Prints every ERROR\n" + "and WARNING. Exits 0 if no description has an ERROR (with --strict: no\n" + "WARNING either), 1 otherwise." + ), + epilog=( + "Examples:\n" + " # one description\n" + " oceanarray validate /data/proc/dsG3_1_2026/dsG3_1_2026.mooring.yaml\n" + " # every mooring of a cruise, failing on warnings too\n" + " oceanarray validate /data/proc/*/*.mooring.yaml --strict\n" + " # by name\n" + " oceanarray validate dsG3_1_2026 --proc-dir /data/proc" ), formatter_class=argparse.RawDescriptionHelpFormatter, ) - p_validate.add_argument("yaml", nargs="+", help="Path(s) to .mooring.yaml file(s)") + p_validate.add_argument( + "mooring", + nargs="+", + help="Mooring description(s): a .mooring.yaml, its folder, or a name with --proc-dir.", + ) + _add_dir_args(p_validate, raw_needed=False) + p_validate.add_argument( + "--strict", action="store_true", help="Exit 1 on warnings as well as errors." + ) p_validate.set_defaults(func=cmd_validate) -def _print_report(yaml_path: str) -> bool: - """Print a human-readable validation report; return True when there are no errors.""" +def _print_report(yaml_path: str, *, strict: bool = False) -> bool: + """Print a human-readable validation report; return True when the description passes.""" from oceanarray import mooring as _mooring issues = _mooring.check(yaml_path) @@ -60,4 +95,4 @@ def _print_report(yaml_path: str) -> bool: summary.append(f"{len(warnings)} warning(s)") print(f" {', '.join(summary)}") - return len(errors) == 0 + return not errors and not (strict and warnings) diff --git a/oceanarray/config/parameters.py b/oceanarray/config/parameters.py index d5045aac..da6f54ae 100644 --- a/oceanarray/config/parameters.py +++ b/oceanarray/config/parameters.py @@ -596,7 +596,7 @@ "current_direction": { "label": "Current direction", "units": "degree", - "standard_name": "direction_of_sea_water_velocity", + "standard_name": "sea_water_velocity_to_direction", "cmap": None, }, "bin_pressure": { diff --git a/oceanarray/mooring.py b/oceanarray/mooring.py index 6b3dd052..26e69d6a 100644 --- a/oceanarray/mooring.py +++ b/oceanarray/mooring.py @@ -77,8 +77,11 @@ "deployment_longitude", "planned_latitude", "planned_longitude", - # QC thresholds (read by the QC pass). + # QC thresholds (read by the QC pass; shapes checked by _check_qc_overrides). "qc_ranges", + "qc_spike", + "tilt_qc", + "qc_flat_line", # Deprecated: a raw-data root in the description is a run-time fact; use # raw_dir / --raw-dir. check() WARNs; the override is removed in 0.7.0. "directory", @@ -224,6 +227,7 @@ def check( if isinstance(data, Mapping): issues += _check_settings(data) issues += _check_export_codes(data) + issues += _check_qc_overrides(data) for key in data: if key not in KNOWN_KEYS: issues.append( @@ -271,6 +275,123 @@ def _check_export_codes(data: Mapping) -> list[ValidationIssue]: return issues +#: The QC override blocks, at mooring level and in each instrument entry: whether +#: the block is keyed by variable, and the fields each entry may hold. +_QC_BLOCKS: dict[str, tuple[bool, frozenset[str]]] = { + "qc_ranges": (True, frozenset({"fail_span", "suspect_span"})), + "qc_spike": (True, frozenset({"suspect_threshold", "fail_threshold"})), + "qc_flat_line": (True, frozenset({"suspect_n", "fail_n", "tolerance"})), + "tilt_qc": (False, frozenset({"suspect_threshold", "fail_threshold"})), +} + + +def _is_number(value: object) -> bool: + """Return True for an int or float (not a bool).""" + return isinstance(value, int | float) and not isinstance(value, bool) + + +def _qc_field_issue(field: str, value: object) -> str | None: + """Return what is wrong with one QC field's value, or None if it is valid.""" + if field.endswith("_span"): + if ( + not isinstance(value, list | tuple) + or len(value) != 2 + or not all(_is_number(v) for v in value) + or not value[0] < value[1] + ): + return "must be two increasing numbers, e.g. [0, 40]" + elif field in ("suspect_n", "fail_n"): + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + return "must be a whole number of samples, 1 or more" + elif not _is_number(value) or value < 0: + return "must be a number, 0 or more" + return None + + +def _check_qc_block(block: str, value: object, where: str) -> list[ValidationIssue]: + """Check one QC override block's shape; unknown variables are only a WARNING.""" + from oceanarray.config import parameters as params + + by_variable, fields = _QC_BLOCKS[block] + allowed = ", ".join(sorted(fields)) + if not isinstance(value, Mapping): + return [ + ValidationIssue("ERROR", f"{where}{block} must be a mapping, not {value!r}") + ] + entries = value.items() if by_variable else [(None, value)] + issues: list[ValidationIssue] = [] + for var, entry in entries: + label = f"{where}{block}" + (f".{var}" if var is not None else "") + if var is not None: + if not isinstance(entry, Mapping): + issues.append( + ValidationIssue("ERROR", f"{label} must be a mapping of {allowed}") + ) + continue + if var not in params.VARIABLES and var != "tilt": + # A configured variable the registry does not know is most likely a + # typo; it is still allowed (the file may hold such a variable). + issues.append( + ValidationIssue( + "WARNING", + f"{label}: {var!r} is not a registered variable; check the " + "spelling (thresholds apply only if a file has it)", + ) + ) + if ( + block == "qc_ranges" + and var not in params.QC_GROSS_RANGE + and var != "tilt" + and "fail_span" not in entry + ): + issues.append( + ValidationIssue( + "ERROR", + f"{label} needs a fail_span: {var!r} has no default to start from", + ) + ) + for field, field_value in entry.items(): + if field not in fields: + issues.append( + ValidationIssue( + "ERROR", f"{label}: unknown field {field!r}; allowed: {allowed}" + ) + ) + continue + problem = _qc_field_issue(field, field_value) + if problem: + issues.append( + ValidationIssue( + "ERROR", f"{label}.{field} = {field_value!r} {problem}" + ) + ) + return issues + + +def _check_qc_overrides(data: Mapping) -> list[ValidationIssue]: + """Check the QC override blocks at mooring level and in every instrument entry. + + A misspelt field or a malformed span would otherwise be dropped silently by + the threshold merge, and QC would run with the defaults. + """ + issues: list[ValidationIssue] = [] + for block in _QC_BLOCKS: + if block in data: + issues += _check_qc_block(block, data[block], "") + lists = [data.get(key) for key in ("clamp", "instruments", "inline")] + for entries in lists: + if not isinstance(entries, list): + continue + for i, entry in enumerate(entries): + if not isinstance(entry, Mapping): + continue + where = f"instrument {entry.get('serial', f'index {i}')}: " + for block in _QC_BLOCKS: + if block in entry: + issues += _check_qc_block(block, entry[block], where) + return issues + + def _deployment_month(deployment_time: object) -> str | None: """Return ``YYYYMM`` of a deployment time, or None if it cannot be read.""" if deployment_time is None: diff --git a/oceanarray/pipeline/coordinate.py b/oceanarray/pipeline/coordinate.py index 0d92db91..f818ceff 100644 --- a/oceanarray/pipeline/coordinate.py +++ b/oceanarray/pipeline/coordinate.py @@ -254,7 +254,7 @@ def _warn(msg: str) -> None: ( "current_direction", direction, - "direction_of_sea_water_velocity", + "sea_water_velocity_to_direction", "degrees", "Current direction (0=N, clockwise)", ), diff --git a/oceanarray/pipeline/export.py b/oceanarray/pipeline/export.py index 8757e870..b7596c71 100644 --- a/oceanarray/pipeline/export.py +++ b/oceanarray/pipeline/export.py @@ -42,6 +42,16 @@ def _array_description(array: str | PathLike | Mapping | None) -> Mapping: return data +def _put(out: xr.Dataset, target: Path, *, overwrite: bool, dry_run: bool) -> Path: + """Write *out* under its OceanSITES name in *target*, or only name it on a dry run.""" + path = target / oceansites_filename(out) + if dry_run: + if path.exists() and not overwrite: + raise FileExistsError(f"{path} exists (use overwrite=True / --force)") + return path + return write(out, path, overwrite=overwrite) + + def export( mooring: str | PathLike | Mapping[str, Any], *, @@ -50,6 +60,7 @@ def export( array: str | PathLike | Mapping | None = None, grid: bool = True, overwrite: bool = False, + dry_run: bool = False, ) -> list[Path]: """Write a mooring's stage-3 files and grid as OceanSITES 1.4 files. @@ -75,6 +86,9 @@ def export( overwrite : bool, optional Replace files that exist (default False). + dry_run : bool, optional + Convert but write nothing; return the paths that would be written. + Returns ------- list of pathlib.Path @@ -117,9 +131,7 @@ def export( out = to_oceansites( ds.load(), cfg, array=array_cfg, source_file=path.name ) - written.append( - write(out, target / oceansites_filename(out), overwrite=overwrite) - ) + written.append(_put(out, target, overwrite=overwrite, dry_run=dry_run)) except _SKIPPABLE as e: # One instrument that cannot be exported does not stop the others. left_out.append(f"{serial}: {e}") @@ -137,9 +149,7 @@ def export( stack=stack, source_file=grid_path.name, ) - written.append( - write(out, target / oceansites_filename(out), overwrite=overwrite) - ) + written.append(_put(out, target, overwrite=overwrite, dry_run=dry_run)) except _SKIPPABLE as e: left_out.append(f"grid: {e}") finally: diff --git a/tests/integration/test_cli_paths.py b/tests/integration/test_cli_paths.py index 71e5f904..f7275182 100644 --- a/tests/integration/test_cli_paths.py +++ b/tests/integration/test_cli_paths.py @@ -53,13 +53,18 @@ def test_only_and_the_old_serial_spelling(folder, capsys): ) assert other.stat().st_mtime == t_other assert target.exists() - assert "--serial is now --only" in capsys.readouterr().err + assert ( + "--serial is deprecated and removed in 0.7.0; use --only" + in capsys.readouterr().err + ) -def test_process_stack_without_proc_dir_points_at_run(folder): +def test_process_stack_without_proc_dir_points_at_run(folder, capsys): """The stack and grid through process need --proc-dir; the message names run.""" - with pytest.raises(SystemExit, match="oceanarray run"): + with pytest.raises(SystemExit) as exit_info: _cli(["process", str(folder), "--stage", "stack"]) + assert exit_info.value.code == 2 # a usage error + assert "oceanarray run" in capsys.readouterr().err def test_a_path_that_does_not_exist_is_refused(tmp_path): @@ -83,7 +88,10 @@ def test_old_plot_flag_spellings_still_parse(capsys): ] ) assert (ns.var_y, ns.var_color) == ("pressure", "temperature") - assert "--var_y is now --var-y" in capsys.readouterr().err + assert ( + "--var_y is deprecated and removed in 0.7.0; use --var-y" + in capsys.readouterr().err + ) def test_verbose_lowers_the_stage_console_level(monkeypatch): @@ -216,14 +224,17 @@ def test_report_short_flags_are_old_spellings(folder, capsys): assert (ns.instruments, ns.stack, ns.grid, ns.all_reports) == (True,) * 4 err = capsys.readouterr().err for new in ("--instruments", "--stack", "--grid", "--all"): - assert f"is now {new}" in err + assert f"use {new}" in err def test_dt_is_the_old_spelling_of_dt_seconds(capsys): """--dt still sets the stack time step, and says --dt-seconds.""" ns = build_parser().parse_args(["run", "m", "--proc-dir", ".", "--dt", "600"]) assert ns.dt_seconds == 600 - assert "--dt is now --dt-seconds" in capsys.readouterr().err + assert ( + "--dt is deprecated and removed in 0.7.0; use --dt-seconds" + in capsys.readouterr().err + ) def test_process_dry_run_lists_every_stage_and_the_interpolated_pressure( @@ -243,12 +254,15 @@ def test_process_dry_run_lists_every_stage_and_the_interpolated_pressure( assert {p: p.stat().st_mtime for p in before} == before -def test_process_strict_with_stack_points_to_run(tmp_path): +def test_process_strict_with_stack_points_to_run(tmp_path, capsys): """process --stage stack --strict is refused, naming oceanarray run --strict.""" from oceanarray.cli import build_parser + (tmp_path / "m").mkdir() ns = build_parser().parse_args( ["process", "m", "--proc-dir", str(tmp_path), "--stage", "stack", "--strict"] ) - with pytest.raises(SystemExit, match="oceanarray run MOORING --strict"): + with pytest.raises(SystemExit) as exit_info: ns.func(ns) + assert exit_info.value.code == 2 + assert "oceanarray run MOORING --strict" in capsys.readouterr().err diff --git a/tests/integration/test_stamp.py b/tests/integration/test_stamp.py index 3c436f04..088a6b0a 100644 --- a/tests/integration/test_stamp.py +++ b/tests/integration/test_stamp.py @@ -86,9 +86,25 @@ def test_cli_stamp_exits_nonzero_when_any_file_is_refused(tmp_path, capsys): """`oceanarray stamp` marks what it can and exits 1 when one file is refused.""" good = _unmarked_copy(_STAGE3, tmp_path / _STAGE3.name) bad = shutil.copy(_FIX / "dune2_1_2026_grid.nc", tmp_path) - args = build_parser().parse_args(["stamp", str(good), str(bad), "--stage", "3"]) + args = build_parser().parse_args(["stamp", str(good), str(bad), "--declare", "3"]) assert args.func(args) == 1 out = capsys.readouterr().out assert "stamped: dune2_1_2026_2941_stage3.nc" in out assert "not a stage file" in out assert _open(good).attrs[STAGE_ATTR] == 3 + + +def test_cli_stamp_old_stage_flag_warns_and_still_works(tmp_path): + """``--stage`` is the old spelling of ``--declare``: it works for one release, with a warning.""" + good = _unmarked_copy(_STAGE3, tmp_path / _STAGE3.name) + with pytest.warns(DeprecationWarning, match="--declare"): + args = build_parser().parse_args(["stamp", str(good), "--stage", "3"]) + assert args.func(args) == 0 + assert _open(good).attrs[STAGE_ATTR] == 3 + + +def test_cli_stamp_without_declare_is_a_usage_error(tmp_path): + """No stage given: exit 2, nothing written.""" + good = _unmarked_copy(_STAGE3, tmp_path / _STAGE3.name) + args = build_parser().parse_args(["stamp", str(good)]) + assert args.func(args) == 2 diff --git a/tests/unit/test_export.py b/tests/unit/test_export.py index fd641bcb..72f6cb6a 100644 --- a/tests/unit/test_export.py +++ b/tests/unit/test_export.py @@ -88,3 +88,31 @@ def test_export_written_coordinates_have_no_fill_value(tmp_path: Path) -> None: with netCDF4.Dataset(path) as nc: for name in ("TIME", "DEPTH", "LATITUDE", "LONGITUDE"): assert "_FillValue" not in nc[name].ncattrs(), (path.name, name) + + +def test_cli_export_dry_run_writes_nothing_and_force_replaces( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture +) -> None: + """-n lists the files without writing; --force replaces existing ones.""" + from oceanarray.cli import main + + yaml_path = _copy(tmp_path) + out = tmp_path / "os" + + def run(*flags: str) -> int: + monkeypatch.setattr( + sys, + "argv", + ["oceanarray", "export", str(yaml_path), "-o", str(out), *flags], + ) + with pytest.raises(SystemExit) as exit_info: + main() + return exit_info.value.code + + assert run("-n") == 0 + assert not out.exists() + assert "would write" in capsys.readouterr().out + assert run() == 0 + assert run("--force") == 0 + with pytest.warns(DeprecationWarning, match="--force"): + assert run("--overwrite") == 0 diff --git a/tests/unit/test_mooring.py b/tests/unit/test_mooring.py index dbed51b0..4d329f61 100644 --- a/tests/unit/test_mooring.py +++ b/tests/unit/test_mooring.py @@ -196,3 +196,95 @@ def test_init_writes_the_optional_blocks_commented_out(tmp_path: Path) -> None: assert not [ i for i in mooring.check(doc) if "stack" in i.message or "grid" in i.message ] + + +def _qc_issues(**keys: object) -> list[tuple[str, str]]: + """Return (level, message) for the QC-override findings on the fixture plus *keys*.""" + cfg = {**mooring.load(FIXTURE), **keys} + return [ + (i.level, i.message) + for i in mooring.check(cfg, dir_name="dune2_1_2026") + if any( + b in i.message for b in ("qc_ranges", "qc_spike", "tilt_qc", "qc_flat_line") + ) + ] + + +def test_qc_overrides_as_documented_pass() -> None: + """The documented qc_ranges example and well-formed spike/tilt/flat-line blocks raise nothing.""" + assert ( + _qc_issues( + qc_ranges={ + "temperature": { + "fail_span": [-2.0, 35.0], + "suspect_span": [-1.5, 30.0], + }, + "pitch": {"fail_span": [-30.0, 30.0], "suspect_span": [-20.0, 20.0]}, + "tilt": {"suspect_span": [-20, 20], "fail_span": [-30, 30]}, + }, + qc_spike={"pressure": {"suspect_threshold": 10.0, "fail_threshold": 50.0}}, + tilt_qc={"suspect_threshold": 20, "fail_threshold": 30}, + qc_flat_line={ + "pressure": {"suspect_n": 3, "fail_n": 10, "tolerance": 0.001} + }, + ) + == [] + ) + + +@pytest.mark.parametrize( + ("keys", "fragment"), + [ + ( + {"qc_ranges": {"temperature": {"suspect_spn": [0, 1]}}}, + "unknown field 'suspect_spn'", + ), + ({"qc_ranges": {"salinity": {"fail_span": [40, 0]}}}, "two increasing numbers"), + ( + {"qc_ranges": {"salinity": {"fail_span": [0, 40, 1]}}}, + "two increasing numbers", + ), + ({"qc_spike": {"pressure": {"fail_threshold": "ten"}}}, "must be a number"), + ({"qc_flat_line": {"pressure": {"fail_n": 2.5}}}, "whole number"), + ({"tilt_qc": {"fail": 30}}, "unknown field 'fail'"), + ({"tilt_qc": [20, 30]}, "must be a mapping"), + ({"qc_ranges": {"sigma0": {"suspect_span": [20, 30]}}}, "needs a fail_span"), + ], +) +def test_qc_override_shape_mistakes_are_errors(keys: dict, fragment: str) -> None: + """A misspelt field or malformed value is an ERROR, so load() refuses the description.""" + issues = _qc_issues(**keys) + assert any(level == "ERROR" and fragment in msg for level, msg in issues), issues + with pytest.raises(MooringError): + mooring.load({**mooring.load(FIXTURE), **keys}) + + +def test_qc_override_unknown_variable_only_warns() -> None: + """A variable outside the registry is a WARNING (likely a typo, but allowed).""" + issues = _qc_issues(qc_spike={"temprature": {"fail_threshold": 5.0}}) + assert [level for level, _ in issues] == ["WARNING"] + + +def test_instrument_level_qc_overrides_are_checked() -> None: + """The same checks run on each instrument entry, naming its serial.""" + cfg = mooring.load(FIXTURE) + cfg["clamp"][0] = { + **cfg["clamp"][0], + "qc_ranges": {"temperature": {"fail_spn": [0, 1]}}, + } + serial = cfg["clamp"][0]["serial"] + messages = [i.message for i in mooring.check(cfg, dir_name="dune2_1_2026")] + assert any(f"instrument {serial}: qc_ranges.temperature" in m for m in messages) + + +def test_mooring_level_qc_blocks_are_known_keys() -> None: + """qc_spike, tilt_qc and qc_flat_line are read by the QC pass, so not 'unknown'.""" + cfg = { + **mooring.load(FIXTURE), + "qc_spike": {}, + "tilt_qc": {}, + "qc_flat_line": {}, + } + assert not [ + i for i in mooring.check(cfg, dir_name="dune2_1_2026") if "Unknown" in i.message + ] From 1ee62064ffae82f68aba1f8be5bee58148d99496 Mon Sep 17 00:00:00 2001 From: Eleanor Frajka-Williams Date: Thu, 5 Nov 2026 07:48:09 +0100 Subject: [PATCH 2/2] Let an instrument's qc_ranges take its fail_span from the mooring level --- docs/source/yaml_configuration.rst | 3 ++- oceanarray/mooring.py | 15 +++++++++++++-- tests/unit/test_mooring.py | 15 +++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/docs/source/yaml_configuration.rst b/docs/source/yaml_configuration.rst index de9c2491..cd36a81b 100644 --- a/docs/source/yaml_configuration.rst +++ b/docs/source/yaml_configuration.rst @@ -842,7 +842,8 @@ the defaults): - a threshold that is not a number of 0 or more; - a sample count that is not a whole number of 1 or more; - a block that is not a mapping; -- a ``qc_ranges`` variable without a default and without a ``fail_span``. +- a ``qc_ranges`` variable without a default and without a ``fail_span`` (an + instrument entry may take the ``fail_span`` from the mooring-level block). A variable that is not in the registry is only a WARNING (most often a typo): its thresholds apply if a file has that variable. diff --git a/oceanarray/mooring.py b/oceanarray/mooring.py index 26e69d6a..0e55e2b7 100644 --- a/oceanarray/mooring.py +++ b/oceanarray/mooring.py @@ -308,7 +308,9 @@ def _qc_field_issue(field: str, value: object) -> str | None: return None -def _check_qc_block(block: str, value: object, where: str) -> list[ValidationIssue]: +def _check_qc_block( + block: str, value: object, where: str, inherited: object = None +) -> list[ValidationIssue]: """Check one QC override block's shape; unknown variables are only a WARNING.""" from oceanarray.config import parameters as params @@ -343,6 +345,13 @@ def _check_qc_block(block: str, value: object, where: str) -> list[ValidationIss and var not in params.QC_GROSS_RANGE and var != "tilt" and "fail_span" not in entry + # An instrument entry inherits the mooring-level spans (the two + # are merged before QC runs). + and not ( + isinstance(inherited, Mapping) + and isinstance(inherited.get(var), Mapping) + and "fail_span" in inherited[var] + ) ): issues.append( ValidationIssue( @@ -388,7 +397,9 @@ def _check_qc_overrides(data: Mapping) -> list[ValidationIssue]: where = f"instrument {entry.get('serial', f'index {i}')}: " for block in _QC_BLOCKS: if block in entry: - issues += _check_qc_block(block, entry[block], where) + issues += _check_qc_block( + block, entry[block], where, data.get(block) + ) return issues diff --git a/tests/unit/test_mooring.py b/tests/unit/test_mooring.py index 4d329f61..fd5fc191 100644 --- a/tests/unit/test_mooring.py +++ b/tests/unit/test_mooring.py @@ -277,6 +277,21 @@ def test_instrument_level_qc_overrides_are_checked() -> None: assert any(f"instrument {serial}: qc_ranges.temperature" in m for m in messages) +def test_instrument_qc_ranges_inherit_the_mooring_fail_span() -> None: + """A fail_span set at mooring level counts for an instrument that adds only a suspect_span.""" + cfg = mooring.load(FIXTURE) + cfg["qc_ranges"] = {"sigma0": {"fail_span": [20.0, 30.0]}} + cfg["clamp"][0] = { + **cfg["clamp"][0], + "qc_ranges": {"sigma0": {"suspect_span": [25.0, 28.0]}}, + } + messages = [i.message for i in mooring.check(cfg, dir_name="dune2_1_2026")] + assert not [m for m in messages if "needs a fail_span" in m], messages + del cfg["qc_ranges"] + messages = [i.message for i in mooring.check(cfg, dir_name="dune2_1_2026")] + assert any("needs a fail_span" in m for m in messages) + + def test_mooring_level_qc_blocks_are_known_keys() -> None: """qc_spike, tilt_qc and qc_flat_line are read by the QC pass, so not 'unknown'.""" cfg = {