From 3387dfaa0bde64da556b5acc4fb8c86d0e359e30 Mon Sep 17 00:00:00 2001 From: Aljes Date: Tue, 3 Nov 2026 15:37:39 +0000 Subject: [PATCH 1/5] B10: escape store-derived text, clean stderr off a terminal, honour argv; docs sweep Names, labels, paths and attribute values are escaped before Rich prints them, so view, ls, split and every message show them as the store spells them (F042); convert --help keeps pyadata-cli[spatial] and [pickle] (F160). view on a missing entry fails on stderr with exit 1 (F096). The stderr console no longer forces a terminal, so a log holds no colour or cursor codes and an error stays on one line (F151). main(argv) and main_deprecated(argv) run the arguments given (F161). Help and reference contradictions fixed (F159); AGENTS.md, developers.md, testing.md, troubleshooting.md, pipelines.md and STATUS.md brought up to date (F149, F150, F155, F157). The docs test now also checks that every test file is in testing.md's map and every option in commands.md. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 62 +++++-- README.md | 6 +- STATUS.md | 52 ++++-- docs/changelog.md | 6 +- docs/developers.md | 49 ++++-- docs/guides/convert.md | 11 +- docs/guides/pipelines.md | 15 +- docs/index.md | 5 +- docs/reference/commands.md | 55 +++++- docs/reference/testing.md | 70 +++++--- docs/troubleshooting.md | 10 +- pyproject.toml | 10 +- src/adata/cli.py | 172 ++++++++++-------- src/adata/commands/convert.py | 14 +- src/adata/commands/create.py | 3 +- src/adata/commands/export.py | 3 +- src/adata/commands/import_data.py | 5 +- src/adata/commands/info.py | 41 +++-- src/adata/commands/ls.py | 16 +- src/adata/commands/recast.py | 7 +- src/adata/commands/split.py | 12 +- src/adata/conversion/__init__.py | 24 ++- src/adata/conversion/anndata_copy.py | 3 +- src/adata/conversion/cellbender.py | 3 +- src/adata/conversion/cistopic.py | 3 +- src/adata/conversion/hdf5.py | 3 +- src/adata/conversion/loom.py | 7 +- src/adata/conversion/mtx.py | 11 +- src/adata/conversion/mudata.py | 5 +- src/adata/conversion/parse.py | 9 +- src/adata/conversion/tenx.py | 5 +- src/adata/conversion/text.py | 3 +- src/adata/conversion/visium.py | 3 +- src/adata/core/concat.py | 25 +-- src/adata/core/recast.py | 5 +- src/adata/core/subset.py | 7 +- src/adata/formats/array.py | 9 +- src/adata/formats/dataframe.py | 9 +- src/adata/formats/image.py | 3 +- src/adata/formats/json_data.py | 5 +- src/adata/formats/sparse.py | 9 +- src/adata/formats/validate.py | 3 +- tests/test_cli.py | 8 +- tests/test_cli_output.py | 253 +++++++++++++++++++++++++++ tests/test_docs_are_accurate.py | 91 ++++++++++ 45 files changed, 863 insertions(+), 267 deletions(-) create mode 100644 tests/test_cli_output.py diff --git a/AGENTS.md b/AGENTS.md index dfa9b56..6c29d5a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,9 +24,10 @@ Naming: - The import package and the command are `adata`. - The repo directory is still called `h5ad-cli`. - `h5ad` is a deprecated alias of the command (`main_deprecated` in `cli.py`). +- `info` is a hidden, deprecated alias of `view`. - `convert` given recast's old flags forwards to `recast` with a warning. -Both of these deprecated paths stay until 1.0.0. +These deprecated paths stay until 1.0.0. ## Commands @@ -48,10 +49,17 @@ Test markers: in `tests/reference_stores.py`, so they need the network on the first run. - `perf` tests are slow, but CI runs them in the main job. -CI also installs the package from a fresh resolution, without the lock, -imports every module on the base dependencies alone, and runs `convert`. -That catches a module-level import of something undeclared; one inside a -function is caught only on the paths it runs. Don't import click: typer +CI (`.github/workflows/tests.yml`) runs on every pull request: +- the unit suite on Python 3.12 and 3.13, with the coverage floor; +- a fresh install: the package from a fresh resolution, without the lock, + every module imported on the base dependencies alone, and `convert` run. + That catches a module-level import of something undeclared; one inside a + function is caught only on the paths it runs; +- the unit suite at each direct dependency's floor; +- the `integration` tests of `test_anndata_versions.py` and + `test_external_readers.py`, each its own job; +- `workflow lint` (actionlint and zizmor) and `container image`, which + builds the Docker image and smoke-tests it. Don't import click: typer 0.26 and later no longer install it. Compare the enums typer hands back (such as `ParameterSource`) by `.name`. @@ -60,7 +68,9 @@ surrounding code. ## Architecture -Each layer calls only the layers below it: +Each layer calls only the layers below it, with one exception: `storage/` +uses `elements/` (the spec, `strings`, and the readers and writers that +`copy_tree` and `create_dataset` need to copy an element faithfully). - **`cli.py`**: the whole Typer app, including the `export` and `import` sub-apps. - Commands parse options, validate cheap things such as `--zarr-format`, call @@ -69,7 +79,11 @@ Each layer calls only the layers below it: `--chunk` its `min=1`, and `_space_check_option` is `--no-space-check`. A new command that takes either uses these, not its own `typer.Option`. - `console` writes to **stderr**. Only `out_console` and data go to stdout, - so output pipes cleanly. + so output pipes cleanly. Rich asks each stream whether it is a terminal: + off one, as in a log file or a pipe, it writes no colour, cursor or + spinner codes, and `console` keeps each message on one line. + - `raise _fail(e)` prints an exception as the one `Error:` line on stderr + and exits 1. `main(argv)` runs the arguments it is given. - **`commands/`**: one module per command. Each orchestrates its command: - planning and refusals; - progress output; @@ -126,7 +140,11 @@ Each layer calls only the layers below it: in the matrix's order; it also reads the `analysis/` (clusters, UMAP, PCA) that Xenium and Space Ranger (spots, bins and cells) share. - **`formats/`**: the per-file-type halves of `export` and `import`: npy, mtx, - csv, json and images. `mtx.py` is a streaming Matrix Market codec (two + csv, json and images, with `validate.py` checking an element's kind and + dimensions before an import writes it. Beside them are what converters + read: `parquet.py` (Visium HD's positions, the only pyarrow import), + `pickled.py` (pycisTopic's pickles, through an allowlist) and + `pandas_frame.py` (pandas frames written as anndata writes them). `mtx.py` is a streaming Matrix Market codec (two passes in, block writes out) shared by `import`/`export sparse` and the `10x-mtx` and `parse` converters; `textio.py` holds its gzip-aware block I/O, and `table.py` a streaming CSV reader that types columns as pandas @@ -184,7 +202,15 @@ These are easy to break without any test noticing straight away: no cleanup runs. Honour `--no-space-check`. - **Validate before the output exists.** Resolve selections, check casts and build plans first, so that a refusal writes nothing. An empty selection is - a refusal, never an empty store. + a refusal, never an empty store. What only the data can show (an index out + of range) refuses during the write; staging keeps the output from + appearing. +- **Escape what the store or the user wrote before Rich prints it.** Rich + reads `[word]` as a style tag, so `pct_counts[mt]` printed as it is shows + `pct_counts`, and `[/x]` crashes the command. Member names, labels, paths + and attribute values go through `rich.markup.escape`, in messages, status + lines and progress descriptions alike. Help text is markup too: a literal + `[` there is `\\[` in the source, which the docstring holds as `\[`. - **Zarr outputs keep the source's version.** - Pass `zarr_format=` to `open_store`. Without it zarr-python writes v3, and a v2 input gets silently upgraded. @@ -199,20 +225,30 @@ These are easy to break without any test noticing straight away: `zarr3`. Write element-level tests against it, so one test body covers all three backends. - CLI tests use a module-level `CliRunner`. Strip ANSI codes and collapse - whitespace before matching messages, because Rich wraps them. The CLI test - files define an `_out(result)` helper for this. + whitespace before matching messages: `FORCE_COLOR` brings the codes back, + and help text and stdout's trees wrap at Rich's width. The CLI test files + define an `_out(result)` helper for this. - Correctness is checked against an oracle: anndata reads back what we write, and scipy checks `recast`. Self-consistency alone is not accepted. - `tests/test_performance.py` guards complexity by **counting** elements and bytes read through hooks on h5py and zarr (`tests/perf_counters.py`). It never uses a clock. A change that adds reads per row or per group will fail it. + - The same hooks count bytes: the largest single read (the block a loop + holds), and the whole chunks and shards each read decodes and each write + encodes. + - `perf_counters.peak_rss(argv)` runs `adata` in a fresh interpreter and + returns the peak memory it added. The memory guards grow the input + along n_obs and n_var and assert the peak is a small share of it. + - `tests/perf_fixtures.py` builds stores as anndata writes them: every + layout, h5ad, Zarr v2 and anndata 0.13's sharded Zarr v3. - `tests/test_docs_are_accurate.py` parses every `adata …` line in the README, `docs/` and the `--help` examples, and checks it against the real Click tree. A new or renamed option has to be documented correctly in `docs/reference/commands.md`. A new command also needs a section there, an entry in the table of contents and a mention in the app help. The same file - checks that relative links name a page and an anchor that exist, and that - every page has `title` front matter and a `parent` that exists. + checks that relative links name a page and an anchor that exist, that + every page has `title` front matter and a `parent` that exists, and that + every test file has a row in `docs/reference/testing.md`'s map. - House style: - test names read as sentences (`test_a_failed_selection_keeps_an_existing_output`); diff --git a/README.md b/README.md index 894baac..02a2835 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ Explore and edit huge AnnData stores (`.h5ad` and `.zarr`) from the command line without loading them into memory and without a Python session. `adata` inspects, subsets, splits, concatenates, recasts and converts stores, and moves elements in and out as CSV, JSON, `.npy`, `.mtx` or images. It implements the AnnData on-disk -spec directly against h5py and zarr, so it streams stores larger than RAM, and -converts to and from 10x, STARsolo, Visium, Xenium, CellBender, Parse, loom, MuData -and more. +spec directly against h5py and zarr, so it streams stores larger than RAM. It +reads 10x, STARsolo, Visium, Xenium, CellBender, Parse, loom, MuData and more, +and writes 10x, loom and MuData. **Documentation: ** diff --git a/STATUS.md b/STATUS.md index d927b4c..fbd503c 100644 --- a/STATUS.md +++ b/STATUS.md @@ -54,9 +54,42 @@ Steps: - [x] Wave 1, then 0.17.4 - [x] Wave 2, then 0.17.5 - [ ] Waves 3 and 4, then 0.17.6 + - [x] B6a: the memory harness, then recast streaming (#118) + - [x] B8c: pickles, Xenium and images (#120) + - [ ] B6b: block sizing and Zarr v3 shards (#121) + - [ ] B10: CLI output and the documentation sweep + - [ ] Release 0.17.6 ## Follow-ups noticed +- From waves 3 and 4 of the audit fixes: + - The streaming transpose and other users of `csr_build.BucketSorter` + append every input block to every bucket, (nnz/chunk)² appends, and + each bucket holds three growable datasets (1.8 GB at 30,000 buckets). + The transpose knows its row sizes, so fixed regions as in concat's + `_ColumnBuckets` would do; concat's `_csc_runs` and recast's + `_bucket_runs` could then move into `csr_build` (B3b, B6a). + - The transpose's scratch lives inside a Zarr output and is appended to + unbuffered, about 135 times its writes; an HDF5 scratch beside the + output, as concat has, would fix it. It is also missing from recast's + free-space estimate (B3b, B6a, B6b). + - recast's sparse reads are a fixed million entries, not `--chunk`; and + `concat`, `recast` and `export` read `--chunk` rows where one chunk + height is over the read budget, with column tiles only in `subset`, + `split` and `convert` (B6a, B6b). + - `copy_dataset` writes HDF5 text uncompressed: a Zarr to h5ad + conversion's obs index is about nine times anndata's (B6b). + - `convert` has no `--chunk`: the cisTopic and MuData converters use a + fixed byte budget (B8b, B8c). + - A pickle holding pandas 3's Arrow strings is refused; pycisTopic's + pandas 1.5 never writes one (B8c). + - Python warnings reach stderr with their source line; printing them as + `Warning: ...` lines would match the rest of the output (B5). + - CI's `external-readers` job is named "10x h5 against scanpy and + PyTables" and its `compatibility` job "anndata 0.8-0.13", though the + first checks every converter's reader and the second reads 0.7.8 too. + Renaming a job changes its required-check name, so it waits for the + repository settings (B10). - From wave 2 of the audit fixes: - CI: a Python 3.14 leg; Dependabot for the pinned actions, uv and base image (the pinned actions run on Node 20, which GitHub is retiring); the @@ -67,6 +100,14 @@ Steps: that fix's scope (B5). - Flex's `target_sets` group is not carried, and `--make-unique` names export with their suffixes (B8a). + - `formats/textio.py` does not unquote a field that starts with `"`, as + pandas does; if it did, MTX export could quote such names rather than + refuse them (B8a). + - A fixed-width text dataset that HDF5 compressed can inflate far beyond + its stored size when read whole, as the loom import reads var columns; + only loom's global attributes are bounded (B8b). + - Under an outer join, an input lacking some genes counts as lacking var + columns, varm and varp entirely; anndata pads them (B3b). - A numeric `in` list with thousands of values is slow (B4). - From wave 1 of the audit fixes: - `core/csr_build.py`'s on-disk sort makes many small appends, slow for @@ -86,17 +127,6 @@ Steps: - Dropped: spatial bundles whose matrix is a MEX folder only. 10x always writes the `.h5` beside it, and the refusal points to `--from 10x-mtx`. -- Fixed in 0.16.2 (#77): `export image` without `-o` crashed (`'NoneType' - object has no attribute 'parent'`); it now writes PNG to stdout, as every - other export writes there. Found by the docs review. -- Fixed in 0.16.1 (#74): `subset`, `split` and the MuData export kept - every category of a categorical column, where anndata's subsetting keeps - only those used (and narrows `uns["_colors"]`). The ~50% "slack" - noted here before was mostly a measuring error -- `get_storage_size()` - leaves out the variable-length string heap -- but the unpruned categories - were real: the pbmc3k Multiome `.h5mu` drops from 84.7 to 74.2 MB, and a - subset's obs and var now equal anndata's own subset. - ## Next candidates (#31) - alevin-fry USA mode, kallisto | bustools diff --git a/docs/changelog.md b/docs/changelog.md index 9732a7e..db07345 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -645,9 +645,9 @@ Parquet positions. ### Changed -- **Images are written a strip of rows at a time**, as they are read, rather - than loaded before the write starts: Visium HD's hires image is 6,000 - pixels, some 100 MB. +- **Images are copied into the store a strip of rows at a time**, rather + than as a second whole copy: Visium HD's hires image is 6,000 pixels, some + 100 MB. The image itself is still decoded whole, as a PNG can only be. - **Barcodes are matched as bytes**, a quarter of the memory of numpy's unicode, for Visium, Visium HD and Xenium alike. diff --git a/docs/developers.md b/docs/developers.md index e739246..dac8646 100644 --- a/docs/developers.md +++ b/docs/developers.md @@ -38,7 +38,7 @@ installed by the `integration` tests in environments of their own. | `src/adata/elements/` | The AnnData on-disk spec: encodings, readers for every layout since anndata 0.7, writers of the current one | | `src/adata/storage/` | One interface over h5py and zarr | | `src/adata/conversion/` | `adata convert`: one module per format, through AnnData | -| `src/adata/formats/` | The file types of `export` and `import`: npy, Matrix Market, CSV, JSON, images; and what the converters read: Parquet, GeoJSON polygons | +| `src/adata/formats/` | The file types of `export` and `import`: npy, Matrix Market, CSV, JSON, images; and what the converters read: Parquet, GeoJSON polygons, pycisTopic's pickles | | `src/adata/util/` | Output staging, path checks, names | | `tests/` | The test suite; [Testing](reference/testing.md) describes it | | `benchmarks/` | The comparative benchmark against anndata and scanpy | @@ -56,9 +56,18 @@ Each layer calls only those below it: cli.py ──► commands/ ──► core/, conversion/, formats/ ──► elements/ ──► storage/ ``` +The one exception: `storage/` uses `elements/` (the spec, `strings`, and the +readers and writers that `copy_tree` and `create_dataset` need to copy an +element faithfully). + - **`cli.py`** parses options and checks cheap things such as `--zarr-format`. Its `console` writes to stderr; only data and `out_console` go to stdout, - which is what lets output pipe cleanly. + which is what lets output pipe cleanly. Off a terminal, in a log file or a + pipe, neither writes colour, cursor or spinner codes, and a message on + stderr stays on one line. `raise _fail(e)` turns an exception into that + line, `Error: ...`, and exit code 1. `--chunk` and `--no-space-check` are + defined once, as `_chunk_option` and `_space_check_option`; a new command + reuses them. - **`commands/`** orchestrates one command: what to read, what to refuse, what to write where, and which Zarr version. - **`core/`** holds the algorithms that stream. `subset.py`'s writer serves @@ -131,7 +140,15 @@ before changing anything that writes: `--no-space-check` skips it. - **Validate before the output exists.** Resolve selections, check casts and plan, so that a refusal writes nothing. An empty selection is a refusal, - not an empty store. + not an empty store. What only the data can show, such as an index out of + range, refuses during the write, and staging keeps the output from + appearing. +- **Escape what the store or the user wrote before Rich prints it.** Rich + reads `[word]` as a style tag: printed as it is, `pct_counts[mt]` shows as + `pct_counts`, and `[/x]` crashes the command. Names, labels, paths and + attribute values go through `rich.markup.escape` in every message, status + line and progress description. Help text is markup too, so a literal `[` + in a docstring is written `\\[`. - **Zarr outputs keep the source's version.** Pass `zarr_format=` to `open_store`; without it zarr-python writes v3, upgrading a v2 input. `concat` uses the version its Zarr inputs share. @@ -155,13 +172,22 @@ before changing anything that writes: `concat` outputs. - **Complexity guards that count, not time.** `tests/test_performance.py` counts the elements and bytes read through hooks on h5py and zarr. A change - that adds reads per row or per group fails it on any machine. + that adds reads per row or per group fails it on any machine. The same + hooks (`tests/perf_counters.py`) count the largest read, which is the block + a loop holds, and the whole chunks and shards each read decodes and each + write encodes. +- **Memory measured as a share of the input.** `perf_counters.peak_rss` runs + `adata` in a fresh interpreter and returns the peak memory it added; the + memory guards grow a store along n_obs and n_var, built by + `tests/perf_fixtures.py` as anndata writes it (h5ad, Zarr v2 and anndata + 0.13's sharded Zarr v3), and assert the peak stays a small share of it. - **The docs are tested.** `tests/test_docs_are_accurate.py` checks every `adata …` line in the README, these pages and the `--help` examples against the real Click tree; that each command has a section and a contents entry in [Commands](reference/commands.md) and a mention in the app help; that - every relative link names a page and an anchor that exist; and that every - page has the front matter the sidebar needs. + every relative link names a page and an anchor that exist; that every + page has the front matter the sidebar needs; and that every test file has + a row in [Testing](reference/testing.md)'s map. ```bash uv run pytest -m "not integration" --cov=adata --cov-fail-under=90 # CI's coverage floor @@ -170,10 +196,13 @@ ADATA_REQUIRE_VERSION_FIXTURES=1 uv run pytest tests/test_anndata_versions.py -m ``` `integration` tests need the network on their first run, to build their -environments. CI also installs the package alone from a fresh resolution, -without the lock, imports every module and runs `convert`, so a module-level -import of something undeclared fails there. Don't import click, which typer -0.26 and later no longer install. House style: test names read as sentences, a regression test's +environments. CI runs the unit suite on Python 3.12 and 3.13 and again at +each dependency's floor, the anndata-release and external-reader tests as +jobs of their own, a lint of the workflows and a build and smoke test of the +image. It also installs the package alone from a fresh resolution, without +the lock, imports every module and runs `convert`, so a module-level import +of something undeclared fails there. Don't import click, which typer 0.26 +and later no longer install. House style: test names read as sentences, a regression test's docstring says what used to go wrong, and comments say why. No linter or formatter runs; match the code around you. diff --git a/docs/guides/convert.md b/docs/guides/convert.md index ff7ba5a..6d7fee6 100644 --- a/docs/guides/convert.md +++ b/docs/guides/convert.md @@ -96,9 +96,12 @@ output's extension names the other backend. ## Safe by construction -Every check runs before the output exists: an unknown format, a flag that -does not apply to this pair of formats, a 10x export with no feature ids. -The output is written beside its destination and moved into place only when -complete, and an existing output is replaced only with `--overwrite`. +The input and the options are checked before the output exists: an +unknown format, a flag that does not apply to this pair of formats, a 10x +export with no feature ids. What only the data can show, such as a matrix +entry whose feature index lies outside the features, stops the copy instead. +Either way the output is written beside its destination and moved into place +only when complete, so a refusal leaves nothing behind, and an existing +output is replaced only with `--overwrite`. [Troubleshooting](../troubleshooting.md#before-anything-is-written) lists the refusals. diff --git a/docs/guides/pipelines.md b/docs/guides/pipelines.md index 91afae0..268ba32 100644 --- a/docs/guides/pipelines.md +++ b/docs/guides/pipelines.md @@ -132,6 +132,17 @@ whole, which is what grows with the number of cells. A loom output is dense by definition, so `convert` to `.loom` warns above 10 GB before compression. +What `--chunk` does not bound, held whole by design: + +- obs and var columns, which is what grows with the number of cells; +- an image: `export image` reads its array whole, `import image` decodes + its file whole, and a Visium conversion decodes each tissue image whole + (a PNG can only be read whole) before copying it in strips; + `--no-images` leaves them out; +- a pycisTopic pickle, which cannot be read in part; +- the whole matrix, when asked for with `--in-memory` (`recast`'s transpose, + `export sparse`). + ## Exit codes and output streams | Exit code | Meaning | @@ -143,7 +154,9 @@ A loom output is dense by definition, so `convert` to `.loom` warns above Data goes to stdout and everything else (progress, notes, warnings, errors) to stderr, so `adata export dataframe in.h5ad obs > obs.csv` is clean, and -`2> adata.log` keeps the log. An export piped to a reader that stops early, +`2> adata.log` keeps the log. Written to a file or a pipe rather than a +terminal, neither stream holds colour codes or a spinner, and the error is +one line however long, so `grep '^Error:' adata.log` finds it whole. An export piped to a reader that stops early, such as `head`, ends quietly with exit code 0. ## Zarr v2 and v3 diff --git a/docs/index.md b/docs/index.md index 18ddcaa..894a9c7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -73,8 +73,9 @@ Status goes to stderr and data to stdout, so every command pipes cleanly. (Cell Ranger, STARsolo, velocyto), Visium and Xenium spatial outputs with their coordinates and images, CellBender output, Parse Biosciences runs, looms (velocyto's and SCope's), text tables, plain HDF5 matrices, - pycisTopic pickles and MuData. CI checks them against the readers people - use for them: scanpy, squidpy, loompy, anndata, mudata. + pycisTopic pickles and MuData are read; 10x HDF5 and MTX, loom and MuData + are written too. CI checks them against the readers people use for them: + scanpy, squidpy, loompy, anndata, mudata. - **Outputs that keep the source's choices**: Zarr version, index dtypes, chunking and compression carry over unless a flag changes them. - **Outputs that appear whole or not at all**: every store is written beside diff --git a/docs/reference/commands.md b/docs/reference/commands.md index 770aee7..74d0ca8 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -68,6 +68,12 @@ adata ls data.h5ad uns --depth 2 # only below a path adata ls data.h5ad -1 | grep spatial # bare paths, for piping ``` +| Flag | Meaning | +|---|---| +| `--depth N`, `-d` | Maximum depth to descend | +| `--long`, `-l` | Show each entry's type, shape, dtype and encoding | +| `-1`, `--plain` | One bare path per line, with no tree, for piping | + ## `subset` Write a filtered copy. Select by name list, by expression, or both axes at @@ -84,7 +90,7 @@ adata subset data.h5ad --inplace --obs barcodes.txt | Flag | Meaning | |---|---| | `--output`, `-o` | Output path. Required unless `--inplace` | -| `--inplace` | Replace the source (written to a temporary path first) | +| `--inplace` | Replace the source; the new store is written beside it first | | `--obs` / `--var` | File of names to keep, one per line | | `--obs-query`, `-q` / `--var-query` | Keep rows matching an [expression](query.md) | | `--chunk`, `-C` | Row chunk size for dense matrices (at least 1) | @@ -154,7 +160,7 @@ adata convert multiome.h5ad -o multiome.h5mu |---|---| | `--output`, `-o` | Output path (required) | | `--from` | Input format, if it cannot be detected | -| `--to` | Output format, if the extension does not say | +| `--to` | Output format, if the extension does not say: one the Write column below allows | | `--zarr-format` | Zarr version to write; defaults to the source's, else 3 | | `--compression` | `none`, `gzip` or `lzf` for h5ad; `none`, `gzip`, `zstd` or `blosc` for Zarr; `gzip` (default) or `none` for an MTX folder | | `--overwrite` | Replace an existing output | @@ -189,9 +195,11 @@ written to Zarr gets zarr-python's default codec. `--compression` replaces all of that with one choice. Elements are not re-encoded, so a store written by anndata 0.7 keeps its legacy layout. -Every check runs before anything is written. The output is written beside -its destination (`out.adata-tmp.zarr`) and moved into place only once -complete, so a failed conversion leaves nothing behind. An existing output is +The input and the options are checked before anything is written; what only +the data can show, such as an index out of range, stops the copy. Either way +the output is written beside its destination (`out.adata-tmp.zarr`) and +moved into place only once complete, so a failed conversion leaves nothing +behind. An existing output is refused unless `--overwrite` is given, and is replaced only once its successor is finished. `--overwrite` replaces a store, or a Matrix Market folder holding only its own files, never a folder holding anything else. @@ -216,7 +224,7 @@ adata recast data.h5ad layers/counts -o out.h5ad --layout dense --force | Flag | Meaning | |---|---| | `--output`, `-o` | Output path. Required unless `--inplace` | -| `--inplace` | Replace the source (written to a temporary path first) | +| `--inplace` | Replace the source; the new store is written beside it first | | `--all` | Recast `X`, every layer and `raw/X` | | `--dtype` | New dtype for the values, e.g. `float32` | | `--indices-dtype` | New dtype for sparse indices: `int32` or `int64` | @@ -283,6 +291,7 @@ adata split data.h5ad --by cluster -o out/ --min-size 50 | `--output-dir`, `-o` | Where to write the stores | | `--axis` | `obs` (default) or `var` | | `--min-size` | Skip groups smaller than this | +| `--suffix` | Output extension, which picks the backend (`.zarr` for Zarr); defaults to the source's, `.h5ad` or `.zarr` | | `--dry-run` | Print the plan without writing | | `--manifest / --no-manifest` | Write a CSV listing the outputs (default on) | | `--chunk`, `-C` | Row chunk size for dense matrices (at least 1) | @@ -316,6 +325,7 @@ adata concat a.h5ad b.h5ad -o m.h5ad --keys a,b --index-unique - --uns-merge sam | Flag | Meaning | |---|---| +| `--output`, `-o` | Output path (required) | | `--join`, `-j` | `inner` (shared vars only) or `outer` (their union) | | `--label` | Add an obs column recording each cell's source | | `--keys` | Names for the inputs; defaults to their filenames | @@ -417,8 +427,16 @@ adata create out.h5ad --n-obs 5000 --n-var 2000 adata create out.zarr --obs-names cells.txt --var-names genes.txt ``` -Give each axis either a size (names are generated as `cell_0000`, `gene_0000`) -or a file of names. `--force` overwrites an existing store, and only a store: +| Flag | Meaning | +|---|---| +| `--n-obs`, `--n-var` | Number of observations, of variables | +| `--obs-names`, `--var-names` | A file of names, one per line, which sets the number | +| `--zarr-format` | Zarr version to write, 2 or 3; 3 by default | +| `--force`, `-f` | Overwrite an existing store | + +Give each axis either a size or a file of names. Generated names are +zero-padded to the axis length: `--n-obs 5000` gives `cell_0000` to +`cell_4999`, `--n-obs 3` gives `cell_0` to `cell_2`. `--force` overwrites an existing store, and only a store: a folder that is not a Zarr store, or a file that is not HDF5, is refused. The new store is written beside the old one and swapped in once complete, so a failed `create --force` leaves the old store as it was. @@ -441,6 +459,17 @@ adata export array data.h5ad obsm/X_umap > umap.npy adata export dict data.h5ad uns -o metadata.json ``` +| Flag | Subcommands | Meaning | +|---|---|---| +| `--output`, `-o` | all | Output file; stdout when omitted (`export dict` also takes it as a third argument) | +| `--columns`, `-c` | `dataframe` | Comma separated columns to include | +| `--head N`, `-n` | `dataframe`, `sparse` | Only the first N rows, or entries | +| `--chunk`, `-C` | `dataframe`, `array`, `sparse` | Rows a block (`dataframe`, 10,000; also `--chunk-rows`, `-r`), elements (`array`, 100,000), or rows or columns of the matrix's layout (`sparse`, 1,000); at least 1 | +| `--in-memory`, `-m` | `sparse` | Load the whole matrix first, which can be faster for a small one | +| `--max-elements` | `dict` | Refuse an array larger than this (100,000 elements) | +| `--include-attrs` | `dict` | Write the elements' attributes too | +| `--scale` | `image` | Stretch the values onto 0-255 (below) | + Omit `--output` to write to stdout; `export image` writes PNG there, and otherwise takes its format from the output's extension. `export dataframe`, `export array` and `export sparse` stream in chunks, to a file or to stdout @@ -504,6 +533,16 @@ adata import dict data.h5ad uns/params params.json --inplace adata import image data.h5ad uns/spatial/hires tissue.png --inplace ``` +| Flag | Subcommands | Meaning | +|---|---|---| +| `--output`, `-o` | all | Output path; required unless `--inplace` | +| `--inplace` | all | Replace the source; the new store is written beside it first | +| `--no-space-check` | all | Write even if the disk looks too small | +| `--index-column`, `-i` | `dataframe` | Column that becomes the index; the first by default | +| `--categorical` | `dataframe` | Comma separated text columns to make categoricals | +| `--auto-categorical / --no-auto-categorical` | `dataframe` | Make text columns that repeat a value categoricals (default on) | +| `--chunk`, `-C` | `array` | Elements a block (below) | + Dimensions are validated against the existing obs/var where the path implies an axis. For `array` and `sparse` that is done from the file's header, before the store is copied, so a matrix of the wrong shape is refused at diff --git a/docs/reference/testing.md b/docs/reference/testing.md index 812cb43..eb00ef3 100644 --- a/docs/reference/testing.md +++ b/docs/reference/testing.md @@ -16,18 +16,27 @@ uv run pytest -m perf # just the tracing complexity guards uv run pytest -m integration # anndata releases, scanpy and PyTables ``` -CI runs the unit suite on Python 3.12 and 3.13 with a **90% coverage floor**, -and the anndata compatibility suite and the 10x external-reader suite as -separate jobs. A fresh-install job installs the package alone from a fresh -resolution, without `uv.lock`, as `pip install pyadata-cli` would. It -imports every module on the base dependencies alone, runs `adata convert` -with and without click beside typer, and runs the docs-accuracy suite with no -click installed. The locked suite cannot see a dependency the lock brings in -by accident: 0.7.0 to 0.17.2 imported click, which typer had stopped -installing, so `convert` crashed on every fresh install. A floors job runs -the unit suite once more, on both interpreters, with each direct dependency -at the lowest version `pyproject.toml` allows, after checking that each one -is at its floor and imports. +CI (`.github/workflows/tests.yml`) runs these jobs on every pull request, under +the names GitHub shows: + +| Job | What it runs | +|---|---| +| `tests (py3.12)`, `tests (py3.13)` | The unit suite, `-m "not integration"`, with a **90% coverage floor** | +| `fresh install, no lock` | The package alone from a fresh resolution, without `uv.lock`, as `pip install pyadata-cli` would install it: every module imported on the base dependencies, `adata convert` with and without click beside typer, and the docs-accuracy suite with no click installed | +| `tests at the dependency floors (py3.12)`, `(py3.13)` | The unit suite once more, with each direct dependency at the lowest version `pyproject.toml` allows, after checking that each one is at its floor and imports | +| `anndata 0.8-0.13 compatibility` | `test_anndata_versions.py`: every anndata release from 0.7.8 to 0.13 (see below) | +| `10x h5 against scanpy and PyTables` | `test_external_readers.py`: every converter checked by the reader people use for that format, not only 10x's (see the table below) | +| `workflow lint` | [actionlint](https://github.com/rhysd/actionlint) and [zizmor](https://docs.zizmor.sh) over the workflows | +| `container image` | Builds the Docker image and runs `.github/scripts/smoke-image.sh` against it | + +The last two jobs' names are older than what they check, and renaming a job +renames the check a branch rule requires, so they stay. `test_concat_versions.py` +is an `integration` test that no job runs; run it locally. + +The fresh-install job exists because the locked suite cannot see a +dependency the lock brings in by accident: 0.7.0 to 0.17.2 imported click, +which typer had stopped installing, so `convert` crashed on every fresh +install. Each leg of the unit suite checks that it runs the Python it is named after before testing anything. uv takes its interpreter from `.python-version` @@ -37,13 +46,10 @@ suite on 3.12; the workflow now sets `UV_PYTHON` for each leg. The workflows are tested too. Publishing, the image and the benchmark run only on a tag, so a pull request never runs them. `test_release_pipeline.py` reads them, works out which jobs publish in each kind of run, and runs their -scripts against stand-ins. The `workflow lint` job runs -[actionlint](https://github.com/rhysd/actionlint) and -[zizmor](https://docs.zizmor.sh) over them on every pull request, though not -inside a release, where an advisory published that day would stop it. The -`container image` job builds the Docker image and runs -`.github/scripts/smoke-image.sh` against it, on every pull request and inside -every release before PyPI has it. +scripts against stand-ins. The `workflow lint` job runs on every pull +request, though not inside a release, where an advisory published that day +would stop it. The `container image` job runs on every pull request and +inside every release before PyPI has it. ## How the suite is organised @@ -51,7 +57,11 @@ every release before PyPI has it. |---|---| | `test_elements.py` | The element layer — encodings, string dtypes, readers and writers — run against HDF5, Zarr v2 and Zarr v3 | | `test_storage.py` | Backend detection, copying, Zarr versions, consolidated metadata | +| `test_storage_root_attrs.py` | The root's AnnData attributes: written when a store is opened for writing, and a warning on reading a store that lacks them, though not for anndata 0.7's layout, which never had them | +| `test_staging.py` | `util/staging.py` on its own: write beside and rename into place, several outputs together, a failed swap put back, leftovers refused rather than deleted | +| `test_write_safety.py` | Every way a command used to destroy or corrupt a store while reporting success, or leave a half-written one: outputs that alias the input, failed selections, `--inplace`, failed closes, a full disk, two runs on one output, SIGTERM | | `test_formats.py` | `.npy`, `.mtx`, image and JSON export/import, including their failure modes | +| `test_cli_output.py` | What the CLI prints: names, labels and paths with square brackets as the store spells them, a failure as one plain `Error:` line in a log file, help text that keeps its brackets, `main(argv)` | | `test_validate.py` | Dimension validation for every axis-bearing path, including `raw/` | | `test_invariants.py` | Relationships that must hold for any data: subsetting everything is the identity, split partitions exactly, concat undoes split | | `test_anndata_roundtrip.py` | anndata writes the fixtures, reads back our output | @@ -61,6 +71,11 @@ every release before PyPI has it. | `test_convert.py` | `convert` between every backend pair, read back by anndata; its refusals, staging, and the forwarding of recast's old flags | | `test_recast.py` | `recast` against scipy: casts, transposes, density changes and every refusal | | `test_recast_streaming.py` | `recast` on HDF5, Zarr v2 and Zarr v3 in blocks of a few rows and entries: every layout to every other, every dtype in every layout, matrices at any depth, index ranges with and without `--force`; against scipy and read back by anndata | +| `test_concat_correctness.py` | `concat` against `anndata.concat`, on inputs whose encodings each test sets: plain strings, nullable columns, legacy categoricals, missing values, joins and merges | +| `test_concat_scale.py` | `concat` on the paths that scale, against `anndata.concat`: CSC matrices rebuilt through buckets on disk, and more inputs than can be open at once | +| `test_concat_versions.py` | `concat` outputs read back by the anndata release that wrote the inputs, 0.8 to 0.13 (`integration`) | +| `test_subset_fan_out.py` | The streamed sparse writers of `subset`, `split` and `concat`: index dtypes and compression kept as the source has them, and `split` reading each matrix once for all groups; against anndata and scipy | +| `test_scatter.py` | `core/scatter.py`: rows read in one order written in another through scratch | | `test_tenx.py` | 10x HDF5 import and export against Cell Ranger-shaped fixtures (v2, v3, multimodal, probe-based): mapping, filters, round trips, refusals | | `test_mtx.py` | 10x / STARsolo MTX folders against hand-written fixtures (Cell Ranger 2 and 3, STARsolo, velocyto, GEO prefixes): mapping, entry orders, refusals that leave nothing, round trips | | `test_cellbender.py` | CellBender 0.2 and 0.3+ output, full and filtered: each latent at its barcode, `--min-cell-probability`, `--raw` alignment, refusals | @@ -68,7 +83,12 @@ every release before PyPI has it. | `test_loom.py` | loom import and export against loompy- and velocyto-shaped fixtures (spec 2 and 3): X choice, names, attributes, graphs, `--dense`, round trips | | `test_text.py` | Text matrices in every layout anndata's `read_text` tells apart (corner, short and comment headers, no header, no row names), each delimiter, gzip, `--transpose` CSR and dense, dtypes, refusals by line, small blocks | | `test_hdf5.py` | One dataset of a plain HDF5 file: the listing without `--key`, `read_hdf`'s `row_names` / `col_names` and other names, `--transpose` CSR and dense, slabs of chunks, `--from hdf5` on `.h5` | +| `test_dense.py` | `conversion/dense.py`, which the loom and HDF5 importers read a dense matrix through: datasets chunked every way that changes how it reads them, on each output backend, read back by anndata | | `test_cistopic.py` | pycisTopic pickles read through the allowlist: a CistopicObject's matrices, frames, model and projections, an imputed-features pickle, a pickle naming `os.system` refused unrun, the missing-extra hint | +| `test_pickled.py` | The restricted unpickler against crafted pickles that name only allowlisted globals, each loaded in a child process so a crash cannot take the run down | +| `test_visium.py` | Space Ranger output: standard Visium (Space Ranger 1 to 4) laid out as `squidpy.read.visium` lays it out, Visium HD bins at each size, segmented cells, Space Ranger's analysis, images, refusals | +| `test_xenium.py` | Xenium bundles (XOA 1 to 4): the cell table in or out of the matrix's order, cells the matrix lacks, Xenium's analysis with gaps | +| `test_geojson.py` | `formats/geojson.py`: Space Ranger 4's cell outlines read a feature at a time, centroids and areas against shapely | | `test_pandas_frame.py` | Each pandas dtype written as anndata writes it, on every backend | | `test_concat_axis_elements.py` | `concat` on obsm dataframes and varm, against `anndata.concat` for each join and merge strategy | | `test_legacy_copies.py` | anndata 0.7's dataframes (categoricals as codes with an object reference into `__categories`) copied by `subset` (h5ad and Zarr), `convert` to Zarr v2/v3 and a MuData import: written as current `categorical` groups in a 0.2.0 frame, read back by anndata with their categories | @@ -77,10 +97,11 @@ every release before PyPI has it. | `test_scope.py` | SCope looms, read as pySCENIC's `export2loom` writes them and written with `--scope` (round trips, plain AnnData, refusals, markers): regulons as dataframes, embeddings and clusterings named from `MetaData`, plain or compressed `MetaData`, region looms, R's logicals and factors, renamed fields, small blocks | | `test_table.py` | The streaming CSV reader against pandas: column types, quoting, row subsets, blocks, refusals | | `test_mtx_codec.py`, `test_csr_build.py` | The streaming Matrix Market reader and writer against scipy, and the bucket sort they and `recast` share | -| `test_external_readers.py` | 10x conversions, HDF5 and MTX, Parse runs and looms, checked by scanpy (Parse's own recipe for Parse), anndata with loompy (the pySCENIC tutorial's recipe for SCope looms), anndata's `read_csv`, `read_text` and `read_hdf`, a pandas 1.5 pycisTopic pickle read back there, mudata (a file it writes read as it reads it; ours read by it), and PyTables in their own uv environments (`integration`) | +| `test_external_readers.py` | Every converter checked by the reader people use, each in its own uv environment pinned in `reference_stores.ORACLES` (`integration`): 10x HDF5 and MTX and CellBender by scanpy, and our 10x export by PyTables; Parse runs by Parse's own scanpy recipe; looms by anndata with loompy (the pySCENIC tutorial's recipe for SCope looms); anndata's `read_csv`, `read_text` and `read_hdf`; a pandas 1.5 pycisTopic pickle read back there; mudata (a file it writes read as it reads it; ours read by it); Visium by squidpy, Visium HD by scanpy with pandas and pyarrow, its segmented cells by 10x's geopandas recipe, Xenium by 10x's scanpy recipe; scanpy's `rank_genes_groups` through every conversion | | `test_performance.py` | Complexity guards -- what an operation costs, not how long it takes (see below); its fixtures are in `perf_fixtures.py` | | `test_benchmark_harness.py` | That the benchmark's measurement is trustworthy, and that each run compares with the release before it | | `test_release_pipeline.py` | The workflows and the Dockerfile: which jobs publish in each kind of run, which hold a write token and what they run, pinned actions and base image, no untrusted value pasted into a script, each Python leg on its own Python | +| `test_docs_are_accurate.py` | The docs against the code: every `adata ...` line in the README, the site and the `--help` examples uses real commands and options; each command has a section in the reference; links resolve; every page has its front matter; every test file is in this table | | `test_subset_oracle.py` | `subset` and `split` against anndata's `adata[obs, var]`, element by element: X, every layer layout, obsm and varm of any rank and as dataframes, obsp, varp, raw; on HDF5, Zarr v2 and Zarr v3 | | `test_subset_selection.py`, `test_select.py`, `test_split_names.py` | Name lists that repeat a name, queries on every column layout against pandas, missing labels apart from empty ones, and output names that never share a file | | `test_subset.py`, `test_export.py`, `test_import.py`, `test_info_read.py`, `test_zarr.py`, `test_query.py` | Per-feature unit tests | @@ -101,9 +122,12 @@ def test_something(new_store): ``` For CLI surfaces, use the module-level `CliRunner` and assert on -`result.stdout + (result.stderr or "")`, since status goes to stderr. Strip -ANSI before matching message text — Rich also wraps long lines, so collapse -whitespace. +`result.stdout` for results and `result.stderr` for status and errors. Off a +terminal, as under `CliRunner`, Rich writes no colour codes and keeps each +stderr message on one line, but `FORCE_COLOR` in the environment brings the +codes back and help text and stdout's trees still wrap at Rich's width: strip +ANSI and collapse whitespace before matching text, as the `_out(result)` +helpers of the CLI test files do. ## Performance diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 52d429b..aff4da7 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -6,8 +6,8 @@ nav_order: 5 # Troubleshooting {: .no_toc } -Find the message you got, then the fix. Errors start `Error:` and end the -command with exit code 1; a usage error (an unknown option, a missing +Find the message you got, then the fix. Errors are one line on stderr +starting `Error:`, and end the command with exit code 1; a usage error (an unknown option, a missing argument, an input that does not exist) is exit code 2. Messages are shown here with real paths replaced by short ones. @@ -337,8 +337,10 @@ store it wrote; for a store another tool changed, run ### The command was killed for running out of memory Lower `--chunk`: a dense block holds `--chunk` × n_var values, 1,024 rows by -default. Two things are held whole by design: obs and var columns, and a -pycisTopic pickle, which cannot be read in part. See +default. Some things are held whole by design: obs and var columns, an +image (`export image`, `import image`, and the tissue images of a Visium +conversion, which `--no-images` leaves out), and a pycisTopic pickle, which +cannot be read in part. See [Pipelines and containers](guides/pipelines.md#memory-and---chunk). ### `A loom holds its matrices dense: 54.0 GB to compress and write. It is written, but may take a while` diff --git a/pyproject.toml b/pyproject.toml index a6fcd25..c20d738 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,9 +77,9 @@ pickle = [ "pandas>=2.2.3", "scipy>=1.15", ] -# Reading Visium HD: its bin positions are Parquet only. Imported only by -# the Visium converter, and only for an HD input. 18 is the first with -# wheels for Python 3.13. +# Reading Visium HD: its bin positions are Parquet only. Imported only in +# formats/parquet.py, which the Visium converter calls for an HD input. 18 +# is the first with wheels for Python 3.13. spatial = [ "pyarrow>=18", ] @@ -97,8 +97,8 @@ dev = [ # Turns a hang into a failure with a traceback. Without it a stuck test # burns the CI job's whole timeout and reports nothing useful. "pytest-timeout>=2.3.1", - # Used only by tests/test_anndata_roundtrip.py, to check that what this - # tool reads and writes matches what anndata itself produces. + # The oracle most tests check against: anndata writes their inputs and + # reads back what this tool writes. The tool itself never imports it. "anndata>=0.13", ] diff --git a/src/adata/cli.py b/src/adata/cli.py index b97532c..c8244df 100644 --- a/src/adata/cli.py +++ b/src/adata/cli.py @@ -1,13 +1,13 @@ """CLI for AnnData stores (.h5ad / .zarr) with export and import subcommands.""" from pathlib import Path -from typing import Optional, Sequence, List +from typing import Any, List, Optional, Sequence from rich.console import Console import typer from rich.markup import escape -from adata.conversion import FORMATS, Options +from adata.conversion import FORMATS, Options, writable_formats from adata.core.recast import LAYOUTS from adata.formats.array import IMPORT_CHUNK_ELEMENTS from adata.commands import ( @@ -32,13 +32,43 @@ help="Streaming CLI for huge AnnData .h5ad and .zarr stores " "(view, ls, create, subset, split, concat, convert, recast, export, import)." ) -# Use stderr for status/progress to keep stdout clean for data output -# force_terminal=True ensures Rich output is visible even in non-TTY environments -console = Console(stderr=True, force_terminal=True) -# Results go to stdout so they can be piped; status and errors stay on stderr. + + +class _MessageConsole(Console): + """Rich's console, keeping each message on one line when not on a terminal. + + Rich wraps at the terminal's width, or at 80 columns in a log file, where + a wrapped `Error:` line hides the rest of the message from `grep`. A + renderable (a progress display) keeps Rich's layout, which would otherwise + be cropped rather than wrapped. + """ + + def print(self, *objects: Any, soft_wrap: Optional[bool] = None, **kwargs: Any) -> None: + if soft_wrap is None: + soft_wrap = not self.is_terminal and all(isinstance(o, str) for o in objects) + super().print(*objects, soft_wrap=soft_wrap, **kwargs) + + +# Status, progress and errors go to stderr, results to stdout, so that stdout +# pipes cleanly. Rich asks each stream whether it is a terminal: a log file or +# a pipe gets no colour, no cursor movement and no spinner, so the one line a +# failure leaves on stderr starts `Error:`. +console = _MessageConsole(stderr=True) out_console = Console() +def _fail(error: BaseException) -> typer.Exit: + """Print `error` as one `Error:` line on stderr; return the exit to raise. + + A KeyError's str() is the repr of its message, quotes and all, so its + message is taken as given. Messages quote names and paths from the store + and the command line, which Rich would read as markup: they are escaped. + """ + message = error.args[0] if isinstance(error, KeyError) and error.args else error + console.print(f"[bold red]Error:[/] {escape(str(message))}") + return typer.Exit(code=1) + + def _version_callback(value: bool) -> None: if value: from adata import __version__ @@ -140,8 +170,7 @@ def view( out_console=out_console, ) except Exception as e: - console.print(f"[bold red]Error:[/] {escape(str(e))}") - raise typer.Exit(code=1) + raise _fail(e) @app.command("info", hidden=True) @@ -236,8 +265,7 @@ def create( force=force, ) except Exception as e: - console.print(f"[bold red]Error:[/] {escape(str(e))}") - raise typer.Exit(code=1) + raise _fail(e) # ============================================================================ @@ -288,8 +316,7 @@ def ls( plain=plain, ) except Exception as e: - console.print(f"[bold red]Error:[/] {escape(str(e))}") - raise typer.Exit(code=1) + raise _fail(e) # ============================================================================ @@ -316,7 +343,7 @@ def subset( inplace: bool = typer.Option( False, "--inplace", - help="Modify source file directly.", + help="Replace the source; the new store is written beside it first", ), obs: Optional[Path] = typer.Option( None, @@ -392,8 +419,7 @@ def subset( space_check=not no_space_check, ) except Exception as e: - console.print(f"[bold red]Error:[/] {escape(str(e))}") - raise typer.Exit(code=1) + raise _fail(e) # ============================================================================ @@ -502,8 +528,7 @@ def concat( space_check=not no_space_check, ) except Exception as e: - console.print(f"[bold red]Error:[/] {escape(str(e))}") - raise typer.Exit(code=1) + raise _fail(e) # ============================================================================ @@ -619,7 +644,8 @@ def convert( to_format: Optional[str] = typer.Option( None, "--to", - help=f"Output format, if the extension does not say: {', '.join(FORMATS)}", + help="Output format, if the extension does not say: " + f"{', '.join(writable_formats())}", ), zarr_format: Optional[int] = typer.Option( None, @@ -836,33 +862,39 @@ def convert( """ Convert a store to another format. - The output's extension picks the format: .h5ad, .zarr, .loom, or .h5 for - a 10x Genomics matrix (Cell Ranger's *_feature_bc_matrix.h5); --to - 10x-mtx writes a Matrix Market folder. Between h5ad and zarr every element is - copied, streamed, keeping the source's chunking and compression where the - target can express them. CellBender output keeps its latent variables, as - CellBender's own loader names them. A 10x matrix is read with the - columns scanpy's read_10x_h5 or read_10x_mtx (gex_only=False) gives, - keeping integer counts, and written as Cell Ranger writes it. A Space Ranger outs/ - folder keeps its spot positions, images and scale factors where squidpy - puts them; a Visium HD one is read at the bin size --bin-size names - (needs pyadata-cli[spatial]), or as Space Ranger 4's segmented cells - with --cells; and a Xenium bundle its cell table, - centroids and Xenium's own clusters, UMAP and PCA. MTX folders from Cell - Ranger and STARsolo, velocyto's included, and Parse split-pipe output - are read in two streaming passes, as are text tables (.csv, .tsv, .txt, - gzipped too), read as anndata's read_csv and read_text read them; a plain - HDF5 dataset is read as read_hdf reads it (--key names it); a pycisTopic - pickle without pycisTopic (needs pyadata-cli[pickle]). A loom - is read as anndata.read_loom reads it; a SCope loom (pySCENIC, SCENIC+, + The output's extension picks the format: .h5ad, .zarr, .loom, .h5mu, or + .h5 for a 10x Genomics matrix (Cell Ranger's *_feature_bc_matrix.h5); + --to 10x-mtx writes a Matrix Market folder. Between h5ad and zarr every + element is copied, streamed, keeping the source's chunking and + compression where the target can express them. + + Read and written: a 10x matrix, .h5 or an MTX folder (Cell Ranger's, and + STARsolo's, velocyto's included), read with the columns scanpy's + read_10x_h5 or read_10x_mtx (gex_only=False) gives, keeping integer + counts, and written as Cell Ranger writes it; a loom, read as + anndata.read_loom reads it, where a SCope loom (pySCENIC, SCENIC+, pycistopic) also gives its regulons as dataframes in obsm and varm, its embeddings and clusterings named from its MetaData, and the MetaData in - uns["scope"]; --scope writes them back as SCope reads them. Features are named by gene id, unique but in probe-level - (Flex) matrices; --var-names gene_symbols names them as scanpy does. - - Nothing is written until every check has passed, and the output is moved - into place only once complete: a failed conversion leaves nothing - behind. An existing output is refused unless --overwrite is given. + uns["scope"], and --scope writes them back as SCope reads them; and + MuData. Features are named by gene id, unique but in probe-level (Flex) + matrices; --var-names gene_symbols names them as scanpy does. + + Read only: CellBender output, keeping its latent variables as + CellBender's own loader names them; a Space Ranger outs/ folder, with its + spot positions, images and scale factors where squidpy puts them (Visium + HD at the bin size --bin-size names, which needs pyadata-cli\\[spatial], + or as Space Ranger 4's segmented cells with --cells); a Xenium bundle, + with its cell table, centroids and Xenium's own clusters, UMAP and PCA; + Parse split-pipe output; text tables (.csv, .tsv, .txt, gzipped too), as + anndata's read_csv and read_text read them; one dataset of a plain HDF5 + file, as read_hdf reads it (--key names it); and a pycisTopic pickle, + without pycisTopic (needs pyadata-cli\\[pickle]). + + The input and the options are checked before anything is written; what + only the data shows, such as an index out of range, stops the copy. The + output is written beside its destination and moved into place only once + complete, so a failed conversion leaves nothing behind. An existing + output is refused unless --overwrite is given. To change a matrix's dtype, layout or density, use `recast`. Until 1.0.0, `convert` given recast's flags still forwards to it, with a warning. @@ -1003,8 +1035,7 @@ def convert( space_check=not no_space_check, ) except Exception as e: - console.print(f"[bold red]Error:[/] {escape(str(e))}") - raise typer.Exit(code=1) + raise _fail(e) # ============================================================================ @@ -1035,7 +1066,7 @@ def recast( inplace: bool = typer.Option( False, "--inplace", - help="Modify source file directly.", + help="Replace the source; the new store is written beside it first", ), convert_all: bool = typer.Option( False, @@ -1123,8 +1154,7 @@ def recast( space_check=not no_space_check, ) except Exception as e: - console.print(f"[bold red]Error:[/] {escape(str(e))}") - raise typer.Exit(code=1) + raise _fail(e) # ============================================================================ @@ -1205,8 +1235,7 @@ def split( space_check=not no_space_check, ) except Exception as e: - console.print(f"[bold red]Error:[/] {escape(str(e))}") - raise typer.Exit(code=1) + raise _fail(e) # ============================================================================ @@ -1268,8 +1297,7 @@ def export_dataframe( console=console, ) except Exception as e: - console.print(f"[bold red]Error:[/] {escape(str(e))}") - raise typer.Exit(code=1) + raise _fail(e) @export_app.command("array") @@ -1317,8 +1345,7 @@ def export_array( console=console, ) except Exception as e: - console.print(f"[bold red]Error:[/] {escape(str(e))}") - raise typer.Exit(code=1) + raise _fail(e) @export_app.command("sparse") @@ -1375,8 +1402,7 @@ def export_sparse( console=console, ) except Exception as e: - console.print(f"[bold red]Error:[/] {escape(str(e))}") - raise typer.Exit(code=1) + raise _fail(e) @export_app.command("dict") @@ -1424,8 +1450,7 @@ def export_dict( console=console, ) except Exception as e: - console.print(f"[bold red]Error:[/] {escape(str(e))}") - raise typer.Exit(code=1) + raise _fail(e) @export_app.command("image") @@ -1470,8 +1495,7 @@ def export_image( try: export_image_cmd(file=file, obj=entry, out=output, console=console, scale=scale) except Exception as e: - console.print(f"[bold red]Error:[/] {escape(str(e))}") - raise typer.Exit(code=1) + raise _fail(e) # ============================================================================ @@ -1504,7 +1528,7 @@ def import_dataframe( inplace: bool = typer.Option( False, "--inplace", - help="Modify source file directly.", + help="Replace the source; the new store is written beside it first", ), no_space_check: bool = _space_check_option(), index_column: Optional[str] = typer.Option( @@ -1562,8 +1586,7 @@ def import_dataframe( space_check=not no_space_check, ) except Exception as e: - console.print(f"[bold red]Error:[/] {escape(str(e))}") - raise typer.Exit(code=1) + raise _fail(e) @import_app.command("array") @@ -1593,7 +1616,7 @@ def import_array( inplace: bool = typer.Option( False, "--inplace", - help="Modify source file directly.", + help="Replace the source; the new store is written beside it first", ), chunk_elements: int = _chunk_option( IMPORT_CHUNK_ELEMENTS, "Number of elements to copy per chunk" @@ -1626,8 +1649,7 @@ def import_array( space_check=not no_space_check, ) except Exception as e: - console.print(f"[bold red]Error:[/] {escape(str(e))}") - raise typer.Exit(code=1) + raise _fail(e) @import_app.command("sparse") @@ -1657,7 +1679,7 @@ def import_sparse( inplace: bool = typer.Option( False, "--inplace", - help="Modify source file directly.", + help="Replace the source; the new store is written beside it first", ), no_space_check: bool = _space_check_option(), ) -> None: @@ -1685,8 +1707,7 @@ def import_sparse( space_check=not no_space_check, ) except Exception as e: - console.print(f"[bold red]Error:[/] {escape(str(e))}") - raise typer.Exit(code=1) + raise _fail(e) @import_app.command("dict") @@ -1716,7 +1737,7 @@ def import_dict( inplace: bool = typer.Option( False, "--inplace", - help="Modify source file directly.", + help="Replace the source; the new store is written beside it first", ), no_space_check: bool = _space_check_option(), ) -> None: @@ -1740,8 +1761,7 @@ def import_dict( space_check=not no_space_check, ) except Exception as e: - console.print(f"[bold red]Error:[/] {escape(str(e))}") - raise typer.Exit(code=1) + raise _fail(e) @import_app.command("image") @@ -1769,7 +1789,7 @@ def import_image_cmd( file_okay=True, ), inplace: bool = typer.Option( - False, "--inplace", help="Modify source file directly." + False, "--inplace", help="Replace the source; the new store is written beside it first" ), no_space_check: bool = _space_check_option(), ) -> None: @@ -1792,13 +1812,13 @@ def import_image_cmd( space_check=not no_space_check, ) except Exception as e: - console.print(f"[bold red]Error:[/] {escape(str(e))}") - raise typer.Exit(code=1) + raise _fail(e) def main(argv: Optional[Sequence[str]] = None) -> None: + """Run the CLI on `argv`, or on the command line when it is None.""" try: - app(standalone_mode=True) + app(args=list(argv) if argv is not None else None, standalone_mode=True) except SystemExit as done: if isinstance(done.code, int) and done.code != 0: _leave(done.code) diff --git a/src/adata/commands/convert.py b/src/adata/commands/convert.py index c172b68..fd9641c 100644 --- a/src/adata/commands/convert.py +++ b/src/adata/commands/convert.py @@ -1,9 +1,11 @@ """The `convert` command: rewrite a store in another format. -Every refusal happens while building the plan, before an output exists, and -the outputs are written beside their destination and moved into place only -once complete. A conversion that fails therefore leaves nothing behind, and -never a half-written store that the next command would read as a whole one. +What the input's layout and the options can show is refused while building +the plan, before an output exists; what only the data shows, such as an +index out of range, stops the copy. Either way the outputs are written beside +their destination and moved into place only once complete. A conversion that +fails therefore leaves nothing behind, and never a half-written store that +the next command would read as a whole one. """ from __future__ import annotations @@ -95,7 +97,7 @@ def convert_store( ): check_free_space(plan.outputs[0], lambda: disk_size(file), check=space_check) - console.print(f"[cyan]Converting[/] {file}: {plan.summary}") + console.print(f"[cyan]Converting[/] {escape(str(file))}: {escape(plan.summary)}") for warning in plan.warnings: console.print(f"[yellow]Warning:[/] {escape(warning)}") @@ -103,5 +105,5 @@ def convert_store( converter.write(plan, staged, console) for out in plan.outputs: - console.print(f"[green]Wrote[/] {out}") + console.print(f"[green]Wrote[/] {escape(str(out))}") return plan.outputs diff --git a/src/adata/commands/create.py b/src/adata/commands/create.py index ddc422c..0cc678d 100644 --- a/src/adata/commands/create.py +++ b/src/adata/commands/create.py @@ -12,6 +12,7 @@ from typing import List, Optional from rich.console import Console +from rich.markup import escape from adata.elements.write import ensure_anndata_skeleton, write_dataframe_header from adata.storage import open_store @@ -91,6 +92,6 @@ def create_store( ensure_anndata_skeleton(root) console.print( - f"[green]Created[/] {output} " + f"[green]Created[/] {escape(str(output))} " f"({len(obs_index)} obs x {len(var_index)} var, empty)" ) diff --git a/src/adata/commands/export.py b/src/adata/commands/export.py index 67d1bea..3028553 100644 --- a/src/adata/commands/export.py +++ b/src/adata/commands/export.py @@ -16,6 +16,7 @@ from typing import Iterator, List, Optional from rich.console import Console +from rich.markup import escape from adata.formats.array import export_npy as export_npy_format from adata.formats.dataframe import export_dataframe @@ -47,7 +48,7 @@ def _output(file: Path, out: Optional[Path], console: Console) -> Iterator[Optio out.parent.mkdir(parents=True, exist_ok=True) with staged_outputs([out]) as (temp,): yield temp - console.print(f"[green]Wrote[/] {out}") + console.print(f"[green]Wrote[/] {escape(str(out))}") def export_table( diff --git a/src/adata/commands/import_data.py b/src/adata/commands/import_data.py index 54d9211..2e85296 100644 --- a/src/adata/commands/import_data.py +++ b/src/adata/commands/import_data.py @@ -16,6 +16,7 @@ import h5py import numpy as np from rich.console import Console +from rich.markup import escape from adata.formats.array import IMPORT_CHUNK_ELEMENTS, import_npy from adata.formats.dataframe import import_dataframe @@ -46,13 +47,13 @@ def _copy_source( if src_backend == dst_backend: copy_path(file, temp) if not inplace: - console.print(f"[dim]Copied {file} → {output}[/]") + console.print(f"[dim]Copied {escape(str(file))} → {escape(str(output))}[/]") return with open_store(file, "r") as src_store, open_store(temp, "w") as dst_store: copy_store_contents(src_store.root, dst_store.root) console.print( - f"[dim]Converted {file} ({src_backend}) → {output} ({dst_backend})[/]" + f"[dim]Converted {escape(str(file))} ({src_backend}) → {escape(str(output))} ({dst_backend})[/]" ) diff --git a/src/adata/commands/info.py b/src/adata/commands/info.py index 81e67b7..25219f7 100644 --- a/src/adata/commands/info.py +++ b/src/adata/commands/info.py @@ -2,6 +2,7 @@ from typing import Any, Optional from rich.console import Console +from rich.markup import escape from rich.tree import Tree from adata.core.info import axis_len, format_type_info, get_entry_type @@ -68,8 +69,10 @@ def show_info( ] if sub_keys and key != "X": out.print( - f"\t[bold yellow]{key}:[/]\t" - + ", ".join(f"[bright_white]{sub}[/]" for sub in sub_keys) + f"\t[bold yellow]{escape(key)}:[/]\t" + + ", ".join( + f"[bright_white]{escape(sub)}[/]" for sub in sub_keys + ) ) @@ -83,8 +86,11 @@ def _show_types_tree( - X: top level only - obsm/obsp/varm/varp/layers: 1 level (show matrices) - uns: 2 levels deep + + Names come from the store and are escaped: Rich would read the `[mt]` of + `pct_counts[mt]` as a style tag and drop it. """ - tree = Tree(f"[bold]{root_label}[/]") + tree = Tree(f"[bold]{escape(root_label)}[/]") # Define max depth for each top-level group max_depth_map = { @@ -111,11 +117,11 @@ def add_node( if is_dataset(obj): shape_str = f"[dim]{obj.shape}[/]" if obj.shape else "" - node_text = f"[bright_white]{name}[/] {shape_str} {type_str}" + node_text = f"[bright_white]{escape(name)}[/] {shape_str} {type_str}" parent_tree.add(node_text) else: # Group - node_text = f"[bold yellow]{name}/[/] {type_str}" + node_text = f"[bold yellow]{escape(name)}/[/] {type_str}" subtree = parent_tree.add(node_text) # Recurse only if within allowed depth @@ -149,30 +155,35 @@ def add_node( def _show_object_info(f: Any, entry_path: str, console: Console) -> None: - """Show detailed info for a specific object path.""" + """Show detailed info for a specific object path. + + Every value printed comes from the store or the command line, so each is + escaped before Rich reads the line as markup. + """ # Normalize path entry_path = entry_path.strip().lstrip("/") if entry_path not in f: - console.print(f"[bold red]Error:[/] '{entry_path}' not found in the file.") - return + # Raised, not printed: the CLI reports it on stderr with exit 1, as + # `ls` does, so a script never takes the message for the entry. + raise KeyError(f"'{entry_path}' not found in the file.") entry = f[entry_path] info = get_entry_type(entry) - console.print(f"\n[bold cyan]Path:[/] {entry_path}") - console.print(f"[bold cyan]Type:[/] {info['type']}") + console.print(f"\n[bold cyan]Path:[/] {escape(entry_path)}") + console.print(f"[bold cyan]Type:[/] {escape(info['type'])}") if info["encoding"]: - console.print(f"[bold cyan]Encoding:[/] {info['encoding']}") + console.print(f"[bold cyan]Encoding:[/] {escape(str(info['encoding']))}") if info["shape"]: console.print(f"[bold cyan]Shape:[/] {info['shape']}") if info["dtype"]: - console.print(f"[bold cyan]Dtype:[/] {info['dtype']}") + console.print(f"[bold cyan]Dtype:[/] {escape(info['dtype'])}") - console.print(f"[bold cyan]Details:[/] {info['details']}") + console.print(f"[bold cyan]Details:[/] {escape(info['details'])}") # Show attributes if any if entry.attrs: @@ -181,7 +192,7 @@ def _show_object_info(f: Any, entry_path: str, console: Console) -> None: v_str = v.decode("utf-8") if isinstance(v, bytes) else str(v) if len(v_str) > 80: v_str = v_str[:77] + "..." - console.print(f" [dim]{k}:[/] {v_str}") + console.print(f" [dim]{escape(k)}:[/] {escape(v_str)}") # If it's a group, show children if is_group(entry): @@ -196,4 +207,4 @@ def _show_object_info(f: Any, entry_path: str, console: Console) -> None: child_entry = entry[child_name] child_info = get_entry_type(child_entry) type_str = format_type_info(child_info) - console.print(f" [bright_white]{child_name}[/] {type_str}") + console.print(f" [bright_white]{escape(child_name)}[/] {type_str}") diff --git a/src/adata/commands/ls.py b/src/adata/commands/ls.py index 1c04003..2b6a383 100644 --- a/src/adata/commands/ls.py +++ b/src/adata/commands/ls.py @@ -3,6 +3,9 @@ Unlike `view`, this makes no AnnData assumptions: there is no n_obs x n_var header and no requirement that `obs`/`var` exist, so it works on `.loom` files and arbitrary `.h5` stores as well as AnnData ones. + +Member names are escaped before Rich renders the tree: scanpy names a QC +column `pct_counts[mt]`, and Rich would read `[mt]` as a style tag and drop it. """ from __future__ import annotations @@ -12,6 +15,7 @@ from typing import Any, Iterator, Optional, Tuple from rich.console import Console +from rich.markup import escape from rich.tree import Tree from adata.core.info import format_type_info, get_entry_type @@ -42,9 +46,9 @@ def _describe(obj: Any, long: bool) -> str: if shape: bits.append(f"[dim]{tuple(shape)}[/]") if is_dataset(obj) and getattr(obj, "dtype", None) is not None: - bits.append(f"[dim]{obj.dtype}[/]") + bits.append(f"[dim]{escape(str(obj.dtype))}[/]") if info["encoding"]: - bits.append(f"[dim]{info['encoding']}[/]") + bits.append(f"[dim]{escape(str(info['encoding']))}[/]") return " " + " ".join(bits) @@ -93,10 +97,10 @@ def _print_tree( long: bool, ) -> None: root_label = f"{label}:{entry_path}" if entry_path else label - tree = Tree(f"[bold]{root_label}[/]") + tree = Tree(f"[bold]{escape(root_label)}[/]") if is_dataset(root): - tree.add(f"[bright_white]{entry_path or label}[/]{_describe(root, long)}") + tree.add(f"[bright_white]{escape(entry_path or label)}[/]{_describe(root, long)}") console.print(tree) return @@ -106,9 +110,9 @@ def _print_tree( parent = nodes.get(parent_path, tree) if is_group(obj): nodes[path] = parent.add( - f"[bold yellow]{name}/[/]{_describe(obj, long)}" + f"[bold yellow]{escape(name)}/[/]{_describe(obj, long)}" ) else: - parent.add(f"[bright_white]{name}[/]{_describe(obj, long)}") + parent.add(f"[bright_white]{escape(name)}[/]{_describe(obj, long)}") console.print(tree) diff --git a/src/adata/commands/recast.py b/src/adata/commands/recast.py index 2914669..03a62ff 100644 --- a/src/adata/commands/recast.py +++ b/src/adata/commands/recast.py @@ -14,6 +14,7 @@ from typing import Any, List, Optional from rich.console import Console +from rich.markup import escape from adata.core.recast import ( DEFAULT_CHUNK, @@ -136,7 +137,7 @@ def recast_store( console.print( f"[cyan]Converting {len(targets)} matrix/matrices:[/] " - + ", ".join(targets) + + escape(", ".join(targets)) ) with open_store(temp, "w", zarr_format=zarr_format) as dst_store: @@ -154,9 +155,9 @@ def recast_store( ensure_anndata_skeleton(dst) if inplace: - console.print(f"[green]Converted[/] {file}") + console.print(f"[green]Converted[/] {escape(str(file))}") else: - console.print(f"[green]Wrote[/] {dst_path}") + console.print(f"[green]Wrote[/] {escape(str(dst_path))}") def _extra(plan: Plan) -> float: diff --git a/src/adata/commands/split.py b/src/adata/commands/split.py index 2058a67..b9a4710 100644 --- a/src/adata/commands/split.py +++ b/src/adata/commands/split.py @@ -19,6 +19,7 @@ import numpy as np from rich.console import Console +from rich.markup import escape from adata.core.select import group_indices from adata.core.subset import check_subsettable, split_h5ad @@ -76,7 +77,7 @@ def split_store( # it ("" or "NA"), so NA always holds the rows with no value. names = unique_names(order) console.print( - f"[cyan]Splitting on {axis}[{column}]: " + f"[cyan]Splitting on {axis}\\[{escape(column)}]: " f"{len(order)} group{'s' if len(order) != 1 else ''}[/]" ) @@ -93,7 +94,7 @@ def split_store( planned.append((label, out_path, len(indices))) for label, out_path, count in planned: - console.print(f" {_shown(label)} -> {out_path} ({count} {axis})") + console.print(f" {_shown(label)} -> {escape(str(out_path))} ({count} {axis})") for _, out_path, _ in planned: try: @@ -138,12 +139,13 @@ def split_store( if manifest: _write_manifest(file, output_dir, column, planned, console) - console.print(f"[green]Wrote[/] {len(planned)} stores to {output_dir}") + console.print(f"[green]Wrote[/] {len(planned)} stores to {escape(str(output_dir))}") return planned def _shown(label: Optional[str]) -> str: - return "(missing)" if label is None else repr(label) + """A label as a message prints it, escaped: labels are data, not markup.""" + return "(missing)" if label is None else escape(repr(label)) def _manifest_path(file: Path, output_dir: Path) -> Path: @@ -172,5 +174,5 @@ def _write_manifest( for label, out_path, count in planned: value = "" if label is None else label writer.writerow([source_id, column, value, str(out_path), count]) - console.print(f"[dim]Manifest written to {path}[/]") + console.print(f"[dim]Manifest written to {escape(str(path))}[/]") return path diff --git a/src/adata/conversion/__init__.py b/src/adata/conversion/__init__.py index 0b3dee2..bdcbc6f 100644 --- a/src/adata/conversion/__init__.py +++ b/src/adata/conversion/__init__.py @@ -35,6 +35,7 @@ "converter_for", "detect_format", "output_format", + "writable_formats", ] @@ -184,6 +185,14 @@ def _names() -> str: return ", ".join(FORMATS) +def writable_formats() -> List[str]: + """The formats `convert` can write: AnnData, and those with an exporter.""" + return [ + name for name, fmt in FORMATS.items() + if fmt.anndata or fmt.exporter is not None + ] + + def detect_format(path: Path) -> str: """Name the format of an existing input: by content, then by extension. @@ -234,11 +243,14 @@ def output_format(path: Path, requested: Optional[str] = None) -> str: """ path = Path(path) if requested is not None and requested not in FORMATS: - raise ValueError(f"Unknown format {requested!r}; use one of: {_names()}.") - if requested is not None and not ( - FORMATS[requested].anndata or FORMATS[requested].exporter is not None - ): - raise ValueError(f"{requested} can be read but not written.") + raise ValueError( + f"Unknown format {requested!r}; use one of: {', '.join(writable_formats())}." + ) + if requested is not None and requested not in writable_formats(): + raise ValueError( + f"{requested} can be read but not written; --to takes one of: " + f"{', '.join(writable_formats())}." + ) suffix = path.suffix.lower() # Only formats that can be written: `.h5` names CellBender output too, @@ -255,7 +267,7 @@ def output_format(path: Path, requested: Optional[str] = None) -> str: if by_extension is None: raise ValueError( f"Cannot tell the output format from '{path.name}'. " - f"Use a known extension or pass --to, one of: {_names()} " + f"Use a known extension or pass --to, one of: {', '.join(writable_formats())} " "(--to 10x-mtx for a folder)." ) return by_extension diff --git a/src/adata/conversion/anndata_copy.py b/src/adata/conversion/anndata_copy.py index 607ecb0..a6b30b8 100644 --- a/src/adata/conversion/anndata_copy.py +++ b/src/adata/conversion/anndata_copy.py @@ -10,6 +10,7 @@ from typing import Optional, Sequence from rich.console import Console +from rich.markup import escape from adata.conversion.base import ConversionPlan, Converter, Options from adata.elements.write import ensure_anndata_skeleton @@ -82,7 +83,7 @@ def write( copy_attrs(src.root.attrs, dst.root.attrs, target_backend=backend) ensure_anndata_root_attrs(dst.root) for key in src.root.keys(): - with console.status(f"[cyan]Copying {key}...[/]"): + with console.status(f"[cyan]Copying {escape(key)}...[/]"): copy_tree( src.root[key], dst.root, key, compression=detail["compression"], diff --git a/src/adata/conversion/cellbender.py b/src/adata/conversion/cellbender.py index 5bbbe6a..55c13aa 100644 --- a/src/adata/conversion/cellbender.py +++ b/src/adata/conversion/cellbender.py @@ -38,6 +38,7 @@ import h5py import numpy as np from rich.console import Console +from rich.markup import escape from adata.conversion import tenx from adata.conversion.base import ConversionPlan, Converter, Options @@ -579,7 +580,7 @@ def write( ) if detail.raw is not None: - with console.status(f"[cyan]Reading the raw counts of {detail.raw.path.name}...[/]"): + with console.status(f"[cyan]Reading the raw counts of {escape(detail.raw.path.name)}...[/]"): _write_raw( detail.raw, roots, detail.targets, n_out, matrix.n_features, group, ) diff --git a/src/adata/conversion/cistopic.py b/src/adata/conversion/cistopic.py index d303476..1db573e 100644 --- a/src/adata/conversion/cistopic.py +++ b/src/adata/conversion/cistopic.py @@ -38,6 +38,7 @@ import numpy as np from rich.console import Console +from rich.markup import escape from adata.conversion.base import ConversionPlan, Converter, Options from adata.conversion.features import STRING_STEP, Target, write_obs_index, write_var @@ -210,7 +211,7 @@ def write( write_dense(root[key], label, values) _write_uns(root["uns"], detail, warnings) for message in warnings: - console.print(f"[yellow]Warning:[/] {message}") + console.print(f"[yellow]Warning:[/] {escape(message)}") def _is_sparse(value: Any) -> bool: diff --git a/src/adata/conversion/hdf5.py b/src/adata/conversion/hdf5.py index c812a72..a305ffd 100644 --- a/src/adata/conversion/hdf5.py +++ b/src/adata/conversion/hdf5.py @@ -24,6 +24,7 @@ import h5py import numpy as np from rich.console import Console +from rich.markup import escape from adata.conversion.base import ConversionPlan, Converter, Options from adata.conversion.dense import write_dense_matrix @@ -234,7 +235,7 @@ def obs_names() -> Iterator[List[str]]: for axis, path in (("obs_names", detail.obs_names), ("var_names", detail.var_names)): if path is not None: write_scalar(meta, axis, path) - with console.status(f"[cyan]Reading {detail.key}...[/]"): + with console.status(f"[cyan]Reading {escape(detail.key)}...[/]"): write_dense_matrix( handle[detail.key], [(root, "X")], cells_axis=1 if detail.transpose else 0, dense=detail.dense, compression=detail.compression, diff --git a/src/adata/conversion/loom.py b/src/adata/conversion/loom.py index 8e526de..8b7ec3a 100644 --- a/src/adata/conversion/loom.py +++ b/src/adata/conversion/loom.py @@ -54,6 +54,7 @@ import h5py import numpy as np from rich.console import Console +from rich.markup import escape from adata.conversion import scope from adata.conversion.base import ConversionPlan, Converter, Options @@ -489,7 +490,7 @@ def var_names(step: int) -> Iterator[List[str]]: if not places: continue ds = handle["matrix"] if loom_name == "" else handle["layers"][loom_name] - with console.status(f"[cyan]Reading {loom_name or 'matrix'}...[/]"): + with console.status(f"[cyan]Reading {escape(loom_name or 'matrix')}...[/]"): _write_matrix(ds, root, places, detail) @@ -947,11 +948,11 @@ def write( staged[0], "w" ) as out: root = store.root - with console.status(f"[cyan]Writing {detail.matrix} dense...[/]"): + with console.status(f"[cyan]Writing {escape(detail.matrix)} dense...[/]"): _export_matrix(resolve(root, detail.matrix), out, "matrix") layers = out.create_group("layers") for name, where in detail.layers: - with console.status(f"[cyan]Writing layer {name} dense...[/]"): + with console.status(f"[cyan]Writing layer {escape(name)} dense...[/]"): _export_matrix(resolve(root, where), layers, name) with console.status("[cyan]Writing the attributes...[/]"): diff --git a/src/adata/conversion/mtx.py b/src/adata/conversion/mtx.py index b6cf425..61c4d65 100644 --- a/src/adata/conversion/mtx.py +++ b/src/adata/conversion/mtx.py @@ -37,6 +37,7 @@ import numpy as np from rich.console import Console +from rich.markup import escape from adata.conversion.base import ConversionPlan, Converter, Options from adata.conversion.features import ( @@ -434,7 +435,7 @@ def write( # not leave a store holding obs and var and nothing else. scans = {} for name, header in detail.headers.items(): - with console.status(f"[cyan]Checking {header.path.name}...[/]"): + with console.status(f"[cyan]Checking {escape(header.path.name)}...[/]"): scans[name] = scan(header, emap) with ExitStack() as stack: @@ -473,7 +474,7 @@ def write( ) for parent, key in places ]) - with console.status(f"[cyan]Reading {header.path.name}...[/]"): + with console.status(f"[cyan]Reading {escape(header.path.name)}...[/]"): summed = read_into(header, emap, scanned, destinations, roots[0]) for appenders in destinations: for appender in appenders: @@ -481,7 +482,7 @@ def write( if summed: console.print( f"[yellow]Warning:[/] {summed:,} duplicate entries in " - f"{header.path.name} were summed, as scipy does." + f"{escape(header.path.name)} were summed, as scipy does." ) @@ -627,7 +628,7 @@ def write( matrix = describe(resolve(root, source.path)) check_sparse(matrix) - with console.status(f"[cyan]Counting the entries of {source.path}...[/]"): + with console.status(f"[cyan]Counting the entries of {escape(source.path)}...[/]"): nnz = count_entries(matrix) with create_binary(folder / f"matrix.mtx{ext}", gzipped=gzipped, level=GZIP_LEVEL) as fh: write_header( @@ -638,7 +639,7 @@ def write( nnz=nnz, comments=_metadata_comment(root), ) - with console.status(f"[cyan]Writing {source.path}...[/]"): + with console.status(f"[cyan]Writing {escape(source.path)}...[/]"): write_entries(fh, iter_matrix_entries(matrix, transpose=True)) diff --git a/src/adata/conversion/mudata.py b/src/adata/conversion/mudata.py index 315b3b3..928b3b4 100644 --- a/src/adata/conversion/mudata.py +++ b/src/adata/conversion/mudata.py @@ -34,6 +34,7 @@ import h5py import numpy as np from rich.console import Console +from rich.markup import escape from adata.conversion.base import ConversionPlan, Converter, Options from adata.conversion.features import type_counts, write_index_frame @@ -415,10 +416,10 @@ def write(self, plan: ConversionPlan, staged: Sequence[Path], console: Console) copy_attrs(group.attrs, dst.root.attrs, target_backend=backend) ensure_anndata_root_attrs(dst.root) for key in group.keys(): - with console.status(f"[cyan]Copying {mod.name}/{key}...[/]"): + with console.status(f"[cyan]Copying {escape(mod.name)}/{escape(key)}...[/]"): copy_tree(group[key], dst.root, key, compression=detail.compression) ensure_anndata_skeleton(dst.root) - with console.status(f"[cyan]Adding the global parts to {mod.name}...[/]"): + with console.status(f"[cyan]Adding the global parts to {escape(mod.name)}...[/]"): for axis, (_, mapping, graphs) in zip(mod.axes, _AXES): _write_axis(src.root, dst.root, axis, mapping, graphs, detail.compression) for key in mod.uns: diff --git a/src/adata/conversion/parse.py b/src/adata/conversion/parse.py index 58244eb..f089035 100644 --- a/src/adata/conversion/parse.py +++ b/src/adata/conversion/parse.py @@ -29,6 +29,7 @@ import numpy as np from rich.console import Console +from rich.markup import escape from adata.conversion.base import ConversionPlan, Converter, Options from adata.conversion.features import ( @@ -298,7 +299,7 @@ def write( # cannot be read must not leave a store holding obs and var alone. scans = [] for item in detail.items: - with console.status(f"[cyan]Checking {_label(item)}...[/]"): + with console.status(f"[cyan]Checking {escape(_label(item))}...[/]"): scans.append(scan(item.header, item.emap)) for item, scanned, path in zip(detail.items, scans, staged): @@ -306,7 +307,7 @@ def write( root = stack.enter_context( open_store(path, "w", zarr_format=detail.zarr_format) ).root - with console.status(f"[cyan]Writing obs and var of {item.dge.sample}...[/]"): + with console.status(f"[cyan]Writing obs and var of {escape(str(item.dge.sample))}...[/]"): write_table(root, "obs", item.cells, rows=item.rows) write_var(root, item.target) ensure_anndata_skeleton(root) @@ -316,13 +317,13 @@ def write( data_dtype=value_dtype(item.header, scanned), compression=detail.compression, ) - with console.status(f"[cyan]Reading {_label(item)}...[/]"): + with console.status(f"[cyan]Reading {escape(_label(item))}...[/]"): summed = read_into(item.header, item.emap, scanned, [[appender]], root) appender.finish() if summed: console.print( f"[yellow]Warning:[/] {summed:,} duplicate entries in " - f"{_label(item)} were summed, as scipy does." + f"{escape(_label(item))} were summed, as scipy does." ) diff --git a/src/adata/conversion/tenx.py b/src/adata/conversion/tenx.py index e91fb51..6dafa67 100644 --- a/src/adata/conversion/tenx.py +++ b/src/adata/conversion/tenx.py @@ -34,6 +34,7 @@ import h5py import numpy as np from rich.console import Console +from rich.markup import escape from adata.conversion.base import ConversionPlan, Converter, Options from adata.conversion.features import ( @@ -596,7 +597,7 @@ def write( ) as out: root = store.root matrix = describe(resolve(root, detail["path"])) - with console.status(f"[cyan]Copying {detail['path']}...[/]"): + with console.status(f"[cyan]Copying {escape(detail['path'])}...[/]"): group = _write_matrix(matrix, out) n_obs, n_var = matrix.shape group.create_dataset("shape", data=np.array([n_var, n_obs], dtype=np.int32)) @@ -605,7 +606,7 @@ def write( obs_index, _ = resolve_index(obs, "obs") n_text, example = _write_fixed_strings_streamed(group, "barcodes", obs_index, obs) if n_text: - console.print(f"[yellow]Warning:[/] {_ascii_warning(n_text, 'barcodes', example)}") + console.print(f"[yellow]Warning:[/] {escape(_ascii_warning(n_text, 'barcodes', example))}") flags = obs.get("filtered_barcodes") if is_dataset(flags) and "categories" not in flags.attrs and flags.dtype.kind == "b": # Space Ranger's and Flex's barcodes in the filtered matrix, diff --git a/src/adata/conversion/text.py b/src/adata/conversion/text.py index f79e447..88aa2cc 100644 --- a/src/adata/conversion/text.py +++ b/src/adata/conversion/text.py @@ -40,6 +40,7 @@ import numpy as np from rich.console import Console +from rich.markup import escape from adata.conversion.base import ConversionPlan, Converter, Options from adata.conversion.features import Target, write_obs_index, write_var @@ -552,7 +553,7 @@ def write( write_var(root, Target(label=None, keep=None, var_names=detail.var_names, columns={})) ensure_anndata_skeleton(root) _write_uns(root["uns"], detail) - with console.status(f"[cyan]Reading {Path(plan.source).name}...[/]"): + with console.status(f"[cyan]Reading {escape(Path(plan.source).name)}...[/]"): if detail.transpose: obs = _write_names(root, detail.obs_names) _write_transposed(root, reader, detail) diff --git a/src/adata/conversion/visium.py b/src/adata/conversion/visium.py index ab47562..5ed5ba6 100644 --- a/src/adata/conversion/visium.py +++ b/src/adata/conversion/visium.py @@ -49,6 +49,7 @@ import h5py import numpy as np from rich.console import Console +from rich.markup import escape from adata.conversion import tenx from adata.conversion.base import ConversionPlan, Converter, Options @@ -732,7 +733,7 @@ def _write_spatial_uns(root: Any, detail: _Plan, console: Console) -> None: # anndata 0.8 to 0.11 cannot read a null, and leave out None themselves. console.print( f"[yellow]Warning:[/] scalefactors_json.json holds null for " - f"{', '.join(path.rsplit('/', 1)[-1] for path in skipped)}; left out, as " + f"{escape(', '.join(path.rsplit('/', 1)[-1] for path in skipped))}; left out, as " "anndata leaves out None." ) metadata = write_mapping(library, "metadata") diff --git a/src/adata/core/concat.py b/src/adata/core/concat.py index 2cb539a..165541e 100644 --- a/src/adata/core/concat.py +++ b/src/adata/core/concat.py @@ -27,6 +27,7 @@ import h5py import numpy as np from rich.console import Console +from rich.markup import escape from adata.core.info import axis_len from adata.core.subset import _open_output_limit, aligned_rows @@ -558,7 +559,7 @@ def _concat_categorical( ) if any(ordered) and not keeps_order: console.print( - f"[yellow]Column {name!r}: the inputs disagree on its category order, " + f"[yellow]Column {escape(repr(name))}: the inputs disagree on its category order, " "so it is written unordered (anndata keeps no order either).[/]" ) lookup = {category: i for i, category in enumerate(categories.tolist())} @@ -723,8 +724,8 @@ def _concat_obs_column( _concat_text(parent, name, column, lengths) else: console.print( - f"[yellow]Dropping column {name!r}: " - f"cannot concatenate encodings {sorted({kind for _, kind in kinds})}[/]" + f"[yellow]Dropping column {escape(repr(name))}: " + f"cannot concatenate encodings {escape(str(sorted({kind for _, kind in kinds})))}[/]" ) return False return True @@ -1281,13 +1282,13 @@ def _concat_obsm( return True kinds = {_matrix_kind(s) for s in inputs.each(path)} if "other" in kinds: - console.print(f"[yellow]Skipping obsm/{name}: not an array, or not a dataframe, in every input[/]") + console.print(f"[yellow]Skipping obsm/{escape(name)}: not an array, or not a dataframe, in every input[/]") return False widths = {_trailing_shape(s) for s in inputs.each(path)} if len(widths) != 1: console.print( - f"[yellow]Skipping obsm/{name}: inputs disagree on shape {sorted(widths)}[/]" + f"[yellow]Skipping obsm/{escape(name)}: inputs disagree on shape {sorted(widths)}[/]" ) return False if len(kinds) != 1: @@ -1508,7 +1509,7 @@ def _write_pairwise( return enc = spec.encoding_type(obj) if enc not in spec.SPARSE_TYPES: - console.print(f"[yellow]Skipping varp/{name}: neither an array nor a sparse matrix[/]") + console.print(f"[yellow]Skipping varp/{escape(name)}: neither an array nor a sparse matrix[/]") return if len(take) == _shape(obj)[0] and np.array_equal(take, np.arange(len(take))): copy_tree(obj, parent, name) @@ -1588,7 +1589,7 @@ def aligned() -> Iterator[Any]: _write_var_column(frame, column, obj[column], take, obj) set_column_order(frame, columns) else: - console.print(f"[yellow]Skipping varm/{name}: neither an array nor a dataframe[/]") + console.print(f"[yellow]Skipping varm/{escape(name)}: neither an array nor a dataframe[/]") def _write_varp( @@ -1900,10 +1901,10 @@ def concat_on_disk( console.print( f"[cyan]Concatenating {len(files)} stores: " - f"{n_obs} obs x {n_var} var ({join} join)[/]" + f"{n_obs} obs x {n_var} var ({escape(join)} join)[/]" ) for f, key, count in zip(files, input_keys, obs_counts): - console.print(f" [dim]{key}[/]: {count} obs from {f.name}") + console.print(f" [dim]{escape(str(key))}[/]: {count} obs from {escape(f.name)}") if len(set(target_obs)) != len(target_obs): console.print( @@ -1961,7 +1962,7 @@ def concat_on_disk( if any(inputs.get(i, "raw") is not None for i in range(len(inputs))): console.print("[yellow]Dropping raw/: not concatenated.[/]") - console.print(f"[green]Wrote[/] {output} ({n_obs} obs x {n_var} var)") + console.print(f"[green]Wrote[/] {escape(str(output))} ({n_obs} obs x {n_var} var)") def _write_obs( @@ -2121,7 +2122,7 @@ def _write_matrices( path = f"layers/{name}" if not all(layer is not None for layer in inputs.each(path)): console.print( - f"[yellow]Skipping layer {name!r}: not present in every input[/]" + f"[yellow]Skipping layer {escape(repr(name))}: not present in every input[/]" ) continue _concat_matrix(layers, name, inputs, path, col_maps, row_counts, n_obs, n_var, @@ -2149,7 +2150,7 @@ def _write_obsm( path = f"obsm/{name}" if not all(entry is not None for entry in inputs.each(path)): console.print( - f"[yellow]Skipping obsm/{name}: not present in every input[/]" + f"[yellow]Skipping obsm/{escape(name)}: not present in every input[/]" ) continue _concat_obsm( diff --git a/src/adata/core/recast.py b/src/adata/core/recast.py index c34c19a..9d3d5a0 100644 --- a/src/adata/core/recast.py +++ b/src/adata/core/recast.py @@ -28,6 +28,7 @@ import numpy as np from rich.console import Console +from rich.markup import escape from adata.elements import spec from adata.elements.write import set_shape_attr @@ -773,7 +774,7 @@ def sparsify( density = nnz / max(1, n_rows * n_cols) if console is not None and density > 0.5: console.print( - f"[yellow]{name} is {density:.0%} nonzero; the sparse form is " + f"[yellow]{escape(name)} is {density:.0%} nonzero; the sparse form is " f"larger than the dense one below about 33%.[/]" ) return nnz, density @@ -1043,7 +1044,7 @@ def plan_conversion( raise ValueError(f"{name}: {message}. Pass --force to convert anyway.") if console is not None: colour = "dim" if report.lossless else "yellow" - console.print(f"[{colour}]{name}: {message}[/]") + console.print(f"[{colour}]{escape(f'{name}: {message}')}[/]") return Plan(src, target_layout, data_dtype, idx_dtype, ptr_dtype, report) diff --git a/src/adata/core/subset.py b/src/adata/core/subset.py index e604c94..f7b0cee 100644 --- a/src/adata/core/subset.py +++ b/src/adata/core/subset.py @@ -10,6 +10,7 @@ import numpy as np from rich.console import Console +from rich.markup import escape from rich.progress import ( Progress, SpinnerColumn, @@ -974,7 +975,7 @@ def _write_targets( console.print( "[yellow]Copying unrecognised top-level " f"{'keys' if len(passthrough) > 1 else 'key'} verbatim: " - f"{', '.join(sorted(passthrough))}[/]" + f"{escape(', '.join(sorted(passthrough)))}[/]" ) tasks.extend(f"copy:{k}" for k in passthrough) @@ -990,7 +991,7 @@ def _write_targets( transient=False, ) as progress: for task in tasks: - task_id = progress.add_task(f"[cyan]{verb} {task}...[/]", total=None) + task_id = progress.add_task(f"[cyan]{verb} {escape(task)}...[/]", total=None) if task in ("obs", "var"): for target, narrowed in zip(targets, pruned): indices = _axis_idx(target, task) @@ -1057,7 +1058,7 @@ def _write_targets( ) progress.update( task_id, - description=f"[green]{verb} {task}[/]", + description=f"[green]{verb} {escape(task)}[/]", completed=1, total=1, ) diff --git a/src/adata/formats/array.py b/src/adata/formats/array.py index fa5bdb2..6d1e301 100644 --- a/src/adata/formats/array.py +++ b/src/adata/formats/array.py @@ -18,6 +18,7 @@ import numpy as np from rich.console import Console +from rich.markup import escape from adata.elements import spec from adata.elements.read import decode_str_array @@ -147,7 +148,7 @@ def export_npy( elif missing: dtype = np.dtype(np.float64) console.print( - f"[yellow]Note:[/] '{obj}' has {missing:,} missing values, written " + f"[yellow]Note:[/] '{escape(obj)}' has {missing:,} missing values, written " "as NaN, so the array is float64." ) else: @@ -176,7 +177,7 @@ def convert(start: Optional[int], end: Optional[int]) -> np.ndarray: status = ( nullcontext() if to_stdout - else console.status(f"[magenta]Exporting {obj} to {out}...[/]") + else console.status(f"[magenta]Exporting {escape(obj)} to {escape(str(out))}...[/]") ) with target as fh, status as progress, stdout_reader_may_stop(): fh.write(header) @@ -186,7 +187,7 @@ def convert(start: Optional[int], end: Optional[int]) -> np.ndarray: n = shape[0] for start, end in _row_blocks(shape, chunk_elements): if progress: - progress.update(f"[magenta]Exporting {obj}: {start}-{end} of {n}...[/]") + progress.update(f"[magenta]Exporting {escape(obj)}: {start}-{end} of {n}...[/]") fh.write(convert(start, end).tobytes()) @@ -250,4 +251,4 @@ def import_npy( target[start:end] = block shape_str = "×".join(str(d) for d in arr.shape) - console.print(f"[green]Imported[/] {shape_str} array into '{obj}'") + console.print(f"[green]Imported[/] {shape_str} array into '{escape(obj)}'") diff --git a/src/adata/formats/dataframe.py b/src/adata/formats/dataframe.py index a015181..aff3a13 100644 --- a/src/adata/formats/dataframe.py +++ b/src/adata/formats/dataframe.py @@ -7,6 +7,7 @@ from typing import Any, List, Optional from rich.console import Console +from rich.markup import escape from adata.core.read import col_chunk_as_strings from adata.elements.read import dataframe_columns, element_len, resolve_index @@ -86,7 +87,7 @@ def export_dataframe( status_ctx = ( nullcontext() if to_stdout - else console.status(f"[magenta]Exporting {axis} table to {out}...[/]") + else console.status(f"[magenta]Exporting {escape(axis)} table to {escape(str(out))}...[/]") ) with status_ctx as status: @@ -159,12 +160,12 @@ def import_dataframe( if column.name in forced: if kind not in ("str", "category"): console.print( - f"[yellow]Note:[/] {column.name!r} holds {kind} values, so it " + f"[yellow]Note:[/] {escape(repr(column.name))} holds {kind} values, so it " "stays as it is rather than a categorical." ) elif column.seen is None: console.print( - f"[yellow]Warning:[/] {column.name!r} has more than " + f"[yellow]Warning:[/] {escape(repr(column.name))} has more than " f"{CATEGORY_CAP:,} distinct values, so it is written as strings." ) else: @@ -184,5 +185,5 @@ def import_dataframe( detail = f" ({n_categorical} categorical)" if n_categorical else "" console.print( f"[green]Imported[/] {scan.n_rows} rows x {len(scan.columns)} columns " - f"into '{obj}'{detail}" + f"into '{escape(obj)}'{detail}" ) diff --git a/src/adata/formats/image.py b/src/adata/formats/image.py index 6150d41..fb68b7c 100644 --- a/src/adata/formats/image.py +++ b/src/adata/formats/image.py @@ -7,6 +7,7 @@ import numpy as np from PIL import Image from rich.console import Console +from rich.markup import escape from adata.elements.write import write_dense, write_mapping from adata.formats.validate import DATAFRAME_PATHS, check_target, validate_dimensions @@ -149,5 +150,5 @@ def import_image(root: Any, obj: str, input_file: Path, console: Console) -> Non write_dense(parent, parts[-1], arr, replace=True) console.print( f"[green]Imported[/] {'x'.join(str(d) for d in arr.shape)} " - f"image ({arr.dtype}) into '{obj}'" + f"image ({arr.dtype}) into '{escape(obj)}'" ) diff --git a/src/adata/formats/json_data.py b/src/adata/formats/json_data.py index b2627fe..66c98f2 100644 --- a/src/adata/formats/json_data.py +++ b/src/adata/formats/json_data.py @@ -8,6 +8,7 @@ import numpy as np from rich.console import Console +from rich.markup import escape from adata.core.read import decode_str_array from adata.elements import spec @@ -186,11 +187,11 @@ def import_json( console.print( f"[yellow]Warning:[/] {len(skipped)} null " f"{'value was' if len(skipped) == 1 else 'values were'} left out, as " - f"anndata leaves out None: {', '.join(skipped[:5])}" + f"anndata leaves out None: {escape(', '.join(skipped[:5]))}" f"{', ...' if len(skipped) > 5 else ''}" ) - console.print(f"[green]Imported[/] JSON data into '{obj}'") + console.print(f"[green]Imported[/] JSON data into '{escape(obj)}'") def write_json_value(parent: Any, name: str, value: Any) -> List[str]: diff --git a/src/adata/formats/sparse.py b/src/adata/formats/sparse.py index 78f390d..6590e0c 100644 --- a/src/adata/formats/sparse.py +++ b/src/adata/formats/sparse.py @@ -12,6 +12,7 @@ from typing import Any from rich.console import Console +from rich.markup import escape from adata.core.csr_build import CsrAppender from adata.core.recast import describe @@ -87,7 +88,7 @@ def export_mtx( status = ( nullcontext() if to_stdout - else console.status(f"[magenta]Exporting {obj} to {out}...[/]") + else console.status(f"[magenta]Exporting {escape(obj)} to {escape(str(out))}...[/]") ) with target as fh, status, stdout_reader_may_stop(): write_header( @@ -121,7 +122,7 @@ def import_mtx( validate_dimensions(root, obj, shape, console) emap = EntryMap(n_rows=header.n_rows, n_cols=header.n_cols) - with console.status(f"[magenta]Checking {input_file.name}...[/]"): + with console.status(f"[magenta]Checking {escape(input_file.name)}...[/]"): scanned = scan(header, emap) dtype = value_dtype(header, scanned) @@ -137,7 +138,7 @@ def import_mtx( appender = CsrAppender( parent, name, shape=shape, data_dtype=dtype, compression=compression ) - with console.status(f"[magenta]Importing {input_file.name}...[/]"): + with console.status(f"[magenta]Importing {escape(input_file.name)}...[/]"): summed = read_into(header, emap, scanned, [[appender]], parent) nnz = appender.finish() @@ -148,5 +149,5 @@ def import_mtx( ) console.print( f"[green]Imported[/] {shape[0]}×{shape[1]} sparse matrix ({nnz} non-zero, " - f"{dtype}) into '{obj}'" + f"{dtype}) into '{escape(obj)}'" ) diff --git a/src/adata/formats/validate.py b/src/adata/formats/validate.py index 2b8ee61..23fa7bf 100644 --- a/src/adata/formats/validate.py +++ b/src/adata/formats/validate.py @@ -3,6 +3,7 @@ from typing import Optional, Tuple, Any from rich.console import Console +from rich.markup import escape from adata.core.info import axis_len from adata.util.path import norm_path @@ -277,4 +278,4 @@ def validate_dimensions( ) return - console.print(f"[dim]Note: No dimension validation for path '{obj_path}'[/]") + console.print(f"[dim]Note: No dimension validation for path '{escape(obj_path)}'[/]") diff --git a/tests/test_cli.py b/tests/test_cli.py index 97e0d1a..1d293d3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -92,11 +92,11 @@ def test_view_entry_nested_path(self, sample_h5ad_file): assert "Path:" in output def test_view_entry_not_found(self, sample_h5ad_file): - """Test info command with non-existent object path.""" + """A missing entry is an error on stderr with exit 1, as `ls` gives (F096).""" result = runner.invoke(app, ["view", str(sample_h5ad_file), "nonexistent"]) - assert result.exit_code == 0 # Doesn't exit with error, just shows message - output = result.stdout + (result.stderr or "") - assert "not found" in output + assert result.exit_code == 1 + assert result.stdout == "" + assert "not found" in result.stderr class TestExportDataframeCommand: diff --git a/tests/test_cli_output.py b/tests/test_cli_output.py new file mode 100644 index 0000000..fb3bdc1 --- /dev/null +++ b/tests/test_cli_output.py @@ -0,0 +1,253 @@ +"""What the CLI prints: names as the store spells them, errors where a pipeline +looks for them, help text that keeps its brackets. + +Rich reads `[word]` in anything it prints as a style tag. Store member names, +labels and paths can hold square brackets (`pct_counts[mt]` is scanpy's own +QC column), so every such string has to be escaped before it reaches Rich, or +it is silently rewritten, or crashes the command on a closing tag. +""" + +from __future__ import annotations + +import os +import re +import subprocess +import sys +from importlib.metadata import version +from pathlib import Path + +import anndata as ad +import h5py +import numpy as np +import pandas as pd +import pytest +from typer.main import get_command +from typer.testing import CliRunner + +from adata import cli +from adata.cli import app + +runner = CliRunner() + +#: Names scanpy and users really write, and ones Rich would read as tags. +OBS_COLUMNS = ["pct_counts[mt]", "qc[bold]", "[red]flag"] +OBSM_KEY = "X_[red]umap" +LABELS = ["s[1]", "[red]b", "x[/y]"] + + +def _bracketed_store(path: Path) -> Path: + """An h5ad whose obs columns, obsm key and labels hold square brackets.""" + obs = pd.DataFrame( + { + OBS_COLUMNS[0]: np.arange(6.0), + OBS_COLUMNS[1]: list("ababab"), + OBS_COLUMNS[2]: [True, False] * 3, + "batch[1]": pd.Categorical([LABELS[0]] * 2 + [LABELS[1]] * 2 + [LABELS[2]] * 2), + }, + index=[f"c{i}" for i in range(6)], + ) + adata = ad.AnnData(np.ones((6, 3), dtype="float32"), obs=obs) + adata.obsm[OBSM_KEY] = np.zeros((6, 2)) + adata.write_h5ad(path) + # An attribute value holding a closing tag: `view ENTRY` prints attributes. + with h5py.File(path, "a") as f: + f[f"obsm/{OBSM_KEY}"].attrs["note"] = "[/oops]" + return path + + +@pytest.fixture +def bracketed(tmp_path) -> Path: + return _bracketed_store(tmp_path / "names.h5ad") + + +def test_view_and_ls_print_member_names_with_square_brackets_as_they_are(bracketed): + """F042: Rich took `[mt]` in `pct_counts[mt]` for a style tag, so `view` + listed `pct_counts` and `ls` showed `X_umap` for `X_[red]umap`, names + the next command then could not find.""" + truth = ad.read_h5ad(bracketed) + names = list(truth.obs.columns) + list(truth.obsm.keys()) + for argv in ( + ["view", str(bracketed)], + ["view", "--tree", "--depth", "1", str(bracketed)], + ["ls", str(bracketed)], + ["ls", "--long", str(bracketed)], + ): + result = runner.invoke(app, argv) + assert result.exit_code == 0, (argv, result.output) + for name in names: + assert name in result.stdout, (argv, name, result.stdout) + + +def test_view_of_an_entry_prints_its_path_and_attributes_as_they_are(bracketed): + """F042: an attribute value holding `[/oops]` raised Rich's MarkupError, + so `view` failed on a valid store, and the path lost its `[red]`.""" + result = runner.invoke(app, ["view", str(bracketed), f"obsm/{OBSM_KEY}"]) + assert result.exit_code == 0, result.output + assert f"obsm/{OBSM_KEY}" in result.stdout + with h5py.File(bracketed, "r") as f: + note = f[f"obsm/{OBSM_KEY}"].attrs["note"] + assert f"note: {note}" in result.stdout + + +def test_split_names_its_column_and_groups_with_square_brackets_as_they_are( + bracketed, tmp_path +): + """F042: `Splitting on obs[batch]` lost `[batch]`, a `[red]b` label was + coloured and renamed `b`, and an `x[/y]` label crashed the plan.""" + labels = list(ad.read_h5ad(bracketed).obs["batch[1]"].cat.categories) + result = runner.invoke( + app, ["split", str(bracketed), "--by", "batch[1]", "-o", str(tmp_path / "out"), "--dry-run"] + ) + assert result.exit_code == 0, result.output + assert "obs[batch[1]]" in result.stderr + for label in labels: + assert repr(label) in result.stderr, (label, result.stderr) + + +def test_messages_name_an_output_whose_path_holds_square_brackets(bracketed, tmp_path): + """F042: paths are user input too; `Wrote [red]out.h5ad` lost `[red]`.""" + output = tmp_path / "[red]out.h5ad" + result = runner.invoke( + app, ["recast", str(bracketed), "X", "-o", str(output), "--dtype", "float64"] + ) + assert result.exit_code == 0, result.output + assert ad.read_h5ad(output).X.dtype == np.float64 + assert f"Wrote {output}" in result.stderr + + +def test_view_of_a_missing_entry_fails_on_stderr_with_exit_1(sample_h5ad_file): + """F096: the `Error:` line went to stdout with exit 0, so a script took + it for the entry's description; `ls` already failed properly.""" + result = runner.invoke(app, ["view", str(sample_h5ad_file), "obsm/X_missing"]) + assert result.exit_code == 1 + assert result.stdout == "" + assert result.stderr.startswith("Error:") + assert "obsm/X_missing" in result.stderr + + +def _run_redirected(argv, tmp_path, env=None): + """Run the CLI as a pipeline step does: every stream a file, no terminal.""" + out, err = tmp_path / "out.txt", tmp_path / "err.txt" + clean = {k: v for k, v in os.environ.items() if k not in ("FORCE_COLOR", "COLUMNS", "NO_COLOR")} + clean["TERM"] = "xterm-256color" + clean.update(env or {}) + with open(out, "wb") as stdout, open(err, "wb") as stderr: + code = subprocess.run( + [sys.executable, "-c", "from adata.cli import main; main()", *argv], + stdin=subprocess.DEVNULL, + stdout=stdout, + stderr=stderr, + env=clean, + timeout=120, + ).returncode + return code, out.read_text(), err.read_text() + + +def test_an_error_in_a_log_file_is_one_plain_line_starting_error(sample_h5ad_file, tmp_path): + """F151: the stderr console forced a terminal, so `2> log` held cursor + and colour codes before `Error:` and the message wrapped at column 80: + `grep '^Error:' log` found nothing, against what the pipeline docs say.""" + deep = tmp_path / ("a_rather_long_folder_name_" * 4) / "and_a_long_output_name.h5ad" + code, out, err = _run_redirected( + ["subset", str(sample_h5ad_file), "-o", str(deep), "--obs-query", "nope == 1"], tmp_path + ) + assert code == 1 + assert out == "" + assert "\x1b" not in err + errors = [line for line in err.splitlines() if line.startswith("Error:")] + assert len(errors) == 1, err + assert "nope" in errors[0] + assert err.endswith(errors[0] + "\n"), err + + +def test_output_piped_to_a_file_has_no_terminal_codes(sample_h5ad_file, tmp_path): + """F151: neither stream holds ANSI codes when it is not a terminal.""" + code, out, err = _run_redirected(["view", "--tree", str(sample_h5ad_file)], tmp_path) + assert code == 0, err + assert "\x1b" not in out + err + assert "obs" in out + + +def _help_sources(): + """(command path, help text) for every command and option, as written.""" + found = [] + + def walk(command, path): + found.append((path, command.help or "")) + for param in command.params: + found.append((path, getattr(param, "help", None) or "")) + for name, sub in (getattr(command, "commands", None) or {}).items(): + walk(sub, path + [name]) + + walk(get_command(app), []) + return found + + +def _rendered_help(path): + result = runner.invoke(app, path + ["--help"], env={"COLUMNS": "400"}, terminal_width=400) + assert result.exit_code == 0, result.output + return " ".join(result.output.split()) + + +def test_convert_help_keeps_the_extras_in_its_install_hints(): + """F160: Rich read `[spatial]` and `[pickle]` as tags, so the hints said + only "(needs pyadata-cli)", which anyone reading them already has.""" + rendered = _rendered_help(["convert"]) + assert "pyadata-cli[spatial]" in rendered + assert "pyadata-cli[pickle]" in rendered + + +def test_every_bracketed_word_in_help_text_survives_rendering(): + """F160, in general: any `[word]` a help string holds reaches the reader.""" + tag = re.compile(r"\\?\[([a-z#/@][^\[\]]*)\]") + missing = [] + for path, text in _help_sources(): + words = {f"[{m}]" for m in tag.findall(text)} + if not words: + continue + rendered = _rendered_help(path) + missing += [(path, w) for w in sorted(words) if w not in rendered] + assert not missing, missing + + +def _leave_raises(monkeypatch): + """`main` ends a failed run with os._exit, which would end pytest too.""" + + def leave(code): + raise SystemExit(code) + + monkeypatch.setattr(cli, "_leave", leave) + + +def test_main_runs_the_arguments_it_is_given(monkeypatch, capsys): + """F161: `main(argv)` ignored argv and parsed the host's sys.argv, so an + embedding caller ran some other command, or failed on its arguments.""" + _leave_raises(monkeypatch) + monkeypatch.setattr(sys, "argv", ["adata", "no-such-command"]) + with pytest.raises(SystemExit) as done: + cli.main(["--version"]) + assert done.value.code == 0 + assert capsys.readouterr().out.strip() == version("pyadata-cli") + + +def test_the_deprecated_entry_point_warns_on_stderr_and_runs_its_arguments( + monkeypatch, capsys +): + """F161: `main_deprecated(argv)` passed argv on to a `main` that dropped it.""" + _leave_raises(monkeypatch) + monkeypatch.setattr(sys, "argv", ["h5ad", "no-such-command"]) + with pytest.raises(SystemExit) as done: + cli.main_deprecated(["--version"]) + assert done.value.code == 0 + captured = capsys.readouterr() + assert captured.out.strip() == version("pyadata-cli") + assert "'h5ad' command is deprecated" in captured.err + + +def test_main_exits_with_the_failing_code(monkeypatch, tmp_path): + """A failed run still leaves through `_leave`, with its exit code.""" + codes = [] + monkeypatch.setattr(cli, "_leave", lambda code: codes.append(code)) + with pytest.raises(SystemExit): + cli.main(["view", str(tmp_path / "missing.h5ad")]) + assert codes == [2] diff --git a/tests/test_docs_are_accurate.py b/tests/test_docs_are_accurate.py index abcb4d7..30965b4 100644 --- a/tests/test_docs_are_accurate.py +++ b/tests/test_docs_are_accurate.py @@ -510,3 +510,94 @@ def _problems_with_example(line: str) -> List[str]: def test_help_examples_name_real_commands_and_options(line, source): problems = _problems_with_example(line) assert not problems, f"{source}: `{line}` -- " + "; ".join(problems) + + +# --------------------------------------------------------------------------- +# the test suite's map +# +# docs/reference/testing.md has a table of what each test file covers. It had +# fallen eight files behind (the spatial converters, the row scatter, staging +# and write safety among them), so a contributor looking for where Visium or +# the scatter is tested found nothing. Both directions are checked: every file +# has a row, and every row names a file that exists. + +TESTING_MD = _front_matter((SITE / "reference" / "testing.md").read_text())[1] +TEST_FILES = sorted(p.name for p in (REPO / "tests").glob("test_*.py")) + + +@pytest.mark.parametrize("name", TEST_FILES) +def test_every_test_file_is_in_the_suite_map(name): + assert f"`{name}`" in TESTING_MD, ( + f"tests/{name} has no row in the table of docs/reference/testing.md " + "(\"How the suite is organised\"): say what it covers." + ) + + +def test_the_suite_map_names_only_test_files_that_exist(): + named = set(re.findall(r"`(test_[a-z0-9_]+\.py)`", TESTING_MD)) + assert named, "no test files found in docs/reference/testing.md" + gone = sorted(named - set(TEST_FILES)) + assert not gone, f"docs/reference/testing.md names test files that do not exist: {gone}" + + +# --------------------------------------------------------------------------- +# every option is in the reference +# +# The checks above catch an option the docs name that does not exist, not one +# that exists and the docs never name: `split`'s table left out `--suffix`, +# and `export`'s `--chunk` and `--in-memory`, `create`'s `--zarr-format` and +# `import dataframe`'s `--index-column` were in no table at all. Each visible +# option has to appear in its command's section of commands.md. A format's +# own `convert` flags live on its page under reference/formats/ instead. + +FORMAT_PAGES = "\n".join( + p.read_text() for p in sorted((SITE / "reference" / "formats").glob("*.md")) +) + + +def _section(name: str) -> str: + match = re.search( + rf"^## `{re.escape(name)}`\n(.*?)(?=^## |\Z)", COMMANDS_MD, re.S | re.M + ) + return match.group(1) if match else "" + + +def _documented_options() -> List[Tuple[str, str, List[str]]]: + found = [] + + def walk(command: Any, path: List[str]) -> None: + subs = _subcommands(command) + for name, sub in subs.items(): + if not getattr(sub, "hidden", False): + walk(sub, path + [name]) + if subs or not path: + return + for param in command.params: + if getattr(param, "hidden", False) or getattr(param, "param_type_name", "") != "option": + continue + flags = [o for o in list(param.opts) + list(param.secondary_opts) if o.startswith("--")] + if flags and "--help" not in flags: + found.append((" ".join(path), path[0], flags)) + + walk(ROOT, []) + return found + + +DOCUMENTED_OPTIONS = _documented_options() + + +@pytest.mark.parametrize( + "command,section,flags", + DOCUMENTED_OPTIONS, + ids=[f"{c}:{f[0]}" for c, _, f in DOCUMENTED_OPTIONS], +) +def test_every_option_is_named_in_the_reference(command, section, flags): + text = _section(section) + if section == "convert": + text += FORMAT_PAGES + named = [f for f in flags if re.search(rf"`[^`]*{re.escape(f)}(?![\w-])[^`]*`", text)] + assert named, ( + f"`adata {command}` takes {' / '.join(flags)}, which the `{section}` " + "section of docs/reference/commands.md never names" + + (" (nor does any page under reference/formats/)" if section == "convert" else "") + ) From fd41e740158da676b4230f7dda6acd33d9a221a2 Mon Sep 17 00:00:00 2001 From: Aljes Date: Tue, 3 Nov 2026 15:39:48 +0000 Subject: [PATCH 2/5] Help text: the auto-categorical rule, --include-attrs, export sparse's --chunk Co-Authored-By: Claude Opus 5.5 --- src/adata/cli.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/adata/cli.py b/src/adata/cli.py index c8244df..94d2554 100644 --- a/src/adata/cli.py +++ b/src/adata/cli.py @@ -1373,7 +1373,7 @@ def export_sparse( ), chunk_elements: int = _chunk_option( 1_000, - "Number of rows/columns (depends on compression format) to process per chunk", + "Rows of a CSR matrix, or columns of a CSC one, per block", ), in_memory: bool = typer.Option( False, @@ -1428,7 +1428,7 @@ def export_dict( help="Maximum array elements for JSON export", ), include_attrs: bool = typer.Option( - False, "--include-attrs", help="Include HDF5 attributes in JSON export" + False, "--include-attrs", help="Include the elements' attributes in the JSON" ), ) -> None: """ @@ -1545,15 +1545,15 @@ def import_dataframe( auto_categorical: bool = typer.Option( True, "--auto-categorical/--no-auto-categorical", - help="Infer categoricals from low-cardinality string columns", + help="Make text columns that repeat a value categoricals, as anndata does", ), ) -> None: """ Import a CSV file as a dataframe. - String columns with few distinct values become categoricals; pass - --no-auto-categorical to keep them as plain strings, or --categorical to - force specific ones. + A text column that repeats a value becomes a categorical, as anndata + makes it; pass --no-auto-categorical to keep text as plain strings, or + --categorical to force specific columns. Examples: adata import dataframe data.h5ad obs cells.csv -o out.h5ad -i cell_id From 40bf6caeb430426ae768360417951b532f4c8e0c Mon Sep 17 00:00:00 2001 From: Aljes Date: Tue, 3 Nov 2026 17:54:43 +0000 Subject: [PATCH 3/5] Review fixes: literal() for markup, split's header, main(argv) raises, docs - util/markup.literal replaces rich.markup.escape everywhere: escape doubles a trailing backslash, which printed twice wherever text followed. - split's header escapes the whole obs[column]: a column with an unclosed bracket closed into a tag on the template's own ]. - main(argv) raises SystemExit for its caller; only the command itself ends the process at once on failure. - Docs: usage errors are Typer's box, not an Error: line; the held-whole lists gain JSON and --in-memory; export sparse's --chunk row; testing.md's job names and the rank_genes_groups row; AGENTS.md's CI list and the click note; developers.md in step. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 36 ++++++++----- docs/developers.md | 20 +++++--- docs/guides/pipelines.md | 9 +++- docs/reference/testing.md | 9 ++-- docs/troubleshooting.md | 12 +++-- src/adata/cli.py | 13 +++-- src/adata/commands/convert.py | 8 +-- src/adata/commands/create.py | 4 +- src/adata/commands/export.py | 4 +- src/adata/commands/import_data.py | 6 +-- src/adata/commands/info.py | 26 +++++----- src/adata/commands/ls.py | 14 +++--- src/adata/commands/recast.py | 8 +-- src/adata/commands/split.py | 12 ++--- src/adata/conversion/anndata_copy.py | 4 +- src/adata/conversion/cellbender.py | 4 +- src/adata/conversion/cistopic.py | 4 +- src/adata/conversion/hdf5.py | 4 +- src/adata/conversion/loom.py | 8 +-- src/adata/conversion/mtx.py | 12 ++--- src/adata/conversion/mudata.py | 6 +-- src/adata/conversion/parse.py | 10 ++-- src/adata/conversion/tenx.py | 6 +-- src/adata/conversion/text.py | 4 +- src/adata/conversion/visium.py | 4 +- src/adata/core/concat.py | 26 +++++----- src/adata/core/recast.py | 6 +-- src/adata/core/subset.py | 8 +-- src/adata/formats/array.py | 10 ++-- src/adata/formats/dataframe.py | 10 ++-- src/adata/formats/image.py | 4 +- src/adata/formats/json_data.py | 6 +-- src/adata/formats/sparse.py | 10 ++-- src/adata/formats/validate.py | 4 +- src/adata/util/markup.py | 29 +++++++++++ tests/test_cli_output.py | 75 ++++++++++++++++++++++++++-- 36 files changed, 281 insertions(+), 154 deletions(-) create mode 100644 src/adata/util/markup.py diff --git a/AGENTS.md b/AGENTS.md index 6c29d5a..31be111 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,19 +52,21 @@ Test markers: CI (`.github/workflows/tests.yml`) runs on every pull request: - the unit suite on Python 3.12 and 3.13, with the coverage floor; - a fresh install: the package from a fresh resolution, without the lock, - every module imported on the base dependencies alone, and `convert` run. - That catches a module-level import of something undeclared; one inside a - function is caught only on the paths it runs; + every module imported on the base dependencies alone, `convert` run with + and without click beside typer, and the docs-accuracy suite run with no + click. That catches a module-level import of something undeclared; one + inside a function is caught only on the paths it runs; - the unit suite at each direct dependency's floor; - the `integration` tests of `test_anndata_versions.py` and `test_external_readers.py`, each its own job; - `workflow lint` (actionlint and zizmor) and `container image`, which - builds the Docker image and smoke-tests it. Don't import click: typer -0.26 and later no longer install it. Compare the enums typer hands back -(such as `ParameterSource`) by `.name`. + builds the Docker image and smoke-tests it. -Neither CI nor the repo runs a linter or formatter. Match the style of the -surrounding code. +Don't import click: typer 0.26 and later no longer install it. Compare the +enums typer hands back (such as `ParameterSource`) by `.name`. + +No linter or formatter runs on the code; CI lints only the workflows. Match +the style of the surrounding code. ## Architecture @@ -83,7 +85,9 @@ uses `elements/` (the spec, `strings`, and the readers and writers that off one, as in a log file or a pipe, it writes no colour, cursor or spinner codes, and `console` keeps each message on one line. - `raise _fail(e)` prints an exception as the one `Error:` line on stderr - and exits 1. `main(argv)` runs the arguments it is given. + and exits 1. `main(argv)` runs the arguments it is given and raises + `SystemExit` for its caller; run as the command (`argv` None), a failed + run ends the process at once (`_leave`), past h5py's teardown. - **`commands/`**: one module per command. Each orchestrates its command: - planning and refusals; - progress output; @@ -153,7 +157,8 @@ uses `elements/` (the spec, `strings`, and the readers and writers that (categorical when values repeat); the loom import uses it. `geojson.py` walks a GeoJSON FeatureCollection a feature at a time and measures each polygon's centroid and area (Space Ranger 4's cell outlines). -- **`util/`**: `staging.py` and `path.py` (below), plus `names.py`. +- **`util/`**: `staging.py` and `path.py` (below), plus `names.py` and + `markup.py` (`literal`, below). - `path.check_output(output, inputs)` is the one output check, and `path.output_target(source, output, inplace)` wraps it for commands with `--inplace`. @@ -208,9 +213,14 @@ These are easy to break without any test noticing straight away: - **Escape what the store or the user wrote before Rich prints it.** Rich reads `[word]` as a style tag, so `pct_counts[mt]` printed as it is shows `pct_counts`, and `[/x]` crashes the command. Member names, labels, paths - and attribute values go through `rich.markup.escape`, in messages, status - lines and progress descriptions alike. Help text is markup too: a literal - `[` there is `\\[` in the source, which the docstring holds as `\[`. + and attribute values go through `util/markup.literal`, in messages, status + lines, progress descriptions and tree labels alike. It escapes as + `rich.markup.escape` does, but a trailing backslash prints once whatever + follows. Never write brackets around a value in a template + (`obs\\[{literal(column)}]`): a value with an unclosed `[` closes into a + tag there; escape the whole `obs[column]` instead. Help text is markup + too: a literal `[` there is `\\[` in the source, which the docstring holds + as `\[`. - **Zarr outputs keep the source's version.** - Pass `zarr_format=` to `open_store`. Without it zarr-python writes v3, and a v2 input gets silently upgraded. diff --git a/docs/developers.md b/docs/developers.md index dac8646..f61b977 100644 --- a/docs/developers.md +++ b/docs/developers.md @@ -38,8 +38,8 @@ installed by the `integration` tests in environments of their own. | `src/adata/elements/` | The AnnData on-disk spec: encodings, readers for every layout since anndata 0.7, writers of the current one | | `src/adata/storage/` | One interface over h5py and zarr | | `src/adata/conversion/` | `adata convert`: one module per format, through AnnData | -| `src/adata/formats/` | The file types of `export` and `import`: npy, Matrix Market, CSV, JSON, images; and what the converters read: Parquet, GeoJSON polygons, pycisTopic's pickles | -| `src/adata/util/` | Output staging, path checks, names | +| `src/adata/formats/` | The file types of `export` and `import`: npy, Matrix Market, CSV, JSON, images, and the check of an element's kind and dimensions before an import writes it; what the converters read: Parquet, GeoJSON polygons, pycisTopic's pickles; and pandas frames written as anndata writes them | +| `src/adata/util/` | Output staging, path checks, names, and `literal` for putting a value into Rich markup | | `tests/` | The test suite; [Testing](reference/testing.md) describes it | | `benchmarks/` | The comparative benchmark against anndata and scanpy | | `docs/` | This site | @@ -65,7 +65,9 @@ element faithfully). which is what lets output pipe cleanly. Off a terminal, in a log file or a pipe, neither writes colour, cursor or spinner codes, and a message on stderr stays on one line. `raise _fail(e)` turns an exception into that - line, `Error: ...`, and exit code 1. `--chunk` and `--no-space-check` are + line, `Error: ...`, and exit code 1. `main(argv)` runs the arguments it is + given and raises `SystemExit` for the caller; run as the command, a failed + run ends the process at once. `--chunk` and `--no-space-check` are defined once, as `_chunk_option` and `_space_check_option`; a new command reuses them. - **`commands/`** orchestrates one command: what to read, what to refuse, what @@ -146,9 +148,12 @@ before changing anything that writes: - **Escape what the store or the user wrote before Rich prints it.** Rich reads `[word]` as a style tag: printed as it is, `pct_counts[mt]` shows as `pct_counts`, and `[/x]` crashes the command. Names, labels, paths and - attribute values go through `rich.markup.escape` in every message, status - line and progress description. Help text is markup too, so a literal `[` - in a docstring is written `\\[`. + attribute values go through `util/markup.literal` in every message, status + line, progress description and tree label; it escapes as + `rich.markup.escape` does, but a trailing backslash prints once whatever + follows. Never write brackets around a value in a template: a value with + an unclosed `[` closes into a tag there. Help text is markup too, so a + literal `[` in a docstring is written `\\[`. - **Zarr outputs keep the source's version.** Pass `zarr_format=` to `open_store`; without it zarr-python writes v3, upgrading a v2 input. `concat` uses the version its Zarr inputs share. @@ -204,7 +209,8 @@ the lock, imports every module and runs `convert`, so a module-level import of something undeclared fails there. Don't import click, which typer 0.26 and later no longer install. House style: test names read as sentences, a regression test's docstring says what used to go wrong, and comments say why. No linter or -formatter runs; match the code around you. +formatter runs on the code (CI lints only the workflows); match the code +around you. [Testing](reference/testing.md) has the detail: what each file covers, what the guards count and why, and the anndata releases checked. diff --git a/docs/guides/pipelines.md b/docs/guides/pipelines.md index 268ba32..916109e 100644 --- a/docs/guides/pipelines.md +++ b/docs/guides/pipelines.md @@ -120,6 +120,7 @@ most data: | `subset`, `split`, `concat`, `recast` | 1,024 | rows of a dense matrix | | `export dataframe` | 10,000 | rows | | `export array` | 100,000 | elements | +| `export sparse` | 1,000 | rows of a CSR matrix, columns of a CSC one | | `import array` | 1,048,576 | elements | A dense block holds `--chunk` × n_var values, so lower it for very wide dense @@ -140,6 +141,8 @@ What `--chunk` does not bound, held whole by design: (a PNG can only be read whole) before copying it in strips; `--no-images` leaves them out; - a pycisTopic pickle, which cannot be read in part; +- a JSON document: `import dict` parses the file whole, and `export dict` + builds its JSON whole (`--max-elements` bounds each array, not the sum); - the whole matrix, when asked for with `--in-memory` (`recast`'s transpose, `export sparse`). @@ -155,8 +158,10 @@ What `--chunk` does not bound, held whole by design: Data goes to stdout and everything else (progress, notes, warnings, errors) to stderr, so `adata export dataframe in.h5ad obs > obs.csv` is clean, and `2> adata.log` keeps the log. Written to a file or a pipe rather than a -terminal, neither stream holds colour codes or a spinner, and the error is -one line however long, so `grep '^Error:' adata.log` finds it whole. An export piped to a reader that stops early, +terminal, neither stream holds colour codes or a spinner, and a refusal or +failure (exit code 1) is one line however long, so `grep '^Error:' +adata.log` finds it whole. A usage error (exit code 2) is the argument +parser's own message, in a box. An export piped to a reader that stops early, such as `head`, ends quietly with exit code 0. ## Zarr v2 and v3 diff --git a/docs/reference/testing.md b/docs/reference/testing.md index eb00ef3..c9866dc 100644 --- a/docs/reference/testing.md +++ b/docs/reference/testing.md @@ -29,8 +29,9 @@ the names GitHub shows: | `workflow lint` | [actionlint](https://github.com/rhysd/actionlint) and [zizmor](https://docs.zizmor.sh) over the workflows | | `container image` | Builds the Docker image and runs `.github/scripts/smoke-image.sh` against it | -The last two jobs' names are older than what they check, and renaming a job -renames the check a branch rule requires, so they stay. `test_concat_versions.py` +The compatibility and external-reader jobs' names are older than what they +check, and renaming a job renames the check a branch rule requires, so they +stay. `test_concat_versions.py` is an `integration` test that no job runs; run it locally. The fresh-install job exists because the locked suite cannot see a @@ -61,7 +62,7 @@ inside every release before PyPI has it. | `test_staging.py` | `util/staging.py` on its own: write beside and rename into place, several outputs together, a failed swap put back, leftovers refused rather than deleted | | `test_write_safety.py` | Every way a command used to destroy or corrupt a store while reporting success, or leave a half-written one: outputs that alias the input, failed selections, `--inplace`, failed closes, a full disk, two runs on one output, SIGTERM | | `test_formats.py` | `.npy`, `.mtx`, image and JSON export/import, including their failure modes | -| `test_cli_output.py` | What the CLI prints: names, labels and paths with square brackets as the store spells them, a failure as one plain `Error:` line in a log file, help text that keeps its brackets, `main(argv)` | +| `test_cli_output.py` | What the CLI prints: names, labels and paths with square brackets or a trailing backslash as the store spells them (`util/markup.literal`), a failure as one plain `Error:` line in a log file, help text that keeps its brackets, `main(argv)` raising rather than ending the process | | `test_validate.py` | Dimension validation for every axis-bearing path, including `raw/` | | `test_invariants.py` | Relationships that must hold for any data: subsetting everything is the identity, split partitions exactly, concat undoes split | | `test_anndata_roundtrip.py` | anndata writes the fixtures, reads back our output | @@ -97,7 +98,7 @@ inside every release before PyPI has it. | `test_scope.py` | SCope looms, read as pySCENIC's `export2loom` writes them and written with `--scope` (round trips, plain AnnData, refusals, markers): regulons as dataframes, embeddings and clusterings named from `MetaData`, plain or compressed `MetaData`, region looms, R's logicals and factors, renamed fields, small blocks | | `test_table.py` | The streaming CSV reader against pandas: column types, quoting, row subsets, blocks, refusals | | `test_mtx_codec.py`, `test_csr_build.py` | The streaming Matrix Market reader and writer against scipy, and the bucket sort they and `recast` share | -| `test_external_readers.py` | Every converter checked by the reader people use, each in its own uv environment pinned in `reference_stores.ORACLES` (`integration`): 10x HDF5 and MTX and CellBender by scanpy, and our 10x export by PyTables; Parse runs by Parse's own scanpy recipe; looms by anndata with loompy (the pySCENIC tutorial's recipe for SCope looms); anndata's `read_csv`, `read_text` and `read_hdf`; a pandas 1.5 pycisTopic pickle read back there; mudata (a file it writes read as it reads it; ours read by it); Visium by squidpy, Visium HD by scanpy with pandas and pyarrow, its segmented cells by 10x's geopandas recipe, Xenium by 10x's scanpy recipe; scanpy's `rank_genes_groups` through every conversion | +| `test_external_readers.py` | Every converter checked by the reader people use, each in its own uv environment pinned in `reference_stores.ORACLES` (`integration`): 10x HDF5 and MTX and CellBender by scanpy, and our 10x export by PyTables; Parse runs by Parse's own scanpy recipe; looms by anndata with loompy (the pySCENIC tutorial's recipe for SCope looms); anndata's `read_csv`, `read_text` and `read_hdf`; a pandas 1.5 pycisTopic pickle read back there; mudata (a file it writes read as it reads it; ours read by it); Visium by squidpy, Visium HD by scanpy with pandas and pyarrow, its segmented cells by 10x's geopandas recipe, Xenium by 10x's scanpy recipe; scanpy's `rank_genes_groups` through h5ad to Zarr v2 and v3, Zarr to h5ad and a recompressed h5ad | | `test_performance.py` | Complexity guards -- what an operation costs, not how long it takes (see below); its fixtures are in `perf_fixtures.py` | | `test_benchmark_harness.py` | That the benchmark's measurement is trustworthy, and that each run compares with the release before it | | `test_release_pipeline.py` | The workflows and the Dockerfile: which jobs publish in each kind of run, which hold a write token and what they run, pinned actions and base image, no untrusted value pasted into a script, each Python leg on its own Python | diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index aff4da7..89fecaa 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -6,9 +6,10 @@ nav_order: 5 # Troubleshooting {: .no_toc } -Find the message you got, then the fix. Errors are one line on stderr -starting `Error:`, and end the command with exit code 1; a usage error (an unknown option, a missing -argument, an input that does not exist) is exit code 2. Messages are shown +Find the message you got, then the fix. A refusal or failure is one line on +stderr starting `Error:`, and ends the command with exit code 1; a usage +error (an unknown option, a missing argument, an input that does not exist) +is the argument parser's own message, in a box, with exit code 2. Messages are shown here with real paths replaced by short ones. 1. TOC @@ -339,8 +340,9 @@ store it wrote; for a store another tool changed, run Lower `--chunk`: a dense block holds `--chunk` × n_var values, 1,024 rows by default. Some things are held whole by design: obs and var columns, an image (`export image`, `import image`, and the tissue images of a Visium -conversion, which `--no-images` leaves out), and a pycisTopic pickle, which -cannot be read in part. See +conversion, which `--no-images` leaves out), a JSON document (`import dict`, +`export dict`), a pycisTopic pickle, which cannot be read in part, and the +matrix `--in-memory` asks for. See [Pipelines and containers](guides/pipelines.md#memory-and---chunk). ### `A loom holds its matrices dense: 54.0 GB to compress and write. It is written, but may take a while` diff --git a/src/adata/cli.py b/src/adata/cli.py index 94d2554..309d6b1 100644 --- a/src/adata/cli.py +++ b/src/adata/cli.py @@ -5,7 +5,6 @@ from rich.console import Console import typer -from rich.markup import escape from adata.conversion import FORMATS, Options, writable_formats from adata.core.recast import LAYOUTS @@ -27,6 +26,7 @@ ) from adata.commands import export_image as export_image_cmd +from adata.util.markup import literal app = typer.Typer( help="Streaming CLI for huge AnnData .h5ad and .zarr stores " @@ -65,7 +65,7 @@ def _fail(error: BaseException) -> typer.Exit: and the command line, which Rich would read as markup: they are escaped. """ message = error.args[0] if isinstance(error, KeyError) and error.args else error - console.print(f"[bold red]Error:[/] {escape(str(message))}") + console.print(f"[bold red]Error:[/] {literal(str(message))}") return typer.Exit(code=1) @@ -1816,11 +1816,16 @@ def import_image_cmd( def main(argv: Optional[Sequence[str]] = None) -> None: - """Run the CLI on `argv`, or on the command line when it is None.""" + """Run the CLI on `argv`, or on the command line when it is None. + + Run as the `adata` command, a failed run ends the process at once + (`_leave`). Given `argv`, it raises SystemExit as Click does, for the + caller, a test or a notebook, to catch. + """ try: app(args=list(argv) if argv is not None else None, standalone_mode=True) except SystemExit as done: - if isinstance(done.code, int) and done.code != 0: + if argv is None and isinstance(done.code, int) and done.code != 0: _leave(done.code) raise diff --git a/src/adata/commands/convert.py b/src/adata/commands/convert.py index fd9641c..81e63c0 100644 --- a/src/adata/commands/convert.py +++ b/src/adata/commands/convert.py @@ -14,7 +14,6 @@ from typing import List, Optional from rich.console import Console -from rich.markup import escape from adata.conversion import ( Options, @@ -24,6 +23,7 @@ output_format, ) from adata.storage import check_hdf5_inputs +from adata.util.markup import literal from adata.util.path import check_output, is_zarr_path, same_path from adata.util.staging import check_free_space, disk_size, staged_outputs @@ -97,13 +97,13 @@ def convert_store( ): check_free_space(plan.outputs[0], lambda: disk_size(file), check=space_check) - console.print(f"[cyan]Converting[/] {escape(str(file))}: {escape(plan.summary)}") + console.print(f"[cyan]Converting[/] {literal(str(file))}: {literal(plan.summary)}") for warning in plan.warnings: - console.print(f"[yellow]Warning:[/] {escape(warning)}") + console.print(f"[yellow]Warning:[/] {literal(warning)}") with staged_outputs(plan.outputs) as staged: converter.write(plan, staged, console) for out in plan.outputs: - console.print(f"[green]Wrote[/] {escape(str(out))}") + console.print(f"[green]Wrote[/] {literal(str(out))}") return plan.outputs diff --git a/src/adata/commands/create.py b/src/adata/commands/create.py index 0cc678d..7e731f4 100644 --- a/src/adata/commands/create.py +++ b/src/adata/commands/create.py @@ -12,10 +12,10 @@ from typing import List, Optional from rich.console import Console -from rich.markup import escape from adata.elements.write import ensure_anndata_skeleton, write_dataframe_header from adata.storage import open_store +from adata.util.markup import literal from adata.util.path import check_output from adata.util.staging import staged_outputs @@ -92,6 +92,6 @@ def create_store( ensure_anndata_skeleton(root) console.print( - f"[green]Created[/] {escape(str(output))} " + f"[green]Created[/] {literal(str(output))} " f"({len(obs_index)} obs x {len(var_index)} var, empty)" ) diff --git a/src/adata/commands/export.py b/src/adata/commands/export.py index 3028553..0ba19c2 100644 --- a/src/adata/commands/export.py +++ b/src/adata/commands/export.py @@ -16,7 +16,6 @@ from typing import Iterator, List, Optional from rich.console import Console -from rich.markup import escape from adata.formats.array import export_npy as export_npy_format from adata.formats.dataframe import export_dataframe @@ -24,6 +23,7 @@ from adata.formats.json_data import export_json as export_json_format from adata.formats.sparse import export_mtx as export_mtx_format from adata.storage import open_store +from adata.util.markup import literal from adata.util.path import check_output from adata.util.staging import staged_outputs @@ -48,7 +48,7 @@ def _output(file: Path, out: Optional[Path], console: Console) -> Iterator[Optio out.parent.mkdir(parents=True, exist_ok=True) with staged_outputs([out]) as (temp,): yield temp - console.print(f"[green]Wrote[/] {escape(str(out))}") + console.print(f"[green]Wrote[/] {literal(str(out))}") def export_table( diff --git a/src/adata/commands/import_data.py b/src/adata/commands/import_data.py index 2e85296..f16aea8 100644 --- a/src/adata/commands/import_data.py +++ b/src/adata/commands/import_data.py @@ -16,7 +16,6 @@ import h5py import numpy as np from rich.console import Console -from rich.markup import escape from adata.formats.array import IMPORT_CHUNK_ELEMENTS, import_npy from adata.formats.dataframe import import_dataframe @@ -26,6 +25,7 @@ from adata.formats.sparse import import_mtx from adata.formats.validate import check_target, validate_dimensions from adata.storage import copy_path, copy_store_contents, detect_backend, open_store +from adata.util.markup import literal from adata.util.path import output_target from adata.util.staging import check_free_space, disk_size, staged_outputs @@ -47,13 +47,13 @@ def _copy_source( if src_backend == dst_backend: copy_path(file, temp) if not inplace: - console.print(f"[dim]Copied {escape(str(file))} → {escape(str(output))}[/]") + console.print(f"[dim]Copied {literal(str(file))} → {literal(str(output))}[/]") return with open_store(file, "r") as src_store, open_store(temp, "w") as dst_store: copy_store_contents(src_store.root, dst_store.root) console.print( - f"[dim]Converted {escape(str(file))} ({src_backend}) → {escape(str(output))} ({dst_backend})[/]" + f"[dim]Converted {literal(str(file))} ({src_backend}) → {literal(str(output))} ({dst_backend})[/]" ) diff --git a/src/adata/commands/info.py b/src/adata/commands/info.py index 25219f7..3576a4e 100644 --- a/src/adata/commands/info.py +++ b/src/adata/commands/info.py @@ -2,11 +2,11 @@ from typing import Any, Optional from rich.console import Console -from rich.markup import escape from rich.tree import Tree from adata.core.info import axis_len, format_type_info, get_entry_type from adata.storage import is_dataset, is_group, open_store +from adata.util.markup import literal # Preferred display order for top-level keys KEY_ORDER = ["X", "obs", "var", "obsm", "varm", "layers", "obsp", "varp", "uns"] @@ -69,9 +69,9 @@ def show_info( ] if sub_keys and key != "X": out.print( - f"\t[bold yellow]{escape(key)}:[/]\t" + f"\t[bold yellow]{literal(key)}:[/]\t" + ", ".join( - f"[bright_white]{escape(sub)}[/]" for sub in sub_keys + f"[bright_white]{literal(sub)}[/]" for sub in sub_keys ) ) @@ -90,7 +90,7 @@ def _show_types_tree( Names come from the store and are escaped: Rich would read the `[mt]` of `pct_counts[mt]` as a style tag and drop it. """ - tree = Tree(f"[bold]{escape(root_label)}[/]") + tree = Tree(f"[bold]{literal(root_label)}[/]") # Define max depth for each top-level group max_depth_map = { @@ -117,11 +117,11 @@ def add_node( if is_dataset(obj): shape_str = f"[dim]{obj.shape}[/]" if obj.shape else "" - node_text = f"[bright_white]{escape(name)}[/] {shape_str} {type_str}" + node_text = f"[bright_white]{literal(name)}[/] {shape_str} {type_str}" parent_tree.add(node_text) else: # Group - node_text = f"[bold yellow]{escape(name)}/[/] {type_str}" + node_text = f"[bold yellow]{literal(name)}/[/] {type_str}" subtree = parent_tree.add(node_text) # Recurse only if within allowed depth @@ -171,19 +171,19 @@ def _show_object_info(f: Any, entry_path: str, console: Console) -> None: entry = f[entry_path] info = get_entry_type(entry) - console.print(f"\n[bold cyan]Path:[/] {escape(entry_path)}") - console.print(f"[bold cyan]Type:[/] {escape(info['type'])}") + console.print(f"\n[bold cyan]Path:[/] {literal(entry_path)}") + console.print(f"[bold cyan]Type:[/] {literal(info['type'])}") if info["encoding"]: - console.print(f"[bold cyan]Encoding:[/] {escape(str(info['encoding']))}") + console.print(f"[bold cyan]Encoding:[/] {literal(str(info['encoding']))}") if info["shape"]: console.print(f"[bold cyan]Shape:[/] {info['shape']}") if info["dtype"]: - console.print(f"[bold cyan]Dtype:[/] {escape(info['dtype'])}") + console.print(f"[bold cyan]Dtype:[/] {literal(info['dtype'])}") - console.print(f"[bold cyan]Details:[/] {escape(info['details'])}") + console.print(f"[bold cyan]Details:[/] {literal(info['details'])}") # Show attributes if any if entry.attrs: @@ -192,7 +192,7 @@ def _show_object_info(f: Any, entry_path: str, console: Console) -> None: v_str = v.decode("utf-8") if isinstance(v, bytes) else str(v) if len(v_str) > 80: v_str = v_str[:77] + "..." - console.print(f" [dim]{escape(k)}:[/] {escape(v_str)}") + console.print(f" [dim]{literal(k)}:[/] {literal(v_str)}") # If it's a group, show children if is_group(entry): @@ -207,4 +207,4 @@ def _show_object_info(f: Any, entry_path: str, console: Console) -> None: child_entry = entry[child_name] child_info = get_entry_type(child_entry) type_str = format_type_info(child_info) - console.print(f" [bright_white]{escape(child_name)}[/] {type_str}") + console.print(f" [bright_white]{literal(child_name)}[/] {type_str}") diff --git a/src/adata/commands/ls.py b/src/adata/commands/ls.py index 2b6a383..d1aa7d4 100644 --- a/src/adata/commands/ls.py +++ b/src/adata/commands/ls.py @@ -15,11 +15,11 @@ from typing import Any, Iterator, Optional, Tuple from rich.console import Console -from rich.markup import escape from rich.tree import Tree from adata.core.info import format_type_info, get_entry_type from adata.storage import is_dataset, is_group, open_store +from adata.util.markup import literal from adata.util.path import norm_path @@ -46,9 +46,9 @@ def _describe(obj: Any, long: bool) -> str: if shape: bits.append(f"[dim]{tuple(shape)}[/]") if is_dataset(obj) and getattr(obj, "dtype", None) is not None: - bits.append(f"[dim]{escape(str(obj.dtype))}[/]") + bits.append(f"[dim]{literal(str(obj.dtype))}[/]") if info["encoding"]: - bits.append(f"[dim]{escape(str(info['encoding']))}[/]") + bits.append(f"[dim]{literal(str(info['encoding']))}[/]") return " " + " ".join(bits) @@ -97,10 +97,10 @@ def _print_tree( long: bool, ) -> None: root_label = f"{label}:{entry_path}" if entry_path else label - tree = Tree(f"[bold]{escape(root_label)}[/]") + tree = Tree(f"[bold]{literal(root_label)}[/]") if is_dataset(root): - tree.add(f"[bright_white]{escape(entry_path or label)}[/]{_describe(root, long)}") + tree.add(f"[bright_white]{literal(entry_path or label)}[/]{_describe(root, long)}") console.print(tree) return @@ -110,9 +110,9 @@ def _print_tree( parent = nodes.get(parent_path, tree) if is_group(obj): nodes[path] = parent.add( - f"[bold yellow]{escape(name)}/[/]{_describe(obj, long)}" + f"[bold yellow]{literal(name)}/[/]{_describe(obj, long)}" ) else: - parent.add(f"[bright_white]{escape(name)}[/]{_describe(obj, long)}") + parent.add(f"[bright_white]{literal(name)}[/]{_describe(obj, long)}") console.print(tree) diff --git a/src/adata/commands/recast.py b/src/adata/commands/recast.py index 03a62ff..3e62394 100644 --- a/src/adata/commands/recast.py +++ b/src/adata/commands/recast.py @@ -14,7 +14,6 @@ from typing import Any, List, Optional from rich.console import Console -from rich.markup import escape from adata.core.recast import ( DEFAULT_CHUNK, @@ -29,6 +28,7 @@ from adata.elements import spec from adata.elements.write import ensure_anndata_skeleton from adata.storage import copy_tree, detect_backend, is_dataset, is_group, open_store +from adata.util.markup import literal from adata.util.path import output_target from adata.util.staging import check_free_space, disk_size, staged_outputs @@ -137,7 +137,7 @@ def recast_store( console.print( f"[cyan]Converting {len(targets)} matrix/matrices:[/] " - + escape(", ".join(targets)) + + literal(", ".join(targets)) ) with open_store(temp, "w", zarr_format=zarr_format) as dst_store: @@ -155,9 +155,9 @@ def recast_store( ensure_anndata_skeleton(dst) if inplace: - console.print(f"[green]Converted[/] {escape(str(file))}") + console.print(f"[green]Converted[/] {literal(str(file))}") else: - console.print(f"[green]Wrote[/] {escape(str(dst_path))}") + console.print(f"[green]Wrote[/] {literal(str(dst_path))}") def _extra(plan: Plan) -> float: diff --git a/src/adata/commands/split.py b/src/adata/commands/split.py index b9a4710..3e752d3 100644 --- a/src/adata/commands/split.py +++ b/src/adata/commands/split.py @@ -19,12 +19,12 @@ import numpy as np from rich.console import Console -from rich.markup import escape from adata.core.select import group_indices from adata.core.subset import check_subsettable, split_h5ad from adata.elements.read import element_len, resolve_index from adata.storage import detect_backend, open_store +from adata.util.markup import literal from adata.util.names import unique_names from adata.util.path import check_output from adata.util.staging import check_free_space, disk_size, staged_outputs @@ -77,7 +77,7 @@ def split_store( # it ("" or "NA"), so NA always holds the rows with no value. names = unique_names(order) console.print( - f"[cyan]Splitting on {axis}\\[{escape(column)}]: " + f"[cyan]Splitting on {literal(f'{axis}[{column}]')}: " f"{len(order)} group{'s' if len(order) != 1 else ''}[/]" ) @@ -94,7 +94,7 @@ def split_store( planned.append((label, out_path, len(indices))) for label, out_path, count in planned: - console.print(f" {_shown(label)} -> {escape(str(out_path))} ({count} {axis})") + console.print(f" {_shown(label)} -> {literal(str(out_path))} ({count} {axis})") for _, out_path, _ in planned: try: @@ -139,13 +139,13 @@ def split_store( if manifest: _write_manifest(file, output_dir, column, planned, console) - console.print(f"[green]Wrote[/] {len(planned)} stores to {escape(str(output_dir))}") + console.print(f"[green]Wrote[/] {len(planned)} stores to {literal(str(output_dir))}") return planned def _shown(label: Optional[str]) -> str: """A label as a message prints it, escaped: labels are data, not markup.""" - return "(missing)" if label is None else escape(repr(label)) + return "(missing)" if label is None else literal(repr(label)) def _manifest_path(file: Path, output_dir: Path) -> Path: @@ -174,5 +174,5 @@ def _write_manifest( for label, out_path, count in planned: value = "" if label is None else label writer.writerow([source_id, column, value, str(out_path), count]) - console.print(f"[dim]Manifest written to {escape(str(path))}[/]") + console.print(f"[dim]Manifest written to {literal(str(path))}[/]") return path diff --git a/src/adata/conversion/anndata_copy.py b/src/adata/conversion/anndata_copy.py index a6b30b8..82cf5e6 100644 --- a/src/adata/conversion/anndata_copy.py +++ b/src/adata/conversion/anndata_copy.py @@ -10,7 +10,6 @@ from typing import Optional, Sequence from rich.console import Console -from rich.markup import escape from adata.conversion.base import ConversionPlan, Converter, Options from adata.elements.write import ensure_anndata_skeleton @@ -21,6 +20,7 @@ ensure_anndata_root_attrs, open_store, ) +from adata.util.markup import literal def _backend(fmt: str) -> str: @@ -83,7 +83,7 @@ def write( copy_attrs(src.root.attrs, dst.root.attrs, target_backend=backend) ensure_anndata_root_attrs(dst.root) for key in src.root.keys(): - with console.status(f"[cyan]Copying {escape(key)}...[/]"): + with console.status(f"[cyan]Copying {literal(key)}...[/]"): copy_tree( src.root[key], dst.root, key, compression=detail["compression"], diff --git a/src/adata/conversion/cellbender.py b/src/adata/conversion/cellbender.py index 55c13aa..eba4d0a 100644 --- a/src/adata/conversion/cellbender.py +++ b/src/adata/conversion/cellbender.py @@ -38,7 +38,6 @@ import h5py import numpy as np from rich.console import Console -from rich.markup import escape from adata.conversion import tenx from adata.conversion.base import ConversionPlan, Converter, Options @@ -64,6 +63,7 @@ ) from adata.formats.mtx import EntryMap, iter_matrix_entries from adata.storage import create_dataset, open_store +from adata.util.markup import literal #: Datasets of a 0.2 matrix group that are the matrix, not a latent. _MATRIX_KEYS = frozenset({ @@ -580,7 +580,7 @@ def write( ) if detail.raw is not None: - with console.status(f"[cyan]Reading the raw counts of {escape(detail.raw.path.name)}...[/]"): + with console.status(f"[cyan]Reading the raw counts of {literal(detail.raw.path.name)}...[/]"): _write_raw( detail.raw, roots, detail.targets, n_out, matrix.n_features, group, ) diff --git a/src/adata/conversion/cistopic.py b/src/adata/conversion/cistopic.py index 1db573e..f01f117 100644 --- a/src/adata/conversion/cistopic.py +++ b/src/adata/conversion/cistopic.py @@ -38,7 +38,6 @@ import numpy as np from rich.console import Console -from rich.markup import escape from adata.conversion.base import ConversionPlan, Converter, Options from adata.conversion.features import STRING_STEP, Target, write_obs_index, write_var @@ -57,6 +56,7 @@ ) from adata.formats import pickled from adata.storage import COMPRESSIONS, create_dataset, is_zarr_group, open_store, with_compression +from adata.util.markup import literal from adata.util.names import claim, count_duplicates, make_index_unique, sanitize_label EXTENSIONS = (".pkl", ".pickle", ".pkl.gz", ".pickle.gz") @@ -211,7 +211,7 @@ def write( write_dense(root[key], label, values) _write_uns(root["uns"], detail, warnings) for message in warnings: - console.print(f"[yellow]Warning:[/] {escape(message)}") + console.print(f"[yellow]Warning:[/] {literal(message)}") def _is_sparse(value: Any) -> bool: diff --git a/src/adata/conversion/hdf5.py b/src/adata/conversion/hdf5.py index a305ffd..ea13459 100644 --- a/src/adata/conversion/hdf5.py +++ b/src/adata/conversion/hdf5.py @@ -24,13 +24,13 @@ import h5py import numpy as np from rich.console import Console -from rich.markup import escape from adata.conversion.base import ConversionPlan, Converter, Options from adata.conversion.dense import write_dense_matrix from adata.conversion.features import STRING_STEP, Target, write_obs_index, write_var from adata.elements.write import ensure_anndata_skeleton, set_column_order, write_mapping, write_scalar from adata.storage import COMPRESSIONS, open_store +from adata.util.markup import literal from adata.util.names import count_duplicates, make_index_unique EXTENSIONS = (".hdf5", ".he5", ".hdf") @@ -235,7 +235,7 @@ def obs_names() -> Iterator[List[str]]: for axis, path in (("obs_names", detail.obs_names), ("var_names", detail.var_names)): if path is not None: write_scalar(meta, axis, path) - with console.status(f"[cyan]Reading {escape(detail.key)}...[/]"): + with console.status(f"[cyan]Reading {literal(detail.key)}...[/]"): write_dense_matrix( handle[detail.key], [(root, "X")], cells_axis=1 if detail.transpose else 0, dense=detail.dense, compression=detail.compression, diff --git a/src/adata/conversion/loom.py b/src/adata/conversion/loom.py index 8b7ec3a..f43b7e4 100644 --- a/src/adata/conversion/loom.py +++ b/src/adata/conversion/loom.py @@ -54,7 +54,6 @@ import h5py import numpy as np from rich.console import Console -from rich.markup import escape from adata.conversion import scope from adata.conversion.base import ConversionPlan, Converter, Options @@ -97,6 +96,7 @@ is_zarr_group, open_store, ) +from adata.util.markup import literal from adata.util.names import claim, count_duplicates, make_index_unique, storable #: Attributes that name the cells, the gene ids and the gene symbols, in the @@ -490,7 +490,7 @@ def var_names(step: int) -> Iterator[List[str]]: if not places: continue ds = handle["matrix"] if loom_name == "" else handle["layers"][loom_name] - with console.status(f"[cyan]Reading {escape(loom_name or 'matrix')}...[/]"): + with console.status(f"[cyan]Reading {literal(loom_name or 'matrix')}...[/]"): _write_matrix(ds, root, places, detail) @@ -948,11 +948,11 @@ def write( staged[0], "w" ) as out: root = store.root - with console.status(f"[cyan]Writing {escape(detail.matrix)} dense...[/]"): + with console.status(f"[cyan]Writing {literal(detail.matrix)} dense...[/]"): _export_matrix(resolve(root, detail.matrix), out, "matrix") layers = out.create_group("layers") for name, where in detail.layers: - with console.status(f"[cyan]Writing layer {escape(name)} dense...[/]"): + with console.status(f"[cyan]Writing layer {literal(name)} dense...[/]"): _export_matrix(resolve(root, where), layers, name) with console.status("[cyan]Writing the attributes...[/]"): diff --git a/src/adata/conversion/mtx.py b/src/adata/conversion/mtx.py index 61c4d65..a3dd12a 100644 --- a/src/adata/conversion/mtx.py +++ b/src/adata/conversion/mtx.py @@ -37,7 +37,6 @@ import numpy as np from rich.console import Console -from rich.markup import escape from adata.conversion.base import ConversionPlan, Converter, Options from adata.conversion.features import ( @@ -82,6 +81,7 @@ read_tsv, ) from adata.storage import COMPRESSIONS, GZIP_LEVEL, is_dataset, open_store +from adata.util.markup import literal from adata.util.path import is_zarr_path #: The velocyto matrices STARsolo writes, in the order they become layers. @@ -435,7 +435,7 @@ def write( # not leave a store holding obs and var and nothing else. scans = {} for name, header in detail.headers.items(): - with console.status(f"[cyan]Checking {escape(header.path.name)}...[/]"): + with console.status(f"[cyan]Checking {literal(header.path.name)}...[/]"): scans[name] = scan(header, emap) with ExitStack() as stack: @@ -474,7 +474,7 @@ def write( ) for parent, key in places ]) - with console.status(f"[cyan]Reading {escape(header.path.name)}...[/]"): + with console.status(f"[cyan]Reading {literal(header.path.name)}...[/]"): summed = read_into(header, emap, scanned, destinations, roots[0]) for appenders in destinations: for appender in appenders: @@ -482,7 +482,7 @@ def write( if summed: console.print( f"[yellow]Warning:[/] {summed:,} duplicate entries in " - f"{escape(header.path.name)} were summed, as scipy does." + f"{literal(header.path.name)} were summed, as scipy does." ) @@ -628,7 +628,7 @@ def write( matrix = describe(resolve(root, source.path)) check_sparse(matrix) - with console.status(f"[cyan]Counting the entries of {escape(source.path)}...[/]"): + with console.status(f"[cyan]Counting the entries of {literal(source.path)}...[/]"): nnz = count_entries(matrix) with create_binary(folder / f"matrix.mtx{ext}", gzipped=gzipped, level=GZIP_LEVEL) as fh: write_header( @@ -639,7 +639,7 @@ def write( nnz=nnz, comments=_metadata_comment(root), ) - with console.status(f"[cyan]Writing {escape(source.path)}...[/]"): + with console.status(f"[cyan]Writing {literal(source.path)}...[/]"): write_entries(fh, iter_matrix_entries(matrix, transpose=True)) diff --git a/src/adata/conversion/mudata.py b/src/adata/conversion/mudata.py index 928b3b4..f906091 100644 --- a/src/adata/conversion/mudata.py +++ b/src/adata/conversion/mudata.py @@ -34,7 +34,6 @@ import h5py import numpy as np from rich.console import Console -from rich.markup import escape from adata.conversion.base import ConversionPlan, Converter, Options from adata.conversion.features import type_counts, write_index_frame @@ -69,6 +68,7 @@ write_legacy_categorical, zarr_format_of, ) +from adata.util.markup import literal from adata.util.names import claim, sanitize_label, storable from adata.util.path import same_path @@ -416,10 +416,10 @@ def write(self, plan: ConversionPlan, staged: Sequence[Path], console: Console) copy_attrs(group.attrs, dst.root.attrs, target_backend=backend) ensure_anndata_root_attrs(dst.root) for key in group.keys(): - with console.status(f"[cyan]Copying {escape(mod.name)}/{escape(key)}...[/]"): + with console.status(f"[cyan]Copying {literal(mod.name)}/{literal(key)}...[/]"): copy_tree(group[key], dst.root, key, compression=detail.compression) ensure_anndata_skeleton(dst.root) - with console.status(f"[cyan]Adding the global parts to {escape(mod.name)}...[/]"): + with console.status(f"[cyan]Adding the global parts to {literal(mod.name)}...[/]"): for axis, (_, mapping, graphs) in zip(mod.axes, _AXES): _write_axis(src.root, dst.root, axis, mapping, graphs, detail.compression) for key in mod.uns: diff --git a/src/adata/conversion/parse.py b/src/adata/conversion/parse.py index f089035..72d2b11 100644 --- a/src/adata/conversion/parse.py +++ b/src/adata/conversion/parse.py @@ -29,7 +29,6 @@ import numpy as np from rich.console import Console -from rich.markup import escape from adata.conversion.base import ConversionPlan, Converter, Options from adata.conversion.features import ( @@ -43,6 +42,7 @@ from adata.formats.mtx import EntryMap, MtxHeader, read_header, read_into, scan, value_dtype from adata.formats.table import TableScan, read_small_table, scan_table, write_table from adata.storage import COMPRESSIONS, open_store +from adata.util.markup import literal from adata.util.names import unique_names from adata.util.path import is_zarr_path @@ -299,7 +299,7 @@ def write( # cannot be read must not leave a store holding obs and var alone. scans = [] for item in detail.items: - with console.status(f"[cyan]Checking {escape(_label(item))}...[/]"): + with console.status(f"[cyan]Checking {literal(_label(item))}...[/]"): scans.append(scan(item.header, item.emap)) for item, scanned, path in zip(detail.items, scans, staged): @@ -307,7 +307,7 @@ def write( root = stack.enter_context( open_store(path, "w", zarr_format=detail.zarr_format) ).root - with console.status(f"[cyan]Writing obs and var of {escape(str(item.dge.sample))}...[/]"): + with console.status(f"[cyan]Writing obs and var of {literal(str(item.dge.sample))}...[/]"): write_table(root, "obs", item.cells, rows=item.rows) write_var(root, item.target) ensure_anndata_skeleton(root) @@ -317,13 +317,13 @@ def write( data_dtype=value_dtype(item.header, scanned), compression=detail.compression, ) - with console.status(f"[cyan]Reading {escape(_label(item))}...[/]"): + with console.status(f"[cyan]Reading {literal(_label(item))}...[/]"): summed = read_into(item.header, item.emap, scanned, [[appender]], root) appender.finish() if summed: console.print( f"[yellow]Warning:[/] {summed:,} duplicate entries in " - f"{escape(_label(item))} were summed, as scipy does." + f"{literal(_label(item))} were summed, as scipy does." ) diff --git a/src/adata/conversion/tenx.py b/src/adata/conversion/tenx.py index 6dafa67..40a9041 100644 --- a/src/adata/conversion/tenx.py +++ b/src/adata/conversion/tenx.py @@ -34,7 +34,6 @@ import h5py import numpy as np from rich.console import Console -from rich.markup import escape from adata.conversion.base import ConversionPlan, Converter, Options from adata.conversion.features import ( @@ -75,6 +74,7 @@ is_dataset, open_store, ) +from adata.util.markup import literal #: What a group must hold to be a matrix, in each layout. _V3_KEYS = frozenset({"barcodes", "data", "indices", "indptr", "shape", "features"}) @@ -597,7 +597,7 @@ def write( ) as out: root = store.root matrix = describe(resolve(root, detail["path"])) - with console.status(f"[cyan]Copying {escape(detail['path'])}...[/]"): + with console.status(f"[cyan]Copying {literal(detail['path'])}...[/]"): group = _write_matrix(matrix, out) n_obs, n_var = matrix.shape group.create_dataset("shape", data=np.array([n_var, n_obs], dtype=np.int32)) @@ -606,7 +606,7 @@ def write( obs_index, _ = resolve_index(obs, "obs") n_text, example = _write_fixed_strings_streamed(group, "barcodes", obs_index, obs) if n_text: - console.print(f"[yellow]Warning:[/] {escape(_ascii_warning(n_text, 'barcodes', example))}") + console.print(f"[yellow]Warning:[/] {literal(_ascii_warning(n_text, 'barcodes', example))}") flags = obs.get("filtered_barcodes") if is_dataset(flags) and "categories" not in flags.attrs and flags.dtype.kind == "b": # Space Ranger's and Flex's barcodes in the filtered matrix, diff --git a/src/adata/conversion/text.py b/src/adata/conversion/text.py index 88aa2cc..c35e274 100644 --- a/src/adata/conversion/text.py +++ b/src/adata/conversion/text.py @@ -40,7 +40,6 @@ import numpy as np from rich.console import Console -from rich.markup import escape from adata.conversion.base import ConversionPlan, Converter, Options from adata.conversion.features import Target, write_obs_index, write_var @@ -63,6 +62,7 @@ open_store, with_compression, ) +from adata.util.markup import literal from adata.util.names import count_duplicates, make_index_unique #: Extensions read as text, each also gzipped; and the delimiter each implies. @@ -553,7 +553,7 @@ def write( write_var(root, Target(label=None, keep=None, var_names=detail.var_names, columns={})) ensure_anndata_skeleton(root) _write_uns(root["uns"], detail) - with console.status(f"[cyan]Reading {escape(Path(plan.source).name)}...[/]"): + with console.status(f"[cyan]Reading {literal(Path(plan.source).name)}...[/]"): if detail.transpose: obs = _write_names(root, detail.obs_names) _write_transposed(root, reader, detail) diff --git a/src/adata/conversion/visium.py b/src/adata/conversion/visium.py index 5ed5ba6..47c39cc 100644 --- a/src/adata/conversion/visium.py +++ b/src/adata/conversion/visium.py @@ -49,7 +49,6 @@ import h5py import numpy as np from rich.console import Console -from rich.markup import escape from adata.conversion import tenx from adata.conversion.base import ConversionPlan, Converter, Options @@ -84,6 +83,7 @@ from adata.formats.textio import open_binary from adata.storage import create_dataset, is_zarr_path, open_store from adata.elements import spec +from adata.util.markup import literal #: The columns of `tissue_positions.csv`, which Space Ranger 1 wrote without #: a header, in this order. @@ -733,7 +733,7 @@ def _write_spatial_uns(root: Any, detail: _Plan, console: Console) -> None: # anndata 0.8 to 0.11 cannot read a null, and leave out None themselves. console.print( f"[yellow]Warning:[/] scalefactors_json.json holds null for " - f"{escape(', '.join(path.rsplit('/', 1)[-1] for path in skipped))}; left out, as " + f"{literal(', '.join(path.rsplit('/', 1)[-1] for path in skipped))}; left out, as " "anndata leaves out None." ) metadata = write_mapping(library, "metadata") diff --git a/src/adata/core/concat.py b/src/adata/core/concat.py index 165541e..85de194 100644 --- a/src/adata/core/concat.py +++ b/src/adata/core/concat.py @@ -27,7 +27,6 @@ import h5py import numpy as np from rich.console import Console -from rich.markup import escape from adata.core.info import axis_len from adata.core.subset import _open_output_limit, aligned_rows @@ -66,6 +65,7 @@ open_store, zarr_format_of, ) +from adata.util.markup import literal from adata.util.names import natural_sorted from adata.util.staging import check_free_space, disk_size, staged_outputs @@ -559,7 +559,7 @@ def _concat_categorical( ) if any(ordered) and not keeps_order: console.print( - f"[yellow]Column {escape(repr(name))}: the inputs disagree on its category order, " + f"[yellow]Column {literal(repr(name))}: the inputs disagree on its category order, " "so it is written unordered (anndata keeps no order either).[/]" ) lookup = {category: i for i, category in enumerate(categories.tolist())} @@ -724,8 +724,8 @@ def _concat_obs_column( _concat_text(parent, name, column, lengths) else: console.print( - f"[yellow]Dropping column {escape(repr(name))}: " - f"cannot concatenate encodings {escape(str(sorted({kind for _, kind in kinds})))}[/]" + f"[yellow]Dropping column {literal(repr(name))}: " + f"cannot concatenate encodings {literal(str(sorted({kind for _, kind in kinds})))}[/]" ) return False return True @@ -1282,13 +1282,13 @@ def _concat_obsm( return True kinds = {_matrix_kind(s) for s in inputs.each(path)} if "other" in kinds: - console.print(f"[yellow]Skipping obsm/{escape(name)}: not an array, or not a dataframe, in every input[/]") + console.print(f"[yellow]Skipping obsm/{literal(name)}: not an array, or not a dataframe, in every input[/]") return False widths = {_trailing_shape(s) for s in inputs.each(path)} if len(widths) != 1: console.print( - f"[yellow]Skipping obsm/{escape(name)}: inputs disagree on shape {sorted(widths)}[/]" + f"[yellow]Skipping obsm/{literal(name)}: inputs disagree on shape {sorted(widths)}[/]" ) return False if len(kinds) != 1: @@ -1509,7 +1509,7 @@ def _write_pairwise( return enc = spec.encoding_type(obj) if enc not in spec.SPARSE_TYPES: - console.print(f"[yellow]Skipping varp/{escape(name)}: neither an array nor a sparse matrix[/]") + console.print(f"[yellow]Skipping varp/{literal(name)}: neither an array nor a sparse matrix[/]") return if len(take) == _shape(obj)[0] and np.array_equal(take, np.arange(len(take))): copy_tree(obj, parent, name) @@ -1589,7 +1589,7 @@ def aligned() -> Iterator[Any]: _write_var_column(frame, column, obj[column], take, obj) set_column_order(frame, columns) else: - console.print(f"[yellow]Skipping varm/{escape(name)}: neither an array nor a dataframe[/]") + console.print(f"[yellow]Skipping varm/{literal(name)}: neither an array nor a dataframe[/]") def _write_varp( @@ -1901,10 +1901,10 @@ def concat_on_disk( console.print( f"[cyan]Concatenating {len(files)} stores: " - f"{n_obs} obs x {n_var} var ({escape(join)} join)[/]" + f"{n_obs} obs x {n_var} var ({literal(join)} join)[/]" ) for f, key, count in zip(files, input_keys, obs_counts): - console.print(f" [dim]{escape(str(key))}[/]: {count} obs from {escape(f.name)}") + console.print(f" [dim]{literal(str(key))}[/]: {count} obs from {literal(f.name)}") if len(set(target_obs)) != len(target_obs): console.print( @@ -1962,7 +1962,7 @@ def concat_on_disk( if any(inputs.get(i, "raw") is not None for i in range(len(inputs))): console.print("[yellow]Dropping raw/: not concatenated.[/]") - console.print(f"[green]Wrote[/] {escape(str(output))} ({n_obs} obs x {n_var} var)") + console.print(f"[green]Wrote[/] {literal(str(output))} ({n_obs} obs x {n_var} var)") def _write_obs( @@ -2122,7 +2122,7 @@ def _write_matrices( path = f"layers/{name}" if not all(layer is not None for layer in inputs.each(path)): console.print( - f"[yellow]Skipping layer {escape(repr(name))}: not present in every input[/]" + f"[yellow]Skipping layer {literal(repr(name))}: not present in every input[/]" ) continue _concat_matrix(layers, name, inputs, path, col_maps, row_counts, n_obs, n_var, @@ -2150,7 +2150,7 @@ def _write_obsm( path = f"obsm/{name}" if not all(entry is not None for entry in inputs.each(path)): console.print( - f"[yellow]Skipping obsm/{escape(name)}: not present in every input[/]" + f"[yellow]Skipping obsm/{literal(name)}: not present in every input[/]" ) continue _concat_obsm( diff --git a/src/adata/core/recast.py b/src/adata/core/recast.py index 9d3d5a0..d3fc8c4 100644 --- a/src/adata/core/recast.py +++ b/src/adata/core/recast.py @@ -28,7 +28,6 @@ import numpy as np from rich.console import Console -from rich.markup import escape from adata.elements import spec from adata.elements.write import set_shape_attr @@ -51,6 +50,7 @@ with_compression, zarr_format_of, ) +from adata.util.markup import literal #: Layouts the user can ask for. "sparse" means "whichever sparse encoding #: keeps the major axis it already has", so densify/sparsify round-trips. @@ -774,7 +774,7 @@ def sparsify( density = nnz / max(1, n_rows * n_cols) if console is not None and density > 0.5: console.print( - f"[yellow]{escape(name)} is {density:.0%} nonzero; the sparse form is " + f"[yellow]{literal(name)} is {density:.0%} nonzero; the sparse form is " f"larger than the dense one below about 33%.[/]" ) return nnz, density @@ -1044,7 +1044,7 @@ def plan_conversion( raise ValueError(f"{name}: {message}. Pass --force to convert anyway.") if console is not None: colour = "dim" if report.lossless else "yellow" - console.print(f"[{colour}]{escape(f'{name}: {message}')}[/]") + console.print(f"[{colour}]{literal(f'{name}: {message}')}[/]") return Plan(src, target_layout, data_dtype, idx_dtype, ptr_dtype, report) diff --git a/src/adata/core/subset.py b/src/adata/core/subset.py index f7b0cee..5ea0032 100644 --- a/src/adata/core/subset.py +++ b/src/adata/core/subset.py @@ -10,7 +10,6 @@ import numpy as np from rich.console import Console -from rich.markup import escape from rich.progress import ( Progress, SpinnerColumn, @@ -49,6 +48,7 @@ write_legacy_categorical, zarr_format_of, ) +from adata.util.markup import literal from adata.util.path import output_target from adata.util.staging import check_free_space, disk_size, staged_outputs @@ -975,7 +975,7 @@ def _write_targets( console.print( "[yellow]Copying unrecognised top-level " f"{'keys' if len(passthrough) > 1 else 'key'} verbatim: " - f"{escape(', '.join(sorted(passthrough)))}[/]" + f"{literal(', '.join(sorted(passthrough)))}[/]" ) tasks.extend(f"copy:{k}" for k in passthrough) @@ -991,7 +991,7 @@ def _write_targets( transient=False, ) as progress: for task in tasks: - task_id = progress.add_task(f"[cyan]{verb} {escape(task)}...[/]", total=None) + task_id = progress.add_task(f"[cyan]{verb} {literal(task)}...[/]", total=None) if task in ("obs", "var"): for target, narrowed in zip(targets, pruned): indices = _axis_idx(target, task) @@ -1058,7 +1058,7 @@ def _write_targets( ) progress.update( task_id, - description=f"[green]{verb} {escape(task)}[/]", + description=f"[green]{verb} {literal(task)}[/]", completed=1, total=1, ) diff --git a/src/adata/formats/array.py b/src/adata/formats/array.py index 6d1e301..6294df1 100644 --- a/src/adata/formats/array.py +++ b/src/adata/formats/array.py @@ -18,7 +18,6 @@ import numpy as np from rich.console import Console -from rich.markup import escape from adata.elements import spec from adata.elements.read import decode_str_array @@ -27,6 +26,7 @@ from adata.formats.validate import check_target, validate_dimensions from adata.storage import create_dataset, is_dataset, is_group, is_zarr_group from adata.elements.strings import is_string_dtype +from adata.util.markup import literal #: Elements of a `.npy` copied into the store at once by `import array`. IMPORT_CHUNK_ELEMENTS = 1 << 20 @@ -148,7 +148,7 @@ def export_npy( elif missing: dtype = np.dtype(np.float64) console.print( - f"[yellow]Note:[/] '{escape(obj)}' has {missing:,} missing values, written " + f"[yellow]Note:[/] '{literal(obj)}' has {missing:,} missing values, written " "as NaN, so the array is float64." ) else: @@ -177,7 +177,7 @@ def convert(start: Optional[int], end: Optional[int]) -> np.ndarray: status = ( nullcontext() if to_stdout - else console.status(f"[magenta]Exporting {escape(obj)} to {escape(str(out))}...[/]") + else console.status(f"[magenta]Exporting {literal(obj)} to {literal(str(out))}...[/]") ) with target as fh, status as progress, stdout_reader_may_stop(): fh.write(header) @@ -187,7 +187,7 @@ def convert(start: Optional[int], end: Optional[int]) -> np.ndarray: n = shape[0] for start, end in _row_blocks(shape, chunk_elements): if progress: - progress.update(f"[magenta]Exporting {escape(obj)}: {start}-{end} of {n}...[/]") + progress.update(f"[magenta]Exporting {literal(obj)}: {start}-{end} of {n}...[/]") fh.write(convert(start, end).tobytes()) @@ -251,4 +251,4 @@ def import_npy( target[start:end] = block shape_str = "×".join(str(d) for d in arr.shape) - console.print(f"[green]Imported[/] {shape_str} array into '{escape(obj)}'") + console.print(f"[green]Imported[/] {shape_str} array into '{literal(obj)}'") diff --git a/src/adata/formats/dataframe.py b/src/adata/formats/dataframe.py index aff3a13..5ebd93c 100644 --- a/src/adata/formats/dataframe.py +++ b/src/adata/formats/dataframe.py @@ -7,7 +7,6 @@ from typing import Any, List, Optional from rich.console import Console -from rich.markup import escape from adata.core.read import col_chunk_as_strings from adata.elements.read import dataframe_columns, element_len, resolve_index @@ -16,6 +15,7 @@ from adata.formats.table import CATEGORY_CAP, scan_table, write_table from adata.formats.validate import check_element_name, check_target, validate_dimensions from adata.storage import is_group +from adata.util.markup import literal def export_dataframe( @@ -87,7 +87,7 @@ def export_dataframe( status_ctx = ( nullcontext() if to_stdout - else console.status(f"[magenta]Exporting {escape(axis)} table to {escape(str(out))}...[/]") + else console.status(f"[magenta]Exporting {literal(axis)} table to {literal(str(out))}...[/]") ) with status_ctx as status: @@ -160,12 +160,12 @@ def import_dataframe( if column.name in forced: if kind not in ("str", "category"): console.print( - f"[yellow]Note:[/] {escape(repr(column.name))} holds {kind} values, so it " + f"[yellow]Note:[/] {literal(repr(column.name))} holds {kind} values, so it " "stays as it is rather than a categorical." ) elif column.seen is None: console.print( - f"[yellow]Warning:[/] {escape(repr(column.name))} has more than " + f"[yellow]Warning:[/] {literal(repr(column.name))} has more than " f"{CATEGORY_CAP:,} distinct values, so it is written as strings." ) else: @@ -185,5 +185,5 @@ def import_dataframe( detail = f" ({n_categorical} categorical)" if n_categorical else "" console.print( f"[green]Imported[/] {scan.n_rows} rows x {len(scan.columns)} columns " - f"into '{escape(obj)}'{detail}" + f"into '{literal(obj)}'{detail}" ) diff --git a/src/adata/formats/image.py b/src/adata/formats/image.py index fb68b7c..9317db9 100644 --- a/src/adata/formats/image.py +++ b/src/adata/formats/image.py @@ -7,12 +7,12 @@ import numpy as np from PIL import Image from rich.console import Console -from rich.markup import escape from adata.elements.write import write_dense, write_mapping from adata.formats.validate import DATAFRAME_PATHS, check_target, validate_dimensions from adata.formats.common import _resolve from adata.storage import is_dataset +from adata.util.markup import literal from adata.util.path import norm_path @@ -150,5 +150,5 @@ def import_image(root: Any, obj: str, input_file: Path, console: Console) -> Non write_dense(parent, parts[-1], arr, replace=True) console.print( f"[green]Imported[/] {'x'.join(str(d) for d in arr.shape)} " - f"image ({arr.dtype}) into '{escape(obj)}'" + f"image ({arr.dtype}) into '{literal(obj)}'" ) diff --git a/src/adata/formats/json_data.py b/src/adata/formats/json_data.py index 66c98f2..4fde58f 100644 --- a/src/adata/formats/json_data.py +++ b/src/adata/formats/json_data.py @@ -8,7 +8,6 @@ import numpy as np from rich.console import Console -from rich.markup import escape from adata.core.read import decode_str_array from adata.elements import spec @@ -21,6 +20,7 @@ write_string_array, ) from adata.storage import is_dataset, is_group +from adata.util.markup import literal def export_json( @@ -187,11 +187,11 @@ def import_json( console.print( f"[yellow]Warning:[/] {len(skipped)} null " f"{'value was' if len(skipped) == 1 else 'values were'} left out, as " - f"anndata leaves out None: {escape(', '.join(skipped[:5]))}" + f"anndata leaves out None: {literal(', '.join(skipped[:5]))}" f"{', ...' if len(skipped) > 5 else ''}" ) - console.print(f"[green]Imported[/] JSON data into '{escape(obj)}'") + console.print(f"[green]Imported[/] JSON data into '{literal(obj)}'") def write_json_value(parent: Any, name: str, value: Any) -> List[str]: diff --git a/src/adata/formats/sparse.py b/src/adata/formats/sparse.py index 6590e0c..9a6cafc 100644 --- a/src/adata/formats/sparse.py +++ b/src/adata/formats/sparse.py @@ -12,7 +12,6 @@ from typing import Any from rich.console import Console -from rich.markup import escape from adata.core.csr_build import CsrAppender from adata.core.recast import describe @@ -34,6 +33,7 @@ from adata.formats.textio import create_binary from adata.formats.validate import check_target, validate_dimensions from adata.storage import is_dataset, is_group, is_zarr_group +from adata.util.markup import literal def export_mtx( @@ -88,7 +88,7 @@ def export_mtx( status = ( nullcontext() if to_stdout - else console.status(f"[magenta]Exporting {escape(obj)} to {escape(str(out))}...[/]") + else console.status(f"[magenta]Exporting {literal(obj)} to {literal(str(out))}...[/]") ) with target as fh, status, stdout_reader_may_stop(): write_header( @@ -122,7 +122,7 @@ def import_mtx( validate_dimensions(root, obj, shape, console) emap = EntryMap(n_rows=header.n_rows, n_cols=header.n_cols) - with console.status(f"[magenta]Checking {escape(input_file.name)}...[/]"): + with console.status(f"[magenta]Checking {literal(input_file.name)}...[/]"): scanned = scan(header, emap) dtype = value_dtype(header, scanned) @@ -138,7 +138,7 @@ def import_mtx( appender = CsrAppender( parent, name, shape=shape, data_dtype=dtype, compression=compression ) - with console.status(f"[magenta]Importing {escape(input_file.name)}...[/]"): + with console.status(f"[magenta]Importing {literal(input_file.name)}...[/]"): summed = read_into(header, emap, scanned, [[appender]], parent) nnz = appender.finish() @@ -149,5 +149,5 @@ def import_mtx( ) console.print( f"[green]Imported[/] {shape[0]}×{shape[1]} sparse matrix ({nnz} non-zero, " - f"{dtype}) into '{escape(obj)}'" + f"{dtype}) into '{literal(obj)}'" ) diff --git a/src/adata/formats/validate.py b/src/adata/formats/validate.py index 23fa7bf..21471e2 100644 --- a/src/adata/formats/validate.py +++ b/src/adata/formats/validate.py @@ -3,9 +3,9 @@ from typing import Optional, Tuple, Any from rich.console import Console -from rich.markup import escape from adata.core.info import axis_len +from adata.util.markup import literal from adata.util.path import norm_path @@ -278,4 +278,4 @@ def validate_dimensions( ) return - console.print(f"[dim]Note: No dimension validation for path '{escape(obj_path)}'[/]") + console.print(f"[dim]Note: No dimension validation for path '{literal(obj_path)}'[/]") diff --git a/src/adata/util/markup.py b/src/adata/util/markup.py new file mode 100644 index 0000000..e320d60 --- /dev/null +++ b/src/adata/util/markup.py @@ -0,0 +1,29 @@ +"""Text from a store or the command line, put into Rich markup. + +Everything the CLI prints goes through Rich, which reads `[word]` as a style +tag. Member names, labels, paths and attribute values are data, and scanpy's +own QC column is `pct_counts[mt]`: printed as it is, it shows as +`pct_counts`, and a value holding `[/x]` stops the command. Every such value +goes into a message, status line, progress description or tree label through +`literal`. +""" + +from __future__ import annotations + +from rich.markup import escape + + +def literal(value: object) -> str: + """`value` as markup that prints it as it is, whatever follows it. + + `rich.markup.escape` doubles a trailing backslash, so that a tag after it + is still read as a tag; where text follows instead, the doubled one is + printed. Here trailing backslashes are doubled and followed by an empty + span, which reads the same before a tag and before text. + """ + text = str(value) + body = text.rstrip("\\") + trailing = len(text) - len(body) + if not trailing: + return escape(body) + return escape(body) + "\\" * (2 * trailing) + "[i][/i]" diff --git a/tests/test_cli_output.py b/tests/test_cli_output.py index fb3bdc1..b13acc0 100644 --- a/tests/test_cli_output.py +++ b/tests/test_cli_output.py @@ -244,10 +244,79 @@ def test_the_deprecated_entry_point_warns_on_stderr_and_runs_its_arguments( assert "'h5ad' command is deprecated" in captured.err -def test_main_exits_with_the_failing_code(monkeypatch, tmp_path): - """A failed run still leaves through `_leave`, with its exit code.""" +def test_main_run_from_the_command_line_leaves_with_the_failing_code(monkeypatch, tmp_path): + """Run as the `adata` command, a failed run still ends through `_leave`.""" codes = [] monkeypatch.setattr(cli, "_leave", lambda code: codes.append(code)) + monkeypatch.setattr(sys, "argv", ["adata", "view", str(tmp_path / "missing.h5ad")]) with pytest.raises(SystemExit): - cli.main(["view", str(tmp_path / "missing.h5ad")]) + cli.main() assert codes == [2] + + +@pytest.mark.parametrize( + "value", ["plain", "pct_counts[mt]", "[red]b", "x[/y]", "x[red", "a]b", "end\\", "two\\\\", "a\\[b]", "[link=x]y"] +) +@pytest.mark.parametrize( + "template,expected", + [ + ("[cyan]{}[/]", "{}"), + ("{}: tail", "{}: tail"), + ("'{}'", "'{}'"), + ("[dim]{}[/dim] [green]ok[/]", "{} ok"), + ], +) +def test_literal_prints_any_value_as_it_is_whatever_follows_it(value, template, expected): + """F042, from the review: `rich.markup.escape` doubles a trailing + backslash so a tag after it still counts, and where text followed instead + the name printed with two (`Skipping obsm/emb\\\\: ...`). + + A template never writes brackets around a value (`obs\\[{}]`): a value + with an unclosed `[` would close into a tag there, so split escapes the + whole `obs[column]`.""" + from rich.text import Text + + from adata.util.markup import literal + + assert Text.from_markup(template.format(literal(value))).plain == expected.format(value) + + +def test_split_names_a_column_holding_an_unclosed_bracket_as_it_is(tmp_path): + """F042, from the review: `obs\\[{column}]` closed a column `x[red` into the + tag `[red]`, so the header read `Splitting on obs[x` in red.""" + obs = pd.DataFrame({"x[red": pd.Categorical(["a", "a", "b"]), "y\\": ["u", "v", "v"]}, + index=["c0", "c1", "c2"]) + path = tmp_path / "open.h5ad" + ad.AnnData(np.ones((3, 2), dtype="float32"), obs=obs).write_h5ad(path) + for column in ad.read_h5ad(path).obs.columns: + result = runner.invoke( + app, ["split", str(path), "--by", column, "-o", str(tmp_path / "out"), "--dry-run"] + ) + assert result.exit_code == 0, result.output + assert f"Splitting on obs[{column}]: 2 groups" in result.stderr + + +def test_a_name_ending_in_a_backslash_prints_with_one(tmp_path): + """F042, from the review: view printed `Path: uns/slash\\\\`.""" + path = tmp_path / "slash.h5ad" + adata = ad.AnnData(np.ones((2, 2), dtype="float32")) + adata.uns["slash\\"] = np.arange(3) + adata.write_h5ad(path) + assert "slash\\" in ad.read_h5ad(path).uns + viewed = runner.invoke(app, ["view", str(path), "uns/slash\\"]) + assert viewed.exit_code == 0, viewed.output + assert "Path: uns/slash\\\n" in viewed.stdout + listed = runner.invoke(app, ["ls", str(path), "uns"]) + assert listed.exit_code == 0, listed.output + assert "slash\\\n" in listed.stdout and "slash\\\\" not in listed.stdout + + +def test_main_given_arguments_raises_on_failure_rather_than_ending_the_process(monkeypatch): + """F161, from the review: a failure in `main(argv)` ended the process with + os._exit, so a test or notebook embedding the CLI died with it.""" + left = [] + monkeypatch.setattr(cli, "_leave", lambda code: left.append(code)) + with pytest.raises(SystemExit) as done: + cli.main(["view", "/no/such/store.h5ad"]) + assert done.value.code == 2 + assert left == [] From 9d488c2ac44c97c91b72be78afa8ba678ebc8f09 Mon Sep 17 00:00:00 2001 From: Aljes Date: Tue, 3 Nov 2026 18:10:48 +0000 Subject: [PATCH 4/5] STATUS: two test-suite follow-ups noticed in B10 Co-Authored-By: Claude Opus 5.5 --- STATUS.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/STATUS.md b/STATUS.md index fbd503c..a3ba3d6 100644 --- a/STATUS.md +++ b/STATUS.md @@ -89,7 +89,13 @@ Steps: PyTables" and its `compatibility` job "anndata 0.8-0.13", though the first checks every converter's reader and the second reads 0.7.8 too. Renaming a job changes its required-check name, so it waits for the - repository settings (B10). + repository settings (B10). `test_concat_versions.py` is an + `integration` test no CI job runs (B10). + - `test_write_safety.py::test_export_writes_to_devices_and_pipes_as_they_are` + failed once in a dozen local runs on macOS: its FIFO reader stayed + blocked with no writer left. The reader thread is not a daemon, so it + also kept pytest from exiting; a daemon thread would at least let the + run end (B10). - From wave 2 of the audit fixes: - CI: a Python 3.14 leg; Dependabot for the pinned actions, uv and base image (the pinned actions run on Node 20, which GitHub is retiring); the From 2e496ff1596e93820ea12e2f9fe016599a32db7f Mon Sep 17 00:00:00 2001 From: Aljes Date: Wed, 4 Nov 2026 11:21:30 +0000 Subject: [PATCH 5/5] Help text is Rich markup on typer 0.20 too typer 0.21 made Rich markup the default for help text; 0.20, the floor, printed it raw. So on the floors job convert --help showed 'pyadata-cli\[spatial]' with its escape, and the F160 test failed there. Setting rich_markup_mode on the app and both sub-apps gives the same help on every typer the floor allows (checked on 0.20.0, 0.21.1 and 0.27.3). Co-Authored-By: Claude Opus 5.5 --- src/adata/cli.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/adata/cli.py b/src/adata/cli.py index 2752b5c..02454f7 100644 --- a/src/adata/cli.py +++ b/src/adata/cli.py @@ -28,9 +28,13 @@ from adata.commands import export_image as export_image_cmd from adata.util.markup import literal +# Help text is Rich markup on every typer the floor allows: typer 0.21 made +# that the default, but 0.20 printed it raw, so the escaped brackets of +# `pyadata-cli\[spatial]` showed their backslash there. app = typer.Typer( help="Streaming CLI for huge AnnData .h5ad and .zarr stores " - "(view, ls, create, subset, split, concat, convert, recast, export, import)." + "(view, ls, create, subset, split, concat, convert, recast, export, import).", + rich_markup_mode="rich", ) @@ -78,8 +82,8 @@ def _version_callback(value: bool) -> None: # Create sub-apps for export and import -export_app = typer.Typer(help="Export objects from an AnnData store.") -import_app = typer.Typer(help="Import objects into an AnnData store.") +export_app = typer.Typer(help="Export objects from an AnnData store.", rich_markup_mode="rich") +import_app = typer.Typer(help="Import objects into an AnnData store.", rich_markup_mode="rich") app.add_typer(export_app, name="export") app.add_typer(import_app, name="import")