Support aggregating RetroRules datasets within one release

README.md

@@ -39,6 +39,10 @@ python -m rrparser --release 3.1.0 --dataset rhea --diameters 0-8 -o rules.csv
 # Equivalent selection using radii.
 python -m rrparser --dataset rhea --radii 0,1,2,3,4 -o rules.csv
 
+# Aggregate datasets within one release (comma or + separator).
+python -m rrparser --release 3.1.0 --dataset metanetx,rhea -d 0-8 -o rules.csv
+python -m rrparser --release 3.1.0 --dataset all -o rules.csv
+
 # Historical rr02 rules, restricted to retro usage and diameter 10.
 python -m rrparser --release rr02 --rule-type retro -d 10 -o rules.csv
 
@@ -59,6 +63,26 @@ reused from cache, including offline. To update, list available releases and
 choose a version explicitly, or delete `retrorules-latest-<dataset>.txt` from
 the cache to resolve `latest` again. The default remains pinned to `3.1.0`.
 
+### Aggregate datasets
+
+Use `--dataset metanetx,rhea` or `--dataset metanetx+rhea` to combine datasets,
+or `--dataset all` to include every dataset discovered for the selected release.
+Repeated names are ignored. With rr02, `all` selects its only dataset, MetaNetX.
+
+Aggregation keeps one row per template ID. Shared templates combine their
+source reactions, EC annotations, radii, and dataset provenance as unique
+semicolon-separated values. Counts and radius bounds are recalculated, the
+highest available score is retained, and validation must hold in every source.
+Different SMARTS for the same template ID cause an error. Diameter/radius and
+EC filtering, followed by optional score replacement, apply to the merged data.
+
+Each original dataset uses its own cache file; no combined archive is created.
+`latest` resolves a single release containing all explicitly requested datasets,
+so versions are never mixed. `all` and `latest` selections are recorded in
+`retrorules-selection-<release>-<selection>.txt` for offline reuse. Delete that
+selection file to discover the choices again. An explicit cached combination
+such as `metanetx,rhea` also works offline without a selection file.
+
 ### Diameter and radius selection
 
 `--diameters` accepts comma-separated values, inclusive ranges, and comparisons:
@@ -130,7 +154,7 @@ archive cache.
 | --- | --- |
 | `--rules_file` | Local input file, or `retrorules` (default) to use a cached/downloaded dataset. |
 | `--release` | Version, `rr02`, or `latest`; default `3.1.0`. |
-| `--dataset` | Dataset name; default `metanetx`. See `--list-releases`. |
+| `--dataset` | Dataset name, comma/plus-separated names, or `all`; default `metanetx`. See `--list-releases`. |
 | `--list-releases` | Print available release/dataset pairs and exit. |
 | `--cache-dir`, `--rules-dir` | Source cache directory; default package-local `rrparser/cache`. |
 | `-d`, `--diameters` | List, range, or comparison; default all even diameters 0–20. |
@@ -172,6 +196,14 @@ csv_text = parse_rules(
     radii="0,1,2",
 )
 
+# Aggregate datasets; dataset="all" is also supported.
+parse_rules(
+    outfile="combined.csv",
+    release="3.1.0",
+    dataset="metanetx+rhea",
+    diameters="<=8",
+)
+
 # Parse a local file with EC filtering and replacement scores.
 parse_rules(
     outfile="filtered.csv",

rrparser/Args.py

@@ -59,7 +59,7 @@ def _add_arguments(parser):
     parser.add_argument(
         "--dataset",
         default="metanetx",
-        help="RetroRules dataset (default: metanetx; see --list-releases)",
+        help="Dataset names (metanetx,rhea or metanetx+rhea), or all (default: metanetx; see --list-releases)",
     )
     parser.add_argument(
         "-o",

rrparser/Downloads.py

@@ -87,7 +87,9 @@
     return ValueError(f"{message}\nAvailable releases and datasets:\n{choices}")
 
 
-def fetch_dataset(rules_dir, release=DEFAULT_RELEASE, dataset="metanetx"):
+def fetch_dataset(
+    rules_dir, release=DEFAULT_RELEASE, dataset="metanetx", available=None
+):
     """Reuse cached rules; resolve latest only on the first download."""
     if release == "rr02":
         if dataset != "metanetx":
@@ -101,17 +103,17 @@
     if not fullmatch(r"[a-z0-9_-]+", dataset):
         raise ValueError("Invalid RetroRules dataset name")
     cache = Path(rules_dir)
     selection = cache / (f"retrorules-latest-{dataset}.txt")
     if release == "latest" and selection.is_file():
         saved = selection.read_text().strip()
         if fullmatch(r"\d+\.\d+(?:\.\d+)?", saved):
             cached = cache / (f"retrorules-{saved}-{dataset}.csv.gz")
             if cached.is_file():
                 return str(cached)
     filename = cache / (f"retrorules-{release}-{dataset}.csv.gz")
     if release != "latest" and filename.is_file():
         return str(filename)
-    datasets = list_retrorules()
+    datasets = available if available is not None else list_retrorules()
     if release == "latest":
         release = next(
             (
@@ -127,8 +129,67 @@
             f"Release/dataset unavailable: {release} / {dataset}", datasets
         )
     cache.mkdir(parents=True, exist_ok=True)
     filename = cache / (f"retrorules-{release}-{dataset}.csv.gz")
     _download(url, filename)
     if latest:
         selection.write_text(release)
     return str(filename)
+
+
+def fetch_datasets(rules_dir, release=DEFAULT_RELEASE, dataset="metanetx"):
+    """Select datasets from one release, preserving their individual caches."""
+    names = sorted({name.strip() for name in dataset.replace("+", ",").split(",")})
+    if any(not fullmatch(r"[a-z0-9_-]+", name) for name in names):
+        raise ValueError("Use dataset names separated by commas, or all")
+    if "all" in names and names != ["all"]:
+        raise ValueError("Use all alone, or a list of dataset names")
+    if release == "rr02" and names == ["all"]:
+        names = ["metanetx"]
+    if len(names) == 1 and names != ["all"]:
+        return [fetch_dataset(rules_dir, release, names[0])]
+    if release == "rr02":
+        raise ValueError("RetroRules rr02 is only available for metanetx")
+    if release != "latest" and not fullmatch(r"\d+\.\d+(?:\.\d+)?", release):
+        raise _selection_error(f"Invalid release {release!r}")
+    cache = Path(rules_dir)
+    selection = cache / f"retrorules-selection-{release}-{'-'.join(names)}.txt"
+    if selection.is_file():
+        saved = selection.read_text().splitlines()
+        if len(saved) == 2 and fullmatch(r"\d+\.\d+(?:\.\d+)?", saved[0]):
+            selected = saved[1].split(",")
+            if selected and all(fullmatch(r"[a-z0-9_-]+", name) for name in selected):
+                files = [
+                    cache / f"retrorules-{saved[0]}-{name}.csv.gz" for name in selected
+                ]
+                if (release == "latest" or release == saved[0]) and all(
+                    file.is_file() for file in files
+                ):
+                    return list(map(str, files))
+    if release != "latest" and names != ["all"]:
+        files = [cache / f"retrorules-{release}-{name}.csv.gz" for name in names]
+        if all(file.is_file() for file in files):
+            return list(map(str, files))
+    available = list_retrorules()
+    versions = list(
+        dict.fromkeys(version for version, _ in available if version != "rr02")
+    )
+    version = next(
+        (
+            version
+            for version in versions
+            if (release == "latest" or release == version)
+            and (
+                names == ["all"] or all((version, name) in available for name in names)
+            )
+        ),
+        None,
+    )
+    if version is None:
+        raise _selection_error(
+            f"Release/datasets unavailable: {release} / {dataset}", available
+        )
+    if names == ["all"]:
+        names = [name for candidate, name in available if candidate == version]
+    files = [fetch_dataset(cache, version, name, available) for name in names]
+    selection.write_text(version + "\n" + ",".join(names))
+    return files

rrparser/Parser.py

@@ -10,18 +10,18 @@
 
 from csv import QUOTE_ALL, QUOTE_NONE
 from logging import Logger, getLogger
-from re import fullmatch
+from re import fullmatch, split
 from tarfile import open as open_tar
 
-from pandas import DataFrame, read_csv
+from pandas import DataFrame, concat, read_csv
 
 from .Args import (
     DEFAULT_DIAMETERS,
     DEFAULT_RELEASE,
     DEFAULT_RULES_DIR,
     DEFAULT_RULES_FILE,
 )
-from .Downloads import RR02_FILE, fetch_dataset
+from .Downloads import RR02_FILE, fetch_dataset, fetch_datasets
 
 LOGGER = getLogger(__name__)
 
@@ -48,6 +48,84 @@
         return ec
 
 
+def _read_rules(rules_file, **read_options):
+    if str(rules_file).endswith(".tar.gz"):
+        with open_tar(rules_file, mode="r:gz") as archive:
+            member = next(
+                (
+                    item
+                    for item in archive
+                    if item.name in (RR02_FILE, "./" + RR02_FILE) and item.isfile()
+                ),
+                None,
+            )
+            if member is None:
+                raise ValueError(
+                    f"RetroRules rr02 archive does not contain {RR02_FILE}"
+                )
+            with archive.extractfile(member) as source:
+                return read_csv(source, **read_options)
+    else:
+        return read_csv(rules_file, **read_options)
+
+
+def _merge_templates(frames):
+    """Union shared template annotations and retain the best source score."""
+    templates = concat(frames, ignore_index=True)
+    duplicate = templates.duplicated("TEMPLATE_ID", keep=False)
+    if not duplicate.any():
+        return templates
+    groups = templates[duplicate].groupby("TEMPLATE_ID", sort=False)
+    if (groups["TEMPLATE"].nunique() > 1).any():
+        raise ValueError("Conflicting SMARTS for the same template ID across datasets")
+
+    def union(values):
+        return ";".join(
+            sorted(
+                {
+                    item
+                    for value in values.dropna().astype(str)
+                    for item in split(r"[;,]", value)
+                    if item
+                }
+            )
+        )
+
+    aggregations = {
+        column: "first" for column in templates.columns if column != "TEMPLATE_ID"
+    }
+    for column in ("REACTIONS", "ECS", "DATASETS", "RADII"):
+        if column in aggregations:
+            aggregations[column] = union
+    aggregations["SCORE"] = "max"
+    if "VALID" in aggregations:
+        aggregations["VALID"] = "all"
+    merged = groups.agg(aggregations).reset_index()
+    for column in ("REACTIONS", "ECS"):
+        if column + "_COUNT" in merged.columns:
+            merged[column + "_COUNT"] = merged[column].map(
+                lambda value: len(value.split(";")) if value else 0
+            )
+    if "RADII" in merged.columns:
+        radii = merged["RADII"].map(
+            lambda value: sorted(map(int, value.split(";"))) if value else []
+        )
+        merged["RADII"] = radii.map(lambda values: ";".join(map(str, values)))
+        for column, position in (("RADIUS_MIN", 0), ("RADIUS_MAX", -1)):
+            if column in merged.columns:
+                merged[column] = radii.map(
+                    lambda values, position=position: (
+                        values[position] if values else None
+                    )
+                )
+    result = concat([templates[~duplicate], merged], ignore_index=True)
+    return (
+        result.set_index("TEMPLATE_ID")
+        .loc[templates["TEMPLATE_ID"].drop_duplicates()]
+        .reset_index()
+    )
+
+
 def parse_rules(
     outfile: str | None,
     rules_file: str = DEFAULT_RULES_FILE,
@@ -94,7 +172,7 @@
     release: str
         RetroRules version (default 3.1.0), rr02, or latest.
     dataset: str
-        Dataset discovered on retrorules.org (default metanetx).
+        Dataset name, comma/plus-separated names, or all (default metanetx).
     radii: str
         Comma-separated radii 0–10, as an alternative to diameters.
 
@@ -116,36 +194,20 @@
         rule_type, diameters, input_format, output_format
     )
 
-    # If 'rules_file' is set to RetroRules, then fetch RetroRules on Internet
+    rules_files = [rules_file]
     if rules_file == "retrorules":
-        rules_file = fetch_retrorules(rules_dir, logger, release, dataset)
+        rules_files = fetch_datasets(rules_dir, release, dataset)
         sep = ","
 
     logger.debug("Reading values...")
     # Read 'csv' as 'tsv' by specying separator
     read_options = {
         "sep": sep,
-        "dtype": {"RADII": str},
+        "dtype": {"RADII": str, "ECS": str, "EC number": str},
         "float_precision": "round_trip",
     }
-    if str(rules_file).endswith(".tar.gz"):
-        with open_tar(rules_file, mode="r:gz") as archive:
-            member = next(
-                (
-                    item
-                    for item in archive
-                    if item.name in (RR02_FILE, "./" + RR02_FILE) and item.isfile()
-                ),
-                None,
-            )
-            if member is None:
-                raise ValueError(
-                    f"RetroRules rr02 archive does not contain {RR02_FILE}"
-                )
-            with archive.extractfile(member) as source:
-                rf = read_csv(source, **read_options)
-    else:
-        rf = read_csv(rules_file, **read_options)
+    frames = [_read_rules(filename, **read_options) for filename in rules_files]
+    rf = _merge_templates(frames) if len(frames) > 1 else frames[0]
 
     # Filter rules according to 'rule_type' and 'diameters'
     templates = "TEMPLATE_ID" in rf.columns
@@ -158,7 +220,7 @@
                 "SCORE": "Score",
             }
         )
     results = filter_(rf, rule_type, diameters, ecx, ec, logger=logger)
 
     # If 'rules_scores_file' is set, then read it and merge with 'results'
     if scores_file:
@@ -249,7 +311,7 @@
     ecx = ecx if ecx is not None else []
     ec = ec if ec is not None else []
     ec_filter = ecx if ecx else ec
     filter = "in" if ec else "not in"
     logger.info("Filtering rules:")
     logger.info(f" -> EC numbers ({filter}): {ec_filter}")
     logger.info(f" -> Diameters: {diameters}")
@@ -303,7 +365,7 @@
 
 
 def filter_rules_by_ec(
     df: DataFrame, ec_filter: list[str], logger: Logger = LOGGER
 ) -> list[int]:
     """
     Filter out rules that contain at least one EC number that starts with one in 'ecx'.
@@ -312,7 +374,7 @@
     ----------
     df: DataFrame
         Pandas dataframe
     ec: List[str]
         List of EC numbers to filter
     logger : Logger
         The logger object.
@@ -321,7 +383,7 @@
     ec_numbers = df["EC number"].fillna("").str.split(";")
 
     # Find indices of rows where any EC number starts with one in 'ec_filter'
     def filter_ec_numbers(ec_list, ec_filter):
         return any(any(ec.startswith(ec_f) for ec_f in ec_filter) for ec in ec_list)
 
     # Apply filtering function and get indices
@@ -350,9 +412,9 @@
 
 def fetch_retrorules(
     rules_dir: str = DEFAULT_RULES_DIR,
     logger: Logger = LOGGER,
     release: str = DEFAULT_RELEASE,
     dataset: str = "metanetx",
 ) -> str:
     """
     Fetch RetroRules over Internet if not already on disk.