From 2cc1128d085aa04276e7c8ba49b5a46aabc7a2d2 Mon Sep 17 00:00:00 2001
From: dev-belly <dev_belly@outlook.com>
Date: Fri, 23 Oct 2026 22:09:57 +0800
Subject: [PATCH] fix: retain conflicting audit evidence and reject nonfinite
 amounts

---
 src/data_cleaning.py        | 31 ++++++++---------------
 tests/test_data_cleaning.py | 49 +++++++++++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+), 21 deletions(-)

diff --git a/src/data_cleaning.py b/src/data_cleaning.py
index 0567a60..68dbeb8 100644
--- a/src/data_cleaning.py
+++ b/src/data_cleaning.py
@@ -299,7 +299,9 @@ def coerce_numeric(df: pd.DataFrame, columns: tuple[str, ...]) -> pd.DataFrame:
 
         series = df[column]
         if pd.api.types.is_numeric_dtype(series):
-            df[column] = pd.to_numeric(series, errors="coerce")
+            df[column] = pd.to_numeric(series, errors="coerce").replace(
+                [float("inf"), float("-inf")], float("nan")
+            )
             continue
 
         text = series.astype("string")
@@ -316,27 +318,12 @@ def coerce_numeric(df: pd.DataFrame, columns: tuple[str, ...]) -> pd.DataFrame:
 def remove_duplicate_rows(df: pd.DataFrame) -> tuple[pd.DataFrame, int]:
     """Drop exact duplicate rows, keeping the first occurrence.
 
-    Duplicates are compared on the full business key rather than on
-    ``transaction_id`` alone, because a duplicated ``transaction_id`` with a
-    different amount is a *different* problem (a broken key) from a fully
-    duplicated extract row.
+    Compare every supplied column. Repeated IDs with different currencies,
+    approvals or other source evidence are conflicting records, not exact
+    duplicate extracts, and must remain visible for downstream validation.
     """
-    business_key = [
-        column
-        for column in (
-            "transaction_id",
-            "journal_id",
-            "transaction_date",
-            "account_code",
-            "debit_amount",
-            "credit_amount",
-            "vendor_id",
-            "description",
-        )
-        if column in df.columns
-    ]
     before = len(df)
-    deduped = df.drop_duplicates(subset=business_key, keep="first").reset_index(drop=True)
+    deduped = df.drop_duplicates(keep="first").reset_index(drop=True)
     return deduped, before - len(deduped)
 
 
@@ -478,7 +465,9 @@ def clean_transactions(
     # ---- 3. Derived columns used downstream ---------------------------------
     # The functional currency is CNY; foreign documents are translated so that
     # every downstream aggregate is apples-to-apples.
-    df["amount_cny"] = (df["amount_original"] * df["fx_rate"]).round(2)
+    df["amount_cny"] = (df["amount_original"] * df["fx_rate"]).round(2).replace(
+        [float("inf"), float("-inf")], float("nan")
+    )
     # Where the translation is missing, fall back to the functional amount.
     df["amount_cny"] = df["amount_cny"].fillna(df["amount"]).round(2)
 
diff --git a/tests/test_data_cleaning.py b/tests/test_data_cleaning.py
index 7b96d60..190ea5a 100644
--- a/tests/test_data_cleaning.py
+++ b/tests/test_data_cleaning.py
@@ -237,6 +237,13 @@ def test_missing_column_is_ignored(self) -> None:
 
         assert list(result.columns) == ["other"]
 
+    def test_nonfinite_numeric_values_become_missing(self) -> None:
+        frame = pd.DataFrame({"amount": [float("inf"), float("-inf"), 100.0]})
+        result = coerce_numeric(frame, ("amount",))
+
+        assert result["amount"].isna().tolist() == [True, True, False]
+        assert result["amount"].iloc[2] == 100.0
+
 
 # --------------------------------------------------------------------------- #
 # Duplicates
@@ -266,6 +273,28 @@ def test_vouchers_sharing_an_invoice_are_not_duplicates(self) -> None:
         assert removed == 0
         assert len(frame) == 2
 
+    @pytest.mark.parametrize(
+        ("field", "changed"),
+        [
+            ("currency", "USD"),
+            ("fx_rate", 7.0),
+            ("invoice_id", "INV-DIFFERENT"),
+            ("approved_by", "E9999"),
+            ("credit_account_code", "1002"),
+            ("posting_date", WEEKDAY + pd.Timedelta(days=1)),
+        ],
+    )
+    def test_conflicting_source_evidence_is_not_silently_deduplicated(
+        self, field: str, changed: object
+    ) -> None:
+        original = dict(BASE_TRANSACTION)
+        conflicting = {**original, field: changed}
+        frame, removed = remove_duplicate_rows(pd.DataFrame([original, conflicting, original]))
+
+        assert removed == 1
+        assert len(frame) == 2
+        assert frame[field].tolist() == [original[field], changed]
+
 
 # --------------------------------------------------------------------------- #
 # Account names
@@ -521,6 +550,26 @@ def test_cleaning_is_idempotent(self) -> None:
         assert len(twice) == len(once)
         assert report.duplicates_removed == 0
 
+    def test_infinite_functional_amount_is_removed_and_reported(self) -> None:
+        transactions = _frame([{}, {"amount": float("inf")}])
+        vendors = pd.DataFrame({"vendor_id": ["V0001"]})
+        clean, report = clean_transactions(transactions, vendors)
+
+        assert len(clean) == 1
+        assert report.invalid_amount_rows == 1
+        assert clean["transaction_id"].tolist() == ["TX0001"]
+
+    @pytest.mark.parametrize(
+        ("original", "rate"),
+        [(float("inf"), 1.0), (100.0, float("inf")), (1e308, 7.0)],
+    )
+    def test_nonfinite_translation_uses_finite_functional_amount(self, original, rate) -> None:
+        transactions = _frame([{"amount_original": original, "fx_rate": rate}])
+        vendors = pd.DataFrame({"vendor_id": ["V0001"]})
+        clean, _ = clean_transactions(transactions, vendors)
+
+        assert clean["amount_cny"].tolist() == [BASE_TRANSACTION["amount"]]
+
 
 def _masters(vendor_rows: list[dict] | None = None, employee_rows: list[dict] | None = None):
     vendor = {