diff --git a/README.md b/README.md index ff333c2..fb39f7d 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![CI](https://github.com/dev-belly/AuditLens/actions/workflows/ci.yml/badge.svg)](https://github.com/dev-belly/AuditLens/actions/workflows/ci.yml) [![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/downloads/) [![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) -[![Tests: 508](https://img.shields.io/badge/tests-508%20passing-brightgreen.svg)](#testing) +[![Tests: 519](https://img.shields.io/badge/tests-519%20passing-brightgreen.svg)](#testing) **Financial anomaly detection and audit analytics over a 30,000-voucher general ledger.** @@ -58,7 +58,7 @@ Three design commitments drive everything else: | Anomalies caught by neither | **15** | | Isolation Forest | ROC-AUC **0.816**, precision 0.291, recall 0.286 | | Benford first-digit MAD | **0.00218** — close conformity | -| Test suite | **508 tests**, including dashboard render, warehouse and review-plan checks | +| Test suite | **519 tests**, including dashboard render, warehouse and review-plan checks | ### The most important number here is 98.2%, and it is a warning @@ -578,7 +578,7 @@ AuditLens/ │ ├── components.py # KPI cards, risk badges, charts, tables │ └── views/ # the six pages ├── sql/audit_queries.sql # 15 named business queries -├── tests/ # 508 tests, incl. cross-process reproducibility +├── tests/ # 519 tests, incl. cross-process reproducibility ├── docs/ │ ├── architecture.md # design decisions, data contracts, what is deliberately excluded │ ├── methodology.md # every threshold, every weight, every mistake @@ -592,7 +592,7 @@ AuditLens/ │ ├── notebook_content.py # what the three notebooks contain, kept apart from the builder │ ├── capture_screenshots.py # headless captures of all six dashboard pages, via DevTools Protocol │ └── ci_summary.py # headline figures for the CI job summary -├── .github/workflows/ci.yml # pipeline + 508 tests; Ubuntu 3.11/3.12 and Windows 3.12 +├── .github/workflows/ci.yml # pipeline + 519 tests; Ubuntu 3.11/3.12 and Windows 3.12 └── Makefile ``` @@ -765,7 +765,7 @@ builder, so two consecutive builds are byte-identical. ```bash pip install -r requirements.txt python src/run_pipeline.py # ~30 seconds, deterministic -python -m pytest tests/ -q # 508 tests +python -m pytest tests/ -q # 519 tests ``` Two consecutive runs produce byte-identical reports under `outputs/reports/`. This is @@ -773,11 +773,11 @@ enforced by `tests/test_reproducibility.py`, not assumed. ## Testing -508 tests. `make test` runs the lot; `make test-fast` skips the dashboard render suite. +519 tests. `make test` runs the lot; `make test-fast` skips the dashboard render suite. | File | Tests | What it pins down | |---|---|---| -| `test_data_cleaning.py` | 93 | Repairs, full-record master integrity, source-preserving conflicts, and staged cleaning publication with failure rollback | +| `test_data_cleaning.py` | 104 | Repairs, full-record master integrity, conflicting voucher evidence, nonfinite/overflowing amounts, and staged cleaning publication with failure rollback | | `test_audit_rules.py` | 38 | One class per procedure, plus mixed-size split-cluster regressions and the text-encoded-label fallback in `evaluate()` | | `test_benford.py` | 56 | Expected frequencies, MAD bands, χ², the per-bucket z-score, and the schema stability of the insufficient-data branch | | `test_risk_scoring.py` | 55 | Component scores, the noisy-OR combination, band boundaries, `risk_reasons` wording | diff --git a/docs/methodology.md b/docs/methodology.md index 8f2f05a..50d315b 100644 --- a/docs/methodology.md +++ b/docs/methodology.md @@ -496,6 +496,14 @@ one reason — a score with no explanation is treated as a bug. ## 7. Audit review selection +Cleaning removes a repeated normalized extract row only when every supplied +column agrees. Rows sharing the former eight-column subset but differing in +currency, FX, invoice, approval, credit account or posting evidence remain +visible to downstream key validation. Numeric infinities become missing values; +an overflowing FX product also becomes missing before the documented functional +amount fallback. These are input-quality rules, not evidence of a confirmed +accounting exception. Raw extracts are retained separately. + The default workpaper budgets 300 vouchers. Selection is made only from scored transactions and rule alerts; `anomaly_label` and `anomaly_type` are absent from every decision field and from the auditor-facing CSV. diff --git a/src/data_cleaning.py b/src/data_cleaning.py index 0567a60..68dbeb8 100644 --- a/src/data_cleaning.py +++ b/src/data_cleaning.py @@ -299,7 +299,9 @@ def coerce_numeric(df: pd.DataFrame, columns: tuple[str, ...]) -> pd.DataFrame: series = df[column] if pd.api.types.is_numeric_dtype(series): - df[column] = pd.to_numeric(series, errors="coerce") + df[column] = pd.to_numeric(series, errors="coerce").replace( + [float("inf"), float("-inf")], float("nan") + ) continue text = series.astype("string") @@ -316,27 +318,12 @@ def coerce_numeric(df: pd.DataFrame, columns: tuple[str, ...]) -> pd.DataFrame: def remove_duplicate_rows(df: pd.DataFrame) -> tuple[pd.DataFrame, int]: """Drop exact duplicate rows, keeping the first occurrence. - Duplicates are compared on the full business key rather than on - ``transaction_id`` alone, because a duplicated ``transaction_id`` with a - different amount is a *different* problem (a broken key) from a fully - duplicated extract row. + Compare every supplied column. Repeated IDs with different currencies, + approvals or other source evidence are conflicting records, not exact + duplicate extracts, and must remain visible for downstream validation. """ - business_key = [ - column - for column in ( - "transaction_id", - "journal_id", - "transaction_date", - "account_code", - "debit_amount", - "credit_amount", - "vendor_id", - "description", - ) - if column in df.columns - ] before = len(df) - deduped = df.drop_duplicates(subset=business_key, keep="first").reset_index(drop=True) + deduped = df.drop_duplicates(keep="first").reset_index(drop=True) return deduped, before - len(deduped) @@ -478,7 +465,9 @@ def clean_transactions( # ---- 3. Derived columns used downstream --------------------------------- # The functional currency is CNY; foreign documents are translated so that # every downstream aggregate is apples-to-apples. - df["amount_cny"] = (df["amount_original"] * df["fx_rate"]).round(2) + df["amount_cny"] = (df["amount_original"] * df["fx_rate"]).round(2).replace( + [float("inf"), float("-inf")], float("nan") + ) # Where the translation is missing, fall back to the functional amount. df["amount_cny"] = df["amount_cny"].fillna(df["amount"]).round(2) diff --git a/tests/test_data_cleaning.py b/tests/test_data_cleaning.py index 7b96d60..190ea5a 100644 --- a/tests/test_data_cleaning.py +++ b/tests/test_data_cleaning.py @@ -237,6 +237,13 @@ def test_missing_column_is_ignored(self) -> None: assert list(result.columns) == ["other"] + def test_nonfinite_numeric_values_become_missing(self) -> None: + frame = pd.DataFrame({"amount": [float("inf"), float("-inf"), 100.0]}) + result = coerce_numeric(frame, ("amount",)) + + assert result["amount"].isna().tolist() == [True, True, False] + assert result["amount"].iloc[2] == 100.0 + # --------------------------------------------------------------------------- # # Duplicates @@ -266,6 +273,28 @@ def test_vouchers_sharing_an_invoice_are_not_duplicates(self) -> None: assert removed == 0 assert len(frame) == 2 + @pytest.mark.parametrize( + ("field", "changed"), + [ + ("currency", "USD"), + ("fx_rate", 7.0), + ("invoice_id", "INV-DIFFERENT"), + ("approved_by", "E9999"), + ("credit_account_code", "1002"), + ("posting_date", WEEKDAY + pd.Timedelta(days=1)), + ], + ) + def test_conflicting_source_evidence_is_not_silently_deduplicated( + self, field: str, changed: object + ) -> None: + original = dict(BASE_TRANSACTION) + conflicting = {**original, field: changed} + frame, removed = remove_duplicate_rows(pd.DataFrame([original, conflicting, original])) + + assert removed == 1 + assert len(frame) == 2 + assert frame[field].tolist() == [original[field], changed] + # --------------------------------------------------------------------------- # # Account names @@ -521,6 +550,26 @@ def test_cleaning_is_idempotent(self) -> None: assert len(twice) == len(once) assert report.duplicates_removed == 0 + def test_infinite_functional_amount_is_removed_and_reported(self) -> None: + transactions = _frame([{}, {"amount": float("inf")}]) + vendors = pd.DataFrame({"vendor_id": ["V0001"]}) + clean, report = clean_transactions(transactions, vendors) + + assert len(clean) == 1 + assert report.invalid_amount_rows == 1 + assert clean["transaction_id"].tolist() == ["TX0001"] + + @pytest.mark.parametrize( + ("original", "rate"), + [(float("inf"), 1.0), (100.0, float("inf")), (1e308, 7.0)], + ) + def test_nonfinite_translation_uses_finite_functional_amount(self, original, rate) -> None: + transactions = _frame([{"amount_original": original, "fx_rate": rate}]) + vendors = pd.DataFrame({"vendor_id": ["V0001"]}) + clean, _ = clean_transactions(transactions, vendors) + + assert clean["amount_cny"].tolist() == [BASE_TRANSACTION["amount"]] + def _masters(vendor_rows: list[dict] | None = None, employee_rows: list[dict] | None = None): vendor = {