From 2c60bcf2332b00f60785caffe392d6b367bdba57 Mon Sep 17 00:00:00 2001 From: sharmis-tp Date: Wed, 28 Oct 2026 09:00:57 +0000 Subject: [PATCH] feat(core): accept X_df and futr_exog_list in detect_anomalies --- README.md | 4 +- docs/exogenous-variables.md | 51 ++++++++++--- docs/forecasting-parameters.md | 5 ++ foundationforecast/core/exog/__init__.py | 2 + foundationforecast/core/exog/covariates.py | 40 ++++++++++ foundationforecast/core/forecaster.py | 22 +++++- foundationforecast/core/multi_model.py | 22 +++++- tests/core/test_forecaster.py | 86 ++++++++++++++++++++++ tests/helpers.py | 35 ++++++++- tests/models/test_models.py | 35 +++++++++ tests/test_foundation_forecast.py | 44 ++++++++++- 11 files changed, 328 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 29f48aa..84e8db5 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ Your DataFrame needs three columns: `unique_id`, `ds`, and `y`. For best results ## Highlights -- πŸ“ˆ **Exogenous variables (covariates).** Pass known-future features with **`X_df`** on `forecast()` (and exog columns in `df` for `cross_validation()`). Native support on Chronos-2, TimesFM-3, TimeGPT, T0, TimeH2O, TiRex-2, and Timestral; see [Exogenous variables](https://github.com/TimeCopilot/foundationforecast/blob/main/docs/exogenous-variables.md) and the [example notebook](https://github.com/TimeCopilot/foundationforecast/blob/main/docs/examples/exogenous-variables.ipynb). +- πŸ“ˆ **Exogenous variables (covariates).** Pass known-future features with **`X_df`** on `forecast()` and `detect_anomalies()` (or exog columns in `df` for `cross_validation()` and `detect_anomalies()`). Native support on Chronos-2, TimesFM-3, TimeGPT, T0, TimeH2O, TiRex-2, and Timestral; see [Exogenous variables](https://github.com/TimeCopilot/foundationforecast/blob/main/docs/exogenous-variables.md) and the [example notebook](https://github.com/TimeCopilot/foundationforecast/blob/main/docs/examples/exogenous-variables.ipynb). - πŸ’Έ **Full GIFT-Eval replication for $37.** 16 models, 97 dataset configurations each, in about 34 GPU-hours, every model with an official submission within 2% of its leaderboard MASE/CRPS. TimesFM 3 is the most accurate, for under $8. Read the [writeup](https://github.com/TimeCopilot/foundationforecast/blob/main/experiments/gift-eval/README.md). - 🎯 **Benchmark replication in CI.** Wrappers are regression-tested against official leaderboard submissions: [`experiments/gift-eval`](https://github.com/TimeCopilot/foundationforecast/tree/main/experiments/gift-eval) (GIFT-Eval MASE/CRPS on Modal GPU) and [`experiments/fev-bench`](https://github.com/TimeCopilot/foundationforecast/tree/main/experiments/fev-bench) (fev-bench `test_error` on known-dynamic tasks). [CI](https://github.com/TimeCopilot/foundationforecast/actions/workflows/ci.yaml) re-runs these checks on every change. - πŸš€ **GPU-native**. Automatically runs on GPU when available, without model-specific device configuration. @@ -75,7 +75,7 @@ Your DataFrame needs three columns: `unique_id`, `ds`, and `y`. For best results ## Supported models -Every model supports **forecast**, **cross-validation**, and **anomaly detection** through the same API. **Intervals** means prediction intervals via `level` or quantile forecasts. **Exogenous** marks [native known-future exog](https://github.com/TimeCopilot/foundationforecast/blob/main/docs/exogenous-variables.md) (`X_df` on `forecast()`, exog columns in `df` for `cross_validation()`) for that wrapper when using a supported checkpoint; see ΒΆ. **Finetuning** marks models that can adapt to your data at inference time. **License** is the [weight/checkpoint license](https://huggingface.co/models) on the default Hugging Face repo (or provider terms for hosted APIs). See the note below for production use. +Every model supports **forecast**, **cross-validation**, and **anomaly detection** through the same API. **Intervals** means prediction intervals via `level` or quantile forecasts. **Exogenous** marks [native known-future exog](https://github.com/TimeCopilot/foundationforecast/blob/main/docs/exogenous-variables.md) (`X_df` on `forecast()` and `detect_anomalies()`, exog columns in `df` for `cross_validation()` and `detect_anomalies()`) for that wrapper when using a supported checkpoint; see ΒΆ. **Finetuning** marks models that can adapt to your data at inference time. **License** is the [weight/checkpoint license](https://huggingface.co/models) on the default Hugging Face repo (or provider terms for hosted APIs). See the note below for production use. Pass any Hugging Face `repo_id` (or local checkpoint path) supported by the underlying model class. diff --git a/docs/exogenous-variables.md b/docs/exogenous-variables.md index 6de91c6..c0e9998 100644 --- a/docs/exogenous-variables.md +++ b/docs/exogenous-variables.md @@ -6,27 +6,33 @@ horizon. Iteration 1 supports **future-known dynamic** covariates only. ## Data API - **`df`**: `unique_id`, `ds`, `y`, plus exog columns. For **history**, exog must cover - the observed period. For **`cross_validation()`**, also include exog (and `y`) through - the end of each fold so horizon covariates are read from `df` (no separate horizon frame). -- **`X_df`**: `unique_id`, `ds`, exog columns for the **next `h` steps** (no `y`). Used by - **`forecast()`** only. + the observed period. For **`cross_validation()`** and **`detect_anomalies()`**, also + include exog (and `y`) through the end of each fold so horizon covariates are read + from `df`. +- **`X_df`**: `unique_id`, `ds`, exog columns (no `y`). + - On **`forecast()`**: values for the **next `h` steps**. + - On **`detect_anomalies()`**: optional alternative to exog columns in `df`, with a + row for **every** `(unique_id, ds)` in `df` (extra rows are ignored). See + [Anomaly detection](#anomaly-detection). - Alias: **`futr_df`** = same as **`X_df`** on **`forecast()`**. - **`futr_exog_list`** (optional): exog column names only (not `unique_id`, `ds`, or `y`). On **`forecast()`**, infer from `X_df` if omitted. On **`cross_validation()`**, infer - from non-target columns in `df` if omitted. + from non-target columns in `df` if omitted. On **`detect_anomalies()`**, infer from + `X_df` when passed, otherwise from non-target columns in `df`. ## Model constructor: `exog_strategy` | Value | Behavior when horizon exog is provided | |-------|----------------------------------------| | `"auto"` (default) | Use **native** exog when the checkpoint supports it; otherwise **raise** | -| `False` | Ignore `X_df` / `futr_exog_list`; univariate forecast (also in CV) | +| `False` | Ignore `X_df` / `futr_exog_list`; univariate forecast (also in CV and anomaly detection) | When a model does not support native exog and you pass `X_df`, the library logs a warning and raises `ValueError`. **`FoundationForecast`** checks every member model -before calling `forecast()` or `cross_validation()` so a `fallback_model` is not used -instead of that error. For mixed runs, set `exog_strategy=False` on models that do not -support native exog so they ignore horizon exog. +before calling `forecast()`, `cross_validation()`, or `detect_anomalies()` so a +`fallback_model` is not used instead of that error. For mixed runs, set +`exog_strategy=False` on models that do not support native exog so they ignore horizon +exog. ## Native exog by checkpoint @@ -55,6 +61,21 @@ Local checkpoint paths follow the same rules as Hugging Face `repo_id` strings p wrapper. For **`cross_validation()`**, pass exog in **`df`** through the holdout window; the same native rules apply per checkpoint. +## Anomaly detection + +`detect_anomalies()` runs in-sample cross-validation and flags values whose residual +falls outside the `level` interval, so covariate-aware models should see the same +covariates there as in `forecast()`. Every window is scored inside `df`, so covariates +must cover the whole history. Either: + +- keep exog columns in **`df`**, exactly as for `cross_validation()`, or +- pass them separately in **`X_df`** with a row for every `(unique_id, ds)` in `df`. + `X_df` is merged into `df` before the windows are built; a covariate cannot appear in + both. + +In mixed `FoundationForecast` runs, set `exog_strategy=False` on models without native +exog so they score anomalies univariately. + ## Example ```python @@ -84,6 +105,18 @@ fcst_panel = ff.forecast(df=df, h=24, X_df=X_df) # Cross-validation: exog columns in `df` through the holdout window (no X_df) cv = model.cross_validation(df=df, h=24) + +# Anomaly detection: exog columns in `df`, as in cross-validation +anomalies = ff.detect_anomalies(df=df, h=24, n_windows=2, level=99) + +# ...or covariates kept apart in an X_df covering every (unique_id, ds) in df +exog_cols = ["ex_1", "ex_2"] +anomalies = model.detect_anomalies( + df=df.drop(columns=exog_cols), + h=24, + n_windows=2, + X_df=df[["unique_id", "ds", *exog_cols]], +) ``` See `docs/examples/exogenous-variables.ipynb` for more models. diff --git a/docs/forecasting-parameters.md b/docs/forecasting-parameters.md index 5a87924..093c603 100644 --- a/docs/forecasting-parameters.md +++ b/docs/forecasting-parameters.md @@ -52,6 +52,11 @@ When calling `detect_anomalies()` without `h`: anomalies_df = ff.detect_anomalies(df=df, freq="MS", level=99) ``` +`detect_anomalies()` also accepts known-future covariates, like `forecast()` and +`cross_validation()`: keep exog columns in `df`, or pass them in `X_df` (with a row for +every `(unique_id, ds)` in `df`) and optionally `futr_exog_list`. See +[Exogenous variables](exogenous-variables.md#anomaly-detection). + ## Choosing sensible values * **`freq`** must match your data's timestamp spacing. Irregular or gapped series may fail inference. diff --git a/foundationforecast/core/exog/__init__.py b/foundationforecast/core/exog/__init__.py index e7d3f83..e2e670d 100644 --- a/foundationforecast/core/exog/__init__.py +++ b/foundationforecast/core/exog/__init__.py @@ -6,6 +6,7 @@ ExogStrategyConfig, ExogStrategyName, infer_futr_exog_columns, + merge_insample_exog, normalize_exog_strategy, resolve_futr_exog_list, resolve_horizon_exog_df, @@ -29,6 +30,7 @@ "dispatch_futr_exog_forecast", "futr_exog_unsupported_message", "infer_futr_exog_columns", + "merge_insample_exog", "normalize_exog_strategy", "prepare_futr_exog_context", "resolve_futr_exog_list", diff --git a/foundationforecast/core/exog/covariates.py b/foundationforecast/core/exog/covariates.py index c8beea7..36c2066 100644 --- a/foundationforecast/core/exog/covariates.py +++ b/foundationforecast/core/exog/covariates.py @@ -95,6 +95,46 @@ def resolve_exog_columns_from_df( return [c for c in df.columns if c not in META_COLS] +def merge_insample_exog( + df: pd.DataFrame, + X_df: pd.DataFrame, + futr_exog_list: list[str] | None, +) -> tuple[pd.DataFrame, list[str]]: + """Attach exog from ``X_df`` to every ``(unique_id, ds)`` row of ``df``. + + Used by ``detect_anomalies()``, which scores in-sample cross-validation + windows: covariates must cover the history and every scored window, so + ``X_df`` needs a row for each timestamp in ``df`` (extra rows are ignored). + """ + if ID_COL not in X_df.columns or TIME_COL not in X_df.columns: + raise ValueError(f"X_df must include columns '{ID_COL}' and '{TIME_COL}'.") + cols = resolve_futr_exog_list(X_df, futr_exog_list) + in_both = [c for c in cols if c in df.columns] + if in_both: + raise ValueError( + f"Exogenous columns {in_both} are present in both df and X_df. " + "Pass each covariate in only one of them." + ) + if X_df.duplicated(subset=[ID_COL, TIME_COL]).any(): + raise ValueError( + f"X_df must not contain duplicate ({ID_COL}, {TIME_COL}) rows." + ) + exog = X_df[[ID_COL, TIME_COL, *cols]] + if not pd.api.types.is_datetime64_any_dtype(exog[TIME_COL]): + exog = exog.assign(**{TIME_COL: pd.to_datetime(exog[TIME_COL])}) + indicator = "__exog_merge__" + merged = df.merge(exog, on=[ID_COL, TIME_COL], how="left", indicator=indicator) + missing = merged[indicator] == "left_only" + if missing.any(): + first = merged.loc[missing, [ID_COL, TIME_COL]].iloc[0] + raise ValueError( + f"X_df is missing {int(missing.sum())} ({ID_COL}, {TIME_COL}) rows " + f"present in df, e.g. ({first[ID_COL]!r}, {first[TIME_COL]}). " + "For detect_anomalies, X_df must cover every timestamp in df." + ) + return merged.drop(columns=indicator), cols + + def resolve_futr_exog_list( horizon_df: pd.DataFrame, futr_exog_list: list[str] | None, diff --git a/foundationforecast/core/forecaster.py b/foundationforecast/core/forecaster.py index f1d06aa..129e968 100644 --- a/foundationforecast/core/forecaster.py +++ b/foundationforecast/core/forecaster.py @@ -473,6 +473,9 @@ def detect_anomalies( freq: str | None = None, n_windows: int | None = None, level: int | float = 99, + X_df: pd.DataFrame | None = None, + *, + futr_exog_list: list[str] | None = None, ) -> pd.DataFrame: """Flag observations that fall outside the model's prediction interval. @@ -481,13 +484,21 @@ def detect_anomalies( the per-series residual standard deviation. Args: - df: Historical data with columns `unique_id`, `ds`, and `y`. + df: Historical data with columns `unique_id`, `ds`, and `y`. As in + `cross_validation`, any exogenous columns in `df` are used as + known-future covariates for each window. h: Forecast horizon for each window. Defaults to the seasonality implied by `freq`. freq: Pandas frequency of `ds`. Inferred from `df` when omitted. n_windows: Number of windows. Defaults to as many as the shortest series allows. level: Interval level in percent used as the anomaly threshold. + X_df: Exogenous values with `unique_id`, `ds`, and the exogenous + columns (no `y`), kept apart from `df`. Must have a row for + every `(unique_id, ds)` in `df`, since every window is scored + in-sample. See the Exogenous variables guide. + futr_exog_list: Exogenous column names to use. Inferred from `X_df` + (or from the non-target columns of `df`) when omitted. Returns: One row per series and checked timestamp, with `unique_id`, `ds`, @@ -496,8 +507,16 @@ def detect_anomalies( """ from scipy import stats + from .exog.covariates import exog_strategy_disabled, merge_insample_exog + freq = self._maybe_infer_freq(df, freq) df = maybe_convert_col_to_datetime(df, "ds") + exog_kwargs: dict = {} + if not exog_strategy_disabled(self.exog_strategy): + if X_df is not None: + df, futr_exog_list = merge_insample_exog(df, X_df, futr_exog_list) + if futr_exog_list is not None: + exog_kwargs["futr_exog_list"] = futr_exog_list if h is None: h = self._maybe_get_seasonality(freq) min_series_length = counts_by_id(df, "unique_id")["counts"].min() @@ -520,6 +539,7 @@ def detect_anomalies( freq=freq, n_windows=_n_windows, step_size=h, + **exog_kwargs, ) cv_results["residuals"] = cv_results["y"] - cv_results[self.alias] residual_stats = grouped_std_by_id( diff --git a/foundationforecast/core/multi_model.py b/foundationforecast/core/multi_model.py index ab898f8..2b87ed0 100644 --- a/foundationforecast/core/multi_model.py +++ b/foundationforecast/core/multi_model.py @@ -69,7 +69,9 @@ def _model_uses_horizon_exog( kwargs.get("futr_df"), # type: ignore[arg-type] ) return horizon_df is not None - if attr == "cross_validation": + if attr == "detect_anomalies" and kwargs.get("X_df") is not None: + return True + if attr in ("cross_validation", "detect_anomalies"): futr_exog_list = kwargs.get("futr_exog_list") cols = resolve_exog_columns_from_df( df, @@ -294,20 +296,32 @@ def detect_anomalies( freq: str | None = None, n_windows: int | None = None, level: int | float = 99, + X_df: pd.DataFrame | None = None, + *, + futr_exog_list: list[str] | None = None, ) -> pd.DataFrame: """Flag anomalies with every model. Each model backtests the history and marks values outside its `level` - prediction interval. + prediction interval. Exogenous variables are passed to every model; + set `exog_strategy=False` on models without native exog support so + they ignore them. Args: - df: Historical data with columns `unique_id`, `ds`, and `y`. + df: Historical data with columns `unique_id`, `ds`, and `y`. As in + `cross_validation`, any exogenous columns in `df` are used as + known-future covariates for each window. h: Forecast horizon for each window. Defaults to the seasonality implied by `freq`. freq: Pandas frequency of `ds`. Inferred from `df` when omitted. n_windows: Number of windows. Defaults to as many as the shortest series allows. level: Interval level in percent used as the anomaly threshold. + X_df: Exogenous values with `unique_id`, `ds`, and the exogenous + columns (no `y`), kept apart from `df`. Must have a row for + every `(unique_id, ds)` in `df`. + futr_exog_list: Exogenous column names to use. Inferred from `X_df` + (or from the non-target columns of `df`) when omitted. Returns: One row per series and checked timestamp, with `unique_id`, `ds`, @@ -323,4 +337,6 @@ def detect_anomalies( level=level, # type: ignore[arg-type] quantiles=None, n_windows=n_windows, + X_df=X_df, + futr_exog_list=futr_exog_list, ) diff --git a/tests/core/test_forecaster.py b/tests/core/test_forecaster.py index 8625c5c..f6bea45 100644 --- a/tests/core/test_forecaster.py +++ b/tests/core/test_forecaster.py @@ -4,7 +4,9 @@ from tests.helpers import ( DummyModel, + ExogSpyModel, SeasonalNaiveModel, + generate_panel_with_futr_exog, generate_series, generate_series_with_anomalies, ) @@ -326,6 +328,90 @@ def test_detect_anomalies_short_series_error(): model.detect_anomalies(df, h=5, freq="D") +def _split_exog(df: pd.DataFrame, cols: list[str]): + return df.drop(columns=cols), df[["unique_id", "ds", *cols]] + + +def test_detect_anomalies_passes_x_df_to_forecast(): + h = 3 + df, _, exog_cols = generate_panel_with_futr_exog(2, freq="D", h=h) + df_y, X_df = _split_exog(df, exog_cols) + model = ExogSpyModel() + result = model.detect_anomalies(df_y, h=h, freq="D", n_windows=2, X_df=X_df) + assert len(model.horizon_dfs) == 2 + seen = pd.concat(model.horizon_dfs, ignore_index=True) + expected = seen[["unique_id", "ds"]].merge(X_df, on=["unique_id", "ds"]) + pd.testing.assert_frame_equal(seen, expected) + assert len(result) == 2 * 2 * h + scored = result.merge(X_df, on=["unique_id", "ds"]) + np.testing.assert_allclose( + scored[model.alias], scored["x1"] + scored["x2"], rtol=1e-12 + ) + + +def test_detect_anomalies_x_df_matches_exog_columns_in_df(): + h = 3 + df, _, exog_cols = generate_panel_with_futr_exog(2, freq="D", h=h) + df_y, X_df = _split_exog(df, exog_cols) + from_df = ExogSpyModel().detect_anomalies(df, h=h, freq="D", n_windows=2) + from_x_df = ExogSpyModel().detect_anomalies( + df_y, h=h, freq="D", n_windows=2, X_df=X_df.sample(frac=1, random_state=0) + ) + pd.testing.assert_frame_equal(from_df, from_x_df) + + +def test_detect_anomalies_futr_exog_list_selects_x_df_columns(): + h = 3 + df, _, exog_cols = generate_panel_with_futr_exog(1, freq="D", h=h) + df_y, X_df = _split_exog(df, exog_cols) + model = ExogSpyModel() + model.detect_anomalies( + df_y, h=h, freq="D", n_windows=1, X_df=X_df, futr_exog_list=["x1"] + ) + assert list(model.horizon_dfs[0].columns) == ["unique_id", "ds", "x1"] + + +def test_detect_anomalies_exog_strategy_false_ignores_x_df(): + h = 3 + df, _, exog_cols = generate_panel_with_futr_exog(1, freq="D", h=h) + df_y, X_df = _split_exog(df, exog_cols) + model = ExogSpyModel(exog_strategy=False) + with_x_df = model.detect_anomalies(df_y, h=h, freq="D", n_windows=2, X_df=X_df) + without = model.detect_anomalies(df_y, h=h, freq="D", n_windows=2) + assert model.horizon_dfs == [] + pd.testing.assert_frame_equal(with_x_df, without) + + +@pytest.mark.parametrize( + "mutate,match", + [ + (lambda X: X.iloc[:-1], "X_df is missing 1"), + (lambda X: pd.concat([X, X.iloc[[0]]]), "duplicate"), + (lambda X: X.drop(columns="ds"), "must include columns"), + ], + ids=["missing-row", "duplicate-row", "missing-ds"], +) +def test_detect_anomalies_rejects_invalid_x_df(mutate, match): + df, _, exog_cols = generate_panel_with_futr_exog(1, freq="D", h=3) + df_y, X_df = _split_exog(df, exog_cols) + with pytest.raises(ValueError, match=match): + ExogSpyModel().detect_anomalies(df_y, h=3, freq="D", X_df=mutate(X_df)) + + +def test_detect_anomalies_rejects_exog_in_both_df_and_x_df(): + df, _, exog_cols = generate_panel_with_futr_exog(1, freq="D", h=3) + X_df = df[["unique_id", "ds", *exog_cols]] + with pytest.raises(ValueError, match="both df and X_df"): + ExogSpyModel().detect_anomalies(df, h=3, freq="D", X_df=X_df) + + +def test_detect_anomalies_non_native_model_rejects_x_df(): + df, _, exog_cols = generate_panel_with_futr_exog(1, freq="D", h=3) + df_y, X_df = _split_exog(df, exog_cols) + with pytest.raises(ValueError, match="does not support known-future"): + DummyModel().detect_anomalies(df_y, h=3, freq="D", X_df=X_df) + + def test_validate_input(): df = generate_series(n_series=1, freq="D", min_length=10, max_length=10) DummyModel().validate_input(df, h=2) diff --git a/tests/helpers.py b/tests/helpers.py index 6e6ba0c..3f97fa2 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -9,6 +9,7 @@ from utilsforecast.data import generate_series as _generate_series from utilsforecast.processing import make_future_dataframe +from foundationforecast.core.exog import ExogStrategyConfig from foundationforecast.core.forecaster import ExogCapableForecaster, QuantileConverter from foundationforecast.core.quantiles import quantile_column_name @@ -99,8 +100,16 @@ def generate_series_with_anomalies( class DummyModel(ExogCapableForecaster): - def __init__(self, alias: str = "dummy", reuse_loaded_model: bool = True): - super().__init__(reuse_loaded_model=reuse_loaded_model) + def __init__( + self, + alias: str = "dummy", + reuse_loaded_model: bool = True, + exog_strategy: ExogStrategyConfig = "auto", + ): + super().__init__( + reuse_loaded_model=reuse_loaded_model, + exog_strategy=exog_strategy, + ) self.alias = alias def _forecast_univariate( @@ -125,6 +134,28 @@ def _forecast_univariate( return fcst +class ExogSpyModel(DummyModel): + """Native-exog dummy that records horizon exog and forecasts their sum.""" + + def __init__( + self, alias: str = "ExogSpy", exog_strategy: ExogStrategyConfig = "auto" + ): + super().__init__(alias=alias, exog_strategy=exog_strategy) + self.horizon_dfs: list[pd.DataFrame] = [] + + def supports_native_futr_exog(self) -> bool: + return True + + def _forecast_native_futr_exog( + self, df, h, freq, level, quantiles, panel, horizon_df, futr_exog_list + ): + _ = df, h, freq, level, quantiles, panel + self.horizon_dfs.append(horizon_df[["unique_id", "ds", *futr_exog_list]]) + fcst = horizon_df[["unique_id", "ds"]].copy() + fcst[self.alias] = horizon_df[futr_exog_list].sum(axis=1).to_numpy() + return fcst + + class SeasonalNaiveModel(ExogCapableForecaster): alias = "SeasonalNaive" diff --git a/tests/models/test_models.py b/tests/models/test_models.py index 1216650..86d529a 100644 --- a/tests/models/test_models.py +++ b/tests/models/test_models.py @@ -283,6 +283,41 @@ def test_exog_cross_validation_using_level(model): assert f"{model.alias}-hi-{lv}" in cv_df.columns +@pytest.mark.parametrize("model", models) +def test_exog_detect_anomalies_with_X_df(model): + if not model.supports_native_futr_exog(): + pytest.skip("known-future exog requires native checkpoint support") + h = 2 + n_series = 2 + df, _, futr_exog_list = generate_panel_with_futr_exog( + n_series, + freq="D", + h=h, + min_length=32, + max_length=32, + ) + X_df = df[["unique_id", "ds", *futr_exog_list]] + anomalies_df = model.detect_anomalies( + df=df.drop(columns=futr_exog_list), + h=h, + freq="D", + n_windows=2, + level=95, + X_df=X_df, + futr_exog_list=futr_exog_list, + ) + assert len(anomalies_df) == n_series * 2 * h + for col in ( + model.alias, + f"{model.alias}-lo-95", + f"{model.alias}-hi-95", + f"{model.alias}-anomaly", + ): + assert col in anomalies_df.columns + assert anomalies_df[model.alias].notna().all() + assert anomalies_df[f"{model.alias}-anomaly"].dtype == bool + + @pytest.mark.parametrize("model", models) def test_passing_both_level_and_quantiles(model): df = generate_series(n_series=1, freq="D") diff --git a/tests/test_foundation_forecast.py b/tests/test_foundation_forecast.py index 1881e32..0e516b0 100644 --- a/tests/test_foundation_forecast.py +++ b/tests/test_foundation_forecast.py @@ -2,7 +2,12 @@ import pytest from utilsforecast.processing import make_future_dataframe -from tests.helpers import DummyModel, generate_panel_with_futr_exog, generate_series +from tests.helpers import ( + DummyModel, + ExogSpyModel, + generate_panel_with_futr_exog, + generate_series, +) from foundationforecast import FoundationForecast from foundationforecast.core.forecaster import ExogCapableForecaster, Forecaster from foundationforecast.core.quantiles import quantile_column_name @@ -308,3 +313,40 @@ def _forecast_univariate( ) with pytest.raises(ValueError, match="does not support known-future"): forecaster.forecast(df=df, h=3, freq="D", X_df=X_df) + + +def test_foundation_forecast_detect_anomalies_with_x_df_mixed_models(): + h = 3 + df, _, exog_cols = generate_panel_with_futr_exog(n_series=2, freq="D", h=h) + df_y, X_df = df.drop(columns=exog_cols), df[["unique_id", "ds", *exog_cols]] + spy = ExogSpyModel() + forecaster = FoundationForecast( + models=[spy, DummyModel(alias="Univariate", exog_strategy=False)] + ) + result = forecaster.detect_anomalies(df=df_y, h=h, freq="D", n_windows=2, X_df=X_df) + assert len(spy.horizon_dfs) == 2 + assert len(result) == 2 * 2 * h + for alias in ("ExogSpy", "Univariate"): + assert f"{alias}-anomaly" in result.columns + scored = result.merge(X_df, on=["unique_id", "ds"]) + pd.testing.assert_series_equal( + scored["ExogSpy"], scored["x1"] + scored["x2"], check_names=False + ) + assert (result["Univariate"] == 1.0).all() + + +@pytest.mark.parametrize("pass_x_df", [True, False]) +def test_foundation_forecast_detect_anomalies_validates_exog_before_fallback( + pass_x_df, +): + df, _, exog_cols = generate_panel_with_futr_exog(n_series=1, freq="D", h=3) + forecaster = FoundationForecast( + models=[DummyModel(alias="NonNative")], + fallback_model=ExogSpyModel(alias="Fallback"), + ) + kwargs = {} + if pass_x_df: + kwargs["X_df"] = df[["unique_id", "ds", *exog_cols]] + df = df.drop(columns=exog_cols) + with pytest.raises(ValueError, match="does not support known-future"): + forecaster.detect_anomalies(df=df, h=3, freq="D", **kwargs)