From a11a8d3cbf2df5ab32bef8ee2251574eb4a96fa9 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Wed, 26 Aug 2026 00:55:50 +0200 Subject: [PATCH 1/4] Migrate EqualFrequencyDiscretiser.fit() to narwhals, add polars support fit()'s only pandas dependency was pd.qcut(duplicates="drop"), used to compute quantile-based bin edges per variable. Replaced it with np.quantile() on each column's narwhals-extracted numpy array, plus np.unique() to sort and drop duplicate edges - reproducing qcut's duplicates="drop" behaviour without any per-backend branch, since values come from nw_X.get_column(var).to_numpy() regardless of backend. Getting a bit-exact match (not just numerically close) took two fixes verified against pandas 3.0's pandas.core.reshape.tile.qcut source: - pandas masks out NaN before calling np.quantile(values, qs, method="linear") itself, rather than using np.nanquantile - the two are not always bit-identical. Here this distinction is moot in practice: _fit_setup() already rejects NaN in variables_, so no masking is needed - values reaching the loop are already NaN-free. - qcut nudges each quantile that isn't exactly representable in base 2 up via np.nextafter (np.linspace(0, 1, q+1) then np.putmask(quantiles, q*quantiles != np.arange(q+1), nextafter(quantiles, 1))), rounding up rather than to nearest. Skipping this shifted bin edges by ~1e-13 versus real pd.qcut output and broke an existing exact-equality test. With both applied, verified bit-exact (np.array_equal) against real pd.qcut(retbins=True) across large random floats, many-duplicate-value data, all-identical-value data, negative floats, and n ...004, 1601.6000000000001 -> ...004, 1717.6999999999998 -> 1717.7000000000003) - reproduced identically with the OLD pd.qcut-based fit() on the same dataset/pandas version, so this predates the migration and is a doc-staleness issue, not a regression. Also corrected the "uses pandas.qcut() under the hood" line and added a "With polars" section with a verified worked example. Co-Authored-By: Claude Sonnet 5 --- .../EqualFrequencyDiscretiser.rst | 54 +++++++++++++++++-- .../discretisation/equal_frequency.py | 28 ++++++++-- .../test_equal_frequency_discretiser.py | 50 +++++++++++------ 3 files changed, 107 insertions(+), 25 deletions(-) diff --git a/docs/user_guide/discretisation/EqualFrequencyDiscretiser.rst b/docs/user_guide/discretisation/EqualFrequencyDiscretiser.rst index 9e2c408d7..f29064f0a 100644 --- a/docs/user_guide/discretisation/EqualFrequencyDiscretiser.rst +++ b/docs/user_guide/discretisation/EqualFrequencyDiscretiser.rst @@ -46,8 +46,9 @@ would potentially impact the model's performance in this scenario. EqualFrequencyDiscretiser ------------------------- -Feature-engine's :class:`EqualFrequencyDiscretiser` applies equal frequency discretisation to numerical variables. It uses -the `pandas.qcut()` function under the hood to determine the interval limits. +Feature-engine's :class:`EqualFrequencyDiscretiser` applies equal frequency discretisation to numerical variables. It +determines the interval limits from the variable's quantiles, matching the limits that `pandas.qcut()` would return, and +works with both pandas and polars dataframes. You can specify the variables to be discretised by passing their names in a list when setting up the transformer. Alternatively, :class:`EqualFrequencyDiscretiser` will automatically infer the data types and compute the interval limits for all numeric variables. @@ -138,7 +139,7 @@ In the following output, we see the interval limits calculated for each variable {'LotArea': [-inf, 5000.0, 7105.6, - 8099.200000000003, + 8099.200000000004, 8874.0, 9600.0, 10318.400000000001, @@ -152,8 +153,8 @@ In the following output, we see the interval limits calculated for each variable 1218.0, 1348.4, 1476.5, - 1601.6000000000001, - 1717.6999999999998, + 1601.6000000000004, + 1717.7000000000003, 1893.0000000000005, 2166.3999999999996, inf]} @@ -393,6 +394,49 @@ the value range. .. image:: ../../images/equalfrequencydiscretisation_skewed.png +With polars +----------- + +:class:`EqualFrequencyDiscretiser` works in the same way with a polars dataframe: + +.. code:: python + + import polars as pl + from feature_engine.discretisation import EqualFrequencyDiscretiser + + df = pl.DataFrame({ + "Age": [20, 21, 19, 18, 25, 30, 45, 60, 15, 22], + "Marks": [0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1, 0.95], + }) + + disc = EqualFrequencyDiscretiser(q=5, variables=["Age", "Marks"]) + + print(disc.fit_transform(df)) + +The bin edges and resulting codes match those found with pandas: + +.. code:: text + + shape: (10, 2) + ┌─────┬───────┐ + │ Age ┆ Marks │ + │ --- ┆ --- │ + │ i64 ┆ i64 │ + ╞═════╪═══════╡ + │ 1 ┆ 4 │ + │ 2 ┆ 3 │ + │ 1 ┆ 3 │ + │ 0 ┆ 2 │ + │ 3 ┆ 2 │ + │ 3 ┆ 1 │ + │ 4 ┆ 1 │ + │ 4 ┆ 0 │ + │ 0 ┆ 0 │ + │ 2 ┆ 4 │ + └─────┴───────┘ + +`return_object`, `return_boundaries`, and `get_feature_names_out()` work identically to the pandas examples above. + See Also -------- diff --git a/feature_engine/discretisation/equal_frequency.py b/feature_engine/discretisation/equal_frequency.py index a2137870f..fc2549b3a 100644 --- a/feature_engine/discretisation/equal_frequency.py +++ b/feature_engine/discretisation/equal_frequency.py @@ -3,7 +3,9 @@ from typing import List, Optional, Union -import pandas as pd +import narwhals as nw +import numpy as np +from narwhals.typing import IntoDataFrame, IntoSeries from feature_engine._check_init_parameters.check_init_input_params import ( _check_return_empty_is_bool, @@ -156,13 +158,13 @@ def __init__( self.return_empty = return_empty self.q = q - def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): + def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): """ Learn the limits of the equal frequency intervals. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The training dataset. Can be the entire dataframe, not just the variables to be transformed. y: None @@ -172,10 +174,28 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # check input dataframe X, variables_ = self._fit_setup(X) + nw_X = nw.from_native(X, eager_only=True) + quantiles = np.linspace(0, 1, self.q + 1) + # pandas.qcut nudges each quantile that isn't exactly representable in + # base 2 up via nextafter, to round up rather than to nearest (verified + # against pandas.core.reshape.tile.qcut source); skipping this shifts + # bin edges by ~1e-13 versus the pre-migration pd.qcut output. + np.putmask( + quantiles, + self.q * quantiles != np.arange(self.q + 1), + np.nextafter(quantiles, 1), + ) + binner_dict_ = {} for var in variables_: - tmp, bins = pd.qcut(x=X[var], q=self.q, retbins=True, duplicates="drop") + # _fit_setup() already rejects NaN in variables_, so no NaN-masking + # is needed here. np.quantile replicates pandas.qcut's own quantile + # computation (verified bit-exact against real pd.qcut(retbins=True) + # output); np.unique both sorts and drops duplicate edges, matching + # qcut(duplicates="drop"). + values = nw_X.get_column(var).to_numpy() + bins = np.unique(np.quantile(values, quantiles, method="linear")) # Prepend/Append infinities to accommodate outliers bins = list(bins) diff --git a/tests/test_discretisation/test_equal_frequency_discretiser.py b/tests/test_discretisation/test_equal_frequency_discretiser.py index 112262dd1..329e355b4 100644 --- a/tests/test_discretisation/test_equal_frequency_discretiser.py +++ b/tests/test_discretisation/test_equal_frequency_discretiser.py @@ -1,17 +1,22 @@ import pandas as pd +import polars as pl import pytest from sklearn.exceptions import NotFittedError from feature_engine.discretisation import EqualFrequencyDiscretiser -def test_automatically_find_variables_and_return_as_numeric(df_normal_dist): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_automatically_find_variables_and_return_as_numeric(make_df, df_normal_dist): # test case 1: automatically select variables, return_object=False + data = make_df(df_normal_dist) transformer = EqualFrequencyDiscretiser(q=10, variables=None, return_object=False) - X = transformer.fit_transform(df_normal_dist) + X = transformer.fit_transform(data) - # output expected for fit attr + # output expected for fit attr, computed via pandas.qcut (verified bit-exact + # against the transformer's own numpy-based bin edges on both backends) _, bins = pd.qcut(x=df_normal_dist["var"], q=10, retbins=True, duplicates="drop") + bins = list(bins) bins[0] = float("-inf") bins[len(bins) - 1] = float("inf") @@ -26,17 +31,23 @@ def test_automatically_find_variables_and_return_as_numeric(df_normal_dist): assert transformer.variables_ == ["var"] assert transformer.n_features_in_ == 1 # test transform output - assert (transformer.binner_dict_["var"] == bins).all() - assert all(x for x in X["var"].unique() if x not in X_t) + assert transformer.binner_dict_["var"] == bins + X_pd = X if isinstance(X, pd.DataFrame) else X.to_pandas() + assert all(x for x in X_pd["var"].unique() if x not in X_t) # in equal frequency discretisation, all intervals get same proportion of values - assert len((X["var"].value_counts()).unique()) == 1 + assert len((X_pd["var"].value_counts()).unique()) == 1 -def test_automatically_find_variables_and_return_as_object(df_normal_dist): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_automatically_find_variables_and_return_as_object(make_df, df_normal_dist): # test case 2: return variables cast as object + data = make_df(df_normal_dist) transformer = EqualFrequencyDiscretiser(q=10, variables=None, return_object=True) - X = transformer.fit_transform(df_normal_dist) - assert X["var"].dtypes == "O" + X = transformer.fit_transform(data) + if isinstance(X, pd.DataFrame): + assert X["var"].dtypes == "O" + else: + assert X["var"].dtype == pl.Object def test_error_when_q_not_number(): @@ -49,22 +60,29 @@ def test_error_if_return_object_not_bool(): EqualFrequencyDiscretiser(return_object="other") -def test_error_if_input_df_contains_na_in_fit(df_na): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_error_if_input_df_contains_na_in_fit(make_df, df_na): # test case 3: when dataset contains na, fit method + data = make_df(df_na) with pytest.raises(ValueError): transformer = EqualFrequencyDiscretiser() - transformer.fit(df_na) + transformer.fit(data) -def test_error_if_input_df_contains_na_in_transform(df_vartypes, df_na): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_error_if_input_df_contains_na_in_transform(make_df, df_vartypes, df_na): # test case 4: when dataset contains na, transform method + fit_data = make_df(df_vartypes) + transform_data = make_df(df_na[["Name", "City", "Age", "Marks", "dob"]]) with pytest.raises(ValueError): transformer = EqualFrequencyDiscretiser() - transformer.fit(df_vartypes) - transformer.transform(df_na[["Name", "City", "Age", "Marks", "dob"]]) + transformer.fit(fit_data) + transformer.transform(transform_data) -def test_non_fitted_error(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_non_fitted_error(make_df, df_vartypes): + data = make_df(df_vartypes) with pytest.raises(NotFittedError): transformer = EqualFrequencyDiscretiser() - transformer.transform(df_vartypes) + transformer.transform(data) From 11dbdf66df95d780e45eb868ec881da0005d3a22 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Mon, 14 Sep 2026 22:42:30 +0200 Subject: [PATCH 2/4] Use shared backend test fixtures and helpers in EqualFrequencyDiscretiser tests Build inputs from the data_normal_dist / data_vartypes / data_na fixtures on the backend under test instead of converting pandas fixtures (which needs pyarrow for polars, so the polars cases failed), check isinstance(X, make_df) plus to_dict() contents, and use pytest.raises(match=re.escape(msg)). The check that every bin code is present was vacuous and now compares the exact set of codes. Co-Authored-By: Claude Opus 5 --- .../test_equal_frequency_discretiser.py | 79 ++++++++++--------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/tests/test_discretisation/test_equal_frequency_discretiser.py b/tests/test_discretisation/test_equal_frequency_discretiser.py index 329e355b4..30b158c48 100644 --- a/tests/test_discretisation/test_equal_frequency_discretiser.py +++ b/tests/test_discretisation/test_equal_frequency_discretiser.py @@ -1,28 +1,36 @@ +import re +from collections import Counter + +import narwhals as nw import pandas as pd -import polars as pl import pytest from sklearn.exceptions import NotFittedError from feature_engine.discretisation import EqualFrequencyDiscretiser +from tests.backend_helpers import to_dict + +MSG_NA = ( + "Some of the variables in the dataset contain NaN. Check and " + "remove those before using this transformer." +) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_automatically_find_variables_and_return_as_numeric(make_df, df_normal_dist): +def test_automatically_find_variables_and_return_as_numeric( + make_df, data_normal_dist +): # test case 1: automatically select variables, return_object=False - data = make_df(df_normal_dist) transformer = EqualFrequencyDiscretiser(q=10, variables=None, return_object=False) - X = transformer.fit_transform(data) + X = transformer.fit_transform(make_df(data_normal_dist)) # output expected for fit attr, computed via pandas.qcut (verified bit-exact # against the transformer's own numpy-based bin edges on both backends) - _, bins = pd.qcut(x=df_normal_dist["var"], q=10, retbins=True, duplicates="drop") + _, bins = pd.qcut( + x=pd.Series(data_normal_dist["var"]), q=10, retbins=True, duplicates="drop" + ) bins = list(bins) bins[0] = float("-inf") bins[len(bins) - 1] = float("inf") - # expected transform output - X_t = [x for x in range(0, 10)] - # test init params assert transformer.q == 10 assert transformer.variables is None @@ -30,24 +38,21 @@ def test_automatically_find_variables_and_return_as_numeric(make_df, df_normal_d # test fit attr assert transformer.variables_ == ["var"] assert transformer.n_features_in_ == 1 - # test transform output assert transformer.binner_dict_["var"] == bins - X_pd = X if isinstance(X, pd.DataFrame) else X.to_pandas() - assert all(x for x in X_pd["var"].unique() if x not in X_t) + # test transform output + assert isinstance(X, make_df) + values = to_dict(X)["var"] + assert set(values) == set(range(10)) # in equal frequency discretisation, all intervals get same proportion of values - assert len((X_pd["var"].value_counts()).unique()) == 1 + assert len(set(Counter(values).values())) == 1 -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_automatically_find_variables_and_return_as_object(make_df, df_normal_dist): +def test_automatically_find_variables_and_return_as_object(make_df, data_normal_dist): # test case 2: return variables cast as object - data = make_df(df_normal_dist) transformer = EqualFrequencyDiscretiser(q=10, variables=None, return_object=True) - X = transformer.fit_transform(data) - if isinstance(X, pd.DataFrame): - assert X["var"].dtypes == "O" - else: - assert X["var"].dtype == pl.Object + X = transformer.fit_transform(make_df(data_normal_dist)) + assert isinstance(X, make_df) + assert nw.from_native(X, eager_only=True).schema["var"] == nw.Object def test_error_when_q_not_number(): @@ -60,29 +65,25 @@ def test_error_if_return_object_not_bool(): EqualFrequencyDiscretiser(return_object="other") -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_error_if_input_df_contains_na_in_fit(make_df, df_na): +def test_error_if_input_df_contains_na_in_fit(make_df, data_na): # test case 3: when dataset contains na, fit method - data = make_df(df_na) - with pytest.raises(ValueError): - transformer = EqualFrequencyDiscretiser() - transformer.fit(data) + transformer = EqualFrequencyDiscretiser() + with pytest.raises(ValueError, match=re.escape(MSG_NA)): + transformer.fit(make_df(data_na)) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_error_if_input_df_contains_na_in_transform(make_df, df_vartypes, df_na): +def test_error_if_input_df_contains_na_in_transform(make_df, data_vartypes, data_na): # test case 4: when dataset contains na, transform method - fit_data = make_df(df_vartypes) - transform_data = make_df(df_na[["Name", "City", "Age", "Marks", "dob"]]) - with pytest.raises(ValueError): - transformer = EqualFrequencyDiscretiser() - transformer.fit(fit_data) + transform_data = make_df( + {k: data_na[k] for k in ["Name", "City", "Age", "Marks", "dob"]} + ) + transformer = EqualFrequencyDiscretiser() + transformer.fit(make_df(data_vartypes)) + with pytest.raises(ValueError, match=re.escape(MSG_NA)): transformer.transform(transform_data) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_non_fitted_error(make_df, df_vartypes): - data = make_df(df_vartypes) +def test_non_fitted_error(make_df, data_vartypes): + transformer = EqualFrequencyDiscretiser() with pytest.raises(NotFittedError): - transformer = EqualFrequencyDiscretiser() - transformer.transform(data) + transformer.transform(make_df(data_vartypes)) From 582835916fc05dbbabadfded4cbd3f33d57a6b2e Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 15 Sep 2026 12:06:19 +0200 Subject: [PATCH 3/4] Use frame_to_dict after the shared helper rename in #1045 Co-Authored-By: Claude Opus 5 --- tests/test_discretisation/test_equal_frequency_discretiser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_discretisation/test_equal_frequency_discretiser.py b/tests/test_discretisation/test_equal_frequency_discretiser.py index 30b158c48..65fc1490e 100644 --- a/tests/test_discretisation/test_equal_frequency_discretiser.py +++ b/tests/test_discretisation/test_equal_frequency_discretiser.py @@ -7,7 +7,7 @@ from sklearn.exceptions import NotFittedError from feature_engine.discretisation import EqualFrequencyDiscretiser -from tests.backend_helpers import to_dict +from tests.backend_helpers import frame_to_dict MSG_NA = ( "Some of the variables in the dataset contain NaN. Check and " @@ -41,7 +41,7 @@ def test_automatically_find_variables_and_return_as_numeric( assert transformer.binner_dict_["var"] == bins # test transform output assert isinstance(X, make_df) - values = to_dict(X)["var"] + values = frame_to_dict(X)["var"] assert set(values) == set(range(10)) # in equal frequency discretisation, all intervals get same proportion of values assert len(set(Counter(values).values())) == 1 From db9ddf2e08827675c7762394042b346b179a57fc Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Fri, 18 Sep 2026 13:24:01 +0200 Subject: [PATCH 4/4] Group init tests and match errors in EqualFrequencyDiscretiser tests Co-Authored-By: Claude Opus 5 --- .../test_equal_frequency_discretiser.py | 54 +++++++++++++------ 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/tests/test_discretisation/test_equal_frequency_discretiser.py b/tests/test_discretisation/test_equal_frequency_discretiser.py index 65fc1490e..1291903c6 100644 --- a/tests/test_discretisation/test_equal_frequency_discretiser.py +++ b/tests/test_discretisation/test_equal_frequency_discretiser.py @@ -15,6 +15,40 @@ ) +# init parameters +@pytest.mark.parametrize("q", ["other", 1.5, None, [10]]) +def test_error_when_q_not_number(q): + msg = f"q must be an integer. Got {q} instead." + with pytest.raises(ValueError, match=re.escape(msg)): + EqualFrequencyDiscretiser(q=q) + + +@pytest.mark.parametrize("return_object", ["other", 1, None]) +def test_error_if_return_object_not_bool(return_object): + msg = f"return_object must be True or False. Got {return_object} instead." + with pytest.raises(ValueError, match=re.escape(msg)): + EqualFrequencyDiscretiser(return_object=return_object) + + +@pytest.mark.parametrize( + "q, return_object, return_boundaries, precision", + [(10, False, False, 3), (5, True, False, 1), (2, False, True, 7)], +) +def test_init_param_assignment(q, return_object, return_boundaries, precision): + transformer = EqualFrequencyDiscretiser( + q=q, + return_object=return_object, + return_boundaries=return_boundaries, + precision=precision, + ) + assert transformer.q == q + assert transformer.return_object is return_object + assert transformer.return_boundaries is return_boundaries + assert transformer.precision == precision + + +# fit and transform + def test_automatically_find_variables_and_return_as_numeric( make_df, data_normal_dist ): @@ -31,10 +65,6 @@ def test_automatically_find_variables_and_return_as_numeric( bins[0] = float("-inf") bins[len(bins) - 1] = float("inf") - # test init params - assert transformer.q == 10 - assert transformer.variables is None - assert transformer.return_object is False # test fit attr assert transformer.variables_ == ["var"] assert transformer.n_features_in_ == 1 @@ -55,16 +85,6 @@ def test_automatically_find_variables_and_return_as_object(make_df, data_normal_ assert nw.from_native(X, eager_only=True).schema["var"] == nw.Object -def test_error_when_q_not_number(): - with pytest.raises(ValueError): - EqualFrequencyDiscretiser(q="other") - - -def test_error_if_return_object_not_bool(): - with pytest.raises(ValueError): - EqualFrequencyDiscretiser(return_object="other") - - def test_error_if_input_df_contains_na_in_fit(make_df, data_na): # test case 3: when dataset contains na, fit method transformer = EqualFrequencyDiscretiser() @@ -85,5 +105,9 @@ def test_error_if_input_df_contains_na_in_transform(make_df, data_vartypes, data def test_non_fitted_error(make_df, data_vartypes): transformer = EqualFrequencyDiscretiser() - with pytest.raises(NotFittedError): + msg = ( + "This EqualFrequencyDiscretiser instance is not fitted yet. Call 'fit' with " + "appropriate arguments before using this estimator." + ) + with pytest.raises(NotFittedError, match=re.escape(msg)): transformer.transform(make_df(data_vartypes))