Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix zero percentage replacement in get_binned_data Function and update parameter name #1278

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Fix zero percentage replacement in get_binned_data Function and update parameter name #1278

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
63de8d5
fixed min non-zero issue v1
boemer00 Sep 2, 2024
a9bac3f
name correction feel -> fill
boemer00 Sep 2, 2024
bd0b12c
updated function with new parameters
boemer00 Sep 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 28 additions & 16 deletions src/evidently/calculations/stattests/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,18 @@ def get_unique_not_nan_values_list_from_series(current_data: pd.Series, referenc


def get_binned_data(
reference_data: pd.Series, current_data: pd.Series, feature_type: ColumnType, n: int, feel_zeroes: bool = True
reference_data: pd.Series, current_data: pd.Series, feature_type: ColumnType, n: int, fill_zeroes: bool=True,
fill_method: str='auto', dynamic_scale: bool=False
):
"""Split variable into n buckets based on reference quantiles
Args:
reference_data: reference data
current_data: current data
feature_type: feature type
n: number of quantiles
fill_zeroes: whether to fill zero percentages
fill_method: method to calculate fill value ('auto', 'min', 'mean')
dynamic_scale: whether to use dynamic scaling for fill value
Returns:
reference_percents: % of records in each bucket for reference
current_percents: % of records in each bucket for current
Expand All @@ -38,21 +42,29 @@ def get_binned_data(
reference_percents = np.array([ref_feature_dict[key] / len(reference_data) for key in keys])
current_percents = np.array([current_feature_dict[key] / len(current_data) for key in keys])

if feel_zeroes:
np.place(
reference_percents,
reference_percents == 0,
min(reference_percents[reference_percents != 0]) / 10**6
if min(reference_percents[reference_percents != 0]) <= 0.0001
else 0.0001,
)
np.place(
current_percents,
current_percents == 0,
min(current_percents[current_percents != 0]) / 10**6
if min(current_percents[current_percents != 0]) <= 0.0001
else 0.0001,
)
if fill_zeroes:
min_non_zero_ref = np.min(reference_percents[reference_percents != 0])
min_non_zero_cur = np.min(current_percents[current_percents != 0])

if fill_method == 'auto':
fill_value = min(min_non_zero_ref, min_non_zero_cur) / 10
fill_value = min(fill_value, min(min_non_zero_ref, min_non_zero_cur) / 2)
elif fill_method == 'min':
fill_value = min(min_non_zero_ref, min_non_zero_cur)
elif fill_method == 'mean':
fill_value = (min_non_zero_ref + min_non_zero_cur) / 2
else:
raise ValueError("Invalid fill_method. Choose 'auto', 'min', or 'mean'.")

if dynamic_scale:
scale_factor = min(min_non_zero_ref, min_non_zero_cur) / max(min_non_zero_ref, min_non_zero_cur)
fill_value *= scale_factor

np.place(reference_percents, reference_percents == 0, fill_value)
np.place(current_percents, current_percents == 0, fill_value)

reference_percents = reference_percents / np.sum(reference_percents)
current_percents = current_percents / np.sum(current_percents)

return reference_percents, current_percents

Expand Down
Loading