wehub-resource-sync
114 lines
4.1 KiB
Python
114 lines
4.1 KiB
Python
from dataclasses import dataclass
|
|
from typing import List, Optional, Union
|
|
import warnings
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
from sklearn.model_selection import cross_val_score
|
|
from sklearn.naive_bayes import GaussianNB
|
|
|
|
warnings.filterwarnings("ignore")
|
|
|
|
|
|
@dataclass
|
|
class SpuriousCorrelations:
|
|
data: pd.DataFrame
|
|
labels: Union[np.ndarray, list]
|
|
properties_of_interest: Optional[List[str]] = None
|
|
|
|
def __post_init__(self):
|
|
# Must have same number of rows
|
|
if not len(self.data) == len(self.labels):
|
|
raise ValueError(
|
|
"The number of rows in the data dataframe must be the same as the number of labels."
|
|
)
|
|
|
|
# Set default properties_of_interest if not provided
|
|
if self.properties_of_interest is None:
|
|
self.properties_of_interest = self.data.columns.tolist()
|
|
|
|
if not all(isinstance(p, str) for p in self.properties_of_interest):
|
|
raise TypeError("properties_of_interest must be a list of strings.")
|
|
|
|
def calculate_correlations(self) -> pd.DataFrame:
|
|
"""Calculates the spurious correlation scores for each property of interest found in the dataset."""
|
|
baseline_accuracy = self._get_baseline()
|
|
assert (
|
|
self.properties_of_interest is not None
|
|
), "properties_of_interest must be set, but is None."
|
|
property_scores = {
|
|
str(property_of_interest): self.calculate_spurious_correlation(
|
|
property_of_interest, baseline_accuracy
|
|
)
|
|
for property_of_interest in self.properties_of_interest
|
|
}
|
|
data_score = pd.DataFrame(list(property_scores.items()), columns=["property", "score"])
|
|
return data_score
|
|
|
|
def _get_baseline(self) -> float:
|
|
"""Calculates the baseline accuracy of the dataset. The baseline model is predicting the most common label."""
|
|
baseline_accuracy = np.bincount(self.labels).argmax() / len(self.labels)
|
|
return float(baseline_accuracy)
|
|
|
|
def calculate_spurious_correlation(
|
|
self, property_of_interest, baseline_accuracy: float
|
|
) -> float:
|
|
"""Scores the dataset based on a given property of interest.
|
|
|
|
Parameters
|
|
----------
|
|
property_of_interest :
|
|
The property of interest to score the dataset on.
|
|
|
|
baseline_accuracy :
|
|
The accuracy of the baseline model.
|
|
|
|
Returns
|
|
-------
|
|
score :
|
|
A correlation score of the dataset's labels to the property of interest.
|
|
"""
|
|
X = self.data[property_of_interest].values.reshape(-1, 1)
|
|
y = self.labels
|
|
mean_accuracy = _train_and_eval(X, y)
|
|
return relative_room_for_improvement(baseline_accuracy, float(mean_accuracy))
|
|
|
|
|
|
def _train_and_eval(X, y, cv=5) -> float:
|
|
classifier = GaussianNB() # TODO: Make this a parameter
|
|
cv_accuracies = cross_val_score(classifier, X, y, cv=cv, scoring="accuracy")
|
|
mean_accuracy = float(np.mean(cv_accuracies))
|
|
return mean_accuracy
|
|
|
|
|
|
def relative_room_for_improvement(
|
|
baseline_accuracy: float, mean_accuracy: float, eps: float = 1e-8
|
|
) -> float:
|
|
"""
|
|
Calculate the relative room for improvement given a baseline and trial accuracy.
|
|
|
|
This function computes the ratio of the difference between perfect accuracy (1.0)
|
|
and the trial accuracy to the difference between perfect accuracy and the baseline accuracy.
|
|
If the baseline accuracy is perfect (i.e., 1.0), an epsilon value is added to the denominator
|
|
to avoid division by zero.
|
|
|
|
Parameters
|
|
----------
|
|
baseline_accuracy :
|
|
The accuracy of the baseline model. Must be between 0 and 1.
|
|
mean_accuracy :
|
|
The accuracy of the trial model being compared. Must be between 0 and 1.
|
|
eps :
|
|
A small constant to avoid division by zero when baseline accuracy is 1. Defaults to 1e-8.
|
|
|
|
Returns
|
|
-------
|
|
score :
|
|
The relative room for improvement, bounded between 0 and 1.
|
|
"""
|
|
numerator = 1 - mean_accuracy
|
|
denominator = 1 - baseline_accuracy
|
|
if baseline_accuracy == 1:
|
|
denominator += eps
|
|
return min(1, numerator / denominator)
|