wehub-resource-sync
119 lines
5.1 KiB
Python
119 lines
5.1 KiB
Python
"""Helper methods used internally for computing label quality scores."""
|
|
|
|
import warnings
|
|
import numpy as np
|
|
from typing import Optional
|
|
from scipy.special import xlogy
|
|
|
|
from cleanlab.count import get_confident_thresholds
|
|
|
|
|
|
def _subtract_confident_thresholds(
|
|
labels: Optional[np.ndarray],
|
|
pred_probs: np.ndarray,
|
|
multi_label: bool = False,
|
|
confident_thresholds: Optional[np.ndarray] = None,
|
|
) -> np.ndarray:
|
|
"""
|
|
Return adjusted predicted probabilities by subtracting the class confident thresholds and renormalizing.
|
|
|
|
The confident class threshold for a class j is the expected (average) "self-confidence" for class j.
|
|
The purpose of this adjustment is to handle class imbalance.
|
|
|
|
Parameters
|
|
----------
|
|
labels : np.ndarray
|
|
Labels in the same format expected by the `cleanlab.count.get_confident_thresholds()` method.
|
|
If labels is None, confident_thresholds needs to be passed in as it will not be calculated.
|
|
pred_probs : np.ndarray (shape (N, K))
|
|
Predicted-probabilities in the same format expected by the `cleanlab.count.get_confident_thresholds()` method.
|
|
confident_thresholds : np.ndarray (shape (K,))
|
|
Pre-calculated confident thresholds. If passed in, function will subtract these thresholds instead of calculating
|
|
confident_thresholds from the given labels and pred_probs.
|
|
multi_label : bool, optional
|
|
If ``True``, labels should be an iterable (e.g. list) of iterables, containing a
|
|
list of labels for each example, instead of just a single label.
|
|
The multi-label setting supports classification tasks where an example has 1 or more labels.
|
|
Example of a multi-labeled `labels` input: ``[[0,1], [1], [0,2], [0,1,2], [0], [1], ...]``.
|
|
The major difference in how this is calibrated versus single-label is that
|
|
the total number of errors considered is based on the number of labels,
|
|
not the number of examples. So, the calibrated `confident_joint` will sum
|
|
to the number of total labels.
|
|
|
|
Returns
|
|
-------
|
|
pred_probs_adj : np.ndarray (float)
|
|
Adjusted pred_probs.
|
|
"""
|
|
# Get expected (average) self-confidence for each class
|
|
# TODO: Test this for multi-label
|
|
if confident_thresholds is None:
|
|
if labels is None:
|
|
raise ValueError(
|
|
"Cannot calculate confident_thresholds without labels. Pass in either labels or already calculated "
|
|
"confident_thresholds parameter. "
|
|
)
|
|
confident_thresholds = get_confident_thresholds(labels, pred_probs, multi_label=multi_label)
|
|
|
|
# Subtract the class confident thresholds
|
|
pred_probs_adj = pred_probs - confident_thresholds
|
|
|
|
# Re-normalize by shifting data to take care of negative values from the subtraction
|
|
pred_probs_adj += confident_thresholds.max()
|
|
pred_probs_adj /= pred_probs_adj.sum(axis=1, keepdims=True)
|
|
|
|
return pred_probs_adj
|
|
|
|
|
|
def get_normalized_entropy(
|
|
pred_probs: np.ndarray, min_allowed_prob: Optional[float] = None
|
|
) -> np.ndarray:
|
|
"""Return the normalized entropy of pred_probs.
|
|
|
|
Normalized entropy is between 0 and 1. Higher values of entropy indicate higher uncertainty in the model's prediction of the correct label.
|
|
|
|
Read more about normalized entropy `on Wikipedia <https://en.wikipedia.org/wiki/Entropy_(information_theory)>`_.
|
|
|
|
Normalized entropy is used in active learning for uncertainty sampling: https://medium.com/data-science/uncertainty-sampling-cheatsheet-ec57bc067c0b
|
|
|
|
Unlike label-quality scores, entropy only depends on the model's predictions, not the given label.
|
|
|
|
Parameters
|
|
----------
|
|
pred_probs : np.ndarray (shape (N, K))
|
|
Each row of this matrix corresponds to an example x and contains the model-predicted
|
|
probabilities that x belongs to each possible class: P(label=k|x)
|
|
|
|
min_allowed_prob : float, default: None, deprecated
|
|
Minimum allowed probability value. If not `None` (default),
|
|
entries of `pred_probs` below this value will be clipped to this value.
|
|
|
|
.. deprecated:: 2.5.0
|
|
This keyword is deprecated and should be left to the default.
|
|
The entropy is well-behaved even if `pred_probs` contains zeros,
|
|
clipping is unnecessary and (slightly) changes the results.
|
|
|
|
Returns
|
|
-------
|
|
entropy : np.ndarray (shape (N, ))
|
|
Each element is the normalized entropy of the corresponding row of ``pred_probs``.
|
|
|
|
Raises
|
|
------
|
|
ValueError
|
|
An error is raised if any of the probabilities is not in the interval [0, 1].
|
|
"""
|
|
if np.any(pred_probs < 0) or np.any(pred_probs > 1):
|
|
raise ValueError("All probabilities are required to be in the interval [0, 1].")
|
|
num_classes = pred_probs.shape[1]
|
|
|
|
if min_allowed_prob is not None:
|
|
warnings.warn(
|
|
"Using `min_allowed_prob` is not necessary anymore and will be removed.",
|
|
DeprecationWarning,
|
|
)
|
|
pred_probs = np.clip(pred_probs, a_min=min_allowed_prob, a_max=None)
|
|
|
|
# Note that dividing by log(num_classes) changes the base of the log which rescales entropy to 0-1 range
|
|
return -np.sum(xlogy(pred_probs, pred_probs), axis=1) / np.log(num_classes)
|