mirrors/cleanlab--cleanlab
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:49:22 +08:00
commit a41b2ab474
303 changed files with 64772 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cleanlab/multilabel_classification/__init__.py
+4
View File
@@ -0,0 +1,4 @@
from .rank import get_label_quality_scores
from . import rank
from . import dataset
from . import filter
cleanlab/multilabel_classification/dataset.py
+326
View File
@@ -0,0 +1,326 @@
"""
Methods to summarize overall labeling issues across a multi-label classification dataset.
Here each example can belong to one or more classes, or none of the classes at all.
Unlike in standard multi-class classification, model-predicted class probabilities need not sum to 1 for each row in multi-label classification.
"""
import pandas as pd
import numpy as np
from typing import Optional, cast, Dict, Any # noqa: F401
from cleanlab.multilabel_classification.filter import (
find_multilabel_issues_per_class,
find_label_issues,
)
from cleanlab.internal.multilabel_utils import get_onehot_num_classes
from collections import defaultdict
def common_multilabel_issues(
labels=list,
pred_probs=None,
*,
class_names=None,
confident_joint=None,
) -> pd.DataFrame:
"""Summarizes which classes in a multi-label dataset appear most often mislabeled overall.
Since classes are not mutually exclusive in multi-label classification, this method summarizes the label issues for each class independently of the others.
Parameters
----------
labels : List[List[int]]
List of noisy labels for multi-label classification where each example can belong to multiple classes.
Refer to documentation for this argument in :py:func:`multilabel_classification.filter.find_label_issues <cleanlab.multilabel_classification.filter.find_label_issues>` for further details.
pred_probs : np.ndarray
An array of shape ``(N, K)`` of model-predicted class probabilities.
Refer to documentation for this argument in :py:func:`multilabel_classification.filter.find_label_issues <cleanlab.multilabel_classification.filter.find_label_issues>` for further details.
class_names : Iterable[str], optional
A list or other iterable of the string class names. Its order must match the label indices.
If class 0 is 'dog' and class 1 is 'cat', then ``class_names = ['dog', 'cat']``.
If provided, the returned DataFrame will have an extra *Class Name* column with this info.
confident_joint : np.ndarray, optional
An array of shape ``(K, 2, 2)`` representing a one-vs-rest formatted confident joint.
Refer to documentation for this argument in :py:func:`multilabel_classification.filter.find_label_issues <cleanlab.multilabel_classification.filter.find_label_issues>` for details.
Returns
-------
common_multilabel_issues : pd.DataFrame
DataFrame where each row corresponds to a class summarized by the following columns:
- *Class Name*: The name of the class if class_names is provided.
- *Class Index*: The index of the class.
- *In Given Label*: Whether the Class is originally annotated True or False in the given label.
- *In Suggested Label*: Whether the Class should be True or False in the suggested label (based on model's prediction).
- *Num Examples*: Number of examples flagged as a label issue where this Class is True/False "In Given Label" but cleanlab estimates the annotation should actually be as specified "In Suggested Label". I.e. the number of examples in your dataset where this Class was labeled as True but likely should have been False (or vice versa).
- *Issue Probability*: The *Num Examples* column divided by the total number of examples in the dataset; i.e. the relative overall frequency of each type of label issue in your dataset.
By default, the rows in this DataFrame are ordered by "Issue Probability" (descending).
"""
num_examples = _get_num_examples_multilabel(labels=labels, confident_joint=confident_joint)
summary_issue_counts = defaultdict(list)
y_one, num_classes = get_onehot_num_classes(labels, pred_probs)
label_issues_list, labels_list, pred_probs_list = find_multilabel_issues_per_class(
labels=labels,
pred_probs=pred_probs,
confident_joint=confident_joint,
return_indices_ranked_by="self_confidence",
)
for class_num, (label, issues_for_class) in enumerate(zip(y_one.T, label_issues_list)):
binary_label_issues = np.zeros(len(label)).astype(bool)
binary_label_issues[issues_for_class] = True
true_but_false_count = sum(np.logical_and(label == 1, binary_label_issues))
false_but_true_count = sum(np.logical_and(label == 0, binary_label_issues))
if class_names is not None:
summary_issue_counts["Class Name"].append(class_names[class_num])
summary_issue_counts["Class Index"].append(class_num)
summary_issue_counts["In Given Label"].append(True)
summary_issue_counts["In Suggested Label"].append(False)
summary_issue_counts["Num Examples"].append(true_but_false_count)
summary_issue_counts["Issue Probability"].append(true_but_false_count / num_examples)
if class_names is not None:
summary_issue_counts["Class Name"].append(class_names[class_num])
summary_issue_counts["Class Index"].append(class_num)
summary_issue_counts["In Given Label"].append(False)
summary_issue_counts["In Suggested Label"].append(True)
summary_issue_counts["Num Examples"].append(false_but_true_count)
summary_issue_counts["Issue Probability"].append(false_but_true_count / num_examples)
return (
pd.DataFrame.from_dict(summary_issue_counts)
.sort_values(by=["Issue Probability"], ascending=False)
.reset_index(drop=True)
)
def rank_classes_by_multilabel_quality(
labels=None,
pred_probs=None,
*,
class_names=None,
joint=None,
confident_joint=None,
) -> pd.DataFrame:
"""
Returns a DataFrame with three overall label quality scores per class for a multi-label dataset.
These numbers summarize all examples annotated with the class (details listed below under the Returns parameter).
By default, classes are ordered by "Label Quality Score", so the most problematic classes are reported first in the DataFrame.
Score values are unnormalized and may be very small. What matters is their relative ranking across the classes.
**Parameters**:
For information about the arguments to this method, see the documentation of
`~cleanlab.multilabel_classification.dataset.common_multilabel_issues`.
Returns
-------
overall_label_quality : pd.DataFrame
Pandas DataFrame with one row per class and columns: "Class Index", "Label Issues",
"Inverse Label Issues", "Label Issues", "Inverse Label Noise", "Label Quality Score".
Some entries are overall quality scores between 0 and 1, summarizing how good overall the labels
appear to be for that class (lower values indicate more erroneous labels).
Other entries are estimated counts of annotation errors related to this class.
Here is what each column represents:
- *Class Name*: The name of the class if class_names is provided.
- *Class Index*: The index of the class in 0, 1, ..., K-1.
- *Label Issues*: Estimated number of examples in the dataset that are labeled as belonging to class k but actually should not belong to this class.
- *Inverse Label Issues*: Estimated number of examples in the dataset that should actually be labeled as class k but did not receive this label.
- *Label Noise*: Estimated proportion of examples in the dataset that are labeled as class k but should not be. For each class k: this is computed by dividing the number of examples with "Label Issues" that were labeled as class k by the total number of examples labeled as class k.
- *Inverse Label Noise*: Estimated proportion of examples in the dataset that should actually be labeled as class k but did not receive this label.
- *Label Quality Score*: Estimated proportion of examples labeled as class k that have been labeled correctly, i.e. ``1 - label_noise``.
By default, the DataFrame is ordered by "Label Quality Score" (in ascending order), so the classes with the most label issues appear first.
"""
issues_df = common_multilabel_issues(
labels=labels, pred_probs=pred_probs, class_names=class_names, confident_joint=joint
)
issues_dict = defaultdict(defaultdict) # type: Dict[str, Any]
num_examples = _get_num_examples_multilabel(labels=labels, confident_joint=confident_joint)
return_columns = [
"Class Name",
"Class Index",
"Label Issues",
"Inverse Label Issues",
"Label Noise",
"Inverse Label Noise",
"Label Quality Score",
]
if class_names is None:
return_columns = return_columns[1:]
for class_num, row in issues_df.iterrows():
if row["In Given Label"]:
if class_names is not None:
issues_dict[row["Class Index"]]["Class Name"] = row["Class Name"]
issues_dict[row["Class Index"]]["Label Issues"] = int(
row["Issue Probability"] * num_examples
)
issues_dict[row["Class Index"]]["Label Noise"] = row["Issue Probability"]
issues_dict[row["Class Index"]]["Label Quality Score"] = (
1 - issues_dict[row["Class Index"]]["Label Noise"]
)
else:
if class_names is not None:
issues_dict[row["Class Index"]]["Class Name"] = row["Class Name"]
issues_dict[row["Class Index"]]["Inverse Label Issues"] = int(
row["Issue Probability"] * num_examples
)
issues_dict[row["Class Index"]]["Inverse Label Noise"] = row["Issue Probability"]
issues_df_dict = defaultdict(list)
for i in issues_dict:
issues_df_dict["Class Index"].append(i)
for j in issues_dict[i]:
issues_df_dict[j].append(issues_dict[i][j])
return (
pd.DataFrame.from_dict(issues_df_dict)
.sort_values(by="Label Quality Score", ascending=True)
.reset_index(drop=True)
)[return_columns]
def _get_num_examples_multilabel(labels=None, confident_joint: Optional[np.ndarray] = None) -> int:
"""Helper method that finds the number of examples from the parameters or throws an error
if neither parameter is provided.
Parameters
----------
For parameter info, see the docstring of `~cleanlab.multilabel_classification.dataset.common_multilabel_issues`.
Returns
-------
num_examples : int
The number of examples in the dataset.
Raises
------
ValueError
If `labels` is None.
"""
if labels is None and confident_joint is None:
raise ValueError(
"Error: num_examples is None. You must either provide confident_joint, "
"or provide both num_example and joint as input parameters."
)
_confident_joint = cast(np.ndarray, confident_joint)
num_examples = len(labels) if labels is not None else cast(int, np.sum(_confident_joint[0]))
return num_examples
def overall_multilabel_health_score(
labels=None,
pred_probs=None,
*,
confident_joint=None,
) -> float:
"""Returns a single score between 0 and 1 measuring the overall quality of all labels in a multi-label classification dataset.
Intuitively, the score is the average correctness of the given labels across all examples in the
dataset. So a score of 1 suggests your data is perfectly labeled and a score of 0.5 suggests
half of the examples in the dataset may be incorrectly labeled. Thus, a higher
score implies a higher quality dataset.
**Parameters**: For information about the arguments to this method, see the documentation of
`~cleanlab.multilabel_classification.dataset.common_multilabel_issues`.
Returns
-------
health_score : float
A overall score between 0 and 1, where 1 implies all labels in the dataset are estimated to be correct.
A score of 0.5 implies that half of the dataset's labels are estimated to have issues.
"""
num_examples = _get_num_examples_multilabel(labels=labels)
issues = find_label_issues(
labels=labels, pred_probs=pred_probs, confident_joint=confident_joint
)
return 1.0 - sum(issues) / num_examples
def multilabel_health_summary(
labels=None,
pred_probs=None,
*,
class_names=None,
num_examples=None,
confident_joint=None,
verbose=True,
) -> Dict:
"""Prints a health summary of your multi-label dataset.
This summary includes useful statistics like:
* The classes with the most and least label issues.
* Overall label quality scores, summarizing how accurate the labels appear across the entire dataset.
**Parameters**: For information about the arguments to this method, see the documentation of
`~cleanlab.multilabel_classification.dataset.common_multilabel_issues`.
Returns
-------
summary : dict
A dictionary containing keys (see the corresponding functions' documentation to understand the values):
- ``"overall_label_health_score"``, corresponding to output of `~cleanlab.multilabel_classification.dataset.overall_multilabel_health_score`
- ``"classes_by_multilabel_quality"``, corresponding to output of `~cleanlab.multilabel_classification.dataset.rank_classes_by_multilabel_quality`
- ``"common_multilabel_issues"``, corresponding to output of `~cleanlab.multilabel_classification.dataset.common_multilabel_issues`
"""
from cleanlab.internal.util import smart_display_dataframe
if num_examples is None:
num_examples = _get_num_examples_multilabel(labels=labels)
if verbose:
longest_line = f"| for your dataset with {num_examples:,} examples "
print(
"-" * (len(longest_line) - 1)
+ "\n"
+ f"| Generating a Cleanlab Dataset Health Summary{' ' * (len(longest_line) - 49)}|\n"
+ longest_line
+ f"| Note, Cleanlab is not a medical doctor... yet.{' ' * (len(longest_line) - 51)}|\n"
+ "-" * (len(longest_line) - 1)
+ "\n",
)
df_class_label_quality = rank_classes_by_multilabel_quality(
labels=labels,
pred_probs=pred_probs,
class_names=class_names,
confident_joint=confident_joint,
)
if verbose:
print("Overall Class Quality and Noise across your dataset (below)")
print("-" * 60, "\n", flush=True)
smart_display_dataframe(df_class_label_quality)
df_common_issues = common_multilabel_issues(
labels=labels,
pred_probs=pred_probs,
class_names=class_names,
confident_joint=confident_joint,
)
if verbose:
print(
"\nCommon multilabel issues are" + "\n" + "-" * 83 + "\n",
flush=True,
)
smart_display_dataframe(df_common_issues)
print()
health_score = overall_multilabel_health_score(
labels=labels,
pred_probs=pred_probs,
confident_joint=confident_joint,
)
if verbose:
print("\nGenerated with <3 from Cleanlab.\n")
return {
"overall_multilabel_health_score": health_score,
"classes_by_multilabel_quality": df_class_label_quality,
"common_multilabel_issues": df_common_issues,
}
cleanlab/multilabel_classification/filter.py
+303
View File
@@ -0,0 +1,303 @@
"""
Methods to flag which examples have label issues in multi-label classification datasets.
Here each example can belong to one or more classes, or none of the classes at all.
Unlike in standard multi-class classification, model-predicted class probabilities need not sum to 1 for each row in multi-label classification.
"""
import warnings
import inspect
from typing import Optional, Union, Tuple, List, Any
import numpy as np
def find_label_issues(
labels: list,
pred_probs: np.ndarray,
return_indices_ranked_by: Optional[str] = None,
rank_by_kwargs={},
filter_by: str = "prune_by_noise_rate",
frac_noise: float = 1.0,
num_to_remove_per_class: Optional[List[int]] = None,
min_examples_per_class=1,
confident_joint: Optional[np.ndarray] = None,
n_jobs: Optional[int] = None,
verbose: bool = False,
low_memory: bool = False,
) -> np.ndarray:
"""
Identifies potentially mislabeled examples in a multi-label classification dataset.
An example is flagged as with a label issue if *any* of the classes appear to be incorrectly annotated for this example.
Parameters
----------
labels : List[List[int]]
List of noisy labels for multi-label classification where each example can belong to multiple classes.
This is an iterable of iterables where the i-th element of `labels` corresponds to a list of classes that the i-th example belongs to,
according to the original data annotation (e.g. ``labels = [[1,2],[1],[0],..]``).
This method will return the indices i where the inner list ``labels[i]`` is estimated to have some error.
For a dataset with K classes, each class must be represented as an integer in 0, 1, ..., K-1 within the labels.
pred_probs : np.ndarray
An array of shape ``(N, K)`` of model-predicted class probabilities.
Each row of this matrix corresponds to an example `x`
and contains the predicted probability that `x` belongs to each possible class,
for each of the K classes (along its columns).
The columns need not sum to 1 but must be ordered such that
these probabilities correspond to class 0, 1, ..., K-1.
Note
----
Estimated label quality scores are most accurate when they are computed based on out-of-sample ``pred_probs`` from your model.
To obtain out-of-sample predicted probabilities for every example in your dataset, you can use :ref:`cross-validation <pred_probs_cross_val>`.
This is encouraged to get better results.
return_indices_ranked_by : {None, 'self_confidence', 'normalized_margin', 'confidence_weighted_entropy'}, default = None
This function can return a boolean mask (if None) or an array of the example-indices with issues sorted based on the specified ranking method.
Refer to documentation for this argument in :py:func:`filter.find_label_issues <cleanlab.filter.find_label_issues>` for details.
rank_by_kwargs : dict, optional
Optional keyword arguments to pass into scoring functions for ranking by
label quality score (see :py:func:`rank.get_label_quality_scores
<cleanlab.rank.get_label_quality_scores>`).
filter_by : {'prune_by_class', 'prune_by_noise_rate', 'both', 'confident_learning', 'predicted_neq_given', 'low_normalized_margin', 'low_self_confidence'}, default='prune_by_noise_rate'
The specific Confident Learning method to determine precisely which examples have label issues in a dataset.
Refer to documentation for this argument in :py:func:`filter.find_label_issues <cleanlab.filter.find_label_issues>` for details.
frac_noise : float, default = 1.0
This will return the "top" frac_noise * num_label_issues estimated label errors, dependent on the filtering method used,
Refer to documentation for this argument in :py:func:`filter.find_label_issues <cleanlab.filter.find_label_issues>` for details.
num_to_remove_per_class : array_like
An iterable that specifies the number of mislabeled examples to return from each class.
Refer to documentation for this argument in :py:func:`filter.find_label_issues <cleanlab.filter.find_label_issues>` for details.
min_examples_per_class : int, default = 1
The minimum number of examples required per class below which examples from this class will not be flagged as label issues.
Refer to documentation for this argument in :py:func:`filter.find_label_issues <cleanlab.filter.find_label_issues>` for details.
confident_joint : np.ndarray, optional
An array of shape ``(K, 2, 2)`` representing a one-vs-rest formatted confident joint, as is appropriate for multi-label classification tasks.
Entry ``(c, i, j)`` in this array is the number of examples confidently counted into a ``(class c, noisy label=i, true label=j)`` bin,
where `i, j` are either 0 or 1 to denote whether this example belongs to class `c` or not
(recall examples can belong to multiple classes in multi-label classification).
The `confident_joint` can be computed using :py:func:`count.compute_confident_joint <cleanlab.count.compute_confident_joint>` with ``multi_label=True``.
If not provided, it is computed from the given (noisy) `labels` and `pred_probs`.
n_jobs : optional
Number of processing threads used by multiprocessing.
Refer to documentation for this argument in :py:func:`filter.find_label_issues <cleanlab.filter.find_label_issues>` for details.
verbose : optional
If ``True``, prints when multiprocessing happens.
low_memory: bool, default=False
Set as ``True`` if you have a big dataset with limited memory.
Uses :py:func:`experimental.label_issues_batched.find_label_issues_batched <cleanlab.experimental.label_issues_batched>`
Returns
-------
label_issues : np.ndarray
If `return_indices_ranked_by` left unspecified, returns a boolean **mask** for the entire dataset
where ``True`` represents an example suffering from some label issue and
``False`` represents an example that appears accurately labeled.
If `return_indices_ranked_by` is specified, this method instead returns a list of **indices** of examples identified with
label issues (i.e. those indices where the mask would be ``True``).
Indices are sorted by the likelihood that *all* classes are correctly annotated for the corresponding example.
Note
----
Obtain the *indices* of examples with label issues in your dataset by setting
`return_indices_ranked_by`.
"""
from cleanlab.filter import _find_label_issues_multilabel
if low_memory:
if rank_by_kwargs:
warnings.warn(f"`rank_by_kwargs` is not used when `low_memory=True`.")
func_signature = inspect.signature(find_label_issues)
default_args = {
k: v.default
for k, v in func_signature.parameters.items()
if v.default is not inspect.Parameter.empty
}
arg_values = {
"filter_by": filter_by,
"num_to_remove_per_class": num_to_remove_per_class,
"confident_joint": confident_joint,
"n_jobs": n_jobs,
"num_to_remove_per_class": num_to_remove_per_class,
"frac_noise": frac_noise,
"min_examples_per_class": min_examples_per_class,
}
for arg_name, arg_val in arg_values.items():
if arg_val != default_args[arg_name]:
warnings.warn(f"`{arg_name}` is not used when `low_memory=True`.")
return _find_label_issues_multilabel(
labels=labels,
pred_probs=pred_probs,
return_indices_ranked_by=return_indices_ranked_by,
rank_by_kwargs=rank_by_kwargs,
filter_by=filter_by,
frac_noise=frac_noise,
num_to_remove_per_class=num_to_remove_per_class,
min_examples_per_class=min_examples_per_class,
confident_joint=confident_joint,
n_jobs=n_jobs,
verbose=verbose,
low_memory=low_memory,
)
def find_multilabel_issues_per_class(
labels: list,
pred_probs: np.ndarray,
return_indices_ranked_by: Optional[str] = None,
rank_by_kwargs={},
filter_by: str = "prune_by_noise_rate",
frac_noise: float = 1.0,
num_to_remove_per_class: Optional[List[int]] = None,
min_examples_per_class=1,
confident_joint: Optional[np.ndarray] = None,
n_jobs: Optional[int] = None,
verbose: bool = False,
low_memory: bool = False,
) -> Union[np.ndarray, Tuple[List[np.ndarray], List[Any], List[np.ndarray]]]:
"""
Identifies potentially bad labels for each example and each class in a multi-label classification dataset.
Whereas `~cleanlab.multilabel_classification.filter.find_label_issues`
estimates which examples have an erroneous annotation for *any* class, this method estimates which specific classes are incorrectly annotated as well.
This method returns a list of size K, the number of classes in the dataset.
Parameters
----------
labels : List[List[int]]
List of noisy labels for multi-label classification where each example can belong to multiple classes.
Refer to documentation for this argument in `~cleanlab.multilabel_classification.filter.find_label_issues` for further details.
This method will identify whether ``labels[i][k]`` appears correct, for every example ``i`` and class ``k``.
pred_probs : np.ndarray
An array of shape ``(N, K)`` of model-predicted class probabilities.
Refer to documentation for this argument in `~cleanlab.multilabel_classification.filter.find_label_issues` for further details.
return_indices_ranked_by : {None, 'self_confidence', 'normalized_margin', 'confidence_weighted_entropy'}, default = None
This function can return a boolean mask (if this argument is ``None``) or a sorted array of indices based on the specified ranking method (if not ``None``).
Refer to documentation for this argument in :py:func:`filter.find_label_issues <cleanlab.filter.find_label_issues>` for details.
rank_by_kwargs : dict, optional
Optional keyword arguments to pass into scoring functions for ranking by.
label quality score (see :py:func:`rank.get_label_quality_scores
<cleanlab.rank.get_label_quality_scores>`).
filter_by : {'prune_by_class', 'prune_by_noise_rate', 'both', 'confident_learning', 'predicted_neq_given', 'low_normalized_margin', 'low_self_confidence'}, default = 'prune_by_noise_rate'
The specific method that can be used to filter or prune examples with label issues from a dataset.
Refer to documentation for this argument in :py:func:`filter.find_label_issues <cleanlab.filter.find_label_issues>` for details.
frac_noise : float, default = 1.0
This will return the "top" frac_noise * num_label_issues estimated label errors, dependent on the filtering method used,
Refer to documentation for this argument in :py:func:`filter.find_label_issues <cleanlab.filter.find_label_issues>` for details.
num_to_remove_per_class : array_like
This parameter is an iterable that specifies the number of mislabeled examples to return from each class.
Refer to documentation for this argument in :py:func:`filter.find_label_issues <cleanlab.filter.find_label_issues>` for details.
min_examples_per_class : int, default = 1
The minimum number of examples required per class to avoid flagging as label issues.
Refer to documentation for this argument in :py:func:`filter.find_label_issues <cleanlab.filter.find_label_issues>` for details.
confident_joint : np.ndarray, optional
An array of shape ``(K, 2, 2)`` representing a one-vs-rest formatted confident joint.
Refer to documentation for this argument in `~cleanlab.multilabel_classification.filter.find_label_issues` for details.
n_jobs : optional
Number of processing threads used by multiprocessing.
Refer to documentation for this argument in :py:func:`filter.find_label_issues <cleanlab.filter.find_label_issues>` for details.
verbose : optional
If ``True``, prints when multiprocessing happens.
Returns
-------
per_class_label_issues : list(np.ndarray)
By default, this is a list of length K containing the examples where each class appears incorrectly annotated.
``per_class_label_issues[k]`` is a Boolean mask of the same length as the dataset,
where ``True`` values indicate examples where class ``k`` appears incorrectly annotated.
For more details, refer to `~cleanlab.multilabel_classification.filter.find_label_issues`.
Otherwise if `return_indices_ranked_by` is not ``None``, then this method returns 3 objects (each of length K, the number of classes): `label_issues_list`, `labels_list`, `pred_probs_list`.
- *label_issues_list*: an ordered list of indices of examples where class k appears incorrectly annotated, sorted by the likelihood that class k is correctly annotated.
- *labels_list*: a binary one-hot representation of the original labels, useful if you want to compute label quality scores.
- *pred_probs_list*: a one-vs-rest representation of the original predicted probabilities of shape ``(N, 2)``, useful if you want to compute label quality scores.
``pred_probs_list[k][i][0]`` is the estimated probability that example ``i`` belongs to class ``k``, and is equal to: ``1 - pred_probs_list[k][i][1]``.
"""
import cleanlab.filter
from cleanlab.internal.multilabel_utils import get_onehot_num_classes, stack_complement
from cleanlab.experimental.label_issues_batched import find_label_issues_batched
y_one, num_classes = get_onehot_num_classes(labels, pred_probs)
if return_indices_ranked_by is None:
bissues = np.zeros(y_one.shape).astype(bool)
else:
label_issues_list = []
labels_list = []
pred_probs_list = []
if confident_joint is not None and not low_memory:
confident_joint_shape = confident_joint.shape
if confident_joint_shape == (num_classes, num_classes):
warnings.warn(
f"The new recommended format for `confident_joint` in multi_label settings is (num_classes,2,2) as output by compute_confident_joint(...,multi_label=True). Your K x K confident_joint in the old format is being ignored."
)
confident_joint = None
elif confident_joint_shape != (num_classes, 2, 2):
raise ValueError("confident_joint should be of shape (num_classes, 2, 2)")
for class_num, (label, pred_prob_for_class) in enumerate(zip(y_one.T, pred_probs.T)):
pred_probs_binary = stack_complement(pred_prob_for_class)
if low_memory:
quality_score_kwargs = (
{"method": return_indices_ranked_by} if return_indices_ranked_by else None
)
binary_label_issues = find_label_issues_batched(
labels=label,
pred_probs=pred_probs_binary,
verbose=verbose,
quality_score_kwargs=quality_score_kwargs,
return_mask=return_indices_ranked_by is None,
)
else:
if confident_joint is None:
conf = None
else:
conf = confident_joint[class_num]
if num_to_remove_per_class is not None:
ml_num_to_remove_per_class = [num_to_remove_per_class[class_num], 0]
else:
ml_num_to_remove_per_class = None
binary_label_issues = cleanlab.filter.find_label_issues(
labels=label,
pred_probs=pred_probs_binary,
return_indices_ranked_by=return_indices_ranked_by,
frac_noise=frac_noise,
rank_by_kwargs=rank_by_kwargs,
filter_by=filter_by,
num_to_remove_per_class=ml_num_to_remove_per_class,
min_examples_per_class=min_examples_per_class,
confident_joint=conf,
n_jobs=n_jobs,
verbose=verbose,
)
if return_indices_ranked_by is None:
bissues[:, class_num] = binary_label_issues
else:
label_issues_list.append(binary_label_issues)
labels_list.append(label)
pred_probs_list.append(pred_probs_binary)
if return_indices_ranked_by is None:
return bissues
else:
return label_issues_list, labels_list, pred_probs_list
cleanlab/multilabel_classification/rank.py
+179
View File
@@ -0,0 +1,179 @@
"""
Methods to rank the severity of label issues in multi-label classification datasets.
Here each example can belong to one or more classes, or none of the classes at all.
Unlike in standard multi-class classification, model-predicted class probabilities need not sum to 1 for each row in multi-label classification.
"""
from __future__ import annotations
import numpy as np # noqa: F401: Imported for type annotations
from typing import List, TypeVar, Dict, Any, Optional, Tuple, TYPE_CHECKING
from cleanlab.internal.validation import assert_valid_inputs
from cleanlab.internal.util import get_num_classes
from cleanlab.internal.multilabel_utils import int2onehot
from cleanlab.internal.multilabel_scorer import MultilabelScorer, ClassLabelScorer, Aggregator
if TYPE_CHECKING: # pragma: no cover
import numpy.typing as npt
T = TypeVar("T", bound=npt.NBitBase)
def _labels_to_binary(
labels: List[List[int]],
pred_probs: npt.NDArray["np.floating[T]"],
) -> np.ndarray:
"""Validate the inputs to the multilabel scorer. Also transform the labels to a binary representation."""
assert_valid_inputs(
X=None, y=labels, pred_probs=pred_probs, multi_label=True, allow_one_class=True
)
num_classes = get_num_classes(labels=labels, pred_probs=pred_probs, multi_label=True)
binary_labels = int2onehot(labels, K=num_classes)
return binary_labels
def _create_multilabel_scorer(
method: str,
adjust_pred_probs: bool,
aggregator_kwargs: Optional[Dict[str, Any]] = None,
) -> Tuple[MultilabelScorer, Dict]:
"""This function acts as a factory that creates a MultilabelScorer."""
base_scorer = ClassLabelScorer.from_str(method)
base_scorer_kwargs = {"adjust_pred_probs": adjust_pred_probs}
if aggregator_kwargs:
aggregator = Aggregator(**aggregator_kwargs)
scorer = MultilabelScorer(base_scorer, aggregator)
else:
scorer = MultilabelScorer(base_scorer)
return scorer, base_scorer_kwargs
def get_label_quality_scores(
labels: List[List[int]],
pred_probs: npt.NDArray["np.floating[T]"],
*,
method: str = "self_confidence",
adjust_pred_probs: bool = False,
aggregator_kwargs: Dict[str, Any] = {"method": "exponential_moving_average", "alpha": 0.8},
) -> npt.NDArray["np.floating[T]"]:
"""Computes a label quality score for each example in a multi-label classification dataset.
Scores are between 0 and 1 with lower scores indicating examples whose label more likely contains an error.
For each example, this method internally computes a separate score for each individual class
and then aggregates these per-class scores into an overall label quality score for the example.
Parameters
----------
labels : List[List[int]]
List of noisy labels for multi-label classification where each example can belong to multiple classes.
Refer to documentation for this argument in :py:func:`multilabel_classification.filter.find_label_issues <cleanlab.multilabel_classification.filter.find_label_issues>` for further details.
pred_probs : np.ndarray
An array of shape ``(N, K)`` of model-predicted class probabilities.
Refer to documentation for this argument in :py:func:`multilabel_classification.filter.find_label_issues <cleanlab.multilabel_classification.filter.find_label_issues>` for further details.
method : {"self_confidence", "normalized_margin", "confidence_weighted_entropy"}, default = "self_confidence"
Method to calculate separate per-class annotation scores for an example that are then aggregated into an overall label quality score for the example.
These scores are separately calculated for each class based on the corresponding column of `pred_probs` in a one-vs-rest manner,
and are standard label quality scores for binary classification (based on whether the class should or should not apply to this example).
See also
--------
:py:func:`rank.get_label_quality_scores <cleanlab.rank.get_label_quality_scores>` function for details about each option.
adjust_pred_probs : bool, default = False
Account for class imbalance in the label-quality scoring by adjusting predicted probabilities.
Refer to documentation for this argument in :py:func:`rank.get_label_quality_scores <cleanlab.rank.get_label_quality_scores>` for details.
aggregator_kwargs : dict, default = {"method": "exponential_moving_average", "alpha": 0.8}
A dictionary of hyperparameter values to use when aggregating per-class scores into an overall label quality score for each example.
Options for ``"method"`` include: ``"exponential_moving_average"`` or ``"softmin"`` or your own callable function.
See :py:class:`internal.multilabel_scorer.Aggregator <cleanlab.internal.multilabel_scorer.Aggregator>` for details about each option and other possible hyperparameters.
To get a score for each class annotation for each example, use the `~cleanlab.multilabel_classification.rank.get_label_quality_scores_per_class` method instead.
Returns
-------
label_quality_scores : np.ndarray
A 1D array of shape ``(N,)`` with a label quality score (between 0 and 1) for each example in the dataset.
Lower scores indicate examples whose label is more likely to contain some annotation error (for any of the classes).
Examples
--------
>>> from cleanlab.multilabel_classification import get_label_quality_scores
>>> import numpy as np
>>> labels = [[1], [0,2]]
>>> pred_probs = np.array([[0.1, 0.9, 0.1], [0.4, 0.1, 0.9]])
>>> scores = get_label_quality_scores(labels, pred_probs)
>>> scores
array([0.9, 0.5])
"""
binary_labels = _labels_to_binary(labels, pred_probs)
scorer, base_scorer_kwargs = _create_multilabel_scorer(
method=method,
adjust_pred_probs=adjust_pred_probs,
aggregator_kwargs=aggregator_kwargs,
)
return scorer(binary_labels, pred_probs, base_scorer_kwargs=base_scorer_kwargs)
def get_label_quality_scores_per_class(
labels: List[List[int]],
pred_probs: npt.NDArray["np.floating[T]"],
*,
method: str = "self_confidence",
adjust_pred_probs: bool = False,
) -> np.ndarray:
"""
Computes a quality score quantifying how likely each individual class annotation is correct in a multi-label classification dataset.
This is similar to `~cleanlab.multilabel_classification.rank.get_label_quality_scores`
but instead returns the per-class results without aggregation.
For a dataset with K classes, each example receives K scores from this method.
Refer to documentation in `~cleanlab.multilabel_classification.rank.get_label_quality_scores` for details.
Parameters
----------
labels : List[List[int]]
List of noisy labels for multi-label classification where each example can belong to multiple classes.
Refer to documentation for this argument in :py:func:`find_label_issues <cleanlab.multilabel_classification.filter.find_label_issues>` for further details.
pred_probs : np.ndarray
An array of shape ``(N, K)`` of model-predicted class probabilities.
Refer to documentation for this argument in :py:func:`find_label_issues <cleanlab.multilabel_classification.filter.find_label_issues>` for further details.
method : {"self_confidence", "normalized_margin", "confidence_weighted_entropy"}, default = "self_confidence"
Method to calculate separate per-class annotation scores (that quantify how likely a particular class annotation is correct for a particular example).
Refer to documentation for this argument in `~cleanlab.multilabel_classification.rank.get_label_quality_scores` for further details.
adjust_pred_probs : bool, default = False
Account for class imbalance in the label-quality scoring by adjusting predicted probabilities.
Refer to documentation for this argument in :py:func:`rank.get_label_quality_scores <cleanlab.rank.get_label_quality_scores>` for details.
Returns
-------
label_quality_scores : list(np.ndarray)
A list containing K arrays, each of shape (N,). Here K is the number of classes in the dataset and N is the number of examples.
``label_quality_scores[k][i]`` is a score between 0 and 1 quantifying how likely the annotation for class ``k`` is correct for example ``i``.
Examples
--------
>>> from cleanlab.multilabel_classification import get_label_quality_scores
>>> import numpy as np
>>> labels = [[1], [0,2]]
>>> pred_probs = np.array([[0.1, 0.9, 0.1], [0.4, 0.1, 0.9]])
>>> scores = get_label_quality_scores(labels, pred_probs)
>>> scores
array([0.9, 0.5])
"""
binary_labels = _labels_to_binary(labels, pred_probs)
scorer, base_scorer_kwargs = _create_multilabel_scorer(
method=method,
adjust_pred_probs=adjust_pred_probs,
)
return scorer.get_class_label_quality_scores(
labels=binary_labels, pred_probs=pred_probs, base_scorer_kwargs=base_scorer_kwargs
)