chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,3 @@
|
||||
from . import rank
|
||||
from . import filter
|
||||
from . import summary
|
||||
@@ -0,0 +1,101 @@
|
||||
"""
|
||||
Methods to find label issues in token classification datasets (text data), where each token in a sentence receives its own class label.
|
||||
|
||||
The underlying algorithms are described in `this paper <https://arxiv.org/abs/2210.03920>`_.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
from typing import List, Tuple
|
||||
import warnings
|
||||
|
||||
from cleanlab.filter import find_label_issues as find_label_issues_main
|
||||
from cleanlab.experimental.label_issues_batched import find_label_issues_batched
|
||||
|
||||
|
||||
def find_label_issues(
|
||||
labels: list,
|
||||
pred_probs: list,
|
||||
*,
|
||||
return_indices_ranked_by: str = "self_confidence",
|
||||
low_memory: bool = False,
|
||||
**kwargs,
|
||||
) -> List[Tuple[int, int]]:
|
||||
"""Identifies tokens with label issues in a token classification dataset.
|
||||
|
||||
Tokens identified with issues will be ranked by their individual label quality score.
|
||||
|
||||
Instead use :py:func:`token_classification.rank.get_label_quality_scores <cleanlab.token_classification.rank.get_label_quality_scores>`
|
||||
if you prefer to rank the sentences based on their overall label quality.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
labels:
|
||||
Nested list of given labels for all tokens, such that `labels[i]` is a list of labels, one for each token in the `i`-th sentence.
|
||||
|
||||
For a dataset with K classes, each class label must be integer in 0, 1, ..., K-1.
|
||||
|
||||
pred_probs:
|
||||
List of np arrays, such that `pred_probs[i]` has shape ``(T, K)`` if the `i`-th sentence contains T tokens.
|
||||
|
||||
Each row of `pred_probs[i]` corresponds to a token `t` in the `i`-th sentence,
|
||||
and contains model-predicted probabilities that `t` belongs to each of the K possible classes.
|
||||
|
||||
Columns of each `pred_probs[i]` should be ordered such that the probabilities correspond to class 0, 1, ..., K-1.
|
||||
|
||||
return_indices_ranked_by: {"self_confidence", "normalized_margin", "confidence_weighted_entropy"}, default="self_confidence"
|
||||
Returned token-indices are sorted by their label quality score.
|
||||
|
||||
See :py:func:`cleanlab.filter.find_label_issues <cleanlab.filter.find_label_issues>`
|
||||
documentation for more details on each label quality scoring method.
|
||||
|
||||
kwargs:
|
||||
Additional keyword arguments to pass into :py:func:`filter.find_label_issues <cleanlab.filter.find_label_issues>`
|
||||
which is internally applied at the token level. Can include values like `n_jobs` to control parallel processing, `frac_noise`, etc.
|
||||
|
||||
Returns
|
||||
-------
|
||||
issues:
|
||||
List of label issues identified by cleanlab, such that each element is a tuple ``(i, j)``, which
|
||||
indicates that the `j`-th token of the `i`-th sentence has a label issue.
|
||||
|
||||
These tuples are ordered in `issues` list based on the likelihood that the corresponding token is mislabeled.
|
||||
|
||||
Use :py:func:`token_classification.summary.display_issues <cleanlab.token_classification.summary.display_issues>`
|
||||
to view these issues within the original sentences.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> import numpy as np
|
||||
>>> from cleanlab.token_classification.filter import find_label_issues
|
||||
>>> labels = [[0, 0, 1], [0, 1]]
|
||||
>>> pred_probs = [
|
||||
... np.array([[0.9, 0.1], [0.7, 0.3], [0.05, 0.95]]),
|
||||
... np.array([[0.8, 0.2], [0.8, 0.2]]),
|
||||
... ]
|
||||
>>> find_label_issues(labels, pred_probs)
|
||||
[(1, 1)]
|
||||
"""
|
||||
labels_flatten = [l for label in labels for l in label]
|
||||
pred_probs_flatten = np.array([pred for pred_prob in pred_probs for pred in pred_prob])
|
||||
|
||||
if low_memory:
|
||||
for arg_name, _ in kwargs.items():
|
||||
warnings.warn(f"`{arg_name}` is not used when `low_memory=True`.")
|
||||
quality_score_kwargs = {"method": return_indices_ranked_by}
|
||||
issues_main = find_label_issues_batched(
|
||||
labels_flatten, pred_probs_flatten, quality_score_kwargs=quality_score_kwargs
|
||||
)
|
||||
else:
|
||||
issues_main = find_label_issues_main(
|
||||
labels_flatten,
|
||||
pred_probs_flatten,
|
||||
return_indices_ranked_by=return_indices_ranked_by,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
lengths = [len(label) for label in labels]
|
||||
mapping = [[(i, j) for j in range(length)] for i, length in enumerate(lengths)]
|
||||
mapping_flatten = [index for indicies in mapping for index in indicies]
|
||||
|
||||
issues = [mapping_flatten[issue] for issue in issues_main]
|
||||
return issues
|
||||
@@ -0,0 +1,274 @@
|
||||
"""
|
||||
Methods to rank and score sentences in a token classification dataset (text data), based on how likely they are to contain label errors.
|
||||
|
||||
The underlying algorithms are described in `this paper <https://arxiv.org/abs/2210.03920>`_.
|
||||
"""
|
||||
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from typing import List, Optional, Union, Tuple
|
||||
|
||||
from cleanlab.rank import get_label_quality_scores as main_get_label_quality_scores
|
||||
from cleanlab.internal.numerics import softmax
|
||||
|
||||
|
||||
def get_label_quality_scores(
|
||||
labels: list,
|
||||
pred_probs: list,
|
||||
*,
|
||||
tokens: Optional[list] = None,
|
||||
token_score_method: str = "self_confidence",
|
||||
sentence_score_method: str = "min",
|
||||
sentence_score_kwargs: dict = {},
|
||||
) -> Tuple[np.ndarray, list]:
|
||||
"""
|
||||
Returns overall quality scores for the labels in each sentence, as well as for the individual tokens' labels in a token classification dataset.
|
||||
|
||||
Each score is between 0 and 1.
|
||||
|
||||
Lower scores indicate token labels that are less likely to be correct, or sentences that are more likely to contain a mislabeled token.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
labels:
|
||||
Nested list of given labels for all tokens, such that `labels[i]` is a list of labels, one for each token in the `i`-th sentence.
|
||||
|
||||
For a dataset with K classes, each label must be in 0, 1, ..., K-1.
|
||||
|
||||
pred_probs:
|
||||
List of np arrays, such that `pred_probs[i]` has shape ``(T, K)`` if the `i`-th sentence contains T tokens.
|
||||
|
||||
Each row of `pred_probs[i]` corresponds to a token `t` in the `i`-th sentence,
|
||||
and contains model-predicted probabilities that `t` belongs to each of the K possible classes.
|
||||
|
||||
Columns of each `pred_probs[i]` should be ordered such that the probabilities correspond to class 0, 1, ..., K-1.
|
||||
|
||||
tokens:
|
||||
Nested list such that `tokens[i]` is a list of tokens (strings/words) that comprise the `i`-th sentence.
|
||||
|
||||
These strings are used to annotated the returned `token_scores` object, see its documentation for more information.
|
||||
|
||||
sentence_score_method: {"min", "softmin"}, default="min"
|
||||
Method to aggregate individual token label quality scores into a single score for the sentence.
|
||||
|
||||
- `min`: sentence score = minimum of token scores in the sentence
|
||||
- `softmin`: sentence score = ``<s, softmax(1-s, t)>``, where `s` denotes the token label scores of the sentence, and ``<a, b> == np.dot(a, b)``.
|
||||
Here parameter `t` controls the softmax temperature, such that the score converges toward `min` as ``t -> 0``.
|
||||
Unlike `min`, `softmin` is affected by the scores of all tokens in the sentence.
|
||||
|
||||
token_score_method: {"self_confidence", "normalized_margin", "confidence_weighted_entropy"}, default="self_confidence"
|
||||
Label quality scoring method for each token.
|
||||
|
||||
See :py:func:`cleanlab.rank.get_label_quality_scores <cleanlab.rank.get_label_quality_scores>` documentation for more info.
|
||||
|
||||
sentence_score_kwargs:
|
||||
Optional keyword arguments for `sentence_score_method` function (for advanced users only).
|
||||
|
||||
See `~cleanlab.token_classification.rank._softmin_sentence_score` for more info about keyword arguments supported for that scoring method.
|
||||
|
||||
Returns
|
||||
-------
|
||||
sentence_scores:
|
||||
Array of shape ``(N, )`` of scores between 0 and 1, one per sentence in the dataset.
|
||||
|
||||
Lower scores indicate sentences more likely to contain a label issue.
|
||||
|
||||
token_scores:
|
||||
List of ``pd.Series``, such that `token_info[i]` contains the
|
||||
label quality scores for individual tokens in the `i`-th sentence.
|
||||
|
||||
If `tokens` strings were provided, they are used as index for each ``Series``.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> import numpy as np
|
||||
>>> from cleanlab.token_classification.rank import get_label_quality_scores
|
||||
>>> labels = [[0, 0, 1], [0, 1]]
|
||||
>>> pred_probs = [
|
||||
... np.array([[0.9, 0.1], [0.7, 0.3], [0.05, 0.95]]),
|
||||
... np.array([[0.8, 0.2], [0.8, 0.2]]),
|
||||
... ]
|
||||
>>> sentence_scores, token_scores = get_label_quality_scores(labels, pred_probs)
|
||||
>>> sentence_scores
|
||||
array([0.7, 0.2])
|
||||
>>> token_scores
|
||||
[0 0.90
|
||||
1 0.70
|
||||
2 0.95
|
||||
dtype: float64, 0 0.8
|
||||
1 0.2
|
||||
dtype: float64]
|
||||
"""
|
||||
methods = ["min", "softmin"]
|
||||
assert sentence_score_method in methods, "Select from the following methods:\n%s" % "\n".join(
|
||||
methods
|
||||
)
|
||||
|
||||
labels_flatten = np.array([l for label in labels for l in label])
|
||||
pred_probs_flatten = np.array([p for pred_prob in pred_probs for p in pred_prob])
|
||||
|
||||
sentence_length = [len(label) for label in labels]
|
||||
|
||||
def nested_list(x, sentence_length):
|
||||
i = iter(x)
|
||||
return [[next(i) for _ in range(length)] for length in sentence_length]
|
||||
|
||||
token_scores = main_get_label_quality_scores(
|
||||
labels=labels_flatten, pred_probs=pred_probs_flatten, method=token_score_method
|
||||
)
|
||||
scores_nl = nested_list(token_scores, sentence_length)
|
||||
|
||||
if sentence_score_method == "min":
|
||||
sentence_scores = np.array(list(map(np.min, scores_nl)))
|
||||
else:
|
||||
assert sentence_score_method == "softmin"
|
||||
temperature = sentence_score_kwargs.get("temperature", 0.05)
|
||||
sentence_scores = _softmin_sentence_score(scores_nl, temperature=temperature)
|
||||
|
||||
if tokens:
|
||||
token_info = [pd.Series(scores, index=token) for scores, token in zip(scores_nl, tokens)]
|
||||
else:
|
||||
token_info = [pd.Series(scores) for scores in scores_nl]
|
||||
return sentence_scores, token_info
|
||||
|
||||
|
||||
def issues_from_scores(
|
||||
sentence_scores: np.ndarray, *, token_scores: Optional[list] = None, threshold: float = 0.1
|
||||
) -> Union[list, np.ndarray]:
|
||||
"""
|
||||
Converts scores output by `~cleanlab.token_classification.rank.get_label_quality_scores`
|
||||
to a list of issues of similar format as output by :py:func:`token_classification.filter.find_label_issues <cleanlab.token_classification.filter.find_label_issues>`.
|
||||
|
||||
Issues are sorted by label quality score, from most to least severe.
|
||||
|
||||
Only considers as issues those tokens with label quality score lower than `threshold`,
|
||||
so this parameter determines the number of issues that are returned.
|
||||
This method is intended for converting the most severely mislabeled examples to a format compatible with
|
||||
``summary`` methods like :py:func:`token_classification.summary.display_issues <cleanlab.token_classification.summary.display_issues>`.
|
||||
This method does not estimate the number of label errors since the `threshold` is arbitrary,
|
||||
for that instead use :py:func:`token_classification.filter.find_label_issues <cleanlab.token_classification.filter.find_label_issues>`,
|
||||
which estimates the label errors via Confident Learning rather than score thresholding.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
sentence_scores:
|
||||
Array of shape `(N, )` of overall sentence scores, where `N` is the number of sentences in the dataset.
|
||||
|
||||
Same format as the `sentence_scores` returned by `~cleanlab.token_classification.rank.get_label_quality_scores`.
|
||||
|
||||
token_scores:
|
||||
Optional list such that `token_scores[i]` contains the individual token scores for the `i`-th sentence.
|
||||
|
||||
Same format as the `token_scores` returned by `~cleanlab.token_classification.rank.get_label_quality_scores`.
|
||||
|
||||
threshold:
|
||||
Tokens (or sentences, if `token_scores` is not provided) with quality scores above the `threshold` are not
|
||||
included in the result.
|
||||
|
||||
Returns
|
||||
---------
|
||||
issues:
|
||||
List of label issues identified by comparing quality scores to threshold, such that each element is a tuple ``(i, j)``, which
|
||||
indicates that the `j`-th token of the `i`-th sentence has a label issue.
|
||||
|
||||
These tuples are ordered in `issues` list based on the token label quality score.
|
||||
|
||||
Use :py:func:`token_classification.summary.display_issues <cleanlab.token_classification.summary.display_issues>`
|
||||
to view these issues within the original sentences.
|
||||
|
||||
If `token_scores` is not provided, returns array of integer indices (rather than tuples) of the sentences whose label quality score
|
||||
falls below the `threshold` (also sorted by overall label quality score of each sentence).
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> import numpy as np
|
||||
>>> from cleanlab.token_classification.rank import issues_from_scores
|
||||
>>> sentence_scores = np.array([0.1, 0.3, 0.6, 0.2, 0.05, 0.9, 0.8, 0.0125, 0.5, 0.6])
|
||||
>>> issues_from_scores(sentence_scores)
|
||||
array([7, 4])
|
||||
|
||||
Changing the score threshold
|
||||
|
||||
>>> issues_from_scores(sentence_scores, threshold=0.5)
|
||||
array([7, 4, 0, 3, 1])
|
||||
|
||||
Providing token scores along with sentence scores finds issues at the token level
|
||||
|
||||
>>> token_scores = [
|
||||
... [0.9, 0.6],
|
||||
... [0.0, 0.8, 0.8],
|
||||
... [0.8, 0.8],
|
||||
... [0.1, 0.02, 0.3, 0.4],
|
||||
... [0.1, 0.2, 0.03, 0.4],
|
||||
... [0.1, 0.2, 0.3, 0.04],
|
||||
... [0.1, 0.2, 0.4],
|
||||
... [0.3, 0.4],
|
||||
... [0.08, 0.2, 0.5, 0.4],
|
||||
... [0.1, 0.2, 0.3, 0.4],
|
||||
... ]
|
||||
>>> issues_from_scores(sentence_scores, token_scores=token_scores)
|
||||
[(1, 0), (3, 1), (4, 2), (5, 3), (8, 0)]
|
||||
"""
|
||||
if token_scores:
|
||||
issues_with_scores = []
|
||||
for sentence_index, scores in enumerate(token_scores):
|
||||
for token_index, score in enumerate(scores):
|
||||
if score < threshold:
|
||||
issues_with_scores.append((sentence_index, token_index, score))
|
||||
|
||||
issues_with_scores = sorted(issues_with_scores, key=lambda x: x[2])
|
||||
issues = [(i, j) for i, j, _ in issues_with_scores]
|
||||
return issues
|
||||
|
||||
else:
|
||||
ranking = np.argsort(sentence_scores)
|
||||
cutoff = 0
|
||||
while sentence_scores[ranking[cutoff]] < threshold and cutoff < len(ranking):
|
||||
cutoff += 1
|
||||
return ranking[:cutoff]
|
||||
|
||||
|
||||
def _softmin_sentence_score(
|
||||
token_scores: List[np.ndarray], *, temperature: float = 0.05
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Sentence overall label quality scoring using the "softmin" method.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
token_scores:
|
||||
Per-token label quality scores in nested list format,
|
||||
where `token_scores[i]` is a list of scores for each toke in the i'th sentence.
|
||||
|
||||
temperature:
|
||||
Temperature of the softmax function.
|
||||
|
||||
Lower values encourage this method to converge toward the label quality score of the token with the lowest quality label in the sentence.
|
||||
|
||||
Higher values encourage this method to converge toward the average label quality score of all tokens in the sentence.
|
||||
|
||||
Returns
|
||||
---------
|
||||
sentence_scores:
|
||||
Array of shape ``(N, )``, where N is the number of sentences in the dataset, with one overall label quality score for each sentence.
|
||||
|
||||
Examples
|
||||
---------
|
||||
>>> from cleanlab.token_classification.rank import _softmin_sentence_score
|
||||
>>> token_scores = [[0.9, 0.6], [0.0, 0.8, 0.8], [0.8]]
|
||||
>>> _softmin_sentence_score(token_scores)
|
||||
array([6.00741787e-01, 1.80056239e-07, 8.00000000e-01])
|
||||
"""
|
||||
if temperature == 0:
|
||||
return np.array([np.min(scores) for scores in token_scores])
|
||||
|
||||
if temperature == np.inf:
|
||||
return np.array([np.mean(scores) for scores in token_scores])
|
||||
|
||||
def fun(scores: np.ndarray) -> float:
|
||||
return np.dot(
|
||||
scores, softmax(x=1 - np.array(scores), temperature=temperature, axis=0, shift=True)
|
||||
)
|
||||
|
||||
sentence_scores = list(map(fun, token_scores))
|
||||
return np.array(sentence_scores)
|
||||
@@ -0,0 +1,345 @@
|
||||
"""
|
||||
Methods to display sentences and their label issues in a token classification dataset (text data), as well as summarize the types of issues identified.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from cleanlab.internal.token_classification_utils import color_sentence, get_sentence
|
||||
|
||||
|
||||
def display_issues(
|
||||
issues: list,
|
||||
tokens: List[List[str]],
|
||||
*,
|
||||
labels: Optional[list] = None,
|
||||
pred_probs: Optional[list] = None,
|
||||
exclude: List[Tuple[int, int]] = [],
|
||||
class_names: Optional[List[str]] = None,
|
||||
top: int = 20,
|
||||
) -> None:
|
||||
"""
|
||||
Display token classification label issues, showing sentence with problematic token(s) highlighted.
|
||||
|
||||
Can also shows given and predicted label for each token identified to have label issue.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
issues:
|
||||
List of tuples ``(i, j)`` representing a label issue for the `j`-th token of the `i`-th sentence.
|
||||
|
||||
Same format as output by :py:func:`token_classification.filter.find_label_issues <cleanlab.token_classification.filter.find_label_issues>`
|
||||
or :py:func:`token_classification.rank.issues_from_scores <cleanlab.token_classification.rank.issues_from_scores>`.
|
||||
|
||||
tokens:
|
||||
Nested list such that `tokens[i]` is a list of tokens (strings/words) that comprise the `i`-th sentence.
|
||||
|
||||
labels:
|
||||
Optional nested list of given labels for all tokens, such that `labels[i]` is a list of labels, one for each token in the `i`-th sentence.
|
||||
For a dataset with K classes, each label must be in 0, 1, ..., K-1.
|
||||
|
||||
If `labels` is provided, this function also displays given label of the token identified with issue.
|
||||
|
||||
pred_probs:
|
||||
Optional list of np arrays, such that `pred_probs[i]` has shape ``(T, K)`` if the `i`-th sentence contains T tokens.
|
||||
|
||||
Each row of `pred_probs[i]` corresponds to a token `t` in the `i`-th sentence,
|
||||
and contains model-predicted probabilities that `t` belongs to each of the K possible classes.
|
||||
|
||||
Columns of each `pred_probs[i]` should be ordered such that the probabilities correspond to class 0, 1, ..., K-1.
|
||||
|
||||
If `pred_probs` is provided, this function also displays predicted label of the token identified with issue.
|
||||
|
||||
exclude:
|
||||
Optional list of given/predicted label swaps (tuples) to be ignored. For example, if `exclude=[(0, 1), (1, 0)]`,
|
||||
tokens whose label was likely swapped between class 0 and 1 are not displayed. Class labels must be in 0, 1, ..., K-1.
|
||||
|
||||
class_names:
|
||||
Optional length K list of names of each class, such that `class_names[i]` is the string name of the class corresponding to `labels` with value `i`.
|
||||
|
||||
If `class_names` is provided, display these string names for predicted and given labels, otherwise display the integer index of classes.
|
||||
|
||||
top: int, default=20
|
||||
Maximum number of issues to be printed.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> from cleanlab.token_classification.summary import display_issues
|
||||
>>> issues = [(2, 0), (0, 1)]
|
||||
>>> tokens = [
|
||||
... ["A", "?weird", "sentence"],
|
||||
... ["A", "valid", "sentence"],
|
||||
... ["An", "sentence", "with", "a", "typo"],
|
||||
... ]
|
||||
>>> display_issues(issues, tokens)
|
||||
Sentence index: 2, Token index: 0
|
||||
Token: An
|
||||
----
|
||||
An sentence with a typo
|
||||
<BLANKLINE>
|
||||
<BLANKLINE>
|
||||
Sentence index: 0, Token index: 1
|
||||
Token: ?weird
|
||||
----
|
||||
A ?weird sentence
|
||||
"""
|
||||
if not class_names and (labels or pred_probs):
|
||||
print(
|
||||
"Classes will be printed in terms of their integer index since `class_names` was not provided.\n"
|
||||
"Specify this argument to see the string names of each class.\n"
|
||||
)
|
||||
|
||||
top = min(top, len(issues))
|
||||
shown = 0
|
||||
is_tuple = isinstance(issues[0], tuple)
|
||||
|
||||
for issue in issues:
|
||||
if is_tuple:
|
||||
i, j = issue
|
||||
sentence = get_sentence(tokens[i])
|
||||
word = tokens[i][j]
|
||||
|
||||
if pred_probs:
|
||||
prediction = pred_probs[i][j].argmax()
|
||||
if labels:
|
||||
given = labels[i][j]
|
||||
if pred_probs and labels:
|
||||
if (given, prediction) in exclude:
|
||||
continue
|
||||
|
||||
if pred_probs and class_names:
|
||||
prediction = class_names[prediction]
|
||||
if labels and class_names:
|
||||
given = class_names[given]
|
||||
|
||||
shown += 1
|
||||
print(f"Sentence index: {i}, Token index: {j}")
|
||||
print(f"Token: {word}")
|
||||
if labels and not pred_probs:
|
||||
print(f"Given label: {given}")
|
||||
elif not labels and pred_probs:
|
||||
print(f"Predicted label according to provided pred_probs: {prediction}")
|
||||
elif labels and pred_probs:
|
||||
print(
|
||||
f"Given label: {given}, predicted label according to provided pred_probs: {prediction}"
|
||||
)
|
||||
print("----")
|
||||
print(color_sentence(sentence, word))
|
||||
else:
|
||||
shown += 1
|
||||
sentence = get_sentence(tokens[issue])
|
||||
print(f"Sentence issue: {sentence}")
|
||||
if shown == top:
|
||||
break
|
||||
print("\n")
|
||||
|
||||
|
||||
def common_label_issues(
|
||||
issues: List[Tuple[int, int]],
|
||||
tokens: List[List[str]],
|
||||
*,
|
||||
labels: Optional[list] = None,
|
||||
pred_probs: Optional[list] = None,
|
||||
class_names: Optional[List[str]] = None,
|
||||
top: int = 10,
|
||||
exclude: List[Tuple[int, int]] = [],
|
||||
verbose: bool = True,
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Display the tokens (words) that most commonly have label issues.
|
||||
|
||||
These may correspond to words that are ambiguous or systematically misunderstood by the data annotators.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
issues:
|
||||
List of tuples ``(i, j)`` representing a label issue for the `j`-th token of the `i`-th sentence.
|
||||
|
||||
Same format as output by :py:func:`token_classification.filter.find_label_issues <cleanlab.token_classification.filter.find_label_issues>`
|
||||
or :py:func:`token_classification.rank.issues_from_scores <cleanlab.token_classification.rank.issues_from_scores>`.
|
||||
|
||||
tokens:
|
||||
Nested list such that `tokens[i]` is a list of tokens (strings/words) that comprise the `i`-th sentence.
|
||||
|
||||
labels:
|
||||
Optional nested list of given labels for all tokens in the same format as `labels` for `~cleanlab.token_classification.summary.display_issues`.
|
||||
|
||||
If `labels` is provided, this function also displays given label of the token identified to commonly suffer from label issues.
|
||||
|
||||
pred_probs:
|
||||
Optional list of model-predicted probabilities (np arrays) in the same format as `pred_probs` for
|
||||
`~cleanlab.token_classification.summary.display_issues`.
|
||||
|
||||
If both `labels` and `pred_probs` are provided, also reports each type of given/predicted label swap for tokens identified to commonly suffer from label issues.
|
||||
|
||||
class_names:
|
||||
Optional length K list of names of each class, such that `class_names[i]` is the string name of the class corresponding to `labels` with value `i`.
|
||||
|
||||
If `class_names` is provided, display these string names for predicted and given labels, otherwise display the integer index of classes.
|
||||
|
||||
top:
|
||||
Maximum number of tokens to print information for.
|
||||
|
||||
exclude:
|
||||
Optional list of given/predicted label swaps (tuples) to be ignored in the same format as `exclude` for
|
||||
`~cleanlab.token_classification.summary.display_issues`.
|
||||
|
||||
verbose:
|
||||
Whether to also print out the token information in the returned DataFrame `df`.
|
||||
|
||||
Returns
|
||||
-------
|
||||
df:
|
||||
If both `labels` and `pred_probs` are provided, DataFrame `df` contains columns ``['token', 'given_label',
|
||||
'predicted_label', 'num_label_issues']``, and each row contains information for a specific token and
|
||||
given/predicted label swap, ordered by the number of label issues inferred for this type of label swap.
|
||||
|
||||
Otherwise, `df` only has columns ['token', 'num_label_issues'], and each row contains the information for a specific
|
||||
token, ordered by the number of total label issues involving this token.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> from cleanlab.token_classification.summary import common_label_issues
|
||||
>>> issues = [(2, 0), (0, 1)]
|
||||
>>> tokens = [
|
||||
... ["A", "?weird", "sentence"],
|
||||
... ["A", "valid", "sentence"],
|
||||
... ["An", "sentence", "with", "a", "typo"],
|
||||
... ]
|
||||
>>> df = common_label_issues(issues, tokens)
|
||||
Token '?weird' is potentially mislabeled 1 times throughout the dataset
|
||||
<BLANKLINE>
|
||||
Token 'An' is potentially mislabeled 1 times throughout the dataset
|
||||
<BLANKLINE>
|
||||
>>> df
|
||||
token num_label_issues
|
||||
0 An 1
|
||||
1 ?weird 1
|
||||
"""
|
||||
count: Dict[str, Any] = {}
|
||||
if not labels or not pred_probs:
|
||||
for issue in issues:
|
||||
i, j = issue
|
||||
word = tokens[i][j]
|
||||
if word not in count:
|
||||
count[word] = 0
|
||||
count[word] += 1
|
||||
|
||||
words = [word for word in count.keys()]
|
||||
freq = [count[word] for word in words]
|
||||
rank = np.argsort(freq)[::-1][:top]
|
||||
|
||||
for r in rank:
|
||||
print(
|
||||
f"Token '{words[r]}' is potentially mislabeled {freq[r]} times throughout the dataset\n"
|
||||
)
|
||||
|
||||
info = [[word, f] for word, f in zip(words, freq)]
|
||||
info = sorted(info, key=lambda x: x[1], reverse=True)
|
||||
return pd.DataFrame(info, columns=["token", "num_label_issues"])
|
||||
|
||||
if not class_names:
|
||||
print(
|
||||
"Classes will be printed in terms of their integer index since `class_names` was not provided. "
|
||||
)
|
||||
print("Specify this argument to see the string names of each class. \n")
|
||||
|
||||
n = pred_probs[0].shape[1]
|
||||
for issue in issues:
|
||||
i, j = issue
|
||||
word = tokens[i][j]
|
||||
label = labels[i][j]
|
||||
pred = pred_probs[i][j].argmax()
|
||||
if word not in count:
|
||||
count[word] = np.zeros([n, n], dtype=int)
|
||||
if (label, pred) not in exclude:
|
||||
count[word][label][pred] += 1
|
||||
words = [word for word in count.keys()]
|
||||
freq = [np.sum(count[word]) for word in words]
|
||||
rank = np.argsort(freq)[::-1][:top]
|
||||
|
||||
for r in rank:
|
||||
matrix = count[words[r]]
|
||||
most_frequent = np.argsort(count[words[r]].flatten())[::-1]
|
||||
print(
|
||||
f"Token '{words[r]}' is potentially mislabeled {freq[r]} times throughout the dataset"
|
||||
)
|
||||
if verbose:
|
||||
print(
|
||||
"---------------------------------------------------------------------------------------"
|
||||
)
|
||||
for f in most_frequent:
|
||||
i, j = f // n, f % n
|
||||
if matrix[i][j] == 0:
|
||||
break
|
||||
if class_names:
|
||||
print(
|
||||
f"labeled as class `{class_names[i]}` but predicted to actually be class `{class_names[j]}` {matrix[i][j]} times"
|
||||
)
|
||||
else:
|
||||
print(
|
||||
f"labeled as class {i} but predicted to actually be class {j} {matrix[i][j]} times"
|
||||
)
|
||||
print()
|
||||
info = []
|
||||
for word in words:
|
||||
for i in range(n):
|
||||
for j in range(n):
|
||||
num = count[word][i][j]
|
||||
if num > 0:
|
||||
if not class_names:
|
||||
info.append([word, i, j, num])
|
||||
else:
|
||||
info.append([word, class_names[i], class_names[j], num])
|
||||
info = sorted(info, key=lambda x: x[3], reverse=True)
|
||||
return pd.DataFrame(
|
||||
info, columns=["token", "given_label", "predicted_label", "num_label_issues"]
|
||||
)
|
||||
|
||||
|
||||
def filter_by_token(
|
||||
token: str, issues: List[Tuple[int, int]], tokens: List[List[str]]
|
||||
) -> List[Tuple[int, int]]:
|
||||
"""
|
||||
Return subset of label issues involving a particular token.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
token:
|
||||
A specific token you are interested in.
|
||||
|
||||
issues:
|
||||
List of tuples ``(i, j)`` representing a label issue for the `j`-th token of the `i`-th sentence.
|
||||
Same format as output by :py:func:`token_classification.filter.find_label_issues <cleanlab.token_classification.filter.find_label_issues>`
|
||||
or :py:func:`token_classification.rank.issues_from_scores <cleanlab.token_classification.rank.issues_from_scores>`.
|
||||
|
||||
tokens:
|
||||
Nested list such that `tokens[i]` is a list of tokens (strings/words) that comprise the `i`-th sentence.
|
||||
|
||||
Returns
|
||||
----------
|
||||
issues_subset:
|
||||
List of tuples ``(i, j)`` representing a label issue for the `j`-th token of the `i`-th sentence, in the same format as `issues`.
|
||||
But restricting to only those issues that involve the specified `token`.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> from cleanlab.token_classification.summary import filter_by_token
|
||||
>>> token = "?weird"
|
||||
>>> issues = [(2, 0), (0, 1)]
|
||||
>>> tokens = [
|
||||
... ["A", "?weird", "sentence"],
|
||||
... ["A", "valid", "sentence"],
|
||||
... ["An", "sentence", "with", "a", "typo"],
|
||||
... ]
|
||||
>>> filter_by_token(token, issues, tokens)
|
||||
[(0, 1)]
|
||||
"""
|
||||
returned_issues = []
|
||||
for issue in issues:
|
||||
i, j = issue
|
||||
if token.lower() == tokens[i][j].lower():
|
||||
returned_issues.append(issue)
|
||||
return returned_issues
|
||||
Reference in New Issue
Block a user