wehub/cleanlab--cleanlab
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:49:22 +08:00
commit a41b2ab474
303 changed files with 64772 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/source/_static/.gitkeep
+6
View File
@@ -0,0 +1,6 @@
Without this directory, Sphinx will warn:
> WARNING: html_static_path entry '_static' does not exist
This file exists to suppress that warning and because empty directories cannot
be checked in to Git.
docs/source/_static/css/custom.css
+41
View File
@@ -0,0 +1,41 @@
details {
margin-bottom: 0.75rem;
margin-top: 0.5rem;
}
details summary {
cursor: pointer;
}
details summary > * {
display: inline;
}
details[open] summary {
padding-bottom: 0.75rem;
border-bottom: 2px dashed #ccc;
}
details[open] {
border-bottom: 2px dashed #ccc;
}
h1 {
font-size: 2em;
}
h2 {
font-size: 1.5em;
}
h3 {
font-size: 1.17em;
}
h5 {
font-size: .83em;
}
h6 {
font-size: .75em;
}
docs/source/_templates/brand.html
+26
View File
@@ -0,0 +1,26 @@
<a style="padding-bottom: 0px;" class="sidebar-brand{% if logo %} centered{% endif %}" href="{{ pathto(master_doc) }}">
{% block brand_content %}
{%- if logo_url %}
<div class="sidebar-logo-container">
<img class="sidebar-logo" src="{{ logo_url }}" alt="Logo" />
</div>
{%- endif %}
{%- if theme_light_logo and theme_dark_logo %}
<div class="sidebar-logo-container">
<img class="sidebar-logo only-light" src="{{ pathto('_static/' + theme_light_logo, 1) }}" alt="Light Logo" />
<img class="sidebar-logo only-dark" src="{{ pathto('_static/' + theme_dark_logo, 1) }}" alt="Dark Logo" />
</div>
{%- endif %}
{% if not theme_sidebar_hide_name %}
<span style="margin-bottom:0px" class="sidebar-brand-text">
{{ docstitle if docstitle else project }}
</span>
{%- endif %}
{% endblock brand_content %}
</a>
<div style="margin-left: 12px; margin-top: 4px;">
<a style="margin-top: 6px;" class="github-button" href="https://github.com/cleanlab/cleanlab" data-size="large" data-show-count="true"
aria-label="Star cleanlab/cleanlab on GitHub">Star</a>
</div>
docs/source/_templates/page.html
+66
View File
@@ -0,0 +1,66 @@
{% extends "!page.html" %} {% block content %}
<noscript>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>
Parts of this site uses JavaScript, but your browser does not
support it.
</p>
</div>
</noscript>
{% if current_version %}
<!-- Start of Version Warning Banner -->
<p id="doc_ver_warning"></p>
<script type="text/javascript" src="{{ DOCS_SITE_URL }}versioning.js"></script>
<script type="text/javascript">
window.addEventListener("load", () => {
const version_number = Version.version_number;
const path_arr = window.location.pathname.split("/");
if (path_arr.includes("master")) {
document.getElementById(
"doc_ver_warning"
).innerHTML = `<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This version of the documentation corresponds to the master branch of <code class="docutils literal notranslate"><span class="pre">cleanlab</span></code> source code from <a href="https://github.com/cleanlab/cleanlab/">GitHub</a>. To see the documentation for the latest <code class="docutils literal notranslate"><span class="pre">pip</span></code>-installed version, click <a href="{{ DOCS_SITE_URL }}">here</a>.</p>
</div>`;
} else if (
!path_arr.includes(version_number) &&
!path_arr.includes("stable")
) {
document.getElementById("doc_ver_warning").innerHTML =
`<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This documentation is for an old version (<code class="docutils literal notranslate"><span class="pre">{{ current_version.name }}</span></code>) of <code class="docutils literal notranslate"><span class="pre">cleanlab</span></code>. To see the documentation for the latest stable version (<code class="docutils literal notranslate"><span class="pre">` +
version_number +
`</span></code>), click <a href="{{ DOCS_SITE_URL }}">here</a>.</p>
</div>`;
} else {
document.getElementById("doc_ver_warning").remove();
}
});
</script>
<!-- End of Version Warning Banner -->
{% endif %} {{ super() }} {% endblock %} {% block footer %} {{ super() }}
<script type="text/javascript">
window.addEventListener("load", () => {
let elements = document.getElementsByClassName("left-details");
elements[0].insertAdjacentHTML(
"afterbegin",
`<code class="docutils literal notranslate"><span class="pre">cleanlab</span></code> is distributed on <a href="https://pypi.org/project/cleanlab/">PyPI</a> and <a href="https://anaconda.org/conda-forge/cleanlab">conda</a>.`
);
});
</script>
{% endblock %} {% block regular_scripts %}{{ super() }}
<script async defer src="https://buttons.github.io/buttons.js"></script>
{% endblock %}
docs/source/_templates/redirect-to-stable.html
+9
View File
@@ -0,0 +1,9 @@
<!DOCTYPE html>
<html>
<head>
<title>Redirecting to the latest release/tag</title>
<meta charset="utf-8" />
<meta http-equiv="refresh" content="0; url=stable_url" />
<link rel="canonical" href="https://docs.cleanlab.ai/" />
</head>
</html>
docs/source/_templates/versioning.html
+65
View File
@@ -0,0 +1,65 @@
{% if versions %}
<!-- Start of versioning -->
<div class="sidebar-tree">
<p class="caption" role="heading">
<span class="caption-text">{{ _('Versions') }}</span>
</p>
<ul>
<li class="toctree-l1">
<a
id="version_number"
class="reference internal"
href="{{ DOCS_SITE_URL }}"
>stable</a
>
</li>
<li class="toctree-l1">
<a
id="commit_hash"
class="reference internal"
href="{{DOCS_SITE_URL}}master/"
>developer</a
>
</li>
{% for version in RELEASE_VERSIONS %}
<li class="toctree-l1">
<a
id="{{ version }}"
class="reference internal"
href="{{DOCS_SITE_URL}}{{version}}/"
>{{version}}</a
>
</li>
{% endfor %}
</ul>
</div>
<br>
<br>
<script
type="text/javascript"
src="{{ DOCS_SITE_URL }}versioning.js"
></script>
<script type="text/javascript">
window.addEventListener("load", () => {
const version_number = Version.version_number;
const commit_hash = Version.commit_hash;
document.getElementById("version_number").innerHTML =
"stable <code class='docutils literal notranslate'><span class='pre'> (" +
version_number +
")</span></code>";
document.getElementById("commit_hash").innerHTML =
"master <code class='docutils literal notranslate'><span class='pre'> (" +
commit_hash.slice(0, 7) +
"&hellip;)</span></code>";
});
</script>
<!-- End of versioning -->
{% endif %}
docs/source/_templates/versioning.js
+4
View File
@@ -0,0 +1,4 @@
var Version = {
version_number: "placeholder_version_number",
commit_hash: "placeholder_commit_hash",
};
docs/source/cleanlab/benchmarking/index.rst
+12
View File
@@ -0,0 +1,12 @@
benchmarking
============
.. automodule:: cleanlab.benchmarking
:autosummary:
:members:
:undoc-members:
:show-inheritance:
.. toctree::
noise_generation
docs/source/cleanlab/benchmarking/noise_generation.rst
+8
View File
@@ -0,0 +1,8 @@
noise_generation
================
.. automodule:: cleanlab.benchmarking.noise_generation
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/classification.rst
+8
View File
@@ -0,0 +1,8 @@
classification
==============
.. automodule:: cleanlab.classification
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/count.rst
+8
View File
@@ -0,0 +1,8 @@
count
=====
.. automodule:: cleanlab.count
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/data_valuation.rst
+8
View File
@@ -0,0 +1,8 @@
data_valuation
==============
.. automodule:: cleanlab.data_valuation
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/datalab/datalab.rst
+9
View File
@@ -0,0 +1,9 @@
datalab
=======
.. automodule:: cleanlab.datalab.datalab
:autosummary:
:members:
:undoc-members:
:show-inheritance:
:ignore-module-all:
docs/source/cleanlab/datalab/guide/_templates/issue_types_tip.rst
+10
View File
@@ -0,0 +1,10 @@
.. tip::
This type of issue has the issue name `"{{issue_name}}"`.
Run a check for this particular kind of issue by calling :py:meth:`Datalab.find_issues() <cleanlab.datalab.datalab.Datalab.find_issues>` like so:
.. code-block:: python
# `lab` is a Datalab instance
lab.find_issues(..., issue_types = {"{{issue_name}}": {}})
docs/source/cleanlab/datalab/guide/custom_issue_manager.rst
+225
View File
@@ -0,0 +1,225 @@
.. _issue_manager_creating_your_own:
Creating Your Own Issues Manager
================================
This guide walks through the process of creating your own
:py:class:`IssueManager <cleanlab.datalab.internal.issue_manager.issue_manager.IssueManager>`
to detect a custom-defined type of issue alongside the pre-defined issue types in
:py:class:`Datalab <cleanlab.datalab.datalab.Datalab>`.
.. seealso::
- :py:meth:`register <cleanlab.datalab.internal.issue_manager_factory.register>`:
You can either use this function at runtime to register a new issue manager:
.. code-block:: python
from cleanlab.datalab.internal.issue_manager_factory import register
register(MyIssueManager) # Defaults to task="classification"
# register(MyIssueManagerForRegression, task="regression") # Alternative for regression tasks
or add as a decorator to the class definition (currently only works for classification tasks):
.. code-block:: python
@register
class MyIssueManager(IssueManager):
...
Prerequisites
-------------
As a starting point for this guide, we'll import the necessary things for the next section and create a dummy dataset.
.. note::
.. include:: ../optional_dependencies.rst
.. code-block:: python
import numpy as np
import pandas as pd
from cleanlab import IssueManager
# Create a dummy dataset
N = 20
data = pd.DataFrame(
{
"text": [f"example {i}" for i in range(N)],
"label": np.random.randint(0, 2, N),
},
)
Implementing IssueManagers
--------------------------
.. _basic_issue_manager:
Basic Issue Check
~~~~~~~~~~~~~~~~~
To create a basic issue manager, inherit from the
:py:class:`IssueManager <cleanlab.datalab.internal.issue_manager.issue_manager.IssueManager>` class,
assign a name to the class as the class-variable, `issue_name`, and implement the
:py:meth:`find_issues <cleanlab.datalab.internal.issue_manager.issue_manager.IssueManager.find_issues>` method.
The :py:meth:`find_issues <cleanlab.datalab.internal.issue_manager.issue_manager.IssueManager.find_issues>`
method should mark each example in the dataset as an issue or not with a boolean array.
It should also provide a score for each example in the dataset that quantifies the quality of the example
with regards to the issue.
.. code-block:: python
class Basic(IssueManager):
# Assign a name to the issue
issue_name = "basic"
def find_issues(self, **kwargs) -> None:
# Compute scores for each example
scores = np.random.rand(len(self.datalab.data))
# Construct a dataframe where examples are marked for issues
# and the score for each example is included.
self.issues = pd.DataFrame(
{
f"is_{self.issue_name}_issue" : scores < 0.1,
self.issue_score_key : scores,
},
)
# Score the dataset as a whole based on this issue type
self.summary = self.make_summary(score = scores.mean())
.. _intermediate_issue_manager:
Intermediate Issue Check
~~~~~~~~~~~~~~~~~~~~~~~~
To create an intermediate issue:
- Perform the same steps as in the :ref:`basic issue check <basic_issue_manager>` section.
- Populate the `info` attribute with a dictionary of information about the identified issues.
The information can be included in a report generated by :py:class:`Datalab <cleanlab.datalab.datalab.Datalab>`,
if you add any of the keys to the `verbosity_levels` class-attribute.
Optionally, you can also add a description of the type of issue this issue manager handles to the `description` class-attribute.
.. code-block:: python
class Intermediate(IssueManager):
issue_name = "intermediate"
# Add a dictionary of information to include in the report
verbosity_levels = {
0: [],
1: ["std"],
2: ["raw_scores"],
}
# Add a description of the issue
description = "Intermediate issues are a bit more involved than basic issues."
def find_issues(self, *, intermediate_arg: int, **kwargs) -> None:
N = len(self.datalab.data)
raw_scores = np.random.rand(N)
std = raw_scores.std()
threshold = min(0, raw_scores.mean() - std)
sin_filter = np.sin(intermediate_arg * np.arange(N) / N)
kernel = sin_filter ** 2
scores = kernel * raw_scores
self.issues = pd.DataFrame(
{
f"is_{self.issue_name}_issue" : scores < threshold,
self.issue_score_key : scores,
},
)
self.summary = self.make_summary(score = scores.mean())
# Useful information that will be available in the Datalab instance
self.info = {
"std": std,
"raw_scores": raw_scores,
"kernel": kernel,
}
Advanced Issue Check
~~~~~~~~~~~~~~~~~~~~
There could be different types of issues detected in a dataset. A local issue which affects individual data points in a dataset and can be tracked via `Datalab.issues` dataframe (to see which data points are exhibiting this type of issue). Alternatively, a global issue which affects the overall dataset but is not easily attributable to individual data points (hard to say one data point exhibits the issue but another does not). Even for global issues, we recommend trying to assign a per data point score (and boolean) if possible, see the Non-IID IssueManager as an example of this. Note that a global issue must have num_issues greater than 0 in its `issue_summary`, otherwise it won't show up in `Datalab.report()` by default.
Use with Datalab
----------------
We can create a
:py:class:`Datalab <cleanlab.datalab.datalab.Datalab>`
instance and run issue checks with the custom issue managers we created like so:
.. code-block:: python
from cleanlab.datalab.internal.issue_manager_factory import register
from cleanlab import Datalab
# Register the issue manager
for issue_manager in [Basic, Intermediate]:
register(issue_manager)
# Instantiate a datalab instance
datalab = Datalab(data, label_name="label")
# Run the issue check
issue_types = {"basic": {}, "intermediate": {"intermediate_arg": 2}}
datalab.find_issues(issue_types=issue_types)
# Print report
datalab.report(verbosity=0)
The report will look something like this:
.. code-block:: text
Here is a summary of the different kinds of issues found in the data:
issue_type score num_issues
basic 0.477762 2
intermediate 0.286455 0
(Note: A lower score indicates a more severe issue across all examples in the dataset.)
------------------------------------------- basic issues -------------------------------------------
Number of examples with this issue: 2
Overall dataset quality in terms of this issue: 0.4778
Examples representing most severe instances of this issue:
is_basic_issue basic_score
13 True 0.003042
8 True 0.058117
11 False 0.121908
15 False 0.169312
17 False 0.229044
--------------------------------------- intermediate issues ----------------------------------------
About this issue:
Intermediate issues are a bit more involved than basic issues.
Number of examples with this issue: 0
Overall dataset quality in terms of this issue: 0.2865
Examples representing most severe instances of this issue:
is_intermediate_issue intermediate_score kernel
0 False 0.000000 0.0
1 False 0.007059 0.009967
3 False 0.010995 0.087332
2 False 0.016296 0.03947
11 False 0.019459 0.794251
docs/source/cleanlab/datalab/guide/generating_cluster_ids.rst
+29
View File
@@ -0,0 +1,29 @@
Generating Cluster IDs
======================
The underperforming group issue manager provides the option for passing pre-computed
cluster IDs to `find_issues`. These cluster IDs can be obtained by clustering
the features using algorithms such as K-Means, DBSCAN, HDBSCAN etc. Note that
* K-Means requires specifying the number of clusters explicitly.
* DBSCAN is sensitive to the choice of `eps` (radius) and `min_samples` (minimum samples for each cluster).
Example:
.. code-block:: python
import datalab
from sklearn.cluster import KMeans
features, labels = your_data() # Get features and labels
pred_probs = get_pred_probs() # Get prediction probabilities for all samples
# Group features into 8 clusters
clusterer = KMeans(n_clusters=5)
clusterer.fit(features)
cluster_ids = clusterer.labels_
lab = Datalab(data={"features": features, "y": labels}, label_name="y")
issue_types = {"underperforming_group": {"cluster_ids": cluster_ids}}
lab.find_issues(features=features, pred_probs=pred_probs, issue_types=issue_types)
docs/source/cleanlab/datalab/guide/index.rst
+41
View File
@@ -0,0 +1,41 @@
Datalab guides
==============
Guides for using Datalab and understanding the issues it detects.
.. note::
.. include:: ../optional_dependencies.rst
Types of issues
---------------
Guides to use Datalab with greater control, selecting what issues to search for and what nondefault settings to use for detecting them.
.. toctree::
:maxdepth: 3
issue_type_description
Customizing issue types
-----------------------
Guides (for developers) to create a custom issue type that Datalab audits for together with its built-in issue types.
.. toctree::
:maxdepth: 3
custom_issue_manager
Cleanlab Studio (Easy Mode)
---------------------------
`Cleanlab Studio <https://cleanlab.ai/blog/data-centric-ai/>`_ is a fully automated platform that can detect the same data issues as this package, as well as `many more types of issues <https://help.cleanlab.ai/guide/concepts/cleanlab_columns/>`_, all without you having to do any Machine Learning (or even write any code). Beyond being 100x faster to use and producing more useful results, `Cleanlab Studio <https://cleanlab.ai/blog/data-centric-ai/>`_ also provides an intelligent data correction interface for you to quickly fix the issues detected in your dataset (a single data scientist can fix millions of data points thanks to AI suggestions).
`Cleanlab Studio <https://cleanlab.ai/blog/data-centric-ai/>`_ offers a powerful AutoML system (with Foundation models) that is useful for more than improving data quality. With a few clicks, you can: find + fix issues in your dataset, identify the best type of ML model and train/tune it, and deploy this model to serve accurate predictions for new data. Also use the same AutoML to auto-label large datasets (a single user can label millions of data points thanks to powerful Foundation models). `Try Cleanlab Studio for free! <https://cleanlab.ai/signup/>`_
.. image:: https://raw.githubusercontent.com/cleanlab/assets/master/cleanlab/ml-with-cleanlab-studio.png
:width: 800
:alt: Stages of modern AI pipeline that can now be automated with Cleanlab Studio
docs/source/cleanlab/datalab/guide/issue_type_description.rst
+857
View File
@@ -0,0 +1,857 @@
Datalab Issue Types
*******************
Types of issues Datalab can detect
===================================
This page describes the various types of issues that Datalab can detect in a dataset.
For each type of issue, we explain: what it says about your data if detected, why this matters, and what parameters you can optionally specify to control the detection of this issue.
In case you didn't know: you can alternatively use `Cleanlab Studio <https://cleanlab.ai/blog/data-centric-ai/>`_ to detect the same data issues as this package, plus `many more types of issues <https://help.cleanlab.ai/guide/concepts/cleanlab_columns/>`_, all without having to do any Machine Learning (or even write any code).
.. include:: table.rst
Estimates for Each Issue Type
------------------------------
Datalab produces three estimates for **each** type of issue (called say `<ISSUE_NAME>` here):
1. A numeric quality score `<ISSUE_NAME>_score` (between 0 and 1) estimating how severe this issue is exhibited in each example from a dataset. Examples with higher scores are less likely to suffer from this issue. Access these via: the :py:attr:`Datalab.issues <cleanlab.datalab.datalab.Datalab.issues>` attribute or the method :py:meth:`Datalab.get_issues(\<ISSUE_NAME\>) <cleanlab.datalab.datalab.Datalab.get_issues>`.
2. A Boolean `is_<ISSUE_NAME>_issue` flag for each example from a dataset. Examples where this has value `True` are those estimated to exhibit this issue. Access these via: the :py:attr:`Datalab.issues <cleanlab.datalab.datalab.Datalab.issues>` attribute or the method :py:meth:`Datalab.get_issues(\<ISSUE_NAME\>) <cleanlab.datalab.datalab.Datalab.get_issues>`.
3. An overall dataset quality score (between 0 and 1), quantifying how severe this issue is overall across the entire dataset. Datasets with higher scores do not exhibit this issue as badly overall. Access these via: the :py:attr:`Datalab.issue_summary <cleanlab.datalab.datalab.Datalab.issue_summary>` attribute or the method :py:meth:`Datalab.get_issue_summary(\<ISSUE_NAME\>) <cleanlab.datalab.datalab.Datalab.get_issue_summary>`.
**Example (for the outlier issue type)**
.. code-block:: python
issue_name = "outlier" # how to reference the outlier issue type in code
issue_score = "outlier_score" # name of column with quality scores for the outlier issue type, atypical datapoints receive lower scores
is_issue = "is_outlier_issue" # name of Boolean column flagging which datapoints are considered outliers in the dataset
**Dataset vs. data point level issues**
Some issues are primarily about the overall dataset (e.g. non-IID, class imbalance, underperforming group), whereas others are primarily about individual examples (e.g. label issue, outlier, near duplicate, null, etc). The former issue types should be first investigated via the global score from :py:meth:`Datalab.get_issue_summary <cleanlab.datalab.datalab.Datalab.get_issue_summary>`, as the per-example results for such issues from :py:meth:`Datalab.get_issues <cleanlab.datalab.datalab.Datalab.get_issues>` require more expertise to interpret.
Inputs to Datalab
-----------------
Datalab estimates various issues based on the four inputs below.
Each input is optional, if you do not provide it, Datalab will skip checks for those types of issues that require this input.
1. ``label_name`` - a field in the dataset that the stores the annotated class for each example in a multi-class classification dataset.
2. ``pred_probs`` - predicted class probabilities output by your trained model for each example in the dataset (these should be out-of-sample, eg. produced via cross-validation).
3. ``features`` - numeric vector representations of the features for each example in the dataset. These may be embeddings from a (pre)trained model, or just a numerically-transformed version of the original data features.
4. ``knn_graph`` - K nearest neighbor graph represented as a sparse matrix of dissimilarity values between examples in the dataset. If both `knn_graph` and `features` are provided, the `knn_graph` takes precedence, and if only `features` is provided, then a `knn_graph` is internally constructed based on the (either euclidean or cosine) distance between different examples’ features.
Label Issue
-----------
Examples whose given label is estimated to be potentially incorrect (e.g. due to annotation error) are flagged as having label issues.
Datalab estimates which examples appear mislabeled as well as a numeric label quality score for each, which quantifies the likelihood that an example is correctly labeled.
For now, Datalab can only detect label issues in multi-class classification datasets, regression datasets, and multi-label classification datasets.
The cleanlab library has alternative methods you can use to detect label issues in other types of datasets (multi-annotator, token classification, etc.).
Label issues are calculated based on provided `pred_probs` from a trained model. If you do not provide this argument, but you do provide `features`, then a K Nearest Neighbor model will be fit to produce `pred_probs` based on your `features`. Otherwise if neither `pred_probs` nor `features` is provided, then this type of issue will not be considered.
For the most accurate results, provide out-of-sample `pred_probs` which can be obtained for a dataset via `cross-validation <https://docs.cleanlab.ai/stable/tutorials/pred_probs_cross_val.html>`_.
Having mislabeled examples in your dataset may hamper the performance of supervised learning models you train on this data.
For evaluating models or performing other types of data analytics, mislabeled examples may lead you to draw incorrect conclusions.
To handle mislabeled examples, you can either filter out the data with label issues or try to correct their labels.
Learn more about the method used to detect label issues in our paper: `Confident Learning: Estimating Uncertainty in Dataset Labels <https://arxiv.org/abs/1911.00068>`_
.. testsetup:: *
import numpy as np
from cleanlab import Datalab
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_predict
# Load a dataset
np.random.seed(0)
X = np.random.rand(100, 10)
X[-1] = X[-2] # Create an exact-duplicate example
y = np.random.randint(0, 3, 100)
X[y == 1] -= 0.85 # Add noise to the features of class 1
X[y == 2] += 0.85 # Add noise to the features of class 2
y[-3] = {0: 1, 1: 2, 2: 0}[y[-3]] # Swap the label of the example at index -3
clf = LogisticRegression(random_state=0)
pred_probs = cross_val_predict(clf, X, y, cv=3, method="predict_proba")
data = {"features": X, "labels": y}
lab = Datalab(data, label_name="labels", task="classification")
.. testsetup::
lab.find_issues(features=X, pred_probs=pred_probs)
lab.find_issues(features=X, pred_probs=pred_probs, issue_types={"data_valuation": {}})
Some metadata about label issues is stored in the `issues` attribute of the Datalab object.
Let's look at one way to access this information.
.. testcode::
lab.get_issues("label").sort_values("label_score").head(5)
The output will look something like this:
.. testoutput::
is_label_issue label_score given_label predicted_label
97 True 0.064045 0 2
58 False 0.680894 2 2
41 False 0.746043 0 0
4 False 0.794894 2 2
98 False 0.802911 1 1
``is_label_issue``
~~~~~~~~~~~~~~~~~~
A boolean column that flags examples with label issues.
If `True`, the example is estimated to have a label issue.
If `False`, the example is estimated to not have a label issue.
``label_score``
~~~~~~~~~~~~~~~
A numeric column that gives the label quality score for each example.
The score lies between 0 and 1.
The lower the score, the less likely the given label is to be correct.
``given_label``
~~~~~~~~~~~~~~~
A column of the actual labels as provided in the original dataset.
``predicted_label``
~~~~~~~~~~~~~~~~~~~
A column of the predicted labels for each example. This column may contain different labels than the given label, especially when the example is estimated to have a label issue or when a model predicts a different label than the given label.
.. jinja ::
{% with issue_name = "label" %}
{% include "cleanlab/datalab/guide/_templates/issue_types_tip.rst" %}
{% endwith %}
Outlier Issue
-------------
Examples that are very different from the rest of the dataset (i.e. potentially out-of-distribution or rare/anomalous instances).
Outlier issues are calculated based on provided `features` , `knn_graph` , or `pred_probs`.
If you do not provide one of these arguments, this type of issue will not be considered.
This article describes how outlier issues are detected in a dataset: `https://cleanlab.ai/blog/outlier-detection/ <https://cleanlab.ai/blog/outlier-detection/>`_.
When based on `features` or `knn_graph`, the outlier quality of each example is scored inversely proportional to its distance to its K nearest neighbors in the dataset.
When based on `pred_probs`, the outlier quality of each example is scored inversely proportional to the uncertainty in its prediction.
Modeling data with outliers may have unexpected consequences.
Closely inspect them and consider removing some outliers that may be negatively affecting your models.
Learn more about the methods used to detect outliers in our article: `Out-of-Distribution Detection via Embeddings or Predictions <https://cleanlab.ai/blog/outlier-detection/>`_
Some metadata about outlier issues is stored in the `issues` attribute of the Datalab object.
Let's look at one way to access this information.
.. testcode::
lab.get_issues("outlier").sort_values("outlier_score").head(5)
The output will look something like this:
.. testoutput::
is_outlier_issue outlier_score
98 True 0.011562
62 False 0.019657
22 False 0.035243
1 False 0.040907
42 False 0.056865
``is_outlier_issue``
~~~~~~~~~~~~~~~~~~~~
A boolean column, where `True` indicates that an example is identified as an outlier.
``outlier_score``
~~~~~~~~~~~~~~~~~
A numeric column with scores between 0 and 1.
A smaller value for an example indicates that it is less common or typical in the dataset, suggesting that it is more likely to be an outlier.
.. jinja ::
{% with issue_name = "outlier" %}
{% include "cleanlab/datalab/guide/_templates/issue_types_tip.rst" %}
{% endwith %}
(Near) Duplicate Issue
----------------------
A (near) duplicate issue refers to two or more examples in a dataset that are extremely similar to each other, relative to the rest of the dataset.
The examples flagged with this issue may be exactly duplicated, or lie atypically close together when represented as vectors (i.e. feature embeddings).
Near duplicated examples may record the same information with different:
- Abbreviations, misspellings, typos, formatting, etc. in text data.
- Compression formats, resolutions, or sampling rates in image, video, and audio data.
- Minor variations which naturally occur in many types of data (e.g. translated versions of an image).
Near Duplicate issues are calculated based on provided `features` or `knn_graph`.
If you do not provide one of these arguments, this type of issue will not be considered.
Datalab defines near duplicates as those examples whose distance to their nearest neighbor (in the space of provided `features`) in the dataset is less than `c * D`, where `0 < c < 1` is a small constant, and `D` is the median (over the full dataset) of such distances between each example and its nearest neighbor.
Scoring the numeric quality of an example in terms of the near duplicate issue type is done proportionally to its distance to its nearest neighbor.
Including near-duplicate examples in a dataset may negatively impact a ML model's generalization performance and lead to overfitting.
In particular, it is questionable to include examples in a test dataset which are (nearly) duplicated in the corresponding training dataset.
More generally, examples which happen to be duplicated can affect the final modeling results much more than other examples — so you should at least be aware of their presence.
Some metadata about near-duplicate issues is stored in the `issues` attribute of the Datalab object.
Let's look at one way to access this information.
.. testcode::
lab.get_issues("near_duplicate").sort_values("near_duplicate_score").head(5)
The output will look something like this:
.. testoutput::
is_near_duplicate_issue near_duplicate_score near_duplicate_sets distance_to_nearest_neighbor
36 True 0.066009 [11, 80] 0.003906
11 True 0.066009 [36] 0.003906
80 True 0.093245 [36] 0.005599
27 False 0.156720 [] 0.009751
72 False 0.156720 [] 0.009751
``is_near_duplicate_issue``
~~~~~~~~~~~~~~~~~~~~~~~~~~~
A boolean column, where `True` indicates that an example is identified as either a near- or exact-duplicate of other examples in the dataset.
``near_duplicate_score``
~~~~~~~~~~~~~~~~~~~~~~~~
A numeric column with scores between 0 and 1. The lower the score, the more likely the example is to be a near-duplicate of another example in the dataset.
Exact duplicates are assigned a score of 0, while near-duplicates are assigned a score close to 0.
``near_duplicate_sets``
~~~~~~~~~~~~~~~~~~~~~~~
A column of lists of integers. The i-th list contains the indices of examples that are considered near-duplicates of example i (not including example i).
``distance_to_nearest_neighbor``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
A numeric column that represents the distance between each example and its nearest neighbor in the dataset.
The distance is calculated based on the provided `features` or `knn_graph`, and is directly related to the `near_duplicate_score`.
A smaller distance indicates that the example is similar to another example in the dataset.
.. jinja ::
{% with issue_name = "near_duplicate" %}
{% include "cleanlab/datalab/guide/_templates/issue_types_tip.rst" %}
{% endwith %}
Non-IID Issue
-------------
Whether the overall dataset exhibits statistically significant violations of the IID assumption like: changepoints or shift, drift, autocorrelation, etc. The specific form of violation considered is whether the examples are ordered within the dataset such that almost adjacent examples tend to have more similar feature values. If you care about this check, do **not** first shuffle your dataset -- this check is entirely based on the sequential order of your data. Learn more via our blog: `https://cleanlab.ai/blog/non-iid-detection/ <https://cleanlab.ai/blog/non-iid-detection/>`_
The Non-IID issue is detected based on provided `features` or `knn_graph`. If you do not provide one of these arguments, this type of issue will not be considered. While the Non-IID check produces per-example information, it is primarily about assessing the overall dataset rather than assessing individual examples. So pay more attention to the overall dataset Non-IID score obtained via :py:meth:`Datalab.get_issue_summary("non_iid") <cleanlab.datalab.datalab.Datalab.get_issue_summary>` than the per-example scores.
The Non-IID issue is really a dataset-level check, not a per-datapoint level check (either a dataset violates the IID assumption or it doesn't). The per-datapoint scores returned for Non-IID issues merely highlight which datapoints you might focus on to better understand this dataset-level issue - there is not necessarily something specifically wrong with these specific datapoints.
Mathematically, the **overall** Non-IID score for the dataset is defined as the p-value of a statistical test for whether the distribution of *index-gap* values differs between group A vs. group B defined as follows. For a pair of examples in the dataset `x1, x2`, we define their *index-gap* as the distance between the indices of these examples in the ordering of the data (e.g. if `x1` is the 10th example and `x2` is the 100th example in the dataset, their index-gap is 90). We construct group A from pairs of examples which are amongst the K nearest neighbors of each other, where neighbors are defined based on the provided `knn_graph` or via distances in the space of the provided vector `features` . Group B is constructed from random pairs of examples in the dataset.
The Non-IID quality score for each example `x` is defined via a similarly computed p-value but with Group A constructed from the K nearest neighbors of `x` and Group B constructed from random examples from the dataset paired with `x`. Learn more about this method in our paper: `Detecting Dataset Drift and Non-IID Sampling via k-Nearest Neighbors <https://arxiv.org/abs/2305.15696>`_ (or the associated `blogpost <https://cleanlab.ai/blog/non-iid-detection/>`_).
The assumption that examples in a dataset are Independent and Identically Distributed (IID) is fundamental to proper modeling. Detecting all possible violations of the IID assumption is statistically impossible. This issue type only considers specific forms of violation where examples that tend to be closer together in the dataset ordering also tend to have more similar feature values. This includes scenarios where:
- The underlying distribution from which examples stem is evolving/drifting over time (not identically distributed).
- An example can influence the values of future examples in the dataset (not independent).
For datasets with low non-IID score, you should consider why your data are not IID and act accordingly. For example, if the data distribution is drifting over time, consider employing a time-based train/test split instead of a random partition. Note that shuffling the data ahead of time will ensure a good non-IID score, but this is not always a fix to the underlying problem (e.g. future deployment data may stem from a different distribution, or you may overlook the fact that examples influence each other). We thus recommend **not** shuffling your data to be able to diagnose this issue if it exists.
Some metadata about non-IID issues is stored in the `issues` attribute of the Datalab object.
Let's look at one way to access this information.
.. testcode::
lab.get_issues("non_iid").sort_values("non_iid_score").head(5)
The output will look something like this:
.. testoutput::
is_non_iid_issue non_iid_score
24 False 0.681458
37 False 0.804582
64 False 0.810646
80 False 0.815691
78 False 0.834293
``is_non_iid_issue``
~~~~~~~~~~~~~~~~~~~~
A boolean column, where `True` values indicate that the dataset exhibits statistically significant violations of the IID assumption.
If the overall dataset does not appear to be Non-IID (p-value > 0.05), then all entries in this column will be `False`.
If the dataset appears to be Non-IID (p-value < 0.05), then one entry will be `True`, specifically the example with the lowest `non_iid_score`.
We do not recommend interpreting the per-example boolean values, as the Non-IID check is more about the overall dataset.
``non_iid_score``
~~~~~~~~~~~~~~~~~
A numeric column with scores between 0 and 1, containing the Non-IID quality scores for each example.
Learn more via our `blogpost <https://cleanlab.ai/blog/non-iid-detection/>`_.
Be cautious when interpreting the non-IID issue score for individual examples.
The dataset as a whole receives a p-value for our non-IID test (obtained via :py:meth:`Datalab.get_issue_summary("non_iid") <cleanlab.datalab.datalab.Datalab.get_issue_summary>`), which better indicates whether the dataset exhibits non-IID behavior.
When this p-value is low, you can use the per-example non-IID scores to identify which examples to look at for better understanding this non-IID behavior.
.. jinja ::
{% with issue_name = "non_iid" %}
{% include "cleanlab/datalab/guide/_templates/issue_types_tip.rst" %}
{% endwith %}
Class Imbalance Issue
---------------------
Class imbalance is diagnosed just using the `labels` provided as part of the dataset. The overall class imbalance quality score of a dataset is the proportion of examples belonging to the rarest class `q`. If this proportion `q` falls below a threshold, then we say this dataset suffers from the class imbalance issue.
In a dataset identified as having class imbalance, the class imbalance quality score for each example is set equal to `q` if it is labeled as the rarest class, and is equal to 1 for all other examples.
Class imbalance in a dataset can lead to subpar model performance for the under-represented class. Consider collecting more data from the under-represented class, or at least take special care while modeling via techniques like over/under-sampling, SMOTE, asymmetric class weighting, etc.
This issue-type is more about the overall dataset vs. individual data points. If severe class imbalance is detected, Datalab will flag the individual data points from the minority class.
Some metadata about class imbalance issues is stored in the `issues` attribute of the Datalab object.
Let's look at one way to access this information.
.. testcode::
lab.get_issues("class_imbalance").sort_values("class_imbalance_score").head(5)
The output will look something like this:
.. testoutput::
is_class_imbalance_issue class_imbalance_score given_label
27 False 0.28 2
72 False 0.28 2
75 False 0.28 2
33 False 0.28 2
68 False 0.28 2
``is_class_imbalance_issue``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
A boolean column, where `True` indicates which examples belong to the minority class (rarest class) in a classification dataset that exhibits severe class imbalance. If the dataset is not considered to have severe class imbalance (i.e. proportion of examples in the rarest class is not to small relative to the number of classes in the dataset), then all values will be `False`.
``class_imbalance_score``
~~~~~~~~~~~~~~~~~~~~~~~~~
A numeric column with scores between 0 and 1.
Any example belonging to the most under-represented class is assigned a score equal to the proportion of examples in the dataset belonging to that class.
All other examples are assigned a score of 1.
All examples sharing the same label also share the same score.
``given_label``
~~~~~~~~~~~~~~~
A column of the actual labels as provided in the original dataset.
.. jinja ::
{% with issue_name = "class_imbalance" %}
{% include "cleanlab/datalab/guide/_templates/issue_types_tip.rst" %}
{% endwith %}
Image-specific Issues
---------------------
Datalab can identify image-specific issues in datasets, such as images that are excessively dark or bright, blurry, lack detail, or have unusual sizes.
To detect these issues, simply specify the `image_key` argument in :py:meth:`~cleanlab.datalab.datalab.Datalab`, indicating the image column name in your dataset.
This functionality currently works only with Hugging Face datasets. You can convert other local dataset formats into a Hugging Face dataset by following `this guide <https://huggingface.co/docs/datasets/en/loading>`_.
More information on these image-specific issues is available in the `CleanVision package <https://github.com/cleanlab/cleanvision?tab=readme-ov-file#clean-your-data-for-better-computer-vision>`_ .
Spurious Correlations between image-specific properties and labels
------------------------------------------------------------------
Based on the :ref:`image properties discussed earlier <Image-specific Issues>`, Datalab can also look for spurious correlations between image properties and the labels in the dataset.
These are unintended relationships between irrelevant features in images and the given labels, which ML models may easily exploit during training without learning the relevant features.
Once deployed, such models would consistently fail to generalize on unseen data where these spurious correlations most likely don't hold.
Spurious correlations may arise in the dataset due to various reasons, such as:
- Images for certain classes might be consistently captured under specific environmental conditions.
- Preprocessing techniques applied to the data might introduce systematic differences across classes.
- Objects of different classes may be systematically photographed in particular ways.
Spurious Correlations are checked for when Datalab is initialized for an image dataset with the `image_key` keyword argument,
after checking for :ref:`Image-specific Issues <Image-specific Issues>` where the image properties are computed.
Each image property (e.g. darkness/brightness) is assigned a label uncorrelatedness score for the entire dataset. The lower the score, the more strongly the property is correlated with the class labels, across images of the dataset. This score is mathematically defined as: 1 minus the relative accuracy improvement in predicting the labels based solely on this image property value (relative to always predicting the most common overall class).
Consider reviewing the relationship between images with high and low values of this property and the labels if the corresponding label uncorrelatedness score is low, because ML models trained on this dataset may latch onto the spurious correlation and fail to generalize.
This issue type is more about the overall dataset vs. individual data points and will only be highlighted by Datalab in its report, if any such troublesome image properties are found.
Metadata about spurious correlations is stored in the `info` attribute of the Datalab object.
It can be accessed like so:
.. code::
lab.get_info("spurious_correlations")["correlations_df"]
The output will look something like this:
.. testoutput::
property score
0 blurry_score 0.559
1 dark_score 0.808
2 light_score 0.723
3 odd_size_score 0.957
4 odd_aspect_ratio_score 0.835
5 grayscale_score 0.003 # Likely to be spuriously correlated with the labels
6 low_information_score 0.688
.. warning::
Note that the label uncorrelatedness scores are *not* stored in the `issues` attribute of Datalab.
``property``
~~~~~~~~~~~~
A categorical column that identifies specific image-related characteristics assessed for potential spurious correlations with the class labels. Each entry in this column represents a distinct property of the images, such as blurriness, darkness, or grayscale, which may or may not be correlated with the labels.
``score``
~~~~~~~~~
A numeric column that gives the level of label uncorrelatedness for a given image-specific property. The score lies between 0 and 1. The lower the score for an image-property, the more correlated the image-property is with the given labels.
.. tip::
This type of issue has the issue name `"spurious_correlations"`.
Run a check for this particular kind of issue by calling :py:meth:`Datalab.find_issues() <cleanlab.datalab.datalab.Datalab.find_issues>` like so:
.. code-block:: python
# `lab` is a Datalab instance
lab.find_issues(..., issue_types = {"spurious_correlations": {}})
Underperforming Group Issue
---------------------------
An underperforming group refers to a cluster of similar examples (i.e. a slice) in the dataset for which the ML model predictions are poor. The examples in this underperforming group may have noisy labels or feature values, or the trained ML model may not have learned how to properly handle them (consider collecting more data from this subpopulation or up-weighting the existing data from this group).
This issue-type is more about the overall dataset vs. individual data points. If an underperforming group is detected, Datalab will flag the individual data points from this group.
Underperforming Group issues are detected based on one of:
- provided `pred_probs` and `features`,
- provided `pred_probs` and `knn_graph`, or
- provided `pred_probs` and `cluster_ids`. (This option is for advanced users, see the `FAQ <../../../tutorials/faq.html#How-do-I-specify-pre-computed-data-slices/clusters-when-detecting-the-Underperforming-Group-Issue?>`_ for more details.)
If you do not provide both these arguments, this type of issue will not be considered.
To find the underperforming group, Cleanlab clusters the data using the provided `features` and determines the cluster `c` with the lowest average model predictive performance. Model predictive performance is evaluated via the model's self-confidence of the given labels, calculated using :py:func:`rank.get_self_confidence_for_each_label <cleanlab.rank.get_self_confidence_for_each_label>`. Suppose the average predictive power across the full dataset is `r` and is `q` within a cluster of examples. This cluster is considered to be an underperforming group if `q/r` falls below a threshold. A dataset suffers from the Underperforming Group issue if there exists such a cluster within it.
The underperforming group quality score is equal to `q/r` for examples belonging to the underperforming group, and is equal to 1 for all other examples.
Advanced users: If you have pre-computed cluster assignments for each example in the dataset, you can pass them explicitly to :py:meth:`Datalab.find_issues <cleanlab.datalab.datalab.Datalab.find_issues>` using the `cluster_ids` key in the `issue_types` dict argument. This is useful for tabular datasets where you want to group/slice the data based on a categorical column. An integer encoding of the categorical column can be passed as cluster assignments for finding the underperforming group, based on the data slices you define.
Some metadata about underperforming group issues is stored in the `issues` attribute of the Datalab object.
Let's look at one way to access this information.
.. testcode::
lab.get_issues("underperforming_group").sort_values("underperforming_group_score").head(5)
The output will look something like this:
.. testoutput::
is_underperforming_group_issue underperforming_group_score
0 False 1.0
72 False 1.0
71 False 1.0
70 False 1.0
69 False 1.0
``is_underperforming_group_issue``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
A boolean column, where `True` indicates which examples belong to the subgroup (i.e. cluster/slice) for which model predictions are significantly worse than for the rest of the dataset.
If there is no such underperforming subgroup detected, then all values will be `False`.
``underperforming_group_score``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
A numeric column with scores between 0 and 1. Only examples belonging to a detected underperforming group receive a score less than 1.
Every example in the underperforming group shares the same score, which is the ratio of group's label quality score vs. the mean label quality score across the dataset.
The lower the score, the worse the model predictions are for this particular subgroup relative to the rest of the dataset.
.. jinja ::
{% with issue_name = "underperforming_group" %}
{% include "cleanlab/datalab/guide/_templates/issue_types_tip.rst" %}
{% endwith %}
Null Issue
----------
Examples identified with the null issue correspond to rows that have null/missing values across all feature columns (i.e. the entire row is missing values).
Null issues are detected based on provided `features`. If you do not provide `features`, this type of issue will not be considered.
Each example's null issue quality score equals the proportion of features values in this row that are not null/missing. The overall dataset null issue quality score
equals the average of the individual examples' quality scores.
Presence of null examples in the dataset can lead to errors when training ML models. It can also
result in the model learning incorrect patterns due to the null values.
Some metadata about null issues is stored in the `issues` attribute of the Datalab object.
Let's look at one way to access this information.
.. testcode::
lab.get_issues("null").sort_values("null_score").head(5)
The output will look something like this:
.. testoutput::
is_null_issue null_score
0 False 1.0
72 False 1.0
71 False 1.0
70 False 1.0
69 False 1.0
``is_null_issue``
~~~~~~~~~~~~~~~~~
A boolean column, where `True` indicates that an example is identified as having null/missing values across all feature columns.
Examples that just have a single non-null value across multiple feature columns are not flagged with this issue.
``null_score``
~~~~~~~~~~~~~~
A numeric column with scores between 0 and 1. The score represents the proportion of non-null (i.e. non-missing) values in each example.
Lower scores indicate examples with more null/missing values.
.. jinja ::
{% with issue_name = "null"%}
{% include "cleanlab/datalab/guide/_templates/issue_types_tip.rst" %}
{% endwith %}
Data Valuation Issue
--------------------
The examples in the dataset with lowest data valuation scores contribute least to a trained ML model's performance (those whose value falls below a threshold are flagged with this type of issue).
Data valuation issues can be detected based on provided `features` or a provided `knn_graph` (or one pre-computed during the computation of other issue types). If you do not provide one of these two arguments and there isn't a `knn_graph` already stored in the Datalab object, this type of issue will not be considered.
The data valuation score is an approximate Data Shapley value, calculated based on the labels of the top k nearest neighbors of an example. The details of this KNN-Shapley value could be found in the papers: `Efficient Task-Specific Data Valuation for Nearest Neighbor Algorithms <https://arxiv.org/abs/1908.08619>`_ and `Scalability vs. Utility: Do We Have to Sacrifice One for the Other in Data Importance Quantification? <https://arxiv.org/abs/1911.07128>`_.
Some metadata about data valuation issues is stored in the `issues` attribute of the Datalab object.
Let's look at one way to access this information.
.. testcode::
lab.get_issues("data_valuation").sort_values("data_valuation_score").head(5)
The output will look something like this:
.. testoutput::
is_data_valuation_issue data_valuation_score
39 False 0.5
32 False 0.5
98 False 0.5
6 False 0.5
7 False 0.5
``is_data_valuation_issue``
~~~~~~~~~~~~~~~~~~~~~~~~~~~
A boolean column, where `True` indicates that an example does not appear to contribute positively to a model's training performance.
``data_valuation_score``
~~~~~~~~~~~~~~~~~~~~~~~~
A numeric column with scores between 0 and 1. The score reflects how valuable each individual example is in terms of improving the performance of the ML model trained on this dataset.
Examples with higher scores more positively influence the resulting model's predictive performance, contributing to better learning. One would expect the model to get worse if many such examples were removed from its training dataset.
.. jinja ::
{% with issue_name = "data_valuation"%}
{% include "cleanlab/datalab/guide/_templates/issue_types_tip.rst" %}
{% endwith %}
Identifier Column Issue
-----------------------
This issue type flags sequential numerical columns in the features of a dataset. A numerical sequential column is most likely an identifier column, which - in most cases - should not be used in training ML models.
More formally, an identifier column is defined as a column i in features such that set(features[:,i]) = set(c, c+1, ..., c+n) for some integer c, where n = num_rows of features. Note we don't consider the ordering of the column in case the dataset was pre-shuffled.
If the condition is met in one or more columns, then we say this dataset has the identifier_column issue. The overall issue-summary-score is binary. The score equals 1 if the issue is present and 0 if the issue is not present.
The info attribute of Datalab shows the indices of the columns with the column_identifier issue.
.. jinja ::
{% with issue_name = "identifier_column"%}
{% include "cleanlab/datalab/guide/_templates/issue_types_tip.rst" %}
{% endwith %}
Optional Issue Parameters
=========================
Here is the dict of possible (**optional**) parameter values that can be specified via the argument `issue_types` to :py:meth:`Datalab.find_issues <cleanlab.datalab.datalab.Datalab.find_issues>`.
Optionally specify these to exert greater control over how issues are detected in your dataset.
Appropriate defaults are used for any parameters you do not specify, so no need to specify all of these!
.. code-block:: python
possible_issue_types = {
"label": label_kwargs, "outlier": outlier_kwargs,
"near_duplicate": near_duplicate_kwargs, "non_iid": non_iid_kwargs,
"class_imbalance": class_imbalance_kwargs, "underperforming_group": underperforming_group_kwargs,
"null": null_kwargs, "data_valuation": data_valuation_kwargs,
"identifier_column": identifier_column_kwargs,
}
where the possible `kwargs` dicts for each key are described in the sections below.
Label Issue Parameters
----------------------
.. code-block:: python
label_kwargs = {
"k": # number of nearest neighbors to consider when computing pred_probs from features,
"health_summary_parameters": # dict of potential keyword arguments to method `dataset.health_summary()`,
"clean_learning_kwargs": # dict of keyword arguments to constructor `CleanLearning()` including keys like: "find_label_issues_kwargs" or "label_quality_scores_kwargs",
"thresholds": # `thresholds` argument to `CleanLearning.find_label_issues()`,
"noise_matrix": # `noise_matrix` argument to `CleanLearning.find_label_issues()`,
"inverse_noise_matrix": # `inverse_noise_matrix` argument to `CleanLearning.find_label_issues()`,
"save_space": # `save_space` argument to `CleanLearning.find_label_issues()`,
"clf_kwargs": # `clf_kwargs` argument to `CleanLearning.find_label_issues()`. Currently has no effect.,
"validation_func": # `validation_func` argument to `CleanLearning.fit()`. Currently has no effect.,
}
.. attention::
``health_summary_parameters`` and ``health_summary_kwargs`` can work in tandem to determine the arguments to be used in the call to :py:meth:`dataset.health_summary <cleanlab.dataset.health_summary>`.
.. note::
For more information, view the source code of: :py:class:`datalab.internal.issue_manager.label.LabelIssueManager <cleanlab.datalab.internal.issue_manager.label.LabelIssueManager>`.
Outlier Issue Parameters
------------------------
.. code-block:: python
outlier_kwargs = {
"threshold": # floating value between 0 and 1 that sets the sensitivity of the outlier detection algorithms, based on either features or pred_probs.
"k": # integer representing the number of nearest neighbors for nearest neighbors search (passed as argument to `NearestNeighbors`), if necessary, Used with features,
"t": # integer used to modulate the strength of the transformation from distances to scores that lie in the range [0, 1]. Used with features,
"scaling_factor": # floating value used to normalize the distances before they are converted into scores. Used with features,
"metric": # string or callable representing the distance metric used in nearest neighbors search (passed as argument to `NearestNeighbors`), if necessary, Used with features,
"ood_kwargs": # dict of keyword arguments to constructor `OutOfDistribution()`{
"params": {
# NOTE: Each of the following keyword arguments can also be provided outside "ood_kwargs"
"adjust_pred_probs": # `adjust_pred_probs` argument to constructor `OutOfDistribution()`. Used with pred_probs,
"method": # `method` argument to constructor `OutOfDistribution()`. Used with pred_probs,
"confident_thresholds": # `confident_thresholds` argument to constructor `OutOfDistribution()`. Used with pred_probs,
},
},
}
.. note::
For more information, view the source code of: :py:class:`datalab.internal.issue_manager.outlier.OutlierIssueManager <cleanlab.datalab.internal.issue_manager.outlier.OutlierIssueManager>`.
Duplicate Issue Parameters
--------------------------
.. code-block:: python
near_duplicate_kwargs = {
"metric": # string or callable representing the distance metric used in nearest neighbors search (passed as argument to `NearestNeighbors`), if necessary,
"k": # integer representing the number of nearest neighbors for nearest neighbors search (passed as argument to `NearestNeighbors`), if necessary,
"threshold": # `threshold` argument to constructor of `NearDuplicateIssueManager()`. Non-negative floating value that determines the maximum distance between two examples to be considered outliers, relative to the median distance to the nearest neighbors,
}
.. attention::
`k` does not affect the results of the (near) duplicate search algorithm. It only affects the construction of the knn graph, if necessary.
.. note::
For more information, view the source code of: :py:class:`datalab.internal.issue_manager.duplicate.NearDuplicateIssueManager <cleanlab.datalab.internal.issue_manager.duplicate.NearDuplicateIssueManager>`.
Non-IID Issue Parameters
------------------------
.. code-block:: python
non_iid_kwargs = {
"metric": # `metric` argument to constructor of `NonIIDIssueManager`. String or callable for the distance metric used for nearest neighbors search if necessary. `metric` argument to constructor of `sklearn.neighbors.NearestNeighbors`,
"k": # `k` argument to constructor of `NonIIDIssueManager`. Integer representing the number of nearest neighbors for nearest neighbors search if necessary. `n_neighbors` argument to constructor of `sklearn.neighbors.NearestNeighbors`,
"num_permutations": # `num_permutations` argument to constructor of `NonIIDIssueManager`,
"seed": # seed for numpy's random number generator (used for permutation tests),
"significance_threshold": # `significance_threshold` argument to constructor of `NonIIDIssueManager`. Floating value between 0 and 1 that determines the overall signicance of non-IID issues found in the dataset.
}
.. note::
For more information, view the source code of: :py:class:`datalab.internal.issue_manager.noniid.NonIIDIssueManager <cleanlab.datalab.internal.issue_manager.noniid.NonIIDIssueManager>`.
Imbalance Issue Parameters
--------------------------
.. code-block:: python
class_imbalance_kwargs = {
"threshold": # `threshold` argument to constructor of `ClassImbalanceIssueManager`. Non-negative floating value between 0 and 1 indicating the minimum fraction of samples of each class that are present in a dataset without class imbalance.
}
.. note::
For more information, view the source code of: :py:class:`datalab.internal.issue_manager.imbalance.ClassImbalanceIssueManager <cleanlab.datalab.internal.issue_manager.imbalance.ClassImbalanceIssueManager>`.
Underperforming Group Issue Parameters
--------------------------------------
.. code-block:: python
underperforming_group_kwargs = {
# Constructor arguments for `UnderperformingGroupIssueManager`
"threshold": # Non-negative floating value between 0 and 1 used for determinining group of points with low confidence.
"metric": # String or callable for the distance metric used for nearest neighbors search if necessary. `metric` argument to constructor of `sklearn.neighbors.NearestNeighbors`.
"k": # Integer representing the number of nearest neighbors for constructing the nearest neighbour graph. `n_neighbors` argument to constructor of `sklearn.neighbors.NearestNeighbors`.
"min_cluster_samples": # Non-negative integer value specifying the minimum number of examples required for a cluster to be considered as the underperforming group. Used in `UnderperformingGroupIssueManager.filter_cluster_ids`.
"clustering_kwargs": # Key-value pairs representing arguments for the constructor of the clustering algorithm class (e.g. `sklearn.cluster.DBSCAN`).
# Argument for the find_issues() method of UnderperformingGroupIssueManager
"cluster_ids": # A 1-D numpy array containing cluster labels for each sample in the dataset. If passed, these cluster labels are used for determining the underperforming group.
}
.. note::
For more information, view the source code of: :py:class:`datalab.internal.issue_manager.underperforming_group.UnderperformingGroupIssueManager <cleanlab.datalab.internal.issue_manager.underperforming_group.UnderperformingGroupIssueManager>`.
For more information on generating `cluster_ids` for this issue manager, refer to this `FAQ Section <../../../tutorials/faq.html#How-do-I-specify-pre-computed-data-slices/clusters-when-detecting-the-Underperforming-Group-Issue?>`_.
Null Issue Parameters
---------------------
.. code-block:: python
null_kwargs = {}
.. note::
For more information, view the source code of: :py:class:`datalab.internal.issue_manager.null.NullIssueManager <cleanlab.datalab.internal.issue_manager.null.NullIssueManager>`.
Data Valuation Issue Parameters
-------------------------------
.. code-block:: python
data_valuation_kwargs = {
"k": # Number of nearest neighbors used to calculate data valuation scores,
"threshold": # Examples with scores below this threshold will be flagged with a data valuation issue
}
.. note::
For more information, view the source code of: :py:class:`datalab.internal.issue_manager.data_valuation.DataValuationIssueManager <cleanlab.datalab.internal.issue_manager.data_valuation.DataValuationIssueManager>`.
Identifier Column Parameters
----------------------------
.. code-block:: python
identifier_column_kwargs = {}
.. note::
For more information, view the source code of: :py:class:`datalab.internal.issue_manager.identifier_column.IdentifierColumnIssueManager <cleanlab.datalab.internal.issue_manager.identifier_column.IdentifierColumnIssueManager>`.
Image Issue Parameters
----------------------
To customize optional parameters for specific image issue types, you can provide a dictionary format corresponding to each image issue. The following codeblock demonstrates how to specify optional parameters for all image issues. However, it's important to note that providing optional parameters for specific image issues is not mandatory. If no specific parameters are provided, defaults will be used for those issues.
.. code-block:: python
image_issue_types_kwargs = {
"dark": {"threshold": 0.32}, # `threshold` argument for dark issue type. Non-negative floating value between 0 and 1, lower value implies fewer samples will be marked as issue and vice versa.
"light": {"threshold": 0.05}, # `threshold` argument for light issue type. Non-negative floating value between 0 and 1, lower value implies fewer samples will be marked as issue and vice versa.
"blurry": {"threshold": 0.29}, # `threshold` argument for blurry issue type. Non-negative floating value between 0 and 1, lower value implies fewer samples will be marked as issue and vice versa.
"low_information": {"threshold": 0.3}, # `threshold` argument for low_information issue type. Non-negative floating value between 0 and 1, lower value implies fewer samples will be marked as issue and vice versa.
"odd_aspect_ratio": {"threshold": 0.35}, # `threshold` argument for odd_aspect_ratio issue type. Non-negative floating value between 0 and 1, lower value implies fewer samples will be marked as issue and vice versa.
"odd_size": {"threshold": 10.0}, # `threshold` argument for odd_size issue type. Non-negative integer value between starting from 0, unlike other issues, here higher value implies fewer samples will be selected.
}
.. note::
For more information, view the cleanvision `docs <https://cleanvision.readthedocs.io/en/latest/tutorials/tutorial.html#5.-Check-for-an-issue-with-a-different-threshold>`_.
Spurious Correlations Issue Parameters
--------------------------------------
.. code-block:: python
spurious_correlations_kwargs = {
"threshold": 0.3, # Non-negative floating value between 0 and 1, lower value implies fewer image properties may have a low enough label uncorrelatedness score to be marked as issue and vice versa.
}
Cleanlab Studio (Easy Mode)
---------------------------
`Cleanlab Studio <https://cleanlab.ai/blog/data-centric-ai/>`_ is a fully automated platform that can detect the same data issues as this package, as well as `many more types of issues <https://help.cleanlab.ai/guide/concepts/cleanlab_columns/>`_, all without you having to do any Machine Learning (or even write any code). Beyond being 100x faster to use and producing more useful results, `Cleanlab Studio <https://cleanlab.ai/blog/data-centric-ai/>`_ also provides an intelligent data correction interface for you to quickly fix the issues detected in your dataset (a single data scientist can fix millions of data points thanks to AI suggestions).
`Cleanlab Studio <https://cleanlab.ai/blog/data-centric-ai/>`_ offers a powerful AutoML system (with Foundation models) that is useful for more than improving data quality. With a few clicks, you can: find + fix issues in your dataset, identify the best type of ML model and train/tune it, and deploy this model to serve accurate predictions for new data. Also use the same AutoML to auto-label large datasets (a single user can label millions of data points thanks to powerful Foundation models). `Try Cleanlab Studio for free! <https://cleanlab.ai/signup/>`_
.. image:: https://raw.githubusercontent.com/cleanlab/assets/master/cleanlab/ml-with-cleanlab-studio.png
:width: 800
:alt: Stages of modern AI pipeline that can now be automated with Cleanlab Studio
docs/source/cleanlab/datalab/guide/table.rst
+186
View File
@@ -0,0 +1,186 @@
.. tabs::
.. tab:: Classification task
.. list-table::
:widths: 20 10 20 50
:header-rows: 1
* - Issue Name
- Default
- Column Name
- Required keyword arguments in :py:meth:`Datalab.find_issues <cleanlab.datalab.datalab.Datalab.find_issues>`
* - :ref:`label<Label Issue>`
- Yes
-
:ref:`is_label_issue<\`\`is_label_issue\`\`>`
:ref:`label_score<\`\`label_score\`\`>`
:ref:`given_label<\`\`given_label\`\`>`
:ref:`predicted_label<\`\`predicted_label\`\`>`
- `pred_probs` or `features`
* - :ref:`outlier<Outlier Issues>`
- Yes
-
:ref:`is_outlier_issue<\`\`is_outlier_issue\`\`>`
:ref:`outlier_score<\`\`outlier_score\`\`>`
- `pred_probs` or `features` or `knn_graph`
* - :ref:`near_duplicate<(Near) Duplicate Issue>`
- Yes
-
:ref:`is_near_duplicate_issue<\`\`is_near_duplicate_issue\`\`>`
:ref:`near_duplicate_score<\`\`near_duplicate_score\`\`>`
:ref:`near_duplicate_sets<\`\`near_duplicate_sets\`\`>`
:ref:`distance_to_nearest_neighbor<\`\`distance_to_nearest_neighbor\`\`>`
- `features` or `knn_graph`
* - :ref:`non_iid<Non-IID Issue>`
- Yes
-
:ref:`is_non_iid_issue<\`\`is_non_iid_issue\`\`>`
:ref:`non_iid_score<\`\`non_iid_score\`\`>`
- `pred_probs` or `features` or `knn_graph`
* - :ref:`class_imbalance<Class Imbalance Issue>`
- Yes
-
:ref:`is_class_imbalance_issue<\`\`is_class_imbalance_issue\`\`>`
:ref:`class_imbalance_score<\`\`class_imbalance_score\`\`>`
- None [#f1]_
* - :ref:`underperforming_group<Underperforming Group Issue>`
- Yes
-
:ref:`is_underperforming_group_issue<\`\`is_underperforming_group_issue\`\`>`
:ref:`underperforming_group_score<\`\`underperforming_group_score\`\`>`
- (`pred_probs`, `features`) or (`pred_probs`, `knn_graph`) or (`pred_probs`, `cluster_ids`) [#f2]_
* - :ref:`null<Null Issue>`
- Yes
-
:ref:`is_null_issue<\`\`is_null_issue\`\`>`
:ref:`null_score<\`\`null_score\`\`>`
- `features`
* - :ref:`data_valuation<Data Valuation Issue>`
- No
-
:ref:`is_data_valuation_issue<\`\`is_data_valuation_issue\`\`>`
:ref:`data_valuation_score<\`\`data_valuation_score\`\`>`
- `knn_graph`
.. rubric:: Notes
.. [#f1] Only runs if `label_name` is provided in `Datalab()` constructor
.. [#f2] `cluster_ids` currently needs to be passed via `issue_types`
.. tab:: Regression task
.. list-table::
:widths: 20 10 20 50
:header-rows: 1
* - Issue Name
- Default
- Column Name
- Required keyword arguments in :py:meth:`Datalab.find_issues <cleanlab.datalab.datalab.Datalab.find_issues>`
* - :ref:`label<Label Issue>`
- Yes
-
:ref:`is_label_issue<\`\`is_label_issue\`\`>`
:ref:`label_score<\`\`label_score\`\`>`
:ref:`given_label<\`\`given_label\`\`>`
:ref:`predicted_label<\`\`predicted_label\`\`>`
- `pred_probs` [#f3]_ or `features` or (`features`, `model`) [#f4]_
* - :ref:`outlier<Outlier Issues>`
- Yes
-
:ref:`is_outlier_issue<\`\`is_outlier_issue\`\`>`
:ref:`outlier_score<\`\`outlier_score\`\`>`
- `features` or `knn_graph`
* - :ref:`near_duplicate<(Near) Duplicate Issue>`
- Yes
-
:ref:`is_near_duplicate_issue<\`\`is_near_duplicate_issue\`\`>`
:ref:`near_duplicate_score<\`\`near_duplicate_score\`\`>`
- `features` or `knn_graph`
* - :ref:`non_iid<Non-IID Issue>`
- Yes
-
:ref:`is_non_iid_issue<\`\`is_non_iid_issue\`\`>`
:ref:`non_iid_score<\`\`non_iid_score\`\`>`
- `features` or `knn_graph`
* - :ref:`null<Null Issue>`
- Yes
-
:ref:`is_null_issue<\`\`is_null_issue\`\`>`
:ref:`null_score<\`\`null_score\`\`>`
- `features`
.. rubric:: Notes
.. [#f3] :abbr:`pred_probs (Predicted Probabilities)` gets reinterpreted as a `predictions` argument internally
.. [#f4] `model` currently needs to be passed as `issue_types = {"label": {"clean_learning_kwargs": {"model": your_regression_model}}}`
.. tab:: Multilabel task
.. list-table::
:widths: 20 10 20 50
:header-rows: 1
* - Issue Name
- Default
- Column Name
- Required keyword arguments in :py:meth:`Datalab.find_issues <cleanlab.datalab.datalab.Datalab.find_issues>`
* - :ref:`label<Label Issue>`
- Yes
-
:ref:`is_label_issue<\`\`is_label_issue\`\`>`
:ref:`label_score<\`\`label_score\`\`>`
:ref:`given_label<\`\`given_label\`\`>`
:ref:`predicted_label<\`\`predicted_label\`\`>`
- `pred_probs` or `features`
* - :ref:`outlier<Outlier Issues>`
- Yes
-
:ref:`is_outlier_issue<\`\`is_outlier_issue\`\`>`
:ref:`outlier_score<\`\`outlier_score\`\`>`
- `features` or `knn_graph`
* - :ref:`near_duplicate<(Near) Duplicate Issue>`
- Yes
-
:ref:`is_near_duplicate_issue<\`\`is_near_duplicate_issue\`\`>`
:ref:`near_duplicate_score<\`\`near_duplicate_score\`\`>`
- `features` or `knn_graph`
* - :ref:`non_iid<Non-IID Issue>`
- Yes
-
:ref:`is_non_iid_issue<\`\`is_non_iid_issue\`\`>`
:ref:`non_iid_score<\`\`non_iid_score\`\`>`
- `features` or `knn_graph`
* - :ref:`null<Null Issue>`
- Yes
-
:ref:`is_null_issue<\`\`is_null_issue\`\`>`
:ref:`null_score<\`\`null_score\`\`>`
- `features`
docs/source/cleanlab/datalab/index.rst
+31
View File
@@ -0,0 +1,31 @@
datalab
=======
.. automodule:: cleanlab.datalab
:autosummary:
:members:
:undoc-members:
:show-inheritance:
Getting Started
---------------
.. include:: optional_dependencies.rst
Guides
------
.. toctree::
:maxdepth: 2
guide/index
API Reference
-------------
.. toctree::
:maxdepth: 2
datalab
internal/index
docs/source/cleanlab/datalab/internal/adapter/imagelab.rst
+9
View File
@@ -0,0 +1,9 @@
imagelab
=============
.. automodule:: cleanlab.datalab.internal.adapter.imagelab
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/datalab/internal/adapter/index.rst
+10
View File
@@ -0,0 +1,10 @@
adapter
=============
.. warning::
Methods in this ``adapter`` module are intended for internal use within the ``cleanlab`` package. They are not guaranteed to be stable between different versions.
.. toctree::
:maxdepth: 2
imagelab
docs/source/cleanlab/datalab/internal/data.rst
+9
View File
@@ -0,0 +1,9 @@
data
====
.. automodule:: cleanlab.datalab.internal.data
:autosummary:
:members:
:undoc-members:
:show-inheritance:
:ignore-module-all:
docs/source/cleanlab/datalab/internal/data_issues.rst
+9
View File
@@ -0,0 +1,9 @@
data_issues
===========
.. automodule:: cleanlab.datalab.internal.data_issues
:autosummary:
:members:
:undoc-members:
:show-inheritance:
:ignore-module-all:
docs/source/cleanlab/datalab/internal/factory.rst
+9
View File
@@ -0,0 +1,9 @@
factory
=======
.. automodule:: cleanlab.datalab.internal.issue_manager_factory
:autosummary:
:members:
:undoc-members:
:show-inheritance:
:ignore-module-all:
docs/source/cleanlab/datalab/internal/index.rst
+26
View File
@@ -0,0 +1,26 @@
internal
========
.. warning::
Methods in this ``internal`` module are intended for internal use within the ``cleanlab`` package. They are not guaranteed to be stable between different versions.
.. automodule:: cleanlab.datalab.internal
:autosummary:
:members:
:undoc-members:
:show-inheritance:
.. toctree::
:maxdepth: 2
adapter/index
data
data_issues
issue_finder
factory
model_outputs
issue_manager/index
report
task
docs/source/cleanlab/datalab/internal/issue_finder.rst
+12
View File
@@ -0,0 +1,12 @@
issue_finder
============
.. note:: This module is not intended to be used directly by users. It is used by the :mod:`cleanlab.datalab.datalab` module.
Specifically, it is used by the :py:meth:`Datalab.find_issues <cleanlab.datalab.datalab.Datalab.find_issues>` method.
.. automodule:: cleanlab.datalab.internal.issue_finder
:autosummary:
:members:
:undoc-members:
:show-inheritance:
:ignore-module-all:
docs/source/cleanlab/datalab/internal/issue_manager/_notices/not_registered.rst
+5
View File
@@ -0,0 +1,5 @@
.. warning::
This issue manager isn't set up for direct Datalab use yet.
Register it first using `~cleanlab.datalab.internal.issue_manager_factory.register`.
docs/source/cleanlab/datalab/internal/issue_manager/data_valuation.rst
+9
View File
@@ -0,0 +1,9 @@
data_valuation
===========
.. automodule:: cleanlab.datalab.internal.issue_manager.data_valuation
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/datalab/internal/issue_manager/duplicate.rst
+9
View File
@@ -0,0 +1,9 @@
duplicate
=========
.. automodule:: cleanlab.datalab.internal.issue_manager.duplicate
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/datalab/internal/issue_manager/imbalance.rst
+9
View File
@@ -0,0 +1,9 @@
imbalance
=========
.. automodule:: cleanlab.datalab.internal.issue_manager.imbalance
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/datalab/internal/issue_manager/index.rst
+29
View File
@@ -0,0 +1,29 @@
issue_manager
=============
.. warning::
Methods in this ``issue_manager`` module are bleeding edge and may have sharp edges. They are not guaranteed to be stable between different ``cleanlab`` versions.
Registered issue managers
-------------------------
These are the issue managers that Datalab has registered.
.. toctree::
Base issue_manager module <issue_manager>
label
outlier
duplicate
noniid
imbalance
underperforming_group
null
data_valuation
ML task-specific issue managers
---------------------------------
.. toctree::
regression/index
multilabel/index
docs/source/cleanlab/datalab/internal/issue_manager/issue_manager.rst
+8
View File
@@ -0,0 +1,8 @@
issue_manager
=============
.. automodule:: cleanlab.datalab.internal.issue_manager.issue_manager
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/datalab/internal/issue_manager/label.rst
+8
View File
@@ -0,0 +1,8 @@
label
=====
.. automodule:: cleanlab.datalab.internal.issue_manager.label
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/datalab/internal/issue_manager/multilabel/index.rst
+8
View File
@@ -0,0 +1,8 @@
multilabel
==========
.. warning::
Methods in this ``multilabel`` module are bleeding edge and may have sharp edges. They are not guaranteed to be stable between different ``cleanlab`` versions.
.. toctree::
label
docs/source/cleanlab/datalab/internal/issue_manager/multilabel/label.rst
+8
View File
@@ -0,0 +1,8 @@
label
=====
.. automodule:: cleanlab.datalab.internal.issue_manager.multilabel.label
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/datalab/internal/issue_manager/noniid.rst
+9
View File
@@ -0,0 +1,9 @@
noniid
=======
.. automodule:: cleanlab.datalab.internal.issue_manager.noniid
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/datalab/internal/issue_manager/null.rst
+10
View File
@@ -0,0 +1,10 @@
null
====
.. include:: _notices/not_registered.rst
.. automodule:: cleanlab.datalab.internal.issue_manager.null
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/datalab/internal/issue_manager/outlier.rst
+9
View File
@@ -0,0 +1,9 @@
outlier
=======
.. automodule:: cleanlab.datalab.internal.issue_manager.outlier
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/datalab/internal/issue_manager/regression/index.rst
+8
View File
@@ -0,0 +1,8 @@
regression
==========
.. warning::
Methods in this ``regression`` module are bleeding edge and may have sharp edges. They are not guaranteed to be stable between different ``cleanlab`` versions.
.. toctree::
label
docs/source/cleanlab/datalab/internal/issue_manager/regression/label.rst
+8
View File
@@ -0,0 +1,8 @@
label
=====
.. automodule:: cleanlab.datalab.internal.issue_manager.regression.label
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/datalab/internal/issue_manager/underperforming_group.rst
+9
View File
@@ -0,0 +1,9 @@
underperforming_group
=========
.. automodule:: cleanlab.datalab.internal.issue_manager.underperforming_group
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/datalab/internal/model_outputs.rst
+7
View File
@@ -0,0 +1,7 @@
model_outputs
=============
.. automodule:: cleanlab.datalab.internal.model_outputs
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/datalab/internal/report.rst
+12
View File
@@ -0,0 +1,12 @@
report
======
.. note:: This module is not intended to be used directly by users. It is used by the :mod:`cleanlab.datalab.datalab` module.
Specifically, it is used by the :py:meth:`Datalab.report <cleanlab.datalab.datalab.Datalab.report>` method.
.. automodule:: cleanlab.datalab.internal.report
:autosummary:
:members:
:undoc-members:
:show-inheritance:
:ignore-module-all:
docs/source/cleanlab/datalab/internal/task.rst
+11
View File
@@ -0,0 +1,11 @@
task
====
.. note:: This module is not intended to be used directly by users. It is used by the :mod:`cleanlab.datalab.datalab` module.
.. automodule:: cleanlab.datalab.internal.task
:autosummary:
:members:
:undoc-members:
:show-inheritance:
:ignore-module-all:
docs/source/cleanlab/datalab/optional_dependencies.rst
+11
View File
@@ -0,0 +1,11 @@
Using Datalab requires additional dependencies beyond the rest of the ``cleanlab`` package. To install them, run:
.. code-block:: console
$ pip install "cleanlab[datalab]"
For the developmental version of the package, install from source:
.. code-block:: console
$ pip install "git+https://github.com/cleanlab/cleanlab.git#egg=cleanlab[datalab]"
docs/source/cleanlab/dataset.rst
+72
View File
@@ -0,0 +1,72 @@
dataset
=======
.. automodule:: cleanlab.dataset
:autosummary:
:members:
:undoc-members:
:show-inheritance:
.. testsetup:: *
import cleanlab
import numpy as np
from cleanlab.benchmarking import noise_generation
SEED = 0
def get_data_labels_from_dataset(
means=[[3, 2], [7, 7], [0, 8], [0, 10]],
covs=[
[[5, -1.5], [-1.5, 1]],
[[1, 0.5], [0.5, 4]],
[[5, 1], [1, 5]],
[[3, 1], [1, 1]],
],
sizes=[100, 50, 50, 50],
avg_trace=0.8,
seed=SEED, # set to None for non-reproducible randomness
):
np.random.seed(seed=SEED)
K = len(means) # number of classes
data = []
labels = []
test_data = []
test_labels = []
for idx in range(K):
data.append(
np.random.multivariate_normal(
mean=means[idx], cov=covs[idx], size=sizes[idx]
)
)
test_data.append(
np.random.multivariate_normal(
mean=means[idx], cov=covs[idx], size=sizes[idx]
)
)
labels.append(np.array([idx for i in range(sizes[idx])]))
test_labels.append(np.array([idx for i in range(sizes[idx])]))
X_train = np.vstack(data)
y_train = np.hstack(labels)
X_test = np.vstack(test_data)
y_test = np.hstack(test_labels)
# Compute p(y=k) the prior distribution over true labels.
py_true = np.bincount(y_train) / float(len(y_train))
noise_matrix_true = noise_generation.generate_noise_matrix_from_trace(
K,
trace=avg_trace * K,
py=py_true,
valid_noise_matrix=True,
seed=SEED,
)
# Generate our noisy labels using the noise_marix.
s = noise_generation.generate_noisy_labels(y_train, noise_matrix_true)
s_test = noise_generation.generate_noisy_labels(y_test, noise_matrix_true)
ps = np.bincount(s) / float(len(s)) # Prior distribution over noisy labels
return X_train, s
docs/source/cleanlab/experimental/coteaching.rst
+8
View File
@@ -0,0 +1,8 @@
coteaching
==========
.. automodule:: cleanlab.experimental.coteaching
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/experimental/index.rst
+22
View File
@@ -0,0 +1,22 @@
experimental
============
.. warning::
Methods in this ``experimental`` module are bleeding edge and may have sharp edges. They are not guaranteed to be stable between different ``cleanlab`` versions.
Useful methods/models adapted for use with cleanlab.
Some of these files include various models that can be used with cleanlab to find issues in specific types of data. These require dependencies on deep learning and other machine learning packages that are not official cleanlab dependencies. You must install these dependencies on your own if you wish to use them.
.. automodule:: cleanlab.experimental
:autosummary:
:members:
:undoc-members:
:show-inheritance:
.. toctree::
label_issues_batched
span_classification
mnist_pytorch
coteaching
cifar_cnn
docs/source/cleanlab/experimental/label_issues_batched.rst
+8
View File
@@ -0,0 +1,8 @@
label_issues_batched
====================
.. automodule:: cleanlab.experimental.label_issues_batched
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/experimental/mnist_pytorch.rst
+8
View File
@@ -0,0 +1,8 @@
mnist_pytorch
=============
.. automodule:: cleanlab.experimental.mnist_pytorch
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/experimental/span_classification.rst
+8
View File
@@ -0,0 +1,8 @@
span_classification
===================
.. automodule:: cleanlab.experimental.span_classification
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/filter.rst
+8
View File
@@ -0,0 +1,8 @@
filter
=======
.. automodule:: cleanlab.filter
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/internal/index.rst
+23
View File
@@ -0,0 +1,23 @@
internal
========
.. warning::
These ``internal`` utility methods are intended for internal use within the ``cleanlab`` package. They are not guaranteed to be stable between different versions.
.. automodule:: cleanlab.internal
:autosummary:
:members:
:undoc-members:
:show-inheritance:
.. toctree::
util
latent_algebra
label_quality_utils
multilabel_utils
multilabel_scorer
neighbor/index
outlier
token_classification_utils
validation
docs/source/cleanlab/internal/label_quality_utils.rst
+8
View File
@@ -0,0 +1,8 @@
label_quality_utils
===================
.. automodule:: cleanlab.internal.label_quality_utils
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/internal/latent_algebra.rst
+8
View File
@@ -0,0 +1,8 @@
latent_algebra
==============
.. automodule:: cleanlab.internal.latent_algebra
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/internal/multiannotator_utils.rst
+8
View File
@@ -0,0 +1,8 @@
multiannotator_utils
==========================
.. automodule:: cleanlab.internal.multiannotator_utils
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/internal/multilabel_scorer.rst
+8
View File
@@ -0,0 +1,8 @@
multilabel_scorer
=================
.. automodule:: cleanlab.internal.multilabel_scorer
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/internal/multilabel_utils.rst
+8
View File
@@ -0,0 +1,8 @@
multilabel_utils
================
.. automodule:: cleanlab.internal.multilabel_utils
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/internal/neighbor/index.rst
+21
View File
@@ -0,0 +1,21 @@
neighbor
========
The `neighbor` modules provide functionality for performing nearest neighbor search and pairwise distance calculations in those searches.
This submodule consists of the following modules:
- `neighbor.knn_graph`: Contains functions for setting up a nearest neighbor search index and constructing knn graphs.
- `neighbor.search`: Contains a helper function that wraps the default implementation of nearest neighbor searches.
- `neighbor.metric`: Contains functions for selecting distance metrics for nearest neighbor searches.
.. automodule:: cleanlab.internal.neighbor
:autosummary:
:members:
:undoc-members:
:show-inheritance:
.. toctree::
knn_graph
metric
search
docs/source/cleanlab/internal/neighbor/knn_graph.rst
+8
View File
@@ -0,0 +1,8 @@
knn_graph
=========
.. automodule:: cleanlab.internal.neighbor.knn_graph
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/internal/neighbor/metric.rst
+8
View File
@@ -0,0 +1,8 @@
metric
======
.. automodule:: cleanlab.internal.neighbor.metric
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/internal/neighbor/search.rst
+8
View File
@@ -0,0 +1,8 @@
search
======
.. automodule:: cleanlab.internal.neighbor.search
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/internal/outlier.rst
+8
View File
@@ -0,0 +1,8 @@
outlier
=======
.. automodule:: cleanlab.internal.outlier
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/internal/token_classification_utils.rst
+8
View File
@@ -0,0 +1,8 @@
token_classification_utils
==========================
.. automodule:: cleanlab.internal.token_classification_utils
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/internal/util.rst
+8
View File
@@ -0,0 +1,8 @@
util
====
.. automodule:: cleanlab.internal.util
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/internal/validation.rst
+8
View File
@@ -0,0 +1,8 @@
validation
==========
.. automodule:: cleanlab.internal.validation
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/models/index.rst
+16
View File
@@ -0,0 +1,16 @@
models
======
.. warning::
Methods in this ``models`` module are not guaranteed to be stable between different ``cleanlab`` versions.
Useful models adapted for use with cleanlab.
Some of these files include various models that can be used with cleanlab to find issues in specific types of data. These require dependencies on deep learning and other machine learning packages that are not official cleanlab dependencies. You must install these dependencies on your own if you wish to use them.
.. automodule:: cleanlab.models
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/multiannotator.rst
+8
View File
@@ -0,0 +1,8 @@
multiannotator
==============
.. automodule:: cleanlab.multiannotator
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/multilabel_classification/dataset.rst
+8
View File
@@ -0,0 +1,8 @@
dataset
=======
.. automodule:: cleanlab.multilabel_classification.dataset
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/multilabel_classification/filter.rst
+8
View File
@@ -0,0 +1,8 @@
filter
======
.. automodule:: cleanlab.multilabel_classification.filter
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/multilabel_classification/index.rst
+22
View File
@@ -0,0 +1,22 @@
multilabel_classification
=========================
Methods to detect data and label issues in multi-label classification datasets.
In multi-class classification, each example in the dataset belongs to exactly 1 out of K classes (e.g. if classifying animals as: {dog, cat, rat}).
In multi-label classification, each example in the dataset can belong to 1 or more classes (out of K possible classes), or none of the classes at all (e.g. if classifying animals as: {predator, pet, reptile}).
.. automodule:: cleanlab.multilabel_classification
:autosummary:
:members:
:undoc-members:
:show-inheritance:
.. toctree::
filter
rank
dataset
docs/source/cleanlab/multilabel_classification/rank.rst
+8
View File
@@ -0,0 +1,8 @@
rank
====
.. automodule:: cleanlab.multilabel_classification.rank
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/object_detection/filter.rst
+9
View File
@@ -0,0 +1,9 @@
filter
====
.. automodule:: cleanlab.object_detection.filter
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/object_detection/index.rst
+16
View File
@@ -0,0 +1,16 @@
object_detection
====================
Methods to detect label issues in object detection datasets.
.. automodule:: cleanlab.object_detection
:autosummary:
:members:
:undoc-members:
:show-inheritance:
.. toctree::
rank
filter
summary
docs/source/cleanlab/object_detection/rank.rst
+9
View File
@@ -0,0 +1,9 @@
rank
====
.. automodule:: cleanlab.object_detection.rank
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/object_detection/summary.rst
+9
View File
@@ -0,0 +1,9 @@
summary
====
.. automodule:: cleanlab.object_detection.summary
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/outlier.rst
+8
View File
@@ -0,0 +1,8 @@
outlier
==============
.. automodule:: cleanlab.outlier
:autosummary:
:members:
:show-inheritance:
:exclude-members: DEFAULT_PARAM_DICT, OOD_PARAMS, OUTLIER_PARAMS
docs/source/cleanlab/rank.rst
+8
View File
@@ -0,0 +1,8 @@
rank
====
.. automodule:: cleanlab.rank
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/regression/index.rst
+15
View File
@@ -0,0 +1,15 @@
regression
====================
Methods to detect data and label issues in regression datasets.
.. automodule:: cleanlab.regression
:autosummary:
:members:
:undoc-members:
:show-inheritance:
.. toctree::
rank
learn
docs/source/cleanlab/regression/learn.rst
+8
View File
@@ -0,0 +1,8 @@
regression.learn
================
.. automodule:: cleanlab.regression.learn
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/regression/rank.rst
+8
View File
@@ -0,0 +1,8 @@
regression.rank
===============
.. automodule:: cleanlab.regression.rank
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/segmentation/filter.rst
+8
View File
@@ -0,0 +1,8 @@
filter
====
.. automodule:: cleanlab.segmentation.filter
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/segmentation/index.rst
+16
View File
@@ -0,0 +1,16 @@
segmentation
============
Methods to detect label issues in segmentation datasets.
.. automodule:: cleanlab.segmentation
:autosummary:
:members:
:undoc-members:
:show-inheritance:
.. toctree::
rank
filter
summary
docs/source/cleanlab/segmentation/rank.rst
+8
View File
@@ -0,0 +1,8 @@
rank
====
.. automodule:: cleanlab.segmentation.rank
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/segmentation/summary.rst
+8
View File
@@ -0,0 +1,8 @@
summary
=======
.. automodule:: cleanlab.segmentation.summary
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/token_classification/filter.rst
+8
View File
@@ -0,0 +1,8 @@
filter
======
.. automodule:: cleanlab.token_classification.filter
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/token_classification/index.rst
+16
View File
@@ -0,0 +1,16 @@
token_classification
====================
Methods to detect data and label issues in token classification datasets.
.. automodule:: cleanlab.token_classification
:autosummary:
:members:
:undoc-members:
:show-inheritance:
.. toctree::
filter
rank
summary
docs/source/cleanlab/token_classification/rank.rst
+8
View File
@@ -0,0 +1,8 @@
rank
====
.. automodule:: cleanlab.token_classification.rank
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/cleanlab/token_classification/summary.rst
+8
View File
@@ -0,0 +1,8 @@
summary
=======
.. automodule:: cleanlab.token_classification.summary
:autosummary:
:members:
:undoc-members:
:show-inheritance:
docs/source/conf.py
+296
View File
@@ -0,0 +1,296 @@
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
import os
import sys
import datetime
import shutil
sys.path.insert(0, os.path.abspath("../../cleanlab"))
# doctest setup
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here.
import pathlib
sys.path.insert(0, pathlib.Path(__file__).parents[2].resolve().as_posix())
# Open Graph extension
ogp_site_url = "https://docs.cleanlab.ai"
ogp_image = (
"https://raw.githubusercontent.com/cleanlab/assets/master/cleanlab/clos-preview-card.png"
)
# -- Project information -----------------------------------------------------
project = "cleanlab"
copyright = f"{datetime.datetime.now().year}, Cleanlab Inc."
author = "Cleanlab Inc."
# -- General configuration ---------------------------------------------------
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
"sphinx.ext.napoleon",
"nbsphinx",
"sphinx.ext.autodoc",
"sphinx.ext.autosectionlabel",
"autodocsumm",
"sphinx.ext.viewcode",
"sphinx.ext.todo",
"sphinx_tabs.tabs",
"sphinx_multiversion",
"sphinx_copybutton",
"sphinxcontrib.katex",
"sphinxcontrib.gtagjs",
"sphinx_jinja",
"sphinx_autodoc_typehints",
"sphinx.ext.doctest",
"sphinxext.opengraph",
]
numpy_show_class_members = True
# Don't add .txt suffix to source files:
html_sourcelink_suffix = ""
# Add any paths that contain templates here, relative to this directory.
templates_path = ["_templates"]
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ["_build"]
autosummary_generate = True
# set the default role of `name` to make cross references
default_role = "py:obj"
# -- Options for apidoc extension ----------------------------------------------
# apidoc_module_dir = "cleanlab/cleanlab"
# -- Options for todo extension ----------------------------------------------
# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True
# -- Options for Napoleon extension -------------------------------------------
napoleon_google_docstring = False
napoleon_numpy_docstring = True
napoleon_include_init_with_doc = False
napoleon_include_private_with_doc = False
napoleon_include_special_with_doc = True
napoleon_use_admonition_for_examples = False
napoleon_use_admonition_for_notes = False
napoleon_use_admonition_for_references = False
napoleon_use_ivar = False
napoleon_use_param = True
napoleon_use_rtype = False
napoleon_preprocess_types = True
napoleon_type_aliases = None
napoleon_attr_annotations = True
# -- Options for autodoc extension -------------------------------------------
# This value selects what content will be inserted into the main body of an autoclass
# directive
#
# http://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#directive-autoclass
autoclass_content = "class"
# Default options to an ..autoXXX directive.
autodoc_default_options = {
"autosummary": True,
"members": None,
"inherited-members": None,
"show-inheritance": None,
"special-members": "__call__",
}
# Subclasses should show parent classes docstrings if they don't override them.
autodoc_inherit_docstrings = True
# Order functions displayed by the order of source code
autodoc_member_order = "bysource"
# -- Options for copybutton extension -----------------------------------------
# Strip input prompts when copying code blocks. Supports:
# - Python Repl + continuation prompt
# - Bash prompt
# - ipython + continuation prompt
# - jupyter-console + continuation prompt
copybutton_prompt_text = r">>> |\.\.\. |\$ |In \[\d*\]: | {2,5}\.\.\.: | {5,8}: "
copybutton_prompt_is_regexp = True
# -- Options for katex extension -------------------------------------------
if os.getenv("CI") or shutil.which("katex") is not None:
# requires that the machine have `katex` installed: `npm install -g katex`
katex_prerender = True
# -- Options for gtagjs extension -------------------------------------------
gtagjs_ids = ["G-EV8RVEFX82"]
# -- Variables Setting ---------------------------------------------------
# Determine doc site URL (DOCS_SITE_URL)
# Check if it's running in production repo
if os.getenv("GITHUB_REPOSITORY") == "cleanlab/cleanlab":
DOCS_SITE_URL = "/"
else:
DOCS_SITE_URL = "/cleanlab-docs/"
gh_env_file = os.getenv("GITHUB_ENV")
if gh_env_file is not None:
with open(gh_env_file, "a") as f:
f.write(f"\nDOCS_SITE_URL={DOCS_SITE_URL}") # Set to Environment Var
GITHUB_REPOSITORY_OWNER = os.getenv("GITHUB_REPOSITORY_OWNER") or "cleanlab"
GITHUB_REF_NAME = os.getenv("GITHUB_REF_NAME") or "master"
# Pass additional variables to Jinja templates
html_context = {
"DOCS_SITE_URL": DOCS_SITE_URL,
# Add new tags to RELEASE_VERSIONS before release
# fmt: off
"RELEASE_VERSIONS": [
"v2.7.1",
"v2.7.0",
"v2.6.6",
"v2.6.5",
"v2.6.4",
"v2.6.3",
"v2.6.2",
"v2.6.1",
"v2.6.0",
"v2.5.0",
"v2.4.0",
"v2.3.1",
"v2.3.0",
"v2.2.0",
"v2.1.0",
"v2.0.0",
"v1.0.1",
],
# fmt: on
}
# -- nbsphinx Configuration ---------------------------------------------------
# This is processed by Jinja2 and inserted before each notebook
nbsphinx_prolog = (
"""
{% set docname = env.doc2path(env.docname, base=None) %}
.. raw:: html
<style>
.nbinput .prompt,
.nboutput .prompt {
display: none;
}
.output_area {
max-height: 300px;
overflow: auto;
}
.dataframe {
background: #D7D7D7;
}
th {
color:black;
}
</style>
<script type="text/javascript">
window.addEventListener('load', () => {
const h1_element = document.getElementsByTagName("h1");
h1_element[0].insertAdjacentHTML("afterend", `
<p>
<a style="background-color:white;color:black;padding:4px 12px;text-decoration:none;display:inline-block;border-radius:8px;box-shadow:0 2px 4px 0 rgba(0, 0, 0, 0.2), 0 3px 10px 0 rgba(0, 0, 0, 0.19)" href="https://colab.research.google.com/github/"""
+ GITHUB_REPOSITORY_OWNER
+ """/cleanlab-docs/blob/master/"""
+ GITHUB_REF_NAME
+ """/{{ docname|e }}" target="_blank">
<img src="https://colab.research.google.com/img/colab_favicon_256px.png" alt="" style="width:40px;height:40px;vertical-align:middle">
<span style="vertical-align:middle">Run in Google Colab</span>
</a>
</p>
`);
})
</script>
"""
)
# Change this to "always" before running in the doc's CI/CD server
if os.getenv("CI"):
nbsphinx_execute = "always"
if os.getenv("SKIP_NOTEBOOKS", "0") != "0":
nbsphinx_execute = "never"
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = "furo"
html_favicon = "https://raw.githubusercontent.com/cleanlab/assets/a4483476d449f2f05a4c7cde329e72358099cc07/cleanlab/cleanlab_favicon.svg"
html_title = "cleanlab"
html_logo = (
"https://raw.githubusercontent.com/cleanlab/assets/master/cleanlab/cleanlab_logo_only.png"
)
html_theme_options = {
"footer_icons": [
{
"name": "GitHub",
"url": "https://github.com/cleanlab/cleanlab",
"html": """
<svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 16 16">
<path fill-rule="evenodd" d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path>
</svg>
""",
"class": "",
},
],
}
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ["_static"]
html_css_files = ["css/custom.css"]
html_sidebars = {
"**": [
"brand.html",
"sidebar/search.html",
"sidebar/scroll-start.html",
"sidebar/navigation.html",
"sidebar/ethical-ads.html",
"versioning.html",
"sidebar/scroll-end.html",
],
}
docs/source/index.rst
+208
View File
@@ -0,0 +1,208 @@
:og:title: Open-Source Documentation | Cleanlab
:og:description: Get started with Cleanlab Open-Source, learn about capabilities, and follow tutorials to improve your own Data and Models.
cleanlab open-source documentation
==================================
`cleanlab <https://github.com/cleanlab/cleanlab>`_ **automatically detects data and label issues in your ML datasets.**
| This helps you improve your data and train robust ML models on noisy real-world datasets. cleanlab has already found thousands of `label errors <https://labelerrors.com>`_ in ImageNet, MNIST, and other popular ML benchmarking datasets. Beyond handling label errors, this is a comprehensive open-source library implementing many data-centric AI capabilities. Start using automation to improve your data!
Quickstart
==========
1. Install ``cleanlab``
-----------------------
.. tabs::
.. tab:: uv
.. code-block:: bash
uv pip install cleanlab
To install the package with all optional dependencies:
.. code-block:: bash
uv pip install "cleanlab[all]"
.. tab:: pip
.. code-block:: bash
pip install cleanlab
To install the package with all optional dependencies:
.. code-block:: bash
pip install "cleanlab[all]"
.. tab:: conda
.. code-block:: bash
conda install -c cleanlab cleanlab
.. tab:: source
.. code-block:: bash
uv pip install git+https://github.com/cleanlab/cleanlab.git
Or with pip:
.. code-block:: bash
pip install git+https://github.com/cleanlab/cleanlab.git
To install the package with all optional dependencies:
.. code-block:: bash
uv pip install "git+https://github.com/cleanlab/cleanlab.git#egg=cleanlab[all]"
2. Check your data for all sorts of issues
------------------------------------------
cleanlab automatically detects various issues in *any dataset that a classifier can be trained on*. The cleanlab package *works with any ML model* by operating on model outputs (predicted class probabilities or feature embeddings) -- it doesn't require that a particular model created those outputs. For any classification dataset, use your trained model to produce `pred_probs` (predicted class probabilities) and/or `feature_embeddings` (numeric vector representations of each datapoint). To automatically check your dataset for common real-world issues (like label errors, outliers, near duplicates, IID violations, underperforming groups, ...), simply run these few lines of code:
.. code-block:: python
from cleanlab import Datalab
lab = Datalab(data=your_dataset, label_name="column_name_of_labels")
lab.find_issues(features=feature_embeddings, pred_probs=pred_probs)
lab.report() # summarize issues in dataset, how severe they are in each data point, ...
While other data quality tools only catch limited types of data issues based on manually pre-defined validation rules, cleanlab applies automated data-centric AI techniques using your trained ML model to detect many more types of data issues that would otherwise be hard to catch. Don't dive into ML model improvement without first using AI to help check your data!
3. Handle label errors and train robust models with noisy labels
----------------------------------------------------------------
Mislabeled data is a particularly concerning issue plaguing real-world datasets. To use a scikit-learn-compatible model for classification with noisy labels, you don't need to train a model to find label issues -- you can pass the untrained model object, data, and labels into :py:meth:`CleanLearning.find_label_issues <cleanlab.classification.CleanLearning.find_label_issues>` and cleanlab will handle model training for you.
.. code-block:: python
from cleanlab.classification import CleanLearning
# This works with any sklearn-compatible model - just input data + labels and cleanlab will detect label issues ツ
label_issues_info = CleanLearning(clf=sklearn_compatible_model).find_label_issues(data, labels)
:py:class:`CleanLearning <cleanlab.classification.CleanLearning>` also works with models from most standard ML frameworks by wrapping the model for scikit-learn compliance, e.g. pytorch (can use `skorch <https://github.com/skorch-dev/skorch>`_ package), etc.
:py:meth:`find_label_issues <cleanlab.classification.CleanLearning.find_label_issues>` returns a boolean mask flagging which examples have label issues and a numeric label quality score for each example quantifying our confidence that its label is correct.
Beyond standard classification tasks, cleanlab can also detect mislabeled examples in: `multi-label data <tutorials/multilabel_classification.ipynb>`_ (e.g. image/document tagging), `sequence prediction <tutorials/token_classification.ipynb>`_ (e.g. entity recognition), and `data labeled by multiple annotators <tutorials/multiannotator.ipynb>`_ (e.g. crowdsourcing).
.. important::
Cleanlab performs better if the ``pred_probs`` from your model are **out-of-sample**. Details on how to compute out-of-sample predicted probabilities for your entire dataset are :ref:`here <pred_probs_cross_val>`.
cleanlab's :py:class:`CleanLearning <cleanlab.classification.CleanLearning>` class trains a more robust version of any existing (`scikit-learn <https://scikit-learn.org/>`_ `compatible <https://scikit-learn.org/stable/developers/develop.html>`_) classification model, `clf`, by fitting it to an automatically filtered version of your dataset with low-quality data removed. It returns a model trained only on the clean data, from which you can get predictions in the same way as your existing classifier.
.. code-block:: python
from sklearn.linear_model import LogisticRegression
from cleanlab.classification import CleanLearning
cl = CleanLearning(clf=LogisticRegression()) # any sklearn-compatible classifier
cl.fit(train_data, labels)
# Estimate the predictions you would have gotten if you trained without mislabeled data
predictions = cl.predict(test_data)
4. Dataset curation: fix dataset-level issues
---------------------------------------------
cleanlab's `dataset <tutorials/dataset_health.ipynb>`_ module helps you deal with dataset-level issues -- :py:meth:`find overlapping classes <cleanlab.dataset.find_overlapping_classes>` (classes to merge), :py:meth:`rank class-level label quality <cleanlab.dataset.rank_classes_by_label_quality>` (classes to keep/delete), and :py:meth:`measure overall dataset health <cleanlab.dataset.overall_label_health_score>` (to track dataset quality as you make adjustments).
View all dataset-level issues in one line of code with :py:meth:`dataset.health_summary() <cleanlab.dataset.health_summary>`.
.. code-block:: python
from cleanlab.dataset import health_summary
health_summary(labels, pred_probs, class_names=class_names)
5. Improve your data via many other techniques
----------------------------------------------
Beyond handling label errors, cleanlab supports other data-centric AI capabilities including:
- Detecting outliers and out-of-distribution examples in both training and future test data `(tutorial) <tutorials/outliers.ipynb>`_
- Analyzing data labeled by multiple annotators to estimate consensus labels and their quality `(tutorial) <tutorials/multiannotator.ipynb>`_
- Active learning with multiple annotators to identify which data is most informative to label or re-label next `(tutorial) <https://github.com/cleanlab/examples/blob/master/active_learning_multiannotator/active_learning.ipynb>`_
If you have questions, check out our `FAQ <tutorials/faq.ipynb>`_ and feel free to open a Github Issue!
.. toctree::
:hidden:
Quickstart <self>
.. toctree::
:hidden:
:caption: Tutorials
Datalab Tutorials <tutorials/datalab/index>
Improving ML Performance <tutorials/improving_ml_performance>
CleanLearning Tutorials <tutorials/clean_learning/index>
Workflows of Data-Centric AI <tutorials/indepth_overview>
Analyze Dataset-level Issues <tutorials/dataset_health>
Outlier Detection <tutorials/outliers>
Improving Consensus Labels for Multiannotator Data <tutorials/multiannotator>
Multi-Label Classification <tutorials/multilabel_classification>
Noisy Labels in Regression <tutorials/regression>
Token Classification (text) <tutorials/token_classification>
Image Segmentation <tutorials/segmentation>
Object Detection <tutorials/object_detection>
Predicted Probabilities via Cross Validation <tutorials/pred_probs_cross_val>
FAQ <tutorials/faq>
.. toctree::
:caption: API Reference
:hidden:
:maxdepth: 3
cleanlab/datalab/index
cleanlab/classification
cleanlab/filter
cleanlab/rank
cleanlab/count
cleanlab/dataset
cleanlab/outlier
cleanlab/multiannotator
cleanlab/data_valuation
cleanlab/multilabel_classification/index
cleanlab/regression/index
cleanlab/token_classification/index
cleanlab/segmentation/index
cleanlab/object_detection/index
cleanlab/benchmarking/index
cleanlab/models/index
cleanlab/experimental/index
cleanlab/internal/index
.. toctree::
:caption: Guides
:hidden:
Datalab issue types <cleanlab/datalab/guide/index>
.. toctree::
:caption: Links
:hidden:
GitHub <https://github.com/cleanlab/cleanlab>
PyPI <https://pypi.org/project/cleanlab/>
Conda <https://anaconda.org/conda-forge/cleanlab>
Blog <https://cleanlab.ai/blog/>
docs/source/migrating/migrate_v2.rst
+87
View File
@@ -0,0 +1,87 @@
How to migrate to versions >= 2.0.0 from pre 1.0.1
==================================================
If you previously used older versions of cleanlab,
this guide helps update your existing code to work with versions >= 2.0.0 in no time!
Below we outline the major updates and code substitutions to be aware of.
A detailed API change-log is listed in the `v2.0.0. Release Notes <https://github.com/cleanlab/cleanlab/releases/tag/v2.0.0>`_.
Function and class name changes
-------------------------------
This section covers the most commonly-used functionality from Cleanlab 1.0.
| **Old:** ``pruning.get_noise_indices(s, psx, prune_method, sorted_index_method, ...)``
| -->
| **New:** :py:func:`filter.find_label_issues <cleanlab.filter.find_label_issues>` ``(labels, pred_probs, filter_by, return_indices_ranked_by, ...)``
Note: ``inverse_noise_matrix`` is no longer a supported input argument, but ``confident_joint`` remains (you can easily convert between these two).
----
| **Old:** ``pruning.order_label_errors(label_errors_bool, psx, labels, sorted_index_method)``
| -->
| **New:** :py:func:`rank.order_label_issues <cleanlab.rank.order_label_issues>` ``(label_issues_mask, labels, pred_probs, rank_by, ...)``
Note: You can now alternatively use :py:func:`rank.get_label_quality_score() <cleanlab.rank.get_label_quality_score>` to numerically score the labels instead of ranking them.
----
| **Old:** ``latent_estimation.num_label_errors(labels, psx, ...)``
| -->
| **New:** :py:func:`count.num_label_issues <cleanlab.count.num_label_issues>` ``(labels, pred_probs, ...)``
Note: This is the most accurate way to estimate the raw *number* of label errors in a dataset.
----
| **Old:** ``classification.LearningWithNoisyLabels(..., prune_method)``
| -->
| **New:** :py:class:`classification.CleanLearning <cleanlab.classification.CleanLearning>` ``(..., find_label_issues_kwargs)``
Note: :py:class:`CleanLearning <cleanlab.classification.CleanLearning>` can now find label errors for you, neatly organizing them in a ``pandas.DataFrame`` as well as computing the required out-of-sample predicted probabilities. You just specify which classifier, we handle the cross-validation!
Module name changes
-------------------
Reorganized modules:
- ``cleanlab.pruning`` --> :py:mod:`cleanlab.filter`
- ``cleanlab.latent_estimation`` --> :py:mod:`cleanlab.count`
- ``cleanlab.noise_generation`` --> :py:mod:`cleanlab.benchmarking.noise_generation`
- ``cleanlab.baseline_methods`` --> incorporated into :py:mod:`cleanlab.filter`
Internal and experimental functionality, marked as such and not guaranteed to be stable between releases:
- ``cleanlab.models`` --> :py:mod:`cleanlab.experimental`
- ``cleanlab.coteaching`` --> :py:mod:`cleanlab.experimental.coteaching`
- ``cleanlab.latent_algebra`` --> :py:mod:`cleanlab.internal.latent_algebra`
- ``cleanlab.util`` --> :py:mod:`cleanlab.internal.util`
New modules
-----------
- :py:mod:`cleanlab.dataset` : New methods to print summaries of overall types of label issues most common in a dataset.
- :py:mod:`cleanlab.rank` : Moved all ranking and ordering functions from ``cleanlab.pruning`` to here. This module contains methods to score the label quality of each example and rank your data by the quality of their labels.
- :py:mod:`cleanlab.internal` and :py:mod:`cleanlab.experimental`: Moved all advanced code and utility methods to this module, including the old ``cleanlab.latent_algebra`` module. Researchers may find useful functions in here.
Removed modules
---------------
- ``cleanlab.polyplex``
Common argument and variable name changes
-----------------------------------------
Here are some common name and terminology changes in Cleanlab 2.0:
- ``s`` --> ``labels`` (the given labels in the data, which are potentially noisy)
- ``psx`` --> ``pred_probs`` (predicted probabilities output by trained classifier)
- ``label_error`` --> ``label_issue`` (a label that is likely to be wrong)
See the documentation for individual functions for details on how argument names changed.
docs/source/tutorials/.gitignore
+3
View File
@@ -0,0 +1,3 @@
*.tar.gz
pretrained_models/
spoken_digits/
docs/source/tutorials/clean_learning/index.rst
+8
View File
@@ -0,0 +1,8 @@
CleanLearning Tutorials
=======================
.. toctree::
:maxdepth: 1
Text Classification <text>
Tabular Classification (Numeric/Categorical) <tabular>
docs/source/tutorials/clean_learning/tabular.ipynb
+506
View File
@@ -0,0 +1,506 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Classification with Structured/Tabular Data and Noisy Labels\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<div class=\"alert alert-info\">\n",
"Consider Using Datalab\n",
"<br/>\n",
"\n",
"If interested in detecting a wide variety of issues in your tabular data, check out the [Datalab tabular tutorial](https://docs.cleanlab.ai/stable/tutorials/datalab/tabular.html). Datalab can detect many other types of data issues beyond label issues, whereas CleanLearning is a convenience method to handle noisy labels with sklearn-compatible classification models.\n",
"</div>"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"In this 5-minute quickstart tutorial, we use cleanlab with scikit-learn models to find potential label errors in a classification dataset with tabular features (numeric/categorical columns). Tabular (or *structured*) data are typically organized in a row/column format and stored in a SQL database or file types like: CSV, Excel, or Parquet. Here we consider a Student Grades dataset, which contains over 900 individuals who have three exam grades and some optional notes, each being assigned a letter grade (their class label). cleanlab automatically identifies _hundreds_ of examples in this dataset that were mislabeled with the incorrect final grade (data entry mistakes). \n",
"\n",
"This tutorial shows how to handle noisy labels and produce more robust classification models for your own tabular datasets. cleanlab's `CleanLearning` class automatically detects and filters out such badly labeled data, in order to train a more robust version of any Machine Learning model. No change to your existing modeling code is required! \n",
"\n",
"\n",
"**Overview of what we'll do in this tutorial:**\n",
"\n",
"- Train a classifier model (here scikit-learn's ExtraTreesClassifier, although any model could be used) and use this classifier to compute (out-of-sample) predicted class probabilities via cross-validation.\n",
"\n",
"- Identify potential label errors in the data with cleanlab's `find_label_issues` method.\n",
"\n",
"- Train a robust version of the same ExtraTrees model via cleanlab's `CleanLearning` wrapper.\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<div class=\"alert alert-info\">\n",
"Quickstart\n",
"<br/>\n",
" \n",
"Already have an sklearn compatible `model`, tabular `data` and given `labels`? Run the code below to train your `model` and get label issues.\n",
"\n",
"\n",
"<div class=markdown markdown=\"1\" style=\"background:white;margin:16px\"> \n",
" \n",
"```python\n",
"\n",
"from cleanlab.classification import CleanLearning\n",
"\n",
"cl = CleanLearning(model)\n",
"_ = cl.fit(train_data, labels)\n",
"label_issues = cl.get_label_issues()\n",
"preds = cl.predict(test_data) # predictions from a version of your model \n",
" # trained on auto-cleaned data\n",
"\n",
"\n",
"```\n",
" \n",
"</div>\n",
" \n",
"Is your model/data not compatible with `CleanLearning`? You can instead run cross-validation on your model to get out-of-sample `pred_probs`. Then run the code below to get label issue indices ranked by their inferred severity.\n",
"\n",
"\n",
"<div class=markdown markdown=\"1\" style=\"background:white;margin:16px\"> \n",
" \n",
"```python\n",
"\n",
"from cleanlab.filter import find_label_issues\n",
"\n",
"ranked_label_issues = find_label_issues(\n",
" labels,\n",
" pred_probs,\n",
" return_indices_ranked_by=\"self_confidence\",\n",
")\n",
" \n",
"\n",
"```\n",
" \n",
"</div>\n",
"</div>"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 1. Install required dependencies\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"You can use `pip` to install all packages required for this tutorial as follows:\n",
"\n",
"```ipython3\n",
"!pip install cleanlab\n",
"# Make sure to install the version corresponding to this tutorial\n",
"# E.g. if viewing master branch documentation:\n",
"# !pip install git+https://github.com/cleanlab/cleanlab.git\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"nbsphinx": "hidden"
},
"outputs": [],
"source": [
"# Package installation (hidden on docs website).\n",
"dependencies = [\"cleanlab\"]\n",
"\n",
"if \"google.colab\" in str(get_ipython()): # Check if it's running in Google Colab\n",
" %pip install cleanlab # for colab\n",
" cmd = ' '.join([dep for dep in dependencies if dep != \"cleanlab\"])\n",
" %pip install $cmd\n",
"else:\n",
" dependencies_test = [dependency.split('>')[0] if '>' in dependency \n",
" else dependency.split('<')[0] if '<' in dependency \n",
" else dependency.split('=')[0] for dependency in dependencies]\n",
" missing_dependencies = []\n",
" for dependency in dependencies_test:\n",
" try:\n",
" __import__(dependency)\n",
" except ImportError:\n",
" missing_dependencies.append(dependency)\n",
"\n",
" if len(missing_dependencies) > 0:\n",
" print(\"Missing required dependencies:\")\n",
" print(*missing_dependencies, sep=\", \")\n",
" print(\"\\nPlease install them before running the rest of this notebook.\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import random\n",
"import numpy as np\n",
"import pandas as pd \n",
"from sklearn.preprocessing import StandardScaler, LabelEncoder\n",
"from sklearn.model_selection import cross_val_predict, train_test_split\n",
"from sklearn.metrics import accuracy_score\n",
"from sklearn.ensemble import ExtraTreesClassifier\n",
"\n",
"from cleanlab.filter import find_label_issues\n",
"from cleanlab.classification import CleanLearning\n",
"\n",
"SEED = 100 \n",
"\n",
"np.random.seed(SEED)\n",
"random.seed(SEED)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 2. Load and process the data\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We first load the data features and labels (which are possibly noisy).\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"grades_data = pd.read_csv(\"https://s.cleanlab.ai/grades-tabular-demo-v2.csv\")\n",
"grades_data.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X_raw = grades_data[[\"exam_1\", \"exam_2\", \"exam_3\", \"notes\"]]\n",
"labels_raw = grades_data[\"letter_grade\"]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Next we preprocess the data. Here we apply one-hot encoding to features with categorical data, and standardize features with numeric data. We also perform label encoding on the labels, as cleanlab's functions require the labels for each example to be an interger integer in 0, 1, …, num_classes - 1. "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"categorical_features = [\"notes\"]\n",
"X_encoded = pd.get_dummies(X_raw, columns=categorical_features, drop_first=True)\n",
"\n",
"numeric_features = [\"exam_1\", \"exam_2\", \"exam_3\"]\n",
"scaler = StandardScaler()\n",
"X_processed = X_encoded.copy()\n",
"X_processed[numeric_features] = scaler.fit_transform(X_encoded[numeric_features])\n",
"\n",
"encoder = LabelEncoder()\n",
"encoder.fit(labels_raw)\n",
"labels = encoder.transform(labels_raw)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<div class=\"alert alert-info\">\n",
"Bringing Your Own Data (BYOD)?\n",
"\n",
"You can easily replace the above with your own tabular dataset, and continue with the rest of the tutorial.\n",
" \n",
"Your classes (and entries of `labels`) should be represented as integer indices 0, 1, ..., num_classes - 1. \n",
"For example, if your dataset has 7 examples from 3 classes, `labels` might look like: `np.array([2,0,0,1,2,0,1])`\n",
"\n",
"</div>\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 3. Select a classification model and compute out-of-sample predicted probabilities\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Here we use a simple ExtraTrees classifier that fits various randomized decision tress on our data, but you can choose any suitable scikit-learn model for this tutorial."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"clf = ExtraTreesClassifier()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"To find potential labeling errors, cleanlab requires a probabilistic prediction from your model for every datapoint. However, these predictions will be _overfitted_ (and thus unreliable) for examples the model was previously trained on. For the best results, cleanlab should be applied with **out-of-sample** predicted class probabilities, i.e., on examples held out from the model during the training.\n",
"\n",
"K-fold cross-validation is a straightforward way to produce out-of-sample predicted probabilities for every datapoint in the dataset by training K copies of our model on different data subsets and using each copy to predict on the subset of data it did not see during training. An additional benefit of cross-validation is that it provides a more reliable evaluation of our model than a single training/validation split. We can implement this via the `cross_val_predict` method from scikit-learn:\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"num_crossval_folds = 5 \n",
"pred_probs = cross_val_predict(\n",
" clf,\n",
" X_processed,\n",
" labels,\n",
" cv=num_crossval_folds,\n",
" method=\"predict_proba\",\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 4. Use cleanlab to find label issues\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Based on the given labels and out-of-sample predicted probabilities, cleanlab can quickly help us identify poorly labeled instances in our data table. For a dataset with N examples from K classes, the labels should be a 1D array of length N and predicted probabilities should be a 2D (N x K) array. Here we request that the indices of the identified label issues be sorted by cleanlab's self-confidence score, which measures the quality of each given label via the probability assigned to it in our model's prediction."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"ranked_label_issues = find_label_issues(\n",
" labels=labels, pred_probs=pred_probs, return_indices_ranked_by=\"self_confidence\"\n",
")\n",
"\n",
"print(f\"Cleanlab found {len(ranked_label_issues)} potential label errors.\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Let's review some of the most likely label errors:\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X_raw.iloc[ranked_label_issues].assign(label=labels_raw.iloc[ranked_label_issues]).head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"These final grades look suspicious and should definitely be carefully re-examined! This is a straightforward approach to visualize the rows in a data table that might be mislabeled."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 5. Train a more robust model from noisy labels\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Following proper ML practice, let's split our data into train and test sets.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X_train, X_test, labels_train, labels_test = train_test_split(\n",
" X_encoded,\n",
" labels,\n",
" test_size=0.2,\n",
" random_state=SEED,\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We again standardize the numeric features, this time fitting the scaling parameters solely on the training set.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"scaler = StandardScaler()\n",
"X_train[numeric_features] = scaler.fit_transform(X_train[numeric_features])\n",
"X_test[numeric_features] = scaler.transform(X_test[numeric_features])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Let's now train and evaluate the original ExtraTrees model.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"clf.fit(X_train, labels_train)\n",
"acc_og = clf.score(X_test, labels_test)\n",
"print(f\"Test accuracy of original model: {acc_og}\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"cleanlab provides a wrapper class that can be easily applied to any scikit-learn compatible model. Once wrapped, the resulting model can still be used in the exact same manner, but it will now train more robustly if the data have noisy labels.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"clf = ExtraTreesClassifier() # Note we first re-initialize clf\n",
"cl = CleanLearning(clf) # cl has same methods/attributes as clf"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The following operations take place when we train the cleanlab-wrapped model: The original model is trained in a cross-validated fashion to produce out-of-sample predicted probabilities. Then, these predicted probabilities are used to identify label issues, which are then removed from the dataset. Finally, the original model is trained on the remaining clean subset of the data once more.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"_ = cl.fit(X_train, labels_train)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We can get predictions from the resulting model and evaluate them, just like how we did it for the original scikit-learn model.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"preds = cl.predict(X_test)\n",
"acc_cl = accuracy_score(labels_test, preds)\n",
"print(f\"Test accuracy of cleanlab-trained model: {acc_cl}\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We can see that the test set accuracy slightly improved as a result of the data cleaning. Note that this will not always be the case, especially when we evaluate on test data that are themselves noisy. The best practice is to run cleanlab to identify potential label issues and then manually review them, before blindly trusting any accuracy metrics. In particular, the most effort should be made to ensure high-quality test data, which is supposed to reflect the expected performance of our model during deployment."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"nbsphinx": "hidden"
},
"outputs": [],
"source": [
"# Note: This cell is only for docs.cleanlab.ai, if running on local Jupyter or Colab, please ignore it.\n",
"\n",
"if acc_og >= acc_cl: # check cleanlab has improved prediction accuracy\n",
" raise Exception(\"Cleanlab training failed to improve model accuracy.\")\n",
" \n",
"# this file contains true and noisy labels\n",
"true_data = pd.read_csv(\"https://s.cleanlab.ai/student-grades-demo.csv\")\n",
"true_errors = np.where(true_data[\"letter_grade\"] != true_data[\"noisy_letter_grade\"])[0]\n",
"if not all(x in true_errors for x in ranked_label_issues[:5]): # check top errors are indeed errors\n",
" raise Exception(\"Some of the top listed errors are not actually label errors.\")"
]
}
],
"metadata": {
"interpreter": {
"hash": "cda20062bc42cfdcaa0f9720c0b28e880bba110e9dfce6c1689934eec9b595a1"
},
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
docs/source/tutorials/clean_learning/text.ipynb
+584
View File
@@ -0,0 +1,584 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Text Classification with Noisy Labels\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<div class=\"alert alert-info\">\n",
"Consider Using Datalab\n",
"<br/>\n",
"\n",
"If you are interested in detecting a wide variety of issues in your text dataset, check out the [Datalab text tutorial](https://docs.cleanlab.ai/stable/tutorials/datalab/text.html). Datalab can detect many other types of data issues beyond label issues, whereas CleanLearning is a convenience method to handle noisy labels with sklearn-compatible classification models.\n",
"</div>"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"In this 5-minute quickstart tutorial, we use cleanlab to find potential label errors in an intent classification dataset composed of (text) customer service requests at an online bank. We consider a subset of the [Banking77-OOS Dataset](https://arxiv.org/abs/2106.04564) containing 1,000 customer service requests which can be classified into 10 categories corresponding to the intent of the request. cleanlab will shortlist examples that confuse our ML model the most; many of which are potential label errors, out-of-scope examples, or otherwise ambiguous examples. cleanlab's `CleanLearning` class automatically detects and filters out such badly labeled data, in order to train a more robust version of any Machine Learning model. No change to your existing modeling code is required!\n",
"\n",
"\n",
"**Overview of what we'll do in this tutorial:**\n",
"\n",
"- Define a ML model that can be trained on our dataset (here we use Logistic Regression applied to text embeddings from a pretrained Transformer network, you can use any text classifier model).\n",
"\n",
"- Use `CleanLearning` to wrap this ML model and compute out-of-sample predicted class probabilites, which allow us to identify potential label errors in the dataset.\n",
"\n",
"- Train a more robust version of the same ML model after dropping the detected label errors using `CleanLearning`."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<div class=\"alert alert-info\">\n",
"Quickstart\n",
"<br/>\n",
" \n",
"Already have an sklearn compatible `model`, `data` and given `labels`? Run the code below to train your `model` and get label issues using `CleanLearning`. \n",
" \n",
"You can subsequently use the same `CleanLearning` object to train a more robust model (only trained on the clean data) by calling the `.fit()` method and passing in the `label_issues` found earlier.\n",
"\n",
"\n",
"<div class=markdown markdown=\"1\" style=\"background:white;margin:16px\"> \n",
" \n",
"```python\n",
"\n",
"from cleanlab.classification import CleanLearning\n",
"\n",
"cl = CleanLearning(model)\n",
"label_issues = cl.find_label_issues(train_data, labels) # identify mislabeled examples \n",
" \n",
"cl.fit(train_data, labels, label_issues=label_issues)\n",
"preds = cl.predict(test_data) # predictions from a version of your model \n",
" # trained on auto-cleaned data\n",
"\n",
"\n",
"```\n",
" \n",
"</div>\n",
" \n",
"Is your model/data not compatible with `CleanLearning`? You can instead run cross-validation on your model to get out-of-sample `pred_probs`. Then run the code below to get label issue indices ranked by their inferred severity.\n",
"\n",
"\n",
"<div class=markdown markdown=\"1\" style=\"background:white;margin:16px\"> \n",
" \n",
"```python\n",
"\n",
"from cleanlab.filter import find_label_issues\n",
"\n",
"ranked_label_issues = find_label_issues(\n",
" labels,\n",
" pred_probs,\n",
" return_indices_ranked_by=\"self_confidence\",\n",
")\n",
" \n",
"\n",
"```\n",
" \n",
"</div>\n",
"</div>"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 1. Install required dependencies\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"You can use `pip` to install all packages required for this tutorial as follows:\n",
"\n",
"```ipython3\n",
"!pip install sentence-transformers\n",
"!pip install cleanlab\n",
"# Make sure to install the version corresponding to this tutorial\n",
"# E.g. if viewing master branch documentation:\n",
"# !pip install git+https://github.com/cleanlab/cleanlab.git\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"nbsphinx": "hidden"
},
"outputs": [],
"source": [
"# Package installation (hidden on docs.cleanlab.ai).\n",
"# If running on Colab, may want to use GPU (select: Runtime > Change runtime type > Hardware accelerator > GPU)\n",
"# Package versions we used:scikit-learn==1.2.0 sentence-transformers>=2.3.0\n",
"\n",
"dependencies = [\"cleanlab\", \"sentence_transformers\"]\n",
"\n",
"# Supress outputs that may appear if tensorflow happens to be improperly installed: \n",
"import os \n",
"os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\" # disable parallelism to avoid deadlocks with huggingface\n",
"\n",
"if \"google.colab\" in str(get_ipython()): # Check if it's running in Google Colab\n",
" %pip install cleanlab # for colab\n",
" cmd = ' '.join([dep for dep in dependencies if dep != \"cleanlab\"])\n",
" %pip install $cmd\n",
"else:\n",
" dependencies_test = [dependency.split('>')[0] if '>' in dependency \n",
" else dependency.split('<')[0] if '<' in dependency \n",
" else dependency.split('=')[0] for dependency in dependencies]\n",
" missing_dependencies = []\n",
" for dependency in dependencies_test:\n",
" try:\n",
" __import__(dependency)\n",
" except ImportError:\n",
" missing_dependencies.append(dependency)\n",
"\n",
" if len(missing_dependencies) > 0:\n",
" print(\"Missing required dependencies:\")\n",
" print(*missing_dependencies, sep=\", \")\n",
" print(\"\\nPlease install them before running the rest of this notebook.\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import re \n",
"import string \n",
"import pandas as pd \n",
"from sklearn.metrics import accuracy_score\n",
"from sklearn.model_selection import train_test_split, cross_val_predict \n",
"from sklearn.preprocessing import LabelEncoder\n",
"from sklearn.linear_model import LogisticRegression\n",
"from sentence_transformers import SentenceTransformer\n",
"\n",
"from cleanlab.classification import CleanLearning"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"nbsphinx": "hidden"
},
"outputs": [],
"source": [
"# This cell is hidden from docs.cleanlab.ai \n",
"\n",
"import random \n",
"import numpy as np \n",
"\n",
"pd.set_option(\"display.max_colwidth\", None) \n",
"\n",
"SEED = 123456 # for reproducibility \n",
"\n",
"np.random.seed(SEED)\n",
"random.seed(SEED)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 2. Load and format the text dataset\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"data = pd.read_csv(\"https://s.cleanlab.ai/banking-intent-classification.csv\")\n",
"data.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"raw_texts, raw_labels = data[\"text\"].values, data[\"label\"].values\n",
"\n",
"raw_train_texts, raw_test_texts, raw_train_labels, raw_test_labels = train_test_split(raw_texts, raw_labels, test_size=0.1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"num_classes = len(set(raw_train_labels))\n",
"\n",
"print(f\"This dataset has {num_classes} classes.\")\n",
"print(f\"Classes: {set(raw_train_labels)}\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Let's print the first example in the train set."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"i = 0\n",
"print(f\"Example Label: {raw_train_labels[i]}\")\n",
"print(f\"Example Text: {raw_train_texts[i]}\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The data is stored as two numpy arrays for each the train and test set:\n",
"\n",
"1. `raw_train_texts` and `raw_test_texts` store the customer service requests utterances in text format\n",
"2. `raw_train_labels` and `raw_test_labels` store the intent categories (labels) for each example\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"First, we need to perform label enconding on the labels, cleanlab's functions require the labels for each example to be an interger integer in 0, 1, \u2026, num_classes - 1. We will use sklearn's `LabelEncoder` to encode our labels.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"encoder = LabelEncoder()\n",
"encoder.fit(raw_train_labels)\n",
"\n",
"train_labels = encoder.transform(raw_train_labels)\n",
"test_labels = encoder.transform(raw_test_labels)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<div class=\"alert alert-info\">\n",
"Bringing Your Own Data (BYOD)?\n",
"\n",
"You can easily replace the above with your own text dataset, and continue with the rest of the tutorial.\n",
"\n",
"Your classes (and entries of `train_labels` / `test_labels`) should be represented as integer indices 0, 1, ..., num_classes - 1.\n",
"For example, if your dataset has 7 examples from 3 classes, `train_labels` might be: `np.array([2,0,0,1,2,0,1])`\n",
"\n",
"</div>\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Next we convert the text strings into vectors better suited as inputs for our ML model. \n",
"\n",
"We will use numeric representations from a pretrained Transformer model as embeddings of our text. The [Sentence Transformers](https://huggingface.co/docs/hub/sentence-transformers) library offers simple methods to compute these embeddings for text data. Here, we load the pretrained `electra-small-discriminator` model, and then run our data through network to extract a vector embedding of each example."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"transformer = SentenceTransformer('google/electra-small-discriminator')\n",
"\n",
"train_texts = transformer.encode(raw_train_texts)\n",
"test_texts = transformer.encode(raw_test_texts)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Our subsequent ML model will directly operate on elements of `train_texts` and `test_texts` in order to classify the customer service requests."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 3. Define a classification model and use cleanlab to find potential label errors\n",
"\n",
"<a id=\"section3\"></a>"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"A typical way to leverage pretrained networks for a particular classification task is to add a linear output layer and fine-tune the network parameters on the new data. However this can be computationally intensive. Alternatively, we can freeze the pretrained weights of the network and only train the output layer without having to rely on GPU(s). Here we do this conveniently by fitting a scikit-learn linear model on top of the extracted embeddings."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model = LogisticRegression(max_iter=400)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We can define the `CleanLearning` object with our Logistic Regression model and use `find_label_issues` to identify potential label errors.\n",
"\n",
"`CleanLearning` provides a wrapper class that can easily be applied to any scikit-learn compatible model, which can be used to find potential label issues and train a more robust model if the original data contains noisy labels."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"cv_n_folds = 5 # for efficiency; values like 5 or 10 will generally work better\n",
"\n",
"cl = CleanLearning(model, cv_n_folds=cv_n_folds)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"label_issues = cl.find_label_issues(X=train_texts, labels=train_labels)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The `find_label_issues` method above will perform cross validation to compute out-of-sample predicted probabilites for each example, which is used to identify label issues.\n",
"\n",
"This method returns a dataframe containing a label quality score for each example. These numeric scores lie between 0 and 1, where lower scores indicate examples more likely to be mislabeled. The dataframe also contains a boolean column specifying whether or not each example is identified to have a label issue (indicating it is likely mislabeled). Note that the given and predicted labels here are encoded as intergers as that was the format expected by `cleanlab`, we will inverse transform them later in this tutorial."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"label_issues.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We can get the subset of examples flagged with label issues, and also sort by label quality score to find the indices of the 10 most likely mislabeled examples in our dataset."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"identified_issues = label_issues[label_issues[\"is_label_issue\"] == True]\n",
"lowest_quality_labels = label_issues[\"label_quality\"].argsort()[:10].to_numpy()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\n",
" f\"cleanlab found {len(identified_issues)} potential label errors in the dataset.\\n\"\n",
" f\"Here are indices of the top 10 most likely errors: \\n {lowest_quality_labels}\"\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Let's review some of the most likely label errors. To help us inspect these datapoints, we define a method to print any example from the dataset, together with its given (original) label and the suggested alternative label from cleanlab.\n",
"\n",
"We then display some of the top-ranked label issues identified by cleanlab:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def print_as_df(index):\n",
" return pd.DataFrame(\n",
" {\n",
" \"text\": raw_train_texts, \n",
" \"given_label\": raw_train_labels,\n",
" \"predicted_label\": encoder.inverse_transform(label_issues[\"predicted_label\"]),\n",
" },\n",
" ).iloc[index]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print_as_df(lowest_quality_labels[:5])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"These are very clear label errors that cleanlab has identified in this data! Note that the `given_label` does not correctly reflect the intent of these requests, whoever produced this dataset made many mistakes that are important to address before modeling the data.\n",
"\n",
"cleanlab has shortlisted the most likely label errors to speed up your data cleaning process. With this list, you can decide whether to fix these label issues or remove ambiguous examples from the dataset."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 4. Train a more robust model from noisy labels\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Fixing the label issues manually may be time-consuming, but cleanlab can filter these noisy examples and train a model on the remaining clean data for you automatically.\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"To establish a baseline, let's first train and evaluate our original Logistic Regression model.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"baseline_model = LogisticRegression(max_iter=400) # note we first re-instantiate the model\n",
"baseline_model.fit(X=train_texts, y=train_labels)\n",
"\n",
"preds = baseline_model.predict(test_texts)\n",
"acc_og = accuracy_score(test_labels, preds)\n",
"print(f\"\\n Test accuracy of original model: {acc_og}\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Now that we have a baseline, let's check if using `CleanLearning` improves our test accuracy.\n",
"\n",
"`CleanLearning` provides a wrapper that can be applied to any scikit-learn compatible model. The resulting model object can be used in the same manner, but it will now train more robustly if the data has noisy labels.\n",
"\n",
"We can use the same `CleanLearning` object defined above, and pass the label issues we already computed into `.fit()` via the `label_issues` argument. This accelerates things; if we did not provide the label issues, then they would be recomputed via cross-validation. After that `CleanLearning` simply deletes the examples with label issues and retrains your model on the remaining data."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"cl.fit(X=train_texts, labels=train_labels, label_issues=cl.get_label_issues())\n",
"\n",
"pred_labels = cl.predict(test_texts)\n",
"acc_cl = accuracy_score(test_labels, pred_labels)\n",
"print(f\"Test accuracy of cleanlab's model: {acc_cl}\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We can see that the test set accuracy slightly improved as a result of the data cleaning. Note that this will not always be the case, especially when we are evaluating on test data that are themselves noisy. The best practice is to run cleanlab to identify potential label issues and then manually review them, before blindly trusting any accuracy metrics. In particular, the most effort should be made to ensure high-quality test data, which is supposed to reflect the expected performance of our model during deployment.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"nbsphinx": "hidden"
},
"outputs": [],
"source": [
"# Note: This cell is only for docs.cleanlab.ai, if running on local Jupyter or Colab, please ignore it.\n",
"\n",
"highlighted_indices = [646, 390, 628, 702] # check these examples were found in find_label_issues\n",
"if not all(x in identified_issues.index for x in highlighted_indices):\n",
" raise Exception(\"Some highlighted examples are missing from ranked_label_issues.\")\n",
"\n",
"# Also check that cleanlab has improved prediction accuracy\n",
"if acc_og >= acc_cl:\n",
" raise Exception(\"Cleanlab training failed to improve model accuracy.\")"
]
}
],
"metadata": {
"colab": {
"collapsed_sections": [],
"name": "Text Classification with CleanLearning",
"provenance": []
},
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
docs/source/tutorials/datalab/audio.ipynb
+791
View File
@@ -0,0 +1,791 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "eVufWTY3jRPx"
},
"source": [
"# Detecting Issues in an Audio Dataset with Datalab\n",
"\n",
"In this 5-minute quickstart tutorial, we use cleanlab to find label issues in the [Spoken Digit dataset](https://www.tensorflow.org/datasets/catalog/spoken_digit) (it's like MNIST for audio). The dataset contains 2,500 audio clips with English pronunciations of the digits 0 to 9 (these are the class labels to predict from the audio).\n",
"\n",
"**Overview of what we'll do in this tutorial:**\n",
"\n",
"- Extract features from audio clips (.wav files) using a [pre-trained Pytorch model](https://huggingface.co/speechbrain/spkrec-xvect-voxceleb) from HuggingFace that was previously fit to the [VoxCeleb](https://www.robots.ox.ac.uk/~vgg/data/voxceleb/) speech dataset.\n",
"\n",
"- Train a cross-validated linear model using the extracted features and generate out-of-sample predicted probabilities.\n",
"\n",
"- Apply cleanlab's `Datalab` audit to these predictions in order to identify which audio clips in the dataset are likely mislabeled.\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<div class=\"alert alert-info\">\n",
"Quickstart\n",
"<br/>\n",
" \n",
"Already have a `model`? Run cross-validation to get out-of-sample `pred_probs`, and then run the code below to audit your dataset and identify any potential issues.\n",
"\n",
"\n",
"<div class=markdown markdown=\"1\" style=\"background:white;margin:16px\"> \n",
" \n",
"```python\n",
"\n",
"from cleanlab import Datalab\n",
"\n",
"lab = Datalab(data=your_dataset, label_name=\"column_name_of_labels\")\n",
"lab.find_issues(pred_probs=your_pred_probs, issue_types={\"label\":{}})\n",
"\n",
"lab.get_issues(\"label\")\n",
" \n",
"```\n",
" \n",
"</div>\n",
"</div>"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "eqsqBq3PiUHA"
},
"source": [
"## 1. Install dependencies and import them\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "i7nT-U9qc8MS"
},
"source": [
"You can use `pip` to install all packages required for this tutorial as follows:\n",
"\n",
"```ipython3\n",
"!pip install huggingface_hub==0.17.0 speechbrain==0.5.13 \n",
"!pip install \"cleanlab[datalab]\"\n",
"# Make sure to install the version corresponding to this tutorial\n",
"# E.g. if viewing master branch documentation:\n",
"# !pip install git+https://github.com/cleanlab/cleanlab.git\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"nbsphinx": "hidden"
},
"outputs": [],
"source": [
"# Package installation (hidden on docs website).\n",
"# Package versions used: torch==2.1.2 torchaudio==2.1.2 speechbrain==0.5.13\n",
"\n",
"dependencies = [\"cleanlab\", \"huggingface_hub==0.17.0\", \"speechbrain==0.5.13\", \"datasets\"]\n",
"\n",
"if \"google.colab\" in str(get_ipython()): # Check if it's running in Google Colab\n",
" %pip install cleanlab # for colab\n",
" cmd = ' '.join([dep for dep in dependencies if dep != \"cleanlab\"])\n",
" %pip install $cmd\n",
"else:\n",
" dependencies_test = [dependency.split('>')[0] if '>' in dependency \n",
" else dependency.split('<')[0] if '<' in dependency \n",
" else dependency.split('=')[0] for dependency in dependencies]\n",
" missing_dependencies = []\n",
" for dependency in dependencies_test:\n",
" try:\n",
" __import__(dependency)\n",
" except ImportError:\n",
" missing_dependencies.append(dependency)\n",
"\n",
" if len(missing_dependencies) > 0:\n",
" print(\"Missing required dependencies:\")\n",
" print(*missing_dependencies, sep=\", \")\n",
" print(\"\\nPlease install them before running the rest of this notebook.\")"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "x-oboEbRdhf6"
},
"source": [
"Let's import some of the packages needed throughout this tutorial.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "LaEiwXUiVHCS"
},
"outputs": [],
"source": [
"import os\n",
"import pandas as pd\n",
"import numpy as np\n",
"import random\n",
"import torch\n",
"import torchaudio\n",
"import torchaudio\n",
"\n",
"from cleanlab import Datalab\n",
"\n",
"SEED = 456 # ensure reproducibility"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"nbsphinx": "hidden"
},
"outputs": [],
"source": [
"# This (optional) cell is hidden from docs.cleanlab.ai \n",
"\n",
"def set_seed(seed=0):\n",
" \"\"\"Ensure reproducibility.\"\"\"\n",
" np.random.seed(seed)\n",
" torch.manual_seed(seed)\n",
" torch.backends.cudnn.deterministic = True\n",
" torch.backends.cudnn.benchmark = False\n",
" torch.cuda.manual_seed_all(seed)\n",
"\n",
"\n",
"set_seed(SEED)\n",
"pd.options.display.max_colwidth = 500"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "SOen_sxQidLC"
},
"source": [
"## 2. Load the data\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "uHVskN2eeNj6"
},
"source": [
"We must first fetch the dataset. To run the below command, you'll need to have `wget` installed; alternatively you can manually navigate to the link in your browser and download from there.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "GRDPEg7-VOQe",
"outputId": "cb886220-e86e-4a77-9f3a-d7844c37c3a6"
},
"outputs": [],
"source": [
"%%capture\n",
"\n",
"!wget https://github.com/Jakobovski/free-spoken-digit-dataset/archive/v1.0.9.tar.gz\n",
"!mkdir spoken_digits\n",
"!tar -xf v1.0.9.tar.gz -C spoken_digits"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "tRvNnyB0e_IE"
},
"source": [
"The audio data are .wav files in the `recordings/` folder. Note that the label for each audio clip (i.e. digit from 0 to 9) is indicated in the prefix of the file name (e.g. `6_nicolas_32.wav` has the label 6). If instead applying cleanlab to your own dataset, its classes should be represented as integer indices 0, 1, ..., num_classes - 1."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "FDA5sGZwUSur",
"outputId": "0cedc509-63fd-4dc3-d32f-4b537dfe3895"
},
"outputs": [],
"source": [
"DATA_PATH = \"spoken_digits/free-spoken-digit-dataset-1.0.9/recordings/\"\n",
"\n",
"# Get list of .wav file names\n",
"# os.listdir order is nondeterministic, so for reproducibility,\n",
"# we sort first and then do a deterministic shuffle\n",
"file_names = sorted(i for i in os.listdir(DATA_PATH) if i.endswith(\".wav\"))\n",
"random.Random(SEED).shuffle(file_names)\n",
"\n",
"file_paths = [os.path.join(DATA_PATH, name) for name in file_names]\n",
"\n",
"# Check out first 3 files\n",
"file_paths[:3]"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "Xi2592bVhSab"
},
"source": [
"Let's listen to some example audio clips from the dataset. We introduce a `display_example` function to process the .wav file so we can listen to it in this notebook (can skip these details)."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<details><summary>See the implementation of `display_example` **(click to expand)**</summary>\n",
"\n",
"```python\n",
"# Note: This pulldown content is for docs.cleanlab.ai, if running on local Jupyter or Colab, please ignore it.\n",
"\n",
"import torch\n",
"import torchaudio\n",
"from pathlib import Path\n",
"from IPython import display\n",
"\n",
"# Utility function for loading audio files and making sure the sample rate is correct.\n",
"def load_wav_16k_mono(filename):\n",
" \"\"\"Load a WAV file, convert it to a float tensor, resample to 16 kHz single-channel audio.\"\"\"\n",
" # Load audio file with torchaudio\n",
" waveform, sample_rate = torchaudio.load(filename)\n",
" \n",
" # Convert to mono if stereo\n",
" if waveform.shape[0] > 1:\n",
" waveform = torch.mean(waveform, dim=0, keepdim=True)\n",
" \n",
" # Resample to 16kHz if needed\n",
" if sample_rate != 16000:\n",
" resampler = torchaudio.transforms.Resample(sample_rate, 16000)\n",
" waveform = resampler(waveform)\n",
" \n",
" return waveform.squeeze()\n",
"\n",
"\n",
"def display_example(wav_file_name, audio_rate=16000):\n",
" \"\"\"Allows us to listen to any wav file and displays its given label in the dataset.\"\"\"\n",
" wav_file_example = load_wav_16k_mono(wav_file_name)\n",
" label = Path(wav_file_name).parts[-1].split(\"_\")[0]\n",
" print(f\"Given label for this example: {label}\")\n",
" display.display(display.Audio(wav_file_example.numpy(), rate=audio_rate))\n",
"```\n",
"\n",
"</details>"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"nbsphinx": "hidden"
},
"outputs": [],
"source": [
"import torch\n",
"import torchaudio\n",
"from pathlib import Path\n",
"from IPython import display\n",
"\n",
"# Utility function for loading audio files and making sure the sample rate is correct.\n",
"def load_wav_16k_mono(filename):\n",
" \"\"\"Load a WAV file, convert it to a float tensor, resample to 16 kHz single-channel audio.\"\"\"\n",
" # Load audio file with torchaudio\n",
" waveform, sample_rate = torchaudio.load(filename)\n",
" \n",
" # Convert to mono if stereo\n",
" if waveform.shape[0] > 1:\n",
" waveform = torch.mean(waveform, dim=0, keepdim=True)\n",
" \n",
" # Resample to 16kHz if needed\n",
" if sample_rate != 16000:\n",
" resampler = torchaudio.transforms.Resample(sample_rate, 16000)\n",
" waveform = resampler(waveform)\n",
" \n",
" return waveform.squeeze()\n",
"\n",
"\n",
"def display_example(wav_file_name, audio_rate=16000):\n",
" \"\"\"Allows us to listen to any wav file and displays its given label in the dataset.\"\"\"\n",
" wav_file_example = load_wav_16k_mono(wav_file_name)\n",
" label = Path(wav_file_name).parts[-1].split(\"_\")[0]\n",
" print(f\"Given label for this example: {label}\")\n",
" display.display(display.Audio(wav_file_example.numpy(), rate=audio_rate))"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "2bLlDRI6hzon"
},
"source": [
"Click the play button below to listen to this example .wav file. Feel free to change the `wav_file_name_example` variable below to listen to other audio clips in the dataset.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 92
},
"id": "dLBvUZLlII5w",
"outputId": "c6a4917f-4a82-4a89-9193-415072e45550"
},
"outputs": [],
"source": [
"wav_file_name_example = \"spoken_digits/free-spoken-digit-dataset-1.0.9/recordings/7_jackson_43.wav\" # change this to hear other examples\n",
"display_example(wav_file_name_example)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "-QvbZA7yHwkh"
},
"source": [
"## 3. Use pre-trained SpeechBrain model to featurize audio\n",
"\n",
"The [SpeechBrain](https://github.com/speechbrain/speechbrain) package offers many Pytorch neural networks that have been pretrained for speech recognition tasks. Here we instantiate an audio feature extractor using SpeechBrain's `EncoderClassifier`. We'll use the \"spkrec-xvect-voxceleb\" network which has been pre-trained on the [VoxCeleb](https://www.robots.ox.ac.uk/~vgg/data/voxceleb/) speech dataset.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "vL9lkiKsHvKr"
},
"outputs": [],
"source": [
"%%capture\n",
"\n",
"from speechbrain.pretrained import EncoderClassifier\n",
"\n",
"feature_extractor = EncoderClassifier.from_hparams(\n",
" \"speechbrain/spkrec-xvect-voxceleb\",\n",
" # run_opts={\"device\":\"cuda\"} # Uncomment this to run on GPU if you have one (optional)\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "vXlE6IK4ibcr"
},
"source": [
"Next, we run the audio clips through the pre-trained model to extract vector features (aka embeddings)."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 143
},
"id": "obQYDKdLiUU6",
"outputId": "4e923d5c-2cf4-4a5c-827b-0a4fea9d87e4"
},
"outputs": [],
"source": [
"# Create dataframe with .wav file names\n",
"df = pd.DataFrame(file_paths, columns=[\"wav_audio_file_path\"])\n",
"df[\"label\"] = df.wav_audio_file_path.map(lambda x: int(Path(x).parts[-1].split(\"_\")[0]))\n",
"df.head(3)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "I8JqhOZgi94g"
},
"outputs": [],
"source": [
"def extract_audio_embeddings(model, wav_audio_file_path: str) -> tuple:\n",
" \"\"\"Feature extractor that embeds audio into a vector.\"\"\"\n",
" signal, fs = torchaudio.load(wav_audio_file_path) # Load audio signal as tensor\n",
" embeddings = model.encode_batch(\n",
" signal\n",
" ) # Pass tensor through pretrained neural net and extract representation\n",
" return embeddings"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "2FSQ2GR9R_YA"
},
"outputs": [],
"source": [
"# Extract audio embeddings\n",
"embeddings_list = []\n",
"for i, file_name in enumerate(df.wav_audio_file_path): # for each .wav file name\n",
" embeddings = extract_audio_embeddings(feature_extractor, file_name)\n",
" embeddings_list.append(embeddings.cpu().numpy())\n",
"\n",
"embeddings_array = np.squeeze(np.array(embeddings_list))"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "dELkcdXgjTn_"
},
"source": [
"Now we have our features in a 2D numpy array. Each row in the array corresponds to an audio clip. We're now able to represent each audio clip as a 512-dimensional feature vector!\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "kAkY31IVXyr8",
"outputId": "fd70d8d6-2f11-48d5-ae9c-a8c97d453632"
},
"outputs": [],
"source": [
"print(embeddings_array)\n",
"print(\"Shape of array: \", embeddings_array.shape)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "o4RBcaARmfVG"
},
"source": [
"## 4. Fit linear model and compute out-of-sample predicted probabilities\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "y9BIVyI9kHa4"
},
"source": [
"A typical way to leverage pretrained networks for a particular classification task is to add a linear output layer and fine-tune the network parameters on the new data. However this can be computationally intensive. Alternatively, we can freeze the pretrained weights of the network and only train the output layer without having to rely on GPU(s). Here we do this conveniently by fitting a scikit-learn linear model on top of the extracted network embeddings.\n",
"\n",
"To identify label issues, cleanlab requires a probabilistic prediction from your model for every datapoint that should be considered. However these predictions will be _overfit_ (and thus unreliable) for datapoints the model was previously trained on. cleanlab is intended to only be used with **out-of-sample** predicted probabilities, i.e. on datapoints held-out from the model during the training.\n",
"\n",
"K-fold cross-validation is a straightforward way to produce out-of-sample predicted probabilities for every datapoint in the dataset, by training K copies of our model on different data subsets and using each copy to predict on the subset of data it did not see during training. An additional benefit of cross-validation is that it provides more reliable evaluation of our model than a single training/validation split. We can obtain cross-validated out-of-sample predicted probabilities from any classifier via the [cross_val_predict](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.cross_val_predict.html) wrapper provided in scikit-learn.\n",
"Make sure that the columns of your `pred_probs` are properly ordered with respect to the ordering of classes, which for Datalab is: lexicographically sorted by class name.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "i_drkY9YOcw4"
},
"outputs": [],
"source": [
"from sklearn.linear_model import LogisticRegression\n",
"from sklearn.model_selection import cross_val_predict\n",
"\n",
"model = LogisticRegression(C=0.01, max_iter=1000, tol=1e-2, random_state=SEED)\n",
"\n",
"num_crossval_folds = 5 # can decrease this value to reduce runtime, or increase it to get better results\n",
"pred_probs = cross_val_predict(\n",
" estimator=model, X=embeddings_array, y=df.label.values, cv=num_crossval_folds, method=\"predict_proba\"\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "FW1yI9Ryrfkj"
},
"source": [
"For each audio clip, the corresponding predicted probabilities in `pred_probs` are produced by a copy of our `LogisticRegression` model that has never been trained on this audio clip. Hence we call these predictions _out-of-sample_. An additional benefit of cross-validation is that it provides more reliable evaluation of our model than a single training/validation split.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "_b-AQeoXOc7q",
"outputId": "15ae534a-f517-4906-b177-ca91931a8954"
},
"outputs": [],
"source": [
"from sklearn.metrics import accuracy_score\n",
"\n",
"predicted_labels = pred_probs.argmax(axis=1)\n",
"cv_accuracy = accuracy_score(df.label.values, predicted_labels)\n",
"print(f\"Cross-validated estimate of accuracy on held-out data: {cv_accuracy}\")"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "SPz8WBwIlxUE"
},
"source": [
"## 5. Use cleanlab to find label issues\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "laui-jXMm6qR"
},
"source": [
"Based on the given labels, out-of-sample predicted probabilities and features, cleanlab can quickly help us identify label issues in our dataset. For a dataset with N examples from K classes, the labels should be a 1D array of length N and predicted probabilities should be a 2D (N x K) array. \n",
"\n",
"Here, we use cleanlab to find potential label errors in our data. `Datalab` has several ways of loading the data. In this case, we can just pass the DataFrame created above to instantiate the object. We will then pass in the predicted probabilites to the `find_issues()` method so that Datalab can use them to find potential label errors in our data."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"lab = Datalab(df, label_name=\"label\")\n",
"lab.find_issues(pred_probs=pred_probs, issue_types={\"label\":{}})"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We can view the results of running Datalab by calling the `report` method:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"lab.report()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We observe from the report that cleanlab has found some label issues in our dataset. Let us investigate these examples further.\n",
"\n",
"We can view the more details about the label quality for each example using the `get_issues` method, specifying `label` as the issue type."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"label_issues = lab.get_issues(\"label\")\n",
"label_issues.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"This method returns a dataframe containing a label quality score for each example. These numeric scores lie between 0 and 1, where lower scores indicate examples more likely to be mislabeled. The dataframe also contains a boolean column specifying whether or not each example is identified to have a label issue (indicating it is likely mislabeled).\n",
"\n",
"We can then filter for the examples that have been identified as a label error:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"identified_label_issues = label_issues[label_issues[\"is_label_issue\"] == True]\n",
"lowest_quality_labels = identified_label_issues.sort_values(\"label_score\").index\n",
"\n",
"print(f\"Here are indices of the most likely errors: \\n {lowest_quality_labels.values}\")"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "iI07jQ0BnTgt"
},
"source": [
"These examples flagged by cleanlab are those worth inspecting more closely."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 237
},
"id": "FQwRHgbclpsO",
"outputId": "fee5c335-c00e-4fcc-f22b-718705e93182"
},
"outputs": [],
"source": [
"df.iloc[lowest_quality_labels]"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "PsDmd5WDnZJG"
},
"source": [
"Let's listen to some audio clips below of label issues that were identified in this list.\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "p9jLn3Lp85rU"
},
"source": [
"In this example, the given label is **6** but it sounds like **8**.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 92
},
"id": "ff1NFVlDoysO",
"outputId": "8141a036-44c1-4349-c338-880432513e37"
},
"outputs": [],
"source": [
"wav_file_name_example = \"spoken_digits/free-spoken-digit-dataset-1.0.9/recordings/6_yweweler_14.wav\"\n",
"display_example(wav_file_name_example)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "HwokyN0bfVsn"
},
"source": [
"In the three examples below, the given label is **6** but they sound quite ambiguous.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 92
},
"id": "GZgovGkdiaiP",
"outputId": "d76b2ccf-8be2-4f3a-df4c-2c5c99150db7"
},
"outputs": [],
"source": [
"wav_file_name_example = \"spoken_digits/free-spoken-digit-dataset-1.0.9/recordings/6_yweweler_36.wav\"\n",
"display_example(wav_file_name_example)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 92
},
"id": "lfa2eHbMwG8R",
"outputId": "6627ebe2-d439-4bf5-e2cb-44f6278ae86c"
},
"outputs": [],
"source": [
"wav_file_name_example = \"spoken_digits/free-spoken-digit-dataset-1.0.9/recordings/6_yweweler_35.wav\"\n",
"display_example(wav_file_name_example)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"wav_file_name_example = \"spoken_digits/free-spoken-digit-dataset-1.0.9/recordings/6_nicolas_8.wav\"\n",
"display_example(wav_file_name_example)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "-rf8iSngtV83"
},
"source": [
"You can see that even widely-used datasets like Spoken Digit contain problematic labels. Never blindly trust your data! You should always check it for potential issues, many of which can be easily identified by cleanlab.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"nbsphinx": "hidden"
},
"outputs": [],
"source": [
"# Note: This cell is only for docs.cleanlab.ai, if running on local Jupyter or Colab, please ignore it.\n",
"\n",
"highlighted_indices = [1946, 516, 469, 2132] # verify these examples were found in find_label_issues\n",
"if not all(x in lowest_quality_labels for x in highlighted_indices):\n",
" raise Exception(\"Some highlighted examples are missing from label_issues_indices.\")"
]
}
],
"metadata": {
"accelerator": "GPU",
"colab": {
"collapsed_sections": [],
"name": "audio_quickstart_tutorial_deterministic.ipynb",
"provenance": []
},
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
docs/source/tutorials/datalab/datalab_advanced.ipynb
+812
View File
@@ -0,0 +1,812 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Datalab: Advanced workflows to audit your data"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Cleanlab offers a `Datalab` object to identify various issues in your machine learning datasets that may negatively impact models if not addressed. By default, `Datalab` can help you identify noisy labels, outliers, (near) duplicates, and other types of problems that commonly occur in real-world data.\n",
"\n",
"`Datalab` performs these checks by utilizing the (probabilistic) predictions from *any* ML model that has already been trained or its learned representations of the data. Underneath the hood, this class calls all the appropriate cleanlab methods for your dataset and provided model outputs, in order to best audit the data and alert you of important issues. This makes it easy to apply many functionalities of this library all within a single line of code. \n",
"\n",
"**This tutorial will demonstrate some advanced functionalities of Datalab including:**\n",
"\n",
"- Incremental issue search\n",
"- Specifying nondefault arguments to issue checks\n",
"- Save and load Datalab objects\n",
"- Adding a custom IssueManager\n",
"\n",
"If you are new to `Datalab`, check out this [quickstart tutorial](datalab_quickstart.html) for a 5-min introduction!"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<div class=\"alert alert-info\">\n",
"Quickstart\n",
"<br/>\n",
" \n",
"Already have (out-of-sample) `pred_probs` from a model trained on an existing set of labels? Maybe you have some `features` as well? Run the code below to examine your dataset for multiple types of issues.\n",
"\n",
"<div class=markdown markdown=\"1\" style=\"background:white;margin:16px\"> \n",
" \n",
"```ipython3 \n",
"from cleanlab import Datalab\n",
"\n",
"lab = Datalab(data=your_dataset, label_name=\"column_name_of_labels\")\n",
"lab.find_issues(features=your_feature_matrix, pred_probs=your_pred_probs)\n",
"\n",
"lab.report()\n",
"```\n",
" \n",
"</div>\n",
"</div>"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Install and import required dependencies"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"`Datalab` has additional dependencies that are not included in the standard installation of cleanlab.\n",
"\n",
"You can use pip to install all packages required for this tutorial as follows:\n",
"\n",
"```ipython3\n",
"!pip install matplotlib \n",
"!pip install \"cleanlab[datalab]\"\n",
"\n",
"# Make sure to install the version corresponding to this tutorial\n",
"# E.g. if viewing master branch documentation:\n",
"# !pip install git+https://github.com/cleanlab/cleanlab.git\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"nbsphinx": "hidden"
},
"outputs": [],
"source": [
"# Package installation (hidden on docs website).\n",
"dependencies = [\"cleanlab\", \"matplotlib\", \"datasets\"] # TODO: make sure this list is updated\n",
"\n",
"if \"google.colab\" in str(get_ipython()): # Check if it's running in Google Colab\n",
" %pip install cleanlab # for colab\n",
" cmd = ' '.join([dep for dep in dependencies if dep != \"cleanlab\"])\n",
" %pip install $cmd\n",
"else:\n",
" dependencies_test = [dependency.split('>')[0] if '>' in dependency \n",
" else dependency.split('<')[0] if '<' in dependency \n",
" else dependency.split('=')[0] for dependency in dependencies]\n",
" missing_dependencies = []\n",
" for dependency in dependencies_test:\n",
" try:\n",
" __import__(dependency)\n",
" except ImportError:\n",
" missing_dependencies.append(dependency)\n",
"\n",
" if len(missing_dependencies) > 0:\n",
" print(\"Missing required dependencies:\")\n",
" print(*missing_dependencies, sep=\", \")\n",
" print(\"\\nPlease install them before running the rest of this notebook.\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"from sklearn.linear_model import LogisticRegression\n",
"from sklearn.model_selection import cross_val_predict\n",
"\n",
"from cleanlab import Datalab"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Create and load the data"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We'll load a toy classification dataset for this tutorial. The dataset has two numerical features and a label column with three classes."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<details><summary>See the code for data generation. **(click to expand)**</summary>\n",
" \n",
"```ipython3\n",
"# Note: This pulldown content is for docs.cleanlab.ai, if running on local Jupyter or Colab, please ignore it.\n",
"\n",
"from sklearn.model_selection import train_test_split\n",
"from cleanlab.benchmarking.noise_generation import (\n",
" generate_noise_matrix_from_trace,\n",
" generate_noisy_labels,\n",
")\n",
"\n",
"SEED = 123\n",
"np.random.seed(SEED)\n",
"\n",
"BINS = {\n",
" \"low\": [-np.inf, 3.3],\n",
" \"mid\": [3.3, 6.6],\n",
" \"high\": [6.6, +np.inf],\n",
"}\n",
"\n",
"BINS_MAP = {\n",
" \"low\": 0,\n",
" \"mid\": 1,\n",
" \"high\": 2,\n",
"}\n",
"\n",
"\n",
"def create_data():\n",
"\n",
" X = np.random.rand(250, 2) * 5\n",
" y = np.sum(X, axis=1)\n",
" # Map y to bins based on the BINS dict\n",
" y_bin = np.array([k for y_i in y for k, v in BINS.items() if v[0] <= y_i < v[1]])\n",
" y_bin_idx = np.array([BINS_MAP[k] for k in y_bin])\n",
"\n",
" # Split into train and test\n",
" X_train, X_test, y_train, y_test, y_train_idx, y_test_idx = train_test_split(\n",
" X, y_bin, y_bin_idx, test_size=0.5, random_state=SEED\n",
" )\n",
"\n",
" # Add several (5) out-of-distribution points. Sliding them along the decision boundaries\n",
" # to make them look like they are out-of-frame\n",
" X_out = np.array(\n",
" [\n",
" [-1.5, 3.0],\n",
" [-1.75, 6.5],\n",
" [1.5, 7.2],\n",
" [2.5, -2.0],\n",
" [5.5, 7.0],\n",
" ]\n",
" )\n",
" # Add a near duplicate point to the last outlier, with some tiny noise added\n",
" near_duplicate = X_out[-1:] + np.random.rand(1, 2) * 1e-6\n",
" X_out = np.concatenate([X_out, near_duplicate])\n",
"\n",
" y_out = np.sum(X_out, axis=1)\n",
" y_out_bin = np.array([k for y_i in y_out for k, v in BINS.items() if v[0] <= y_i < v[1]])\n",
" y_out_bin_idx = np.array([BINS_MAP[k] for k in y_out_bin])\n",
"\n",
" # Add to train\n",
" X_train = np.concatenate([X_train, X_out])\n",
" y_train = np.concatenate([y_train, y_out])\n",
" y_train_idx = np.concatenate([y_train_idx, y_out_bin_idx])\n",
"\n",
" # Add an exact duplicate example to the training set\n",
" exact_duplicate_idx = np.random.randint(0, len(X_train))\n",
" X_duplicate = X_train[exact_duplicate_idx, None]\n",
" y_duplicate = y_train[exact_duplicate_idx, None]\n",
" y_duplicate_idx = y_train_idx[exact_duplicate_idx, None]\n",
"\n",
" # Add to train\n",
" X_train = np.concatenate([X_train, X_duplicate])\n",
" y_train = np.concatenate([y_train, y_duplicate])\n",
" y_train_idx = np.concatenate([y_train_idx, y_duplicate_idx])\n",
"\n",
" py = np.bincount(y_train_idx) / float(len(y_train_idx))\n",
" m = len(BINS)\n",
"\n",
" noise_matrix = generate_noise_matrix_from_trace(\n",
" m,\n",
" trace=0.9 * m,\n",
" py=py,\n",
" valid_noise_matrix=True,\n",
" seed=SEED,\n",
" )\n",
"\n",
" noisy_labels_idx = generate_noisy_labels(y_train_idx, noise_matrix)\n",
" noisy_labels = np.array([list(BINS_MAP.keys())[i] for i in noisy_labels_idx])\n",
"\n",
" return X_train, y_train_idx, noisy_labels, noisy_labels_idx, X_out, X_duplicate\n",
"```\n",
"\n",
"</details>"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"nbsphinx": "hidden"
},
"outputs": [],
"source": [
"from sklearn.model_selection import train_test_split\n",
"from cleanlab.benchmarking.noise_generation import (\n",
" generate_noise_matrix_from_trace,\n",
" generate_noisy_labels,\n",
")\n",
"\n",
"SEED = 123\n",
"np.random.seed(SEED)\n",
"\n",
"BINS = {\n",
" \"low\": [-np.inf, 3.3],\n",
" \"mid\": [3.3, 6.6],\n",
" \"high\": [6.6, +np.inf],\n",
"}\n",
"\n",
"BINS_MAP = {\n",
" \"low\": 0,\n",
" \"mid\": 1,\n",
" \"high\": 2,\n",
"}\n",
"\n",
"\n",
"def create_data():\n",
"\n",
" X = np.random.rand(250, 2) * 5\n",
" y = np.sum(X, axis=1)\n",
" # Map y to bins based on the BINS dict\n",
" y_bin = np.array([k for y_i in y for k, v in BINS.items() if v[0] <= y_i < v[1]])\n",
" y_bin_idx = np.array([BINS_MAP[k] for k in y_bin])\n",
"\n",
" # Split into train and test\n",
" X_train, X_test, y_train, y_test, y_train_idx, y_test_idx = train_test_split(\n",
" X, y_bin, y_bin_idx, test_size=0.5, random_state=SEED\n",
" )\n",
"\n",
" # Add several (5) out-of-distribution points. Sliding them along the decision boundaries\n",
" # to make them look like they are out-of-frame\n",
" X_out = np.array(\n",
" [\n",
" [-1.5, 3.0],\n",
" [-1.75, 6.5],\n",
" [1.5, 7.2],\n",
" [2.5, -2.0],\n",
" [5.5, 7.0],\n",
" ]\n",
" )\n",
" # Add a near duplicate point to the last outlier, with some tiny noise added\n",
" near_duplicate = X_out[-1:] + np.random.rand(1, 2) * 1e-6\n",
" X_out = np.concatenate([X_out, near_duplicate])\n",
"\n",
" y_out = np.sum(X_out, axis=1)\n",
" y_out_bin = np.array([k for y_i in y_out for k, v in BINS.items() if v[0] <= y_i < v[1]])\n",
" y_out_bin_idx = np.array([BINS_MAP[k] for k in y_out_bin])\n",
"\n",
" # Add to train\n",
" X_train = np.concatenate([X_train, X_out])\n",
" y_train = np.concatenate([y_train, y_out])\n",
" y_train_idx = np.concatenate([y_train_idx, y_out_bin_idx])\n",
"\n",
" # Add an exact duplicate example to the training set\n",
" exact_duplicate_idx = np.random.randint(0, len(X_train))\n",
" X_duplicate = X_train[exact_duplicate_idx, None]\n",
" y_duplicate = y_train[exact_duplicate_idx, None]\n",
" y_duplicate_idx = y_train_idx[exact_duplicate_idx, None]\n",
"\n",
" # Add to train\n",
" X_train = np.concatenate([X_train, X_duplicate])\n",
" y_train = np.concatenate([y_train, y_duplicate])\n",
" y_train_idx = np.concatenate([y_train_idx, y_duplicate_idx])\n",
"\n",
" py = np.bincount(y_train_idx) / float(len(y_train_idx))\n",
" m = len(BINS)\n",
"\n",
" noise_matrix = generate_noise_matrix_from_trace(\n",
" m,\n",
" trace=0.9 * m,\n",
" py=py,\n",
" valid_noise_matrix=True,\n",
" seed=SEED,\n",
" )\n",
"\n",
" noisy_labels_idx = generate_noisy_labels(y_train_idx, noise_matrix)\n",
" noisy_labels = np.array([list(BINS_MAP.keys())[i] for i in noisy_labels_idx])\n",
"\n",
" return X_train, y_train_idx, noisy_labels, noisy_labels_idx, X_out, X_duplicate"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X_train, y_train_idx, noisy_labels, noisy_labels_idx, X_out, X_duplicate = create_data()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We make a scatter plot of the features, with a color corresponding to the observed labels. Incorrect given labels are highlighted in red if they do not match the true label, outliers highlighted with an a black cross, and duplicates highlighted with a cyan cross."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<details><summary>See the code to visualize the data. **(click to expand)**</summary>\n",
" \n",
"```ipython3\n",
"# Note: This pulldown content is for docs.cleanlab.ai, if running on local Jupyter or Colab, please ignore it.\n",
"\n",
"import matplotlib.pyplot as plt\n",
"\n",
"def plot_data(X_train, y_train_idx, noisy_labels_idx, X_out, X_duplicate):\n",
" # Plot data with clean labels and noisy labels, use BINS_MAP for the legend\n",
" fig, ax = plt.subplots(figsize=(8, 6.5))\n",
" \n",
" low = ax.scatter(X_train[noisy_labels_idx == 0, 0], X_train[noisy_labels_idx == 0, 1], label=\"low\")\n",
" mid = ax.scatter(X_train[noisy_labels_idx == 1, 0], X_train[noisy_labels_idx == 1, 1], label=\"mid\")\n",
" high = ax.scatter(X_train[noisy_labels_idx == 2, 0], X_train[noisy_labels_idx == 2, 1], label=\"high\")\n",
" \n",
" ax.set_title(\"Noisy labels\")\n",
" ax.set_xlabel(r\"$x_1$\", fontsize=16)\n",
" ax.set_ylabel(r\"$x_2$\", fontsize=16)\n",
"\n",
" # Plot true boundaries (x+y=3.3, x+y=6.6)\n",
" ax.set_xlim(-3.5, 9.0)\n",
" ax.set_ylim(-3.5, 9.0)\n",
" ax.plot([-0.7, 4.0], [4.0, -0.7], color=\"k\", linestyle=\"--\", alpha=0.5)\n",
" ax.plot([-0.7, 7.3], [7.3, -0.7], color=\"k\", linestyle=\"--\", alpha=0.5)\n",
"\n",
" # Draw red circles around the points that are misclassified (i.e. the points that are in the wrong bin)\n",
" for i, (X, y) in enumerate(zip([X_train, X_train], [y_train_idx, noisy_labels_idx])):\n",
" for j, (k, v) in enumerate(BINS_MAP.items()):\n",
" label_err = ax.scatter(\n",
" X[(y == v) & (y != y_train_idx), 0],\n",
" X[(y == v) & (y != y_train_idx), 1],\n",
" s=180,\n",
" marker=\"o\",\n",
" facecolor=\"none\",\n",
" edgecolors=\"red\",\n",
" linewidths=2.5,\n",
" alpha=0.5,\n",
" label=\"Label error\",\n",
" )\n",
"\n",
"\n",
" outlier = ax.scatter(X_out[:, 0], X_out[:, 1], color=\"k\", marker=\"x\", s=100, linewidth=2, label=\"Outlier\")\n",
"\n",
" # Plot the exact duplicate\n",
" dups = ax.scatter(\n",
" X_duplicate[:, 0],\n",
" X_duplicate[:, 1],\n",
" color=\"c\",\n",
" marker=\"x\",\n",
" s=100,\n",
" linewidth=2,\n",
" label=\"Duplicates\",\n",
" )\n",
" \n",
" first_legend = ax.legend(handles=[low, mid, high], loc=[0.75, 0.7], title=\"Given Class Label\", alignment=\"left\", title_fontproperties={\"weight\":\"semibold\"})\n",
" second_legend = ax.legend(handles=[label_err, outlier, dups], loc=[0.75, 0.45], title=\"Type of Issue\", alignment=\"left\", title_fontproperties={\"weight\":\"semibold\"})\n",
" \n",
" ax = plt.gca().add_artist(first_legend)\n",
" ax = plt.gca().add_artist(second_legend)\n",
" plt.tight_layout()\n",
"```\n",
" \n",
"</details>"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"nbsphinx": "hidden"
},
"outputs": [],
"source": [
"import matplotlib.pyplot as plt\n",
"\n",
"def plot_data(X_train, y_train_idx, noisy_labels_idx, X_out, X_duplicate):\n",
" # Plot data with clean labels and noisy labels, use BINS_MAP for the legend\n",
" fig, ax = plt.subplots(figsize=(6, 4))\n",
" \n",
" low = ax.scatter(X_train[noisy_labels_idx == 0, 0], X_train[noisy_labels_idx == 0, 1], label=\"low\")\n",
" mid = ax.scatter(X_train[noisy_labels_idx == 1, 0], X_train[noisy_labels_idx == 1, 1], label=\"mid\")\n",
" high = ax.scatter(X_train[noisy_labels_idx == 2, 0], X_train[noisy_labels_idx == 2, 1], label=\"high\")\n",
" \n",
" ax.set_title(\"Noisy labels\")\n",
" ax.set_xlabel(r\"$x_1$\", fontsize=16)\n",
" ax.set_ylabel(r\"$x_2$\", fontsize=16)\n",
"\n",
" # Plot true boundaries (x+y=3.3, x+y=6.6)\n",
" ax.set_xlim(-2.5, 8.5)\n",
" ax.set_ylim(-3.5, 9.0)\n",
" ax.plot([-0.7, 4.0], [4.0, -0.7], color=\"k\", linestyle=\"--\", alpha=0.5)\n",
" ax.plot([-0.7, 7.3], [7.3, -0.7], color=\"k\", linestyle=\"--\", alpha=0.5)\n",
"\n",
" # Draw red circles around the points that are misclassified (i.e. the points that are in the wrong bin)\n",
" for i, (X, y) in enumerate(zip([X_train, X_train], [y_train_idx, noisy_labels_idx])):\n",
" for j, (k, v) in enumerate(BINS_MAP.items()):\n",
" label_err = ax.scatter(\n",
" X[(y == v) & (y != y_train_idx), 0],\n",
" X[(y == v) & (y != y_train_idx), 1],\n",
" s=180,\n",
" marker=\"o\",\n",
" facecolor=\"none\",\n",
" edgecolors=\"red\",\n",
" linewidths=2.5,\n",
" alpha=0.5,\n",
" label=\"Label error\",\n",
" )\n",
"\n",
"\n",
" outlier = ax.scatter(X_out[:, 0], X_out[:, 1], color=\"k\", marker=\"x\", s=100, linewidth=2, label=\"Outlier\")\n",
"\n",
" # Plot the exact duplicate\n",
" dups = ax.scatter(\n",
" X_duplicate[:, 0],\n",
" X_duplicate[:, 1],\n",
" color=\"c\",\n",
" marker=\"x\",\n",
" s=100,\n",
" linewidth=2,\n",
" label=\"Duplicates\",\n",
" )\n",
" \n",
" title_fontproperties = {\"weight\":\"semibold\", \"size\": 8}\n",
" first_legend = ax.legend(handles=[low, mid, high], loc=[0.76, 0.7], title=\"Given Class Label\", alignment=\"left\", title_fontproperties=title_fontproperties, fontsize=8, markerscale=0.5)\n",
" second_legend = ax.legend(handles=[label_err, outlier, dups], loc=[0.76, 0.46], title=\"Type of Issue\", alignment=\"left\", title_fontproperties=title_fontproperties, fontsize=8, markerscale=0.5)\n",
" \n",
" ax = plt.gca().add_artist(first_legend)\n",
" ax = plt.gca().add_artist(second_legend)\n",
" plt.tight_layout()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"plot_data(X_train, y_train_idx, noisy_labels_idx, X_out, X_duplicate)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"In real-world scenarios, you won't know the true labels or the distribution of the features, so we won't use these in this tutorial, except for evaluation purposes."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Get out-of-sample predicted probabilities from a classifier"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"To detect certain types of issues in classification data (e.g. label errors), `Datalab` relies on predicted class probabilities from a trained model. Ideally, the prediction for each example should be out-of-sample (to avoid overfitting), coming from a copy of the model that was not trained on this example. \n",
"\n",
"This tutorial uses a simple logistic regression model \n",
"and the `cross_val_predict()` function from scikit-learn to generate out-of-sample predicted class probabilities for every example in the training set. You can replace this with *any* other classifier model and train it with cross-validation to get out-of-sample predictions.\n",
"Make sure that the columns of your `pred_probs` are properly ordered with respect to the ordering of classes, which for Datalab is: lexicographically sorted by class name."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model = LogisticRegression()\n",
"pred_probs = cross_val_predict(\n",
" estimator=model, X=X_train, y=noisy_labels, cv=5, method=\"predict_proba\"\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Instantiate Datalab object"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Here we instantiate the Datalab object that will be used in the remainder in the tutorial by passing in the data created above.\n",
"\n",
"`Datalab` has several ways of loading the data. In this case, we'll simply wrap the training features and noisy labels in a dictionary so that we can pass it to `Datalab`.\n",
"\n",
"Other supported data formats for `Datalab` include: [HuggingFace Datasets](https://huggingface.co/docs/datasets/index) and [pandas DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html). `Datalab` works across most data modalities (image, text, tabular, audio, etc). It is intended to find issues that commonly occur in datasets for which you have trained a supervised ML model, regardless of the type of data.\n",
"\n",
"Currently, pandas DataFrames that contain categorical columns might cause some issues when instantiating the `Datalab` object, so it is recommended to ensure that your DataFrame does not contain any categorical columns, or use other data formats (eg. python dictionary, HuggingFace Datasets) to pass in your data."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"data = {\"X\": X_train, \"y\": noisy_labels}\n",
"\n",
"lab = Datalab(data, label_name=\"y\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## **Functionality 1**: Incremental issue search "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We can call `find_issues` multiple times on a `Datalab` object to detect issues one type at a time.\n",
"\n",
"This is done via the `issue_types` argument which accepts a dictionary of issue types and any corresponding keyword arguments to specify nondefault keyword arguments to use for detecting each type of issues. In this first call, we only want to detect label issues, which are detected solely based on `pred_probs`, hence there is no need for us to pass in `features` here."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"lab.find_issues(pred_probs=pred_probs, issue_types={\"label\": {}}) \n",
"lab.report()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We can check for additional types of issues with the same `Datalab`. Here, we would like to detect outliers and near duplicates which both utilize the features of the data.\n",
"\n",
"Notice that this second call to `find_issues()` updates the output of `report()`, we can see the existing label issues detected alongside the new issues."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"lab.find_issues(features=data[\"X\"], issue_types={\"outlier\": {}, \"near_duplicate\": {}})\n",
"lab.report()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## **Functionality 2**: Specifying nondefault arguments"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We can also overwrite previously-executed checks for a type of issue. Here we re-run the detection of outliers, but specify that different non-default settings should be used (in this case, the number of neighbors `k` compared against to determine which datapoints are outliers). \n",
"The results from this new detection will replace the original outlier detection results in the updated `Datalab`. You could similarly specify non-default settings for other issue types in the first call to `Datalab.find_issues()`."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"lab.find_issues(features=data[\"X\"], issue_types={\"outlier\": {\"k\": 30}})\n",
"lab.report()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"You can also increase the verbosity of the `report` to see additional information about the data issues and control how many top-ranked examples are shown for each issue."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"lab.report(num_examples=10, verbosity=2)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Notice how the number of flagged outlier issues has changed after specfying different settings to use for outlier detection."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## **Functionality 3**: Save and load Datalab objects"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"A `Datalab` can be saved to a folder at a specified path. In a future Python process, this path can be used to load the `Datalab` from file back into memory. Your dataset is not saved as part of this process, so you'll need to save/load it separately to keep working with it."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"path = \"datalab-files\"\n",
"lab.save(path, force=True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We can load a `Datalab` object we have on file and view the previously detected issues."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"new_lab = Datalab.load(path)\n",
"new_lab.report()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## **Functionality 4**: Adding a custom IssueManager"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"`Datalab` detects pre-defined types of issues for you in one line of code: `find_issues()`. What if you want to check for other custom types of issues along with these pre-defined types, all within the same line of code?\n",
"\n",
"All issue types in `Datalab` are subclasses of cleanlab's `IssueManager` class.\n",
"To register a custom issue type for use with `Datalab`, simply also make it a subclass of `IssueManager`.\n",
"\n",
"The necessary members to implement in the subclass are:\n",
"\n",
"- A class variable called `issue_name` that acts as a unique identifier for the type of issue.\n",
"- An instance method called `find_issues` that:\n",
" - Computes a quality score for each example in the dataset (between 0-1), in terms of how *unlikely* it is to be an issue.\n",
" - Flags each example as an issue or not (may be based on thresholding the quality scores).\n",
" - Combine these in a dataframe that is assigned to an `issues` attribute of the `IssueManager`.\n",
" - Define a summary score for the overall quality of entire dataset, in terms of this type of issue. Set this score as part of the `summary` attribute of the `IssueManager`.\n",
" \n",
"To demonstrate this, we create an arbitrary issue type that checks the divisibility of an example's index in the dataset by 13."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from cleanlab.datalab.internal.issue_manager import IssueManager\n",
"from cleanlab.datalab.internal.issue_manager_factory import register\n",
"\n",
"\n",
"def scoring_function(idx: int, div: int = 13) -> float:\n",
" if idx == 0:\n",
" # Zero excluded from the divisibility check, gets the highest score\n",
" return 1\n",
" rem = idx % div\n",
" inv_scale = idx // div\n",
" if rem == 0:\n",
" return 0.5 * (1 - np.exp(-0.1*(inv_scale-1)))\n",
" else:\n",
" return 1 - 0.49 * (1 - np.exp(-inv_scale**0.5))*rem/div\n",
"\n",
"\n",
"@register # register this issue type for use with Datalab\n",
"class SuperstitionIssueManager(IssueManager):\n",
" \"\"\"A custom issue manager that keeps track of issue indices that\n",
" are divisible by 13.\n",
" \"\"\"\n",
" description: str = \"Examples with indices that are divisible by 13 may be unlucky.\" # Optional\n",
" issue_name: str = \"superstition\"\n",
"\n",
" def find_issues(self, div=13, **_) -> None:\n",
" ids = self.datalab.issues.index.to_series()\n",
" issues_mask = ids.apply(lambda idx: idx % div == 0 and idx != 0)\n",
" scores = ids.apply(lambda idx: scoring_function(idx, div))\n",
" self.issues = pd.DataFrame(\n",
" {\n",
" f\"is_{self.issue_name}_issue\": issues_mask,\n",
" self.issue_score_key: scores,\n",
" },\n",
" )\n",
" summary_score = 1 - sum(issues_mask) / len(issues_mask)\n",
" self.summary = self.make_summary(score = summary_score)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Once registered, this `IssueManager` will perform custom issue checks when `find_issues` is called on a `Datalab` instance.\n",
"\n",
"As our `Datalab` instance here already has results from the outlier and near duplicate checks, we perform the custom issue check separately."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"lab.find_issues(issue_types={\"superstition\": {}})\n",
"lab.report()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
},
"vscode": {
"interpreter": {
"hash": "d4d1e4263499bec80672ea0156c357c1ee493ec2b1c70f0acce89fc37c4a6abe"
}
}
},
"nbformat": 4,
"nbformat_minor": 4
}
docs/source/tutorials/datalab/datalab_quickstart.ipynb
+843
View File
@@ -0,0 +1,843 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Datalab: A unified audit to detect all kinds of issues in data and labels"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Cleanlab offers a `Datalab` object that can identify various issues in your machine learning datasets, such as noisy labels, outliers, (near) duplicates, drift, and other types of problems common in real-world data. These data issues may negatively impact models if not addressed. `Datalab` utilizes *any* ML model you have already trained for your data to diagnose these issues, it only requires access to either: (probabilistic) predictions from your model or its learned representations of the data.\n",
"\n",
"\n",
"**Overview of what we'll do in this tutorial:**\n",
"\n",
"- Compute out-of-sample predicted probabilities for a sample dataset using cross-validation.\n",
"- Use `Datalab` to identify issues such as noisy labels, outliers, (near) duplicates, and other types of problems \n",
"- View the issue summaries and other information about our sample dataset\n",
"\n",
"You can easily replace our demo dataset with your own image/text/tabular/audio/etc dataset, and then run the same code to discover what sort of issues lurk within it!"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<div class=\"alert alert-info\">\n",
"Quickstart\n",
"<br/>\n",
" \n",
"Already have (out-of-sample) `pred_probs` from a model trained on an existing set of labels? Maybe you also have some numeric `features` (or model embeddings of data)? Run the code below to examine your dataset for multiple types of issues.\n",
"\n",
"<div class=markdown markdown=\"1\" style=\"background:white;margin:16px\"> \n",
" \n",
"```ipython3 \n",
"from cleanlab import Datalab\n",
"\n",
"lab = Datalab(data=your_dataset, label_name=\"column_name_of_labels\")\n",
"lab.find_issues(features=your_feature_matrix, pred_probs=your_pred_probs)\n",
"\n",
"lab.report()\n",
"```\n",
" \n",
"</div>\n",
"</div>"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 1. Install and import required dependencies"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"`Datalab` has additional dependencies that are not included in the standard installation of cleanlab.\n",
"\n",
"You can use pip to install all packages required for this tutorial as follows:\n",
"\n",
"```ipython3\n",
"!pip install matplotlib\n",
"!pip install \"cleanlab[datalab]\"\n",
"\n",
"# Make sure to install the version corresponding to this tutorial\n",
"# E.g. if viewing master branch documentation:\n",
"# !pip install git+https://github.com/cleanlab/cleanlab.git\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"nbsphinx": "hidden"
},
"outputs": [],
"source": [
"# Package installation (hidden on docs website).\n",
"dependencies = [\"cleanlab\", \"matplotlib\", \"datasets\"] # TODO: make sure this list is updated\n",
"\n",
"if \"google.colab\" in str(get_ipython()): # Check if it's running in Google Colab\n",
" %pip install cleanlab # for colab\n",
" cmd = ' '.join([dep for dep in dependencies if dep != \"cleanlab\"])\n",
" %pip install $cmd\n",
"else:\n",
" dependencies_test = [dependency.split('>')[0] if '>' in dependency \n",
" else dependency.split('<')[0] if '<' in dependency \n",
" else dependency.split('=')[0] for dependency in dependencies]\n",
" missing_dependencies = []\n",
" for dependency in dependencies_test:\n",
" try:\n",
" __import__(dependency)\n",
" except ImportError:\n",
" missing_dependencies.append(dependency)\n",
"\n",
" if len(missing_dependencies) > 0:\n",
" print(\"Missing required dependencies:\")\n",
" print(*missing_dependencies, sep=\", \")\n",
" print(\"\\nPlease install them before running the rest of this notebook.\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"from sklearn.linear_model import LogisticRegression\n",
"from sklearn.model_selection import cross_val_predict\n",
"\n",
"from cleanlab import Datalab"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 2. Create and load the data (can skip these details)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We'll load a toy classification dataset for this tutorial. The dataset has two numerical features and a label column with three possible classes. Each example is classified as either: *low*, *mid* or *high*."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<details><summary>See the code for data generation. **(click to expand)**</summary>\n",
" \n",
"```ipython3\n",
"# Note: This pulldown content is for docs.cleanlab.ai, if running on local Jupyter or Colab, please ignore it.\n",
"\n",
"from sklearn.model_selection import train_test_split\n",
"from cleanlab.benchmarking.noise_generation import (\n",
" generate_noise_matrix_from_trace,\n",
" generate_noisy_labels,\n",
")\n",
"\n",
"SEED = 123\n",
"np.random.seed(SEED)\n",
"\n",
"BINS = {\n",
" \"low\": [-np.inf, 3.3],\n",
" \"mid\": [3.3, 6.6],\n",
" \"high\": [6.6, +np.inf],\n",
"}\n",
"\n",
"BINS_MAP = {\n",
" \"low\": 0,\n",
" \"mid\": 1,\n",
" \"high\": 2,\n",
"}\n",
"\n",
"\n",
"def create_data():\n",
"\n",
" X = np.random.rand(250, 2) * 5\n",
" y = np.sum(X, axis=1)\n",
" # Map y to bins based on the BINS dict\n",
" y_bin = np.array([k for y_i in y for k, v in BINS.items() if v[0] <= y_i < v[1]])\n",
" y_bin_idx = np.array([BINS_MAP[k] for k in y_bin])\n",
"\n",
" # Split into train and test\n",
" X_train, X_test, y_train, y_test, y_train_idx, y_test_idx = train_test_split(\n",
" X, y_bin, y_bin_idx, test_size=0.5, random_state=SEED\n",
" )\n",
"\n",
" # Add several (5) out-of-distribution points. Sliding them along the decision boundaries\n",
" # to make them look like they are out-of-frame\n",
" X_out = np.array(\n",
" [\n",
" [-1.5, 3.0],\n",
" [-1.75, 6.5],\n",
" [1.5, 7.2],\n",
" [2.5, -2.0],\n",
" [5.5, 7.0],\n",
" ]\n",
" )\n",
" # Add a near duplicate point to the last outlier, with some tiny noise added\n",
" near_duplicate = X_out[-1:] + np.random.rand(1, 2) * 1e-6\n",
" X_out = np.concatenate([X_out, near_duplicate])\n",
"\n",
" y_out = np.sum(X_out, axis=1)\n",
" y_out_bin = np.array([k for y_i in y_out for k, v in BINS.items() if v[0] <= y_i < v[1]])\n",
" y_out_bin_idx = np.array([BINS_MAP[k] for k in y_out_bin])\n",
"\n",
" # Add to train\n",
" X_train = np.concatenate([X_train, X_out])\n",
" y_train = np.concatenate([y_train, y_out])\n",
" y_train_idx = np.concatenate([y_train_idx, y_out_bin_idx])\n",
"\n",
" # Add an exact duplicate example to the training set\n",
" exact_duplicate_idx = np.random.randint(0, len(X_train))\n",
" X_duplicate = X_train[exact_duplicate_idx, None]\n",
" y_duplicate = y_train[exact_duplicate_idx, None]\n",
" y_duplicate_idx = y_train_idx[exact_duplicate_idx, None]\n",
"\n",
" # Add to train\n",
" X_train = np.concatenate([X_train, X_duplicate])\n",
" y_train = np.concatenate([y_train, y_duplicate])\n",
" y_train_idx = np.concatenate([y_train_idx, y_duplicate_idx])\n",
"\n",
" py = np.bincount(y_train_idx) / float(len(y_train_idx))\n",
" m = len(BINS)\n",
"\n",
" noise_matrix = generate_noise_matrix_from_trace(\n",
" m,\n",
" trace=0.9 * m,\n",
" py=py,\n",
" valid_noise_matrix=True,\n",
" seed=SEED,\n",
" )\n",
"\n",
" noisy_labels_idx = generate_noisy_labels(y_train_idx, noise_matrix)\n",
" noisy_labels = np.array([list(BINS_MAP.keys())[i] for i in noisy_labels_idx])\n",
"\n",
" return X_train, y_train_idx, noisy_labels, noisy_labels_idx, X_out, X_duplicate\n",
"```\n",
"\n",
"</details>"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"nbsphinx": "hidden"
},
"outputs": [],
"source": [
"from sklearn.model_selection import train_test_split\n",
"from cleanlab.benchmarking.noise_generation import (\n",
" generate_noise_matrix_from_trace,\n",
" generate_noisy_labels,\n",
")\n",
"\n",
"SEED = 123\n",
"np.random.seed(SEED)\n",
"\n",
"BINS = {\n",
" \"low\": [-np.inf, 3.3],\n",
" \"mid\": [3.3, 6.6],\n",
" \"high\": [6.6, +np.inf],\n",
"}\n",
"\n",
"BINS_MAP = {\n",
" \"low\": 0,\n",
" \"mid\": 1,\n",
" \"high\": 2,\n",
"}\n",
"\n",
"\n",
"def create_data():\n",
"\n",
" X = np.random.rand(250, 2) * 5\n",
" y = np.sum(X, axis=1)\n",
" # Map y to bins based on the BINS dict\n",
" y_bin = np.array([k for y_i in y for k, v in BINS.items() if v[0] <= y_i < v[1]])\n",
" y_bin_idx = np.array([BINS_MAP[k] for k in y_bin])\n",
"\n",
" # Split into train and test\n",
" X_train, X_test, y_train, y_test, y_train_idx, y_test_idx = train_test_split(\n",
" X, y_bin, y_bin_idx, test_size=0.5, random_state=SEED\n",
" )\n",
"\n",
" # Add several (5) out-of-distribution points. Sliding them along the decision boundaries\n",
" # to make them look like they are out-of-frame\n",
" X_out = np.array(\n",
" [\n",
" [-1.5, 3.0],\n",
" [-1.75, 6.5],\n",
" [1.5, 7.2],\n",
" [2.5, -2.0],\n",
" [5.5, 7.0],\n",
" ]\n",
" )\n",
" # Add a near duplicate point to the last outlier, with some tiny noise added\n",
" near_duplicate = X_out[-1:] + np.random.rand(1, 2) * 1e-6\n",
" X_out = np.concatenate([X_out, near_duplicate])\n",
"\n",
" y_out = np.sum(X_out, axis=1)\n",
" y_out_bin = np.array([k for y_i in y_out for k, v in BINS.items() if v[0] <= y_i < v[1]])\n",
" y_out_bin_idx = np.array([BINS_MAP[k] for k in y_out_bin])\n",
"\n",
" # Add to train\n",
" X_train = np.concatenate([X_train, X_out])\n",
" y_train = np.concatenate([y_train, y_out])\n",
" y_train_idx = np.concatenate([y_train_idx, y_out_bin_idx])\n",
"\n",
" # Add an exact duplicate example to the training set\n",
" exact_duplicate_idx = np.random.randint(0, len(X_train))\n",
" X_duplicate = X_train[exact_duplicate_idx, None]\n",
" y_duplicate = y_train[exact_duplicate_idx, None]\n",
" y_duplicate_idx = y_train_idx[exact_duplicate_idx, None]\n",
"\n",
" # Add to train\n",
" X_train = np.concatenate([X_train, X_duplicate])\n",
" y_train = np.concatenate([y_train, y_duplicate])\n",
" y_train_idx = np.concatenate([y_train_idx, y_duplicate_idx])\n",
"\n",
" py = np.bincount(y_train_idx) / float(len(y_train_idx))\n",
" m = len(BINS)\n",
"\n",
" noise_matrix = generate_noise_matrix_from_trace(\n",
" m,\n",
" trace=0.9 * m,\n",
" py=py,\n",
" valid_noise_matrix=True,\n",
" seed=SEED,\n",
" )\n",
"\n",
" noisy_labels_idx = generate_noisy_labels(y_train_idx, noise_matrix)\n",
" noisy_labels = np.array([list(BINS_MAP.keys())[i] for i in noisy_labels_idx])\n",
" # Assign few datapoints to rare class\n",
" random_idx = np.random.randint(0, X_train.shape[0], 3)\n",
" noisy_labels[random_idx] = \"max\"\n",
" noisy_labels_idx[random_idx] = np.max(y_bin_idx) + 1\n",
" \n",
"\n",
" return X_train, y_train_idx, noisy_labels, noisy_labels_idx, X_out, X_duplicate"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X_train, y_train_idx, noisy_labels, noisy_labels_idx, X_out, X_duplicate = create_data()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We make a scatter plot of the features, with a color corresponding to the observed labels. Incorrect given labels are highlighted in red if they do not match the true label, outliers highlighted with an a black cross, and duplicates highlighted with a cyan cross."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<details><summary>See the code to visualize the data. **(click to expand)**</summary>\n",
" \n",
"```ipython3\n",
"# Note: This pulldown content is for docs.cleanlab.ai, if running on local Jupyter or Colab, please ignore it.\n",
"\n",
"import matplotlib.pyplot as plt\n",
"\n",
"def plot_data(X_train, y_train_idx, noisy_labels_idx, X_out, X_duplicate):\n",
" # Plot data with clean labels and noisy labels, use BINS_MAP for the legend\n",
" fig, ax = plt.subplots(figsize=(8, 6.5))\n",
" \n",
" low = ax.scatter(X_train[noisy_labels_idx == 0, 0], X_train[noisy_labels_idx == 0, 1], label=\"low\")\n",
" mid = ax.scatter(X_train[noisy_labels_idx == 1, 0], X_train[noisy_labels_idx == 1, 1], label=\"mid\")\n",
" high = ax.scatter(X_train[noisy_labels_idx == 2, 0], X_train[noisy_labels_idx == 2, 1], label=\"high\")\n",
" \n",
" ax.set_title(\"Noisy labels\")\n",
" ax.set_xlabel(r\"$x_1$\", fontsize=16)\n",
" ax.set_ylabel(r\"$x_2$\", fontsize=16)\n",
"\n",
" # Plot true boundaries (x+y=3.3, x+y=6.6)\n",
" ax.set_xlim(-3.5, 9.0)\n",
" ax.set_ylim(-3.5, 9.0)\n",
" ax.plot([-0.7, 4.0], [4.0, -0.7], color=\"k\", linestyle=\"--\", alpha=0.5)\n",
" ax.plot([-0.7, 7.3], [7.3, -0.7], color=\"k\", linestyle=\"--\", alpha=0.5)\n",
"\n",
" # Draw red circles around the points that are misclassified (i.e. the points that are in the wrong bin)\n",
" for i, (X, y) in enumerate(zip([X_train, X_train], [y_train_idx, noisy_labels_idx])):\n",
" for j, (k, v) in enumerate(BINS_MAP.items()):\n",
" label_err = ax.scatter(\n",
" X[(y == v) & (y != y_train_idx), 0],\n",
" X[(y == v) & (y != y_train_idx), 1],\n",
" s=180,\n",
" marker=\"o\",\n",
" facecolor=\"none\",\n",
" edgecolors=\"red\",\n",
" linewidths=2.5,\n",
" alpha=0.5,\n",
" label=\"Label error\",\n",
" )\n",
"\n",
"\n",
" outlier = ax.scatter(X_out[:, 0], X_out[:, 1], color=\"k\", marker=\"x\", s=100, linewidth=2, label=\"Outlier\")\n",
"\n",
" # Plot the exact duplicate\n",
" dups = ax.scatter(\n",
" X_duplicate[:, 0],\n",
" X_duplicate[:, 1],\n",
" color=\"c\",\n",
" marker=\"x\",\n",
" s=100,\n",
" linewidth=2,\n",
" label=\"Duplicates\",\n",
" )\n",
" \n",
" first_legend = ax.legend(handles=[low, mid, high], loc=[0.75, 0.7], title=\"Given Class Label\", alignment=\"left\", title_fontproperties={\"weight\":\"semibold\"})\n",
" second_legend = ax.legend(handles=[label_err, outlier, dups], loc=[0.75, 0.45], title=\"Type of Issue\", alignment=\"left\", title_fontproperties={\"weight\":\"semibold\"})\n",
" \n",
" ax = plt.gca().add_artist(first_legend)\n",
" ax = plt.gca().add_artist(second_legend)\n",
" plt.tight_layout()\n",
"```\n",
" \n",
"</details>"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"nbsphinx": "hidden"
},
"outputs": [],
"source": [
"import matplotlib.pyplot as plt\n",
"\n",
"def plot_data(X_train, y_train_idx, noisy_labels_idx, X_out, X_duplicate):\n",
" # Plot data with clean labels and noisy labels, use BINS_MAP for the legend\n",
" fig, ax = plt.subplots(figsize=(6, 4))\n",
" \n",
" low = ax.scatter(X_train[noisy_labels_idx == 0, 0], X_train[noisy_labels_idx == 0, 1], label=\"low\")\n",
" mid = ax.scatter(X_train[noisy_labels_idx == 1, 0], X_train[noisy_labels_idx == 1, 1], label=\"mid\")\n",
" high = ax.scatter(X_train[noisy_labels_idx == 2, 0], X_train[noisy_labels_idx == 2, 1], label=\"high\")\n",
" \n",
" ax.set_title(\"Noisy labels\")\n",
" ax.set_xlabel(r\"$x_1$\", fontsize=16)\n",
" ax.set_ylabel(r\"$x_2$\", fontsize=16)\n",
"\n",
" # Plot true boundaries (x+y=3.3, x+y=6.6)\n",
" ax.set_xlim(-2.5, 8.5)\n",
" ax.set_ylim(-3.5, 9.0)\n",
" ax.plot([-0.7, 4.0], [4.0, -0.7], color=\"k\", linestyle=\"--\", alpha=0.5)\n",
" ax.plot([-0.7, 7.3], [7.3, -0.7], color=\"k\", linestyle=\"--\", alpha=0.5)\n",
"\n",
" # Draw red circles around the points that are misclassified (i.e. the points that are in the wrong bin)\n",
" for i, (X, y) in enumerate(zip([X_train, X_train], [y_train_idx, noisy_labels_idx])):\n",
" for j, (k, v) in enumerate(BINS_MAP.items()):\n",
" label_err = ax.scatter(\n",
" X[(y == v) & (y != y_train_idx), 0],\n",
" X[(y == v) & (y != y_train_idx), 1],\n",
" s=180,\n",
" marker=\"o\",\n",
" facecolor=\"none\",\n",
" edgecolors=\"red\",\n",
" linewidths=2.5,\n",
" alpha=0.5,\n",
" label=\"Label error\",\n",
" )\n",
"\n",
"\n",
" outlier = ax.scatter(X_out[:, 0], X_out[:, 1], color=\"k\", marker=\"x\", s=100, linewidth=2, label=\"Outlier\")\n",
"\n",
" # Plot the exact duplicate\n",
" dups = ax.scatter(\n",
" X_duplicate[:, 0],\n",
" X_duplicate[:, 1],\n",
" color=\"c\",\n",
" marker=\"x\",\n",
" s=100,\n",
" linewidth=2,\n",
" label=\"Duplicates\",\n",
" )\n",
" \n",
" title_fontproperties = {\"weight\":\"semibold\", \"size\": 8}\n",
" first_legend = ax.legend(handles=[low, mid, high], loc=[0.76, 0.7], title=\"Given Class Label\", alignment=\"left\", title_fontproperties=title_fontproperties, fontsize=8, markerscale=0.5)\n",
" second_legend = ax.legend(handles=[label_err, outlier, dups], loc=[0.76, 0.46], title=\"Type of Issue\", alignment=\"left\", title_fontproperties=title_fontproperties, fontsize=8, markerscale=0.5)\n",
" \n",
" ax = plt.gca().add_artist(first_legend)\n",
" ax = plt.gca().add_artist(second_legend)\n",
" plt.tight_layout()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"plot_data(X_train, y_train_idx, noisy_labels_idx, X_out, X_duplicate)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"In real-world scenarios, you won't know the true labels or the distribution of the features, so we won't use these in this tutorial, except for evaluation purposes.\n",
"\n",
"\n",
"\n",
"`Datalab` has several ways of loading the data.\n",
"In this case, we'll simply wrap the training features and noisy labels in a dictionary so that we can pass it to `Datalab`."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"data = {\"X\": X_train, \"y\": noisy_labels}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Other supported data formats for `Datalab` include: [HuggingFace Datasets](https://huggingface.co/docs/datasets/index) and [pandas DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html). `Datalab` works across most data modalities (image, text, tabular, audio, etc). It is intended to find issues that commonly occur in datasets for which you have trained a supervised ML model, regardless of the type of data.\n",
"\n",
"Currently, pandas DataFrames that contain categorical columns might cause some issues when instantiating the `Datalab` object, so it is recommended to ensure that your DataFrame does not contain any categorical columns, or use other data formats (eg. python dictionary, HuggingFace Datasets) to pass in your data."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 3. Get out-of-sample predicted probabilities from a classifier"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"To detect certain types of issues in classification data (e.g. label errors), `Datalab` relies on predicted class probabilities from a trained model. Ideally, the prediction for each example should be out-of-sample (to avoid overfitting), coming from a copy of the model that was not trained on this example. \n",
"\n",
"This tutorial uses a simple logistic regression model \n",
"and the `cross_val_predict()` function from scikit-learn to generate out-of-sample predicted class probabilities for every example in the training set. You can replace this with *any* other classifier model and train it with cross-validation to get out-of-sample predictions.\n",
"Make sure that the columns of your `pred_probs` are properly ordered with respect to the ordering of classes, which for Datalab is: lexicographically sorted by class name."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model = LogisticRegression()\n",
"pred_probs = cross_val_predict(\n",
" estimator=model, X=data[\"X\"], y=data[\"y\"], cv=5, method=\"predict_proba\"\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 4. Use Datalab to find issues in the dataset"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We create a `Datalab` object from the dataset, also providing the name of the label column in the dataset. Only instantiate one `Datalab` object per dataset, and note that only classification datasets are supported for now.\n",
"\n",
"All that is need to audit your data is to call `find_issues()`.\n",
"This method accepts various inputs like: predicted class probabilities, numeric feature representations of the data. The more information you provide here, the more thoroughly `Datalab` will audit your data! Note that `features` should be some numeric representation of each example, either obtained through preprocessing transformation of your raw data or embeddings from a (pre)trained model. In this case, our data is already entirely numeric so we just provide the features directly."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"lab = Datalab(data, label_name=\"y\")\n",
"lab.find_issues(pred_probs=pred_probs, features=data[\"X\"])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Now let's review the results of this audit using `report()`.\n",
"This provides a high-level summary of each type of issue found in the dataset."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"lab.report()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 5. Learn more about the issues in your dataset"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Datalab detects all sorts of issues in a dataset and what to do with the findings will vary case-by-case.\n",
"\n",
"To conceptually understand how each type of issue is defined and what it means if detected in your data, check out the [Issue Type Descriptions](../../cleanlab/datalab/guide/issue_type_description.html) page. The [Datalab Issue Types](https://docs.cleanlab.ai/stable/cleanlab/datalab/guide/issue_type_description.html) page also lists additional types of issues that `Datalab.find_issues()` can detect, as well as optional parameters you can specify for greater control over how your data are checked.\n",
"\n",
"Datalab offers several methods to understand more details about a particular issue in your dataset.\n",
"The `get_issue_summary()` method fetches summary statistics regarding how severe each type of issue is overall across the whole dataset."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"lab.get_issue_summary()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"In the returned summary DataFrame: LOWER `score` values indicate types of issues that are MORE severe *overall* across the dataset (lower-quality data in terms of this issue), HIGHER `num_issues` values indicate types of issues that are MORE severe *overall* across the dataset (more datapoints appear to exhibit this issue).\n",
"\n",
"We can also only request the summary for a particular type of issue."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"lab.get_issue_summary(\"label\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The `get_issues()` method returns information for each *individual example* in the dataset including: whether or not it is plagued by this issue (Boolean), as well as a *quality score* (numeric value betweeen 0 to 1) quantifying how severe this issue appears to be for this particular example."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"lab.get_issues().head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Each example receives a separate *quality score* for each issue type (eg. `outlier_score` is the *quality score* for the `outlier` issue type, quantifying *how typical* each datapoint appears to be). LOWER scores indicate MORE severe instances of the issue, so the most-concerning datapoints have the lowest quality scores. Sort by these scores to see the most-concerning examples in your dataset for each type of issue. The quality scores are directly comparable between examples/datasets, but not across different issue types.\n",
"\n",
"Similar to above, we can pass the type of issue as a argument to `get_issues()` to get the information for one particular type of issue.\n",
"As an example, let's see the examples identified as having the most severe *label* issues:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"examples_w_issue = (\n",
" lab.get_issues(\"label\")\n",
" .query(\"is_label_issue\")\n",
" .sort_values(\"label_score\")\n",
")\n",
"\n",
"examples_w_issue.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Inspecting the labels for some of these top-ranked examples, we find their given label was indeed incorrect."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Get additional information \n",
"\n",
"Miscellaneous additional information (statistics, intermediate results, etc) related to a particular issue type can be accessed via `get_info(issue_name)`."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"label_issues_info = lab.get_info(\"label\")\n",
"label_issues_info[\"classes_by_label_quality\"]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"This portion of the info shows overall label quality summaries of all examples annotated as a particular class (e.g. the `Label Issues` column is the estimated number of examples labeled as this class that should actually have a different label).\n",
"To learn more about this, see the documentation for the [cleanlab.dataset.rank_classes_by_label_quality](../../cleanlab/dataset.html#cleanlab.dataset.rank_classes_by_label_quality)\n",
"method.\n",
"\n",
"You can view all sorts of information regarding your dataset using the `get_info()` method with no arguments passed. This is not printed here as it returns a huge dictionary but feel free to check it out yourself! Don't worry if you don't understand all of the miscellaneous information in this `info` dictionary, none of it is critical to diagnose the issues in your dataset. Understanding miscellaneous info may require reading the documentation of the miscellaneous cleanlab functions which computed it.\n",
"\n",
"#### Near duplicate issues \n",
"\n",
"Let's also inspect the examples flagged as (near) duplicates.\n",
"For each such example, the `near_duplicate_sets` column below indicates *which* other examples in the dataset are highly similar to it (this value is empty for examples not flagged as nearly duplicated). The `near_duplicate_score` quantifies *how similar* each example is to its nearest neighbor in the dataset."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"lab.get_issues(\"near_duplicate\").query(\"is_near_duplicate_issue\").sort_values(\"near_duplicate_score\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Learn more about handling near duplicates detected in a dataset from [the FAQ](../faq.html#How-to-handle-near-duplicate-data-identified-by-cleanlab?). \n",
"\n",
"Other issues detected in this tutorial dataset include **outliers** and **class imbalance**, see the [Issue Type Descriptions](../../cleanlab/datalab/guide/issue_type_description.html) for more information. `Datalab` makes it very easy to check your datasets for all sorts of issues that are important to deal with for training robust models. The inputs it uses to detect issues can come from *any* model you have trained (the better your model, the more accurate the issue detection will be).\n",
"\n",
"To learn more, check out this [example notebook](https://github.com/cleanlab/examples/blob/master/datalab_image_classification/datalab.ipynb) (demonstrates Datalab applied to a real dataset) and the [advanced Datalab tutorial](datalab_advanced.html) (demonstrates configuration and customization options to exert greater control)."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"nbsphinx": "hidden"
},
"outputs": [],
"source": [
"# Note: This cell is only for docs.cleanlab.ai, if running on local Jupyter or Colab, please ignore it.\n",
"from sklearn.metrics import roc_auc_score\n",
"\n",
"def precision_at_k(predicted_indices, true_indices, k):\n",
" return len(set(predicted_indices[:k]).intersection(set(true_indices))) / k\n",
"\n",
"def recall_at_k(predicted_indices, true_indices, k):\n",
" return len(set(predicted_indices[:k]).intersection(set(true_indices))) / len(true_indices)\n",
"\n",
"def jaccard_similarity(l1, l2):\n",
" s1 = set(l1)\n",
" s2 = set(l2)\n",
" intersect_set = s1.intersection(s2)\n",
" union_set = s1.union(s2)\n",
" if len(intersect_set) == 0:\n",
" return 0\n",
" return len(intersect_set) / len(union_set)\n",
"\n",
"label_issues = lab.get_issues(\"label\")\n",
"predicted_label_issues_indices = (\n",
" label_issues.query(\"is_label_issue\").sort_values(\"label_score\").index.to_list()\n",
")\n",
"predicted_label_issues_indices_by_score = (\n",
" label_issues.sort_values(\"label_score\").index.to_list()\n",
")\n",
"label_issue_indices = np.where(y_train_idx != noisy_labels_idx)[0]\n",
"\n",
"label_quality_scores = label_issues[\"label_score\"].tolist()\n",
"Z = (y_train_idx == noisy_labels_idx).astype(float).tolist()\n",
"\n",
"predicted_outlier_issues_indices = (\n",
" lab.get_issues(\"outlier\").query(\"is_outlier_issue\").index.to_list()\n",
")\n",
"outlier_issue_indices = list(range(125, 130+1))\n",
"exact_duplicate_idx = [index for index, elem in enumerate(X_train) if (elem == X_duplicate).all()][0]\n",
"if exact_duplicate_idx >= 125: # if the random index selected to create a duplicate >= 125, then the last point is also an outlier\n",
" outlier_issue_indices.append(131)\n",
"\n",
"predicted_duplicate_issues_indices = (\n",
" lab.get_issues(\"near_duplicate\").query(\"is_near_duplicate_issue\").index.tolist()\n",
")\n",
"duplicate_issue_indices = [exact_duplicate_idx, 129, 130, 131]\n",
"\n",
"k = len(label_issue_indices)\n",
"assert precision_at_k(predicted_label_issues_indices, label_issue_indices, k) >= 0.75\n",
"assert recall_at_k(predicted_label_issues_indices, label_issue_indices, k) >= 0.75\n",
"assert precision_at_k(predicted_label_issues_indices_by_score, label_issue_indices, k) == 1.0\n",
"assert recall_at_k(predicted_label_issues_indices_by_score, label_issue_indices, k) == 1.0\n",
"assert roc_auc_score(Z, label_quality_scores) > 0.9\n",
"\n",
"assert jaccard_similarity(predicted_outlier_issues_indices, outlier_issue_indices) > 0.9\n",
"assert jaccard_similarity(predicted_duplicate_issues_indices, duplicate_issue_indices) > 0.9\n",
"\n",
"expected_issue_types = set([\"label\", \"outlier\", \"near_duplicate\", \"class_imbalance\"])\n",
"detected_issue_types = set(lab.get_issue_summary()[lab.get_issue_summary()[\"num_issues\"] > 0][\"issue_type\"])\n",
"assert detected_issue_types == expected_issue_types"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
},
"vscode": {
"interpreter": {
"hash": "d4d1e4263499bec80672ea0156c357c1ee493ec2b1c70f0acce89fc37c4a6abe"
}
}
},
"nbformat": 4,
"nbformat_minor": 4
}

Some files were not shown because too many files have changed in this diff Show More