commit a41b2ab474619175e5e9e70535dc0f499362e60b Author: wehub-resource-sync Date: Mon Jul 13 12:49:22 2026 +0800 chore: import upstream snapshot with attribution diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..162c576 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +indent_style = space +trim_trailing_whitespace = true +indent_size = 4 + +[*.ipynb] +insert_final_newline = false + +[*.{yml,yaml}] +indent_size = 2 diff --git a/.github/ISSUE_TEMPLATE/1-bug-report.md b/.github/ISSUE_TEMPLATE/1-bug-report.md new file mode 100644 index 0000000..88b17ae --- /dev/null +++ b/.github/ISSUE_TEMPLATE/1-bug-report.md @@ -0,0 +1,32 @@ +--- +name: Bug Report +about: Use this template to report a bug. +title: "[Short summary of the bug]" +labels: needs triage +assignees: '' + +--- + + + +# Stack trace + + + +# Steps to reproduce + + + +# Additional information + +- **Cleanlab version**: +- **Operating system**: +- **Python version**: + + diff --git a/.github/ISSUE_TEMPLATE/2-feature-request.md b/.github/ISSUE_TEMPLATE/2-feature-request.md new file mode 100644 index 0000000..19226ef --- /dev/null +++ b/.github/ISSUE_TEMPLATE/2-feature-request.md @@ -0,0 +1,26 @@ +--- +name: Feature Request +about: Use this template to ask for new functionality in cleanlab. +title: "[Short summary of the feature]" +labels: needs triage +assignees: '' + +--- + + + +# Details + + diff --git a/.github/ISSUE_TEMPLATE/3-help.md b/.github/ISSUE_TEMPLATE/3-help.md new file mode 100644 index 0000000..8c106de --- /dev/null +++ b/.github/ISSUE_TEMPLATE/3-help.md @@ -0,0 +1,17 @@ +--- +name: Help +about: Use this template to ask for help. +title: "[Short summary of the question]" +labels: question +assignees: '' + +--- + + diff --git a/.github/ISSUE_TEMPLATE/4-other.md b/.github/ISSUE_TEMPLATE/4-other.md new file mode 100644 index 0000000..9c2821d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/4-other.md @@ -0,0 +1,10 @@ +--- +name: Other +about: Use this template when no other template applies. +title: "[Short summary of the issue]" +labels: needs triage +assignees: '' + +--- + + diff --git a/.github/get_min_dependencies.py b/.github/get_min_dependencies.py new file mode 100644 index 0000000..573254e --- /dev/null +++ b/.github/get_min_dependencies.py @@ -0,0 +1,16 @@ +"""This script fetches minimum dependencies of cleanlab package and writes them to the file requirements-min.txt""" + +import json + + +if __name__ == "__main__": + with open("./deps.json", "r") as f: + deps = json.load(f) + + for package in deps: + if package["package"]["package_name"] == "cleanlab": + for dep in package["dependencies"]: + req_version = dep["required_version"] + with open("requirements-min.txt", "a") as f: + if req_version.startswith(">="): + f.write(f"{dep['package_name']}=={req_version[2:]}\n") diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..4768841 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,82 @@ +## Summary + +> 🎯 **Purpose**: Describe the objective of your changes in this Pull-Request. +> +> 📜 **Example Usage**: If adding new functionality, strive to provide a minimal working example (MWE) that demonstrates end-to-end usage and output of your new methods. If creating a MWE is not feasible, a conceptual code snippet showcasing the usage is also appreciated. Ensure that the code provided can be copied, pasted, and run without modifications. + +**[ ✏️ Write your summary here. ]** + +```python +# Example code snippet + +# Necessary Imports: +from my_module import my_new_function + +# Setup if needed (e.g., data loading, model instantiation): +a = 20 +b = 22 + +# Usage and Output: +result = my_new_function(a, b) +# 42 + +# ---- Your code below ---- +# [Replace the example above with your own code, ensuring it is executable and demonstrates your feature or fix.] +[your_uncommented_runnable_code_here] +# [your_output_here_as_a_comment] +``` + +## Impact + +> 🌐 Areas Affected: Enumerate modules, functions, or documentation impacted by your code. +> +> 👥 Who’s Affected: Mention who might be impacted (if applicable). + +**Screenshots** + +> 📸 If your changes modify documentation or a tutorial notebook, please include screenshots or GIFs showing the rendered outputs for a quicker initial review, highlighting the main changes. + + +## Testing + +> 🔍 Testing Done: Outline what kinds of tests you performed. +> +> 🔗 Test Case Link: Directly link to the most end-to-end check of your new functionality. + +**Unaddressed Cases** + +> It's ok if your unit tests are not yet +> comprehensive when you first open the PR, +> we can revisit them later! +> +> ⚠️ Mention any aspects of testing that have *not* been covered, and why. + +- + +## Links to Relevant Issues or Conversations + +> 🔗 What Git or Slack items (Issues, threads, etc) that are specifically related to +> this work? Please link them here. + +- + +## References + +> 📚 Include any extra information that may be helpful to reviewers of your +> Pull-Request here (e.g. relevant URLs or Wiki pages that could help give +> background information). +> +> Share links, docs, or sources that facilitated your changes. +> +> If relevant, please include additional code snippets +> (and their outputs/return values) showing alternative ways to use your newly added methods. + +```python +[additional code snippets and explanations] +``` + +- + +## Reviewer Notes +> 💡 Include any specific points for the reviewer to consider during their review. + diff --git a/.github/workflows/bleeding-edge.yml b/.github/workflows/bleeding-edge.yml new file mode 100644 index 0000000..15dd668 --- /dev/null +++ b/.github/workflows/bleeding-edge.yml @@ -0,0 +1,35 @@ +name: Bleeding Edge Dependencies Test +on: + schedule: + - cron: '0 6 1 * *' # First day of each month at 6:00 UTC + workflow_dispatch: # Allow manual triggering + +jobs: + test-bleeding-edge: + name: "Bleeding Edge Dependencies: Python Latest" + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v6 + - uses: snok/latest-python-versions@v1 + id: python-versions + with: + min-version: '3.14' + include-prereleases: false + - uses: actions/setup-python@v6 + with: + python-version: ${{ fromJson(steps.python-versions.outputs.latest-python-versions)[0] }} + - name: Download cleanvision test data + run: | + git clone https://github.com/cleanlab/assets.git + mv assets/cleanlab_test_data ./tests/datalab/data + - name: Install bleeding-edge dependencies + run: | + python -m pip install --upgrade pip + pip install --pre --upgrade numpy scikit-learn pandas tqdm termcolor + pip install --pre torch torchvision || pip install torch torchvision + pip install -e ".[all]" + - name: Install test dependencies + run: pip install --no-cache-dir -r requirements-test-core.txt + - name: Test with bleeding-edge deps + run: pytest --verbose --maxfail=10 -m "not slow" --ignore=tests/test_frameworks.py --ignore=tests/test_model_pytorch_cnn.py \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..3815719 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,165 @@ +name: CI +on: + push: + pull_request: + schedule: + - cron: '0 12 * * 1' # Monday at 12:00 UTC +jobs: + test-core: + name: "Core Tests: Python ${{ matrix.python }} on ${{ matrix.os }}" + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest + python: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python }} + - name: Download cleanvision test data + run: | + git clone https://github.com/cleanlab/assets.git + mv assets/cleanlab_test_data ./tests/datalab/data + - name: Install cleanlab + run: | + python -m pip install --upgrade pip + pip install -e ".[all]" + - name: Install core test dependencies + run: pip install --no-cache-dir -r requirements-test-core.txt + - name: Audit dependencies for vulnerabilities + uses: pypa/gh-action-pip-audit@v1.1.0 + with: + inputs: requirements-test-core.txt + - name: Run core tests (excluding ML framework tests) + run: pytest --verbose --maxfail=30 --order-tests --cov=cleanlab/ --cov-config .coveragerc --cov-report=xml -m "not slow" --ignore=tests/test_frameworks.py --ignore=tests/test_model_pytorch_cnn.py + - uses: codecov/codecov-action@v5 + with: + version: "v0.7.3" + + test-ml-frameworks: + name: "ML Framework Tests: Python ${{ matrix.python }} on ubuntu-latest" + runs-on: ubuntu-latest + strategy: + matrix: + python: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python }} + - name: Download cleanvision test data + run: | + git clone https://github.com/cleanlab/assets.git + mv assets/cleanlab_test_data ./tests/datalab/data + - name: Install cleanlab + run: | + python -m pip install --upgrade pip + pip install -e ".[all]" + - name: Install ML framework dependencies + run: pip install --no-cache-dir -r requirements-dev.txt + - name: Audit dependencies for vulnerabilities + uses: pypa/gh-action-pip-audit@v1.1.0 + with: + inputs: requirements-dev.txt + - name: Run ML framework tests + run: pytest --verbose --maxfail=30 tests/test_frameworks.py tests/test_model_pytorch_cnn.py + test-without-extras-min-versions: + name: Test without optional dependencies and with minimum compatible versions of dependencies + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: '3.10' + - name: Install cleanlab + run: | + python -m pip install --upgrade pip + pip install . + - name: Install test dependencies + run: | + pip install pytest pipdeptree hypothesis + pipdeptree -j > deps.json + - name: Install minimum versions + run: | + python ./.github/get_min_dependencies.py + pip install -r requirements-min.txt + - name: Run tests + run: | + pytest tests/test_multilabel_classification.py tests/test_multiannotator.py tests/test_filter_count.py + test-max-versions: + name: "Maximum Dependency Versions: Python 3.14" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.14' + - name: Download cleanvision test data + run: | + git clone https://github.com/cleanlab/assets.git + mv assets/cleanlab_test_data ./tests/datalab/data + - name: Install latest stable versions + run: | + python -m pip install --upgrade pip + pip install --upgrade numpy scikit-learn pandas tqdm termcolor + pip install --upgrade torch torchvision + pip install -e ".[all]" + - name: Install test dependencies + run: pip install --no-cache-dir -r requirements-test-core.txt + - name: Test with latest stable versions + run: pytest --verbose --maxfail=10 -m "not slow" --ignore=tests/test_frameworks.py --ignore=tests/test_model_pytorch_cnn.py + typecheck: + name: Type check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: '3.14' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ".[all]" # install dependencies + pip install --no-cache-dir -r requirements-test-core.txt # install core dependencies and type stubs + - name: Type check + run: mypy --install-types --non-interactive cleanlab + fmt: + name: Format + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: psf/black@stable + flake8: + name: Check for unused/wildcard imports + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: '3.14' + - name: Install flake8 + run: pip install flake8 + - name: Lint with flake8 + run: flake8 cleanlab tests + nblint: + name: Lint Notebooks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: cleanlab/nblint-action@v1 + with: + directory: 'docs' \ No newline at end of file diff --git a/.github/workflows/gh-pages.yaml b/.github/workflows/gh-pages.yaml new file mode 100644 index 0000000..8610c70 --- /dev/null +++ b/.github/workflows/gh-pages.yaml @@ -0,0 +1,210 @@ +name: GitHub Pages + +on: + push: + branches: + - "**" + + release: + types: [published] + + workflow_dispatch: + +jobs: + deploy: + runs-on: ubuntu-24.04 + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install Pandoc + # pin Pandoc version + run: | + cd /tmp + wget https://github.com/jgm/pandoc/releases/download/2.19.2/pandoc-2.19.2-linux-amd64.tar.gz + sudo tar xzvf pandoc-2.19.2-linux-amd64.tar.gz --strip-components 1 -C /usr/local + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Setup Node + uses: actions/setup-node@v6 + with: + node-version: "22" + + - name: Install FFmpeg + run: | + sudo apt-get update + sudo apt-get install -y ffmpeg + + - name: Upgrade pip + run: python3 -m pip install --upgrade pip + + # Caching removed to resolve disk space issues + # Heavy ML packages (TensorFlow, PyTorch, HuggingFace) will be downloaded fresh each run + # but this avoids the 3.7GB cache that causes "No space left on device" errors + + - name: Install cleanlab + run: pip install -e ".[all]" + + - name: Install dependencies + run: python3 -m pip install -r docs/requirements.txt + + - name: Install katex + run: npm install -g katex + + - name: Find and replace "%pip install cleanlab" with master branch in .ipynb files + if: ${{ github.ref_type == 'branch' }} + uses: jacobtomlinson/gha-find-replace@v2 + with: + find: "%pip install cleanlab ${{ '#' }} for colab" + replace: "%pip install git+https://github.com/${{ github.repository }}.git@${{ github.sha }}" + include: "docs/source**.ipynb" + + - name: Find and replace "%pip install cleanlab" with release tag in .ipynb files + if: ${{ github.ref_type == 'tag' }} + uses: jacobtomlinson/gha-find-replace@v2 + with: + find: "%pip install cleanlab ${{ '#' }} for colab" + replace: "%pip install cleanlab==${{ github.ref_name }}" + include: "docs/source**.ipynb" + + - name: Add and Commit changes to .ipynb files + run: | + git config --global user.email "actions@github.com" + git config --global user.name "GitHub Actions" + git add -A && git commit -m "Change cleanlab ver in .ipynb files" + + - name: Change tag to the latest commit + if: ${{ github.ref_type == 'tag' }} + run: | + git tag -f $GITHUB_REF_NAME + + - name: Run Sphinx with sphinx-multiversion wrapper - branch trigger + if: ${{ github.ref_type == 'branch' }} + env: + TF_CPP_MIN_LOG_LEVEL: 3 + HF_HOME: /tmp/hf_cache + HF_HUB_DISABLE_SYMLINKS_WARNING: 1 + TRANSFORMERS_CACHE: /tmp/transformers + TORCH_HOME: /tmp/torch + run: sphinx-multiversion docs/source cleanlab-docs -D smv_branch_whitelist=${{ github.ref_name }} -D smv_tag_whitelist=None + + - name: Run Sphinx with sphinx-multiversion wrapper - tag trigger + if: ${{ github.ref_type == 'tag' }} + env: + TF_CPP_MIN_LOG_LEVEL: 3 + HF_HOME: /tmp/hf_cache + HF_HUB_DISABLE_SYMLINKS_WARNING: 1 + TRANSFORMERS_CACHE: /tmp/transformers + TORCH_HOME: /tmp/torch + run: sphinx-multiversion docs/source cleanlab-docs -D smv_branch_whitelist=None -D smv_tag_whitelist=${{ github.ref_name }} + + - name: Debug temporary cache usage (not saved) + run: | + echo "=== Temporary Cache Usage ===" + echo "These caches are created in /tmp and cleaned up after build:" + echo " /tmp/hf_cache: $(du -sh /tmp/hf_cache 2>/dev/null | cut -f1 || echo 'not found')" + echo " /tmp/transformers: $(du -sh /tmp/transformers 2>/dev/null | cut -f1 || echo 'not found')" + echo " /tmp/torch: $(du -sh /tmp/torch 2>/dev/null | cut -f1 || echo 'not found')" + echo "Pip cache directory: $(pip cache dir)" + echo "Pip cache size: $(du -sh $(pip cache dir) 2>/dev/null | cut -f1 || echo 'not found')" + echo "========================" + + - name: Get latest stable release + id: stable_release + continue-on-error: true + uses: pozetroninc/github-action-get-latest-release@master + with: + owner: ${{ github.repository_owner }} + repo: cleanlab + excludes: prerelease, draft + + - name: Create latest_release.txt file + if: ${{ steps.stable_release.outcome == 'success' }} + run: echo ${{ steps.stable_release.outputs.release }} > cleanlab-docs/latest_release.txt + + - name: Find and replace "placeholder_version_number" in versioning.js + if: ${{ steps.stable_release.outcome == 'success' }} + uses: jacobtomlinson/gha-find-replace@v2 + with: + find: "placeholder_version_number" + replace: ${{ steps.stable_release.outputs.release }} + include: "docs/source/_templates/versioning.js" + + - name: Find and replace "placeholder_commit_hash" in versioning.js + uses: jacobtomlinson/gha-find-replace@v2 + with: + find: "placeholder_commit_hash" + replace: ${{ github.sha }} + include: "docs/source/_templates/versioning.js" + + - name: Copy versioning JS file to cleanlab-docs/ + run: | + cp docs/source/_templates/versioning.js cleanlab-docs/versioning.js + + - name: Find and replace "stable_url" in redirect-to-stable.html + if: ${{ github.ref_type == 'tag' && steps.stable_release.outcome == 'success' }} + uses: jacobtomlinson/gha-find-replace@v2 + with: + find: "stable_url" + replace: ${{ env.DOCS_SITE_URL }}stable/index.html + include: "docs/source/_templates/redirect-to-stable.html" + + - name: Copy redirecting HTML file to cleanlab-docs/ + if: ${{ github.ref_type == 'tag' }} + run: cp docs/source/_templates/redirect-to-stable.html cleanlab-docs/index.html + + - name: Update OpenGraph URLs in HTML files + # This code fixes the URLs in the HTML files with OpenGraph tags. + # It uses a regular expression to capture two parts of the URL: + # $1: The first part of the URL, including the protocol and domain. + # $2: The rest of the URL, including the path and query string. + # The replacement string combines the captured parts with the ref name. + # This ensures that the ref name is inserted into the URL correctly. + env: + REF_URL_SEGMENT: ${{ (github.ref_type == 'tag' && steps.stable_release.outcome == 'success') && 'stable' || github.ref_name }} + uses: jacobtomlinson/gha-find-replace@v2 + with: + find: '(- + sudo apt-get install -y + pandoc + - uses: actions/checkout@v6 + - run: | + find . -name '*.html' -delete + - run: | + find . -name '*.md' -exec pandoc -i {} -o {}.html \; + - uses: anishathalye/proof-html@v2 + with: + directory: . + check_html: false + check_favicon: false + ignore_missing_alt: true + tokens: | + {"https://github.com": "${{ secrets.GITHUB_TOKEN }}"} + ignore_url: | + https://jair.org/index.php/jair/article/view/12125 + https://stackoverflow.com/questions/41573587/what-is-the-difference-between-venv-pyvenv-pyenv-virtualenv-virtualenvwrappe + ignore_url_re: | + https://www.gnu.org/software/wget + swap_urls: | + {"^(\\..*)\\.md(#?.*)$": "\\1.md.html\\2", + "^(https://github\\.com/.*)#.*$": "\\1"} diff --git a/.github/workflows/manual-testpypi.yml b/.github/workflows/manual-testpypi.yml new file mode 100644 index 0000000..fc72fc5 --- /dev/null +++ b/.github/workflows/manual-testpypi.yml @@ -0,0 +1,154 @@ +name: Manual TestPyPI Release + +on: + workflow_dispatch: # Manual trigger only + inputs: + reason: + description: 'Reason for this test release' + required: false + default: 'Testing changes before official release' + +jobs: + extract_project_name: + runs-on: ubuntu-latest + outputs: + cleanlab_package_name: ${{ steps.extract_name.outputs.value }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: TOML Reader + uses: SebRollen/toml-action@v1.2.0 + id: extract_name + with: + file: "pyproject.toml" + field: "project.name" + + build: + needs: extract_project_name + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Remove tests directory + run: | + TEST_DIR=$(find . -type d -name "tests") + if [ -n "$TEST_DIR" ]; then + rm -rf $TEST_DIR + else + echo "No tests directory found! This is unexpected." + exit 1 + fi + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: 3.11 + + - name: Modify version for TestPyPI + run: | + # Add .dev suffix with run number for TestPyPI + sed -i "s/__version__ = \"\(.*\)\"/__version__ = \"\1.dev${{ github.run_number }}\"/" cleanlab/version.py + echo "Modified version for TestPyPI:" + grep "__version__" cleanlab/version.py + + - name: Install dependencies + run: pip install --upgrade "setuptools>=65.0,<70.0" wheel twine==5.0.0 pkginfo + + - name: Clean build artifacts + run: | + rm -rf build/ dist/ *.egg-info cleanlab.egg-info/ + + - name: Perform some tests on the metadata of the package + run: | + python setup.py check --metadata --strict + + - name: Build package + run: | + python setup.py sdist bdist_wheel + + - name: Test package + run: | + twine check dist/* + + - name: Display package info + run: | + echo "📦 Built packages:" + ls -lh dist/ + echo "" + echo "📝 Reason: ${{ github.event.inputs.reason }}" + + - name: Store build artifacts + uses: actions/upload-artifact@v5 + with: + name: package + path: dist/* + + publish-testpypi: + needs: build + runs-on: ubuntu-latest + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + + steps: + - name: Fetch build artifacts + uses: actions/download-artifact@v5 + with: + name: package + path: dist/ + + - name: Publish to TestPyPI + uses: pypa/gh-action-pypi-publish@e53eb8b103ffcb59469888563dc324e3c8ba6f06 + with: + repository-url: https://test.pypi.org/legacy/ + verify-metadata: true + + test-basic-import: + needs: [publish-testpypi, extract_project_name] + runs-on: ubuntu-latest + + steps: + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: 3.11 + + - name: Install package from TestPyPI + env: + CLEANLAB_PACKAGE_NAME: ${{ needs.extract_project_name.outputs.cleanlab_package_name }} + run: pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ $CLEANLAB_PACKAGE_NAME + + - name: Test importing package + run: | + python -c "import cleanlab + from cleanlab.outlier import OutOfDistribution + ood = OutOfDistribution() + print(ood) + print('✅ Basic import test passed!') + print('Version:', cleanlab.__version__)" + + test-extras-import: + needs: [publish-testpypi, extract_project_name] + runs-on: ubuntu-latest + + steps: + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: 3.11 + + - name: Install package with extras + env: + CLEANLAB_PACKAGE_NAME: ${{ needs.extract_project_name.outputs.cleanlab_package_name }} + run: pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ $CLEANLAB_PACKAGE_NAME[all] + + - name: Test importing package with extras + run: | + python -c "import cleanlab + from cleanlab import Datalab + lab = Datalab({'a': [1, 2, 3, 4, 5], 'b': ['a', 'b', 'c', 'b', 'a']}) + print(lab) + print('✅ Extras import test passed!') + print('Version:', cleanlab.__version__)" diff --git a/.github/workflows/release-build-publish.yml b/.github/workflows/release-build-publish.yml new file mode 100644 index 0000000..0e98210 --- /dev/null +++ b/.github/workflows/release-build-publish.yml @@ -0,0 +1,260 @@ +name: Release Build Publish + +on: + release: + types: [ published ] + +jobs: + extract_project_name: + runs-on: ubuntu-latest + outputs: + cleanlab_package_name: ${{ steps.extract_name.outputs.value }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: TOML Reader + uses: SebRollen/toml-action@v1.2.0 + id: extract_name + with: + file: "pyproject.toml" + field: "project.name" + + store_published_version: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.compare.outputs.version }} + steps: + + - name: Checkout code + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: 3.11 + + - name: Install package + run: pip install . + + - name: Compare the published version with the package version + id: compare + run: | + TAG_NAME="${{ github.ref_name }}" + # TAG_NAME must be in the form "v1.2.3[...]", error if it doesn't match + if [[ ! $TAG_NAME =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then + echo "Tag name '$TAG_NAME' does not match the expected pattern 'vX.Y.Z'" + exit 1 + fi + # Strip the 'v' prefix from the tag, and store the version in an output variable + VERSION=${TAG_NAME:1} # Strip the 'v' prefix from the tag + echo "version=$VERSION" >> $GITHUB_OUTPUT + # Check that the package version matches the tag version + PACKAGE_VERSION=$(python -c "import cleanlab; print(cleanlab.__version__)") + if [ "$PACKAGE_VERSION" != "$VERSION" ]; then + echo "Package version $PACKAGE_VERSION does not match tag $VERSION" + exit 1 + fi + + build: + needs: [store_published_version, extract_project_name] + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Remove tests directory + run: | + # Remove the tests directory from the package + TEST_DIR=$(find . -type d -name "tests") + if [ -n "$TEST_DIR" ]; then + rm -rf $TEST_DIR + else + echo "No tests directory found! This is unexpected." + exit 1 + fi + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: 3.11 + + - name: Install dependencies + run: pip install --upgrade "setuptools>=65.0,<70.0" wheel twine==5.0.0 pkginfo + + - name: Clean build artifacts + run: | + rm -rf build/ dist/ *.egg-info cleanlab.egg-info/ + + - name: Perform some tests on the metadata of the package + run: | + python setup.py check --metadata --strict + + - name: Build package + run: | + python setup.py sdist bdist_wheel + + - name: Test package + run: | + twine check dist/* + + - name: Verify contents of the tar.gz + env: + CLEANLAB_PACKAGE_NAME: ${{ needs.extract_project_name.outputs.cleanlab_package_name }} + run: | + # Extract the tar.gz to a temporary directory + TMPDIR=$(mktemp -d) + tar -xzf dist/*.tar.gz -C $TMPDIR + PACKAGE_DIR=$TMPDIR/$(ls $TMPDIR) # Assuming there's only one directory extracted + + # Define expected files and directories + EXPECTED_FILES="LICENSE PKG-INFO MANIFEST.in DEVELOPMENT.md CONTRIBUTING.md CODE_OF_CONDUCT.md README.md pyproject.toml setup.cfg setup.py" + EXPECTED_DIRS="cleanlab $CLEANLAB_PACKAGE_NAME.egg-info" + + # Collect actual top-level files and directories + ACTUAL_CONTENTS=$(ls $PACKAGE_DIR) + ACTUAL_FILES=$(find $PACKAGE_DIR -maxdepth 1 -type f -exec basename {} \;) + ACTUAL_DIRS=$(find $PACKAGE_DIR -maxdepth 1 -type d -exec basename {} \; | sed "1d") # Remove the package directory itself from the list + + # Function to check for unexpected or missing files and directories + check_contents() { + local missing=0 + # Check for expected files + for expected_file in $EXPECTED_FILES; do + echo $ACTUAL_FILES | grep -qw $expected_file || { echo "Missing expected file: $expected_file"; missing=1; } + done + + # Check for expected directories + for expected_dir in $EXPECTED_DIRS; do + echo $ACTUAL_DIRS | grep -qw $expected_dir || { echo "Missing expected directory: $expected_dir"; missing=1; } + done + + # Check for unexpected files and directories + for actual in $ACTUAL_CONTENTS; do + echo $EXPECTED_FILES $EXPECTED_DIRS | grep -qw $actual || { echo "Unexpected item in package: $actual"; missing=1; } + done + + # Exit if anything unexpected or missing + if [ $missing -ne 0 ]; then + echo "Package content verification failed." + exit 1 + fi + } + + # Execute checks + check_contents + + - name: Store build artifacts + uses: actions/upload-artifact@v5 + with: + name: package + path: dist/* + + publish-testpypi: + needs: build + runs-on: ubuntu-latest + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + + steps: + - name: Fetch build artifacts + uses: actions/download-artifact@v5 + with: + name: package + path: dist/ + + - name: Publish to TestPyPI + uses: pypa/gh-action-pypi-publish@e53eb8b103ffcb59469888563dc324e3c8ba6f06 + with: + repository-url: https://test.pypi.org/legacy/ + verify-metadata: true + + verify-version: + needs: [publish-testpypi, store_published_version, extract_project_name] + runs-on: ubuntu-latest + environment: testpypi + + steps: + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: 3.11 + + - name: Install package from TestPyPI + env: + CLEANLAB_PACKAGE_NAME: ${{ needs.extract_project_name.outputs.cleanlab_package_name }} + run: pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ $CLEANLAB_PACKAGE_NAME + + - name: Verify the published version + env: + VERSION: ${{ needs.store_published_version.outputs.version }} + run: | + TEST_IMPORT_VERSION=$(python -c "import cleanlab; print(cleanlab.__version__)") + if [ "$TEST_IMPORT_VERSION" != "$VERSION" ]; then + echo "Imported version $TEST_IMPORT_VERSION does not match tag $VERSION" + exit 1 + fi + + test-basic-import: + needs: [verify-version, extract_project_name] + runs-on: ubuntu-latest + + steps: + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: 3.11 + + - name: Install package from TestPyPI + env: + CLEANLAB_PACKAGE_NAME: ${{ needs.extract_project_name.outputs.cleanlab_package_name }} + run: pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ $CLEANLAB_PACKAGE_NAME + + - name: Test importing package + run: | + python -c "import cleanlab + from cleanlab.outlier import OutOfDistribution + ood = OutOfDistribution() + print(ood)" + + test-extras-import: + needs: [verify-version, extract_project_name] + runs-on: ubuntu-latest + + steps: + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: 3.11 + + - name: Install package with extras + env: + CLEANLAB_PACKAGE_NAME: ${{ needs.extract_project_name.outputs.cleanlab_package_name }} + run: pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ $CLEANLAB_PACKAGE_NAME[all] + + - name: Test importing package with extras + run: | + python -c "import cleanlab + from cleanlab import Datalab + lab = Datalab({'a': [1, 2, 3, 4, 5], 'b': ['a', 'b', 'c', 'b', 'a']}) + print(lab)" + + publish-pypi: + needs: [test-basic-import, test-extras-import] + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + + steps: + - name: Fetch build artifacts + uses: actions/download-artifact@v5 + with: + name: package + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@e53eb8b103ffcb59469888563dc324e3c8ba6f06 + with: + verify-metadata: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..05a4fa0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,159 @@ +cleanlab/*.ipynb +.DS_Store +*/.DS_Store +tests/fasttext_data/ +data/ +docs/_build/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ +.coverage* +*.py,cover +.hypothesis/** +.pytest_cache/** + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +logs/ + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +build +_build +cleanlab-docs + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +.direnv + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# Downloaded datasets for docs +/docs/source/notebooks/aclImdb +/docs/source/notebooks/data +/docs/source/notebooks/*.gz +/docs/source/notebooks/spoken_digits +/docs/source/notebooks/pretrained_models +/docs/source/tutorials/datalab/datalab-files/ + +# Editor files +.vscode/ +.idea/ + +# jupyter notebooks +.ipynb_checkpoints/ + +# Misc +results/ +image_files* + +# datasets +cifar* + +# IPython +profile_default/ +ipython_config.py +__pypackages__/ + + +# Data files +**.zip +**-ubyte +**.gz +**.tar +**.7Z +**.BZ2 +**.rar +**.zarr +**.npy + diff --git a/.mypy.ini b/.mypy.ini new file mode 100644 index 0000000..3cd1be9 --- /dev/null +++ b/.mypy.ini @@ -0,0 +1,32 @@ +[mypy] +allow_redefinition = True + +[mypy-sklearn.*] +ignore_missing_imports = True + +[mypy-scikeras.*] +ignore_missing_imports = True + +[mypy-tensorflow.*] +ignore_missing_imports = True + +[mypy-IPython.display.*] +ignore_missing_imports = True + +[mypy-torch.*] +ignore_missing_imports = True + +[mypy-torchvision.*] +ignore_missing_imports = True + +[mypy-tqdm.*] +ignore_missing_imports = True + +[mypy-matplotlib.*] +ignore_missing_imports = True + +[mypy-datasets.*] +ignore_missing_imports = True + +[mypy-scipy.*] +ignore_missing_imports = True diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..640716b --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,5 @@ +repos: + - repo: https://github.com/psf/black + rev: 25.12.0 + hooks: + - id: black diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..925e2a7 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,131 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Providing specific details to ensure your comments are easy to understand +* Valuing other people's time +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Failing to properly attribute original sources +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +team@cleanlab.ai. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..3afcadf --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,59 @@ +# Contributing to cleanlab + +All kinds of contributions to cleanlab are greatly appreciated. If you're not looking to write code, submitting a [feature request](#feature-requests) or +[bug report](#bug-reports) is a great way to contribute. If you want to get +your hands dirty, you can submit [Pull Requests](#pull-requests), either working on your +own ideas or addressing [existing issues][issues]. + +If you are unsure or confused about anything, please go ahead and submit your +issue or pull request anyways! We appreciate all contributions, and we'll do +our best to incorporate your feedback or code into cleanlab. + +For some [ideas on useful contributions](https://github.com/cleanlab/cleanlab/wiki#ideas-for-contributing-to-cleanlab), look for [Issues](https://github.com/cleanlab/cleanlab/issues) or [Projects](https://github.com/cleanlab/cleanlab/projects) with the "good first issue" tag. Join our [Slack Community](https://cleanlab.ai/slack) to discuss other ideas! + +Detailed contributing instructions can be found in the [Development Guide](DEVELOPMENT.md), please read this carefully! + + +## Feature Requests + +Do you have an idea for an awesome new feature for cleanab? Let us know by +[submitting a feature request][issue]. + +If you are inclined to do so, you're welcome to [fork][fork] cleanlab, work on +implementing the feature yourself, and submit a patch. In this case, it's +*highly recommended* that you first [open an issue][issue] describing your +enhancement to get early feedback on the new feature that you are implementing. +This will help avoid wasted efforts and ensure that your work is incorporated +into the cleanlab code base. + +## Bug Reports + +Did something go wrong with cleanlab? Sorry about that! Bug reports are greatly +appreciated! + +When you [submit a bug report][issue], please include as much context as you +can. This includes information like Python version, cleanlab version, an error +message or stack trace, and detailed steps to reproduce the bug (if possible, including toy data that reproduces the error). The more information you can include, the better. + +## Pull Requests + +Want to write code to help improve cleanlab? Awesome! + +If there are [open issues][issues], you're more than welcome to work on those (a good place to start is those tagged "good first issue"). If you have your own ideas, that's great too! In that case, before working on substantial changes to the code base, it is *highly recommended* that you first +[open an issue][issue] describing what you intend to work on. + +To contribute your code to the library, you'll want to [fork the repo](https://docs.github.com/en/get-started/quickstart/fork-a-repo), push your changes to your fork, and finally create a new [Pull Request][pr]. + +Detailed development instructions, such as how to run the tests, are available +in [DEVELOPMENT.md](DEVELOPMENT.md). + +--- + +If you have any questions about contributing to cleanlab, feel free to +[ask][discussions]! + +[issue]: https://github.com/cleanlab/cleanlab/issues/new +[issues]: https://github.com/cleanlab/cleanlab/issues +[fork]: https://github.com/cleanlab/cleanlab/fork +[pr]: https://github.com/cleanlab/cleanlab/pulls +[discussions]: https://github.com/cleanlab/cleanlab/discussions diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 0000000..9b01e37 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,426 @@ +# Development + +This document explains how to set up a development environment for +[contributing](CONTRIBUTING.md) to cleanlab. + +## Setting up a virtual environment + +While this is not required, we recommend that you do development and testing in +a virtual environment. There are a number of tools to do this, including +[uv](https://docs.astral.sh/uv/), [virtualenv](https://virtualenv.pypa.io/), +[pipenv](https://pipenv.pypa.io/), and [venv](https://docs.python.org/3/library/venv.html). +You can [compare](https://stackoverflow.com/questions/41573587/what-is-the-difference-between-venv-pyvenv-pyenv-virtualenv-virtualenvwrappe) +the tools and choose what is right for you. + +### Using uv (fast) + +```shell +uv venv ./ENV # create a new virtual environment in the directory ENV +source ./ENV/bin/activate # switch to using the virtual environment +``` + +### Using venv (built-in) + +```shell +python3 -m venv ./ENV # create a new virtual environment in the directory ENV +source ./ENV/bin/activate # switch to using the virtual environment +``` + +You only need to create the virtual environment once, but you will need to +activate it every time you start a new shell. Once the virtual environment is +activated, the install commands below will install dependencies into the +virtual environment rather than your system Python installation. + +## Installing dependencies and cleanlab + +Run the following commands in the repository's root directory. + +### Using uv + +```shell +uv pip install -r requirements-dev.txt # install development requirements +uv pip install -e . # install cleanlab as an editable package +``` + +### Using pip + +```shell +pip install -r requirements-dev.txt # install development requirements +pip install -e . # install cleanlab as an editable package +``` + +For Macs with Apple silicon: replace `tensorflow` in requirements-dev.txt with: `tensorflow-macos==2.9.2` and `tensorflow-metal==0.5.1` + +### Handling optional dependencies + +When designing a class that relies on an optional, domain-specific runtime dependency, it is better to use lazy-importing to avoid forcing users to install the dependency if they do not need it. + +Depending on the coupling of your class to the dependency, you may want to consider importing it at the module-level or as an instance variable of the class or a function that uses the dependency. + +If the dependency is used by many methods in the module or other classes, it is better to import it at the module-level. +On the other hand, if the dependency is only used by a handful of methods, then it's better to import it inside the method. If the dependency is not installed, an ImportError should be raised when the method is called, along with instructions on how to install the dependency. + +Here is an example of a class that lazily imports CuPy and has a sum method (element-wise) that can be used on both CPU and GPU devices. + +Unless an alternative implementations of the sum method is available, an `ImportError` should be raised when the method is called with instructions on how to install the dependency. + +
Example code + +```python +def lazy_import_cupy(): + try: + import cupy + except ImportError as error: + # If the dependency is required for the class to work, + # replace this block with a raised ImportError containing instructions + print("Warning: cupy is not installed. Please install it with `pip install cupy`.") + cupy = None + return cupy + +class Summation: + def __init__(self): + self.cupy = lazy_import_cupy() + def sum(self, x) -> float: + if self.cupy is None: + return sum(x) + return self.cupy.sum(x) +``` +
+ + +For the build system to recognize the optional dependency, you should add it to the `EXTRAS_REQUIRE` constant in **setup.py**: + +
Example code + +```python +EXTRAS_REQUIRE = { + ... + "gpu": [ + # Explain why the dependency below is needed, + # e.g. "for performing summation on GPU" + "cupy", + ], +} +``` + + +Or assign to a separate variable and add it to `EXTRAS_REQUIRE` + +```python +GPU_REQUIRES = [ + # Explanation ... + "cupy", +] + +EXTAS_REQUIRE = { + ... + "gpu": GPU_REQUIRES, +} +``` +
+ + +The package can be installed with the optional dependency (here called `gpu`) via: + +### PyPI installation + +```shell +uv pip install "cleanlab[gpu]" +``` + +### Editable installation + +```shell +uv pip install -e ".[gpu]" +``` + +## Testing + + +**Download test data** +The test data for cleanlab resides in the assets repository. Use the following commands to download test data before running tests. +```shell +git clone https://github.com/cleanlab/assets.git +mv assets/cleanlab_test_data cleanlab/tests/datalab/data +``` + +**Run all the tests:** + +```shell +pytest +``` + +**Run a specific file or test:** + +```shell +pytest -k +``` + +**Run with verbose output:** + +```shell +pytest --verbose +``` + +**Run with code coverage:** + +```shell +pytest --cov=cleanlab/ --cov-config .coveragerc --cov-report=html +``` + +The coverage report will be available in `coverage_html_report/index.html`, +which you can open with your web browser. + +### Type checking + +Cleanlab uses [mypy](https://mypy.readthedocs.io/en/stable/) typing. Type checking happens automatically during CI but can be run locally. + +**Check typing in all files:** + +```shell +mypy cleanlab +``` + +The above is just a simplified command for demonstration, do NOT run this for testing your own type annotations! +Our CI adds a few additional flags to the `mypy` command it uses in the file: +**.github/workflows/ci.yml**. +To exactly match the `mypy` command that is executed in CI, copy these flags, and also ensure your version of `mypy` and related packages like `pandas-stubs` match the latest released versions (used in our CI). + +### Examples + +You can check that the [examples](https://github.com/cleanlab/examples) still +work with changes you make to cleanlab by manually running the notebooks. +You can also run all example notebooks as follows: + +```shell +git clone https://github.com/cleanlab/examples.git +``` + +Then specify your local version of cleanlab source in the first line of: **examples/requirements.txt**. +E.g. you can edit this line to point to your local version of cleanlab as a relative path such as `../cleanlab` if the `cleanlab` and `examples` repos are sibling directories on your computer. + +Finally execute the bash script: + +```shell +examples/run_all_notebooks.sh +``` + + +## How to style new code contributions + +cleanlab follows the [Black](https://black.readthedocs.io/) code style (see [pyproject.toml](pyproject.toml)). This is +enforced by CI, so please format your code by invoking `black` before submitting a pull request. + +Generally aim to follow the [PEP-8 coding style](https://peps.python.org/pep-0008/). +Please do not use wildcard `import *` in any files, instead you should always import the specific functions that you need from a module. + +All cleanlab code should have a maximum line length of 100 characters. + +### Pre-commit hook + +This repo uses the [pre-commit framework](https://pre-commit.com/) to easily +set up code style checks that run automatically whenever you make a commit. +You can install the git hook scripts with: + +```shell +pre-commit install +``` + +### EditorConfig + +This repo uses [EditorConfig](https://editorconfig.org/) to keep code style +consistent across editors and IDEs. You can install a plugin for your editor, +and then your editor will automatically ensure that indentation and line +endings match the project style. + + +## Adding new modules into the source code + + You should go through the following checklist if you intend to add new functionality to the package in a separate module. +- [x] Add brief description of the module’s purpose in a comment at the top of file and docstrings for every function. +- [x] Import the module `my_module.py` into main [``__init__.py``](cleanlab/__init__.py) +- [x] Create detailed unit tests (typically in a new file `tests/test_my_module.py`) +- [x] Add new module to docs index pages (**docs/source/index.rst**) and create .rst file in **docs/source/cleanlab/** (so that module appears on [docs.cleanlab.ai](https://docs.cleanlab.ai/stable/index.html) -- please verify its documentation also looks good there) +- [x] Create a QuickStart (**docs/source/tutorials**) notebook that runs main module functionality in 5min or less and add it to index pages (**docs/source/tutorials/index.rst**, **docs/source/index.rst**). Clear cell output before pushing. +- [x] Create an [examples](https://github.com/cleanlab/examples) notebook that runs more advanced module functionality with a more real-world application (can have a longer run time). Push with printed cell output. + +## Contributing new issue types to Datalab + +To contribute a new type of issue that Datalab can automatically detect in any dataset, refer to our guide on [Creating Your Own Issues Manager](https://docs.cleanlab.ai/master/cleanlab/datalab/guide/custom_issue_manager.html). + +Do not add your new issue type to the set of issues that Datalab detects by default, our team can add it to this default set later on once it's utility has been thoroughly validated. + +Don't forget to update the [issue type descriptions guide](https://github.com/cleanlab/cleanlab/blob/master/docs/source/cleanlab/datalab/guide/issue_type_description.rst) with a brief description of your new issue type. +It is ideal to stick to a format that maintains consistency and readability. +Generally, the format includes a title, explanation of the issue, required arguments, then any additional information. +It would be helpful to include a tip for users on how to detect the issue using Datalab. + +Try to add tests for this new issue type. It's a good idea to start with some tests in a separate module in the [issue manager test directory](https://github.com/cleanlab/cleanlab/tree/master/tests/datalab/issue_manager). + + +## Documentation + +You can build the docs from your local cleanlab version by following [these +instructions](./docs/README.md#build-the-cleanlab-docs-locally). + +If editing existing docs or adding new tutorials, please first read through our [guidelines](./docs/README.md#tips-for-editing-docstutorials). + +## Documentation style + +cleanlab uses [NumPy +style](https://numpydoc.readthedocs.io/en/latest/format.html#docstring-standard) +docstrings +([example](https://www.sphinx-doc.org/en/master/usage/extensions/example_numpy.html)). + +Aspects that are not covered in the NumPy style or that are different from the +NumPy style are documented below: + +- **Referring to the cleanlab package**: we refer to cleanlab without any + special formatting, so no `cleanlab`, just cleanlab. +- **Cross-referencing**: when mentioning functions/classes/methods, always + [cross-reference](https://www.sphinx-doc.org/en/master/usage/restructuredtext/domains.html#cross-referencing-python-objects) + them to create a clickable link. Cross-referencing code from Jupyter + notebooks is not currently supported. +- **Variable, module, function, and class names**: when not cross-references, + should be written between single back-ticks, like `` `pred_probs` ``. Such + names in Jupyter notebooks (Markdown) can be written between single + back-ticks as well. +- **Math**: We support [LaTeX + math](https://sphinxcontrib-katex.readthedocs.io/en/v0.8.6/examples.html) + with the inline `` :math:`x+y` `` or the block: + + ``` + .. math:: + + \sum_{0}^{n} 2n+1 + ``` +- **Pseudocode vs math**: Prefer pseudocode in double backticks over LaTeX math. +- **Bold vs italics**: Use italics when defining a term, and use bold sparingly + for extra emphasis. +- **Shapes**: Do not include shapes in the type of parameters, instead use + `np.array` or `array_like` as the type and specify allowed shapes in the + description. See, for example, the documentation for + `cleanlab.classification.CleanLearning.fit()`. Format for 1D shape: `(N,1)` +- **Optional arguments**: for the most part, just put `, optional` in the type. +- **Type unions**: if a parameter or return type is something like "a numpy + array or None", you can use "or" to separate types, e.g. `np.array or None`, + and it'll be parsed correctly. +- **Parameterized types**: Use [standard Python type + hints](https://docs.python.org/3/library/typing.html) for referring to + parameters and parameterized types in docs, e.g. `Iterable[int]` or + `list[float]`. + +### Common variable names / terminology used throughout codebase + +- `N` - the number of examples/datapoints in a dataset. + - `num_examples` may also be used when additional clarity is needed. +- `K` - the number of classes (unique labels) for a dataset. + - `num_classes` may also be used when additional clarity is needed. +- `labels` - a label for each example, length should be N (sample-size of dataset) +- `classes` - set of possible labels for any one example, length should be K (number of possible categories in classification problem) + +Try to adhere to this standardized terminology unless you have good reason not to! + +### Relative Link Formatting Instructions + +Use relative linking to connect information between docs and jupyter notebooks, and make sure links will remain valid in the future as new cleanlab versions are released! Sphinx/html works with relative paths so try to specify relative paths if necessary. For specific situations: + +- Link another function or class from within a source code docstring: + - If you just want to specify the function/class name (ie. the function/class is unique throughout our library): `` `~cleanlab.file.function_or_class_name` ``. + + This uses the [Sphinx's](https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-default_role) `default_role = "py:obj"` setting, so the leading tilde shortens the link to only display `function_or_class_name`. + - If you want to additionally specify the module which the function belongs to: + - `` :py:func:`file.function_name ` `` for functions + - ``:py:class:`file.class_name ` `` for classes + + Here you have more control over the text that is displayed to display the module name. When referring to a function that is alternatively defined in other modules as well, always use this option to be more explicit about which module you are referencing. +- Link a tutorial (rst file) from within a source code docstring or rst file: ``:ref:`tutorial_name ` `` +- Link a tutorial notebook (ipynb file) from within a source code docstring or rst file: `` `notebook_name `_ `` . (If the notebook is not the in the same folder as the source code, use a relative path) +- Link a function from within a tutorial notebook: `[function_name](../cleanlab/file.html#cleanlab.file.function_name)` + + Links from master branch tutorials will reference master branch functions, similarly links from tutorials in stable branch will reference stable branch functions since we are using relative paths. +- Link a specific section of a notebook from within the notebook: `[section title](#section-title)` +- Link a different tutorial notebook from within a tutorial notebook: `[another notebook](another_notebook.html)`. (Note this only works when the other notebook is in same folder as this notebook, otherwise may need to try relative path) +- Link another specific section of different notebook from within a tutorial notebook: `[another notebook section title](another_notebook.html#another-notebook-section-title)` +- Linking examples notebooks from inside tutorial notebooks can be simply done by linking global url of the example notebook in master branch of github.com/cleanlab/examples/ + +## Packaging and releasing + +The release process is automated using GitHub Actions. When a release is published on the main [cleanlab](https://github.com/cleanlab/cleanlab) repository, the following workflows are triggered: + +1. Docs are built and pushed to the `cleanlab-docs` depository within the same organization, which handles the deployment to [docs.cleanlab.ai](https://docs.cleanlab.ai/stable/index.html). +2. A new release is created on PyPI with the same version number as the release on GitHub. + +There are other workflows that need to be handled manually in other repositories, but that is outside the score of this section. +This section will focus on the PyPI release process. + +### Developing the PyPI release process + +It's important to test the release process on a separate PyPI project before releasing to the main [cleanlab project](https://pypi.org/project/cleanlab/). For the remainder of this section, we will refer to the test project as `test-cleanlab-`, where `` is your GitHub username. This name should be unique to avoid conflicts with other users' test projects. + +#### Prerequisites + +##### PyPI Prerequisites + +- Create separate user accounts on [PyPI](https://pypi.org/) and [Test PyPI](https://test.pypi.org/). + - [Register here on PyPI](https://pypi.org/account/register/). + - [Register here on Test PyPI](https://test.pypi.org/account/register/). + - Ideally, these accounts should have the same username, but this is not strictly necessary. + +- Add a "[Trusted Publisher](https://docs.pypi.org/trusted-publishers/)" on both PyPI accounts (i.e. Publishing with OpenID Connect). + - This will allow you to publish packages to PyPI and Test PyPI using GitHub Actions, without needing to store your PyPI credentials in the repository. + - Walk through the steps in ["Creating a PyPI project with a trusted publisher"](https://docs.pypi.org/trusted-publishers/creating-a-project-through-oidc/) for both PyPI and Test PyPI. + - The PyPI Project name is: `test-cleanlab-`. + - The owner is: `` (your GitHub username). + - The repository name is: `cleanlab` (your fork of the cleanlab repository). + - The [workflow file](.github/workflows/release-build-publish.yml) name is: `release-build-publish.yml` + - ATTENTION: The environment name should be left empty in the Test PyPI project, and set to `pypi` in the PyPI project. + - See discussion on the environment in the [GitHub Prerequisites](#github-prerequisites) section. + +##### GitHub Prerequisites + +- [Fork the cleanlab repository](https://github.com/cleanlab/cleanlab/fork) to your GitHub account. + - This will allow you to test the release process on your fork, on a separate PyPI project. + +- On your fork of the cleanlab repository, create two environments called `testpypi` and `pypi` in the "Environments" tab, under the repository "Settings". + - For the `testpypi` environment, add a wait timer of 1 minute to allow TestPyPI uploads to complete processing before testing installation. + - For the `pypi` environment, add a protection rule for requiring a review from a maintainer. For extra security, you may disallow a self-review so that a second maintainer must approve the release. + - You may wish to limit which tags can trigger a release in these environment, to avoid accidental releases. + - Github Docs provides instructions on [how to create a new environment](https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#creating-an-environment) with these considerations. + + +##### Local Prerequisites + +- In `pyproject.toml` replace the `project.name` configuration with the value `test-cleanlab-`, where `` is your GitHub username. + +#### Testing the release process + +1. Push a commit with an updated version number in `cleanlab/version.py`. + - Ideally, this should be a patch version bump, e.g. `0.1.0` to `0.1.1` or a minor version bump, e.g. `0.1.X` to `0.2.0`. + - This is typically done via a standalone PR to the cleanlab/cleanlab repository. + + ```diff + # Bump the version number in cleanlab/version.py + - __version__ = "2.6.0" + + __version__ = "2.6.1" + ``` + +2. On the repository's GitHub page, navigate to the "Releases" page and click "Draft a new release". + - Choose a tag version that matches the version number in `cleanlab/version.py`, it should follow the format `vX.Y.Z`, e.g. `v2.6.1` or `v2.7.0`. + - This kind of format will be automatically checked by the release workflow. + - This tag may not exist yet, but Github allows you to create it upon publishing the release. + - Target the `master` branch. + - Select the previous tag to compare against, if it exists. Usually this is the previous release tag, e.g. the previous patch version. + - GitHub should allow you to generate release notes based on this information. + - When you've finalized the release notes and are ready to publish the release, click "Publish release". + - This will kick off the release workflow, which will build and publish the package to Test PyPI, test the package installation, and then publish the package to PyPI. + +3. Open up the "Actions" tab on your fork of the cleanlab repository and monitor the progress of the release workflow. + - A "Release Build Publish" workflow should be triggered by the release, and you can monitor its progress there. + - It will check for the project name (for uploading to the proper PyPI project), and validate the version name/tag. + - When these steps pass, it will build the distribution and check the contents. + - Passing the build step, it will upload the distribution to Test PyPI. + - After the upload, it will kick off several test jobs to install the package from Test PyPI and run various tests. + - Adding more kinds of tests at this stage in the workflow is a good idea, to ensure the package is working as expected. Just create a new job that `needs` the `verify-version` job, and runs the tests you want to add. + - After all the tests pass, it will trigger the final job. However, the environment should be configure to require a review from a maintainer before the final job can be run. + - View the deployment and approve it it everything looks good so far. This will trigger the final job to publish the package to PyPI. \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..7248b11 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +include LICENSE README.md CONTRIBUTING.md DEVELOPMENT.md CODE_OF_CONDUCT.md + +# Exclude development and documentation files from distribution +exclude requirements-dev.txt +exclude requirements-test-core.txt +recursive-exclude docs * diff --git a/README.md b/README.md new file mode 100644 index 0000000..798062f --- /dev/null +++ b/README.md @@ -0,0 +1,286 @@ +
+ +
+ +
+pypi_versions +py_versions +coverage +Github Stars +Twitter +
+ +

+

+ Documentation | + Examples | + Blog | + Research +

+

+ +Cleanlab’s open-source library helps you **clean** data and **lab**els by automatically detecting issues in a ML dataset. To facilitate **machine learning with messy, real-world data**, this data-centric AI package uses your *existing* models to estimate dataset problems that can be fixed to train even *better* models. + + +

+ +

+

+ Examples of various issues in Cat/Dog dataset automatically detected by cleanlab via this code: +

+ +```python + lab = cleanlab.Datalab(data=dataset, label="column_name_for_labels") + # Fit any ML model, get its feature_embeddings & pred_probs for your data + lab.find_issues(features=feature_embeddings, pred_probs=pred_probs) + lab.report() +``` + +- Use cleanlab to automatically check every: [text](https://docs.cleanlab.ai/stable/tutorials/datalab/text.html), [audio](https://docs.cleanlab.ai/stable/tutorials/datalab/audio.html), [image](https://docs.cleanlab.ai/stable/tutorials/datalab/image.html), or [tabular](https://docs.cleanlab.ai/stable/tutorials/datalab/tabular.html) dataset. +- Use cleanlab to automatically: [detect data issues (outliers, duplicates, label errors, etc)](https://docs.cleanlab.ai/stable/tutorials/datalab/datalab_quickstart.html), [train robust models](https://docs.cleanlab.ai/stable/tutorials/indepth_overview.html), [infer consensus + annotator-quality for multi-annotator data](https://docs.cleanlab.ai/stable/tutorials/multiannotator.html), [suggest data to (re)label next (active learning)](https://github.com/cleanlab/examples/blob/master/active_learning_multiannotator/active_learning.ipynb). + + +--- + + +## Run cleanlab open-source + +This cleanlab package runs on Python 3.10+ and supports Linux, macOS, as well as Windows. + +- Get started [here](https://docs.cleanlab.ai/)! Install via `uv`, `pip`, or `conda`. +- Developers who install the bleeding-edge from source should refer to [this master branch documentation](https://docs.cleanlab.ai/master/index.html). + +**Practicing data-centric AI can look like this:** +1. Train initial ML model on original dataset. +2. Utilize this model to diagnose data issues (via cleanlab methods) and improve the dataset. +3. Train the same model on the improved dataset. +4. Try various modeling techniques to further improve performance. + +Most folks jump from Step 1 → 4, but you may achieve big gains without *any* change to your modeling code by using cleanlab! +Continuously boost performance by iterating Steps 2 → 4 (and try to evaluate with *cleaned* data). + +![](https://raw.githubusercontent.com/cleanlab/assets/master/cleanlab/flowchart.png) + + +## Use cleanlab with any model and in most ML tasks + +All features of cleanlab work with **any dataset** and **any model**. Yes, any model: PyTorch, Tensorflow, Keras, JAX, HuggingFace, OpenAI, XGBoost, scikit-learn, etc. + +cleanlab is useful across a wide variety of Machine Learning tasks. Specific tasks this data-centric AI package offers dedicated functionality for include: +1. [Binary and multi-class classification](https://docs.cleanlab.ai/stable/tutorials/indepth_overview.html) +2. [Multi-label classification](https://docs.cleanlab.ai/stable/tutorials/multilabel_classification.html) (e.g. image/document tagging) +3. [Token classification](https://docs.cleanlab.ai/stable/tutorials/token_classification.html) (e.g. entity recognition in text) +4. [Regression](https://docs.cleanlab.ai/stable/tutorials/regression.html) (predicting numerical column in a dataset) +5. [Image segmentation](https://docs.cleanlab.ai/stable/tutorials/segmentation.html) (images with per-pixel annotations) +6. [Object detection](https://docs.cleanlab.ai/stable/tutorials/object_detection.html) (images with bounding box annotations) +7. [Classification with data labeled by multiple annotators](https://docs.cleanlab.ai/stable/tutorials/multiannotator.html) +8. [Active learning with multiple annotators](https://github.com/cleanlab/examples/blob/master/active_learning_multiannotator/active_learning.ipynb) (suggest which data to label or re-label to improve model most) +9. [Outlier detection](https://docs.cleanlab.ai/stable/tutorials/outliers.html) (identify atypical data that appears out of distribution) + +For other ML tasks, cleanlab can still help you improve your dataset if appropriately applied. +See our [Example Notebooks](https://github.com/cleanlab/examples) and [Blog](https://cleanlab.ai/blog/learn/). + + +## So fresh, so cleanlab + +Beyond automatically catching [all sorts of issues](https://docs.cleanlab.ai/stable/cleanlab/datalab/guide/issue_type_description.html) lurking in your data, this data-centric AI package helps you deal with **noisy labels** and train more **robust ML models**. +Here's an example: + +```python + +# cleanlab works with **any classifier**. Yup, you can use PyTorch/TensorFlow/OpenAI/XGBoost/etc. +cl = cleanlab.classification.CleanLearning(sklearn.YourFavoriteClassifier()) + +# cleanlab finds data and label issues in **any dataset**... in ONE line of code! +label_issues = cl.find_label_issues(data, labels) + +# cleanlab trains a robust version of your model that works more reliably with noisy data. +cl.fit(data, labels) + +# cleanlab estimates the predictions you would have gotten if you had trained with *no* label issues. +cl.predict(test_data) + +# A universal data-centric AI tool, cleanlab quantifies class-level issues and overall data quality, for any dataset. +cleanlab.dataset.health_summary(labels, confident_joint=cl.confident_joint) +``` + +cleanlab **clean**s your data's **lab**els via state-of-the-art *confident learning* algorithms, published in this [paper](https://jair.org/index.php/jair/article/view/12125) and [blog](https://l7.curtisnorthcutt.com/confident-learning). See some of the datasets cleaned with cleanlab at [labelerrors.com](https://labelerrors.com). + +cleanlab is: + +1. **backed by theory** -- with [provable guarantees](https://arxiv.org/abs/1911.00068) of exact label noise estimation, even with imperfect models. +2. **fast** -- code is parallelized and scalable. +4. **easy to use** -- one line of code to find mislabeled data, bad annotators, outliers, or train noise-robust models. +6. **general** -- works with **[any dataset](https://labelerrors.com/)** (text, image, tabular, audio,...) + **any model** (PyTorch, OpenAI, XGBoost,...) +
+ +![](https://raw.githubusercontent.com/cleanlab/assets/master/cleanlab/label-errors-examples.png) +

+Examples of incorrect given labels in various image datasets found and corrected using cleanlab. +While these examples are from image datasets, this also works for text, audio, tabular data. +

+ + +## Citation and related publications + +cleanlab is based on peer-reviewed research. Here are relevant papers to cite if you use this package: + +
Confident Learning (JAIR '21) (click to show bibtex) + + @article{northcutt2021confidentlearning, + title={Confident Learning: Estimating Uncertainty in Dataset Labels}, + author={Curtis G. Northcutt and Lu Jiang and Isaac L. Chuang}, + journal={Journal of Artificial Intelligence Research (JAIR)}, + volume={70}, + pages={1373--1411}, + year={2021} + } + +
+ +
Rank Pruning (UAI '17) (click to show bibtex) + + @inproceedings{northcutt2017rankpruning, + author={Northcutt, Curtis G. and Wu, Tailin and Chuang, Isaac L.}, + title={Learning with Confident Examples: Rank Pruning for Robust Classification with Noisy Labels}, + booktitle = {Proceedings of the Thirty-Third Conference on Uncertainty in Artificial Intelligence}, + series = {UAI'17}, + year = {2017}, + location = {Sydney, Australia}, + numpages = {10}, + url = {http://auai.org/uai2017/proceedings/papers/35.pdf}, + publisher = {AUAI Press}, + } + +
+ +
Label Quality Scoring (ICML '22) (click to show bibtex) + + @inproceedings{kuan2022labelquality, + title={Model-agnostic label quality scoring to detect real-world label errors}, + author={Kuan, Johnson and Mueller, Jonas}, + booktitle={ICML DataPerf Workshop}, + year={2022} + } + +
+ +
Label Errors in Token Classification / Entity Recognition (NeurIPS '22) (click to show bibtex) + + @inproceedings{wang2022tokenerrors, + title={Detecting label errors in token classification data}, + author={Wang, Wei-Chen and Mueller, Jonas}, + booktitle={NeurIPS Workshop on Interactive Learning for Natural Language Processing (InterNLP)}, + year={2022} + } + +
+ +
Label Errors in Multi-Label Classification (ICLR '23) (click to show bibtex) + + @inproceedings{thyagarajan2023multilabel, + title={Identifying Incorrect Annotations in Multi-Label Classification Data}, + author={Thyagarajan, Aditya and Snorrason, Elías and Northcutt, Curtis and Mueller, Jonas}, + booktitle={ICLR Workshop on Trustworthy ML}, + year={2023} + } + +
+ +
Label Errors in Object Detection (ICML '23) (click to show bibtex) + + @inproceedings{tkachenko2023objectlab, + title={ObjectLab: Automated Diagnosis of Mislabeled Images in Object Detection Data}, + author={Tkachenko, Ulyana and Thyagarajan, Aditya and Mueller, Jonas}, + booktitle={ICML Workshop on Data-centric Machine Learning Research}, + year={2023} + } + +
+ +
Label Errors in Image Segmentation (ICML '23) (click to show bibtex) + + @inproceedings{lad2023segmentation, + title={Estimating label quality and errors in semantic segmentation data via any model}, + author={Lad, Vedang and Mueller, Jonas}, + booktitle={ICML Workshop on Data-centric Machine Learning Research}, + year={2023} + } + +
+ +
Detecting Errors in Numerical Data (DMLR '24) (click to show bibtex) + + @inproceedings{zhou2023errors, + title={Detecting Errors in a Numerical Response via any Regression Model}, + author={Zhou, Hang and Mueller, Jonas and Kumar, Mayank and Wang, Jane-Ling and Lei, Jing}, + booktitle={Journal of Data-centric Machine Learning Research}, + year={2024} + } + +
+ +
Out-of-Distribution Detection (ICML '22) (click to show bibtex) + + @inproceedings{kuan2022ood, + title={Back to the Basics: Revisiting Out-of-Distribution Detection Baselines}, + author={Kuan, Johnson and Mueller, Jonas}, + booktitle={ICML Workshop on Principles of Distribution Shift}, + year={2022} + } + +
+ +
CROWDLAB for Data with Multiple Annotators (NeurIPS '22) (click to show bibtex) + + @inproceedings{goh2022crowdlab, + title={CROWDLAB: Supervised learning to infer consensus labels and quality scores for data with multiple annotators}, + author={Goh, Hui Wen and Tkachenko, Ulyana and Mueller, Jonas}, + booktitle={NeurIPS Human in the Loop Learning Workshop}, + year={2022} + } + +
+ +
ActiveLab: Active learning with data re-labeling (ICLR '23) (click to show bibtex) + + @inproceedings{goh2023activelab, + title={ActiveLab: Active Learning with Re-Labeling by Multiple Annotators}, + author={Goh, Hui Wen and Mueller, Jonas}, + booktitle={ICLR Workshop on Trustworthy ML}, + year={2023} + } + +
+ +
Detecting Dataset Drift and Non-IID Sampling (ICML '23) (click to show bibtex) + + @inproceedings{cummings2023drift, + title={Detecting Dataset Drift and Non-IID Sampling via k-Nearest Neighbors}, + author={Cummings, Jesse and Snorrason, Elías and Mueller, Jonas}, + booktitle={ICML Workshop on Data-centric Machine Learning Research}, + year={2023} + } + +
+ +To understand/cite other cleanlab functionality not described above, check out our [Blog](https://cleanlab.ai/blog/learn). + + +## Other resources + +- [Example Notebooks demonstrating practical applications of this package](https://github.com/cleanlab/examples) + +- [Cleanlab Blog](https://cleanlab.ai/blog/learn) + +- [Blog post: Introduction to Confident Learning](https://l7.curtisnorthcutt.com/confident-learning) + +- [NeurIPS 2021 paper: Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks](https://arxiv.org/abs/2103.14749) + +- [Introduction to Data-centric AI (MIT IAP Course)](https://dcai.csail.mit.edu/) + +- [Release notes for past versions](https://github.com/cleanlab/cleanlab/releases) + +**Interested in contributing?** See the [contributing guide](CONTRIBUTING.md), [development guide](DEVELOPMENT.md), and [ideas on useful contributions](https://github.com/cleanlab/cleanlab/wiki#ideas-for-contributing-to-cleanlab). + +**Have questions?** Check out [our FAQ](https://docs.cleanlab.ai/stable/tutorials/faq.html) and [Github Issues](https://github.com/cleanlab/cleanlab/issues?q=is%3Aissue). diff --git a/README.wehub.md b/README.wehub.md new file mode 100644 index 0000000..2b2f092 --- /dev/null +++ b/README.wehub.md @@ -0,0 +1,7 @@ +# WeHub 来源说明 + +- 原始项目：`cleanlab/cleanlab` +- 原始仓库：https://github.com/cleanlab/cleanlab +- 导入方式：上游默认分支的最新快照 +- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准 +- 本文件仅用于记录来源，不代表 WeHub 是原项目作者 diff --git a/cleanlab/__init__.py b/cleanlab/__init__.py new file mode 100644 index 0000000..31e363c --- /dev/null +++ b/cleanlab/__init__.py @@ -0,0 +1,57 @@ +from .version import __version__ +from . import classification +from . import count +from . import rank +from . import filter +from . import benchmarking +from . import dataset +from . import multiannotator +from . import outlier +from . import token_classification +from . import multilabel_classification +from . import object_detection +from . import regression +from . import segmentation + + +class DatalabUnavailable: + def __init__(self, message): + self.message = message + + def __getattr__(self, name): + message = self.message + f" (raised when trying to access {name})" + raise ImportError(message) + + def __call__(self, *args, **kwargs): + message = ( + self.message + f" (raised when trying to call with args: {args}, kwargs: {kwargs})" + ) + raise ImportError(message) + + +def _datalab_import_factory(): + try: + from .datalab.datalab import Datalab as _Datalab + + return _Datalab + except ImportError: + return DatalabUnavailable( + "Datalab is not available due to missing dependencies. " + "To install Datalab, run `pip install 'cleanlab[datalab]'`." + ) + + +def _issue_manager_import_factory(): + try: + from .datalab.internal.issue_manager import IssueManager as _IssueManager + + return _IssueManager + except ImportError: + return DatalabUnavailable( + "IssueManager is not available due to missing dependencies for Datalab. " + "To install Datalab, run `pip install 'cleanlab[datalab]'`." + ) + + +Datalab = _datalab_import_factory() +IssueManager = _issue_manager_import_factory() diff --git a/cleanlab/benchmarking/__init__.py b/cleanlab/benchmarking/__init__.py new file mode 100644 index 0000000..35c0212 --- /dev/null +++ b/cleanlab/benchmarking/__init__.py @@ -0,0 +1 @@ +from . import noise_generation diff --git a/cleanlab/benchmarking/noise_generation.py b/cleanlab/benchmarking/noise_generation.py new file mode 100644 index 0000000..cfd8e98 --- /dev/null +++ b/cleanlab/benchmarking/noise_generation.py @@ -0,0 +1,486 @@ +""" +Helper methods that are useful for benchmarking cleanlab’s core algorithms. +These methods introduce synthetic noise into the labels of a classification dataset. +Specifically, this module provides methods for generating valid noise matrices (for which learning with noise is possible), +generating noisy labels given a noise matrix, generating valid noise matrices with a specific trace value, and more. +""" + +from typing import Optional + +import numpy as np +from cleanlab.internal.util import value_counts +from cleanlab.internal.constants import FLOATING_POINT_COMPARISON + + +def noise_matrix_is_valid(noise_matrix, py, *, verbose=False) -> bool: + """Given a prior `py` representing ``p(true_label=k)``, checks if the given `noise_matrix` is a + learnable matrix. Learnability means that it is possible to achieve + better than random performance, on average, for the amount of noise in + `noise_matrix`. + + Parameters + ---------- + noise_matrix : np.ndarray + An array of shape ``(K, K)`` representing the conditional probability + matrix ``P(label=k_s|true_label=k_y)`` containing the fraction of + examples in every class, labeled as every other class. Assumes columns of + `noise_matrix` sum to 1. + + py : np.ndarray + An array of shape ``(K,)`` representing the fraction (prior probability) + of each true class label, ``P(true_label = k)``. + + Returns + ------- + is_valid : bool + Whether the noise matrix is a learnable matrix. + """ + + # Number of classes + K = len(py) + + # let's assume some number of training examples for code readability, + # but it doesn't matter what we choose as it's not actually used. + N = float(10000) + + ps = np.dot(noise_matrix, py) # P(true_label=k) + + # P(label=k, true_label=k') + joint_noise = np.multiply(noise_matrix, py) # / float(N) + + # Check that joint_probs is valid probability matrix + if not (abs(joint_noise.sum() - 1.0) < FLOATING_POINT_COMPARISON): + return False + + # Check that noise_matrix is a valid matrix + # i.e. check p(label=k)*p(true_label=k) < p(label=k, true_label=k) + for i in range(K): + C = N * joint_noise[i][i] + E1 = N * joint_noise[i].sum() - C + E2 = N * joint_noise.T[i].sum() - C + O = N - E1 - E2 - C + if verbose: + print( + "E1E2/C", + round(E1 * E2 / C), + "E1", + round(E1), + "E2", + round(E2), + "C", + round(C), + "|", + round(E1 * E2 / C + E1 + E2 + C), + "|", + round(E1 * E2 / C), + "<", + round(O), + ) + print( + round(ps[i] * py[i]), + "<", + round(joint_noise[i][i]), + ":", + ps[i] * py[i] < joint_noise[i][i], + ) + + if not (ps[i] * py[i] < joint_noise[i][i]): + return False + + return True + + +def generate_noisy_labels(true_labels, noise_matrix) -> np.ndarray: + """Generates noisy `labels` from perfect labels `true_labels`, + "exactly" yielding the provided `noise_matrix` between `labels` and `true_labels`. + + Below we provide a for loop implementation of what this function does. + We do not use this implementation as it is not a fast algorithm, but + it explains as Python pseudocode what is happening in this function. + + Parameters + ---------- + true_labels : np.ndarray + An array of shape ``(N,)`` representing perfect labels, without any + noise. Contains K distinct natural number classes, 0, 1, ..., K-1. + + noise_matrix : np.ndarray + An array of shape ``(K, K)`` representing the conditional probability + matrix ``P(label=k_s|true_label=k_y)`` containing the fraction of + examples in every class, labeled as every other class. Assumes columns of + `noise_matrix` sum to 1. + + Returns + ------- + labels : np.ndarray + An array of shape ``(N,)`` of noisy labels. + + Examples + -------- + + .. code:: python + + # Generate labels + count_joint = (noise_matrix * py * len(y)).round().astype(int) + labels = np.ndarray(y) + for k_s in range(K): + for k_y in range(K): + if k_s != k_y: + idx_flip = np.where((labels==k_y)&(true_label==k_y))[0] + if len(idx_flip): # pragma: no cover + labels[np.random.choice( + idx_flip, + count_joint[k_s][k_y], + replace=False, + )] = k_s + """ + + # Make y a numpy array, if it is not + true_labels = np.asarray(true_labels) + + # Number of classes + K = len(noise_matrix) + + # Compute p(true_label=k) + py = value_counts(true_labels) / float(len(true_labels)) + + # Counts of pairs (labels, y) + count_joint = (noise_matrix * py * len(true_labels)).astype(int) + # Remove diagonal entries as they do not involve flipping of labels. + np.fill_diagonal(count_joint, 0) + + # Generate labels + labels = np.array(true_labels) + for k in range(K): # Iterate over true_label == k + # Get the noisy labels that have non-zero counts + labels_per_class = np.where(count_joint[:, k] != 0)[0] + # Find out how many of each noisy label we need to flip to + label_counts = count_joint[labels_per_class, k] + # Create a list of the new noisy labels + noise = [labels_per_class[i] for i, c in enumerate(label_counts) for z in range(c)] + # Randomly choose y labels for class k and set them to the noisy labels. + idx_flip = np.where((labels == k) & (true_labels == k))[0] + if len(idx_flip) and len(noise) and len(idx_flip) >= len(noise): # pragma: no cover + labels[np.random.choice(idx_flip, len(noise), replace=False)] = noise + + # Validate that labels indeed produces the correct noise_matrix (or close to it) + # Compute the actual noise matrix induced by labels + # counts = confusion_matrix(labels, true_labels).astype(float) + # new_noise_matrix = counts / counts.sum(axis=0) + # assert(np.linalg.norm(noise_matrix - new_noise_matrix) <= 2) + + return labels + + +def generate_noise_matrix_from_trace( + K, + trace, + *, + max_trace_prob=1.0, + min_trace_prob=1e-5, + max_noise_rate=1 - 1e-5, + min_noise_rate=0.0, + valid_noise_matrix=True, + py=None, + frac_zero_noise_rates=0.0, + seed=0, + max_iter=10000, +) -> Optional[np.ndarray]: + """Generates a ``K x K`` noise matrix ``P(label=k_s|true_label=k_y)`` with + ``np.sum(np.diagonal(noise_matrix))`` equal to the given `trace`. + + Parameters + ---------- + K : int + Creates a noise matrix of shape ``(K, K)``. Implies there are + K classes for learning with noisy labels. + + trace : float + Sum of diagonal entries of array of random probabilities returned. + + max_trace_prob : float + Maximum probability of any entry in the trace of the return matrix. + + min_trace_prob : float + Minimum probability of any entry in the trace of the return matrix. + + max_noise_rate : float + Maximum noise_rate (non-diagonal entry) in the returned np.ndarray. + + min_noise_rate : float + Minimum noise_rate (non-diagonal entry) in the returned np.ndarray. + + valid_noise_matrix : bool, default=True + If ``True``, returns a matrix having all necessary conditions for + learning with noisy labels. In particular, ``p(true_label=k)p(label=k) < p(true_label=k,label=k)`` + is satisfied. This requires that ``trace > 1``. + + py : np.ndarray + An array of shape ``(K,)`` representing the fraction (prior probability) of each true class label, ``P(true_label = k)``. + This argument is **required** when ``valid_noise_matrix=True``. + + frac_zero_noise_rates : float + The fraction of the ``n*(n-1)`` noise rates + that will be set to 0. Note that if you set a high trace, it may be + impossible to also have a low fraction of zero noise rates without + forcing all non-1 diagonal values. Instead, when this happens we only + guarantee to produce a noise matrix with `frac_zero_noise_rates` *or + higher*. The opposite occurs with a small trace. + + seed : int + Seeds the random number generator for numpy. + + max_iter : int, default=10000 + The max number of tries to produce a valid matrix before returning ``None``. + + Returns + ------- + noise_matrix : np.ndarray or None + An array of shape ``(K, K)`` representing the noise matrix ``P(label=k_s|true_label=k_y)`` with `trace` + equal to ``np.sum(np.diagonal(noise_matrix))``. This a conditional probability matrix and a + left stochastic matrix. Returns ``None`` if `max_iter` is exceeded. + """ + + if valid_noise_matrix and trace <= 1: + raise ValueError( + "trace = {}. trace > 1 is necessary for a".format(trace) + + " valid noise matrix to be returned (valid_noise_matrix == True)" + ) + + if valid_noise_matrix and py is None and K > 2: + raise ValueError( + "py must be provided (not None) if the input parameter" + " valid_noise_matrix == True" + ) + + if K <= 1: + raise ValueError("K must be >= 2, but K = {}.".format(K)) + + if max_iter < 1: + return None + + np.random.seed(seed) + + # Special (highly constrained) case with faster solution. + # Every 2 x 2 noise matrix with trace > 1 is valid because p(y) is not used + if K == 2: + if frac_zero_noise_rates >= 0.5: # Include a single zero noise rate + noise_mat = np.array( + [ + [1.0, 1 - (trace - 1.0)], + [0.0, trace - 1.0], + ] + ) + return noise_mat if np.random.rand() > 0.5 else np.rot90(noise_mat, k=2) + else: # No zero noise rates + diag = generate_n_rand_probabilities_that_sum_to_m(2, trace) + noise_matrix = np.array( + [ + [diag[0], 1 - diag[1]], + [1 - diag[0], diag[1]], + ] + ) + return noise_matrix + + # K > 2 + for z in range(max_iter): + noise_matrix = np.zeros(shape=(K, K)) + + # Randomly generate noise_matrix diagonal. + nm_diagonal = generate_n_rand_probabilities_that_sum_to_m( + n=K, + m=trace, + max_prob=max_trace_prob, + min_prob=min_trace_prob, + ) + np.fill_diagonal(noise_matrix, nm_diagonal) + + # Randomly distribute number of zero-noise-rates across columns + num_col_with_noise = K - np.count_nonzero(1 == nm_diagonal) + num_zero_noise_rates = int(K * (K - 1) * frac_zero_noise_rates) + # Remove zeros already in [1,0,..,0] columns + num_zero_noise_rates -= (K - num_col_with_noise) * (K - 1) + num_zero_noise_rates = np.maximum(num_zero_noise_rates, 0) # Prevent negative + num_zero_noise_rates_per_col = ( + randomly_distribute_N_balls_into_K_bins( + N=num_zero_noise_rates, + K=num_col_with_noise, + max_balls_per_bin=K - 2, + # 2 = one for diagonal, and one to sum to 1 + min_balls_per_bin=0, + ) + if K > 2 + else np.array([0, 0]) + ) # Special case when K == 2 + stack_nonzero_noise_rates_per_col = list(K - 1 - num_zero_noise_rates_per_col)[::-1] + # Randomly generate noise rates for columns with noise. + for col in np.arange(K)[nm_diagonal != 1]: + num_noise = stack_nonzero_noise_rates_per_col.pop() + # Generate num_noise noise_rates for the given column. + noise_rates_col = list( + generate_n_rand_probabilities_that_sum_to_m( + n=num_noise, + m=1 - nm_diagonal[col], + max_prob=max_noise_rate, + min_prob=min_noise_rate, + ) + ) + # Randomly select which rows of the noisy column to assign the + # random noise rates + rows = np.random.choice( + [row for row in range(K) if row != col], num_noise, replace=False + ) + for row in rows: + noise_matrix[row][col] = noise_rates_col.pop() + if not valid_noise_matrix or noise_matrix_is_valid(noise_matrix, py): + return noise_matrix + + return None + + +def generate_n_rand_probabilities_that_sum_to_m( + n, + m, + *, + max_prob=1.0, + min_prob=0.0, +) -> np.ndarray: + """ + Generates `n` random probabilities that sum to `m`. + + When ``min_prob=0`` and ``max_prob = 1.0``, use + ``np.random.dirichlet(np.ones(n))*m`` instead. + + Parameters + ---------- + n : int + Length of array of random probabilities to be returned. + + m : float + Sum of array of random probabilities that is returned. + + max_prob : float, default=1.0 + Maximum probability of any entry in the returned array. Must be between 0 and 1. + + min_prob : float, default=0.0 + Minimum probability of any entry in the returned array. Must be between 0 and 1. + + Returns + ------- + probabilities : np.ndarray + An array of probabilities. + """ + + if n == 0: + return np.array([]) + if (max_prob + FLOATING_POINT_COMPARISON) < m / float(n): + raise ValueError( + "max_prob must be greater or equal to m / n, but " + + "max_prob = " + + str(max_prob) + + ", m = " + + str(m) + + ", n = " + + str(n) + + ", m / n = " + + str(m / float(n)) + ) + if min_prob > (m + FLOATING_POINT_COMPARISON) / float(n): + raise ValueError( + "min_prob must be less or equal to m / n, but " + + "max_prob = " + + str(max_prob) + + ", m = " + + str(m) + + ", n = " + + str(n) + + ", m / n = " + + str(m / float(n)) + ) + + # When max_prob = 1, min_prob = 0, the next two lines are equivalent to: + # intermediate = np.sort(np.append(np.random.uniform(0, 1, n-1), [0, 1])) + # result = (intermediate[1:] - intermediate[:-1]) * m + result = np.random.dirichlet(np.ones(n)) * m + + min_val = min(result) + max_val = max(result) + while max_val > (max_prob + FLOATING_POINT_COMPARISON): + new_min = min_val + (max_val - max_prob) + # This adjustment prevents the new max from always being max_prob. + adjustment = (max_prob - new_min) * np.random.rand() + result[np.argmin(result)] = new_min + adjustment + result[np.argmax(result)] = max_prob - adjustment + min_val = min(result) + max_val = max(result) + + min_val = min(result) + max_val = max(result) + while min_val < (min_prob - FLOATING_POINT_COMPARISON): + min_val = min(result) + max_val = max(result) + new_max = max_val - (min_prob - min_val) + # This adjustment prevents the new min from always being min_prob. + adjustment = (new_max - min_prob) * np.random.rand() + result[np.argmax(result)] = new_max - adjustment + result[np.argmin(result)] = min_prob + adjustment + min_val = min(result) + max_val = max(result) + + return result + + +def randomly_distribute_N_balls_into_K_bins( + N, # int + K, # int + *, + max_balls_per_bin=None, + min_balls_per_bin=None, +) -> np.ndarray: + """Returns a uniformly random numpy integer array of length `N` that sums + to `K`. + + Parameters + ---------- + N : int + Number of balls. + K : int + Number of bins. + max_balls_per_bin : int + Ensure that each bin contains at most `max_balls_per_bin` balls. + min_balls_per_bin : int + Ensure that each bin contains at least `min_balls_per_bin` balls. + + Returns + ------- + int_array : np.array + Length `N` array that sums to `K`. + """ + + if N == 0: + return np.zeros(K, dtype=int) + if max_balls_per_bin is None: + max_balls_per_bin = N + else: + max_balls_per_bin = min(max_balls_per_bin, N) + if min_balls_per_bin is None: + min_balls_per_bin = 0 + else: + min_balls_per_bin = min(min_balls_per_bin, N / K) + if N / float(K) > max_balls_per_bin: + N = max_balls_per_bin * K + + arr = np.round( + generate_n_rand_probabilities_that_sum_to_m( + n=K, + m=1, + max_prob=max_balls_per_bin / float(N), + min_prob=min_balls_per_bin / float(N), + ) + * N + ) + while sum(arr) != N: + while sum(arr) > N: # pragma: no cover + arr[np.argmax(arr)] -= 1 + while sum(arr) < N: + arr[np.argmin(arr)] += 1 + return arr.astype(int) diff --git a/cleanlab/classification.py b/cleanlab/classification.py new file mode 100644 index 0000000..15d8adc --- /dev/null +++ b/cleanlab/classification.py @@ -0,0 +1,1062 @@ +""" +cleanlab can be used for learning with noisy labels for any dataset and model. + +For regular (multi-class) classification tasks, +the `~cleanlab.classification.CleanLearning` class wraps an instance of an +sklearn classifier. The wrapped classifier must adhere to the `sklearn estimator API +`_, +meaning it must define four functions: + +* ``clf.fit(X, y, sample_weight=None)`` +* ``clf.predict_proba(X)`` +* ``clf.predict(X)`` +* ``clf.score(X, y, sample_weight=None)`` + +where `X` contains data (i.e. features), `y` contains labels (with elements in 0, 1, ..., K-1, +where K is the number of classes). The first index of `X` and of `y` should correspond to the different examples in the dataset, +such that ``len(X) = len(y) = N`` (sample-size). Here `sample_weight` re-weights examples in +the loss function while training (supporting `sample_weight` in your classifier is recommended but optional). + +Furthermore, your estimator should be correctly clonable via +`sklearn.base.clone `_: +cleanlab internally creates multiple instances of the +estimator, and if you e.g. manually wrap a PyTorch model, you must ensure that +every call to the estimator's ``__init__()`` creates an independent instance of +the model (for sklearn compatibility, the weights of neural network models should typically be initialized inside of ``clf.fit()``). + +Note +---- +There are two new notions of confidence in this package: + +1. Confident *examples* --- examples we are confident are labeled correctly. +We prune everything else. Mathematically, this means keeping the examples +with high probability of belong to their provided label class. + +2. Confident *errors* --- examples we are confident are labeled erroneously. +We prune these. Mathematically, this means pruning the examples with +high probability of belong to a different class. + +Examples +-------- +>>> from cleanlab.classification import CleanLearning +>>> from sklearn.linear_model import LogisticRegression as LogReg +>>> cl = CleanLearning(clf=LogReg()) # Pass in any classifier. +>>> cl.fit(X_train, labels_maybe_with_errors) +>>> # Estimate the predictions as if you had trained without label issues. +>>> pred = cl.predict(X_test) + +If the model is not sklearn-compatible by default, it might be the case that +standard packages can adapt the model. For example, you can adapt PyTorch +models using `skorch `_ and adapt Keras models +using `SciKeras `_. + +If an open-source adapter doesn't already exist, you can manually wrap the +model to be sklearn-compatible. This is made easy by inheriting from +`sklearn.base.BaseEstimator +`_: + +.. code:: python + + from sklearn.base import BaseEstimator + + class YourModel(BaseEstimator): + def __init__(self, ): + pass + def fit(self, X, y, sample_weight=None): + pass + def predict(self, X): + pass + def predict_proba(self, X): + pass + def score(self, X, y, sample_weight=None): + pass + +Note +---- + +* `labels` refers to the given labels in the original dataset, which may have errors +* labels must be integers in 0, 1, ..., K-1, where K is the total number of classes + +Note +---- + +Confident learning is the state-of-the-art (`Northcutt et al., 2021 `_) for +weak supervision, finding label issues in datasets, learning with noisy +labels, uncertainty estimation, and more. It works with *any* classifier, +including deep neural networks. See the `clf` parameter. + +Confident learning is a subfield of theory and algorithms of machine learning with noisy labels. +Cleanlab achieves state-of-the-art performance of any open-sourced implementation of confident +learning across a variety of tasks like multi-class classification, multi-label classification, +and PU learning. + +Given any classifier having the `predict_proba` method, an input feature +matrix `X`, and a discrete vector of noisy labels `labels`, confident learning estimates the +classifications that would be obtained if the *true labels* had instead been provided +to the classifier during training. `labels` denotes the noisy labels instead of +the :math:`\\tilde{y}` used in confident learning paper. +""" + +from sklearn.linear_model import LogisticRegression as LogReg +from sklearn.metrics import accuracy_score +from sklearn.base import BaseEstimator +import numpy as np +import pandas as pd +import inspect +import warnings +from typing import Optional, TYPE_CHECKING + +if TYPE_CHECKING: # pragma: no cover + from typing_extensions import Self + +from cleanlab.rank import get_label_quality_scores +from cleanlab import filter +from cleanlab.internal.util import ( + value_counts, + compress_int_array, + subset_X_y, + get_num_classes, + force_two_dimensions, +) +from cleanlab.count import ( + estimate_py_noise_matrices_and_cv_pred_proba, + estimate_py_and_noise_matrices_from_probabilities, + estimate_cv_predicted_probabilities, + estimate_latent, + compute_confident_joint, +) +from cleanlab.internal.latent_algebra import ( + compute_py_inv_noise_matrix, + compute_noise_matrix_from_inverse, +) +from cleanlab.internal.validation import ( + assert_valid_inputs, + labels_to_array, +) +from cleanlab.experimental.label_issues_batched import find_label_issues_batched + + +class CleanLearning(BaseEstimator): # Inherits sklearn classifier + """ + CleanLearning = Machine Learning with cleaned data (even when training on messy, error-ridden data). + + Automated and robust learning with noisy labels using any dataset and any model. This class + trains a model `clf` with error-prone, noisy labels as if the model had been instead trained + on a dataset with perfect labels. It achieves this by cleaning out the error and providing + cleaned data while training. This class is currently intended for standard (multi-class) classification tasks. + + Parameters + ---------- + clf : estimator instance, optional + A classifier implementing the `sklearn estimator API + `_, + defining the following functions: + + * ``clf.fit(X, y, sample_weight=None)`` + * ``clf.predict_proba(X)`` + * ``clf.predict(X)`` + * ``clf.score(X, y, sample_weight=None)`` + + See :py:mod:`cleanlab.models`, the tutorials, and examples/ repo + for examples of sklearn wrappers, e.g. around PyTorch, Keras, or FastText. + + If the model is not sklearn-compatible by default, it might be the case that + standard packages can adapt the model. For example, you can adapt PyTorch + models using `skorch `_ and adapt Keras models + using `SciKeras `_. + + Stores the classifier used in Confident Learning. + Default classifier used is `sklearn.linear_model.LogisticRegression + `_. + Default classifier assumes that indexing along the first dimension of the dataset corresponds to + selecting different training examples. + + seed : int, optional + Set the default state of the random number generator used to split + the cross-validated folds. By default, uses `np.random` current random state. + + cv_n_folds : int, default=5 + This class needs holdout predicted probabilities for every data example + and if not provided, uses cross-validation to compute them. + `cv_n_folds` sets the number of cross-validation folds used to compute + out-of-sample probabilities for each example in `X`. + + converge_latent_estimates : bool, optional + If true, forces numerical consistency of latent estimates. Each is + estimated independently, but they are related mathematically with closed + form equivalences. This will iteratively enforce consistency. + + pulearning : {None, 0, 1}, default=None + Only works for 2 class datasets. Set to the integer of the class that is + perfectly labeled (you are certain that there are no errors in that class). + + find_label_issues_kwargs : dict, optional + Keyword arguments to pass into :py:func:`filter.find_label_issues + `. Particularly useful options include: + `filter_by`, `frac_noise`, `min_examples_per_class` (which all impact ML accuracy), + `n_jobs` (set this to 1 to disable multi-processing if it's causing issues). + + label_quality_scores_kwargs : dict, optional + Keyword arguments to pass into :py:func:`rank.get_label_quality_scores + `. Options include: `method`, `adjust_pred_probs`. + + verbose : bool, default=False + Controls how much output is printed. Set to ``False`` to suppress print + statements. + + low_memory: bool, default=False + Set as ``True`` if you have a big dataset with limited memory. + Uses :py:func:`experimental.label_issues_batched.find_label_issues_batched ` + to find label issues. + """ + + def __init__( + self, + clf=None, + *, + seed=None, + # Hyper-parameters (used by .fit() function) + cv_n_folds=5, + converge_latent_estimates=False, + pulearning=None, + find_label_issues_kwargs={}, + label_quality_scores_kwargs={}, + verbose=False, + low_memory=False, + ): + self._default_clf = False + if clf is None: + # Use logistic regression if no classifier is provided. + clf = LogReg(solver="lbfgs") + self._default_clf = True + + # Make sure the given classifier has the appropriate methods defined. + if not hasattr(clf, "fit"): + raise ValueError("The classifier (clf) must define a .fit() method.") + if not hasattr(clf, "predict_proba"): + raise ValueError("The classifier (clf) must define a .predict_proba() method.") + if not hasattr(clf, "predict"): + raise ValueError("The classifier (clf) must define a .predict() method.") + + if seed is not None: + np.random.seed(seed=seed) + + self.clf = clf + self.seed = seed + self.cv_n_folds = cv_n_folds + self.converge_latent_estimates = converge_latent_estimates + self.pulearning = pulearning + self.find_label_issues_kwargs = find_label_issues_kwargs + self.label_quality_scores_kwargs = label_quality_scores_kwargs + self.verbose = verbose + self.label_issues_df = None + self.label_issues_mask = None + self.sample_weight = None + self.confident_joint = None + self.py = None + self.ps = None + self.num_classes = None + self.noise_matrix = None + self.inverse_noise_matrix = None + self.clf_kwargs = None + self.clf_final_kwargs = None + self.low_memory = low_memory + + def fit( + self, + X, + labels=None, + *, + pred_probs=None, + thresholds=None, + noise_matrix=None, + inverse_noise_matrix=None, + label_issues=None, + sample_weight=None, + clf_kwargs={}, + clf_final_kwargs={}, + validation_func=None, + y=None, + ) -> "Self": + """ + Train the model `clf` with error-prone, noisy labels as if + the model had been instead trained on a dataset with the correct labels. + `fit` achieves this by first training `clf` via cross-validation on the noisy data, + using the resulting predicted probabilities to identify label issues, + pruning the data with label issues, and finally training `clf` on the remaining clean data. + + Parameters + ---------- + X : np.ndarray or DatasetLike + Data features (i.e. training inputs for ML), typically an array of shape ``(N, ...)``, + where N is the number of examples. + Supported `DatasetLike` types beyond ``np.ndarray`` include: + ``pd.DataFrame``, ``scipy.sparse.csr_matrix``, ``torch.utils.data.Dataset``, + or any dataset object ``X`` that supports list-based indexing: + ``X[index_list]`` to select a subset of training examples. + Your classifier that this instance was initialized with, + ``clf``, must be able to ``fit()`` and ``predict()`` data of this format. + + + labels : array_like + An array of shape ``(N,)`` of noisy classification labels, where some labels may be erroneous. + Elements must be integers in the set 0, 1, ..., K-1, where K is the number of classes. + Supported `array_like` types include: ``np.ndarray``, ``pd.Series``, or ``list``. + + pred_probs : np.ndarray, optional + An array of shape ``(N, K)`` of model-predicted probabilities, + ``P(label=k|x)``. Each row of this matrix corresponds + to an example `x` and contains the model-predicted probabilities that + `x` belongs to each possible class, for each of the K classes. The + columns must be ordered such that these probabilities correspond to class 0, 1, ..., K-1. + `pred_probs` should be :ref:`out-of-sample, eg. computed via cross-validation `. + If provided, `pred_probs` will be used to find label issues rather than the ``clf`` classifier. + + Note + ---- + If you are not sure, leave ``pred_probs=None`` (the default) and it + will be computed for you using cross-validation with the provided model. + + thresholds : array_like, optional + An array of shape ``(K, 1)`` or ``(K,)`` of per-class threshold + probabilities, used to determine the cutoff probability necessary to + consider an example as a given class label (see `Northcutt et al., + 2021 `_, Section + 3.1, Equation 2). + + This is for advanced users only. If not specified, these are computed + for you automatically. If an example has a predicted probability + greater than this threshold, it is counted as having true_label = + k. This is not used for pruning/filtering, only for estimating the + noise rates using confident counts. + + noise_matrix : np.ndarray, optional + An array of shape ``(K, K)`` representing the conditional probability + matrix ``P(label=k_s | true label=k_y)``, the + fraction of examples in every class, labeled as every other class. + Assumes columns of `noise_matrix` sum to 1. + + inverse_noise_matrix : np.ndarray, optional + An array of shape ``(K, K)`` representing the conditional probability + matrix ``P(true label=k_y | label=k_s)``, + the estimated fraction observed examples in each class ``k_s`` + that are mislabeled examples from every other class ``k_y``, + Assumes columns of `inverse_noise_matrix` sum to 1. + + label_issues : pd.DataFrame or np.ndarray, optional + Specifies the label issues for each example in dataset. + If ``pd.DataFrame``, must be formatted as the one returned by: + :py:meth:`CleanLearning.find_label_issues + ` or + `~cleanlab.classification.CleanLearning.get_label_issues`. + If ``np.ndarray``, must contain either boolean `label_issues_mask` as output by: + default :py:func:`filter.find_label_issues `, + or integer indices as output by + :py:func:`filter.find_label_issues ` + with its `return_indices_ranked_by` argument specified. + Providing this argument significantly reduces the time this method takes to run by + skipping the slow cross-validation step necessary to find label issues. + Examples identified to have label issues will be + pruned from the data before training the final `clf` model. + + Caution: If you provide `label_issues` without having previously called + `~cleanlab.classification.CleanLearning.find_label_issues` + e.g. as a ``np.ndarray``, then some functionality like training with sample weights may be disabled. + + sample_weight : array_like, optional + Array of weights with shape ``(N,)`` that are assigned to individual samples, + assuming total number of examples in dataset is `N`. + If not provided, samples may still be weighted by the estimated noise in the class they are labeled as. + + clf_kwargs : dict, optional + Optional keyword arguments to pass into `clf`'s ``fit()`` method. + + clf_final_kwargs : dict, optional + Optional extra keyword arguments to pass into the final `clf` ``fit()`` on the cleaned data + but not the `clf` ``fit()`` in each fold of cross-validation on the noisy data. + The final ``fit()`` will also receive `clf_kwargs`, + but these may be overwritten by values in `clf_final_kwargs`. + This can be useful for training differently in the final ``fit()`` + than during cross-validation. + + validation_func : callable, optional + Optional callable function that takes two arguments, `X_val`, `y_val`, and returns a dict + of keyword arguments passed into to ``clf.fit()`` which may be functions of the validation + data in each cross-validation fold. Specifies how to map the validation data split in each + cross-validation fold into the appropriate format to pass into `clf`'s ``fit()`` method, assuming + ``clf.fit()`` can utilize validation data if it is appropriately passed in (eg. for early-stopping). + Eg. if your model's ``fit()`` method is called using ``clf.fit(X, y, X_validation, y_validation)``, + then you could set ``validation_func = f`` where + ``def f(X_val, y_val): return {"X_validation": X_val, "y_validation": y_val}`` + + Note that `validation_func` will be ignored in the final call to `clf.fit()` on the + cleaned subset of the data. This argument is only for allowing `clf` to access the + validation data in each cross-validation fold (eg. for early-stopping or hyperparameter-selection + purposes). If you want to pass in validation data even in the final training call to ``clf.fit()`` + on the cleaned data subset, you should explicitly pass in that data yourself + (eg. via `clf_final_kwargs` or `clf_kwargs`). + + y: array_like, optional + Alternative argument that can be specified instead of `labels`. + Specifying `y` has the same effect as specifying `labels`, + and is offered as an alternative for compatibility with sklearn. + + Returns + ------- + self : CleanLearning + Fitted estimator that has all the same methods as any sklearn estimator. + + + After calling ``self.fit()``, this estimator also stores extra attributes such as: + + * *self.label_issues_df*: a ``pd.DataFrame`` accessible via + `~cleanlab.classification.CleanLearning.get_label_issues` + of similar format as the one returned by: `~cleanlab.classification.CleanLearning.find_label_issues`. + See documentation of :py:meth:`CleanLearning.find_label_issues` + for column descriptions. + + + After calling ``self.fit()``, `self.label_issues_df` may also contain an extra column: + + * *sample_weight*: Numeric values that were used to weight examples during + the final training of `clf` in ``CleanLearning.fit()``. + `sample_weight` column will only be present if automatic sample weights were actually used. + These automatic weights are assigned to each example based on the class it belongs to, + i.e. there are only num_classes unique sample_weight values. + The sample weight for an example belonging to class k is computed as ``1 / p(given_label = k | true_label = k)``. + This sample_weight normalizes the loss to effectively trick `clf` into learning with the distribution + of the true labels by accounting for the noisy data pruned out prior to training on cleaned data. + In other words, examples with label issues were removed, so this weights the data proportionally + so that the classifier trains as if it had all the true labels, + not just the subset of cleaned data left after pruning out the label issues. + + Note + ---- + If ``CleanLearning.fit()`` does not work for your data/model, you can run the same procedure yourself: + * Utilize :ref:`cross-validation ` to get out-of-sample `pred_probs` for each example. + * Call :py:func:`filter.find_label_issues ` with `pred_probs`. + * Filter the examples with detected issues and train your model on the remaining data. + """ + + if labels is not None and y is not None: + raise ValueError("You must specify either `labels` or `y`, but not both.") + if y is not None: + labels = y + if labels is None: + raise ValueError("You must specify `labels`.") + if self._default_clf: + X = force_two_dimensions(X) + + self.clf_final_kwargs = {**clf_kwargs, **clf_final_kwargs} + + if "sample_weight" in clf_kwargs: + raise ValueError( + "sample_weight should be provided directly in fit() or in clf_final_kwargs rather than in clf_kwargs" + ) + + if sample_weight is not None: + if "sample_weight" not in inspect.signature(self.clf.fit).parameters: + raise ValueError( + "sample_weight must be a supported fit() argument for your model in order to be specified here" + ) + + if label_issues is None: + if self.label_issues_df is not None and self.verbose: + print( + "If you already ran self.find_label_issues() and don't want to recompute, you " + "should pass the label_issues in as a parameter to this function next time." + ) + label_issues = self.find_label_issues( + X, + labels, + pred_probs=pred_probs, + thresholds=thresholds, + noise_matrix=noise_matrix, + inverse_noise_matrix=inverse_noise_matrix, + clf_kwargs=clf_kwargs, + validation_func=validation_func, + ) + + else: # set args that may not have been set if `self.find_label_issues()` wasn't called yet + assert_valid_inputs(X, labels, pred_probs) + if self.num_classes is None: + if noise_matrix is not None: + label_matrix = noise_matrix + else: + label_matrix = inverse_noise_matrix + self.num_classes = get_num_classes(labels, pred_probs, label_matrix) + if self.verbose: + print("Using provided label_issues instead of finding label issues.") + if self.label_issues_df is not None: + print( + "These will overwrite self.label_issues_df and will be returned by " + "`self.get_label_issues()`. " + ) + + # label_issues always overwrites self.label_issues_df. Ensure it is properly formatted: + self.label_issues_df = self._process_label_issues_arg(label_issues, labels) + + if "label_quality" not in self.label_issues_df.columns and pred_probs is not None: + if self.verbose: + print("Computing label quality scores based on given pred_probs ...") + self.label_issues_df["label_quality"] = get_label_quality_scores( + labels, pred_probs, **self.label_quality_scores_kwargs + ) + + self.label_issues_mask = self.label_issues_df["is_label_issue"].to_numpy() + x_mask = np.invert(self.label_issues_mask) + x_cleaned, labels_cleaned = subset_X_y(X, labels, x_mask) + if self.verbose: + print(f"Pruning {np.sum(self.label_issues_mask)} examples with label issues ...") + print(f"Remaining clean data has {len(labels_cleaned)} examples.") + + if sample_weight is None: + # Check if sample_weight in args of clf.fit() + if ( + "sample_weight" in inspect.signature(self.clf.fit).parameters + and "sample_weight" not in self.clf_final_kwargs + and self.noise_matrix is not None + ): + # Re-weight examples in the loss function for the final fitting + # such that the "apparent" original number of examples in each class + # is preserved, even though the pruned sets may differ. + if self.verbose: + print( + "Assigning sample weights for final training based on estimated label quality." + ) + sample_weight_auto = np.ones(np.shape(labels_cleaned)) + for k in range(self.num_classes): + sample_weight_k = 1.0 / max( + self.noise_matrix[k][k], 1e-3 + ) # clip sample weights + sample_weight_auto[labels_cleaned == k] = sample_weight_k + + sample_weight_expanded = np.zeros( + len(labels) + ) # pad pruned examples with zeros, length of original dataset + sample_weight_expanded[x_mask] = sample_weight_auto + # Store the sample weight for every example in the original, unfiltered dataset + self.label_issues_df["sample_weight"] = sample_weight_expanded + self.sample_weight = self.label_issues_df[ + "sample_weight" + ] # pointer to here to avoid duplication + self.clf_final_kwargs["sample_weight"] = sample_weight_auto + if self.verbose: + print("Fitting final model on the clean data ...") + else: + if self.verbose: + if "sample_weight" in self.clf_final_kwargs: + print("Fitting final model on the clean data with custom sample_weight ...") + else: + if ( + "sample_weight" in inspect.signature(self.clf.fit).parameters + and self.noise_matrix is None + ): + print( + "Cannot utilize sample weights for final training! " + "Why this matters: during final training, sample weights help account for the amount of removed data in each class. " + "This helps ensure the correct class prior for the learned model. " + "To use sample weights, you need to either provide the noise_matrix or have previously called self.find_label_issues() instead of filter.find_label_issues() which computes them for you." + ) + print("Fitting final model on the clean data ...") + + elif sample_weight is not None and "sample_weight" not in self.clf_final_kwargs: + self.clf_final_kwargs["sample_weight"] = sample_weight[x_mask] + if self.verbose: + print("Fitting final model on the clean data with custom sample_weight ...") + + else: # pragma: no cover + if self.verbose: + if "sample_weight" in self.clf_final_kwargs: + print("Fitting final model on the clean data with custom sample_weight ...") + else: + print("Fitting final model on the clean data ...") + + self.clf.fit(x_cleaned, labels_cleaned, **self.clf_final_kwargs) + + if self.verbose: + print( + "Label issues stored in label_issues_df DataFrame accessible via: self.get_label_issues(). " + "Call self.save_space() to delete this potentially large DataFrame attribute." + ) + return self + + def predict(self, *args, **kwargs) -> np.ndarray: + """Predict class labels using your wrapped classifier `clf`. + Works just like ``clf.predict()``. + + Parameters + ---------- + X : np.ndarray or DatasetLike + Test data in the same format expected by your wrapped classifier. + + Returns + ------- + class_predictions : np.ndarray + Vector of class predictions for the test examples. + """ + if self._default_clf: + if args: + X = args[0] + elif "X" in kwargs: + X = kwargs["X"] + del kwargs["X"] + else: + raise ValueError("No input provided to predict, please provide X.") + X = force_two_dimensions(X) + new_args = (X,) + args[1:] + return self.clf.predict(*new_args, **kwargs) + else: + return self.clf.predict(*args, **kwargs) + + def predict_proba(self, *args, **kwargs) -> np.ndarray: + """Predict class probabilities ``P(true label=k)`` using your wrapped classifier `clf`. + Works just like ``clf.predict_proba()``. + + Parameters + ---------- + X : np.ndarray or DatasetLike + Test data in the same format expected by your wrapped classifier. + + Returns + ------- + pred_probs : np.ndarray + ``(N x K)`` array of predicted class probabilities, one row for each test example. + """ + if self._default_clf: + if args: + X = args[0] + elif "X" in kwargs: + X = kwargs["X"] + del kwargs["X"] + else: + raise ValueError("No input provided to predict, please provide X.") + X = force_two_dimensions(X) + new_args = (X,) + args[1:] + return self.clf.predict_proba(*new_args, **kwargs) + else: + return self.clf.predict_proba(*args, **kwargs) + + def score(self, X, y, sample_weight=None) -> float: + """Evaluates your wrapped classifier `clf`'s score on a test set `X` with labels `y`. + Uses your model's default scoring function, or simply accuracy if your model as no ``"score"`` attribute. + + Parameters + ---------- + X : np.ndarray or DatasetLike + Test data in the same format expected by your wrapped classifier. + + y : array_like + Test labels in the same format as labels previously used in ``fit()``. + + sample_weight : np.ndarray, optional + An array of shape ``(N,)`` or ``(N, 1)`` used to weight each test example when computing the score. + + Returns + ------- + score: float + Number quantifying the performance of this classifier on the test data. + """ + if self._default_clf: + X = force_two_dimensions(X) + if hasattr(self.clf, "score"): + # Check if sample_weight in clf.score() + if "sample_weight" in inspect.signature(self.clf.score).parameters: + return self.clf.score(X, y, sample_weight=sample_weight) + else: + return self.clf.score(X, y) + else: + return accuracy_score( + y, + self.clf.predict(X), + sample_weight=sample_weight, + ) + + def find_label_issues( + self, + X=None, + labels=None, + *, + pred_probs=None, + thresholds=None, + noise_matrix=None, + inverse_noise_matrix=None, + save_space=False, + clf_kwargs={}, + validation_func=None, + ) -> pd.DataFrame: + """ + Identifies potential label issues in the dataset using confident learning. + + Runs cross-validation to get out-of-sample pred_probs from `clf` + and then calls :py:func:`filter.find_label_issues + ` to find label issues. + These label issues are cached internally and returned in a pandas DataFrame. + Kwargs for :py:func:`filter.find_label_issues + ` must have already been specified + in the initialization of this class, not here. + + Unlike :py:func:`filter.find_label_issues + `, which requires `pred_probs`, + this method only requires a classifier and it can do the cross-validation for you. + Both methods return the same boolean mask that identifies which examples have label issues. + This is the preferred method to use if you plan to subsequently invoke: + `~cleanlab.classification.CleanLearning.fit`. + + Note: this method computes the label issues from scratch. To access + previously-computed label issues from this `~cleanlab.classification.CleanLearning` instance, use the + `~cleanlab.classification.CleanLearning.get_label_issues` method. + + This is the method called to find label issues inside + `~cleanlab.classification.CleanLearning.fit` + and they share mostly the same parameters. + + Parameters + ---------- + save_space : bool, optional + If True, then returned `label_issues_df` will not be stored as attribute. + This means some other methods like `self.get_label_issues()` will no longer work. + + + For info about the **other parameters**, see the docstring of `~cleanlab.classification.CleanLearning.fit`. + + Returns + ------- + label_issues_df : pd.DataFrame + DataFrame with info about label issues for each example. + Unless `save_space` argument is specified, same DataFrame is also stored as + `self.label_issues_df` attribute accessible via + `~cleanlab.classification.CleanLearning.get_label_issues`. + Each row represents an example from our dataset and + the DataFrame may contain the following columns: + + * *is_label_issue*: boolean mask for the entire dataset where ``True`` represents a label issue and ``False`` represents an example that is accurately labeled with high confidence. This column is equivalent to `label_issues_mask` output from :py:func:`filter.find_label_issues`. + * *label_quality*: Numeric score that measures the quality of each label (how likely it is to be correct, with lower scores indicating potentially erroneous labels). + * *given_label*: Integer indices corresponding to the class label originally given for this example (same as `labels` input). Included here for ease of comparison against `clf` predictions, only present if "predicted_label" column is present. + * *predicted_label*: Integer indices corresponding to the class predicted by trained `clf` model. Only present if ``pred_probs`` were provided as input or computed during label-issue-finding. + * *sample_weight*: Numeric values used to weight examples during the final training of `clf` in `~cleanlab.classification.CleanLearning.fit`. This column may not be present after `self.find_label_issues()` but may be added after call to `~cleanlab.classification.CleanLearning.fit`. For more precise definition of sample weights, see documentation of `~cleanlab.classification.CleanLearning.fit` + """ + + # Check inputs + assert_valid_inputs(X, labels, pred_probs) + labels = labels_to_array(labels) + if noise_matrix is not None and np.trace(noise_matrix) <= 1: + t = np.round(np.trace(noise_matrix), 2) + raise ValueError("Trace(noise_matrix) is {}, but must exceed 1.".format(t)) + if inverse_noise_matrix is not None and (np.trace(inverse_noise_matrix) <= 1): + t = np.round(np.trace(inverse_noise_matrix), 2) + raise ValueError("Trace(inverse_noise_matrix) is {}. Must exceed 1.".format(t)) + + if self._default_clf: + X = force_two_dimensions(X) + if noise_matrix is not None: + label_matrix = noise_matrix + else: + label_matrix = inverse_noise_matrix + self.num_classes = get_num_classes(labels, pred_probs, label_matrix) + if (pred_probs is None) and (len(labels) / self.num_classes < self.cv_n_folds): + raise ValueError( + "Need more data from each class for cross-validation. " + "Try decreasing cv_n_folds (eg. to 2 or 3) in CleanLearning()" + ) + # 'ps' is p(labels=k) + self.ps = value_counts(labels) / float(len(labels)) + + self.clf_kwargs = clf_kwargs + if self.low_memory: + # If needed, compute P(label=k|x), denoted pred_probs (the predicted probabilities) + if pred_probs is None: + if self.verbose: + print( + "Computing out of sample predicted probabilities via " + f"{self.cv_n_folds}-fold cross validation. May take a while ..." + ) + + pred_probs = estimate_cv_predicted_probabilities( + X=X, + labels=labels, + clf=self.clf, + cv_n_folds=self.cv_n_folds, + seed=self.seed, + clf_kwargs=self.clf_kwargs, + validation_func=validation_func, + ) + + if self.verbose: + print("Using predicted probabilities to identify label issues ...") + + if self.find_label_issues_kwargs: + warnings.warn(f"`find_label_issues_kwargs` is not used when `low_memory=True`.") + arg_values = { + "thresholds": thresholds, + "noise_matrix": noise_matrix, + "inverse_noise_matrix": inverse_noise_matrix, + } + for arg_name, arg_val in arg_values.items(): + if arg_val is not None: + warnings.warn(f"`{arg_name}` is not used when `low_memory=True`.") + label_issues_mask = find_label_issues_batched(labels, pred_probs, return_mask=True) + else: + self._process_label_issues_kwargs(self.find_label_issues_kwargs) + # self._process_label_issues_kwargs might set self.confident_joint. If so, we should use it. + if self.confident_joint is not None: + self.py, noise_matrix, inv_noise_matrix = estimate_latent( + confident_joint=self.confident_joint, + labels=labels, + ) + + # If needed, compute noise rates (probability of class-conditional mislabeling). + if noise_matrix is not None: + self.noise_matrix = noise_matrix + if inverse_noise_matrix is None: + if self.verbose: + print("Computing label noise estimates from provided noise matrix ...") + self.py, self.inverse_noise_matrix = compute_py_inv_noise_matrix( + ps=self.ps, + noise_matrix=self.noise_matrix, + ) + if inverse_noise_matrix is not None: + self.inverse_noise_matrix = inverse_noise_matrix + if noise_matrix is None: + if self.verbose: + print( + "Computing label noise estimates from provided inverse noise matrix ..." + ) + self.noise_matrix = compute_noise_matrix_from_inverse( + ps=self.ps, + inverse_noise_matrix=self.inverse_noise_matrix, + ) + + if noise_matrix is None and inverse_noise_matrix is None: + if pred_probs is None: + if self.verbose: + print( + "Computing out of sample predicted probabilities via " + f"{self.cv_n_folds}-fold cross validation. May take a while ..." + ) + ( + self.py, + self.noise_matrix, + self.inverse_noise_matrix, + self.confident_joint, + pred_probs, + ) = estimate_py_noise_matrices_and_cv_pred_proba( + X=X, + labels=labels, + clf=self.clf, + cv_n_folds=self.cv_n_folds, + thresholds=thresholds, + converge_latent_estimates=self.converge_latent_estimates, + seed=self.seed, + clf_kwargs=self.clf_kwargs, + validation_func=validation_func, + ) + else: # pred_probs is provided by user (assumed holdout probabilities) + if self.verbose: + print("Computing label noise estimates from provided pred_probs ...") + ( + self.py, + self.noise_matrix, + self.inverse_noise_matrix, + self.confident_joint, + ) = estimate_py_and_noise_matrices_from_probabilities( + labels=labels, + pred_probs=pred_probs, + thresholds=thresholds, + converge_latent_estimates=self.converge_latent_estimates, + ) + # If needed, compute P(label=k|x), denoted pred_probs (the predicted probabilities) + if pred_probs is None: + if self.verbose: + print( + "Computing out of sample predicted probabilities via " + f"{self.cv_n_folds}-fold cross validation. May take a while ..." + ) + + pred_probs = estimate_cv_predicted_probabilities( + X=X, + labels=labels, + clf=self.clf, + cv_n_folds=self.cv_n_folds, + seed=self.seed, + clf_kwargs=self.clf_kwargs, + validation_func=validation_func, + ) + # If needed, compute the confident_joint (e.g. occurs if noise_matrix was given) + if self.confident_joint is None: + self.confident_joint = compute_confident_joint( + labels=labels, + pred_probs=pred_probs, + thresholds=thresholds, + ) + + # if pulearning == the integer specifying the class without noise. + if self.num_classes == 2 and self.pulearning is not None: # pragma: no cover + # pulearning = 1 (no error in 1 class) implies p(label=1|true_label=0) = 0 + self.noise_matrix[self.pulearning][1 - self.pulearning] = 0 + self.noise_matrix[1 - self.pulearning][1 - self.pulearning] = 1 + # pulearning = 1 (no error in 1 class) implies p(true_label=0|label=1) = 0 + self.inverse_noise_matrix[1 - self.pulearning][self.pulearning] = 0 + self.inverse_noise_matrix[self.pulearning][self.pulearning] = 1 + # pulearning = 1 (no error in 1 class) implies p(label=1,true_label=0) = 0 + self.confident_joint[self.pulearning][1 - self.pulearning] = 0 + self.confident_joint[1 - self.pulearning][1 - self.pulearning] = 1 + + # Add confident joint to find label issue args if it is not previously specified + if "confident_joint" not in self.find_label_issues_kwargs.keys(): + # however does not add if users specify filter_by="confident_learning", as it will throw a warning + if not self.find_label_issues_kwargs.get("filter_by") == "confident_learning": + self.find_label_issues_kwargs["confident_joint"] = self.confident_joint + + labels = labels_to_array(labels) + if self.verbose: + print("Using predicted probabilities to identify label issues ...") + label_issues_mask = filter.find_label_issues( + labels, + pred_probs, + **self.find_label_issues_kwargs, + ) + label_quality_scores = get_label_quality_scores( + labels, pred_probs, **self.label_quality_scores_kwargs + ) + label_issues_df = pd.DataFrame( + {"is_label_issue": label_issues_mask, "label_quality": label_quality_scores} + ) + if self.verbose: + print(f"Identified {np.sum(label_issues_mask)} examples with label issues.") + + predicted_labels = pred_probs.argmax(axis=1) + label_issues_df["given_label"] = compress_int_array(labels, self.num_classes) + label_issues_df["predicted_label"] = compress_int_array(predicted_labels, self.num_classes) + + if not save_space: + if self.label_issues_df is not None and self.verbose: + print( + "Overwriting previously identified label issues stored at self.label_issues_df. " + "self.get_label_issues() will now return the newly identified label issues. " + ) + self.label_issues_df = label_issues_df + self.label_issues_mask = label_issues_df[ + "is_label_issue" + ] # pointer to here to avoid duplication + elif self.verbose: + print( # pragma: no cover + "Not storing label_issues as attributes since save_space was specified." + ) + + return label_issues_df + + def get_label_issues(self) -> Optional[pd.DataFrame]: + """ + Accessor. Returns `label_issues_df` attribute if previously already computed. + This ``pd.DataFrame`` describes the label issues identified for each example + (each row corresponds to an example). + For column definitions, see the documentation of + `~cleanlab.classification.CleanLearning.find_label_issues`. + + Returns + ------- + label_issues_df : pd.DataFrame + DataFrame with (precomputed) info about label issues for each example. + """ + + if self.label_issues_df is None: + warnings.warn( + "Label issues have not yet been computed. Run `self.find_label_issues()` or `self.fit()` first." + ) + return self.label_issues_df + + def save_space(self): + """ + Clears non-sklearn attributes of this estimator to save space (in-place). + This includes the DataFrame attribute that stored label issues which may be large for big datasets. + You may want to call this method before deploying this model (i.e. if you just care about producing predictions). + After calling this method, certain non-prediction-related attributes/functionality will no longer be available + (e.g. you cannot call ``self.fit()`` anymore). + """ + + if self.label_issues_df is None and self.verbose: + print("self.label_issues_df is already empty") # pragma: no cover + self.label_issues_df = None + self.sample_weight = None + self.label_issues_mask = None + self.find_label_issues_kwargs = None + self.label_quality_scores_kwargs = None + self.confident_joint = None + self.py = None + self.ps = None + self.num_classes = None + self.noise_matrix = None + self.inverse_noise_matrix = None + self.clf_kwargs = None + self.clf_final_kwargs = None + if self.verbose: + print("Deleted non-sklearn attributes such as label_issues_df to save space.") + + def _process_label_issues_kwargs(self, find_label_issues_kwargs): + """ + Private helper function that is used to modify the arguments to passed to + filter.find_label_issues via the CleanLearning.find_label_issues class. Because + this is a classification task, some default parameters change and some errors should + be throne if certain unsupported (for classification) arguments are passed in. This method + handles those parameters inside of find_label_issues_kwargs and throws an error if you pass + in a kwargs argument to filter.find_label_issues that is not supported by the + CleanLearning.find_label_issues() function. + """ + + # Defaults for CleanLearning.find_label_issues() vs filter.find_label_issues() + DEFAULT_FIND_LABEL_ISSUES_KWARGS = {"min_examples_per_class": 10} + find_label_issues_kwargs = {**DEFAULT_FIND_LABEL_ISSUES_KWARGS, **find_label_issues_kwargs} + # Todo: support multi_label classification in the future and remove multi_label from list + unsupported_kwargs = ["return_indices_ranked_by", "multi_label"] + for unsupported_kwarg in unsupported_kwargs: + if unsupported_kwarg in find_label_issues_kwargs: + raise ValueError( + "These kwargs of `find_label_issues()` are not supported " + f"for `CleanLearning`: {unsupported_kwargs}" + ) + # CleanLearning will use this to compute the noise_matrix and inverse_noise_matrix + if "confident_joint" in find_label_issues_kwargs: + self.confident_joint = find_label_issues_kwargs["confident_joint"] + self.find_label_issues_kwargs = find_label_issues_kwargs + + def _process_label_issues_arg(self, label_issues, labels) -> pd.DataFrame: + """ + Helper method to get the label_issues input arg into a formatted DataFrame. + """ + + labels = labels_to_array(labels) + if isinstance(label_issues, pd.DataFrame): + if "is_label_issue" not in label_issues.columns: + raise ValueError( + "DataFrame label_issues must contain column: 'is_label_issue'. " + "See CleanLearning.fit() documentation for label_issues column descriptions." + ) + if len(label_issues) != len(labels): + raise ValueError("label_issues and labels must have same length") + if "given_label" in label_issues.columns and np.any( + label_issues["given_label"].to_numpy() != labels + ): + raise ValueError("labels must match label_issues['given_label']") + return label_issues + elif isinstance(label_issues, np.ndarray): + if not label_issues.dtype in [np.dtype("bool"), np.dtype("int")]: + raise ValueError("If label_issues is numpy.array, dtype must be 'bool' or 'int'.") + if label_issues.dtype is np.dtype("bool") and label_issues.shape != labels.shape: + raise ValueError( + "If label_issues is boolean numpy.array, must have same shape as labels" + ) + if label_issues.dtype is np.dtype("int"): # convert to boolean mask + if len(np.unique(label_issues)) != len(label_issues): + raise ValueError( + "If label_issues.dtype is 'int', must contain unique integer indices " + "corresponding to examples with label issues such as output by: " + "filter.find_label_issues(..., return_indices_ranked_by=...)" + ) + issue_indices = label_issues + label_issues = np.full(len(labels), False, dtype=bool) + if len(issue_indices) > 0: + label_issues[issue_indices] = True + return pd.DataFrame({"is_label_issue": label_issues}) + else: + raise ValueError("label_issues must be either pandas.DataFrame or numpy.array") diff --git a/cleanlab/count.py b/cleanlab/count.py new file mode 100644 index 0000000..2ffa016 --- /dev/null +++ b/cleanlab/count.py @@ -0,0 +1,1489 @@ +""" +Methods to estimate latent structures used for confident learning, including: + +* Latent prior of the unobserved, error-less labels: `py`: ``p(y)`` +* Latent noisy channel (noise matrix) characterizing the flipping rates: `nm`: ``P(given label | true label)`` +* Latent inverse noise matrix characterizing the flipping process: `inv`: ``P(true label | given label)`` +* Latent `confident_joint`, an un-normalized matrix that counts the confident subset of label errors under the joint distribution for true/given label + +These are estimated from a classification dataset. This module considers two types of datasets: + +* standard (multi-class) classification where each example is labeled as belonging to exactly one of K classes (e.g. ``labels = np.array([0,0,1,0,2,1])``) +* multi-label classification where each example can be labeled as belonging to multiple classes (e.g. ``labels = [[1,2],[1],[0],[],...]``) +""" + +import warnings +from typing import Optional, Tuple, Union + +import numpy as np +import sklearn.base +from sklearn.linear_model import LogisticRegression as LogReg +from sklearn.metrics import confusion_matrix +from sklearn.model_selection import StratifiedKFold + +from cleanlab.internal.constants import ( + CONFIDENT_THRESHOLDS_LOWER_BOUND, + FLOATING_POINT_COMPARISON, + TINY_VALUE, +) +from cleanlab.internal.latent_algebra import ( + compute_inv_noise_matrix, + compute_noise_matrix_from_inverse, + compute_py, +) +from cleanlab.internal.multilabel_utils import get_onehot_num_classes, stack_complement +from cleanlab.internal.util import ( + append_extra_datapoint, + clip_noise_rates, + clip_values, + get_num_classes, + get_unique_classes, + is_torch_dataset, + round_preserving_row_totals, + train_val_split, + value_counts_fill_missing_classes, +) +from cleanlab.internal.validation import assert_valid_inputs, labels_to_array +from cleanlab.typing import LabelLike + + +def num_label_issues( + labels: LabelLike, + pred_probs: np.ndarray, + *, + confident_joint: Optional[np.ndarray] = None, + estimation_method: str = "off_diagonal", + multi_label: bool = False, +) -> int: + """Estimates the number of label issues in a classification dataset. Use this method to get the most accurate + estimate of number of label issues when you don't need the indices of the examples with label issues. + + Parameters + ---------- + labels : np.ndarray or list + Given class labels for each example in the dataset, some of which may be erroneous, + in same format expected by :py:func:`filter.find_label_issues ` function. + + pred_probs : + Model-predicted class probabilities for each example in the dataset, + in same format expected by :py:func:`filter.find_label_issues ` function. + + confident_joint : + Array of estimated class label error statisics used for identifying label issues, + in same format expected by :py:func:`filter.find_label_issues ` function. + The `confident_joint` can be computed using `~cleanlab.count.compute_confident_joint`. + It is internally computed from the given (noisy) `labels` and `pred_probs`. + + estimation_method : + Method for estimating the number of label issues in dataset by counting the examples in the off-diagonal of the `confident_joint` ``P(label=i, true_label=j)``. + + * ``'off_diagonal'``: Counts the number of examples in the off-diagonal of the `confident_joint`. Returns the same value as ``sum(find_label_issues(filter_by='confident_learning'))`` + + * ``'off_diagonal_calibrated'``: Calibrates confident joint estimate ``P(label=i, true_label=j)`` such that + ``np.sum(cj) == len(labels)`` and ``np.sum(cj, axis = 1) == np.bincount(labels)`` before counting the number + of examples in the off-diagonal. Number will always be equal to or greater than + ``estimate_issues='off_diagonal'``. You can use this value as the cutoff threshold used with ranking/scoring + functions from :py:mod:`cleanlab.rank` with `num_label_issues` over ``estimation_method='off_diagonal'`` in + two cases: + + #. As we add more label and data quality scoring functions in :py:mod:`cleanlab.rank`, this approach will always work. + #. If you have a custom score to rank your data by label quality and you just need to know the cut-off of likely label issues. + + * ``'off_diagonal_custom'``: Counts the number of examples in the off-diagonal of a provided `confident_joint` matrix. + + TL;DR: Use this method to get the most accurate estimate of number of label issues when you don't need the indices of the label issues. + + Note: ``'off_diagonal'`` may sometimes underestimate issues for data with few classes, so consider using ``'off_diagonal_calibrated'`` instead if your data has < 4 classes. + + multi_label : bool, optional + Set ``False`` if your dataset is for regular (multi-class) classification, where each example belongs to exactly one class. + Set ``True`` if your dataset is for multi-label classification, where each example can belong to multiple classes. + See documentation of `~cleanlab.count.compute_confident_joint` for details. + + Returns + ------- + num_issues : + The estimated number of examples with label issues in the dataset. + """ + valid_methods = ["off_diagonal", "off_diagonal_calibrated", "off_diagonal_custom"] + if isinstance(confident_joint, np.ndarray) and estimation_method != "off_diagonal_custom": + warn_str = ( + "The supplied `confident_joint` is ignored as `confident_joint` is recomuputed internally using " + "the supplied `labels` and `pred_probs`. If you still want to use custom `confident_joint` call function " + "with `estimation_method='off_diagonal_custom'`." + ) + warnings.warn(warn_str) + + if multi_label: + return _num_label_issues_multilabel( + labels=labels, + pred_probs=pred_probs, + confident_joint=confident_joint, + ) + labels = labels_to_array(labels) + assert_valid_inputs(X=None, y=labels, pred_probs=pred_probs) + + if estimation_method == "off_diagonal": + _, cl_error_indices = compute_confident_joint( + labels=labels, + pred_probs=pred_probs, + calibrate=False, + return_indices_of_off_diagonals=True, + ) + + label_issues_mask = np.zeros(len(labels), dtype=bool) + label_issues_mask[cl_error_indices] = True + + # Remove label issues if model prediction is close to given label + mask = _reduce_issues(pred_probs=pred_probs, labels=labels) + label_issues_mask[mask] = False + num_issues = np.sum(label_issues_mask) + elif estimation_method == "off_diagonal_calibrated": + calculated_confident_joint = compute_confident_joint( + labels=labels, + pred_probs=pred_probs, + calibrate=True, + ) + assert isinstance(calculated_confident_joint, np.ndarray) + # Estimate_joint calibrates the row sums to match the prior distribution of given labels and normalizes to sum to 1 + joint = estimate_joint(labels, pred_probs, confident_joint=calculated_confident_joint) + frac_issues = 1.0 - joint.trace() + num_issues = np.rint(frac_issues * len(labels)).astype(int) + elif estimation_method == "off_diagonal_custom": + if not isinstance(confident_joint, np.ndarray): + raise ValueError( + f""" + No `confident_joint` provided. For 'estimation_method' = {estimation_method} you need to provide pre-calculated + `confident_joint` matrix. Use a different `estimation_method` if you want the `confident_joint` matrix to + be calculated for you. + """ + ) + else: + joint = estimate_joint(labels, pred_probs, confident_joint=confident_joint) + frac_issues = 1.0 - joint.trace() + num_issues = np.rint(frac_issues * len(labels)).astype(int) + else: + raise ValueError( + f""" + {estimation_method} is not a valid estimation method! + Please choose a valid estimation method: {valid_methods} + """ + ) + + return num_issues + + +def _num_label_issues_multilabel( + labels: LabelLike, + pred_probs: np.ndarray, + confident_joint: Optional[np.ndarray] = None, +) -> int: + """ + Parameters + ---------- + labels: list + Refer to documentation for this argument in ``count.calibrate_confident_joint()`` with `multi_label=True` for details. + + pred_probs : np.ndarray + Predicted-probabilities in the same format expected by the `~cleanlab.count.get_confident_thresholds` function. + + Returns + ------- + num_issues : int + The estimated number of examples with label issues in the multi-label dataset. + + Note: We set the filter_by method as 'confident_learning' to match the non-multilabel case + (analog to the off_diagonal estimation method) + """ + + from cleanlab.filter import find_label_issues + + issues_idx = find_label_issues( + labels=labels, + pred_probs=pred_probs, + confident_joint=confident_joint, + multi_label=True, + filter_by="confident_learning", # specified to match num_label_issues + ) + return sum(issues_idx) + + +def _reduce_issues(pred_probs, labels): + """Returns a boolean mask denoting correct predictions or predictions within a margin around 0.5 for binary classification, suitable for filtering out indices in 'is_label_issue'.""" + pred_probs_copy = np.copy(pred_probs) # Make a copy of the original array + pred_probs_copy[np.arange(len(labels)), labels] += FLOATING_POINT_COMPARISON + pred = pred_probs_copy.argmax(axis=1) + mask = pred == labels + del pred_probs_copy # Delete copy + return mask + + +def calibrate_confident_joint( + confident_joint: np.ndarray, labels: LabelLike, *, multi_label: bool = False +) -> np.ndarray: + """Calibrates any confident joint estimate ``P(label=i, true_label=j)`` such that + ``np.sum(cj) == len(labels)`` and ``np.sum(cj, axis = 1) == np.bincount(labels)``. + + In other words, this function forces the confident joint to have the + true noisy prior ``p(labels)`` (summed over columns for each row) and also + forces the confident joint to add up to the total number of examples. + + This method makes the confident joint a valid counts estimate + of the actual joint of noisy and true labels. + + Parameters + ---------- + confident_joint : np.ndarray + An array of shape ``(K, K)`` representing the confident joint, the matrix used for identifying label issues, which + estimates a confident subset of the joint distribution of the noisy and true labels, ``P_{noisy label, true label}``. + Entry ``(j, k)`` in the matrix is the number of examples confidently counted into the pair of ``(noisy label=j, true label=k)`` classes. + The `confident_joint` can be computed using `~cleanlab.count.compute_confident_joint`. + If not provided, it is computed from the given (noisy) `labels` and `pred_probs`. + If `multi_label` is True, then the `confident_joint` should be a one-vs-rest array of shape ``(K, 2, 2)``, and an array of the same shape will be returned. + + labels : np.ndarray or list + Given class labels for each example in the dataset, some of which may be erroneous, + in same format expected by :py:func:`filter.find_label_issues ` function. + + multi_label : bool, optional + If ``False``, dataset is for regular (multi-class) classification, where each example belongs to exactly one class. + If ``True``, dataset is for multi-label classification, where each example can belong to multiple classes. + See documentation of `~cleanlab.count.compute_confident_joint` for details. + In multi-label classification, the confident/calibrated joint arrays have shape ``(K, 2, 2)`` + formatted in a one-vs-rest fashion such that they contain a 2x2 matrix for each class + that counts examples which are correctly/incorrectly labeled as belonging to that class. + After calibration, the entries in each class-specific 2x2 matrix will sum to the number of examples. + + Returns + ------- + calibrated_cj : np.ndarray + An array of shape ``(K, K)`` representing a valid estimate of the joint *counts* of noisy and true labels (if `multi_label` is False). + If `multi_label` is True, the returned `calibrated_cj` is instead an one-vs-rest array of shape ``(K, 2, 2)``, + where for class `c`: entry ``(c, 0, 0)`` in this one-vs-rest array is the number of examples whose noisy label contains `c` confidently identified as truly belonging to class `c` as well. + Entry ``(c, 1, 0)`` in this one-vs-rest array is the number of examples whose noisy label contains `c` confidently identified as not actually belonging to class `c`. + Entry ``(c, 0, 1)`` in this one-vs-rest array is the number of examples whose noisy label does not contain `c` confidently identified as truly belonging to class `c`. + Entry ``(c, 1, 1)`` in this one-vs-rest array is the number of examples whose noisy label does not contain `c` confidently identified as actually not belonging to class `c` as well. + + """ + + if multi_label: + if not isinstance(labels, list): + raise TypeError("`labels` must be list when `multi_label=True`.") + else: + return _calibrate_confident_joint_multilabel(confident_joint, labels) + else: + num_classes = len(confident_joint) + label_counts = value_counts_fill_missing_classes(labels, num_classes, multi_label=False) + # Calibrate confident joint to have correct p(labels) prior on noisy labels. + calibrated_cj = ( + confident_joint.T + / np.clip(confident_joint.sum(axis=1), a_min=TINY_VALUE, a_max=None) + * label_counts + ).T + # Calibrate confident joint to sum to: + # The number of examples (for single labeled datasets) + # The number of total labels (for multi-labeled datasets) + calibrated_cj = ( + calibrated_cj + / np.clip(np.sum(calibrated_cj), a_min=TINY_VALUE, a_max=None) + * sum(label_counts) + ) + return round_preserving_row_totals(calibrated_cj) + + +def _calibrate_confident_joint_multilabel(confident_joint: np.ndarray, labels: list) -> np.ndarray: + """Calibrates the confident joint for multi-label classification data. Here + input `labels` is a list of lists (or list of iterable). + This is intended as a helper function. You should probably + be using `calibrate_confident_joint(multi_label=True)` instead. + + + See `calibrate_confident_joint` docstring for more info. + + Parameters + ---------- + confident_joint : np.ndarray + Refer to documentation for this argument in count.calibrate_confident_joint() for details. + + labels : list + Refer to documentation for this argument in count.calibrate_confident_joint() for details. + + multi_label : bool, optional + Refer to documentation for this argument in count.calibrate_confident_joint() for details. + + Returns + ------- + calibrated_cj : np.ndarray + An array of shape ``(K, 2, 2)`` of type float representing a valid + estimate of the joint *counts* of noisy and true labels in a one-vs-rest fashion.""" + y_one, num_classes = get_onehot_num_classes(labels) + calibrate_confident_joint_list: np.ndarray = np.ndarray( + shape=(num_classes, 2, 2), dtype=np.int64 + ) + for class_num, (cj, y) in enumerate(zip(confident_joint, y_one.T)): + calibrate_confident_joint_list[class_num] = calibrate_confident_joint(cj, labels=y) + + return calibrate_confident_joint_list + + +def estimate_joint( + labels: LabelLike, + pred_probs: np.ndarray, + *, + confident_joint: Optional[np.ndarray] = None, + multi_label: bool = False, +) -> np.ndarray: + """ + Estimates the joint distribution of label noise ``P(label=i, true_label=j)`` guaranteed to: + + * Sum to 1 + * Satisfy ``np.sum(joint_estimate, axis = 1) == p(labels)`` + + Parameters + ---------- + labels : np.ndarray or list + Given class labels for each example in the dataset, some of which may be erroneous, + in same format expected by :py:func:`filter.find_label_issues ` function. + + pred_probs : np.ndarray + Model-predicted class probabilities for each example in the dataset, + in same format expected by :py:func:`filter.find_label_issues ` function. + + confident_joint : np.ndarray, optional + Array of estimated class label error statisics used for identifying label issues, + in same format expected by :py:func:`filter.find_label_issues ` function. + The `confident_joint` can be computed using `~cleanlab.count.compute_confident_joint`. + If not provided, it is internally computed from the given (noisy) `labels` and `pred_probs`. + + multi_label : bool, optional + If ``False``, dataset is for regular (multi-class) classification, where each example belongs to exactly one class. + If ``True``, dataset is for multi-label classification, where each example can belong to multiple classes. + See documentation of `~cleanlab.count.compute_confident_joint` for details. + + Returns + ------- + confident_joint_distribution : np.ndarray + An array of shape ``(K, K)`` representing an + estimate of the true joint distribution of noisy and true labels (if `multi_label` is False). + If `multi_label` is True, an array of shape ``(K, 2, 2)`` representing an + estimate of the true joint distribution of noisy and true labels for each class in a one-vs-rest fashion. + Entry ``(c, i, j)`` in this array is the number of examples confidently counted into a ``(class c, noisy label=i, true label=j)`` bin, + where `i, j` are either 0 or 1 to denote whether this example belongs to class `c` or not + (recall examples can belong to multiple classes in multi-label classification). + """ + + if confident_joint is None: + calibrated_cj = compute_confident_joint( + labels, + pred_probs, + calibrate=True, + multi_label=multi_label, + ) + else: + if labels is not None: + calibrated_cj = calibrate_confident_joint( + confident_joint, labels, multi_label=multi_label + ) + else: + calibrated_cj = confident_joint + + assert isinstance(calibrated_cj, np.ndarray) + if multi_label: + if not isinstance(labels, list): + raise TypeError("`labels` must be list when `multi_label=True`.") + else: + return _estimate_joint_multilabel( + labels=labels, pred_probs=pred_probs, confident_joint=confident_joint + ) + else: + return calibrated_cj / np.clip(float(np.sum(calibrated_cj)), a_min=TINY_VALUE, a_max=None) + + +def _estimate_joint_multilabel( + labels: list, pred_probs: np.ndarray, *, confident_joint: Optional[np.ndarray] = None +) -> np.ndarray: + """Parameters + ---------- + labels : list + Refer to documentation for this argument in filter.find_label_issues() for details. + + pred_probs : np.ndarray + Refer to documentation for this argument in count.estimate_joint() for details. + + confident_joint : np.ndarray, optional + Refer to documentation for this argument in filter.find_label_issues() with multi_label=True for details. + + Returns + ------- + confident_joint_distribution : np.ndarray + An array of shape ``(K, 2, 2)`` representing an + estimate of the true joint distribution of noisy and true labels for each class, in a one-vs-rest format employed for multi-label settings. + """ + y_one, num_classes = get_onehot_num_classes(labels, pred_probs) + if confident_joint is None: + calibrated_cj = compute_confident_joint( + labels, + pred_probs, + calibrate=True, + multi_label=True, + ) + else: + calibrated_cj = confident_joint + assert isinstance(calibrated_cj, np.ndarray) + calibrated_cf: np.ndarray = np.ndarray((num_classes, 2, 2)) + for class_num, (label, pred_prob_for_class) in enumerate(zip(y_one.T, pred_probs.T)): + pred_probs_binary = stack_complement(pred_prob_for_class) + calibrated_cf[class_num] = estimate_joint( + labels=label, + pred_probs=pred_probs_binary, + confident_joint=calibrated_cj[class_num], + ) + + return calibrated_cf + + +def compute_confident_joint( + labels: LabelLike, + pred_probs: np.ndarray, + *, + thresholds: Optional[Union[np.ndarray, list]] = None, + calibrate: bool = True, + multi_label: bool = False, + return_indices_of_off_diagonals: bool = False, +) -> Union[np.ndarray, Tuple[np.ndarray, list]]: + """Estimates the confident counts of latent true vs observed noisy labels + for the examples in our dataset. This array of shape ``(K, K)`` is called the **confident joint** + and contains counts of examples in every class, confidently labeled as every other class. + These counts may subsequently be used to estimate the joint distribution of true and noisy labels + (by normalizing them to frequencies). + + Important: this function assumes that `pred_probs` are out-of-sample + holdout probabilities. This can be :ref:`done with cross validation `. If + the probabilities are not computed out-of-sample, overfitting may occur. + + Parameters + ---------- + labels : np.ndarray or list + Given class labels for each example in the dataset, some of which may be erroneous, + in same format expected by :py:func:`filter.find_label_issues ` function. + + pred_probs : np.ndarray + Model-predicted class probabilities for each example in the dataset, + in same format expected by :py:func:`filter.find_label_issues ` function. + + thresholds : array_like, optional + An array of shape ``(K, 1)`` or ``(K,)`` of per-class threshold + probabilities, used to determine the cutoff probability necessary to + consider an example as a given class label (see `Northcutt et al., + 2021 `_, Section + 3.1, Equation 2). + + This is for advanced users only. If not specified, these are computed + for you automatically. If an example has a predicted probability + greater than this threshold, it is counted as having true_label = + k. This is not used for pruning/filtering, only for estimating the + noise rates using confident counts. + + calibrate : bool, default=True + Calibrates confident joint estimate ``P(label=i, true_label=j)`` such that + ``np.sum(cj) == len(labels)`` and ``np.sum(cj, axis = 1) == np.bincount(labels)``. + When ``calibrate=True``, this method returns an estimate of + the latent true joint counts of noisy and true labels. + + multi_label : bool, optional + If ``True``, this is multi-label classification dataset (where each example can belong to more than one class) + rather than a regular (multi-class) classifiction dataset. + In this case, `labels` should be an iterable (e.g. list) of iterables (e.g. ``List[List[int]]``), + containing the list of classes to which each example belongs, instead of just a single class. + Example of `labels` for a multi-label classification dataset: ``[[0,1], [1], [0,2], [0,1,2], [0], [1], [], ...]``. + + return_indices_of_off_diagonals : bool, optional + If ``True``, returns indices of examples that were counted in off-diagonals + of confident joint as a baseline proxy for the label issues. This + sometimes works as well as ``filter.find_label_issues(confident_joint)``. + + + Returns + ------- + confident_joint_counts : np.ndarray + An array of shape ``(K, K)`` representing counts of examples + for which we are confident about their given and true label (if `multi_label` is False). + If `multi_label` is True, + this array instead has shape ``(K, 2, 2)`` representing a one-vs-rest format for the confident joint, where for each class `c`: + Entry ``(c, 0, 0)`` in this one-vs-rest array is the number of examples whose noisy label contains `c` confidently identified as truly belonging to class `c` as well. + Entry ``(c, 1, 0)`` in this one-vs-rest array is the number of examples whose noisy label contains `c` confidently identified as not actually belonging to class `c`. + Entry ``(c, 0, 1)`` in this one-vs-rest array is the number of examples whose noisy label does not contain `c` confidently identified as truly belonging to class `c`. + Entry ``(c, 1, 1)`` in this one-vs-rest array is the number of examples whose noisy label does not contain `c` confidently identified as actually not belonging to class `c` as well. + + + Note + ---- + If `return_indices_of_off_diagonals` is set as True, this function instead returns a tuple `(confident_joint, indices_off_diagonal)` + where `indices_off_diagonal` is a list of arrays and each array contains the indices of examples counted in off-diagonals of confident joint. + + Note + ---- + We provide a for-loop based simplification of the confident joint + below. This implementation is not efficient, not used in practice, and + not complete, but covers the gist of how the confident joint is computed: + + .. code:: python + + # Confident examples are those that we are confident have true_label = k + # Estimate (K, K) matrix of confident examples with label = k_s and true_label = k_y + cj_ish = np.zeros((K, K)) + for k_s in range(K): # k_s is the class value k of noisy labels `s` + for k_y in range(K): # k_y is the (guessed) class k of true_label k_y + cj_ish[k_s][k_y] = sum((pred_probs[:,k_y] >= (thresholds[k_y] - 1e-8)) & (labels == k_s)) + + The following is a vectorized (but non-parallelized) implementation of the + confident joint, again slow, using for-loops/simplified for understanding. + This implementation is 100% accurate, it's just not optimized for speed. + + .. code:: python + + confident_joint = np.zeros((K, K), dtype = int) + for i, row in enumerate(pred_probs): + s_label = labels[i] + confident_bins = row >= thresholds - 1e-6 + num_confident_bins = sum(confident_bins) + if num_confident_bins == 1: + confident_joint[s_label][np.argmax(confident_bins)] += 1 + elif num_confident_bins > 1: + confident_joint[s_label][np.argmax(row)] += 1 + """ + + if multi_label: + if not isinstance(labels, list): + raise TypeError("`labels` must be list when `multi_label=True`.") + + return _compute_confident_joint_multi_label( + labels=labels, + pred_probs=pred_probs, + thresholds=thresholds, + calibrate=calibrate, + return_indices_of_off_diagonals=return_indices_of_off_diagonals, + ) + + # labels needs to be a numpy array + labels = np.asarray(labels) + + # Estimate the probability thresholds for confident counting + if thresholds is None: + # P(we predict the given noisy label is k | given noisy label is k) + thresholds = get_confident_thresholds(labels, pred_probs, multi_label=multi_label) + thresholds = np.asarray(thresholds) + + # Compute confident joint (vectorized for speed). + + # pred_probs_bool is a bool matrix where each row represents a training example as a boolean vector of + # size num_classes, with True if the example confidently belongs to that class and False if not. + pred_probs_bool = pred_probs >= thresholds - 1e-6 + num_confident_bins = pred_probs_bool.sum(axis=1) + # The indices where this is false, are often outliers (not confident of any label) + at_least_one_confident = num_confident_bins > 0 + more_than_one_confident = num_confident_bins > 1 + pred_probs_argmax = pred_probs.argmax(axis=1) + # Note that confident_argmax is meaningless for rows of all False + confident_argmax = pred_probs_bool.argmax(axis=1) + # For each example, choose the confident class (greater than threshold) + # When there is 2+ confident classes, choose the class with largest prob. + true_label_guess = np.where( + more_than_one_confident, + pred_probs_argmax, + confident_argmax, + ) + # true_labels_confident omits meaningless all-False rows + true_labels_confident = true_label_guess[at_least_one_confident] + labels_confident = labels[at_least_one_confident] + + # Handle case where no examples are confident (sklearn >=1.8.0 compatibility) + # In sklearn <1.8.0, confusion_matrix with empty inputs returned zeros matrix + # In sklearn >=1.8.0, it raises ValueError - we emulate the old behavior + if len(true_labels_confident) == 0: + confident_joint = np.zeros((pred_probs.shape[1], pred_probs.shape[1]), dtype=int) + else: + confident_joint = confusion_matrix( + y_true=true_labels_confident, + y_pred=labels_confident, + labels=range(pred_probs.shape[1]), + ).T + # Guarantee at least one correctly labeled example is represented in every class + np.fill_diagonal(confident_joint, confident_joint.diagonal().clip(min=1)) + if calibrate: + confident_joint = calibrate_confident_joint(confident_joint, labels) + + if return_indices_of_off_diagonals: + true_labels_neq_given_labels = true_labels_confident != labels_confident + indices = np.arange(len(labels))[at_least_one_confident][true_labels_neq_given_labels] + + return confident_joint, indices + + return confident_joint + + +def _compute_confident_joint_multi_label( + labels: list, + pred_probs: np.ndarray, + *, + thresholds: Optional[Union[np.ndarray, list]] = None, + calibrate: bool = True, + return_indices_of_off_diagonals: bool = False, +) -> Union[np.ndarray, Tuple[np.ndarray, list]]: + """Computes the confident joint for multi_labeled data. Thus, + input `labels` is a list of lists (or list of iterable). + This is intended as a helper function. You should probably + be using `compute_confident_joint(multi_label=True)` instead. + + The MAJOR DIFFERENCE in how this is computed versus single_label, + is the total number of errors considered is based on the number + of labels, not the number of examples. So, the confident_joint + will have larger values. + + See `compute_confident_joint` docstring for more info. + + Parameters + ---------- + labels : list of list/iterable (length N) + Given noisy labels for multi-label classification. + Must be a list of lists (or a list of np.ndarrays or iterable). + The i-th element is a list containing the classes that the i-th example belongs to. + + pred_probs : np.ndarray (shape (N, K)) + P(label=k|x) is a matrix with K model-predicted probabilities. + Each row of this matrix corresponds to an example `x` and contains the model-predicted + probabilities that `x` belongs to each possible class. + The columns must be ordered such that these probabilities correspond to class 0, 1, 2,..., K-1. + `pred_probs` must be out-of-sample (ideally should have been computed using 3+ fold cross-validation). + + thresholds : iterable (list or np.ndarray) of shape (K, 1) or (K,) + P(label^=k|label=k). If an example has a predicted probability "greater" than + this threshold, it is counted as having true_label = k. This is + not used for filtering/pruning, only for estimating the noise rates using + confident counts. This value should be between 0 and 1. Default is None. + + calibrate : bool, default = True + Calibrates confident joint estimate P(label=i, true_label=j) such that + ``np.sum(cj) == len(labels) and np.sum(cj, axis = 1) == np.bincount(labels)``. + + return_indices_of_off_diagonals: bool, default = False + If true returns indices of examples that were counted in off-diagonals + of confident joint as a baseline proxy for the label issues. This + sometimes works as well as filter.find_label_issues(confident_joint). + + Returns + ------- + confident_joint_counts : np.ndarray + An array of shape ``(K, 2, 2)`` representing the confident joint of noisy and true labels for each class, in a one-vs-rest format employed for multi-label settings. + + Note: if `return_indices_of_off_diagonals` is set as True, this function instead returns a tuple `(confident_joint_counts, indices_off_diagonal)` + where `indices_off_diagonal` is a list of arrays (one per class) and each array contains the indices of examples counted in off-diagonals of confident joint for that class. + """ + + y_one, num_classes = get_onehot_num_classes(labels, pred_probs) + confident_joint_list: np.ndarray = np.ndarray(shape=(num_classes, 2, 2), dtype=np.int64) + indices_off_diagonal = [] + for class_num, (label, pred_prob_for_class) in enumerate(zip(y_one.T, pred_probs.T)): + pred_probs_binary = stack_complement(pred_prob_for_class) + if return_indices_of_off_diagonals: + cj, ind = compute_confident_joint( + labels=label, + pred_probs=pred_probs_binary, + multi_label=False, + thresholds=thresholds, + calibrate=calibrate, + return_indices_of_off_diagonals=return_indices_of_off_diagonals, + ) + indices_off_diagonal.append(ind) + else: + cj = compute_confident_joint( + labels=label, + pred_probs=pred_probs_binary, + multi_label=False, + thresholds=thresholds, + calibrate=calibrate, + return_indices_of_off_diagonals=return_indices_of_off_diagonals, + ) + confident_joint_list[class_num] = cj + + if return_indices_of_off_diagonals: + return confident_joint_list, indices_off_diagonal + + return confident_joint_list + + +def estimate_latent( + confident_joint: np.ndarray, + labels: np.ndarray, + *, + py_method: str = "cnt", + converge_latent_estimates: bool = False, +) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """Computes the latent prior ``p(y)``, the noise matrix ``P(labels|y)`` and the + inverse noise matrix ``P(y|labels)`` from the `confident_joint` ``count(labels, y)``. The + `confident_joint` can be estimated by `~cleanlab.count.compute_confident_joint` + which counts confident examples. + + Parameters + ---------- + confident_joint : np.ndarray + An array of shape ``(K, K)`` representing the confident joint, the matrix used for identifying label issues, which + estimates a confident subset of the joint distribution of the noisy and true labels, ``P_{noisy label, true label}``. + Entry ``(j, k)`` in the matrix is the number of examples confidently counted into the pair of ``(noisy label=j, true label=k)`` classes. + The `confident_joint` can be computed using `~cleanlab.count.compute_confident_joint`. + If not provided, it is computed from the given (noisy) `labels` and `pred_probs`. + + labels : np.ndarray + A 1D array of shape ``(N,)`` containing class labels for a standard (multi-class) classification dataset. Some given labels may be erroneous. + Elements must be integers in the set 0, 1, ..., K-1, where K is the number of classes. + + py_method : {"cnt", "eqn", "marginal", "marginal_ps"}, default="cnt" + `py` is shorthand for the "class proportions (a.k.a prior) of the true labels". + This method defines how to compute the latent prior ``p(true_label=k)``. Default is ``"cnt"``, + which works well even when the noise matrices are estimated poorly by using + the matrix diagonals instead of all the probabilities. + + converge_latent_estimates : bool, optional + If ``True``, forces numerical consistency of estimates. Each is estimated + independently, but they are related mathematically with closed form + equivalences. This will iteratively make them mathematically consistent. + + Returns + ------ + tuple + A tuple containing (py, noise_matrix, inv_noise_matrix). + + Note + ---- + Multi-label classification is not supported in this method. + """ + + num_classes = len(confident_joint) + label_counts = value_counts_fill_missing_classes(labels, num_classes) + # 'ps' is p(labels=k) + ps = label_counts / float(len(labels)) + # Number of training examples confidently counted from each noisy class + labels_class_counts = confident_joint.sum(axis=1).astype(float) + # Number of training examples confidently counted into each true class + true_labels_class_counts = confident_joint.sum(axis=0).astype(float) + # p(label=k_s|true_label=k_y) ~ |label=k_s and true_label=k_y| / |true_label=k_y| + noise_matrix = confident_joint / np.clip(true_labels_class_counts, a_min=TINY_VALUE, a_max=None) + # p(true_label=k_y|label=k_s) ~ |true_label=k_y and label=k_s| / |label=k_s| + inv_noise_matrix = confident_joint.T / np.clip( + labels_class_counts, a_min=TINY_VALUE, a_max=None + ) + # Compute the prior p(y), the latent (uncorrupted) class distribution. + py = compute_py( + ps, + noise_matrix, + inv_noise_matrix, + py_method=py_method, + true_labels_class_counts=true_labels_class_counts, + ) + # Clip noise rates to be valid probabilities. + noise_matrix = clip_noise_rates(noise_matrix) + inv_noise_matrix = clip_noise_rates(inv_noise_matrix) + # Make latent estimates mathematically agree in their algebraic relations. + if converge_latent_estimates: + py, noise_matrix, inv_noise_matrix = _converge_estimates( + ps, py, noise_matrix, inv_noise_matrix + ) + # Again clip py and noise rates into proper range [0,1) + py = clip_values(py, low=1e-5, high=1.0, new_sum=1.0) + noise_matrix = clip_noise_rates(noise_matrix) + inv_noise_matrix = clip_noise_rates(inv_noise_matrix) + + return py, noise_matrix, inv_noise_matrix + + +def estimate_py_and_noise_matrices_from_probabilities( + labels: np.ndarray, + pred_probs: np.ndarray, + *, + thresholds: Optional[Union[np.ndarray, list]] = None, + converge_latent_estimates: bool = True, + py_method: str = "cnt", + calibrate: bool = True, +) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Computes the confident counts + estimate of latent variables `py` and the noise rates + using observed labels and predicted probabilities, `pred_probs`. + + Important: this function assumes that `pred_probs` are out-of-sample + holdout probabilities. This can be :ref:`done with cross validation `. If + the probabilities are not computed out-of-sample, overfitting may occur. + + This function estimates the `noise_matrix` of shape ``(K, K)``. This is the + fraction of examples in every class, labeled as every other class. The + `noise_matrix` is a conditional probability matrix for ``P(label=k_s|true_label=k_y)``. + + Under certain conditions, estimates are exact, and in most + conditions, estimates are within one percent of the actual noise rates. + + Parameters + ---------- + labels : np.ndarray + A 1D array of shape ``(N,)`` containing class labels for a standard (multi-class) classification dataset. Some given labels may be erroneous. + Elements must be integers in the set 0, 1, ..., K-1, where K is the number of classes. + + pred_probs : np.ndarray + Model-predicted class probabilities for each example in the dataset, + in same format expected by :py:func:`filter.find_label_issues ` function. + + thresholds : array_like, optional + An array of shape ``(K, 1)`` or ``(K,)`` of per-class threshold + probabilities, used to determine the cutoff probability necessary to + consider an example as a given class label (see `Northcutt et al., + 2021 `_, Section + 3.1, Equation 2). + + This is for advanced users only. If not specified, these are computed + for you automatically. If an example has a predicted probability + greater than this threshold, it is counted as having true_label = + k. This is not used for pruning/filtering, only for estimating the + noise rates using confident counts. + + converge_latent_estimates : bool, optional + If ``True``, forces numerical consistency of estimates. Each is estimated + independently, but they are related mathematically with closed form + equivalences. This will iteratively make them mathematically consistent. + + py_method : {"cnt", "eqn", "marginal", "marginal_ps"}, default="cnt" + How to compute the latent prior ``p(true_label=k)``. Default is ``"cnt"`` as it often + works well even when the noise matrices are estimated poorly by using + the matrix diagonals instead of all the probabilities. + + calibrate : bool, default=True + Calibrates confident joint estimate ``P(label=i, true_label=j)`` such that + ``np.sum(cj) == len(labels)`` and ``np.sum(cj, axis = 1) == np.bincount(labels)``. + + Returns + ------ + estimates : tuple + A tuple of arrays: (`py`, `noise_matrix`, `inverse_noise_matrix`, `confident_joint`). + + Note + ---- + Multi-label classification is not supported in this method. + """ + + confident_joint = compute_confident_joint( + labels=labels, + pred_probs=pred_probs, + thresholds=thresholds, + calibrate=calibrate, + ) + assert isinstance(confident_joint, np.ndarray) + py, noise_matrix, inv_noise_matrix = estimate_latent( + confident_joint=confident_joint, + labels=labels, + py_method=py_method, + converge_latent_estimates=converge_latent_estimates, + ) + assert isinstance(confident_joint, np.ndarray) + + return py, noise_matrix, inv_noise_matrix, confident_joint + + +def estimate_confident_joint_and_cv_pred_proba( + X, + labels, + clf=LogReg(solver="lbfgs"), + *, + cv_n_folds=5, + thresholds=None, + seed=None, + calibrate=True, + clf_kwargs={}, + validation_func=None, +) -> Tuple[np.ndarray, np.ndarray]: + """Estimates ``P(labels, y)``, the confident counts of the latent + joint distribution of true and noisy labels + using observed `labels` and predicted probabilities `pred_probs`. + + The output of this function is an array of shape ``(K, K)``. + + Under certain conditions, estimates are exact, and in many + conditions, estimates are within one percent of actual. + + Notes: There are two ways to compute the confident joint with pros/cons. + (1) For each holdout set, we compute the confident joint, then sum them up. + (2) Compute pred_proba for each fold, combine, compute the confident joint. + (1) is more accurate because it correctly computes thresholds for each fold + (2) is more accurate when you have only a little data because it computes + the confident joint using all the probabilities. For example if you had 100 + examples, with 5-fold cross validation + uniform p(y) you would only have 20 + examples to compute each confident joint for (1). Such small amounts of data + is bound to result in estimation errors. For this reason, we implement (2), + but we implement (1) as a commented out function at the end of this file. + + Parameters + ---------- + X : np.ndarray or pd.DataFrame + Input feature matrix of shape ``(N, ...)``, where N is the number of + examples. The classifier that this instance was initialized with, + ``clf``, must be able to fit() and predict() data with this format. + + labels : np.ndarray or pd.Series + A 1D array of shape ``(N,)`` containing class labels for a standard (multi-class) classification dataset. + Some given labels may be erroneous. + Elements must be integers in the set 0, 1, ..., K-1, where K is the number of classes. + All classes must be present in the dataset. + + clf : estimator instance, optional + A classifier implementing the `sklearn estimator API + `_. + + cv_n_folds : int, default=5 + The number of cross-validation folds used to compute + out-of-sample predicted probabilities for each example in `X`. + + thresholds : array_like, optional + An array of shape ``(K, 1)`` or ``(K,)`` of per-class threshold + probabilities, used to determine the cutoff probability necessary to + consider an example as a given class label (see `Northcutt et al., + 2021 `_, Section + 3.1, Equation 2). + + This is for advanced users only. If not specified, these are computed + for you automatically. If an example has a predicted probability + greater than this threshold, it is counted as having true_label = + k. This is not used for pruning/filtering, only for estimating the + noise rates using confident counts. + + seed : int, optional + Set the default state of the random number generator used to split + the cross-validated folds. If None, uses np.random current random state. + + calibrate : bool, default=True + Calibrates confident joint estimate ``P(label=i, true_label=j)`` such that + ``np.sum(cj) == len(labels)`` and ``np.sum(cj, axis = 1) == np.bincount(labels)``. + + clf_kwargs : dict, optional + Optional keyword arguments to pass into `clf`'s ``fit()`` method. + + validation_func : callable, optional + Specifies how to map the validation data split in cross-validation as input for ``clf.fit()``. + For details, see the documentation of :py:meth:`CleanLearning.fit` + + Returns + ------ + estimates : tuple + Tuple of two numpy arrays in the form: + (joint counts matrix, predicted probability matrix) + + Note + ---- + Multi-label classification is not supported in this method. + """ + + assert_valid_inputs(X, labels) + labels = labels_to_array(labels) + num_classes = get_num_classes( + labels=labels + ) # This method definitely only works if all classes are present. + + # Create cross-validation object for out-of-sample predicted probabilities. + # CV folds preserve the fraction of noisy positive and + # noisy negative examples in each class. + kf = StratifiedKFold(n_splits=cv_n_folds, shuffle=True, random_state=seed) + + # Initialize pred_probs array + pred_probs = np.zeros(shape=(len(labels), num_classes)) + + # Split X and labels into "cv_n_folds" stratified folds. + # CV indices only require labels: https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.StratifiedKFold.html + # Only split based on labels because X may have various formats: + for k, (cv_train_idx, cv_holdout_idx) in enumerate(kf.split(X=labels, y=labels)): + try: + clf_copy = sklearn.base.clone(clf) # fresh untrained copy of the model + except Exception: + raise ValueError( + "`clf` must be clonable via: sklearn.base.clone(clf). " + "You can either implement instance method `clf.get_params()` to produce a fresh untrained copy of this model, " + "or you can implement the cross-validation outside of cleanlab " + "and pass in the obtained `pred_probs` to skip cleanlab's internal cross-validation" + ) + # Select the training and holdout cross-validated sets. + X_train_cv, X_holdout_cv, s_train_cv, s_holdout_cv = train_val_split( + X, labels, cv_train_idx, cv_holdout_idx + ) + + # dict with keys: which classes missing, values: index of holdout data from this class that is duplicated: + missing_class_inds = {} + if not is_torch_dataset(X): + # Ensure no missing classes in training set. + train_cv_classes = set(s_train_cv) + all_classes = set(range(num_classes)) + if len(train_cv_classes) != len(all_classes): + missing_classes = all_classes.difference(train_cv_classes) + warnings.warn( + "Duplicated some data across multiple folds to ensure training does not fail " + f"because these classes do not have enough data for proper cross-validation: {missing_classes}." + ) + for missing_class in missing_classes: + # Duplicate one instance of missing_class from holdout data to the training data: + holdout_inds = np.where(s_holdout_cv == missing_class)[0] + dup_idx = holdout_inds[0] + s_train_cv = np.append(s_train_cv, s_holdout_cv[dup_idx]) + # labels are always np.ndarray so don't have to consider .iloc above + X_train_cv = append_extra_datapoint( + to_data=X_train_cv, from_data=X_holdout_cv, index=dup_idx + ) + missing_class_inds[missing_class] = dup_idx + + # Map validation data into appropriate format to pass into classifier clf + if validation_func is None: + validation_kwargs = {} + elif callable(validation_func): + validation_kwargs = validation_func(X_holdout_cv, s_holdout_cv) + else: + raise TypeError("validation_func must be callable function with args: X_val, y_val") + + # Fit classifier clf to training set, predict on holdout set, and update pred_probs. + clf_copy.fit(X_train_cv, s_train_cv, **clf_kwargs, **validation_kwargs) + pred_probs_cv = clf_copy.predict_proba(X_holdout_cv) # P(labels = k|x) # [:,1] + + # Replace predictions for duplicated indices with dummy predictions: + for missing_class in missing_class_inds: + dummy_pred = np.zeros(pred_probs_cv[0].shape) + dummy_pred[missing_class] = 1.0 # predict given label with full confidence + dup_idx = missing_class_inds[missing_class] + pred_probs_cv[dup_idx] = dummy_pred + + pred_probs[cv_holdout_idx] = pred_probs_cv + + # Compute the confident counts, a num_classes x num_classes matrix for all pairs of labels. + confident_joint = compute_confident_joint( + labels=labels, + pred_probs=pred_probs, # P(labels = k|x) + thresholds=thresholds, + calibrate=calibrate, + ) + assert isinstance(confident_joint, np.ndarray) + assert isinstance(pred_probs, np.ndarray) + + return confident_joint, pred_probs + + +def estimate_py_noise_matrices_and_cv_pred_proba( + X, + labels, + clf=LogReg(solver="lbfgs"), + *, + cv_n_folds=5, + thresholds=None, + converge_latent_estimates=False, + py_method="cnt", + seed=None, + clf_kwargs={}, + validation_func=None, +) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """This function computes the out-of-sample predicted + probability ``P(label=k|x)`` for every example x in `X` using cross + validation while also computing the confident counts noise + rates within each cross-validated subset and returning + the average noise rate across all examples. + + This function estimates the `noise_matrix` of shape ``(K, K)``. This is the + fraction of examples in every class, labeled as every other class. The + `noise_matrix` is a conditional probability matrix for ``P(label=k_s|true_label=k_y)``. + + Under certain conditions, estimates are exact, and in most + conditions, estimates are within one percent of the actual noise rates. + + Parameters + ---------- + X : np.ndarray + Input feature matrix of shape ``(N, ...)``, where N is the number of + examples. The classifier that this instance was initialized with, + `clf`, must be able to handle data with this shape. + + labels : np.ndarray + A 1D array of shape ``(N,)`` containing class labels for a standard (multi-class) classification dataset. + Some given labels may be erroneous. + Elements must be integers in the set 0, 1, ..., K-1, where K is the number of classes. + All classes must be present in the dataset. + + clf : estimator instance, optional + A classifier implementing the `sklearn estimator API + `_. + + cv_n_folds : int, default=5 + The number of cross-validation folds used to compute + out-of-sample probabilities for each example in `X`. + + thresholds : array_like, optional + An array of shape ``(K, 1)`` or ``(K,)`` of per-class threshold + probabilities, used to determine the cutoff probability necessary to + consider an example as a given class label (see `Northcutt et al., + 2021 `_, Section + 3.1, Equation 2). + + This is for advanced users only. If not specified, these are computed + for you automatically. If an example has a predicted probability + greater than this threshold, it is counted as having true_label = + k. This is not used for pruning/filtering, only for estimating the + noise rates using confident counts. + + converge_latent_estimates : bool, optional + If ``True``, forces numerical consistency of estimates. Each is estimated + independently, but they are related mathematically with closed form + equivalences. This will iteratively make them mathematically consistent. + + py_method : {"cnt", "eqn", "marginal", "marginal_ps"}, default="cnt" + How to compute the latent prior ``p(true_label=k)``. Default is ``"cnt"`` as it often + works well even when the noise matrices are estimated poorly by using + the matrix diagonals instead of all the probabilities. + + seed : int, optional + Set the default state of the random number generator used to split + the cross-validated folds. If ``None``, uses ``np.random`` current random state. + + clf_kwargs : dict, optional + Optional keyword arguments to pass into `clf`'s ``fit()`` method. + + validation_func : callable, optional + Specifies how to map the validation data split in cross-validation as input for ``clf.fit()``. + For details, see the documentation of :py:meth:`CleanLearning.fit` + + Returns + ------ + estimates: tuple + A tuple of five arrays (py, noise matrix, inverse noise matrix, confident joint, predicted probability matrix). + + Note + ---- + Multi-label classification is not supported in this method. + """ + confident_joint, pred_probs = estimate_confident_joint_and_cv_pred_proba( + X=X, + labels=labels, + clf=clf, + cv_n_folds=cv_n_folds, + thresholds=thresholds, + seed=seed, + clf_kwargs=clf_kwargs, + validation_func=validation_func, + ) + + py, noise_matrix, inv_noise_matrix = estimate_latent( + confident_joint=confident_joint, + labels=labels, + py_method=py_method, + converge_latent_estimates=converge_latent_estimates, + ) + + return py, noise_matrix, inv_noise_matrix, confident_joint, pred_probs + + +def estimate_cv_predicted_probabilities( + X, + labels, + clf=LogReg(solver="lbfgs"), + *, + cv_n_folds=5, + seed=None, + clf_kwargs={}, + validation_func=None, +) -> np.ndarray: + """This function computes the out-of-sample predicted + probability [P(label=k|x)] for every example in X using cross + validation. Output is a np.ndarray of shape ``(N, K)`` where N is + the number of training examples and K is the number of classes. + + Parameters + ---------- + X : np.ndarray + Input feature matrix of shape ``(N, ...)``, where N is the number of + examples. The classifier that this instance was initialized with, + `clf`, must be able to handle data with this shape. + + labels : np.ndarray + A 1D array of shape ``(N,)`` containing class labels for a standard (multi-class) classification dataset. + Some given labels may be erroneous. + Elements must be integers in the set 0, 1, ..., K-1, where K is the number of classes. + All classes must be present in the dataset. + + clf : estimator instance, optional + A classifier implementing the `sklearn estimator API + `_. + + cv_n_folds : int, default=5 + The number of cross-validation folds used to compute + out-of-sample probabilities for each example in `X`. + + seed : int, optional + Set the default state of the random number generator used to split + the cross-validated folds. If ``None``, uses ``np.random`` current random state. + + clf_kwargs : dict, optional + Optional keyword arguments to pass into `clf`'s ``fit()`` method. + + validation_func : callable, optional + Specifies how to map the validation data split in cross-validation as input for ``clf.fit()``. + For details, see the documentation of :py:meth:`CleanLearning.fit` + + Returns + -------- + pred_probs : np.ndarray + An array of shape ``(N, K)`` representing ``P(label=k|x)``, the model-predicted probabilities. + Each row of this matrix corresponds to an example `x` and contains the model-predicted + probabilities that `x` belongs to each possible class. + """ + + return estimate_py_noise_matrices_and_cv_pred_proba( + X=X, + labels=labels, + clf=clf, + cv_n_folds=cv_n_folds, + seed=seed, + clf_kwargs=clf_kwargs, + validation_func=validation_func, + )[-1] + + +def estimate_noise_matrices( + X, + labels, + clf=LogReg(solver="lbfgs"), + *, + cv_n_folds=5, + thresholds=None, + converge_latent_estimates=True, + seed=None, + clf_kwargs={}, + validation_func=None, +) -> Tuple[np.ndarray, np.ndarray]: + """Estimates the `noise_matrix` of shape ``(K, K)``. This is the + fraction of examples in every class, labeled as every other class. The + `noise_matrix` is a conditional probability matrix for ``P(label=k_s|true_label=k_y)``. + + Under certain conditions, estimates are exact, and in most + conditions, estimates are within one percent of the actual noise rates. + + Parameters + ---------- + X : np.ndarray + Input feature matrix of shape ``(N, ...)``, where N is the number of + examples. The classifier that this instance was initialized with, + `clf`, must be able to handle data with this shape. + + labels : np.ndarray + An array of shape ``(N,)`` of noisy labels, i.e. some labels may be erroneous. + Elements must be integers in the set 0, 1, ..., K-1, where K is the number of classes. + + clf : estimator instance, optional + A classifier implementing the `sklearn estimator API + `_. + + cv_n_folds : int, default=5 + The number of cross-validation folds used to compute + out-of-sample probabilities for each example in `X`. + + thresholds : array_like, optional + An array of shape ``(K, 1)`` or ``(K,)`` of per-class threshold + probabilities, used to determine the cutoff probability necessary to + consider an example as a given class label (see `Northcutt et al., + 2021 `_, Section + 3.1, Equation 2). + + This is for advanced users only. If not specified, these are computed + for you automatically. If an example has a predicted probability + greater than this threshold, it is counted as having true_label = + k. This is not used for pruning/filtering, only for estimating the + noise rates using confident counts. + + converge_latent_estimates : bool, optional + If ``True``, forces numerical consistency of estimates. Each is estimated + independently, but they are related mathematically with closed form + equivalences. This will iteratively make them mathematically consistent. + + seed : int, optional + Set the default state of the random number generator used to split + the cross-validated folds. If None, uses np.random current random state. + + clf_kwargs : dict, optional + Optional keyword arguments to pass into `clf`'s ``fit()`` method. + + validation_func : callable, optional + Specifies how to map the validation data split in cross-validation as input for ``clf.fit()``. + For details, see the documentation of :py:meth:`CleanLearning.fit` + + Returns + ------ + estimates : tuple + A tuple containing arrays (`noise_matrix`, `inv_noise_matrix`).""" + + return estimate_py_noise_matrices_and_cv_pred_proba( + X=X, + labels=labels, + clf=clf, + cv_n_folds=cv_n_folds, + thresholds=thresholds, + converge_latent_estimates=converge_latent_estimates, + seed=seed, + clf_kwargs=clf_kwargs, + validation_func=validation_func, + )[1:-2] + + +def _converge_estimates( + ps: np.ndarray, + py: np.ndarray, + noise_matrix: np.ndarray, + inverse_noise_matrix: np.ndarray, + *, + inv_noise_matrix_iterations: int = 5, + noise_matrix_iterations: int = 3, +) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """Updates py := P(true_label=k) and both `noise_matrix` and `inverse_noise_matrix` + to be numerically consistent with each other, by iteratively updating their estimates based on + the mathematical relationships between them. + + Forces numerical consistency of estimates. Each is estimated + independently, but they are related mathematically with closed form + equivalences. This will iteratively make them mathematically consistent. + + py := P(true_label=k) and the inverse noise matrix P(true_label=k_y|label=k_s) specify one + another, meaning one can be computed from the other and vice versa. + When numerical discrepancy exists due to poor estimation, they can be made + to agree by repeatedly computing one from the other, + for some a certain number of iterations (3-10 works fine.) + + Do not set iterations too high or performance will decrease as small + deviations will get perturbed over and over and potentially magnified. + + Note that we have to first converge the inverse_noise_matrix and py, + then we can update the noise_matrix, then repeat. This is because the + inverse noise matrix depends on py (which is unknown/latent), but the + noise matrix depends on ps (which is known), so there will be no change in + the noise matrix if we recompute it when py and inverse_noise_matrix change. + + + Parameters + ---------- + ps : np.ndarray (shape (K, ) or (1, K)) + The fraction (prior probability) of each observed, NOISY class P(labels = k). + + py : np.ndarray (shape (K, ) or (1, K)) + The estimated fraction (prior probability) of each TRUE class P(true_label = k). + + noise_matrix : np.ndarray of shape (K, K), K = number of classes + A conditional probability matrix of the form P(label=k_s|true_label=k_y) containing + the fraction of examples in every class, labeled as every other class. + Assumes columns of noise_matrix sum to 1. + + inverse_noise_matrix : np.ndarray of shape (K, K), K = number of classes + A conditional probability matrix of the form P(true_label=k_y|labels=k_s) representing + the estimated fraction observed examples in each class k_s, that are + mislabeled examples from every other class k_y. If None, the + inverse_noise_matrix will be computed from pred_probs and labels. + Assumes columns of inverse_noise_matrix sum to 1. + + inv_noise_matrix_iterations : int, default = 5 + Number of times to converge inverse noise matrix with py and noise mat. + + noise_matrix_iterations : int, default = 3 + Number of times to converge noise matrix with py and inverse noise mat. + + Returns + ------ + estimates: tuple + Three arrays of the form (`py`, `noise_matrix`, `inverse_noise_matrix`) all + having numerical agreement in terms of their mathematical relations.""" + + for j in range(noise_matrix_iterations): + for i in range(inv_noise_matrix_iterations): + inverse_noise_matrix = compute_inv_noise_matrix(py=py, noise_matrix=noise_matrix, ps=ps) + py = compute_py(ps, noise_matrix, inverse_noise_matrix) + noise_matrix = compute_noise_matrix_from_inverse( + ps=ps, inverse_noise_matrix=inverse_noise_matrix, py=py + ) + + return py, noise_matrix, inverse_noise_matrix + + +def get_confident_thresholds( + labels: LabelLike, + pred_probs: np.ndarray, + multi_label: bool = False, +) -> np.ndarray: + """Returns expected (average) "self-confidence" for each class. + + The confident class threshold for a class j is the expected (average) "self-confidence" for class j, + i.e. the model-predicted probability of this class averaged amongst all examples labeled as class j. + + Parameters + ---------- + labels : np.ndarray or list + Given class labels for each example in the dataset, some of which may be erroneous, + in same format expected by :py:func:`filter.find_label_issues ` function. + + pred_probs : np.ndarray + Model-predicted class probabilities for each example in the dataset, + in same format expected by :py:func:`filter.find_label_issues ` function. + + multi_label : bool, default = False + Set ``False`` if your dataset is for regular (multi-class) classification, where each example belongs to exactly one class. + Set ``True`` if your dataset is for multi-label classification, where each example can belong to multiple classes. + See documentation of `~cleanlab.count.compute_confident_joint` for details. + + Returns + ------- + confident_thresholds : np.ndarray + An array of shape ``(K, )`` where K is the number of classes. + """ + if multi_label: + assert isinstance(labels, list) + return _get_confident_thresholds_multilabel(labels=labels, pred_probs=pred_probs) + else: + # When all_classes != unique_classes the class threshold for the missing classes is set to + # BIG_VALUE such that no valid prob >= BIG_VALUE (no example will be counted in missing classes) + # REQUIRES: pred_probs.max() >= 1 + # TODO: if you want this to work for arbitrary softmax outputs where pred_probs.max() + # may exceed 1, change BIG_VALUE = 2 --> BIG_VALUE = 2 * pred_probs.max(). Downside of + # this approach is that there will be no standard value returned for missing classes. + labels = labels_to_array(labels) + all_classes = range(pred_probs.shape[1]) + unique_classes = get_unique_classes(labels, multi_label=multi_label) + BIG_VALUE = 2 + confident_thresholds = [ + np.mean(pred_probs[:, k][labels == k]) if k in unique_classes else BIG_VALUE + for k in all_classes + ] + confident_thresholds = np.clip( + confident_thresholds, a_min=CONFIDENT_THRESHOLDS_LOWER_BOUND, a_max=None + ) + return confident_thresholds + + +def _get_confident_thresholds_multilabel( + labels: list, + pred_probs: np.ndarray, +): + """Returns expected (average) "self-confidence" for each class. + + The confident class threshold for a class j is the expected (average) "self-confidence" for class j in a one-vs-rest setting. + + Parameters + ---------- + labels: list + Refer to documentation for this argument in ``count.calibrate_confident_joint()`` with ``multi_label=True`` for details. + + pred_probs : np.ndarray + Predicted class probabilities in the same format expected by the `~cleanlab.count.get_confident_thresholds` function. + + Returns + ------- + confident_thresholds : np.ndarray + An array of shape ``(K, 2, 2)`` where `K` is the number of classes, in a one-vs-rest format. + """ + y_one, num_classes = get_onehot_num_classes(labels, pred_probs) + confident_thresholds: np.ndarray = np.ndarray((num_classes, 2)) + for class_num, (label_for_class, pred_prob_for_class) in enumerate(zip(y_one.T, pred_probs.T)): + pred_probs_binary = stack_complement(pred_prob_for_class) + confident_thresholds[class_num] = get_confident_thresholds( + pred_probs=pred_probs_binary, labels=label_for_class + ) + return confident_thresholds diff --git a/cleanlab/data_valuation.py b/cleanlab/data_valuation.py new file mode 100644 index 0000000..abe16a4 --- /dev/null +++ b/cleanlab/data_valuation.py @@ -0,0 +1,127 @@ +""" +Methods for quantifying the value of each data point in a Machine Learning dataset. +Data Valuation helps us assess individual training data points' contributions to a ML model's predictive performance. +""" + +from typing import Callable, Optional, Union + +import numpy as np +from scipy.sparse import csr_matrix + +from cleanlab.internal.neighbor.knn_graph import create_knn_graph_and_index + + +def _knn_shapley_score(neighbor_indices: np.ndarray, y: np.ndarray, k: int) -> np.ndarray: + """Compute the Data Shapley values of data points using neighbor indices in a K-Nearest Neighbors (KNN) graph. + + This function leverages equations (18) and (19) from the paper available at https://arxiv.org/abs/1908.08619 + for computational efficiency. + + Parameters + ---------- + neighbor_indices : + A 2D array where each row contains the indices of the k-nearest neighbors for each data point. + y : + A 1D array of target values corresponding to the data points. + k : + The number of nearest neighbors to consider for each data point. + + Notes + ----- + - The training set is used as its own test set for the KNN-Shapley value computation, meaning y_test is the same as y_train. + - `neighbor_indices` are assumed to be pre-sorted by distance, with the nearest neighbors appearing first, and with at least `k` neighbors. + - Unlike the referenced paper, this implementation does not account for an upper error bound epsilon. + Consequently, K* is treated as equal to K instead of K* = max(K, 1/epsilon). + - This simplification implies that the term min(K, j + 1) will always be j + 1, which is offset by the + corresponding denominator term in the inner loop. + - Dividing by K in the end achieves the same result as dividing by K* in the paper. + - The pre-allocated `scores` array incorporates equation (18) for j = k - 1, ensuring efficient computation. + """ + N = y.shape[0] + scores = np.zeros((N, N)) + + for y_alpha, s_alpha, idx in zip(y, scores, neighbor_indices): + y_neighbors = y[idx] + ans_matches = (y_neighbors == y_alpha).flatten() + for j in range(k - 2, -1, -1): + s_alpha[idx[j]] = s_alpha[idx[j + 1]] + float( + int(ans_matches[j]) - int(ans_matches[j + 1]) + ) + return np.mean(scores / k, axis=0) + + +def data_shapley_knn( + labels: np.ndarray, + *, + features: Optional[np.ndarray] = None, + knn_graph: Optional[csr_matrix] = None, + metric: Optional[Union[str, Callable]] = None, + k: int = 10, +) -> np.ndarray: + """ + Compute the Data Shapley values of data points using a K-Nearest Neighbors (KNN) graph. + + This function calculates the contribution (Data Shapley value) of each data point in a dataset + for model training, either directly from data features or using a precomputed KNN graph. + + The examples in the dataset with lowest data valuation scores contribute least + to a trained ML model’s performance (those whose value falls below a threshold are flagged with this type of issue). + The data valuation score is an approximate Data Shapley value, calculated based on the labels of the top k nearest neighbors of an example. Details on this KNN-Shapley value can be found in these papers: + https://arxiv.org/abs/1908.08619 and https://arxiv.org/abs/1911.07128. + + Parameters + ---------- + labels : + An array of labels for the data points(only for multi-class classification datasets). + features : + Feature embeddings (vector representations) of every example in the dataset. + + Necessary if `knn_graph` is not supplied. + + If provided, this must be a 2D array with shape (num_examples, num_features). + knn_graph : + A precomputed sparse KNN graph. If not provided, it will be computed from the `features` using the specified `metric`. + metric : Optional[str or Callable], default=None + The distance metric for KNN graph construction. + Supports metrics available in ``sklearn.neighbors.NearestNeighbors`` + Default metric is ``"cosine"`` for ``dim(features) > 3``, otherwise ``"euclidean"`` for lower-dimensional data. + The euclidean is computed with an efficient implementation from scikit-learn when the number of examples is greater than 100. + When the number of examples is 100 or fewer, a more numerically stable version of the euclidean distance from scipy is used. + k : + The number of neighbors to consider for the KNN graph and Data Shapley value computation. + Must be less than the total number of data points. + The value may not exceed the number of neighbors of each data point stored in the KNN graph. + + Returns + ------- + scores : + An array of transformed Data Shapley values for each data point, calibrated to indicate their relative importance. + These scores have been adjusted to fall within 0 to 1. + Values closer to 1 indicate data points that are highly influential and positively contribute to a trained ML model's performance. + Conversely, scores below 0.5 indicate data points estimated to negatively impact model performance. + + Raises + ------ + ValueError + If neither `knn_graph` nor `features` are provided, or if `k` is larger than the number of examples in `features`. + + Examples + -------- + >>> import numpy as np + >>> from cleanlab.data_valuation import data_shapley_knn + >>> labels = np.array([0, 1, 0, 1, 0]) + >>> features = np.array([[0, 1, 2, 3, 4]]).T + >>> data_shapley_knn(labels=labels, features=features, k=4) + array([0.55 , 0.525, 0.55 , 0.525, 0.55 ]) + """ + if knn_graph is None and features is None: + raise ValueError("Either knn_graph or features must be provided.") + + # Use provided knn_graph or compute it from features + if knn_graph is None: + knn_graph, _ = create_knn_graph_and_index(features, n_neighbors=k, metric=metric) + + num_examples = labels.shape[0] + distances = knn_graph.indices.reshape(num_examples, -1) + scores = _knn_shapley_score(neighbor_indices=distances, y=labels, k=k) + return 0.5 * (scores + 1) diff --git a/cleanlab/datalab/__init__.py b/cleanlab/datalab/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cleanlab/datalab/datalab.py b/cleanlab/datalab/datalab.py new file mode 100644 index 0000000..8aebf48 --- /dev/null +++ b/cleanlab/datalab/datalab.py @@ -0,0 +1,623 @@ +""" +Datalab offers a unified audit to detect all kinds of issues in data and labels. + +.. note:: + .. include:: optional_dependencies.rst +""" + +from __future__ import annotations + +import warnings +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +import numpy as np +import pandas as pd + +import cleanlab +from cleanlab.datalab.internal.adapter.constants import DEFAULT_CLEANVISION_ISSUES +from cleanlab.datalab.internal.adapter.imagelab import create_imagelab +from cleanlab.datalab.internal.data import Data +from cleanlab.datalab.internal.display import _Displayer +from cleanlab.datalab.internal.helper_factory import ( + _DataIssuesBuilder, + issue_finder_factory, + report_factory, +) +from cleanlab.datalab.internal.issue_manager_factory import ( + list_default_issue_types as _list_default_issue_types, + list_possible_issue_types as _list_possible_issue_types, +) +from cleanlab.datalab.internal.serialize import _Serializer +from cleanlab.datalab.internal.task import Task + +if TYPE_CHECKING: # pragma: no cover + import numpy.typing as npt + from datasets.arrow_dataset import Dataset + from scipy.sparse import csr_matrix + + DatasetLike = Union[Dataset, pd.DataFrame, Dict[str, Any], List[Dict[str, Any]], str] + + +__all__ = ["Datalab"] + + +class Datalab: + """ + A single object to automatically detect all kinds of issues in datasets. + This is how we recommend you interface with the cleanlab library if you want to audit the quality of your data and detect issues within it. + If you have other specific goals (or are doing a less standard ML task not supported by Datalab), then consider using the other methods across the library. + Datalab tracks intermediate state (e.g. data statistics) from certain cleanlab functions that can be re-used across other cleanlab functions for better efficiency. + + Parameters + ---------- + data : Union[Dataset, pd.DataFrame, dict, list, str] + Dataset-like object that can be converted to a Hugging Face Dataset object. + + It should contain the labels for all examples, identified by a + `label_name` column in the Dataset object. + + Supported formats: + - datasets.Dataset + - pandas.DataFrame + - dict (keys are strings, values are arrays/lists of length ``N``) + - list (list of dictionaries that each have the same keys) + - str + + - path to a local file: Text (.txt), CSV (.csv), JSON (.json) + - or a dataset identifier on the Hugging Face Hub + + task : str + The type of machine learning task that the dataset is used for. + + Supported tasks: + - "classification" (default): Multiclass classification + - "regression" : Regression + - "multilabel" : Multilabel classification + + label_name : str, optional + The name of the label column in the dataset. + + image_key : str, optional + Optional key that can be specified for image datasets to point to the field (column) containing the actual images themselves (as PIL objects). + If specified, additional image-specific issue types will be checked for in the dataset. + See the `CleanVision package `_ for descriptions of these image-specific issue types. + Currently, this argument is only supported for data formatted as a Hugging Face ``datasets.Dataset`` object. + + + verbosity : int, optional + The higher the verbosity level, the more information + Datalab prints when auditing a dataset. + Valid values are 0 through 4. Default is 1. + + Examples + -------- + >>> import datasets + >>> from cleanlab import Datalab + >>> data = datasets.load_dataset("glue", "sst2", split="train") + >>> datalab = Datalab(data, label_name="label") + """ + + def __init__( + self, + data: "DatasetLike", + task: str = "classification", + label_name: Optional[str] = None, + image_key: Optional[str] = None, + verbosity: int = 1, + ) -> None: + # Assume continuous values of labels for regression task + # Map labels to integers for classification task + self.task = Task.from_str(task) + self._data = Data(data, self.task, label_name) + self.data = self._data._data + self._labels = self._data.labels + self._label_map = self._labels.label_map + self.label_name = self._labels.label_name + self._data_hash = self._data._data_hash + self.cleanlab_version = cleanlab.version.__version__ + self.verbosity = verbosity + self._imagelab = create_imagelab(dataset=self.data, image_key=image_key) + + # Create the builder for DataIssues + builder = _DataIssuesBuilder(self._data) + builder.set_imagelab(self._imagelab).set_task(self.task) + self.data_issues = builder.build() + + # todo: check displayer methods + def __repr__(self) -> str: + return _Displayer(data_issues=self.data_issues, task=self.task).__repr__() + + def __str__(self) -> str: + return _Displayer(data_issues=self.data_issues, task=self.task).__str__() + + @property + def labels(self) -> Union[np.ndarray, List[List[int]]]: + """Labels of the dataset, in a [0, 1, ..., K-1] format.""" + return self._labels.labels + + @property + def has_labels(self) -> bool: + """Whether the dataset has labels, and that they are in a [0, 1, ..., K-1] format.""" + return self._labels.is_available + + @property + def class_names(self) -> List[str]: + """Names of the classes in the dataset. + + If the dataset has no labels, returns an empty list. + """ + return self._labels.class_names + + def find_issues( + self, + *, + pred_probs: Optional[np.ndarray] = None, + features: Optional[npt.NDArray] = None, + knn_graph: Optional[csr_matrix] = None, + issue_types: Optional[Dict[str, Any]] = None, + ) -> None: + """ + Checks the dataset for all sorts of common issues in real-world data (in both labels and feature values). + + You can use Datalab to find issues in your data, utilizing *any* model you have already trained. + This method only interacts with your model via its predictions or embeddings (and other functions thereof). + The more of these inputs you provide, the more types of issues Datalab can detect in your dataset/labels. + If you provide a subset of these inputs, Datalab will output what insights it can based on the limited information from your model. + + NOTE + ---- + The issues are saved in the ``self.issues`` attribute of the ``Datalab`` object, but are not returned. + + Parameters + ---------- + pred_probs : + Out-of-sample predicted class probabilities made by the model for every example in the dataset. + To best detect label issues, provide this input obtained from the most accurate model you can produce. + + For classification data, this must be a 2D array with shape ``(num_examples, K)`` where ``K`` is the number of classes in the dataset. + Make sure that the columns of your `pred_probs` are properly ordered with respect to the ordering of classes, which for Datalab is: lexicographically sorted by class name. + + For regression data, this must be a 1D array with shape ``(num_examples,)`` containing the predicted value for each example. + + For multilabel classification data, this must be a 2D array with shape ``(num_examples, K)`` where ``K`` is the number of classes in the dataset. + Make sure that the columns of your `pred_probs` are properly ordered with respect to the ordering of classes, which for Datalab is: lexicographically sorted by class name. + + + features : Optional[np.ndarray] + Feature embeddings (vector representations) of every example in the dataset. + + If provided, this must be a 2D array with shape (num_examples, num_features). + + knn_graph : + Sparse matrix of precomputed distances between examples in the dataset in a k nearest neighbor graph. + + If provided, this must be a square CSR matrix with shape ``(num_examples, num_examples)`` and ``(k*num_examples)`` non-zero entries (``k`` is the number of nearest neighbors considered for each example), + evenly distributed across the rows. + Each non-zero entry in this matrix is a distance between a pair of examples in the dataset. Self-distances must be omitted + (i.e. diagonal must be all zeros, k nearest neighbors for each example do not include the example itself). + + This CSR format uses three 1D arrays (`data`, `indices`, `indptr`) to store a 2D matrix ``M``: + + - `data`: 1D array containing all the non-zero elements of matrix ``M``, listed in a row-wise fashion (but sorted within each row). + - `indices`: 1D array storing the column indices in matrix ``M`` of these non-zero elements. Each entry in `indices` corresponds to an entry in `data`, indicating the column of ``M`` containing this entry. + - `indptr`: 1D array indicating the start and end indices in `data` for each row of matrix ``M``. The non-zero elements of the i-th row of ``M`` are stored from ``data[indptr[i]]`` to ``data[indptr[i+1]]``. + + Within each row of matrix ``M`` (defined by the ranges in `indptr`), the corresponding non-zero entries (distances) of `knn_graph` must be sorted in ascending order (specifically in the segments of the `data` array that correspond to each row of ``M``). The `indices` array must also reflect this ordering, maintaining the correct column positions for these sorted distances. + + This type of matrix is returned by the method: `sklearn.neighbors.NearestNeighbors.kneighbors_graph `_. + + Below is an example to illustrate: + + .. code-block:: python + + knn_graph.todense() + # matrix([[0. , 0.3, 0.2], + # [0.3, 0. , 0.4], + # [0.2, 0.4, 0. ]]) + + knn_graph.data + # array([0.2, 0.3, 0.3, 0.4, 0.2, 0.4]) + # Here, 0.2 and 0.3 are the sorted distances in the first row, 0.3 and 0.4 in the second row, and so on. + + knn_graph.indices + # array([2, 1, 0, 2, 0, 1]) + # Corresponding neighbor indices for the distances from the `data` array. + + knn_graph.indptr + # array([0, 2, 4, 6]) + # The non-zero entries in the first row are stored from `knn_graph.data[0]` to `knn_graph.data[2]`, the second row from `knn_graph.data[2]` to `knn_graph.data[4]`, and so on. + + For any duplicated examples i,j whose distance is 0, there should be an *explicit* zero stored in the matrix, i.e. ``knn_graph[i,j] = 0``. + + If both `knn_graph` and `features` are provided, the `knn_graph` will take precendence. + If `knn_graph` is not provided, it is constructed based on the provided `features`. + If neither `knn_graph` nor `features` are provided, certain issue types like (near) duplicates will not be considered. + + .. seealso:: + See the + `scipy.sparse.csr_matrix documentation `_ + for more details on the CSR matrix format. + + issue_types : + Collection specifying which types of issues to consider in audit and any non-default parameter settings to use. + If unspecified, a default set of issue types and recommended parameter settings is considered. + + This is a dictionary of dictionaries, where the keys are the issue types of interest + and the values are dictionaries of parameter values that control how each type of issue is detected (only for advanced users). + More specifically, the values are constructor keyword arguments passed to the corresponding ``IssueManager``, + which is responsible for detecting the particular issue type. + + .. seealso:: + :py:class:`IssueManager ` + + Examples + -------- + + Here are some ways to provide inputs to :py:meth:`find_issues`: + + - Passing ``pred_probs``: + .. code-block:: python + + >>> from sklearn.linear_model import LogisticRegression + >>> import numpy as np + >>> from cleanlab import Datalab + >>> X = np.array([[0, 1], [1, 1], [2, 2], [2, 0]]) + >>> y = np.array([0, 1, 1, 0]) + >>> clf = LogisticRegression(random_state=0).fit(X, y) + >>> pred_probs = clf.predict_proba(X) + >>> lab = Datalab(data={"X": X, "y": y}, label_name="y") + >>> lab.find_issues(pred_probs=pred_probs) + + + - Passing ``features``: + .. code-block:: python + + >>> from sklearn.linear_model import LogisticRegression + >>> from sklearn.neighbors import NearestNeighbors + >>> import numpy as np + >>> from cleanlab import Datalab + >>> X = np.array([[0, 1], [1, 1], [2, 2], [2, 0]]) + >>> y = np.array([0, 1, 1, 0]) + >>> lab = Datalab(data={"X": X, "y": y}, label_name="y") + >>> lab.find_issues(features=X) + + .. note:: + + You can pass both ``pred_probs`` and ``features`` to :py:meth:`find_issues` for a more comprehensive audit. + + - Passing a ``knn_graph``: + .. code-block:: python + + >>> from sklearn.neighbors import NearestNeighbors + >>> import numpy as np + >>> from cleanlab import Datalab + >>> X = np.array([[0, 1], [1, 1], [2, 2], [2, 0]]) + >>> y = np.array([0, 1, 1, 0]) + >>> nbrs = NearestNeighbors(n_neighbors=2, metric="euclidean").fit(X) + >>> knn_graph = nbrs.kneighbors_graph(mode="distance") + >>> knn_graph # Pass this to Datalab + <4x4 sparse matrix of type '' + with 8 stored elements in Compressed Sparse Row format> + >>> knn_graph.toarray() # DO NOT PASS knn_graph.toarray() to Datalab, only pass the sparse matrix itself + array([[0. , 1. , 2.23606798, 0. ], + [1. , 0. , 1.41421356, 0. ], + [0. , 1.41421356, 0. , 2. ], + [0. , 1.41421356, 2. , 0. ]]) + >>> lab = Datalab(data={"X": X, "y": y}, label_name="y") + >>> lab.find_issues(knn_graph=knn_graph) + + - Configuring issue types: + Suppose you want to only consider label issues. Just pass a dictionary with the key "label" and an empty dictionary as the value (to use default label issue parameters). + + .. code-block:: python + + >>> issue_types = {"label": {}} + >>> # lab.find_issues(pred_probs=pred_probs, issue_types=issue_types) + + If you are advanced user who wants greater control, you can pass keyword arguments to the issue manager that handles the label issues. + For example, if you want to pass the keyword argument "clean_learning_kwargs" + to the constructor of the :py:class:`LabelIssueManager `, you would pass: + + + .. code-block:: python + + >>> issue_types = { + ... "label": { + ... "clean_learning_kwargs": { + ... "prune_method": "prune_by_noise_rate", + ... }, + ... }, + ... } + >>> # lab.find_issues(pred_probs=pred_probs, issue_types=issue_types) + + """ + + if issue_types is not None and not issue_types: + warnings.warn( + "No issue types were specified so no issues will be found in the dataset. Set `issue_types` as None to consider a default set of issues." + ) + return None + issue_finder = issue_finder_factory(self._imagelab)( + datalab=self, task=self.task, verbosity=self.verbosity + ) + issue_finder.find_issues( + pred_probs=pred_probs, + features=features, + knn_graph=knn_graph, + issue_types=issue_types, + ) + + if self.verbosity: + print( + f"\nAudit complete. {self.data_issues.issue_summary['num_issues'].sum()} issues found in the dataset." + ) + + def report( + self, + *, + num_examples: int = 5, + verbosity: Optional[int] = None, + include_description: bool = True, + show_summary_score: bool = False, + show_all_issues: bool = False, + ) -> None: + """Prints informative summary of all issues. + + Parameters + ---------- + num_examples : + Number of examples to show for each type of issue. + The report shows the top `num_examples` instances in the dataset that suffer the most from each type of issue. + + verbosity : + Higher verbosity levels add more information to the report. + + include_description : + Whether or not to include a description of each issue type in the report. + Consider setting this to ``False`` once you're familiar with how each issue type is defined. + + show_summary_score : + Whether or not to include the overall severity score of each issue type in the report. + These scores are not comparable across different issue types, + see the ``issue_summary`` documentation to learn more. + + show_all_issues : + Whether or not the report should show all issue types that were checked for, or only the types of issues detected in the dataset. + With this set to ``True``, the report may include more types of issues that were not detected in the dataset. + + See Also + -------- + For advanced usage, see documentation for the + :py:class:`Reporter ` class. + """ + if verbosity is None: + verbosity = self.verbosity + if self.data_issues.issue_summary.empty: + print("Please specify some `issue_types` in datalab.find_issues() to see a report.\n") + return + + reporter = report_factory(self._imagelab)( + data_issues=self.data_issues, + task=self.task, + verbosity=verbosity, + include_description=include_description, + show_summary_score=show_summary_score, + show_all_issues=show_all_issues, + imagelab=self._imagelab, + ) + reporter.report(num_examples=num_examples) + + @property + def issues(self) -> pd.DataFrame: + """Issues found in each example from the dataset.""" + return self.data_issues.issues + + @issues.setter + def issues(self, issues: pd.DataFrame) -> None: + self.data_issues.issues = issues + + @property + def issue_summary(self) -> pd.DataFrame: + """Summary of issues found in the dataset and the overall severity of each type of issue. + + Each type of issue has a summary score, which is usually defined as an average of + per-example issue-severity scores (over all examples in the dataset). + So these summary scores are not directly tied to the number of examples estimated to exhibit + a particular type of issue. Issue-severity (ie. quality of each example) is measured differently for each issue type, + and these per-example scores are only comparable across different examples for the same issue-type, but are not comparable across different issue types. + For instance, label quality might be scored via estimated likelihood of the given label, + whereas outlier quality might be scored via distance to K-nearest-neighbors in feature space (fundamentally incomparable quantities). + For some issue types, the summary score is not an average of per-example scores, but rather a global statistic of the dataset + (eg. for `non_iid` issue type, the p-value for hypothesis test that data are IID). + + In summary, you can compare these summary scores across datasets for the same issue type, but never compare them across different issue types. + + Examples + ------- + + If checks for "label" and "outlier" issues were run, + then the issue summary will look something like this: + + >>> datalab.issue_summary + issue_type score + outlier 0.123 + label 0.456 + """ + return self.data_issues.issue_summary + + @issue_summary.setter + def issue_summary(self, issue_summary: pd.DataFrame) -> None: + self.data_issues.issue_summary = issue_summary + + @property + def info(self) -> Dict[str, Dict[str, Any]]: + """Information and statistics about the dataset issues found. + + Examples + ------- + + If checks for "label" and "outlier" issues were run, + then the info will look something like this: + + >>> datalab.info + { + "label": { + "given_labels": [0, 1, 0, 1, 1, 1, 1, 1, 0, 1, ...], + "predicted_label": [0, 0, 0, 1, 0, 1, 0, 1, 0, 1, ...], + ..., + }, + "outlier": { + "nearest_neighbor": [3, 7, 1, 2, 8, 4, 5, 9, 6, 0, ...], + "distance_to_nearest_neighbor": [0.123, 0.789, 0.456, ...], + ..., + }, + } + """ + return self.data_issues.info + + @info.setter + def info(self, info: Dict[str, Dict[str, Any]]) -> None: + self.data_issues.info = info + + def get_issues(self, issue_name: Optional[str] = None) -> pd.DataFrame: + """ + Use this after finding issues to see which examples suffer from which types of issues. + + Parameters + ---------- + issue_name : str or None + The type of issue to focus on. If `None`, returns full DataFrame summarizing all of the types of issues detected in each example from the dataset. + + Raises + ------ + ValueError + If `issue_name` is not a type of issue previously considered in the audit. + + Returns + ------- + specific_issues : + A DataFrame where each row corresponds to an example from the dataset and columns specify: + whether this example exhibits a particular type of issue, and how severely (via a numeric quality score where lower values indicate more severe instances of the issue). + The quality scores lie between 0-1 and are directly comparable between examples (for the same issue type), but not across different issue types. + + Additional columns may be present in the DataFrame depending on the type of issue specified. + """ + + # Validate issue_name + if issue_name is not None and issue_name not in self.list_possible_issue_types(): + raise ValueError( + f"""Invalid issue_name: {issue_name}. Please specify a valid issue_name from the list of possible issue types. + Either, specify one of the following: {self.list_possible_issue_types()} + or set issue_name as None to get all issue types. + """ + ) + return self.data_issues.get_issues(issue_name=issue_name) + + def get_issue_summary(self, issue_name: Optional[str] = None) -> pd.DataFrame: + """Summarize the issues found in dataset of a particular type, + including how severe this type of issue is overall across the dataset. + + See the documentation of the ``issue_summary`` attribute to learn more. + + Parameters + ---------- + issue_name : + Name of the issue type to summarize. If `None`, summarizes each of the different issue types previously considered in the audit. + + Returns + ------- + issue_summary : + DataFrame where each row corresponds to a type of issue, and columns quantify: + the number of examples in the dataset estimated to exhibit this type of issue, + and the overall severity of the issue across the dataset (via a numeric quality score where lower values indicate that the issue is overall more severe). + The quality scores lie between 0-1 and are directly comparable between multiple datasets (for the same issue type), but not across different issue types. + """ + return self.data_issues.get_issue_summary(issue_name=issue_name) + + def get_info(self, issue_name: Optional[str] = None) -> Dict[str, Any]: + """Get the info for the issue_name key. + + This function is used to get the info for a specific issue_name. If the info is not computed yet, it will raise an error. + + Parameters + ---------- + issue_name : + The issue name for which the info is required. + + Returns + ------- + :py:meth:`info ` : + The info for the issue_name. + """ + return self.data_issues.get_info(issue_name) + + def list_possible_issue_types(self) -> List[str]: + """Returns a list of all registered issue types. + + Any issue type that is not in this list cannot be used in the :py:meth:`find_issues` method. + + See Also + -------- + :py:class:`REGISTRY ` : All available issue types and their corresponding issue managers can be found here. + """ + possible_issue_types = _list_possible_issue_types(task=self.task) + if self._imagelab is not None: + possible_issue_types.extend(DEFAULT_CLEANVISION_ISSUES.keys()) + return possible_issue_types + + def list_default_issue_types(self) -> List[str]: + """Returns a list of the issue types that are run by default + when :py:meth:`find_issues` is called without specifying `issue_types`. + + See Also + -------- + :py:class:`REGISTRY ` : All available issue types and their corresponding issue managers can be found here. + """ + default_issue_types = _list_default_issue_types(task=self.task) + if self._imagelab is not None: + default_issue_types.extend(DEFAULT_CLEANVISION_ISSUES.keys()) + return default_issue_types + + def save(self, path: str, force: bool = False) -> None: + """Saves this Datalab object to file (all files are in folder at `path/`). + We do not guarantee saved Datalab can be loaded from future versions of cleanlab. + + Parameters + ---------- + path : + Folder in which all information about this Datalab should be saved. + + force : + If ``True``, overwrites any existing files in the folder at `path`. Use this with caution! + + NOTE + ---- + You have to save the Dataset yourself separately if you want it saved to file. + """ + _Serializer.serialize(path=path, datalab=self, force=force) + save_message = f"Saved Datalab to folder: {path}" + print(save_message) + + @staticmethod + def load(path: str, data: Optional[Dataset] = None) -> "Datalab": + """Loads Datalab object from a previously saved folder. + + Parameters + ---------- + `path` : + Path to the folder previously specified in ``Datalab.save()``. + + `data` : + The dataset used to originally construct the Datalab. + Remember the dataset is not saved as part of the Datalab, + you must save/load the data separately. + + Returns + ------- + `datalab` : + A Datalab object that is identical to the one originally saved. + """ + datalab = _Serializer.deserialize(path=path, data=data) + load_message = f"Datalab loaded from folder: {path}" + print(load_message) + return datalab diff --git a/cleanlab/datalab/internal/__init__.py b/cleanlab/datalab/internal/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cleanlab/datalab/internal/adapter/__init__.py b/cleanlab/datalab/internal/adapter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cleanlab/datalab/internal/adapter/constants.py b/cleanlab/datalab/internal/adapter/constants.py new file mode 100644 index 0000000..3808e8b --- /dev/null +++ b/cleanlab/datalab/internal/adapter/constants.py @@ -0,0 +1,19 @@ +SPURIOUS_CORRELATION_ISSUE = { + "spurious_correlations": { + "threshold": 0.3, + }, +} # Default issue type for spurious correlation in Datalab + +DEFAULT_CLEANVISION_ISSUES = { + "dark": {}, + "light": {}, + "low_information": { + "threshold": 0.15, + }, + "odd_aspect_ratio": {}, + "odd_size": {}, + "grayscale": {}, + "blurry": {}, +} # list of CleanVision issue types considered by default in Datalab may differ from CleanVision package default + +IMAGELAB_ISSUES_MAX_PREVALENCE = 0.1 diff --git a/cleanlab/datalab/internal/adapter/imagelab.py b/cleanlab/datalab/internal/adapter/imagelab.py new file mode 100644 index 0000000..995f40f --- /dev/null +++ b/cleanlab/datalab/internal/adapter/imagelab.py @@ -0,0 +1,430 @@ +"""An internal wrapper around the Imagelab class from the CleanVision package to incorporate it into Datalab. +This allows low-quality images to be detected alongside other issues in computer vision datasets. +The methods/classes in this module are just intended for internal use. +""" + +import warnings +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type, cast, Union + +import numpy as np +import numpy.typing as npt +import pandas as pd +from scipy.sparse import csr_matrix + +from cleanlab.datalab.internal.adapter.constants import ( + DEFAULT_CLEANVISION_ISSUES, + IMAGELAB_ISSUES_MAX_PREVALENCE, + SPURIOUS_CORRELATION_ISSUE, +) +from cleanlab.datalab.internal.data import Data +from cleanlab.datalab.internal.data_issues import DataIssues, _InfoStrategy +from cleanlab.datalab.internal.issue_finder import IssueFinder +from cleanlab.datalab.internal.report import Reporter +from cleanlab.datalab.internal.task import Task +from cleanlab.datalab.internal.spurious_correlation import SpuriousCorrelations + +if TYPE_CHECKING: # pragma: no cover + from cleanvision import Imagelab + from datasets.arrow_dataset import Dataset + + +def create_imagelab(dataset: "Dataset", image_key: Optional[str]) -> Optional["Imagelab"]: + """Creates Imagelab instance for running CleanVision checks. CleanVision checks are only supported for + huggingface datasets as of now. + + Parameters + ---------- + dataset: datasets.Dataset + Huggingface dataset used by Imagelab + image_key: str + key for image feature in the huggingface dataset + + Returns + ------- + Imagelab + """ + imagelab = None + if not image_key: + return imagelab + try: + from cleanvision import Imagelab + from datasets.arrow_dataset import Dataset + + if isinstance(dataset, Dataset): + imagelab = Imagelab(hf_dataset=dataset, image_key=image_key) + else: + raise ValueError( + "For now, only huggingface datasets are supported for running cleanvision checks inside cleanlab. You can easily convert most datasets to the huggingface dataset format." + ) + + except ImportError: + raise ImportError( + "Cannot import required image packages. Please install them via: `pip install cleanlab[image]` or just install cleanlab with " + "all optional dependencies via: `pip install cleanlab[all]`" + ) + return imagelab + + +class ImagelabDataIssuesAdapter(DataIssues): + """ + Class that collects and stores information and statistics on issues found in a dataset. + + Parameters + ---------- + data : + The data object for which the issues are being collected. + strategy : + Strategy used for processing info dictionaries. + + Parameters + ---------- + issues : pd.DataFrame + Stores information about each individual issue found in the data, + on a per-example basis. + issue_summary : pd.DataFrame + Summarizes the overall statistics for each issue type. + info : dict + A dictionary that contains information and statistics about the data and each issue type. + """ + + def __init__(self, data: Data, strategy: Type[_InfoStrategy]) -> None: + super().__init__(data, strategy) + + def _update_issues_imagelab(self, imagelab: "Imagelab", overlapping_issues: List[str]) -> None: + overwrite_columns = [f"is_{issue_type}_issue" for issue_type in overlapping_issues] + overwrite_columns.extend([f"{issue_type}_score" for issue_type in overlapping_issues]) + + if overwrite_columns: + warnings.warn( + f"Overwriting columns {overwrite_columns} in self.issues with " + f"columns from imagelab." + ) + self.issues.drop(columns=overwrite_columns, inplace=True) + new_columnns = list(set(imagelab.issues.columns).difference(self.issues.columns)) + self.issues = self.issues.join(imagelab.issues[new_columnns], how="outer") + + def filter_based_on_max_prevalence(self, issue_summary: pd.DataFrame, max_num: int): + removed_issues = issue_summary[issue_summary["num_images"] > max_num]["issue_type"].tolist() + if len(removed_issues) > 0: + print( + f"Removing {', '.join(removed_issues)} from potential issues in the dataset as it exceeds max_prevalence={IMAGELAB_ISSUES_MAX_PREVALENCE}" + ) + return issue_summary[issue_summary["num_images"] <= max_num].copy() + + def collect_issues_from_imagelab(self, imagelab: "Imagelab", issue_types: List[str]) -> None: + """ + Collect results from Imagelab and update datalab.issues and datalab.issue_summary + + Parameters + ---------- + imagelab: Imagelab + Imagelab instance that run all the checks for image issue types + """ + overlapping_issues = list(set(self.issue_summary["issue_type"]) & set(issue_types)) + self._update_issues_imagelab(imagelab, overlapping_issues) + + if overlapping_issues: + warnings.warn( + f"Overwriting {overlapping_issues} rows in self.issue_summary from imagelab." + ) + self.issue_summary = self.issue_summary[ + ~self.issue_summary["issue_type"].isin(overlapping_issues) + ] + imagelab_summary_copy = imagelab.issue_summary.copy() + imagelab_summary_copy = self.filter_based_on_max_prevalence( + imagelab_summary_copy, int(IMAGELAB_ISSUES_MAX_PREVALENCE * len(self.issues)) + ) + + imagelab_summary_copy.rename({"num_images": "num_issues"}, axis=1, inplace=True) + self.issue_summary = pd.concat( + [self.issue_summary, imagelab_summary_copy], axis=0, ignore_index=True + ) + for issue_type in issue_types: + self._update_issue_info(issue_type, imagelab.info[issue_type]) + + def get_info(self, issue_name: Optional[str] = None) -> Dict[str, Any]: + # Extend method for fetching info about spurious correlations + if issue_name != "spurious_correlations": + return super().get_info(issue_name) + + correlations_info = self.info.get("spurious_correlations", {}) + if not correlations_info: + raise ValueError( + "Spurious correlations have not been calculated. Run find_issues() first." + ) + return correlations_info + + +class CorrelationVisualizer: + """Class to visualize images corresponding to the extreme (minimum and maximum) individual + scores for each of the detected correlated properties. + """ + + def __init__(self): + # Wrapper for VizManager that's from the optional cleanvision dependency + try: + from cleanvision.utils.viz_manager import VizManager + + self.viz_manager = VizManager + except ImportError: + raise ImportError( + "cleanvision is required for correlation visualization. Please install it to use this feature." + ) + + def visualize( + self, images: List, title_info: Dict, ncols: int = 2, cell_size: tuple = (2, 2) + ) -> None: + self.viz_manager.individual_images( + images=images, + title_info=title_info, + ncols=ncols, + cell_size=cell_size, + ) + + +class CorrelationReporter: + """Class to report spurious correlations between image features and class labels detected in the data. + + If no spurious correlations are found, the class will not report anything. + """ + + def __init__(self, data_issues: "DataIssues", imagelab: "Imagelab"): + self.imagelab: "Imagelab" = imagelab + self.data_issues = data_issues + self.threshold = data_issues.get_info("spurious_correlations").get("threshold") + if not self.threshold: + raise ValueError( + "Spurious correlations have not been calculated. Run find_issues() first." + ) + self.visualizer = CorrelationVisualizer() + + def report(self) -> None: + """Reports spurious correlations between image features and class labels detected in the data, + if any are found. + """ + correlated_properties = self._get_correlated_properties() + if not correlated_properties: + return + + self._print_correlation_summary() + correlations_df = cast( + pd.DataFrame, self.data_issues.get_info("spurious_correlations").get("correlations_df") + ) + filtered_correlations_df = self._get_filtered_correlated_properties( + correlations_df, correlated_properties + ) + print(filtered_correlations_df.to_string(index=False) + "\n") + + self._visualize_extremes(correlated_properties, self.data_issues) + + def _print_correlation_summary(self) -> None: + print("\n\n") + report_correlation_header = "Summary of (potentially spurious) correlations between image properties and class labels detected in the data:\n\n" + report_correlation_metric = "Lower scores below correspond to images properties that are more strongly correlated with the class labels.\n\n" + print(report_correlation_header + report_correlation_metric) + + def _visualize_extremes( + self, correlated_properties: List[str], data_issues: "DataIssues" + ) -> None: + report_extremal_images = "Here are the images corresponding to the extreme (minimum and maximum) individual scores for each of the detected correlated properties:\n\n" + print(report_extremal_images) + issues = data_issues.get_issues() + correlated_indices = { + prop: [issues[prop].idxmin(), issues[prop].idxmax()] for prop in correlated_properties + } + self._visualize(correlated_indices, issues) + + def _visualize(self, correlated_indices: Dict[str, List[Any]], issues: pd.DataFrame) -> None: + for prop, image_ids in correlated_indices.items(): + print( + f"{'Images with minimum and maximum individual scores for ' + prop.replace('_score', '') + ' issue:'}\n" + ) + title_info = {"scores": [f"score: {issues.loc[id, prop]:.4f}" for id in image_ids]} + self.visualizer.visualize( + images=[self.imagelab._dataset[id] for id in image_ids], + title_info=title_info, + ) + + def _get_correlated_properties(self) -> List[str]: + correlations_df = self.data_issues.get_info("spurious_correlations").get("correlations_df") + if correlations_df is None or correlations_df.empty: + return [] + return correlations_df.query("score < @self.threshold")["property"].tolist() + + def _get_filtered_correlated_properties( + self, correlations_df: pd.DataFrame, correlated_properties: List[str] + ) -> pd.DataFrame: + query_str = "property in @correlated_properties" + filtered_correlations_df = correlations_df.query(query_str) + filtered_correlations_df.loc[:, "property"] = filtered_correlations_df["property"].apply( + lambda x: x.replace("_score", "") + ) + return filtered_correlations_df + + +class ImagelabReporterAdapter(Reporter): + def __init__( + self, + data_issues: "DataIssues", + imagelab: "Imagelab", + task: Task, + verbosity: int = 1, + include_description: bool = True, + show_summary_score: bool = False, + show_all_issues: bool = False, + ): + super().__init__( + data_issues=data_issues, + task=task, + verbosity=verbosity, + include_description=include_description, + show_summary_score=show_summary_score, + show_all_issues=show_all_issues, + ) + self.imagelab = imagelab + self.correlation_reporter: Optional[CorrelationReporter] = None + try: + self.correlation_reporter = CorrelationReporter(data_issues, imagelab) + except: + # Spurious correlations have not been calculated + self.correlation_reporter = None + + def report(self, num_examples: int) -> None: + super().report(num_examples) + self._report_imagelab(num_examples) + + # Only report spurious correlations if they've been calculated & detected + if self.correlation_reporter is not None: + self.correlation_reporter.report() + + def _report_imagelab(self, num_examples): + print("\n\n") + self.imagelab.report( + num_images=num_examples, + max_prevalence=IMAGELAB_ISSUES_MAX_PREVALENCE, + print_summary=False, + verbosity=0, + show_id=True, + ) + + +class ImagelabIssueFinderAdapter(IssueFinder): + def __init__(self, datalab, task, verbosity): + super().__init__(datalab, task, verbosity) + self.imagelab = self.datalab._imagelab + + def _get_imagelab_issue_types(self, issue_types, **kwargs): + if issue_types is None: + return DEFAULT_CLEANVISION_ISSUES + + if "image_issue_types" not in issue_types: + return None + + issue_types_copy = {} + for issue_type, params in issue_types["image_issue_types"].items(): + if not params: + issue_types_copy[issue_type] = DEFAULT_CLEANVISION_ISSUES[issue_type] + else: + issue_types_copy[issue_type] = params + + return issue_types_copy + + def find_issues( + self, + *, + pred_probs: Optional[np.ndarray] = None, + features: Optional[npt.NDArray] = None, + knn_graph: Optional[csr_matrix] = None, + issue_types: Optional[Dict[str, Any]] = None, + ) -> None: + issue_types_to_ignore_in_datalab = ["image_issue_types", "spurious_correlations"] + datalab_issue_types = ( + {k: v for k, v in issue_types.items() if k not in issue_types_to_ignore_in_datalab} + if issue_types + else issue_types + ) + super().find_issues( + pred_probs=pred_probs, + features=features, + knn_graph=knn_graph, + issue_types=datalab_issue_types, + ) + + issue_types_copy = self._get_imagelab_issue_types(issue_types) + if issue_types_copy: + try: + if self.verbosity: + print(f'Finding {", ".join(issue_types_copy.keys())} images ...') + + self.imagelab.find_issues(issue_types=issue_types_copy, verbose=False) + + self.datalab.data_issues.collect_statistics(self.imagelab) + self.datalab.data_issues.collect_issues_from_imagelab( + self.imagelab, issue_types_copy.keys() + ) + except Exception as e: + print(f"Error in checking for image issues: {e}") + + # if issue_types is neither 'None' nor empty dictionary (non-trivial) but + # there is no mention of 'spurious_correlations', we return. + if issue_types and "spurious_correlations" not in issue_types: + return + + # Check if all vision issue scores are computed + imagelab_columns = self.imagelab.issues.columns.tolist() + if all( + default_cleanvision_issue + "_score" not in imagelab_columns + for default_cleanvision_issue in DEFAULT_CLEANVISION_ISSUES.keys() + ): + print("Skipping spurious correlations check: Image property scores not available.") + print( + "To include this check, run find_issues() without parameters to compute all scores." + ) + return + + # Spurious correlation part must be run + print("Finding spurious correlation issues in the dataset ...") + + # the else part of the following must contain 'spurious_correlations' key + spurious_correlation_issue_types = ( + SPURIOUS_CORRELATION_ISSUE["spurious_correlations"] + if not issue_types + else issue_types["spurious_correlations"] + ) + + # If threshold is not expicitly given (e.g. lab.find_issues("issue_types={"spurious_correlations": {}")) + # we extract the default value from SPURIOUS_CORRELATION_ISSUE + spurious_correlation_issue_threshold = spurious_correlation_issue_types.get( + "threshold", SPURIOUS_CORRELATION_ISSUE["spurious_correlations"]["threshold"] + ) + + try: + if self.datalab.has_labels: + self.datalab.data_issues.info["spurious_correlations"] = ( + handle_spurious_correlations( + imagelab_issues=self.imagelab.issues, + labels=self.datalab.labels, + threshold=spurious_correlation_issue_threshold, + ) + ) + except Exception as e: + print(f"Error in checking for spurious correlations: {e}") + + +def handle_spurious_correlations( + *, + imagelab_issues: pd.DataFrame, + labels: Union[np.ndarray, List[List[int]]], + threshold: float, + **_, +) -> Dict[str, Any]: + imagelab_columns = imagelab_issues.columns.tolist() + + score_columns = [col for col in imagelab_columns if col.endswith("_score")] + correlations_df = SpuriousCorrelations( + data=imagelab_issues[score_columns], labels=labels + ).calculate_correlations() + return { + "correlations_df": correlations_df, + "threshold": threshold, + } diff --git a/cleanlab/datalab/internal/data.py b/cleanlab/datalab/internal/data.py new file mode 100644 index 0000000..3822125 --- /dev/null +++ b/cleanlab/datalab/internal/data.py @@ -0,0 +1,431 @@ +"""Classes and methods for datasets that are loaded into Datalab.""" + +import os +from typing import Any, Callable, Dict, List, Mapping, Optional, Union, cast, TYPE_CHECKING, Tuple + +from cleanlab.datalab.internal.task import Task + +try: + import datasets +except ImportError as error: + raise ImportError( + "Cannot import datasets package. " + "Please install it and try again, or just install cleanlab with " + "all optional dependencies via: `pip install 'cleanlab[all]'`" + ) from error +from abc import ABC, abstractmethod +import numpy as np +import pandas as pd +from datasets.arrow_dataset import Dataset +from datasets import ClassLabel + +# Import Column types for compatibility with datasets 4.0.0+ +try: + from datasets.arrow_dataset import Column + from datasets.iterable_dataset import IterableColumn +except ImportError: + # For backwards compatibility with older datasets versions + Column = None + IterableColumn = None + +from cleanlab.internal.validation import labels_to_array, labels_to_list_multilabel + + +if TYPE_CHECKING: # pragma: no cover + DatasetLike = Union[Dataset, pd.DataFrame, Dict[str, Any], List[Dict[str, Any]], str] + + +class DataFormatError(ValueError): + """Exception raised when the data is not in a supported format.""" + + def __init__(self, data: Any): + self.data = data + message = ( + f"Unsupported data type: {type(data)}\n" + "Supported types: " + "datasets.Dataset, pandas.DataFrame, dict, list, str" + ) + super().__init__(message) + + +class DatasetDictError(ValueError): + """Exception raised when a DatasetDict is passed to Datalab. + + Usually, this means that a dataset identifier was passed to Datalab, but + the dataset is a DatasetDict, which contains multiple splits of the dataset. + + """ + + def __init__(self): + message = ( + "Please pass a single dataset, not a DatasetDict. " + "Try specifying a split, e.g. `dataset = load_dataset('dataset', split='train')` " + "then pass `dataset` to Datalab." + ) + super().__init__(message) + + +class DatasetLoadError(ValueError): + """Exception raised when a dataset cannot be loaded. + + Parameters + ---------- + dataset_type: type + The type of dataset that failed to load. + """ + + def __init__(self, dataset_type: type): + message = f"Failed to load dataset from {dataset_type}.\n" + super().__init__(message) + + +class Data: + """ + Class that holds and validates datasets for Datalab. + + Internally, the data is stored as a datasets.Dataset object and the labels + are integers (ranging from 0 to K-1, where K is the number of classes) stored + in a numpy array. + + Parameters + ---------- + data : + Dataset to be audited by Datalab. + Several formats are supported, which will internally be converted to a Dataset object. + + Supported formats: + - datasets.Dataset + - pandas.DataFrame + - dict + - keys are strings + - values are arrays or lists of equal length + - list + - list of dictionaries with the same keys + - str + - path to a local file + - Text (.txt) + - CSV (.csv) + - JSON (.json) + - or a dataset identifier on the Hugging Face Hub + It checks if the string is a path to a file that exists locally, and if not, + it assumes it is a dataset identifier on the Hugging Face Hub. + + label_name : Union[str, List[str]] + Name of the label column in the dataset. + + task : + The task associated with the dataset. This is used to determine how to + to format the labels. + + Note: + + - If the task is a classification task, the labels + will be mapped to integers, e.g. [0, 1, ..., K-1] where K is the number + of classes. If the task is a regression task, the labels will not be + mapped to integers. + + - If the task is a multilabel task, the labels will be formatted as a + list of lists, e.g. [[0, 1], [1, 2], [0, 2]] where each sublist contains + the labels for a single example. If the task is not a multilabel task, + the labels will be formatted as a 1D numpy array. + + Warnings + -------- + Optional dependencies: + + - datasets : + Dataset, DatasetDict and load_dataset are imported from datasets. + This is an optional dependency of cleanlab, but is required for + :py:class:`Datalab ` to work. + """ + + def __init__( + self, + data: "DatasetLike", + task: Task, + label_name: Optional[str] = None, + ) -> None: + self._validate_data(data) + self._data = self._load_data(data) + self._data_hash = hash(self._data) + self.labels: Label + label_class = MultiLabel if task.is_multilabel else MultiClass + map_to_int = task.is_classification + self.labels = label_class(data=self._data, label_name=label_name, map_to_int=map_to_int) + + def _load_data(self, data: "DatasetLike") -> Dataset: + """Checks the type of dataset and uses the correct loader method and + assigns the result to the data attribute.""" + dataset_factory_map: Dict[type, Callable[..., Dataset]] = { + Dataset: lambda x: x, + pd.DataFrame: Dataset.from_pandas, + dict: self._load_dataset_from_dict, + list: self._load_dataset_from_list, + str: self._load_dataset_from_string, + } + if not isinstance(data, tuple(dataset_factory_map.keys())): + raise DataFormatError(data) + return dataset_factory_map[type(data)](data) + + def __len__(self) -> int: + return len(self._data) + + def __eq__(self, other) -> bool: + if isinstance(other, Data): + # Equality checks + hashes_are_equal = self._data_hash == other._data_hash + labels_are_equal = self.labels == other.labels + return all([hashes_are_equal, labels_are_equal]) + return False + + def __hash__(self) -> int: + return self._data_hash + + @property + def class_names(self) -> List[str]: + return self.labels.class_names + + @property + def has_labels(self) -> bool: + """Check if labels are available.""" + return self.labels.is_available + + @staticmethod + def _validate_data(data) -> None: + if isinstance(data, datasets.DatasetDict): + raise DatasetDictError() + if not isinstance(data, (Dataset, pd.DataFrame, dict, list, str)): + raise DataFormatError(data) + + @staticmethod + def _load_dataset_from_dict(data_dict: Dict[str, Any]) -> Dataset: + try: + return Dataset.from_dict(data_dict) + except Exception as error: + raise DatasetLoadError(dict) from error + + @staticmethod + def _load_dataset_from_list(data_list: List[Dict[str, Any]]) -> Dataset: + try: + return Dataset.from_list(data_list) + except Exception as error: + raise DatasetLoadError(list) from error + + @staticmethod + def _load_dataset_from_string(data_string: str) -> Dataset: + if not os.path.exists(data_string): + try: + dataset = datasets.load_dataset(data_string) + return cast(Dataset, dataset) + except Exception as error: + raise DatasetLoadError(str) from error + + factory: Dict[str, Callable[[str], Any]] = { + ".txt": Dataset.from_text, + ".csv": Dataset.from_csv, + ".json": Dataset.from_json, + } + + extension = os.path.splitext(data_string)[1] + if extension not in factory: + raise DatasetLoadError(type(data_string)) + + dataset = factory[extension](data_string) + dataset_cast = cast(Dataset, dataset) + return dataset_cast + + +class Label(ABC): + """ + Class to represent labels in a dataset. + + It stores the labels as a numpy array and maps them to integers if necessary. + If a mapping is not necessary, e.g. for regression tasks, the mapping will be an empty dictionary. + + Parameters + ---------- + data : + A Hugging Face Dataset object. + + label_name : str + Name of the label column in the dataset. + + map_to_int : bool + Whether to map the labels to integers, e.g. [0, 1, ..., K-1] where K is the number of classes. + If False, the labels are not mapped to integers, e.g. for regression tasks. + """ + + def __init__( + self, *, data: Dataset, label_name: Optional[str] = None, map_to_int: bool = True + ) -> None: + self._data = data + self.label_name = label_name + self.labels = labels_to_array([]) + self.label_map: Mapping[Union[str, int], Any] = {} + if label_name is not None: + self.labels, self.label_map = self._extract_labels(data, label_name, map_to_int) + self._validate_labels() + + def __len__(self) -> int: + if self.labels is None: + return 0 + return len(self.labels) + + def __eq__(self, __value: object) -> bool: + if isinstance(__value, Label): + labels_are_equal = np.array_equal(self.labels, __value.labels) + names_are_equal = self.label_name == __value.label_name + maps_are_equal = self.label_map == __value.label_map + return all([labels_are_equal, names_are_equal, maps_are_equal]) + return False + + def __getitem__(self, __index: Union[int, slice, np.ndarray]) -> np.ndarray: + return self.labels[__index] + + def __bool__(self) -> bool: + return self.is_available + + @property + def class_names(self) -> List[str]: + """A list of class names that are present in the dataset. + + Without labels, this will return an empty list. + """ + return list(self.label_map.values()) + + @property + def is_available(self) -> bool: + """Check if labels are available.""" + empty_labels = self.labels is None or len(self.labels) == 0 + empty_label_map = self.label_map is None or len(self.label_map) == 0 + return not (empty_labels or empty_label_map) + + def _validate_labels(self) -> None: + if self.label_name not in self._data.column_names: + raise ValueError(f"Label column '{self.label_name}' not found in dataset.") + labels = self._data[self.label_name] + error_message = ( + f"Expected labels to be numpy array, list, or Column type, got {type(labels)}" + ) + assert _is_valid_label_column(labels), error_message + assert len(labels) == len(self._data) + + @abstractmethod + def _extract_labels(self, *args, **kwargs) -> Any: + """Extract labels from the dataset and formats them""" + raise NotImplementedError + + +class MultiLabel(Label): + def __init__(self, data, label_name, map_to_int): + super().__init__(data=data, label_name=label_name, map_to_int=map_to_int) + + def _extract_labels( + self, data: Dataset, label_name: str, map_to_int: bool + ) -> Tuple[List[List[int]], Dict[int, Any]]: + # Convert Column types to list for compatibility with validation functions + raw_labels = data[label_name] + converted_labels = _convert_column_to_list(raw_labels) + labels: List[List[int]] = labels_to_list_multilabel(converted_labels) + # label_map needs to be lexicographically sorted. np.unique should sort it + unique_labels = np.unique([x for ele in labels for x in ele]) + label_map = {label: i for i, label in enumerate(unique_labels)} + formatted_labels = [[label_map[item] for item in label] for label in labels] + inverse_map = {i: label for label, i in label_map.items()} + return formatted_labels, inverse_map + + +class MultiClass(Label): + def __init__(self, data, label_name, map_to_int): + super().__init__(data=data, label_name=label_name, map_to_int=map_to_int) + + def _extract_labels(self, data: Dataset, label_name: str, map_to_int: bool): + """ + Picks out labels from the dataset and formats them to be [0, 1, ..., K-1] + where K is the number of classes. Also returns a mapping from the formatted + labels to the original labels in the dataset. + + Note: This function is not meant to be used directly. It is used by + ``cleanlab.data.Data`` to extract the formatted labels from the dataset + and stores them as attributes. + + Parameters + ---------- + data : datasets.Dataset + A Hugging Face Dataset object. + + label_name : str + Name of the column in the dataset that contains the labels. + + map_to_int : bool + Whether to map the labels to integers, e.g. [0, 1, ..., K-1] where K is the number of classes. + If False, the labels are not mapped to integers, e.g. for regression tasks. + Returns + ------- + formatted_labels : np.ndarray + Labels in the format [0, 1, ..., K-1] where K is the number of classes. + + inverse_map : dict + Mapping from the formatted labels to the original labels in the dataset. + """ + + labels = labels_to_array(data[label_name]) # type: ignore[assignment] + if labels.ndim != 1: + raise ValueError("labels must be 1D numpy array.") + + if not map_to_int: + # Don't map labels to integers, e.g. for regression tasks + return labels, {} + label_name_feature = data.features[label_name] + if isinstance(label_name_feature, ClassLabel): + label_map = { + label: label_name_feature.str2int(label) for label in label_name_feature.names + } + formatted_labels = labels + else: + label_map = {label: i for i, label in enumerate(np.unique(labels))} + formatted_labels = np.vectorize(label_map.get, otypes=[int])(labels) + inverse_map = {i: label for label, i in label_map.items()} + + return formatted_labels, inverse_map + + +def _is_valid_label_column(labels: Any) -> bool: + """Helper function to check if labels are a valid type including datasets 4.0.0+ Column types. + + Parameters + ---------- + labels : Any + The labels object to validate. + + Returns + ------- + bool + True if labels are a valid type (numpy array, list, or Column types). + """ + valid_types = [np.ndarray, list] + if Column is not None: + valid_types.append(Column) + if IterableColumn is not None: + valid_types.append(IterableColumn) + return isinstance(labels, tuple(valid_types)) + + +def _convert_column_to_list(labels): + """Helper function to convert Column types to list for compatibility with validation functions. + + Parameters + ---------- + labels : Any + The labels object to convert. + + Returns + ------- + list or original type + Converted to list if it's a Column type, otherwise returns original object. + """ + if Column is not None and isinstance(labels, Column): + return list(labels) + elif IterableColumn is not None and isinstance(labels, IterableColumn): + return list(labels) + return labels diff --git a/cleanlab/datalab/internal/data_issues.py b/cleanlab/datalab/internal/data_issues.py new file mode 100644 index 0000000..1e8ef2d --- /dev/null +++ b/cleanlab/datalab/internal/data_issues.py @@ -0,0 +1,413 @@ +""" +Module for the :py:class:`DataIssues` class, which serves as a central repository for storing +information and statistics about issues found in a dataset. + +It collects information from various +:py:class:`IssueManager ` +instances and keeps track of each issue, a summary for each type of issue, +related information and statistics about the issues. + +The collected information can be accessed using the +`~cleanlab.datalab.internal.data_issues.DataIssues.get_info` method. +We recommend using that method instead of this module, which is just intended for internal use. +""" + +from __future__ import annotations + +import warnings +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type, Union +import numpy as np + +import pandas as pd + +if TYPE_CHECKING: # pragma: no cover + from cleanlab.datalab.internal.data import Data + from cleanlab.datalab.internal.issue_manager import IssueManager + from cleanvision import Imagelab + + +class _InfoStrategy(ABC): + """ + Abstract base class for strategies that fetch information about data issues. + + Subclasses must implement the `get_info` method, which takes a `Data` object, a dictionary of + information about data issues, and an optional issue name, and returns a dictionary of + information about the specified issue, augmented with dataset about the dataset as a whole. + + This class also provides a helper method, `_get_info_helper`, which takes an information + dictionary and an optional issue name, and returns a copy of the information dictionary for + the specified issue. If the issue name is `None`, this method returns `None`. + """ + + @staticmethod + @abstractmethod + def get_info( + data: Data, + info: Dict[str, Dict[str, Any]], + issue_name: Optional[str] = None, + ) -> Dict[str, Any]: + """ + Get information about a data issue from an information dictionary. + + Parameters + ---------- + info : dict + A dictionary of information about data issues. + issue_name : str or None, optional (default=None) + The name of the issue to get information about. If `None`, this method returns `None`. + + Returns + ------- + dict or None + A copy of the information dictionary for the specified issue, or `None` if the issue + name is `None`. + + Raises + ------ + ValueError + If the specified issue name is not found in the information dictionary. + """ + pass # pragma: no cover + + @staticmethod + def _get_info_helper( + info: Dict[str, Dict[str, Any]], + issue_name: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + if issue_name is None: + return None + if issue_name not in info: + raise ValueError( + f"issue_name {issue_name} not found in self.info. These have not been computed yet." + ) + info = info[issue_name].copy() + return info + + +class _ClassificationInfoStrategy(_InfoStrategy): + """Strategy for computing information about data issues related to classification tasks.""" + + @staticmethod + def get_info( + data: Data, + info: Dict[str, Dict[str, Any]], + issue_name: Optional[str] = None, + ) -> Dict[str, Any]: + info_extracted = _InfoStrategy._get_info_helper(info=info, issue_name=issue_name) + info = info_extracted if info_extracted is not None else info + if issue_name in ["label", "class_imbalance"]: + if data.labels.is_available is False: + raise ValueError( + "The labels are not available. " + "Most likely, no label column was provided when creating the Data object." + ) + # Labels that are stored as integers may need to be converted to strings. + label_map = data.labels.label_map + if not label_map: + raise ValueError("The label map is not available.") + for key in ["given_label", "predicted_label"]: + labels = info.get(key, None) + if labels is not None: + info[key] = np.vectorize(label_map.get)(labels) + info["class_names"] = list(label_map.values()) # type: ignore + return info + + +class _RegressionInfoStrategy(_InfoStrategy): + """Strategy for computing information about data issues related to regression tasks.""" + + @staticmethod + def get_info( + data: Data, + info: Dict[str, Dict[str, Any]], + issue_name: Optional[str] = None, + ) -> Dict[str, Any]: + info_extracted = _InfoStrategy._get_info_helper(info=info, issue_name=issue_name) + info = info_extracted if info_extracted is not None else info + if issue_name == "label": + for key in ["given_label", "predicted_label"]: + labels = info.get(key, None) + if labels is not None: + info[key] = labels + return info + + +class _MultilabelInfoStrategy(_InfoStrategy): + """Strategy for computing information about data issues related to multilabel tasks.""" + + @staticmethod + def get_info( + data: Data, + info: Dict[str, Dict[str, Any]], + issue_name: Optional[str] = None, + ) -> Dict[str, Any]: + info_extracted = _InfoStrategy._get_info_helper(info=info, issue_name=issue_name) + info = info_extracted if info_extracted is not None else info + if issue_name == "label": + if data.labels.is_available is False: + raise ValueError( + "The labels are not available. " + "Most likely, no label column was provided when creating the Data object." + ) + # Labels that are stored as integers may need to be converted to strings. + label_map = data.labels.label_map + if not label_map: + raise ValueError("The label map is not available.") + for key in ["given_label", "predicted_label"]: + labels = info.get(key, None) + if labels is not None: + info[key] = [list(map(label_map.get, label)) for label in labels] # type: ignore + info["class_names"] = list(label_map.values()) # type: ignore + return info + + +class DataIssues: + """ + Class that collects and stores information and statistics on issues found in a dataset. + + Parameters + ---------- + data : + The data object for which the issues are being collected. + strategy : + Strategy used for processing info dictionaries. + + Attributes + ---------- + issues : pd.DataFrame + Stores information about each individual issue found in the data, + on a per-example basis. + issue_summary : pd.DataFrame + Summarizes the overall statistics for each issue type. + info : dict + A dictionary that contains information and statistics about the data and each issue type. + """ + + def __init__(self, data: Data, strategy: Type[_InfoStrategy]) -> None: + self.issues: pd.DataFrame = pd.DataFrame(index=range(len(data))) + self.issue_summary: pd.DataFrame = pd.DataFrame( + columns=["issue_type", "score", "num_issues"] + ).astype({"score": np.float64, "num_issues": np.int64}) + self.info: Dict[str, Dict[str, Any]] = { + "statistics": get_data_statistics(data), + } + self._data = data + self._strategy = strategy + + def get_info(self, issue_name: Optional[str] = None) -> Dict[str, Any]: + return self._strategy.get_info(data=self._data, info=self.info, issue_name=issue_name) + + @property + def statistics(self) -> Dict[str, Any]: + """Returns the statistics dictionary. + + Shorthand for self.info["statistics"]. + """ + return self.info["statistics"] + + def get_issues(self, issue_name: Optional[str] = None) -> pd.DataFrame: + """ + Use this after finding issues to see which examples suffer from which types of issues. + + Parameters + ---------- + issue_name : str or None + The type of issue to focus on. If `None`, returns full DataFrame summarizing all of the types of issues detected in each example from the dataset. + + Raises + ------ + ValueError + If `issue_name` is not a type of issue previously considered in the audit. + + Returns + ------- + specific_issues : + A DataFrame where each row corresponds to an example from the dataset and columns specify: + whether this example exhibits a particular type of issue and how severely (via a numeric quality score where lower values indicate more severe instances of the issue). + + Additional columns may be present in the DataFrame depending on the type of issue specified. + """ + if self.issues.empty: + raise ValueError( + """No issues available for retrieval. Please check the following before using `get_issues`: + 1. Ensure `find_issues` was executed. If not, please run it with the necessary parameters. + 2. If `find_issues` was run but you're seeing this message, + it may have encountered limitations preventing full analysis. + However, partial checks can still provide valuable insights. + Review `find_issues` output carefully for any specific actions needed + to facilitate a more comprehensive analysis before calling `get_issues`. + """ + ) + if issue_name is None: + return self.issues + + columns = [col for col in self.issues.columns if issue_name in col] + if not columns: + raise ValueError( + f"""No columns found for issue type '{issue_name}'. Ensure the following: + 1. `find_issues` has been executed. If it hasn't, please run it. + 2. Check `find_issues` output to verify that the issue type '{issue_name}' was included in the checks to + ensure it was not excluded accidentally before the audit. + 3. Review `find_issues` output for any errors or warnings that might indicate the check for '{issue_name}' issues failed to complete. + This can provide better insights into what adjustments may be necessary. + """ + ) + specific_issues = self.issues[columns] + info = self.get_info(issue_name=issue_name) + + if issue_name == "label": + specific_issues = specific_issues.assign( + given_label=info["given_label"], predicted_label=info["predicted_label"] + ) + + if issue_name == "near_duplicate": + column_dict = { + k: info.get(k) + for k in ["near_duplicate_sets", "distance_to_nearest_neighbor"] + if info.get(k) is not None + } + specific_issues = specific_issues.assign(**column_dict) + + if issue_name == "class_imbalance": + specific_issues = specific_issues.assign(given_label=info["given_label"]) + return specific_issues + + def get_issue_summary(self, issue_name: Optional[str] = None) -> pd.DataFrame: + """Summarize the issues found in dataset of a particular type, + including how severe this type of issue is overall across the dataset. + + Parameters + ---------- + issue_name : + Name of the issue type to summarize. If `None`, summarizes each of the different issue types previously considered in the audit. + + Returns + ------- + issue_summary : + DataFrame where each row corresponds to a type of issue, and columns quantify: + the number of examples in the dataset estimated to exhibit this type of issue, + and the overall severity of the issue across the dataset (via a numeric quality score where lower values indicate that the issue is overall more severe). + """ + if self.issue_summary.empty: + raise ValueError( + "No issues found in the dataset. " + "Call `find_issues` before calling `get_issue_summary`." + ) + + if issue_name is None: + return self.issue_summary + + row_mask = self.issue_summary["issue_type"] == issue_name + if not any(row_mask): + raise ValueError(f"Issue type {issue_name} not found in the summary.") + return self.issue_summary[row_mask].reset_index(drop=True) + + def collect_statistics(self, issue_manager: Union[IssueManager, "Imagelab"]) -> None: + """Update the statistics in the info dictionary. + + Parameters + ---------- + statistics : + A dictionary of statistics to add/update in the info dictionary. + + Examples + -------- + + A common use case is to reuse the KNN-graph across multiple issue managers. + To avoid recomputing the KNN-graph for each issue manager, + we can pass it as a statistic to the issue managers. + + >>> from scipy.sparse import csr_matrix + >>> weighted_knn_graph = csr_matrix(...) + >>> issue_manager_that_computes_knn_graph = ... + + """ + key = "statistics" + statistics: Dict[str, Any] = issue_manager.info.get(key, {}) + if statistics: + self.info[key].update(statistics) + + def _update_issues(self, issue_manager): + overlapping_columns = list(set(self.issues.columns) & set(issue_manager.issues.columns)) + if overlapping_columns: + warnings.warn( + f"Overwriting columns {overlapping_columns} in self.issues with " + f"columns from issue manager {issue_manager}." + ) + self.issues.drop(columns=overlapping_columns, inplace=True) + self.issues = self.issues.join(issue_manager.issues, how="outer") + + def _update_issue_info(self, issue_name, new_info): + if issue_name in self.info: + warnings.warn(f"Overwriting key {issue_name} in self.info") + self.info[issue_name] = new_info + + def collect_issues_from_issue_manager(self, issue_manager: IssueManager) -> None: + """ + Collects results from an IssueManager and update the corresponding + attributes of the Datalab object. + + This includes: + - self.issues + - self.issue_summary + - self.info + + Parameters + ---------- + issue_manager : + IssueManager object to collect results from. + """ + self._update_issues(issue_manager) + + if issue_manager.issue_name in self.issue_summary["issue_type"].values: + warnings.warn( + f"Overwriting row in self.issue_summary with " + f"row from issue manager {issue_manager}." + ) + self.issue_summary = self.issue_summary[ + self.issue_summary["issue_type"] != issue_manager.issue_name + ] + issue_column_name: str = f"is_{issue_manager.issue_name}_issue" + num_issues: int = int(issue_manager.issues[issue_column_name].sum()) + self.issue_summary = pd.concat( + [ + self.issue_summary, + issue_manager.summary.assign(num_issues=num_issues), + ], + axis=0, + ignore_index=True, + ) + self._update_issue_info(issue_manager.issue_name, issue_manager.info) + + def collect_issues_from_imagelab(self, imagelab: "Imagelab", issue_types: List[str]) -> None: + pass # pragma: no cover + + def set_health_score(self) -> None: + """Set the health score for the dataset based on the issue summary. + + Currently, the health score is the mean of the scores for each issue type. + """ + self.info["statistics"]["health_score"] = self.issue_summary["score"].mean() + + +def get_data_statistics(data: Data) -> Dict[str, Any]: + """Get statistics about a dataset. + + This function is called to initialize the "statistics" info in all `Datalab` objects. + + Parameters + ---------- + data : Data + Data object containing the dataset. + """ + statistics: Dict[str, Any] = { + "num_examples": len(data), + "multi_label": False, + "health_score": None, + } + if data.labels.is_available: + class_names = data.class_names + statistics["class_names"] = class_names + statistics["num_classes"] = len(class_names) + return statistics diff --git a/cleanlab/datalab/internal/display.py b/cleanlab/datalab/internal/display.py new file mode 100644 index 0000000..3f37fdc --- /dev/null +++ b/cleanlab/datalab/internal/display.py @@ -0,0 +1,135 @@ +""" +Module that handles the string representation of Datalab objects. +""" + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Dict, List, Optional, Type + +from cleanlab.datalab.internal.task import Task + +if TYPE_CHECKING: # pragma: no cover + from cleanlab.datalab.internal.data_issues import DataIssues + + +class RepresentationStrategy(ABC): + def __init__(self, data_issues: "DataIssues"): + self.data_issues = data_issues + + @property + def checks_run(self) -> bool: + return not self.data_issues.issues.empty + + @property + def num_examples(self) -> Optional[int]: + return self.data_issues.get_info("statistics").get("num_examples") + + @property + def num_classes(self) -> Optional[int]: + return self.data_issues.get_info("statistics").get("num_classes") + + @property + def issues_identified(self) -> str: + return ( + self.data_issues.issue_summary["num_issues"].sum() if self.checks_run else "Not checked" + ) + + def show_task(self, task: "Task") -> str: + return f"task={str(task).capitalize()}" + + def show_checks_run(self) -> str: + return f"checks_run={self.checks_run}" + + def show_num_examples(self) -> str: + return f"num_examples={self.num_examples}" if self.num_examples is not None else "" + + def show_num_classes(self) -> str: + return f"num_classes={self.num_classes}" if self.num_classes is not None else "" + + def show_issues_identified(self) -> str: + return f"issues_identified={self.issues_identified}" + + @abstractmethod + def represent(self) -> str: + pass + + def to_string(self, task: "Task") -> str: + """What is displayed if user executes: print(datalab)""" + info_list = [ + f"Task: {str(task).capitalize()}", + f"Checks run: {'Yes' if self.checks_run else 'No'}", + f"Number of examples: {self.num_examples if self.num_examples is not None else 'Unknown'}", + f"Number of classes: {self.num_classes if self.num_classes is not None else 'Unknown'}", + f"Issues identified: {self.issues_identified}", + ] + + return "Datalab:\n" + "\n".join(info_list) + + +class ClassificationRepresentation(RepresentationStrategy): + def represent(self) -> str: + display_strings: List[str] = [ + self.show_task(Task.CLASSIFICATION), + self.show_checks_run(), + self.show_num_examples(), + self.show_num_classes(), + self.show_issues_identified(), + ] + # Drop empty strings + display_strings = [s for s in display_strings if bool(s)] + display_str = ", ".join(display_strings) + return f"Datalab({display_str})" + + +class RegressionRepresentation(RepresentationStrategy): + def represent(self) -> str: + display_strings: List[str] = [ + self.show_task(Task.REGRESSION), + self.show_checks_run(), + self.show_num_examples(), + self.show_issues_identified(), + ] + # Drop empty strings + display_strings = [s for s in display_strings if bool(s)] + display_str = ", ".join(display_strings) + return f"Datalab({display_str})" + + +class MultilabelRepresentation(RepresentationStrategy): + def represent(self) -> str: + display_strings: List[str] = [ + self.show_task(Task.MULTILABEL), + self.show_checks_run(), + self.show_num_examples(), + self.show_num_classes(), + self.show_issues_identified(), + ] + # Drop empty strings + display_strings = [s for s in display_strings if bool(s)] + display_str = ", ".join(display_strings) + return f"Datalab({display_str})" + + +class _Displayer: + def __init__(self, data_issues: "DataIssues", task: "Task") -> None: + self.data_issues = data_issues + self.task = task + self.representation_strategy = self._get_representation_strategy() + + def _get_representation_strategy(self) -> RepresentationStrategy: + strategies: Dict[str, Type[RepresentationStrategy]] = { + "classification": ClassificationRepresentation, + "regression": RegressionRepresentation, + "multilabel": MultilabelRepresentation, + } + strategy_class = strategies.get(self.task.value) + if strategy_class is None: + raise ValueError(f"Unsupported task type: {self.task}") + return strategy_class(self.data_issues) + + def __repr__(self) -> str: + """What is displayed in console if user executes: >>> datalab""" + return self.representation_strategy.represent() + + def __str__(self) -> str: + """What is displayed if user executes: print(datalab)""" + return self.representation_strategy.to_string(self.task) diff --git a/cleanlab/datalab/internal/helper_factory.py b/cleanlab/datalab/internal/helper_factory.py new file mode 100644 index 0000000..4cff486 --- /dev/null +++ b/cleanlab/datalab/internal/helper_factory.py @@ -0,0 +1,92 @@ +from typing import Dict, Optional, Type + +from cleanlab.datalab.internal.adapter.imagelab import ( + ImagelabDataIssuesAdapter, + ImagelabIssueFinderAdapter, + ImagelabReporterAdapter, +) +from cleanlab.datalab.internal.data import Data +from cleanlab.datalab.internal.data_issues import ( + _InfoStrategy, + DataIssues, + _ClassificationInfoStrategy, + _RegressionInfoStrategy, + _MultilabelInfoStrategy, +) +from cleanlab.datalab.internal.issue_finder import IssueFinder +from cleanlab.datalab.internal.report import Reporter +from cleanlab.datalab.internal.task import Task + + +def issue_finder_factory(imagelab): + if imagelab: + return ImagelabIssueFinderAdapter + else: + return IssueFinder + + +def report_factory(imagelab): + if imagelab: + return ImagelabReporterAdapter + else: + return Reporter + + +class _DataIssuesBuilder: + """A helper class for constructing DataIssues instances. + It uses the builder pattern to allow users to specify the desired + configuration of the DataIssues instance. + It uses the `set_X` naming convention for methods that set the + desired configuration, before calling the `build` method to + construct the DataIssues instance. + """ + + def __init__(self, data: Data): + self.data = data + self.imagelab = None + self.task: Optional[Task] = None + + def set_imagelab(self, imagelab): + self.imagelab = imagelab + return self + + def set_task(self, task: Task): + """Set the task that the data is intended for. + + Parameters + ---------- + task : Task + Specific machine learning task that the datset is intended for. + See details about supported tasks in :py:class:`Task `. + """ + self.task = task + return self + + def build(self) -> DataIssues: + data_issues_class = self._data_issues_factory() + strategy = self._select_info_strategy() + return data_issues_class(self.data, strategy) + + def _data_issues_factory(self) -> Type[DataIssues]: + """Factory method that selects the appropriate class for + constructing the DataIssues instance. + """ + if self.imagelab: + return ImagelabDataIssuesAdapter + else: + return DataIssues + + def _select_info_strategy(self) -> Type[_InfoStrategy]: + """The DataIssues class takes in a strategy class + for processing info dictionaries. This method selects + the appropriate strategy class based on the task during + the `build` method-call. + """ + _default_return = _ClassificationInfoStrategy + strategy_lookup: Dict[Task, Type[_InfoStrategy]] = { + Task.REGRESSION: _RegressionInfoStrategy, + Task.MULTILABEL: _MultilabelInfoStrategy, + } + if self.task is None: + return _default_return + return strategy_lookup.get(self.task, _default_return) diff --git a/cleanlab/datalab/internal/issue_finder.py b/cleanlab/datalab/internal/issue_finder.py new file mode 100644 index 0000000..1f2acce --- /dev/null +++ b/cleanlab/datalab/internal/issue_finder.py @@ -0,0 +1,489 @@ +""" +Module for the :class:`IssueFinder` class, which is responsible for configuring, +creating and running issue managers. + +It determines which types of issues to look for, instatiates the IssueManagers +via a factory, run the issue managers +(:py:meth:`IssueManager.find_issues `), +and collects the results to :py:class:`DataIssues `. + +.. note:: + + This module is not intended to be used directly. Instead, use the public-facing + :py:meth:`Datalab.find_issues ` method. +""" + +from __future__ import annotations + +import warnings +from typing import TYPE_CHECKING, Any, Dict, Optional + +import numpy as np +from scipy.sparse import csr_matrix + +from cleanlab.datalab.internal.issue_manager_factory import ( + _IssueManagerFactory, + list_default_issue_types, +) +from cleanlab.datalab.internal.model_outputs import ( + MultiClassPredProbs, + MultiLabelPredProbs, + RegressionPredictions, +) +from cleanlab.datalab.internal.task import Task + +if TYPE_CHECKING: # pragma: no cover + from typing import Callable + + import numpy.typing as npt + + from cleanlab.datalab.datalab import Datalab + + +_CLASSIFICATION_ARGS_DICT = { + "label": ["pred_probs", "features"], + "outlier": ["pred_probs", "features", "knn_graph"], + "near_duplicate": ["features", "knn_graph"], + "non_iid": ["pred_probs", "features", "knn_graph"], + # The underperforming_group issue type requires a pair of inputs: (pred_probs, ) + "underperforming_group": ["pred_probs", "features", "knn_graph", "cluster_ids"], + "data_valuation": ["features", "knn_graph"], + "class_imbalance": [], + "null": ["features"], +} +_REGRESSION_ARGS_DICT = { + "label": ["features", "predictions"], + "outlier": ["features", "knn_graph"], + "near_duplicate": ["features", "knn_graph"], + "non_iid": ["features", "knn_graph"], + "data_valuation": ["features", "knn_graph"], + "null": ["features"], +} + +_MULTILABEL_ARGS_DICT = { + "label": ["pred_probs"], + "outlier": ["features", "knn_graph"], + "near_duplicate": ["features", "knn_graph"], + "non_iid": ["features", "knn_graph"], + "data_valuation": ["features", "knn_graph"], + "null": ["features"], +} + + +def _resolve_required_args_for_classification(**kwargs): + """Resolves the required arguments for each issue type intended for classification tasks.""" + initial_args_dict = _CLASSIFICATION_ARGS_DICT.copy() + args_dict = { + issue_type: {arg: kwargs.get(arg, None) for arg in initial_args_dict[issue_type]} + for issue_type in initial_args_dict + } + + # Some issue types (like class-imbalance) have no required args. + # This conditional lambda is used to include them in args dict. + keep_empty_argument = lambda k: not len(_CLASSIFICATION_ARGS_DICT[k]) + + # Remove None values from argument list, rely on default values in IssueManager + args_dict = { + k: {k2: v2 for k2, v2 in v.items() if v2 is not None} + for k, v in args_dict.items() + if (v or keep_empty_argument(k)) + } + + # Prefer `knn_graph` over `features` if both are provided. + for v in args_dict.values(): + if "cluster_ids" in v and ("knn_graph" in v or "features" in v): + warnings.warn( + "`cluster_ids` have been provided with `knn_graph` or `features`." + "Issue managers that require cluster labels will prefer" + "`cluster_ids` over computation of cluster labels using" + "`knn_graph` or `features`. " + ) + if "knn_graph" in v and "features" in v: + warnings.warn( + "Both `features` and `knn_graph` were provided. " + "Most issue managers will likely prefer using `knn_graph` " + "instead of `features` for efficiency." + ) + + # Only keep issue types that have at least one argument + # or those that require no arguments. + args_dict = {k: v for k, v in args_dict.items() if (v or keep_empty_argument(k))} + + return args_dict + + +def _resolve_required_args_for_regression(**kwargs): + """Resolves the required arguments for each issue type intended for regression tasks.""" + initial_args_dict = _REGRESSION_ARGS_DICT.copy() + args_dict = { + issue_type: {arg: kwargs.get(arg, None) for arg in initial_args_dict[issue_type]} + for issue_type in initial_args_dict + } + # Some issue types have no required args. + # This conditional lambda is used to include them in args dict. + keep_empty_argument = lambda k: not len(_REGRESSION_ARGS_DICT[k]) + + # Remove None values from argument list, rely on default values in IssueManager + args_dict = { + k: {k2: v2 for k2, v2 in v.items() if v2 is not None} + for k, v in args_dict.items() + if v or keep_empty_argument(k) + } + + # Only keep issue types that have at least one argument + # or those that require no arguments. + args_dict = {k: v for k, v in args_dict.items() if (v or keep_empty_argument(k))} + + return args_dict + + +def _resolve_required_args_for_multilabel(**kwargs): + """Resolves the required arguments for each issue type intended for multilabel tasks.""" + initial_args_dict = _MULTILABEL_ARGS_DICT.copy() + args_dict = { + issue_type: {arg: kwargs.get(arg, None) for arg in initial_args_dict[issue_type]} + for issue_type in initial_args_dict + } + # Some issue types have no required args. + # This conditional lambda is used to include them in args dict. + keep_empty_argument = lambda k: not len(_MULTILABEL_ARGS_DICT[k]) + + # Remove None values from argument list, rely on default values in IssueManager + args_dict = { + k: {k2: v2 for k2, v2 in v.items() if v2 is not None} + for k, v in args_dict.items() + if v or keep_empty_argument(k) # Allow label issues to require no arguments + } + + # Only keep issue types that have at least one argument + # or those that require no arguments. + args_dict = {k: v for k, v in args_dict.items() if (v or keep_empty_argument(k))} + + return args_dict + + +def _select_strategy_for_resolving_required_args(task: Task) -> Callable: + """Helper function that selects the strategy for resolving required arguments for each issue type. + + Each strategy resolves the required arguments for each issue type. + + This is a helper function that filters out any issue manager + that does not have the required arguments. + + This does not consider custom hyperparameters for each issue type. + + Parameters + ---------- + task : str + The type of machine learning task that the dataset is used for. + + Returns + ------- + args_dict : + Dictionary of required arguments for each issue type, if available. + """ + strategies = { + Task.CLASSIFICATION: _resolve_required_args_for_classification, + Task.REGRESSION: _resolve_required_args_for_regression, + Task.MULTILABEL: _resolve_required_args_for_multilabel, + } + selected_strategy = strategies.get(task, None) + if selected_strategy is None: + raise ValueError(f"No strategy for resolving required arguments for task '{task}'") + return selected_strategy + + +class IssueFinder: + """ + The IssueFinder class is responsible for managing the process of identifying + issues in the dataset by handling the creation and execution of relevant + IssueManagers. It serves as a coordinator or helper class for the Datalab class + to encapsulate the specific behavior of the issue finding process. + + At a high level, the IssueFinder is responsible for: + + - Determining which types of issues to look for. + - Instantiating the appropriate IssueManagers using a factory. + - Running the IssueManagers' `find_issues` methods. + - Collecting the results into a DataIssues instance. + + Parameters + ---------- + datalab : Datalab + The Datalab instance associated with this IssueFinder. + + task : str + The type of machine learning task that the dataset is used for. + + verbosity : int + Controls the verbosity of the output during the issue finding process. + + Note + ---- + This class is not intended to be used directly. Instead, use the + `Datalab.find_issues` method which internally utilizes an IssueFinder instance. + """ + + def __init__(self, datalab: "Datalab", task: Task, verbosity=1): + self.datalab = datalab + self.task = task + self.verbosity = verbosity + + def find_issues( + self, + *, + pred_probs: Optional[np.ndarray] = None, + features: Optional[npt.NDArray] = None, + knn_graph: Optional[csr_matrix] = None, + issue_types: Optional[Dict[str, Any]] = None, + ) -> None: + """ + Checks the dataset for all sorts of common issues in real-world data (in both labels and feature values). + + You can use Datalab to find issues in your data, utilizing *any* model you have already trained. + This method only interacts with your model via its predictions or embeddings (and other functions thereof). + The more of these inputs you provide, the more types of issues Datalab can detect in your dataset/labels. + If you provide a subset of these inputs, Datalab will output what insights it can based on the limited information from your model. + + Note + ---- + This method is not intended to be used directly. Instead, use the + :py:meth:`Datalab.find_issues ` method. + + Note + ---- + The issues are saved in the ``self.datalab.data_issues.issues`` attribute, but are not returned. + + Parameters + ---------- + pred_probs : + Out-of-sample predicted class probabilities made by the model for every example in the dataset. + To best detect label issues, provide this input obtained from the most accurate model you can produce. + + If provided for classification, this must be a 2D array with shape ``(num_examples, K)`` where K is the number of classes in the dataset. + If provided for regression, this must be a 1D array with shape ``(num_examples,)``. + + features : Optional[np.ndarray] + Feature embeddings (vector representations) of every example in the dataset. + + If provided, this must be a 2D array with shape (num_examples, num_features). + + knn_graph : + Sparse matrix representing distances between examples in the dataset in a k nearest neighbor graph. + + For details, refer to the documentation of the same argument in :py:class:`Datalab.find_issues ` + + issue_types : + Collection specifying which types of issues to consider in audit and any non-default parameter settings to use. + If unspecified, a default set of issue types and recommended parameter settings is considered. + + This is a dictionary of dictionaries, where the keys are the issue types of interest + and the values are dictionaries of parameter values that control how each type of issue is detected (only for advanced users). + More specifically, the values are constructor keyword arguments passed to the corresponding ``IssueManager``, + which is responsible for detecting the particular issue type. + + .. seealso:: + :py:class:`IssueManager ` + """ + + issue_types_copy = self.get_available_issue_types( + pred_probs=pred_probs, + features=features, + knn_graph=knn_graph, + issue_types=issue_types, + ) + + if not issue_types_copy: + return None + + new_issue_managers = [ + factory(datalab=self.datalab, **issue_types_copy.get(factory.issue_name, {})) + for factory in _IssueManagerFactory.from_list( + list(issue_types_copy.keys()), task=self.task + ) + ] + + failed_managers = [] + data_issues = self.datalab.data_issues + for issue_manager, arg_dict in zip(new_issue_managers, issue_types_copy.values()): + try: + if self.verbosity: + print(f"Finding {issue_manager.issue_name} issues ...") + issue_manager.find_issues(**arg_dict) + data_issues.collect_statistics(issue_manager) + data_issues.collect_issues_from_issue_manager(issue_manager) + except Exception as e: + print(f"Error in {issue_manager.issue_name}: {e}") + failed_managers.append(issue_manager) + if failed_managers: + print(f"Failed to check for these issue types: {failed_managers}") + data_issues.set_health_score() + + def _set_issue_types( + self, + issue_types: Optional[Dict[str, Any]], + required_defaults_dict: Dict[str, Any], + ) -> Dict[str, Any]: + """Set necessary configuration for each IssueManager in a dictionary. + + While each IssueManager defines default values for its arguments, + the Datalab class needs to organize the calls to each IssueManager + with different arguments, some of which may be user-provided. + + Parameters + ---------- + issue_types : + Dictionary of issue types and argument configuration for their respective IssueManagers. + If None, then the `required_defaults_dict` is used. + + required_defaults_dict : + Dictionary of default parameter configuration for each issue type. + + Returns + ------- + issue_types_copy : + Dictionary of issue types and their parameter configuration. + The input `issue_types` is copied and updated with the necessary default values. + """ + if issue_types is not None: + issue_types_copy = issue_types.copy() + self._check_missing_args(required_defaults_dict, issue_types_copy) + else: + issue_types_copy = required_defaults_dict.copy() + # keep only default issue types + issue_types_copy = { + issue: issue_types_copy[issue] + for issue in list_default_issue_types(self.task) + if issue in issue_types_copy + } + + # Check that all required arguments are provided. + self._validate_issue_types_dict(issue_types_copy, required_defaults_dict) + + # Remove None values from argument list, rely on default values in IssueManager + for key, value in issue_types_copy.items(): + issue_types_copy[key] = {k: v for k, v in value.items() if v is not None} + + return issue_types_copy + + @staticmethod + def _check_missing_args(required_defaults_dict, issue_types): + for key, issue_type_value in issue_types.items(): + missing_args = set(required_defaults_dict.get(key, {})) - set(issue_type_value.keys()) + # Impute missing arguments with default values. + missing_dict = { + missing_arg: required_defaults_dict[key][missing_arg] + for missing_arg in missing_args + } + issue_types[key].update(missing_dict) + + @staticmethod + def _validate_issue_types_dict( + issue_types: Dict[str, Any], required_defaults_dict: Dict[str, Any] + ) -> None: + missing_required_args_dict = {} + for issue_name, required_args in required_defaults_dict.items(): + if issue_name in issue_types: + missing_args = set(required_args.keys()) - set(issue_types[issue_name].keys()) + if missing_args: + missing_required_args_dict[issue_name] = missing_args + if any(missing_required_args_dict.values()): + error_message = "" + for issue_name, missing_required_args in missing_required_args_dict.items(): + error_message += f"Required argument {missing_required_args} for issue type {issue_name} was not provided.\n" + raise ValueError(error_message) + + def get_available_issue_types(self, **kwargs): + """Returns a dictionary of issue types that can be used in :py:meth:`Datalab.find_issues + ` method.""" + + pred_probs = kwargs.get("pred_probs", None) + features = kwargs.get("features", None) + knn_graph = kwargs.get("knn_graph", None) + issue_types = kwargs.get("issue_types", None) + + model_output = None + if pred_probs is not None: + model_output_dict = { + Task.REGRESSION: RegressionPredictions, + Task.CLASSIFICATION: MultiClassPredProbs, + Task.MULTILABEL: MultiLabelPredProbs, + } + + model_output_class = model_output_dict.get(self.task) + if model_output_class is None: + raise ValueError(f"Unknown task type '{self.task}'") + + model_output = model_output_class(pred_probs) + + if model_output is not None: + # A basic trick to assign the model output to the correct argument + # E.g. Datalab accepts only `pred_probs`, but those are assigned to the `predictions` argument for regression-related issue_managers + kwargs.update({model_output.argument: model_output.collect()}) + + # Determine which parameters are required for each issue type + strategy_for_resolving_required_args = _select_strategy_for_resolving_required_args( + self.task + ) + required_args_per_issue_type = strategy_for_resolving_required_args(**kwargs) + + issue_types_copy = self._set_issue_types(issue_types, required_args_per_issue_type) + if issue_types is None: + # Only run default issue types if no issue types are specified + issue_types_copy = { + issue: issue_types_copy[issue] + for issue in list_default_issue_types(self.task) + if issue in issue_types_copy + } + drop_label_check = ( + "label" in issue_types_copy + and not self.datalab.has_labels + and self.task != Task.REGRESSION + ) + + if drop_label_check: + warnings.warn("No labels were provided. " "The 'label' issue type will not be run.") + issue_types_copy.pop("label") + + outlier_check_needs_features = ( + self.task == "classification" + and "outlier" in issue_types_copy + and not self.datalab.has_labels + ) + if outlier_check_needs_features: + no_features = features is None + no_knn_graph = knn_graph is None + pred_probs_given = issue_types_copy["outlier"].get("pred_probs", None) is not None + + only_pred_probs_given = pred_probs_given and no_features and no_knn_graph + if only_pred_probs_given: + warnings.warn( + "No labels were provided. " "The 'outlier' issue type will not be run." + ) + issue_types_copy.pop("outlier") + + drop_class_imbalance_check = ( + "class_imbalance" in issue_types_copy + and not self.datalab.has_labels + and self.task == Task.CLASSIFICATION + ) + if drop_class_imbalance_check: + issue_types_copy.pop("class_imbalance") + + required_pairs_for_underperforming_group = [ + ("pred_probs", "features"), + ("pred_probs", "knn_graph"), + ("pred_probs", "cluster_ids"), + ] + drop_underperforming_group_check = "underperforming_group" in issue_types_copy and not any( + all( + key in issue_types_copy["underperforming_group"] + and issue_types_copy["underperforming_group"].get(key) is not None + for key in pair + ) + for pair in required_pairs_for_underperforming_group + ) + if drop_underperforming_group_check: + issue_types_copy.pop("underperforming_group") + + return issue_types_copy diff --git a/cleanlab/datalab/internal/issue_manager/__init__.py b/cleanlab/datalab/internal/issue_manager/__init__.py new file mode 100644 index 0000000..f37be1e --- /dev/null +++ b/cleanlab/datalab/internal/issue_manager/__init__.py @@ -0,0 +1,10 @@ +from .issue_manager import IssueManager # isort:skip +from .duplicate import NearDuplicateIssueManager +from .label import LabelIssueManager +from .outlier import OutlierIssueManager +from .noniid import NonIIDIssueManager +from .imbalance import ClassImbalanceIssueManager +from .underperforming_group import UnderperformingGroupIssueManager +from .data_valuation import DataValuationIssueManager +from .null import NullIssueManager +from .identifier_column import IdentifierColumnIssueManager diff --git a/cleanlab/datalab/internal/issue_manager/data_valuation.py b/cleanlab/datalab/internal/issue_manager/data_valuation.py new file mode 100644 index 0000000..fee9044 --- /dev/null +++ b/cleanlab/datalab/internal/issue_manager/data_valuation.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +from typing import ( + TYPE_CHECKING, + Any, + Callable, + ClassVar, + Dict, + List, + Optional, + Union, +) + + +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix + +from cleanlab.data_valuation import data_shapley_knn +from cleanlab.datalab.internal.issue_manager import IssueManager +from cleanlab.datalab.internal.issue_manager.knn_graph_helpers import ( + num_neighbors_in_knn_graph, + set_knn_graph, +) + +if TYPE_CHECKING: # pragma: no cover + import numpy.typing as npt + import pandas as pd + from cleanlab.datalab.datalab import Datalab + + +class DataValuationIssueManager(IssueManager): + """ + Detect which examples in a dataset are least valuable via an approximate Data Shapely value. + + Examples + -------- + .. code-block:: python + + >>> from cleanlab import Datalab + >>> import numpy as np + >>> from sklearn.neighbors import NearestNeighbors + >>> + >>> # Generate two distinct clusters + >>> X = np.vstack([ + ... np.random.normal(-1, 1, (25, 2)), + ... np.random.normal(1, 1, (25, 2)), + ... ]) + >>> y = np.array([0]*25 + [1]*25) + >>> + >>> # Initialize Datalab with data + >>> lab = Datalab(data={"y": y}, label_name="y") + >>> + >>> # Creating a knn_graph for data valuation + >>> knn = NearestNeighbors(n_neighbors=10).fit(X) + >>> knn_graph = knn.kneighbors_graph(mode='distance') + >>> + >>> # Specifying issue types for data valuation + >>> issue_types = {"data_valuation": {}} + >>> lab.find_issues(knn_graph=knn_graph, issue_types=issue_types) + """ + + description: ClassVar[ + str + ] = """ + Examples that contribute minimally to a model's training + receive lower valuation scores. + Since the original knn-shapley value is in [-1, 1], we transform it to [0, 1] by: + + .. math:: + 0.5 \times (\text{shapley} + 1) + + here shapley is the original knn-shapley value. + """ + + issue_name: ClassVar[str] = "data_valuation" + issue_score_key: ClassVar[str] + verbosity_levels: ClassVar[Dict[int, List[str]]] = { + 0: [], + 1: [], + 2: [], + 3: ["average_data_valuation"], + } + + DEFAULT_THRESHOLD = 0.5 + + def __init__( + self, + datalab: Datalab, + metric: Optional[Union[str, Callable]] = None, + threshold: Optional[float] = None, + k: int = 10, + **kwargs, + ): + super().__init__(datalab) + self.metric = metric + self.k = k + self.threshold = threshold if threshold is not None else self.DEFAULT_THRESHOLD + + def find_issues( + self, + features: Optional[npt.NDArray] = None, + **kwargs, + ) -> None: + """Calculate the data valuation score with a provided or existing knn graph. + Based on KNN-Shapley value described in https://arxiv.org/abs/1911.07128 + The larger the score, the more valuable the data point is, the more contribution it will make to the model's training. + + Parameters + ---------- + knn_graph : csr_matrix + A sparse matrix representing the knn graph. + """ + labels = self.datalab.labels + if not isinstance(labels, np.ndarray): + error_msg = ( + f"Expected labels to be a numpy array of shape (n_samples,) to use with DataValuationIssueManager, " + f"but got {type(labels)} instead." + ) + raise TypeError(error_msg) + + knn_graph, self.metric, _ = set_knn_graph( + features=features, + find_issues_kwargs=kwargs, + metric=self.metric, + k=self.k, + statistics=self.datalab.get_info("statistics"), + ) + + # TODO: Check self.k against user-provided knn-graphs across all issue managers + num_neighbors = num_neighbors_in_knn_graph(knn_graph) + if self.k > num_neighbors: + raise ValueError( + f"The provided knn graph has {num_neighbors} neighbors, which is less than the required {self.k} neighbors. " + "Please ensure that the knn graph you provide has at least as many neighbors as the required value of k." + ) + + scores = data_shapley_knn(labels, knn_graph=knn_graph, k=self.k) + + self.issues = pd.DataFrame( + { + f"is_{self.issue_name}_issue": scores < self.threshold, + self.issue_score_key: scores, + }, + ) + self.summary = self.make_summary(score=scores.mean()) + + self.info = self.collect_info(issues=self.issues, knn_graph=knn_graph) + + def collect_info(self, issues: pd.DataFrame, knn_graph: csr_matrix) -> dict: + issues_info = { + "num_low_valuation_issues": sum(issues[f"is_{self.issue_name}_issue"]), + "average_data_valuation": issues[self.issue_score_key].mean(), + } + + params_dict = { + "metric": self.metric, + "k": self.k, + "threshold": self.threshold, + } + + statistics_dict = self._build_statistics_dictionary(knn_graph=knn_graph) + + info_dict = { + **issues_info, + **params_dict, + **statistics_dict, + } + + return info_dict + + def _build_statistics_dictionary(self, knn_graph: csr_matrix) -> Dict[str, Dict[str, Any]]: + statistics_dict: Dict[str, Dict[str, Any]] = {"statistics": {}} + + # Add the knn graph as a statistic if necessary + graph_key = "weighted_knn_graph" + old_knn_graph = self.datalab.get_info("statistics").get(graph_key, None) + old_graph_exists = old_knn_graph is not None + prefer_new_graph = ( + not old_graph_exists + or (old_knn_graph is not None and knn_graph.nnz > old_knn_graph.nnz) + or self.metric != self.datalab.get_info("statistics").get("knn_metric", None) + ) + if prefer_new_graph: + statistics_dict["statistics"][graph_key] = knn_graph + if self.metric is not None: + statistics_dict["statistics"]["knn_metric"] = self.metric + + return statistics_dict diff --git a/cleanlab/datalab/internal/issue_manager/duplicate.py b/cleanlab/datalab/internal/issue_manager/duplicate.py new file mode 100644 index 0000000..f36e076 --- /dev/null +++ b/cleanlab/datalab/internal/issue_manager/duplicate.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, List, Optional, Union +import warnings + +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix + + +from cleanlab.datalab.internal.issue_manager import IssueManager +from cleanlab.datalab.internal.issue_manager.knn_graph_helpers import set_knn_graph +from cleanlab.internal.constants import EPSILON + +if TYPE_CHECKING: # pragma: no cover + import numpy.typing as npt + from cleanlab.datalab.datalab import Datalab + + +class NearDuplicateIssueManager(IssueManager): + """Manages issues related to near-duplicate examples.""" + + description: ClassVar[ + str + ] = """A (near) duplicate issue refers to two or more examples in + a dataset that are extremely similar to each other, relative + to the rest of the dataset. The examples flagged with this issue + may be exactly duplicated, or lie atypically close together when + represented as vectors (i.e. feature embeddings). + """ + issue_name: ClassVar[str] = "near_duplicate" + verbosity_levels = { + 0: [], + 1: [], + 2: ["threshold"], + } + + def __init__( + self, + datalab: Datalab, + metric: Optional[Union[str, Callable]] = None, + threshold: float = 0.13, + k: int = 10, + **_, + ): + super().__init__(datalab) + self.metric = metric + self.threshold = self._set_threshold(threshold) + self.k = k + self.near_duplicate_sets: List[List[int]] = [] + + def find_issues( + self, + features: Optional[npt.NDArray] = None, + **kwargs, + ) -> None: + knn_graph, self.metric, _ = set_knn_graph( + features=features, + find_issues_kwargs=kwargs, + metric=self.metric, + k=self.k, + statistics=self.datalab.get_info("statistics"), + ) + + N = knn_graph.shape[0] + nn_distances = knn_graph.data.reshape(N, -1)[:, 0] + median_nn_distance = max(np.median(nn_distances), EPSILON) # avoid threshold = 0 + self.near_duplicate_sets = self._neighbors_within_radius( + knn_graph, self.threshold, median_nn_distance + ) + + # Flag every example in a near-duplicate set as a near-duplicate issue + all_near_duplicates = np.unique(np.concatenate(self.near_duplicate_sets)) + is_issue_column = np.zeros(N, dtype=bool) + is_issue_column[all_near_duplicates] = True + temperature = 1.0 / median_nn_distance + scores = _compute_scores_with_exp_transform(nn_distances, temperature=temperature) + self.issues = pd.DataFrame( + { + f"is_{self.issue_name}_issue": is_issue_column, + self.issue_score_key: scores, + }, + ) + + self.summary = self.make_summary(score=scores.mean()) + self.info = self.collect_info(knn_graph=knn_graph, median_nn_distance=median_nn_distance) + + @staticmethod + def _neighbors_within_radius(knn_graph: csr_matrix, threshold: float, median: float): + """Returns a list of lists of indices of near-duplicate examples. + + Each list of indices represents a set of near-duplicate examples. + + If the list is empty for a given example, then that example is not + a near-duplicate of any other example. + """ + + N = knn_graph.shape[0] + distances = knn_graph.data.reshape(N, -1) + # Create a mask for the threshold + mask = distances < threshold * median + + # Update the indptr to reflect the new number of neighbors + indptr = np.zeros(knn_graph.indptr.shape, dtype=knn_graph.indptr.dtype) + indptr[1:] = np.cumsum(mask.sum(axis=1)) + + # Filter the knn_graph based on the threshold + indices = knn_graph.indices[mask.ravel()] + near_duplicate_sets = [indices[indptr[i] : indptr[i + 1]] for i in range(N)] + + # Second pass over the data is required to ensure each item is included in the near-duplicate sets of its own near-duplicates. + # This is important because a "near-duplicate" relationship is reciprocal. + # For example, if item A is a near-duplicate of item B, then item B should also be considered a near-duplicate of item A. + # NOTE: This approach does not assure that the sets are ordered by increasing distance. + for i, near_duplicates in enumerate(near_duplicate_sets): + for j in near_duplicates: + if i not in near_duplicate_sets[j]: + near_duplicate_sets[j] = np.append(near_duplicate_sets[j], i) + + return near_duplicate_sets + + def collect_info(self, knn_graph: csr_matrix, median_nn_distance: float) -> dict: + issues_dict = { + "average_near_duplicate_score": self.issues[self.issue_score_key].mean(), + "near_duplicate_sets": self.near_duplicate_sets, + } + + params_dict = { + "metric": self.metric, + "k": self.k, + "threshold": self.threshold, + } + + N = knn_graph.shape[0] + dists = knn_graph.data.reshape(N, -1)[:, 0] + nn_ids = knn_graph.indices.reshape(N, -1)[:, 0] + + knn_info_dict = { + "nearest_neighbor": nn_ids.tolist(), + "distance_to_nearest_neighbor": dists.tolist(), + "median_distance_to_nearest_neighbor": median_nn_distance, + } + + statistics_dict = self._build_statistics_dictionary(knn_graph=knn_graph) + + info_dict = { + **issues_dict, + **params_dict, + **knn_info_dict, + **statistics_dict, + } + return info_dict + + def _build_statistics_dictionary(self, knn_graph: csr_matrix) -> Dict[str, Dict[str, Any]]: + statistics_dict: Dict[str, Dict[str, Any]] = {"statistics": {}} + + # Add the knn graph as a statistic if necessary + graph_key = "weighted_knn_graph" + old_knn_graph = self.datalab.get_info("statistics").get(graph_key, None) + old_graph_exists = old_knn_graph is not None + prefer_new_graph = ( + not old_graph_exists + or (old_knn_graph is not None and knn_graph.nnz > old_knn_graph.nnz) + or self.metric != self.datalab.get_info("statistics").get("knn_metric", None) + ) + if prefer_new_graph: + statistics_dict["statistics"][graph_key] = knn_graph + if self.metric is not None: + statistics_dict["statistics"]["knn_metric"] = self.metric + + return statistics_dict + + def _set_threshold( + self, + threshold: float, + ) -> float: + """Computes nearest-neighbors thresholding for near-duplicate detection.""" + if threshold < 0: + warnings.warn( + f"Computed threshold {threshold} is less than 0. " + "Setting threshold to 0." + "This may indicate that either the only a few examples are in the dataset, " + "or the data is heavily skewed." + ) + threshold = 0 + return threshold + + +def _compute_scores_with_exp_transform(nn_distances: np.ndarray, temperature: float) -> np.ndarray: + r"""Compute near-duplicate scores from nearest neighbor distances. + + This is a non-linear transformation of the nearest neighbor distances that + maps distances to scores in the range [0, 1]. + + Note + ---- + + This transformation is given by the following formula: + + .. math:: + + \text{score}(d, t) = 1 - e^{-dt} + + where :math:`d` is the nearest neighbor distance and :math:`t > 0` is a temperature parameter. + + Parameters + ---------- + nn_distances : + The nearest neighbor distances for each example. + + Returns + ------- + scores : + The near-duplicate scores for each example. The scores are in the range [0, 1]. + A lower score indicates that an example is more likely to be a near-duplicate than + an example with a higher score. + A score of 0 indicates that an example has an exact duplicate. + """ + if temperature <= 0: + raise ValueError("Temperature must be greater than 0.") + + scores = 1 - np.exp(-temperature * nn_distances) + + # Ensure that for nn_distances approximately equal to 0, the score is set to 0 + inds = np.isclose(nn_distances, 0) + scores[inds] = 0 + + return scores diff --git a/cleanlab/datalab/internal/issue_manager/identifier_column.py b/cleanlab/datalab/internal/issue_manager/identifier_column.py new file mode 100644 index 0000000..553a6b8 --- /dev/null +++ b/cleanlab/datalab/internal/issue_manager/identifier_column.py @@ -0,0 +1,131 @@ +from typing import ClassVar, List, Optional, Union + +import numpy as np +import numpy.typing as npt +import pandas as pd + +from cleanlab.datalab.internal.issue_manager import IssueManager + + +class IdentifierColumnIssueManager(IssueManager): + """Manages issues related to identifier columns in feature columns""" + + description: ClassVar[ + str + ] = """Checks whether there is an identifier_column in the features of a dataset. + Identifier columns are defined as a column i in features such that + set(features[:,i]) = set(c, c+1, ..., c+n) for some integer c, + where n = num-rows of features. If there is such a column, the dataset has + the identifier_column issue + """ + issue_name: ClassVar[str] = "identifier_column" + verbosity_levels = { + 0: [], + 1: ["identifier_columns"], + 2: [], + } + + def _is_sequential(self, arr: npt.NDArray) -> bool: + """ + Check if the elements in the array are sequential. + + Parameters: + arr: The input array. + + Returns: + A boolean indicating whether the elements in the array are sequential. + """ + if arr.size == 0: + return False + unique_sorted = np.unique(arr) # Returns a sorted unique list + min_val, max_val = unique_sorted[0], unique_sorted[-1] + expected_range = np.arange(min_val, max_val + 1) + if expected_range.size == 1 or unique_sorted.size != expected_range.size: + return False + return bool((expected_range == unique_sorted).all()) + + def _prepare_features( + self, features: Optional[Union[npt.NDArray, pd.DataFrame, list, dict]] + ) -> Union[npt.NDArray, List[npt.NDArray]]: + """ + Prepare the features for issue check. + + Args: + features: The input features. + + Returns: + features: features as npt.NDArray + """ + if isinstance(features, np.ndarray): + return features.T # Transpose if it's a NumPy array + # to keep the datatype of the string columns for dicts and pandas dataframes consistent + # we convert the string columns to dtype=str, otherwise we ran into error in our tests + elif isinstance(features, pd.DataFrame): + result = [] + for col in features.columns: + if pd.api.types.is_string_dtype( + features[col] + ): # detect string columns in both pandas 2.x and 3.x. + arr = np.array(features[col].values).astype(str) + else: + arr = np.array(features[col].values) + result.append(arr) + return result + elif isinstance(features, dict): + result = [] + for value in features.values(): + if isinstance(value[0], str): + arr = np.array(value).astype(str) + else: + arr = np.array(value) + result.append(arr) + return result + elif isinstance(features, list): + for col_list in features: + if not isinstance(col_list, list) and not isinstance(col_list, np.ndarray): + raise ValueError( + "features must be a list of lists or numpy arrays if a list is passed." + ) + return [np.array(col_list) for col_list in features] + else: + raise ValueError("features must be a numpy array, pandas DataFrame, list, or dict.") + + def find_issues( + self, features: Optional[Union[npt.NDArray, pd.DataFrame, list, dict]], **kwargs + ) -> None: + """ + Find identifier columns in the given dataset. + + Parameters: + features (Optional[npt.NDArray | pd.DataFrame | list | dict]): + The dataset to check for identifier columns. + Returns: + None + """ + if features is None: + raise ValueError("features must be provided to check for identifier columns.") + processed_features = self._prepare_features(features) + + is_identifier_column = np.array( + [ + np.issubdtype(feature.dtype, np.integer) and self._is_sequential(feature) + for feature in processed_features + ] + ) + identifier_column_indices = np.where(is_identifier_column) + # this issue does not reflect rows at all so we set the score to 1.0 for all rows in the issue attribute + # and set the is_identifier_column_issue to False + num_rows = processed_features[0].size + self.issues = pd.DataFrame( + { + f"is_{self.issue_name}_issue": False, + self.issue_score_key: np.ones(num_rows), + }, + ) + # score in summary should be 1.0 if the issue is not present and 0.0 if at least one column is an identifier column + self.summary = self.make_summary(score=1.0 - float(is_identifier_column.any())) + # more elegant way to set the score in summary + self.info = { + "identifier_columns": identifier_column_indices[0].tolist(), + "num_identifier_columns": identifier_column_indices[0].size, + } diff --git a/cleanlab/datalab/internal/issue_manager/imbalance.py b/cleanlab/datalab/internal/issue_manager/imbalance.py new file mode 100644 index 0000000..beb4a2f --- /dev/null +++ b/cleanlab/datalab/internal/issue_manager/imbalance.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, ClassVar + +import numpy as np +import pandas as pd +from cleanlab.datalab.internal.issue_manager import IssueManager + +if TYPE_CHECKING: # pragma: no cover + from cleanlab.datalab.datalab import Datalab + + +class ClassImbalanceIssueManager(IssueManager): + """Manages issues related to imbalance class examples. + + Parameters + ---------- + datalab: + The Datalab instance that this issue manager searches for issues in. + + threshold: + Minimum fraction of samples of each class that are present in a dataset without class imbalance. + + """ + + description: ClassVar[str] = ( + """Examples belonging to the most under-represented class in the dataset.""" + ) + + issue_name: ClassVar[str] = "class_imbalance" + verbosity_levels = { + 0: ["Rarest Class"], + 1: [], + 2: [], + } + + def __init__(self, datalab: Datalab, threshold: float = 0.1, **_): + super().__init__(datalab) + self.threshold = threshold + + def find_issues( + self, + **kwargs, + ) -> None: + labels = self.datalab.labels + if not isinstance(labels, np.ndarray): + error_msg = ( + f"Expected labels to be a numpy array of shape (n_samples,) to use with ClassImbalanceIssueManager, " + f"but got {type(labels)} instead." + ) + raise TypeError(error_msg) + K = len(self.datalab.class_names) + class_probs = np.bincount(labels) / len(labels) + rarest_class_idx = int(np.argmin(class_probs)) + # solely one class is identified as rarest, ties go to class w smaller integer index + scores = np.where(labels == rarest_class_idx, class_probs[rarest_class_idx], 1) + imbalance_exists = class_probs[rarest_class_idx] < self.threshold * (1 / K) + rarest_class_issue = rarest_class_idx if imbalance_exists else -1 + is_issue_column = labels == rarest_class_issue + rarest_class_name = self.datalab._label_map.get(rarest_class_issue, "NA") + + self.issues = pd.DataFrame( + { + f"is_{self.issue_name}_issue": is_issue_column, + self.issue_score_key: scores, + }, + ) + self.summary = self.make_summary(score=class_probs[rarest_class_idx]) + self.info = self.collect_info(class_name=rarest_class_name, labels=labels) + + def collect_info(self, class_name: str, labels: np.ndarray) -> dict: + params_dict = { + "threshold": self.threshold, + "Rarest Class": class_name, + "given_label": labels, + } + info_dict = {**params_dict} + return info_dict diff --git a/cleanlab/datalab/internal/issue_manager/issue_manager.py b/cleanlab/datalab/internal/issue_manager/issue_manager.py new file mode 100644 index 0000000..4873d2b --- /dev/null +++ b/cleanlab/datalab/internal/issue_manager/issue_manager.py @@ -0,0 +1,330 @@ +from __future__ import annotations + +from abc import ABC, ABCMeta, abstractmethod +from itertools import chain +from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Set, Tuple, Type, TypeVar +import json + +import numpy as np +import pandas as pd + +if TYPE_CHECKING: # pragma: no cover + from cleanlab.datalab.datalab import Datalab + + +T = TypeVar("T", bound="IssueManager") +TM = TypeVar("TM", bound="IssueManagerMeta") + + +class IssueManagerMeta(ABCMeta): + """Metaclass for IssueManager that adds issue_score_key to the class. + + :meta private: + """ + + issue_name: ClassVar[str] + issue_score_key: ClassVar[str] + verbosity_levels: ClassVar[Dict[int, List[str]]] = { + 0: [], + 1: [], + 2: [], + 3: [], + } + + def __new__( + meta: Type[TM], + name: str, + bases: Tuple[Type[Any], ...], + class_dict: Dict[str, Any], + ) -> TM: # Classes that inherit from ABC don't need to be modified + if ABC in bases: + return super().__new__(meta, name, bases, class_dict) + + # Ensure that the verbosity levels don't have keys other than those in ["issue", "info"] + verbosity_levels = class_dict.get("verbosity_levels", meta.verbosity_levels) + for level, level_list in verbosity_levels.items(): + if not isinstance(level_list, list): + raise ValueError( + f"Verbosity levels must be lists. " + f"Got {level_list} in {name}.verbosity_levels" + ) + prohibited_keys = [key for key in level_list if not isinstance(key, str)] + if prohibited_keys: + raise ValueError( + f"Verbosity levels must be lists of strings. " + f"Got {prohibited_keys} in {name}.verbosity_levels[{level}]" + ) + + # Concrete classes need to have an issue_name attribute + if "issue_name" not in class_dict: + raise TypeError("IssueManagers need an issue_name class variable") + + # Add issue_score_key to class + class_dict["issue_score_key"] = f"{class_dict['issue_name']}_score" + return super().__new__(meta, name, bases, class_dict) + + +class IssueManager(ABC, metaclass=IssueManagerMeta): + """Base class for managing data issues of a particular type in a Datalab. + + For each example in a dataset, the IssueManager for a particular type of issue should compute: + - A numeric severity score between 0 and 1, + with values near 0 indicating severe instances of the issue. + - A boolean `is_issue` value, which is True + if we believe this example suffers from the issue in question. + `is_issue` may be determined by thresholding the severity score + (with an a priori determined reasonable threshold value), + or via some other means (e.g. Confident Learning for flagging label issues). + + The IssueManager should also report: + - A global value between 0 and 1 summarizing how severe this issue is in the dataset overall + (e.g. the average severity across all examples in dataset + or count of examples where `is_issue=True`). + - Other interesting `info` about the issue and examples in the dataset, + and statistics estimated from current dataset that may be reused + to score this issue in future data. + For example, `info` for label issues could contain the: + confident_thresholds, confident_joint, predicted label for each example, etc. + Another example is for (near)-duplicate detection issue, where `info` could contain: + which set of examples in the dataset are all (nearly) identical. + + Implementing a new IssueManager: + - Define the `issue_name` class attribute, e.g. "label", "duplicate", "outlier", etc. + - Implement the abstract methods `find_issues` and `collect_info`. + - `find_issues` is responsible for computing computing the `issues` and `summary` dataframes. + - `collect_info` is responsible for computing the `info` dict. It is called by `find_issues`, + once the manager has set the `issues` and `summary` dataframes as instance attributes. + """ + + description: ClassVar[str] = "" + """Short text that summarizes the type of issues handled by this IssueManager. + + :meta hide-value: + """ + issue_name: ClassVar[str] + """Returns a key that is used to store issue summary results about the assigned Lab.""" + issue_score_key: ClassVar[str] + """Returns a key that is used to store issue score results about the assigned Lab.""" + verbosity_levels: ClassVar[Dict[int, List[str]]] = { + 0: [], + 1: [], + 2: [], + 3: [], + } + """A dictionary of verbosity levels and their corresponding dictionaries of + report items to print. + + :meta hide-value: + + Example + ------- + + >>> verbosity_levels = { + ... 0: [], + ... 1: ["some_info_key"], + ... 2: ["additional_info_key"], + ... } + """ + + def __init__(self, datalab: Datalab, **_): + self.datalab = datalab + self.info: Dict[str, Any] = {} + self.issues: pd.DataFrame = pd.DataFrame() + self.summary: pd.DataFrame = pd.DataFrame() + + def __repr__(self): + class_name = self.__class__.__name__ + return class_name + + @classmethod + def __init_subclass__(cls): + required_class_variables = [ + "issue_name", + ] + for var in required_class_variables: + if not hasattr(cls, var): + raise NotImplementedError(f"Class {cls.__name__} must define class variable {var}") + + @abstractmethod + def find_issues(self, *args, **kwargs) -> None: + """Finds occurrences of this particular issue in the dataset. + + Computes the `issues` and `summary` dataframes. Calls `collect_info` to compute the `info` dict. + """ + raise NotImplementedError + + def collect_info(self, *args, **kwargs) -> dict: + """Collects data for the info attribute of the Datalab. + + NOTE + ---- + This method is called by :py:meth:`find_issues` after :py:meth:`find_issues` has set the `issues` and `summary` dataframes + as instance attributes. + """ + raise NotImplementedError + + @classmethod + def make_summary(cls, score: float) -> pd.DataFrame: + """Construct a summary dataframe. + + Parameters + ---------- + score : + The overall score for this issue. + + Returns + ------- + summary : + A summary dataframe. + """ + if not 0 <= score <= 1: + raise ValueError(f"Score must be between 0 and 1. Got {score}.") + + return pd.DataFrame( + { + "issue_type": [cls.issue_name], + "score": [score], + }, + ) + + @classmethod + def report( + cls, + issues: pd.DataFrame, + summary: pd.DataFrame, + info: Dict[str, Any], + num_examples: int = 5, + verbosity: int = 0, + include_description: bool = False, + info_to_omit: Optional[List[str]] = None, + ) -> str: + """Compose a report of the issues found by this IssueManager. + + Parameters + ---------- + issues : + An issues dataframe. + + Example + ------- + >>> import pandas as pd + >>> issues = pd.DataFrame( + ... { + ... "is_X_issue": [True, False, True], + ... "X_score": [0.2, 0.9, 0.4], + ... }, + ... ) + + summary : + The summary dataframe. + + Example + ------- + >>> summary = pd.DataFrame( + ... { + ... "issue_type": ["X"], + ... "score": [0.5], + ... }, + ... ) + + info : + The info dict. + + Example + ------- + >>> info = { + ... "A": "val_A", + ... "B": ["val_B1", "val_B2"], + ... } + + num_examples : + The number of examples to print. + + verbosity : + The verbosity level of the report. + + include_description : + Whether to include a description of the issue in the report. + + Returns + ------- + report_str : + A string containing the report. + """ + + max_verbosity = max(cls.verbosity_levels.keys()) + top_level = max_verbosity + 1 + if verbosity not in list(cls.verbosity_levels.keys()) + [top_level]: + raise ValueError( + f"Verbosity level {verbosity} not supported. " + f"Supported levels: {cls.verbosity_levels.keys()}" + f"Use verbosity={top_level} to print all info." + ) + if issues.empty: + print(f"No issues found") + + topk_ids = issues.sort_values(by=cls.issue_score_key, ascending=True).index[:num_examples] + + score = summary["score"].loc[0] + report_str = f"{' ' + cls.issue_name + ' issues ':-^60}\n\n" + + if include_description and cls.description: + description = cls.description + if verbosity == 0: + description = description.split("\n\n", maxsplit=1)[0] + report_str += "About this issue:\n\t" + description + "\n\n" + report_str += ( + f"Number of examples with this issue: {issues[f'is_{cls.issue_name}_issue'].sum()}\n" + f"Overall dataset quality in terms of this issue: {score:.4f}\n\n" + ) + + info_to_print: Set[str] = set() + _info_to_omit = set(issues.columns).union(info_to_omit or []) + verbosity_levels_values = chain.from_iterable( + list(cls.verbosity_levels.values())[: verbosity + 1] + ) + info_to_print.update(set(verbosity_levels_values) - _info_to_omit) + if verbosity == top_level: + info_to_print.update(set(info.keys()) - _info_to_omit) + + report_str += "Examples representing most severe instances of this issue:\n" + report_str += issues.loc[topk_ids].to_string() + + def truncate(s, max_len=4) -> str: + if hasattr(s, "shape") or hasattr(s, "ndim"): + s = np.array(s) + if s.ndim > 1: + description = f"array of shape {s.shape}\n" + with np.printoptions(threshold=max_len): + if s.ndim == 2: + description += f"{s}" + if s.ndim > 2: + description += f"{s}" + return description + s = s.tolist() + + if isinstance(s, list): + if all([isinstance(s_, list) for s_ in s]): + return truncate(np.array(s, dtype=object), max_len=max_len) + if len(s) > max_len: + s = s[:max_len] + ["..."] + return str(s) + + if info_to_print: + info_to_print_dict = {key: info[key] for key in info_to_print} + # Print the info dict, truncating arrays to 4 elements, + report_str += f"\n\nAdditional Information: " + for key, value in info_to_print_dict.items(): + if key == "statistics": + continue + if isinstance(value, dict): + report_str += f"\n{key}:\n{json.dumps(value, indent=4)}" + elif isinstance(value, pd.DataFrame): + max_rows = 5 + df_str = value.head(max_rows).to_string() + if len(value) > max_rows: + df_str += f"\n... (total {len(value)} rows)" + report_str += f"\n{key}:\n{df_str}" + else: + report_str += f"\n{key}: {truncate(value)}" + return report_str diff --git a/cleanlab/datalab/internal/issue_manager/knn_graph_helpers.py b/cleanlab/datalab/internal/issue_manager/knn_graph_helpers.py new file mode 100644 index 0000000..499c526 --- /dev/null +++ b/cleanlab/datalab/internal/issue_manager/knn_graph_helpers.py @@ -0,0 +1,69 @@ +import numpy.typing as npt +from scipy.sparse import csr_matrix + + +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, cast + + +from cleanlab.internal.neighbor.knn_graph import create_knn_graph_and_index +from cleanlab.typing import Metric + +if TYPE_CHECKING: + from sklearn.neighbors import NearestNeighbors + + +def num_neighbors_in_knn_graph(knn_graph: csr_matrix) -> int: + """Calculate the number of neighbors per row in a knn graph.""" + return knn_graph.nnz // knn_graph.shape[0] + + +def _process_knn_graph_from_inputs( + user_find_issues_kwargs: Dict[str, Any], statistics: Dict[str, Any], k_for_recomputation: int +) -> Optional[csr_matrix]: + """Determine if a knn_graph is provided in the kwargs or if one is already stored in the associated Datalab instance.""" + provided_knn_graph: Optional[csr_matrix] = user_find_issues_kwargs.get("knn_graph", None) + existing_knn_graph = statistics.get("weighted_knn_graph", None) + + knn_graph: Optional[csr_matrix] = None + if provided_knn_graph is not None: + knn_graph = provided_knn_graph + elif existing_knn_graph is not None: + knn_graph = existing_knn_graph + num_neighbors = num_neighbors_in_knn_graph(knn_graph) if knn_graph is not None else -1 + needs_recompute = k_for_recomputation > num_neighbors + if needs_recompute: + # If the provided knn graph is insufficient, then we need to recompute the knn graph + # with the provided features + knn_graph = None + return knn_graph + + +def knn_exists(kwargs: Dict[str, Any], statistics: Dict[str, Any], k_needed: int) -> bool: + """Check if a sufficiently large knn graph exists in the kwargs or statistics.""" + return ( + _process_knn_graph_from_inputs(kwargs, statistics, k_for_recomputation=k_needed) is not None + ) + + +def set_knn_graph( + features: Optional[npt.NDArray], + find_issues_kwargs: Dict[str, Any], + metric: Optional[Metric], + k: int, + statistics: Dict[str, Any], +) -> Tuple[csr_matrix, Metric, Optional["NearestNeighbors"]]: + # This only fetches graph (optionally) + knn_graph = _process_knn_graph_from_inputs( + find_issues_kwargs, statistics, k_for_recomputation=k + ) + old_knn_metric = statistics.get("knn_metric", metric) + + missing_knn_graph = knn_graph is None + metric_changes = metric and metric != old_knn_metric + + knn: Optional[NearestNeighbors] = None + if missing_knn_graph or metric_changes: + assert features is not None, "Features must be provided to compute the knn graph." + knn_graph, knn = create_knn_graph_and_index(features, n_neighbors=k, metric=metric) + metric = knn.metric + return cast(csr_matrix, knn_graph), cast(Metric, metric), knn diff --git a/cleanlab/datalab/internal/issue_manager/label.py b/cleanlab/datalab/internal/issue_manager/label.py new file mode 100644 index 0000000..962a87a --- /dev/null +++ b/cleanlab/datalab/internal/issue_manager/label.py @@ -0,0 +1,273 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional + +import numpy as np +from sklearn.neighbors import KNeighborsClassifier +from sklearn.preprocessing import OneHotEncoder + +from cleanlab.classification import CleanLearning +from cleanlab.count import get_confident_thresholds +from cleanlab.datalab.internal.issue_manager import IssueManager +from cleanlab.internal.validation import assert_valid_inputs + +if TYPE_CHECKING: # pragma: no cover + import numpy.typing as npt + import pandas as pd + + from cleanlab.datalab.datalab import Datalab + + +class LabelIssueManager(IssueManager): + """Manages label issues in a Datalab. + + Parameters + ---------- + datalab : + A Datalab instance. + + k : + The number of nearest neighbors to consider when computing pred_probs from features. + Only applicable if features are provided and pred_probs are not. + + clean_learning_kwargs : + Keyword arguments to pass to the :py:meth:`CleanLearning ` constructor. + + health_summary_parameters : + Keyword arguments to pass to the :py:meth:`health_summary ` function. + """ + + description: ClassVar[ + str + ] = """Examples whose given label is estimated to be potentially incorrect + (e.g. due to annotation error) are flagged as having label issues. + """ + + issue_name: ClassVar[str] = "label" + verbosity_levels = { + 0: [], + 1: [], + 2: [], + 3: ["classes_by_label_quality", "overlapping_classes"], + } + + def __init__( + self, + datalab: Datalab, + k: int = 10, + clean_learning_kwargs: Optional[Dict[str, Any]] = None, + health_summary_parameters: Optional[Dict[str, Any]] = None, + **_, + ): + super().__init__(datalab) + self.cl = CleanLearning(**(clean_learning_kwargs or {})) + self.k = k + self.health_summary_parameters: Dict[str, Any] = ( + health_summary_parameters.copy() if health_summary_parameters else {} + ) + self._find_issues_inputs: Dict[str, bool] = {"features": False, "pred_probs": False} + self._reset() + + @staticmethod + def _process_find_label_issues_kwargs(**kwargs) -> Dict[str, Any]: + """Searches for keyword arguments that are meant for the + CleanLearning.find_label_issues method call + + Examples + -------- + >>> from cleanlab.datalab.internal.issue_manager.label import LabelIssueManager + >>> LabelIssueManager._process_find_label_issues_kwargs(thresholds=[0.1, 0.9]) + {'thresholds': [0.1, 0.9]} + """ + accepted_kwargs = [ + "thresholds", + "noise_matrix", + "inverse_noise_matrix", + "save_space", + "clf_kwargs", + "validation_func", + ] + return {k: v for k, v in kwargs.items() if k in accepted_kwargs and v is not None} + + def _reset(self) -> None: + """Reset the attributes of this manager based on the available datalab info + and the keyword arguments stored as instance attributes. + + This allows the builder to use pre-computed info from the datalab to speed up + some computations in the :py:meth:`find_issues` method. + """ + if not self.health_summary_parameters: + statistics_dict = self.datalab.get_info("statistics") + self.health_summary_parameters = { + "labels": self.datalab.labels, + "class_names": list(self.datalab._label_map.values()), + "num_examples": statistics_dict.get("num_examples"), + "joint": statistics_dict.get("joint", None), + "confident_joint": statistics_dict.get("confident_joint", None), + "multi_label": statistics_dict.get("multi_label", None), + "asymmetric": statistics_dict.get("asymmetric", None), + "verbose": False, + } + self.health_summary_parameters = { + k: v for k, v in self.health_summary_parameters.items() if v is not None + } + + def find_issues( + self, + pred_probs: Optional[npt.NDArray] = None, + features: Optional[npt.NDArray] = None, + **kwargs, + ) -> None: + """Find label issues in the datalab. + + Parameters + ---------- + pred_probs : + The predicted probabilities for each example. + + features : + The features for each example. + """ + if pred_probs is not None: + self._find_issues_inputs.update({"pred_probs": True}) + if pred_probs is None: + self._find_issues_inputs.update({"features": True}) + if features is None: + raise ValueError( + "Either pred_probs or features must be provided to find label issues." + ) + # produce out-of-sample pred_probs from features + labels = self.datalab.labels + if not isinstance(labels, np.ndarray): + error_msg = ( + f"Expected labels to be a numpy array of shape (n_samples,) to use in LabelIssueManager, " + f"but got {type(labels)} instead." + ) + raise TypeError(error_msg) + + knn = KNeighborsClassifier(n_neighbors=self.k + 1) + knn.fit(features, labels) + pred_probs = knn.predict_proba(features) + + encoder = OneHotEncoder() + label_transform = labels.reshape(-1, 1) + one_hot_label = encoder.fit_transform(label_transform) + + # adjust pred_probs so it is out-of-sample + pred_probs = np.asarray( + (pred_probs - 1 / (self.k + 1) * one_hot_label) * (self.k + 1) / self.k + ) + + self.health_summary_parameters.update({"pred_probs": pred_probs}) + # Find examples with label issues + labels = self.datalab.labels + self.issues = self.cl.find_label_issues( + labels=labels, + pred_probs=pred_probs, + **self._process_find_label_issues_kwargs(**kwargs), + ) + self.issues.rename(columns={"label_quality": self.issue_score_key}, inplace=True) + + summary_dict = self.get_health_summary(pred_probs=pred_probs) + + # Get a summarized dataframe of the label issues + self.summary = self.make_summary(score=summary_dict["overall_label_health_score"]) + + confident_thresholds = get_confident_thresholds(labels=labels, pred_probs=pred_probs) + # Collect info about the label issues + self.info = self.collect_info( + issues=self.issues, + summary_dict=summary_dict, + confident_thresholds=confident_thresholds, + ) + + # Drop columns from issues that are in the info + self.issues = self.issues.drop(columns=["given_label", "predicted_label"]) + + def get_health_summary(self, pred_probs) -> dict: + """Returns a short summary of the health of this Lab.""" + from cleanlab.dataset import health_summary + + # Validate input + self._validate_pred_probs(pred_probs) + + summary_kwargs = self._get_summary_parameters(pred_probs) + summary = health_summary(**summary_kwargs) + return summary + + def _get_summary_parameters(self, pred_probs) -> Dict["str", Any]: + """Collects a set of input parameters for the health summary function based on + any info available in the datalab. + + Parameters + ---------- + pred_probs : + The predicted probabilities for each example. + + kwargs : + Keyword arguments to pass to the health summary function. + + Returns + ------- + summary_parameters : + A dictionary of parameters to pass to the health summary function. + """ + if "confident_joint" in self.health_summary_parameters: + summary_parameters = { + "confident_joint": self.health_summary_parameters["confident_joint"] + } + elif all([x in self.health_summary_parameters for x in ["joint", "num_examples"]]): + summary_parameters = { + k: self.health_summary_parameters[k] for k in ["joint", "num_examples"] + } + else: + summary_parameters = { + "pred_probs": pred_probs, + "labels": self.datalab.labels, + } + + summary_parameters["class_names"] = self.health_summary_parameters["class_names"] + + for k in ["asymmetric", "verbose"]: + # Start with the health_summary_parameters, then override with kwargs + if k in self.health_summary_parameters: + summary_parameters[k] = self.health_summary_parameters[k] + + return ( + summary_parameters # will be called in `dataset.health_summary(**summary_parameters)` + ) + + def collect_info( + self, issues: pd.DataFrame, summary_dict: dict, confident_thresholds: np.ndarray + ) -> dict: + issues_info = { + "num_label_issues": sum(issues[f"is_{self.issue_name}_issue"]), + "average_label_quality": issues[self.issue_score_key].mean(), + "given_label": issues["given_label"].tolist(), + "predicted_label": issues["predicted_label"].tolist(), + } + + health_summary_info = { + "confident_joint": summary_dict["joint"], + "classes_by_label_quality": summary_dict["classes_by_label_quality"], + "overlapping_classes": summary_dict["overlapping_classes"], + } + + cl_info = {} + for k in self.cl.__dict__: + if k not in ["py", "noise_matrix", "inverse_noise_matrix", "confident_joint"]: + continue + cl_info[k] = self.cl.__dict__[k] + + info_dict = { + **issues_info, + **health_summary_info, + **cl_info, + "confident_thresholds": confident_thresholds.tolist(), + "find_issues_inputs": self._find_issues_inputs, + } + + return info_dict + + def _validate_pred_probs(self, pred_probs) -> None: + assert_valid_inputs(X=None, y=self.datalab.labels, pred_probs=pred_probs) diff --git a/cleanlab/datalab/internal/issue_manager/multilabel/__init__.py b/cleanlab/datalab/internal/issue_manager/multilabel/__init__.py new file mode 100644 index 0000000..174216c --- /dev/null +++ b/cleanlab/datalab/internal/issue_manager/multilabel/__init__.py @@ -0,0 +1 @@ +from .label import MultilabelIssueManager diff --git a/cleanlab/datalab/internal/issue_manager/multilabel/label.py b/cleanlab/datalab/internal/issue_manager/multilabel/label.py new file mode 100644 index 0000000..a43b33f --- /dev/null +++ b/cleanlab/datalab/internal/issue_manager/multilabel/label.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, ClassVar, Dict, List + +import pandas as pd + +from cleanlab.datalab.internal.issue_manager import IssueManager +from cleanlab.internal.multilabel_utils import onehot2int +from cleanlab.multilabel_classification.filter import find_label_issues +from cleanlab.multilabel_classification.rank import get_label_quality_scores + +if TYPE_CHECKING: # pragma: no cover + import numpy.typing as npt + import pandas as pd + + from cleanlab.datalab.datalab import Datalab + + +class MultilabelIssueManager(IssueManager): + """Manages label issues in Datalab for multilabel tasks. + + Parameters + ---------- + datalab : + A Datalab instance. + """ + + description: ClassVar[ + str + ] = """Examples whose given label(s) are estimated to be potentially incorrect + (e.g. due to annotation error) are flagged as having label issues. + """ + + _PREDICTED_LABEL_THRESH = 0.5 + """Internal variable specifying threshold for predicted label.""" + + issue_name: ClassVar[str] = "label" + verbosity_levels = { + 0: [], + 1: [], + 2: [], + 3: [], + } + + def __init__( + self, + datalab: Datalab, + **_, + ): + super().__init__(datalab) + + @staticmethod + def _process_find_label_issues_kwargs(**kwargs: Dict[str, Any]) -> Dict[str, Any]: + """Searches for keyword arguments that are meant for the + multilabel_classification.filter.find_label_issues method call. + + Examples + -------- + >>> from cleanlab.datalab.internal.issue_manager.multilabel.label import MultilabelIssueManager + >>> MultilabelIssueManager._process_find_label_issues_kwargs(frac_noise=0.9) + {'frac_noise': 0.9} + """ + accepted_kwargs = [ + "filter_by", + "frac_noise", + "num_to_remove_per_class", + "min_examples_per_class", + "confident_joint", + "n_jobs", + "verbose", + "low_memory", + ] + return {k: v for k, v in kwargs.items() if k in accepted_kwargs and v is not None} + + @staticmethod + def _process_get_label_quality_scores_kwargs(**kwargs: Dict[str, Any]) -> Dict[str, Any]: + """Searches for keyword arguments that are meant for the + multilabel_classification.rank.get_label_quality_scores method call. + + Examples + -------- + >>> from cleanlab.datalab.internal.issue_manager.multilabel.label import MultilabelIssueManager + >>> MultilabelIssueManager._process_get_label_quality_scores_kwargs(method="self_confidence") + {'method': 'self_confidence'} + """ + accepted_kwargs = ["method", "adjust_pred_probs", "aggregator_kwargs"] + return {k: v for k, v in kwargs.items() if k in accepted_kwargs and v is not None} + + def find_issues( + self, + pred_probs: npt.NDArray, + **kwargs, + ) -> None: + """Find label issues in a multilabel dataset. + + Parameters + ---------- + pred_probs : + The predicted probabilities for each example. + """ + predicted_labels = onehot2int(pred_probs > self._PREDICTED_LABEL_THRESH) + + # Find examples with label issues + assert isinstance(self.datalab.labels, List) # Type Narrowing + is_issue_column = find_label_issues( + labels=self.datalab.labels, + pred_probs=pred_probs, + **self._process_find_label_issues_kwargs(**kwargs), + ) + scores = get_label_quality_scores( + labels=self.datalab.labels, + pred_probs=pred_probs, + **self._process_get_label_quality_scores_kwargs(**kwargs), + ) + + self.issues = pd.DataFrame( + { + f"is_{self.issue_name}_issue": is_issue_column, + self.issue_score_key: scores, + }, + ) + # Get a summarized dataframe of the label issues + self.summary = self.make_summary(score=scores.mean()) + + # Collect info about the label issues + self.info = self.collect_info(self.datalab.labels, predicted_labels) + + def collect_info( + self, given_labels: List[List[int]], predicted_labels: List[List[int]] + ) -> Dict[str, Any]: + issues_info = { + "given_label": given_labels, + "predicted_label": predicted_labels, + } + return issues_info diff --git a/cleanlab/datalab/internal/issue_manager/noniid.py b/cleanlab/datalab/internal/issue_manager/noniid.py new file mode 100644 index 0000000..056b4d3 --- /dev/null +++ b/cleanlab/datalab/internal/issue_manager/noniid.py @@ -0,0 +1,446 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, Optional, Union, cast +import itertools + +from scipy.stats import gaussian_kde +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix + +from cleanlab.datalab.internal.issue_manager import IssueManager +from cleanlab.datalab.internal.issue_manager.knn_graph_helpers import knn_exists, set_knn_graph + +if TYPE_CHECKING: # pragma: no cover + import numpy.typing as npt + from cleanlab.datalab.datalab import Datalab + + +def simplified_kolmogorov_smirnov_test( + neighbor_histogram: npt.NDArray[np.float64], + non_neighbor_histogram: npt.NDArray[np.float64], +) -> float: + """Computes the Kolmogorov-Smirnov statistic between two groups of data. + The statistic is the largest difference between the empirical cumulative + distribution functions (ECDFs) of the two groups. + + Parameters + ---------- + neighbor_histogram : + Histogram data for the nearest neighbor group. + + non_neighbor_histogram : + Histogram data for the non-neighbor group. + + Returns + ------- + statistic : + The KS statistic between the two ECDFs. + + Note + ---- + - Both input arrays should have the same length. + - The input arrays are histograms, which means they contain the count + or frequency of values in each group. The data in the histograms + should be normalized so that they sum to one. + + To calculate the KS statistic, the function first calculates the ECDFs + for both input arrays, which are step functions that show the cumulative + sum of the data up to each point. The function then calculates the + largest absolute difference between the two ECDFs. + """ + + neighbor_cdf = np.cumsum(neighbor_histogram) + non_neighbor_cdf = np.cumsum(non_neighbor_histogram) + + statistic = np.max(np.abs(neighbor_cdf - non_neighbor_cdf)) + return statistic + + +class NonIIDIssueManager(IssueManager): + """Manages issues related to non-iid data distributions. + + Parameters + ---------- + datalab : + The Datalab instance that this issue manager searches for issues in. + + metric : + The distance metric used to compute the KNN graph of the examples in the dataset. + If set to `None`, the metric will be automatically selected based on the dimensionality + of the features used to represent the examples in the dataset. + + k : + The number of nearest neighbors to consider when computing the KNN graph of the examples. + + num_permutations : + The number of trials to run when performing permutation testing to determine whether + the distribution of index-distances between neighbors in the dataset is IID or not. + + Note + ---- + This class will only flag a single example as an issue if the dataset is considered non-IID. This type of issue + is more relevant to the entire dataset as a whole, rather than to individual examples. + + """ + + description: ClassVar[ + str + ] = """Whether the dataset exhibits statistically significant + violations of the IID assumption like: + changepoints or shift, drift, autocorrelation, etc. + The specific violation considered is whether the + examples are ordered such that almost adjacent examples + tend to have more similar feature values. + """ + issue_name: ClassVar[str] = "non_iid" + verbosity_levels = { + 0: ["p-value"], + 1: [], + 2: [], + } + + def __init__( + self, + datalab: Datalab, + metric: Optional[Union[str, Callable]] = None, + k: int = 10, + num_permutations: int = 25, + seed: Optional[int] = 0, + significance_threshold: float = 0.05, + **_, + ): + super().__init__(datalab) + self.metric = metric + self.k = k + self.num_permutations = num_permutations + self.tests = { + "ks": simplified_kolmogorov_smirnov_test, + } + self.background_distribution = None + self.seed = seed + self.significance_threshold = significance_threshold + + # TODO: Temporary flag introduced to decide on storing knn graphs based on pred_probs. + # Revisit and finalize the implementation. + self._skip_storing_knn_graph_for_pred_probs: bool = False + + @staticmethod + def _determine_optional_features( + features: Optional[npt.NDArray], + pred_probs: Optional[np.ndarray], + ) -> Optional[npt.NDArray]: + """ + Determines the feature array to be used for constructing a knn-graph. Prioritizing the original features array over pred_probs. + If neither are provided, returns None. + + Parameters + ---------- + features : + Original feature array or None. + + pred_probs : + Predicted probabilities array or None. + + Returns + ------- + features_to_use : + Either the original feature array or the predicted probabilities array, + intended for constructing the knn-graph. + + Notes + ----- + A knn-graph constructed from predicted probabilities should not be stored in the statistics. But this kind + of knn-graph is allowed for the purpose of running a non-IID check. + """ + if features is not None: + return features + + if pred_probs is not None: + return pred_probs + + return None + + def find_issues( + self, + features: Optional[npt.NDArray] = None, + pred_probs: Optional[np.ndarray] = None, + **kwargs, + ) -> None: + statistics = self.datalab.get_info("statistics") + + # Crucial when building knn graphs with pred_probs instead of features, where only the + # latter is preferred for storage. + self._determine_if_knn_graph_storage_should_be_skipped( + features, pred_probs, kwargs, statistics, self.k + ) + + knn_graph, self.metric, _ = set_knn_graph( + features=self._determine_optional_features(features, pred_probs), + find_issues_kwargs=kwargs, + metric=self.metric, + k=self.k, + statistics=statistics, + ) + + self.neighbor_index_choices = self._get_neighbors(knn_graph=knn_graph) + + self.num_neighbors = self.k + + indices = np.arange(self.N) + self.neighbor_index_distances = np.abs(indices.reshape(-1, 1) - self.neighbor_index_choices) + + self.statistics = self._get_statistics(self.neighbor_index_distances) + + self.p_value = self._permutation_test(num_permutations=self.num_permutations) + + scores = self._score_dataset() + issue_mask = np.zeros(self.N, dtype=bool) + if self.p_value < self.significance_threshold: + issue_mask[scores.argmin()] = True + self.issues = pd.DataFrame( + { + f"is_{self.issue_name}_issue": issue_mask, + self.issue_score_key: scores, + }, + ) + + self.summary = self.make_summary(score=self.p_value) + + self.info = self.collect_info(knn_graph=knn_graph) + + def _determine_if_knn_graph_storage_should_be_skipped( + self, features, pred_probs, kwargs, statistics, k + ) -> None: + """Decide whether to skip storing the knn graph based on the availability of pred_probs. + + Should only happend when a new knn graph needs to be computed, and that it + can only be computed from pred_probs. + """ + sufficient_knn_graph_available = knn_exists(kwargs, statistics, k) + pred_probs_needed = ( + not sufficient_knn_graph_available and features is None and pred_probs is not None + ) + if pred_probs_needed: + self._skip_storing_knn_graph_for_pred_probs = True + + def collect_info(self, knn_graph: csr_matrix) -> dict: + issues_dict = { + "p-value": self.p_value, + } + + params_dict = { + "metric": self.metric, + "k": self.k, + } + + statistics_dict = self._build_statistics_dictionary(knn_graph=knn_graph) + + info_dict = { + **issues_dict, + **params_dict, # type: ignore[arg-type] + **statistics_dict, # type: ignore[arg-type] + } + return info_dict + + def _build_statistics_dictionary(self, knn_graph: csr_matrix) -> Dict[str, Dict[str, Any]]: + statistics_dict: Dict[str, Dict[str, Any]] = {"statistics": {}} + + if self._skip_storing_knn_graph_for_pred_probs: + return statistics_dict + # Add the knn graph as a statistic if necessary + graph_key = "weighted_knn_graph" + old_knn_graph = self.datalab.get_info("statistics").get(graph_key, None) + old_graph_exists = old_knn_graph is not None + prefer_new_graph = ( + (knn_graph is not None and not old_graph_exists) + or (old_knn_graph is not None and knn_graph.nnz > old_knn_graph.nnz) + or self.metric != self.datalab.get_info("statistics").get("knn_metric", None) + ) + if prefer_new_graph: + statistics_dict["statistics"][graph_key] = knn_graph + if self.metric is not None: + statistics_dict["statistics"]["knn_metric"] = self.metric + + return statistics_dict + + def _permutation_test(self, num_permutations) -> float: + N = self.N + + if self.seed is not None: + np.random.seed(self.seed) + perms = np.fromiter( + itertools.chain.from_iterable( + np.random.permutation(N) for i in range(num_permutations) + ), + dtype=int, + ).reshape(num_permutations, N) + + neighbor_index_choices = self.neighbor_index_choices + neighbor_index_choices = neighbor_index_choices.reshape(1, *neighbor_index_choices.shape) + perm_neighbor_choices = perms[:, neighbor_index_choices].reshape( + num_permutations, *neighbor_index_choices.shape[1:] + ) + neighbor_index_distances = np.abs(perms[..., None] - perm_neighbor_choices).reshape( + num_permutations, -1 + ) + + statistics = [] + for neighbor_index_dist in neighbor_index_distances: + stats = self._get_statistics( + neighbor_index_dist, + ) + statistics.append(stats) + + ks_stats = np.array([stats["ks"] for stats in statistics]) + ks_stats_kde = gaussian_kde(ks_stats) + p_value = ks_stats_kde.integrate_box(self.statistics["ks"], 100) + + return p_value + + def _score_dataset(self) -> npt.NDArray[np.float64]: + """This function computes a variant of the KS statistic for each + datapoint. Rather than computing the maximum difference + between the CDF of the neighbor distances (foreground + distribution) and the CDF of the all index distances + (background distribution), we compute the absolute difference + in area-under-the-curve of the two CDFs. + + The foreground distribution is computed by sampling the + neighbor distances from the KNN graph, but the background + distribution is computed analytically. The background CDF for + a datapoint i can be split up into three parts. Let d = min(i, + N - i - 1). + + 1. For 0 < j <= d, the slope of the CDF is 2 / (N - 1) since + there are two datapoints in the dataset that are distance j + from datapoint i. We call this threshold the 'double distance + threshold' + + 2. For d < j <= N - d - 1, the slope of the CDF is + 1 / (N - 1) since there is only one datapoint in the dataset + that is distance j from datapoint i. + + 3. For j > N - d - 1, the slope of the CDF is 0 and is + constant at 1.0 since there are no datapoints in the dataset + that are distance j from datapoint i. + + We compute the area differences on each of the k intervals for + which the foreground CDF is constant which allows for the + possibility that the background CDF may intersect the + foreground CDF on this interval. We do not account for these + cases when computing absolute AUC difference. + + Our algorithm is simple, sort the k sampled neighbor + distances. Then, for each of the k neighbor distances sampled, + compute the AUC for each CDF up to that point. Then, subtract + from each area the previous area in the sorted order to get + the AUC of the CDF on the interval between those two + points. Subtract the background interval AUCs from the + foreground interval AUCs, take the absolute value, and + sum. The algorithm is vectorized such that this statistic is + computed for each of the N datapoints simultaneously. + + The statistics are then normalized by their respective maximum + possible distance (N - d - 1) and then mapped to [0,1] via + tanh. + """ + N = self.N + + sorted_neighbors = np.sort(self.neighbor_index_distances, axis=1) + + # find the maximum distance that occurs with double probability + middle_idx = np.floor((N - 1) / 2).astype(int) + double_distances = np.arange(N).reshape(N, 1) + double_distances[double_distances > middle_idx] -= N - 1 + double_distances = np.abs(double_distances) + + sorted_neighbors = np.hstack([sorted_neighbors, np.ones((N, 1)) * (N - 1)]).astype(int) + + # the set of distances that are less than the double distance threshold + set_beginning = sorted_neighbors <= double_distances + # the set of distances that are greater than the double distance threshold but have nonzero probability + set_middle = (sorted_neighbors > double_distances) & ( + sorted_neighbors <= (N - double_distances - 1) + ) + # the set of distances that occur with 0 probability + set_end = sorted_neighbors > (N - double_distances - 1) + + shifted_neighbors = np.zeros(sorted_neighbors.shape) + shifted_neighbors[:, 1:] = sorted_neighbors[:, :-1] + diffs = sorted_neighbors - shifted_neighbors # the distances between the sorted indices + + area_beginning = (double_distances**2) / (N - 1) + length = N - 2 * double_distances - 1 + a = 2 * double_distances / (N - 1) + area_middle = 0.5 * (a + 1) * length + + # compute the area under the CDF for each of the indices in sorted_neighbors + background_area = np.zeros(diffs.shape) + background_diffs = np.zeros(diffs.shape) + background_area[set_beginning] = ((sorted_neighbors**2) / (N - 1))[set_beginning] + background_area[set_middle] = ( + area_beginning + + 0.5 + * ( + (sorted_neighbors + 3 * double_distances) + * (sorted_neighbors - double_distances) + / (N - 1) + ) + )[set_middle] + background_area[set_end] = ( + area_beginning + area_middle + (sorted_neighbors - (N - double_distances - 1) * 1.0) + )[set_end] + + # compute the area under the CDF between indices in sorted_neighbors + shifted_background = np.zeros(background_area.shape) + shifted_background[:, 1:] = background_area[:, :-1] + background_diffs = background_area - shifted_background + + # compute the foreground CDF and AUC between indices in sorted_neighbors + foreground_cdf = np.arange(sorted_neighbors.shape[1]) / (sorted_neighbors.shape[1] - 1) + foreground_diffs = foreground_cdf.reshape(1, -1) * diffs + + # compute the differences between foreground and background area intervals + area_diffs = np.abs(foreground_diffs - background_diffs) + stats = np.sum(area_diffs, axis=1) + + # normalize scores by the index and transform to [0, 1] + indices = np.arange(N) + reverse = N - indices + normalizer = np.where(indices > reverse, indices, reverse) + + scores = stats / normalizer + scores = np.tanh(-1 * scores) + 1 + return scores + + def _get_neighbors(self, knn_graph: csr_matrix) -> np.ndarray: + """ + Given a knn graph, returns an (N, k) array in + which j is in A[i] if item i and j are nearest neighbors. + """ + self.N = knn_graph.shape[0] + kneighbors = knn_graph.indices.reshape(self.N, -1) + return kneighbors + + def _get_statistics( + self, + neighbor_index_distances, + ) -> dict[str, float]: + neighbor_index_distances = neighbor_index_distances.flatten() + sorted_neighbors = np.sort(neighbor_index_distances) + sorted_neighbors = np.hstack([sorted_neighbors, np.ones((1)) * (self.N - 1)]).astype(int) + + if self.background_distribution is None: + self.background_distribution = (self.N - np.arange(1, self.N)) / ( + self.N * (self.N - 1) / 2 + ) + + background_distribution = cast(np.ndarray, self.background_distribution) + background_cdf = np.cumsum(background_distribution) + + foreground_cdf = np.arange(sorted_neighbors.shape[0]) / (sorted_neighbors.shape[0] - 1) + + statistic = np.max(np.abs(foreground_cdf - background_cdf[sorted_neighbors - 1])) + statistics = {"ks": statistic} + return statistics diff --git a/cleanlab/datalab/internal/issue_manager/null.py b/cleanlab/datalab/internal/issue_manager/null.py new file mode 100644 index 0000000..5f1fde5 --- /dev/null +++ b/cleanlab/datalab/internal/issue_manager/null.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +from collections import Counter +from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional + +import numpy as np +import pandas as pd + +from cleanlab.datalab.internal.issue_manager import IssueManager + +if TYPE_CHECKING: # pragma: no cover + import numpy.typing as npt + + +class NullIssueManager(IssueManager): + """Manages issues related to null/missing values in the rows of features. + + Parameters + ---------- + datalab : + The Datalab instance that this issue manager searches for issues in. + """ + + description: ClassVar[ + str + ] = """Examples identified with the null issue correspond to rows that have null/missing values across all feature columns (i.e. the entire row is missing values). + """ + issue_name: ClassVar[str] = "null" + verbosity_levels = { + 0: [], + 1: [], + 2: ["most_common_issue"], + } + + @staticmethod + def _calculate_null_issues( + features: npt.NDArray[Any], + ) -> tuple[npt.NDArray[np.bool_], npt.NDArray[np.float64], npt.NDArray[np.bool_]]: + """Tracks the number of null values in each row of a feature array, + computes quality scores based on the fraction of null values in each row, + and returns a boolean array indicating whether each row only has null values.""" + cols = features.shape[1] + null_tracker = pd.isna(features) + non_null_count = cols - null_tracker.sum(axis=1) + scores = non_null_count / cols + is_null_issue = non_null_count == 0 + return is_null_issue, scores, null_tracker + + def find_issues( + self, + features: Optional[npt.NDArray | pd.DataFrame] = None, + **kwargs, + ) -> None: + if features is None: + raise ValueError("features must be provided to check for null values.") + # Support features as a numpy array. Temporarily allow this issuecheck to convert a DataFrame to a numpy array. + if isinstance(features, pd.DataFrame): + features = features.to_numpy() + + is_null_issue, scores, null_tracker = self._calculate_null_issues(features=features) + + self.issues = pd.DataFrame( + { + f"is_{self.issue_name}_issue": is_null_issue, + self.issue_score_key: scores, + }, + ) + + self.summary = self.make_summary(score=scores.mean()) + self.info = self.collect_info(null_tracker) + + @staticmethod + def _most_common_issue( + null_tracker: np.ndarray, + ) -> dict[str, dict[str, str | int | list[int] | list[int | None]]]: + """ + Identify and return the most common null value pattern across all rows + and count the number of rows with this pattern. + + Parameters + ------------ + null_tracker : np.ndarray + A boolean array of the same shape as features, where True indicates null/missing entries. + + Returns + -------- + Dict[str, Any] + A dictionary containing the most common issue pattern and the count of rows with this pattern. + """ + # Convert the boolean null_tracker matrix into a list of strings. + most_frequent_pattern = "no_null" + rows_affected: List[int] = [] + occurrence_of_most_frequent_pattern = 0 + if np.any(null_tracker, axis=None): + null_row_indices = np.where(np.any(null_tracker, axis=1))[0] + null_patterns_as_strings = [ + "".join(map(str, null_tracker[i].astype(int).tolist())) for i in null_row_indices + ] + + # Use Counter to efficiently count occurrences and find the most common pattern. + pattern_counter = Counter(null_patterns_as_strings) + ( + most_frequent_pattern, + occurrence_of_most_frequent_pattern, + ) = pattern_counter.most_common(1)[0] + rows_affected = [] + for idx, row in enumerate(null_patterns_as_strings): + if row == most_frequent_pattern: + rows_affected.append(int(null_row_indices[idx])) + return { + "most_common_issue": { + "pattern": most_frequent_pattern, + "rows_affected": rows_affected, + "count": occurrence_of_most_frequent_pattern, + } + } + + @staticmethod + def _column_impact(null_tracker: np.ndarray) -> Dict[str, List[float]]: + """ + Calculate and return the impact of null values per column, represented as the proportion + of rows having null values in each column. + + Parameters + ---------- + null_tracker : np.ndarray + A boolean array of the same shape as features, where True indicates null/missing entries. + + Returns + ------- + Dict[str, List[float]] + A dictionary containing the impact per column, with values being a list + where each element is the percentage of rows having null values in the corresponding column. + """ + # Calculate proportion of nulls in each column + proportion_of_nulls_per_column = null_tracker.mean(axis=0) + + # Return result as a dictionary containing a list of proportions + return {"column_impact": proportion_of_nulls_per_column.tolist()} + + def collect_info(self, null_tracker: np.ndarray) -> dict: + most_common_issue = self._most_common_issue(null_tracker=null_tracker) + column_impact = self._column_impact(null_tracker=null_tracker) + average_null_score = {"average_null_score": self.issues[self.issue_score_key].mean()} + issues_dict = {**average_null_score, **most_common_issue, **column_impact} + info_dict: Dict[str, Any] = {**issues_dict} + return info_dict + + @classmethod + def report(cls, *args, **kwargs) -> str: + """ + Return a report of issues found by the NullIssueManager. + + This method extends the superclass method by identifying and reporting + specific issues related to null values in the dataset. + + Parameters + ---------- + *args : list + Variable length argument list. + **kwargs : dict + Arbitrary keyword arguments. + + Returns + ------- + report_str : + A string containing the report. + + See Also + -------- + :meth:`cleanlab.datalab.Datalab.report` + + Notes + ----- + This method differs from other IssueManager report methods. It checks for issues + and prompts the user to address them to enable other issue managers to run effectively. + """ + # Generate the base report using the superclass method + original_report = super().report(*args, **kwargs) + + # Retrieve the 'issues' dataframe from keyword arguments + issues = kwargs["issues"] + + # Identify examples that have null values in all features + issue_filter = f"is_{cls.issue_name}_issue" + examples_with_full_nulls = issues.query(issue_filter).index.tolist() + + # Identify examples that have some null values (but not in all features) + partial_null_filter = f"{cls.issue_score_key} < 1.0 and not {issue_filter}" + examples_with_partial_nulls = issues.query(partial_null_filter).index.tolist() + + # Append information about examples with null values in all features + if examples_with_full_nulls: + report_addition = ( + f"\n\nFound {len(examples_with_full_nulls)} examples with null values in all features. " + f"These examples should be removed from the dataset before running other issue managers." + # TODO: Add a link to the documentation on how to handle null examples + ) + original_report += report_addition + + # Append information about examples with some null values + if examples_with_partial_nulls: + report_addition = ( + f"\n\nFound {len(examples_with_partial_nulls)} examples with null values in some features. " + f"Please address these issues before running other issue managers." + # TODO: Add a link to the documentation on how to handle partially null examples + ) + original_report += report_addition + + return original_report diff --git a/cleanlab/datalab/internal/issue_manager/outlier.py b/cleanlab/datalab/internal/issue_manager/outlier.py new file mode 100644 index 0000000..057e020 --- /dev/null +++ b/cleanlab/datalab/internal/issue_manager/outlier.py @@ -0,0 +1,301 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Tuple + +from scipy.sparse import csr_matrix +from scipy.stats import iqr +import numpy as np +import pandas as pd + +from cleanlab.datalab.internal.issue_manager import IssueManager +from cleanlab.datalab.internal.issue_manager.knn_graph_helpers import knn_exists, set_knn_graph +from cleanlab.internal.outlier import correct_precision_errors +from cleanlab.outlier import OutOfDistribution, transform_distances_to_scores + +if TYPE_CHECKING: # pragma: no cover + from sklearn.neighbors import NearestNeighbors + import numpy.typing as npt + from cleanlab.datalab.datalab import Datalab + from cleanlab.typing import Metric + + +class OutlierIssueManager(IssueManager): + """Manages issues related to out-of-distribution examples.""" + + description: ClassVar[ + str + ] = """Examples that are very different from the rest of the dataset + (i.e. potentially out-of-distribution or rare/anomalous instances). + """ + issue_name: ClassVar[str] = "outlier" + verbosity_levels = { + 0: [], + 1: [], + 2: ["average_ood_score"], + 3: [], + } + + DEFAULT_THRESHOLDS = { + "features": 0.37037, + "pred_probs": 0.13, + } + """Default thresholds for outlier detection. + + If outlier detection is performed on the features, an example whose average + distance to their k nearest neighbors is greater than + Q3_avg_dist + (1 / threshold - 1) * IQR_avg_dist is considered an outlier. + + If outlier detection is performed on the predicted probabilities, an example + whose average score is lower than threshold * median_outlier_score is + considered an outlier. + """ + + def __init__( + self, + datalab: Datalab, + k: int = 10, + t: int = 1, + metric: Optional[Metric] = None, + scaling_factor: Optional[float] = None, + threshold: Optional[float] = None, + **kwargs, + ): + super().__init__(datalab) + + ood_kwargs = kwargs.get("ood_kwargs", {}) + + valid_ood_params = OutOfDistribution.DEFAULT_PARAM_DICT.keys() + params = { + key: value + for key, value in ((k, kwargs.get(k, None)) for k in valid_ood_params) + if value is not None + } + + # Simplified API: directly specify k and metric instead of NearestNeighbors object + # This reduces dependency on OutOfDistribution and aligns with Datalab's approach + params["k"] = k + self.k = k + self.t = t + self.metric: Optional[Metric] = metric + self.scaling_factor = scaling_factor + + if params: + ood_kwargs["params"] = params + + # OutOfDistribution still used for pred-prob based outlier detection + self.ood: OutOfDistribution = OutOfDistribution(**ood_kwargs) + + self._find_issues_inputs: Dict[str, bool] = { + "features": False, + "pred_probs": False, + "knn_graph": False, + } + + # Used for both methods of outlier detection + self.threshold = threshold + + def find_issues( + self, + features: Optional[npt.NDArray] = None, + pred_probs: Optional[np.ndarray] = None, + **kwargs, + ) -> None: + statistics = self.datalab.get_info("statistics") + + # Determine if we can use kNN-based outlier detection + knn_graph_works: bool = self._knn_graph_works(features, kwargs, statistics, self.k) + knn_graph = None + knn = None + if knn_graph_works: + # Set up or retrieve the kNN graph + knn_graph, self.metric, knn = set_knn_graph( + features=features, + find_issues_kwargs=kwargs, + metric=self.metric, + k=self.k, + statistics=statistics, + ) + + # Compute distances and thresholds for outlier detection + distances = knn_graph.data.reshape(knn_graph.shape[0], -1) + assert isinstance(distances, np.ndarray) + ( + self.threshold, + issue_threshold, # Useful info for detecting issues in test data + is_issue_column, + ) = self._compute_threshold_and_issue_column_from_distances(distances, self.threshold) + + # Calculate outlier scores based on average distances + avg_distances = distances.mean(axis=1) + median_avg_distance = np.median(avg_distances) + self._find_issues_inputs.update({"knn_graph": True}) + + # Ensure scaling factor is not too small to avoid numerical issues + if self.scaling_factor is None: + self.scaling_factor = float( + max(median_avg_distance, 100 * np.finfo(np.float64).eps) + ) + scores = transform_distances_to_scores( + avg_distances, t=self.t, scaling_factor=self.scaling_factor + ) + + # Apply precision error correction if metric is available + _metric = self.metric + if _metric is not None: + _metric = _metric if isinstance(_metric, str) else _metric.__name__ + scores = correct_precision_errors(scores, avg_distances, _metric) + elif pred_probs is not None: + # Fallback to prediction probabilities-based outlier detection + scores = self._score_with_pred_probs(pred_probs, **kwargs) + self._find_issues_inputs.update({"pred_probs": True}) + + # Set threshold for pred_probs-based detection + if self.threshold is None: + self.threshold = self.DEFAULT_THRESHOLDS["pred_probs"] + if not 0 <= self.threshold: + raise ValueError(f"threshold must be non-negative, but got {self.threshold}.") + issue_threshold = float( + self.threshold * np.median(scores) + ) # Useful info for detecting issues in test data + is_issue_column = scores < issue_threshold + + else: + # Handle case where neither kNN nor pred_probs-based detection is possible + if ( + kwargs.get("knn_graph", None) is not None + or statistics.get("weighted_knn_graph", None) is not None + ): + raise ValueError( + "knn_graph is provided, but not sufficiently large to compute the scores based on the provided hyperparameters." + ) + raise ValueError(f"Either features pred_probs must be provided.") + + # Store results + self.issues = pd.DataFrame( + { + f"is_{self.issue_name}_issue": is_issue_column, + self.issue_score_key: scores, + }, + ) + + self.summary = self.make_summary(score=scores.mean()) + + self.info = self.collect_info(issue_threshold=issue_threshold, knn_graph=knn_graph, knn=knn) + + def _knn_graph_works(self, features, kwargs, statistics, k: int) -> bool: + """Decide whether to skip the knn-based outlier detection and rely on pred_probs instead.""" + sufficient_knn_graph_available = knn_exists(kwargs, statistics, k) + return (features is not None) or sufficient_knn_graph_available + + def _compute_threshold_and_issue_column_from_distances( + self, distances: np.ndarray, threshold: Optional[float] = None + ) -> Tuple[float, float, np.ndarray]: + avg_distances = distances.mean(axis=1) + if threshold: + if not (isinstance(threshold, (int, float)) and 0 <= threshold <= 1): + raise ValueError( + f"threshold must be a number between 0 and 1, got {threshold} of type {type(threshold)}." + ) + if threshold is None: + threshold = OutlierIssueManager.DEFAULT_THRESHOLDS["features"] + + def compute_issue_threshold(avg_distances: np.ndarray, threshold: float) -> float: + q3_distance = np.percentile(avg_distances, 75) + iqr_scale = 1 / threshold - 1 if threshold != 0 else np.inf + issue_threshold = q3_distance + iqr_scale * iqr(avg_distances) + return float(issue_threshold) + + issue_threshold = compute_issue_threshold(avg_distances, threshold) + return threshold, issue_threshold, avg_distances > issue_threshold + + def collect_info( + self, + *, + issue_threshold: float, + knn_graph: Optional[csr_matrix], + knn: Optional["NearestNeighbors"], + ) -> dict: + issues_dict = { + "average_ood_score": self.issues[self.issue_score_key].mean(), + "threshold": self.threshold, + "issue_threshold": issue_threshold, + } + pred_probs_issues_dict: Dict[str, Any] = {} + feature_issues_dict = {} + + if knn_graph is not None: + N = knn_graph.shape[0] + k = knn_graph.nnz // N + dists = knn_graph.data.reshape(N, -1)[:, 0] + nn_ids = knn_graph.indices.reshape(N, -1)[:, 0] + + feature_issues_dict.update( + { + "k": self.k, # type: ignore[union-attr] + "nearest_neighbor": nn_ids.tolist(), + "distance_to_nearest_neighbor": dists.tolist(), + "metric": self.metric, # type: ignore[union-attr] + "scaling_factor": self.scaling_factor, + "t": self.t, + "knn": knn, + } + ) + + if self.ood.params["confident_thresholds"] is not None: + pass # + statistics_dict = self._build_statistics_dictionary(knn_graph=knn_graph) + ood_params_dict = { + "ood": self.ood, + **self.ood.params, + } + knn_dict = { + **pred_probs_issues_dict, + **feature_issues_dict, + } + info_dict: Dict[str, Any] = { + **issues_dict, + **ood_params_dict, # type: ignore[arg-type] + **knn_dict, + **statistics_dict, + "find_issues_inputs": self._find_issues_inputs, + } + return info_dict + + def _build_statistics_dictionary( + self, *, knn_graph: Optional[csr_matrix] + ) -> Dict[str, Dict[str, Any]]: + statistics_dict: Dict[str, Dict[str, Any]] = {"statistics": {}} + + # Add the knn graph as a statistic if necessary + graph_key = "weighted_knn_graph" + old_knn_graph = self.datalab.get_info("statistics").get(graph_key, None) + old_graph_exists = old_knn_graph is not None + prefer_new_graph = ( + not old_graph_exists + or ( + isinstance(knn_graph, csr_matrix) + and old_knn_graph is not None + and knn_graph.nnz > old_knn_graph.nnz + ) + or self.metric != self.datalab.get_info("statistics").get("knn_metric", None) + ) + if prefer_new_graph: + if knn_graph is not None: + statistics_dict["statistics"][graph_key] = knn_graph + if self.metric is not None: + statistics_dict["statistics"]["knn_metric"] = self.metric + + return statistics_dict + + def _score_with_pred_probs(self, pred_probs: np.ndarray, **kwargs) -> np.ndarray: + # Remove "threshold" from kwargs if it exists + kwargs.pop("threshold", None) + labels = self.datalab.labels + if not isinstance(labels, np.ndarray): + error_msg = ( + f"labels must be a numpy array of shape (n_samples,) to use the OutlierIssueManager " + f"with pred_probs, but got {type(labels)}." + ) + raise TypeError(error_msg) + scores = self.ood.fit_score(pred_probs=pred_probs, labels=labels, **kwargs) + return scores diff --git a/cleanlab/datalab/internal/issue_manager/regression/__init__.py b/cleanlab/datalab/internal/issue_manager/regression/__init__.py new file mode 100644 index 0000000..ac67cab --- /dev/null +++ b/cleanlab/datalab/internal/issue_manager/regression/__init__.py @@ -0,0 +1 @@ +from .label import RegressionLabelIssueManager diff --git a/cleanlab/datalab/internal/issue_manager/regression/label.py b/cleanlab/datalab/internal/issue_manager/regression/label.py new file mode 100644 index 0000000..24cabca --- /dev/null +++ b/cleanlab/datalab/internal/issue_manager/regression/label.py @@ -0,0 +1,241 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional +import numpy as np +import pandas as pd + +from cleanlab.regression.learn import CleanLearning +from cleanlab.datalab.internal.issue_manager import IssueManager +from cleanlab.regression.rank import get_label_quality_scores + +if TYPE_CHECKING: # pragma: no cover + from cleanlab.datalab.datalab import Datalab + + +class RegressionLabelIssueManager(IssueManager): + """Manages label issues in a Datalab for regression tasks. + + Parameters + ---------- + datalab : + A Datalab instance. + + clean_learning_kwargs : + Keyword arguments to pass to the :py:meth:`regression.learn.CleanLearning ` constructor. + + threshold : + The threshold to use to determine if an example has a label issue. It is a multiplier + of the median label quality score that sets the absolute threshold. Only used if + predictions are provided to `~RegressionLabelIssueManager.find_issues`, not if + features are provided. Default is 0.05. + """ + + description: ClassVar[ + str + ] = """Examples whose given label is estimated to be potentially incorrect + (e.g. due to annotation error) are flagged as having label issues. + """ + + issue_name: ClassVar[str] = "label" + verbosity_levels = { + 0: [], + 1: [], + 2: [], + 3: [], # TODO + } + + def __init__( + self, + datalab: Datalab, + clean_learning_kwargs: Optional[Dict[str, Any]] = None, + threshold: float = 0.05, + health_summary_parameters: Optional[Dict[str, Any]] = None, + **_, + ): + super().__init__(datalab) + self.cl = CleanLearning(**(clean_learning_kwargs or {})) + # This is a field for prioritizing features only when using a custom model + self._uses_custom_model = "model" in (clean_learning_kwargs or {}) + self.threshold = threshold + + def find_issues( + self, + features: Optional[np.ndarray] = None, + predictions: Optional[np.ndarray] = None, + **kwargs, + ) -> None: + """Find label issues in the datalab. + + .. admonition:: Priority Order for finding issues: + + 1. Custom Model: Requires `features` to be passed to this method. Used if a model is set up in the constructor. + 2. Predictions: Uses `predictions` if provided and no model is set up in the constructor. + 3. Default Model: Defaults to a standard model using `features` if no model or predictions are provided. + """ + if features is None and predictions is None: + raise ValueError( + "Regression requires numerical `features` or `predictions` " + "to be passed in as an argument to `find_issues`." + ) + if features is None and self._uses_custom_model: + raise ValueError( + "Regression requires numerical `features` to be passed in as an argument to `find_issues` " + "when using a custom model." + ) + # If features are provided and either a custom model is used or no predictions are provided + use_features = features is not None and (self._uses_custom_model or predictions is None) + labels = self.datalab.labels + if not isinstance(labels, np.ndarray): + error_msg = ( + f"Expected labels to be a numpy array of shape (n_samples,) to use with RegressionLabelIssueManager, " + f"but got {type(labels)} instead." + ) + raise TypeError(error_msg) + if use_features: + assert features is not None # mypy won't narrow the type for some reason + self.issues = find_issues_with_features( + features=features, + y=labels, + cl=self.cl, + **kwargs, # function sanitizes kwargs + ) + self.issues.rename(columns={"label_quality": self.issue_score_key}, inplace=True) + + # Otherwise, if predictions are provided, process them + else: + assert predictions is not None # mypy won't narrow the type for some reason + self.issues = find_issues_with_predictions( + predictions=predictions, + y=labels, + **{**kwargs, **{"threshold": self.threshold}}, # function sanitizes kwargs + ) + + # Get a summarized dataframe of the label issues + self.summary = self.make_summary(score=self.issues[self.issue_score_key].mean()) + + # Collect info about the label issues + self.info = self.collect_info(issues=self.issues) + + # Drop columns from issues that are in the info + self.issues = self.issues.drop(columns=["given_label", "predicted_label"]) + + def collect_info(self, issues: pd.DataFrame) -> dict: + issues_info = { + "num_label_issues": sum(issues[f"is_{self.issue_name}_issue"]), + "average_label_quality": issues[self.issue_score_key].mean(), + "given_label": issues["given_label"].tolist(), + "predicted_label": issues["predicted_label"].tolist(), + } + + # health_summary_info, cl_info kept just for consistency with classification, but it could be just return issues_info + health_summary_info: dict = {} + cl_info: dict = {} + + info_dict = { + **issues_info, + **health_summary_info, + **cl_info, + } + + return info_dict + + +def find_issues_with_predictions( + predictions: np.ndarray, + y: np.ndarray, + threshold: float, + **kwargs, +) -> pd.DataFrame: + """Find label issues in a regression dataset based on predictions. + This uses a threshold to determine if an example has a label issue + based on the quality score. + + Parameters + ---------- + predictions : + The predictions from a regression model. + + y : + The given labels. + + threshold : + The threshold to use to determine if an example has a label issue. It is a multiplier + of the median label quality score that sets the absolute threshold. + + **kwargs : + Various keyword arguments. + + Returns + ------- + issues : + A dataframe of the issues. It contains the following columns: + - is_label_issue : bool + True if the example has a label issue. + - label_score : float + The quality score of the label. + - given_label : float + The given label. It is the same as the y parameter. + - predicted_label : float + The predicted label. It is the same as the predictions parameter. + """ + _accepted_kwargs = ["method"] + _kwargs = {k: kwargs.get(k) for k in _accepted_kwargs} + _kwargs = {k: v for k, v in _kwargs.items() if v is not None} + quality_scores = get_label_quality_scores(labels=y, predictions=predictions, **_kwargs) + + median_score = np.median(quality_scores) + is_label_issue_mask = quality_scores < median_score * threshold + + issues = pd.DataFrame( + { + "is_label_issue": is_label_issue_mask, + "label_score": quality_scores, + "given_label": y, + "predicted_label": predictions, + } + ) + return issues + + +def find_issues_with_features( + features: np.ndarray, + y: np.ndarray, + cl: CleanLearning, + **kwargs, +) -> pd.DataFrame: + """Find label issues in a regression dataset based on features. + This delegates the work to the CleanLearning.find_label_issues method. + + Parameters + ---------- + features : + The numerical features from a regression dataset. + + y : + The given labels. + + **kwargs : + Various keyword arguments. + + Returns + ------- + issues : + A dataframe of the issues. It contains the following columns: + - is_label_issue : bool + True if the example has a label issue. + - label_score : float + The quality score of the label. + - given_label : float + The given label. It is the same as the y parameter. + - predicted_label : float + The predicted label. It is determined by the CleanLearning.find_label_issues method. + """ + _accepted_kwargs = [ + "uncertainty", + "coarse_search_range", + "fine_search_size", + "save_space", + "model_kwargs", + ] + _kwargs = {k: v for k, v in kwargs.items() if k in _accepted_kwargs and v is not None} + return cl.find_label_issues(X=features, y=y, **_kwargs) diff --git a/cleanlab/datalab/internal/issue_manager/underperforming_group.py b/cleanlab/datalab/internal/issue_manager/underperforming_group.py new file mode 100644 index 0000000..fd26025 --- /dev/null +++ b/cleanlab/datalab/internal/issue_manager/underperforming_group.py @@ -0,0 +1,345 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, Optional, Union, Tuple +import warnings +import inspect + +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix +from sklearn.cluster import DBSCAN + +from cleanlab.datalab.internal.issue_manager import IssueManager +from cleanlab.datalab.internal.issue_manager.knn_graph_helpers import set_knn_graph +from cleanlab.rank import get_self_confidence_for_each_label + +if TYPE_CHECKING: # pragma: no cover + import numpy.typing as npt + from cleanlab.datalab.datalab import Datalab + + +CLUSTERING_ALGO = "DBSCAN" +CLUSTERING_PARAMS_DEFAULT = {"metric": "precomputed"} + + +class UnderperformingGroupIssueManager(IssueManager): + """ + Manages issues related to underperforming group examples. + + Note: The `min_cluster_samples` argument should not be confused with the + `min_samples` argument of sklearn.cluster.DBSCAN. + + Examples + -------- + >>> from cleanlab import Datalab + >>> import numpy as np + >>> X = np.random.normal(size=(50, 2)) + >>> y = np.random.randint(2, size=50) + >>> pred_probs = X / X.sum(axis=1, keepdims=True) + >>> data = {"X": X, "y": y} + >>> lab = Datalab(data, label_name="y") + >>> issue_types={"underperforming_group": {"clustering_kwargs": {"eps": 0.5}}} + >>> lab.find_issues(pred_probs=pred_probs, features=X, issue_types=issue_types) + """ + + description: ClassVar[ + str + ] = """An underperforming group refers to a cluster of similar examples + (i.e. a slice) in the dataset for which the ML model predictions + are particularly poor (loss evaluation over this subpopulation is high). + """ + issue_name: ClassVar[str] = "underperforming_group" + verbosity_levels = { + 0: [], + 1: [], + 2: ["threshold"], + } + OUTLIER_CLUSTER_LABELS: ClassVar[Tuple[int]] = (-1,) + """Specifies labels considered as outliers by the clustering algorithm.""" + NO_UNDERPERFORMING_CLUSTER_ID: ClassVar[int] = min(OUTLIER_CLUSTER_LABELS) - 1 + """Constant to signify absence of any underperforming cluster.""" + + def __init__( + self, + datalab: Datalab, + metric: Optional[Union[str, Callable]] = None, + threshold: float = 0.1, + k: int = 10, + clustering_kwargs: Dict[str, Any] = {}, + min_cluster_samples: int = 5, + **_: Any, + ): + super().__init__(datalab) + self.metric = metric + self.threshold = self._set_threshold(threshold) + self.k = k + self.clustering_kwargs = clustering_kwargs + self.min_cluster_samples = min_cluster_samples + + def find_issues( + self, + pred_probs: npt.NDArray, + features: Optional[npt.NDArray] = None, + cluster_ids: Optional[npt.NDArray[np.int_]] = None, + **kwargs: Any, + ) -> None: + labels = self.datalab.labels + if not isinstance(labels, np.ndarray): + error_msg = ( + f"Labels must be a numpy array of shape (n_samples,) for UnderperformingGroupIssueManager. " + f"Got {type(labels)} instead." + ) + raise TypeError(error_msg) + if cluster_ids is None: + statistics = self.datalab.get_info("statistics") + knn_graph, self.metric, _ = set_knn_graph( + features, kwargs, self.metric, self.k, statistics + ) + cluster_ids = self.perform_clustering(knn_graph) + performed_clustering = True + else: + if self.clustering_kwargs: + warnings.warn( + "`clustering_kwargs` will not be used since `cluster_ids` have been passed." + ) + performed_clustering = False + knn_graph = None + unique_cluster_ids = self.filter_cluster_ids(cluster_ids) + if not unique_cluster_ids.size: + raise ValueError( + "No meaningful clusters were generated for determining underperforming group." + ) + n_clusters = len(unique_cluster_ids) + cluster_id_to_score, worst_cluster_id, worst_cluster_ratio = ( + self.get_underperforming_clusters(cluster_ids, unique_cluster_ids, labels, pred_probs) + ) + is_issue_column = cluster_ids == worst_cluster_id + scores = np.ones(is_issue_column.shape[0]) + for cluster_id, cluster_score in cluster_id_to_score.items(): + scores[cluster_ids == cluster_id] = cluster_score + self.issues = pd.DataFrame( + { + f"is_{self.issue_name}_issue": is_issue_column, + self.issue_score_key: scores, + }, + ) + self.summary = self.make_summary(score=worst_cluster_ratio) + self.info = self.collect_info( + knn_graph=knn_graph, + n_clusters=n_clusters, + cluster_ids=cluster_ids, + performed_clustering=performed_clustering, + worst_cluster_id=worst_cluster_id, + ) + + def perform_clustering(self, knn_graph: csr_matrix) -> npt.NDArray[np.int_]: + """Perform clustering of datapoints using a knn graph as distance matrix. + + Args: + knn_graph (csr_matrix): Sparse Distance Matrix. + + Returns: + cluster_ids (npt.NDArray[np.int_]): Cluster IDs for each datapoint. + """ + DBSCAN_VALID_KEYS = inspect.signature(DBSCAN).parameters.keys() + dbscan_params = { + key: value + for key, value in ((k, self.clustering_kwargs.get(k, None)) for k in DBSCAN_VALID_KEYS) + if value is not None + } + dbscan_params["metric"] = "precomputed" + clusterer = DBSCAN(**dbscan_params) + cluster_ids = clusterer.fit_predict( + knn_graph.copy() + ) # Copy to avoid modification by DBSCAN + return cluster_ids + + def filter_cluster_ids(self, cluster_ids: npt.NDArray[np.int_]) -> npt.NDArray[np.int_]: + """Remove outlier clusters and return IDs of clusters with at least `self.min_cluster_samples` number of datapoints. + + + Args: + cluster_ids (npt.NDArray[np.int_]): Cluster IDs for each datapoint. + + Returns: + unique_cluster_ids (npt.NDArray[np.int_]): List of unique cluster IDs after + removing outlier clusters and clusters with less than `self.min_cluster_samples` + number of datapoints. + """ + unique_cluster_ids = np.array( + [label for label in set(cluster_ids) if label not in self.OUTLIER_CLUSTER_LABELS] + ) + frequencies = np.bincount(cluster_ids[~np.isin(cluster_ids, self.OUTLIER_CLUSTER_LABELS)]) + unique_cluster_ids = np.array( + [ + cluster_id + for cluster_id in unique_cluster_ids + if frequencies[cluster_id] >= self.min_cluster_samples + ] + ) + return unique_cluster_ids + + def get_underperforming_clusters( + self, + cluster_ids: npt.NDArray[np.int_], + unique_cluster_ids: npt.NDArray[np.int_], + labels: npt.NDArray, + pred_probs: npt.NDArray, + ) -> Tuple[Dict[int, float], int, float]: + """Get ID and quality score of each underperforming cluster. + + Args: + cluster_ids (npt.NDArray[np.int_]): Cluster IDs corresponding to each sample + unique_cluster_ids (npt.NDArray[np.int_]): Unique cluster IDs excluding noisy clusters + labels (npt.NDArray): Label of each sample + pred_probs (npt.NDArray): Prediction probability + + Returns: + Tuple[Dict[int, float], int, float]: (Cluster IDs and their scores, Worst Cluster ID, Worst Cluster Quality Score) + """ + worst_cluster_ratio = 1.0 # Largest possible probability value + worst_cluster_id = min(unique_cluster_ids) - 1 + # For calculating mean_performance of the dataset, choose labels and pred-probs of samples belonging to non-noisy clusters + filtered_cluster_id_mask = np.isin(cluster_ids, unique_cluster_ids) + filtered_labels = labels[filtered_cluster_id_mask] + filtered_pred_probs = pred_probs[filtered_cluster_id_mask] + mean_performance = get_self_confidence_for_each_label( + filtered_labels, filtered_pred_probs + ).mean() + cluster_ids_to_score = {} + for cluster_id in unique_cluster_ids: + cluster_mask = cluster_ids == cluster_id + cur_cluster_ids = labels[cluster_mask] + cur_cluster_pred_probs = pred_probs[cluster_mask] + cluster_performance = get_self_confidence_for_each_label( + cur_cluster_ids, cur_cluster_pred_probs + ).mean() + if cluster_performance < mean_performance: + cluster_ids_to_score[cluster_id] = cluster_performance / mean_performance + if cluster_performance < worst_cluster_ratio: + worst_cluster_ratio = cluster_ids_to_score[cluster_id] + worst_cluster_id = cluster_id + worst_cluster_id = ( + worst_cluster_id + if worst_cluster_ratio < self.threshold + else self.NO_UNDERPERFORMING_CLUSTER_ID + ) + return cluster_ids_to_score, worst_cluster_id, worst_cluster_ratio + + def collect_info( + self, + knn_graph: csr_matrix, + n_clusters: int, + cluster_ids: npt.NDArray[np.int_], + performed_clustering: bool, + worst_cluster_id: int, + ) -> Dict[str, Any]: + params_dict = { + "k": self.k, + "metric": self.metric, + "threshold": self.threshold, + } + + knn_info_dict = {} + if knn_graph is not None: + N = knn_graph.shape[0] + dists = knn_graph.data.reshape(N, -1)[:, 0] + nn_ids = knn_graph.indices.reshape(N, -1)[:, 0] + + knn_info_dict = { + "nearest_neighbor": nn_ids.tolist(), + "distance_to_nearest_neighbor": dists.tolist(), + } + statistics_dict = self._build_statistics_dictionary(knn_graph=knn_graph) + + cluster_stat_dict = self._get_cluster_statistics( + n_clusters=n_clusters, + cluster_ids=cluster_ids, + performed_clustering=performed_clustering, + worst_cluster_id=worst_cluster_id, + ) + info_dict = { + **params_dict, + **knn_info_dict, + **statistics_dict, + **cluster_stat_dict, + } + + return info_dict + + def _build_statistics_dictionary(self, knn_graph: csr_matrix) -> Dict[str, Dict[str, Any]]: + statistics_dict: Dict[str, Dict[str, Any]] = {"statistics": {}} + + # Add the knn graph as a statistic if necessary + graph_key = "weighted_knn_graph" + old_knn_graph = self.datalab.get_info("statistics").get(graph_key, None) + old_graph_exists = old_knn_graph is not None + prefer_new_graph = ( + not old_graph_exists + or ( + isinstance(knn_graph, csr_matrix) + and old_knn_graph is not None + and knn_graph.nnz > old_knn_graph.nnz + ) + or self.metric != self.datalab.get_info("statistics").get("knn_metric", None) + ) + if prefer_new_graph: + if knn_graph is not None: + statistics_dict["statistics"][graph_key] = knn_graph + if self.metric is not None: + statistics_dict["statistics"]["knn_metric"] = self.metric + + return statistics_dict + + def _get_cluster_statistics( + self, + n_clusters: int, + cluster_ids: npt.NDArray[np.int_], + performed_clustering: bool, + worst_cluster_id: int, + ) -> Dict[str, Dict[str, Any]]: + """Get relevant cluster statistics. + + Args: + n_clusters (int): Number of clusters + cluster_ids (npt.NDArray[np.int_]): Cluster IDs for each datapoint. + performed_clustering (bool): Set to True to indicate that clustering was performed on + `features` passed to `find_issues`. Set to False to suggest that `cluster_ids` were explicitly + passed to `find_issues`. + worst_cluster_id (int): Uderperforming cluster ID. + + Returns: + cluster_stats (Dict[str, Dict[str, Any]]): Cluster Statistics + """ + cluster_stats: Dict[str, Dict[str, Any]] = { + "clustering": { + "algorithm": None, + "params": {}, + "stats": { + "n_clusters": n_clusters, + "cluster_ids": cluster_ids, + "underperforming_cluster_id": worst_cluster_id, + }, + } + } + if performed_clustering: + cluster_stats["clustering"].update( + {"algorithm": CLUSTERING_ALGO, "params": CLUSTERING_PARAMS_DEFAULT} + ) + + return cluster_stats + + def _set_threshold( + self, + threshold: float, + ) -> float: + """Computes nearest-neighbors thresholding for near-duplicate detection.""" + if threshold < 0: + warnings.warn( + f"Computed threshold {threshold} is less than 0. " + "Setting threshold to 0." + "This may indicate that either the only a few examples are in the dataset, " + "or the data is heavily skewed." + ) + threshold = 0 + return threshold diff --git a/cleanlab/datalab/internal/issue_manager_factory.py b/cleanlab/datalab/internal/issue_manager_factory.py new file mode 100644 index 0000000..8609b9d --- /dev/null +++ b/cleanlab/datalab/internal/issue_manager_factory.py @@ -0,0 +1,267 @@ +"""The factory module provides a factory class for constructing concrete issue managers +and a decorator for registering new issue managers. + +This module provides the :py:meth:`register` decorator for users to register new subclasses of +:py:class:`IssueManager ` +in the registry. Each IssueManager detects some particular type of issue in a dataset. + + +Note +---- + +The :class:`REGISTRY` variable is used by the factory class to keep track +of registered issue managers. +The factory class is used as an implementation detail by +:py:class:`Datalab `, +which provides a simplified API for constructing concrete issue managers. +:py:class:`Datalab ` is intended to be used by users +and provides detailed documentation on how to use the API. + +Warning +------- +Neither the :class:`REGISTRY` variable nor the factory class should be used directly by users. +""" + +from __future__ import annotations + +from typing import Dict, List, Type + +from cleanlab.datalab.internal.issue_manager import ( + ClassImbalanceIssueManager, + DataValuationIssueManager, + IssueManager, + LabelIssueManager, + NearDuplicateIssueManager, + NonIIDIssueManager, + ClassImbalanceIssueManager, + UnderperformingGroupIssueManager, + DataValuationIssueManager, + OutlierIssueManager, + NullIssueManager, +) +from cleanlab.datalab.internal.issue_manager.regression import RegressionLabelIssueManager +from cleanlab.datalab.internal.issue_manager.multilabel.label import MultilabelIssueManager +from cleanlab.datalab.internal.task import Task + + +REGISTRY: Dict[Task, Dict[str, Type[IssueManager]]] = { + Task.CLASSIFICATION: { + "outlier": OutlierIssueManager, + "label": LabelIssueManager, + "near_duplicate": NearDuplicateIssueManager, + "non_iid": NonIIDIssueManager, + "class_imbalance": ClassImbalanceIssueManager, + "underperforming_group": UnderperformingGroupIssueManager, + "data_valuation": DataValuationIssueManager, + "null": NullIssueManager, + }, + Task.REGRESSION: { + "label": RegressionLabelIssueManager, + "outlier": OutlierIssueManager, + "near_duplicate": NearDuplicateIssueManager, + "non_iid": NonIIDIssueManager, + "data_valuation": DataValuationIssueManager, + "null": NullIssueManager, + }, + Task.MULTILABEL: { + "label": MultilabelIssueManager, + "outlier": OutlierIssueManager, + "near_duplicate": NearDuplicateIssueManager, + "non_iid": NonIIDIssueManager, + "data_valuation": DataValuationIssueManager, + "null": NullIssueManager, + }, +} +"""Registry of issue managers that can be constructed from a task and issue type +and used in the Datalab class. + +:meta hide-value: + +Currently, the following issue managers are registered by default for a given task: + +- Classification: + + - ``"outlier"``: :py:class:`OutlierIssueManager ` + - ``"label"``: :py:class:`LabelIssueManager ` + - ``"near_duplicate"``: :py:class:`NearDuplicateIssueManager ` + - ``"non_iid"``: :py:class:`NonIIDIssueManager ` + - ``"class_imbalance"``: :py:class:`ClassImbalanceIssueManager ` + - ``"underperforming_group"``: :py:class:`UnderperformingGroupIssueManager ` + - ``"data_valuation"``: :py:class:`DataValuationIssueManager ` + - ``"null"``: :py:class:`NullIssueManager ` + +- Regression: + + - ``"label"``: :py:class:`RegressionLabelIssueManager ` + - ``"outlier"``: :py:class:`OutlierIssueManager ` + - ``"near_duplicate"``: :py:class:`NearDuplicateIssueManager ` + - ``"non_iid"``: :py:class:`NonIIDIssueManager ` + - ``"null"``: :py:class:`NullIssueManager ` + +- Multilabel: + + - ``"label"``: :py:class:`MultilabelIssueManager ` + - ``"outlier"``: :py:class:`OutlierIssueManager ` + - ``"near_duplicate"``: :py:class:`NearDuplicateIssueManager ` + - ``"non_iid"``: :py:class:`NonIIDIssueManager ` + - ``"null"``: :py:class:`NullIssueManager ` + +Warning +------- +This variable should not be used directly by users. +""" + + +# Construct concrete issue manager with a from_str method +class _IssueManagerFactory: + """Factory class for constructing concrete issue managers.""" + + @classmethod + def from_str(cls, issue_type: str, task: Task) -> Type[IssueManager]: + """Constructs a concrete issue manager class from a string.""" + if isinstance(issue_type, list): + raise ValueError( + "issue_type must be a string, not a list. Try using from_list instead." + ) + + if task not in REGISTRY: + raise ValueError(f"Invalid task type: {task}, must be in {list(REGISTRY.keys())}") + if issue_type not in REGISTRY[task]: + raise ValueError(f"Invalid issue type: {issue_type} for task {task}") + + return REGISTRY[task][issue_type] + + @classmethod + def from_list(cls, issue_types: List[str], task: Task) -> List[Type[IssueManager]]: + """Constructs a list of concrete issue manager classes from a list of strings.""" + return [cls.from_str(issue_type, task) for issue_type in issue_types] + + +def register(cls: Type[IssueManager], task: str = str(Task.CLASSIFICATION)) -> Type[IssueManager]: + """Registers the issue manager factory. + + Parameters + ---------- + cls : + A subclass of + :py:class:`IssueManager `. + + task : + Specific machine learning task like classification or regression. + See :py:meth:`Task.from_str `` for more details, + to see which task type corresponds to which string. + + Returns + ------- + cls : + The same class that was passed in. + + Example + ------- + + When defining a new subclass of + :py:class:`IssueManager `, + you can register it like so: + + .. code-block:: python + + from cleanlab import IssueManager + from cleanlab.datalab.internal.issue_manager_factory import register + + @register + class MyIssueManager(IssueManager): + issue_name: str = "my_issue" + def find_issues(self, **kwargs): + # Some logic to find issues + pass + + or in a function call: + + .. code-block:: python + + from cleanlab import IssueManager + from cleanlab.datalab.internal.issue_manager_factory import register + + class MyIssueManager(IssueManager): + issue_name: str = "my_issue" + def find_issues(self, **kwargs): + # Some logic to find issues + pass + + register(MyIssueManager, task="classification") + """ + + if not issubclass(cls, IssueManager): + raise ValueError(f"Class {cls} must be a subclass of IssueManager") + + name: str = str(cls.issue_name) + + try: + _task = Task.from_str(task) + if _task not in REGISTRY: + raise ValueError(f"Invalid task type: {_task}, must be in {list(REGISTRY.keys())}") + except KeyError: + raise ValueError(f"Invalid task type: {task}, must be in {list(REGISTRY.keys())}") + + if name in REGISTRY[_task]: + print( + f"Warning: Overwriting existing issue manager {name} with {cls} for task {_task}." + "This may cause unexpected behavior." + ) + + REGISTRY[_task][name] = cls + return cls + + +def list_possible_issue_types(task: Task) -> List[str]: + """Returns a list of all registered issue types. + + Any issue type that is not in this list cannot be used in the :py:meth:`find_issues` method. + + See Also + -------- + :py:class:`REGISTRY ` : All available issue types and their corresponding issue managers can be found here. + """ + return list(REGISTRY.get(task, [])) + + +def list_default_issue_types(task: Task) -> List[str]: + """Returns a list of the issue types that are run by default + when :py:meth:`find_issues` is called without specifying `issue_types`. + + task : + Specific machine learning task supported by Datalab. + + See Also + -------- + :py:class:`REGISTRY ` : All available issue types and their corresponding issue managers can be found here. + """ + default_issue_types_dict = { + Task.CLASSIFICATION: [ + "null", + "label", + "outlier", + "near_duplicate", + "non_iid", + "class_imbalance", + "underperforming_group", + ], + Task.REGRESSION: [ + "null", + "label", + "outlier", + "near_duplicate", + "non_iid", + ], + Task.MULTILABEL: [ + "null", + "label", + "outlier", + "near_duplicate", + "non_iid", + ], + } + if task not in default_issue_types_dict: + task = Task.CLASSIFICATION + default_issue_types = default_issue_types_dict[task] + return default_issue_types diff --git a/cleanlab/datalab/internal/model_outputs.py b/cleanlab/datalab/internal/model_outputs.py new file mode 100644 index 0000000..a266bd8 --- /dev/null +++ b/cleanlab/datalab/internal/model_outputs.py @@ -0,0 +1,116 @@ +""" +This module contains the ModelOutput class, which is used internally within Datalab +to represent model outputs (e.g. predictions, probabilities, etc.) and process them +for issue finding. +This class and associated naming conventions are subject to change and is not meant +to be used by users. +""" + +from abc import ABC, abstractmethod +import numpy as np +from dataclasses import dataclass + + +@dataclass +class ModelOutput(ABC): + """ + An abstract class for representing model outputs (e.g. predictions, probabilities, etc.) + for internal use within Datalab. This class is not meant to be used by users. + + It is used internally within the issue-finding process Datalab runs to assign + types to the data and process it accordingly. + + Parameters + ---------- + data : array-like + The model outputs. Not to be confused with the data used to train the model. + This is mainly intended for NumPy arrays. + """ + + data: np.ndarray + + @abstractmethod + def validate(self): + """ + Validate the data format and content. + E.g. a pred_probs object used for classification + should be a 2D array with values between 0 and 1 and sum to 1 for each row. + """ + pass + + @abstractmethod + def collect(self): + """ + Fetch the data for issue finding. + Usually this is just the data itself, but sometimes it may be a transformation + of the data (e.g. a 1D array of predictions from a 2D array of predicted probabilities). + """ + pass + + +class MultiClassPredProbs(ModelOutput): + """ + A class for representing a model's predicted probabilities for each class + in a multi-class classification problem. This class is not meant to be used by users. + """ + + argument = "pred_probs" + + def validate(self): + pred_probs = self.data + if pred_probs.ndim != 2: + raise ValueError("pred_probs must be a 2D array for multi-class classification") + if not np.all((pred_probs >= 0) & (pred_probs <= 1)): + incorrect_range = (np.min(pred_probs), np.max(pred_probs)) + raise ValueError( + "Expected pred_probs to be between 0 and 1 for multi-label classification," + f" but got values in range {incorrect_range} instead." + ) + if not np.allclose(np.sum(pred_probs, axis=1), 1): + raise ValueError("pred_probs must sum to 1 for each row for multi-class classification") + + def collect(self): + return self.data + + +class RegressionPredictions(ModelOutput): + """ + A class for representing a model's predictions for a regression problem. + This class is not meant to be used by users. + """ + + argument = "predictions" + + def validate(self): + predictions = self.data + if predictions.ndim != 1: + raise ValueError("pred_probs must be a 1D array for regression") + + def collect(self): + return self.data + + +class MultiLabelPredProbs(ModelOutput): + """ + A class for representing a model's predicted probabilities for each class + in a multilabel classification problem. This class is not meant to be used by users. + """ + + argument = "pred_probs" + + def validate(self): + pred_probs = self.data + if pred_probs.ndim != 2: + raise ValueError( + f"Expected pred_probs to be a 2D array for multi-label classification," + " but got {pred_probs.ndim}D array instead." + ) + if not np.all((pred_probs >= 0) & (pred_probs <= 1)): + incorrect_range = (np.min(pred_probs), np.max(pred_probs)) + raise ValueError( + "Expected pred_probs to be between 0 and 1 for multi-label classification," + f" but got values in range {incorrect_range} instead." + ) + + def collect(self): + return self.data diff --git a/cleanlab/datalab/internal/report.py b/cleanlab/datalab/internal/report.py new file mode 100644 index 0000000..0862321 --- /dev/null +++ b/cleanlab/datalab/internal/report.py @@ -0,0 +1,194 @@ +""" +Module that handles reporting of all types of issues identified in the data. +""" + +from typing import TYPE_CHECKING, List + +import pandas as pd + +from cleanlab.datalab.internal.adapter.constants import DEFAULT_CLEANVISION_ISSUES +from cleanlab.datalab.internal.issue_manager_factory import _IssueManagerFactory +from cleanlab.datalab.internal.task import Task + +if TYPE_CHECKING: # pragma: no cover + from cleanlab.datalab.internal.data_issues import DataIssues + + +class Reporter: + """Class that generates a report about the issues stored in a :py:class:`DataIssues` object. + + Parameters + ---------- + data_issues : + The :py:class:`DataIssues` object containing the issues to report on. This is usually + generated by the :py:class:`Datalab` class, stored in the :py:attr:`data_issues` attribute, + and then passed to the :py:class:`Reporter` class to generate a report. + + task : + Specific machine learning task that the datset is intended for. + See details about supported tasks in :py:class:`Task `. + + verbosity : + The default verbosity of the report to generate. Each :py:class`IssueManager` + specifies the available verbosity levels and what additional information + is included at each level. + + include_description : + Whether to include the description of each issue type in the report. The description + is included by default, but can be excluded by setting this parameter to ``False``. + + Note + ---- + This class is not intended to be used directly. Instead, use the + `Datalab.find_issues` method which internally utilizes an IssueFinder instance. + """ + + def __init__( + self, + data_issues: "DataIssues", + task: Task, + verbosity: int = 1, + include_description: bool = True, + show_summary_score: bool = False, + show_all_issues: bool = False, + **kwargs, + ): + self.data_issues = data_issues + self.task = task + self.verbosity = verbosity + self.include_description = include_description + self.show_summary_score = show_summary_score + self.show_all_issues = show_all_issues + + def _get_empty_report(self) -> str: + """This method is used to return a report when there are + no issues found in the data with Datalab.find_issues(). + """ + report_str = "No issues found in the data. Good job!" + if not self.show_summary_score: + recommendation_msg = ( + "Try re-running Datalab.report() with " + "`show_summary_score = True` and `show_all_issues = True`." + ) + report_str += f"\n\n{recommendation_msg}" + return report_str + + def report(self, num_examples: int) -> None: + """Prints a report about identified issues in the data. + + Parameters + ---------- + num_examples : + The number of examples to include in the report for each issue type. + """ + print(self.get_report(num_examples=num_examples)) + + def get_report(self, num_examples: int) -> str: + """Constructs a report about identified issues in the data. + + Parameters + ---------- + num_examples : + The number of examples to include in the report for each issue type. + + + Returns + ------- + report_str : + A string containing the report. + + Examples + -------- + >>> from cleanlab.datalab.internal.report import Reporter + >>> reporter = Reporter(data_issues=data_issues, include_description=False) + >>> report_str = reporter.get_report(num_examples=5) + >>> print(report_str) + """ + report_str = "" + issue_summary = self.data_issues.issue_summary + should_return_empty_report = not ( + self.show_all_issues or issue_summary.empty or issue_summary["num_issues"].sum() > 0 + ) + + if should_return_empty_report: + return self._get_empty_report() + issue_summary_sorted = issue_summary.sort_values(by="num_issues", ascending=False) + report_str += self._write_summary(summary=issue_summary_sorted) + + issue_types = self._get_issue_types(issue_summary_sorted) + + def add_issue_to_report(issue_name: str) -> bool: + """Returns True if the issue should be added to the report. + It is excluded if show_all_issues is False and there are no issues of that type + found in the data. + """ + if self.show_all_issues: + return True + summary = self.data_issues.get_issue_summary(issue_name=issue_name) + has_issues = summary["num_issues"][0] > 0 + return has_issues + + issue_reports = [ + _IssueManagerFactory.from_str(issue_type=key, task=self.task).report( + issues=self.data_issues.get_issues(issue_name=key), + summary=self.data_issues.get_issue_summary(issue_name=key), + info=self.data_issues.get_info(issue_name=key), + num_examples=num_examples, + verbosity=self.verbosity, + include_description=self.include_description, + ) + for key in issue_types + ] + + report_str += "\n\n\n".join(issue_reports) + return report_str + + def _write_summary(self, summary: pd.DataFrame) -> str: + statistics = self.data_issues.get_info("statistics") + num_examples = statistics["num_examples"] + num_classes = statistics.get( + "num_classes" + ) # This may not be required for all types of datasets in the future (e.g. unlabeled/regression) + + dataset_information = f"Dataset Information: num_examples: {num_examples}" + if num_classes is not None: + dataset_information += f", num_classes: {num_classes}" + + if not self.show_all_issues: + # Drop any items in the issue_summary that have no issues (any issue detected in data needs to have num_issues > 0) + summary = summary.query("num_issues > 0") + + report_header = ( + f"{dataset_information}\n\n" + + "Here is a summary of various issues found in your data:\n\n" + ) + report_footer = ( + "\n\n" + + "Learn about each issue: https://docs.cleanlab.ai/stable/cleanlab/datalab/guide/issue_type_description.html\n" + + "See which examples in your dataset exhibit each issue via: `datalab.get_issues()`\n\n" + + "Data indices corresponding to top examples of each issue are shown below.\n\n\n" + ) + + if self.show_summary_score: + return ( + report_header + + summary.to_string(index=False) + + "\n\n" + + "(Note: A lower score indicates a more severe issue across all examples in the dataset.)" + + report_footer + ) + + return ( + report_header + summary.drop(columns=["score"]).to_string(index=False) + report_footer + ) + + def _get_issue_types(self, issue_summary: pd.DataFrame) -> List[str]: + issue_types = [ + issue_type + for issue_type, num_issues in zip( + issue_summary["issue_type"].tolist(), issue_summary["num_issues"].tolist() + ) + if issue_type not in DEFAULT_CLEANVISION_ISSUES + and (self.show_all_issues or num_issues > 0) + ] + return issue_types diff --git a/cleanlab/datalab/internal/serialize.py b/cleanlab/datalab/internal/serialize.py new file mode 100644 index 0000000..b4b1b4c --- /dev/null +++ b/cleanlab/datalab/internal/serialize.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import os +import pickle +import warnings +from typing import TYPE_CHECKING, Optional + +import pandas as pd + +import cleanlab +from cleanlab.datalab.internal.data import Data + +if TYPE_CHECKING: # pragma: no cover + from datasets.arrow_dataset import Dataset + + from cleanlab.datalab.datalab import Datalab + + +# Constants: +OBJECT_FILENAME = "datalab.pkl" +ISSUES_FILENAME = "issues.csv" +ISSUE_SUMMARY_FILENAME = "summary.csv" +INFO_FILENAME = "info.pkl" +DATA_DIRNAME = "data" + + +class _Serializer: + @staticmethod + def _save_data_issues(path: str, datalab: Datalab) -> None: + """Saves the issues to disk.""" + issues_path = os.path.join(path, ISSUES_FILENAME) + datalab.data_issues.issues.to_csv(issues_path, index=False) + + issue_summary_path = os.path.join(path, ISSUE_SUMMARY_FILENAME) + datalab.data_issues.issue_summary.to_csv(issue_summary_path, index=False) + + @staticmethod + def _save_data(path: str, datalab: Datalab) -> None: + """Saves the dataset to disk.""" + data_path = os.path.join(path, DATA_DIRNAME) + datalab.data.save_to_disk(data_path) + + @staticmethod + def _validate_version(datalab: Datalab) -> None: + current_version = cleanlab.__version__ # type: ignore[attr-defined] + datalab_version = datalab.cleanlab_version + if current_version != datalab_version: + warnings.warn( + f"Saved Datalab was created using different version of cleanlab " + f"({datalab_version}) than current version ({current_version}). " + f"Things may be broken!" + ) + + @classmethod + def serialize(cls, path: str, datalab: Datalab, force: bool) -> None: + """Serializes the datalab object to disk. + + Parameters + ---------- + path : str + Path to save the datalab object to. + + datalab : Datalab + The datalab object to save. + + force : bool + If True, will overwrite existing files at the specified path. + """ + path_exists = os.path.exists(path) + if not path_exists: + os.mkdir(path) + else: + if not force: + raise FileExistsError("Please specify a new path or set force=True") + print(f"WARNING: Existing files will be overwritten by newly saved files at: {path}") + + # Save the datalab object to disk. + with open(os.path.join(path, OBJECT_FILENAME), "wb") as f: + pickle.dump(datalab, f) + + # Save the issues to disk. Use placeholder method for now. + cls._save_data_issues(path=path, datalab=datalab) + + # Save the dataset to disk + cls._save_data(path=path, datalab=datalab) + + @classmethod + def deserialize(cls, path: str, data: Optional[Dataset] = None) -> Datalab: + """Deserializes the datalab object from disk.""" + + if not os.path.exists(path): + raise ValueError(f"No folder found at specified path: {path}") + + with open(os.path.join(path, OBJECT_FILENAME), "rb") as f: + datalab: Datalab = pickle.load(f) + + cls._validate_version(datalab) + + # Load the issues from disk. + issues_path = os.path.join(path, ISSUES_FILENAME) + if not hasattr(datalab.data_issues, "issues") and os.path.exists(issues_path): + datalab.data_issues.issues = pd.read_csv(issues_path) + + issue_summary_path = os.path.join(path, ISSUE_SUMMARY_FILENAME) + if not hasattr(datalab.data_issues, "issue_summary") and os.path.exists(issue_summary_path): + datalab.data_issues.issue_summary = pd.read_csv(issue_summary_path) + + if data is not None: + if hash(data) != hash(datalab._data): + raise ValueError( + "Data has been modified since Lab was saved. " + "Cannot load Lab with modified data." + ) + + if len(data) != len(datalab.labels): + raise ValueError( + f"Length of data ({len(data)}) does not match length of labels ({len(datalab.labels)})" + ) + + datalab._data = Data(data, datalab.task, datalab.label_name) + datalab.data = datalab._data._data + + return datalab diff --git a/cleanlab/datalab/internal/spurious_correlation.py b/cleanlab/datalab/internal/spurious_correlation.py new file mode 100644 index 0000000..550451b --- /dev/null +++ b/cleanlab/datalab/internal/spurious_correlation.py @@ -0,0 +1,113 @@ +from dataclasses import dataclass +from typing import List, Optional, Union +import warnings + +import numpy as np +import pandas as pd +from sklearn.model_selection import cross_val_score +from sklearn.naive_bayes import GaussianNB + +warnings.filterwarnings("ignore") + + +@dataclass +class SpuriousCorrelations: + data: pd.DataFrame + labels: Union[np.ndarray, list] + properties_of_interest: Optional[List[str]] = None + + def __post_init__(self): + # Must have same number of rows + if not len(self.data) == len(self.labels): + raise ValueError( + "The number of rows in the data dataframe must be the same as the number of labels." + ) + + # Set default properties_of_interest if not provided + if self.properties_of_interest is None: + self.properties_of_interest = self.data.columns.tolist() + + if not all(isinstance(p, str) for p in self.properties_of_interest): + raise TypeError("properties_of_interest must be a list of strings.") + + def calculate_correlations(self) -> pd.DataFrame: + """Calculates the spurious correlation scores for each property of interest found in the dataset.""" + baseline_accuracy = self._get_baseline() + assert ( + self.properties_of_interest is not None + ), "properties_of_interest must be set, but is None." + property_scores = { + str(property_of_interest): self.calculate_spurious_correlation( + property_of_interest, baseline_accuracy + ) + for property_of_interest in self.properties_of_interest + } + data_score = pd.DataFrame(list(property_scores.items()), columns=["property", "score"]) + return data_score + + def _get_baseline(self) -> float: + """Calculates the baseline accuracy of the dataset. The baseline model is predicting the most common label.""" + baseline_accuracy = np.bincount(self.labels).argmax() / len(self.labels) + return float(baseline_accuracy) + + def calculate_spurious_correlation( + self, property_of_interest, baseline_accuracy: float + ) -> float: + """Scores the dataset based on a given property of interest. + + Parameters + ---------- + property_of_interest : + The property of interest to score the dataset on. + + baseline_accuracy : + The accuracy of the baseline model. + + Returns + ------- + score : + A correlation score of the dataset's labels to the property of interest. + """ + X = self.data[property_of_interest].values.reshape(-1, 1) + y = self.labels + mean_accuracy = _train_and_eval(X, y) + return relative_room_for_improvement(baseline_accuracy, float(mean_accuracy)) + + +def _train_and_eval(X, y, cv=5) -> float: + classifier = GaussianNB() # TODO: Make this a parameter + cv_accuracies = cross_val_score(classifier, X, y, cv=cv, scoring="accuracy") + mean_accuracy = float(np.mean(cv_accuracies)) + return mean_accuracy + + +def relative_room_for_improvement( + baseline_accuracy: float, mean_accuracy: float, eps: float = 1e-8 +) -> float: + """ + Calculate the relative room for improvement given a baseline and trial accuracy. + + This function computes the ratio of the difference between perfect accuracy (1.0) + and the trial accuracy to the difference between perfect accuracy and the baseline accuracy. + If the baseline accuracy is perfect (i.e., 1.0), an epsilon value is added to the denominator + to avoid division by zero. + + Parameters + ---------- + baseline_accuracy : + The accuracy of the baseline model. Must be between 0 and 1. + mean_accuracy : + The accuracy of the trial model being compared. Must be between 0 and 1. + eps : + A small constant to avoid division by zero when baseline accuracy is 1. Defaults to 1e-8. + + Returns + ------- + score : + The relative room for improvement, bounded between 0 and 1. + """ + numerator = 1 - mean_accuracy + denominator = 1 - baseline_accuracy + if baseline_accuracy == 1: + denominator += eps + return min(1, numerator / denominator) diff --git a/cleanlab/datalab/internal/task.py b/cleanlab/datalab/internal/task.py new file mode 100644 index 0000000..abd6bdf --- /dev/null +++ b/cleanlab/datalab/internal/task.py @@ -0,0 +1,130 @@ +""" +This module contains the Task enum, which internally represents the tasks +supported by Datalab, so that the appropriate task-specific logic can be applied. +This class and associated naming conventions are subject to change and is not meant +to be used by users. +""" + +from enum import Enum + + +class Task(Enum): + """ + Represents a task supported by Datalab. + + Datalab supports the following tasks: + + * **Classification**: for predicting discrete class labels. + * **Regression**: for predicting continuous numerical values. + * **Multilabel**: for predicting multiple binary labels simultaneously. + + Example + ------- + >>> task = Task.CLASSIFICATION + >>> task + + """ + + CLASSIFICATION = "classification" + """Classification task.""" + REGRESSION = "regression" + """Regression task.""" + MULTILABEL = "multilabel" + """Multilabel task.""" + + def __str__(self): + """ + Returns the string representation of the task. + + Returns: + str: The string representation of the task. + """ + return self.value + + @classmethod + def from_str(cls, task_str: str) -> "Task": + """ + Converts a string representation of a task to a Task enum value. + + Parameters + ---------- + task_str : + The string representation of the task. + + Returns + ------- + Task : + The corresponding Task enum value. + + Raises + ------ + ValueError : + If the provided task_str is not a valid task supported by Datalab. + + Examples + -------- + >>> Task.from_str("classification") + + >>> print(Task.from_str("regression")) + regression + """ + _value_to_enum = {task.value: task for task in Task} + try: + return _value_to_enum[task_str] + except KeyError: + valid_tasks = list(_value_to_enum.keys()) + raise ValueError(f"Invalid task: {task_str}. Datalab only supports {valid_tasks}.") + + @property + def is_classification(self): + """ + Checks if the task is classification. + + Returns + ------- + bool : + True if the task is classification, False otherwise. + + Examples + -------- + >>> task = Task.CLASSIFICATION + >>> print(task.is_classification) + True + """ + return self == Task.CLASSIFICATION + + @property + def is_regression(self): + """ + Checks if the task is regression. + + Returns + ------- + bool : + True if the task is regression, False otherwise. + + Examples + -------- + >>> task = Task.CLASSIFICATION + >>> print(task.is_regression) + False + """ + return self == Task.REGRESSION + + @property + def is_multilabel(self): + """ + Checks if the task is multilabel. + + Returns + ------- + bool : + True if the task is multilabel, False otherwise. + + Examples + -------- + >>> task = Task.CLASSIFICATION + >>> print(task.is_multilabel) + False + """ + return self == Task.MULTILABEL diff --git a/cleanlab/dataset.py b/cleanlab/dataset.py new file mode 100644 index 0000000..2fa6a66 --- /dev/null +++ b/cleanlab/dataset.py @@ -0,0 +1,514 @@ +""" +Provides dataset-level and class-level overviews of issues in your classification dataset. +If your task allows you to modify the classes in your dataset, this module can help you determine +which classes to remove (see `~cleanlab.dataset.rank_classes_by_label_quality`) +and which classes to merge (see `~cleanlab.dataset.find_overlapping_classes`). +""" + +from typing import Optional, cast +import numpy as np +import pandas as pd + +from cleanlab.count import estimate_joint, num_label_issues +from cleanlab.internal.constants import EPSILON + + +def rank_classes_by_label_quality( + labels=None, + pred_probs=None, + *, + class_names=None, + num_examples=None, + joint=None, + confident_joint=None, + multi_label=False, +) -> pd.DataFrame: + """ + Returns a Pandas DataFrame with all classes and three overall class label quality scores + (details about each score are listed in the Returns parameter). By default, classes are ordered + by "Label Quality Score", ascending, so the most problematic classes are reported first. + + Score values are unnormalized and may tend to be very small. What matters is their relative + ranking across the classes. + + This method works by providing any one (and only one) of the following inputs: + + 1. ``labels`` and ``pred_probs``, or + 2. ``joint`` and ``num_examples``, or + 3. ``confident_joint`` + + Only provide **exactly one of the above input options**, do not provide a combination. + + Examples + -------- + >>> from cleanlab.dataset import rank_classes_by_label_quality + >>> from sklearn.linear_model import LogisticRegression + >>> from sklearn.model_selection import cross_val_predict + >>> data, labels = get_data_labels_from_dataset() + >>> yourFavoriteModel = LogisticRegression() + >>> pred_probs = cross_val_predict(yourFavoriteModel, data, labels, cv=3, method="predict_proba") + >>> df = rank_classes_by_label_quality(labels=labels, pred_probs=pred_probs) + + **Parameters**: For parameter info, see the docstring of `~cleanlab.dataset.find_overlapping_classes`. + + Returns + ------- + overall_label_quality : pd.DataFrame + Pandas DataFrame with cols "Class Index", "Label Issues", "Inverse Label Issues", + "Label Issues", "Inverse Label Noise", "Label Quality Score", + with a description of each of these columns below. + The length of the DataFrame is ``num_classes`` (one row per class). + Noise scores are between 0 and 1, where 0 implies no label issues + in the class. The "Label Quality Score" is also between 0 and 1 where 1 implies + perfect quality. Columns: + + * *Class Index*: The index of the class in 0, 1, ..., K-1. + * *Label Issues*: ``count(given_label = k, true_label != k)``, estimated number of examples in the dataset that are labeled as class k but should have a different label. + * *Inverse Label Issues*: ``count(given_label != k, true_label = k)``, estimated number of examples in the dataset that should actually be labeled as class k but have been given another label. + * *Label Noise*: ``prob(true_label != k | given_label = k)``, estimated proportion of examples in the dataset that are labeled as class k but should have a different label. For each class k: this is computed by dividing the number of examples with "Label Issues" that were labeled as class k by the total number of examples labeled as class k. + * *Inverse Label Noise*: ``prob(given_label != k | true_label = k)``, estimated proportion of examples in the dataset that should actually be labeled as class k but have been given another label. + * *Label Quality Score*: ``p(true_label = k | given_label = k)``. This is the proportion of examples with given label k that have been labeled correctly, i.e. ``1 - label_noise``. + + By default, the DataFrame is ordered by "Label Quality Score", ascending. + """ + if multi_label: + raise ValueError( + "For multilabel data, please instead call: multilabel_classification.dataset.overall_multilabel_health_score()" + ) + + if joint is None: + joint = estimate_joint( + labels=labels, + pred_probs=pred_probs, + confident_joint=confident_joint, + ) + if num_examples is None: + num_examples = _get_num_examples(labels=labels) + given_label_noise = joint.sum(axis=1) - joint.diagonal() # p(s=k) - p(s=k,y=k) = p(y!=k, s=k) + true_label_noise = joint.sum(axis=0) - joint.diagonal() # p(y=k) - p(s=k,y=k) = p(s!=k,y=k) + given_conditional_noise = given_label_noise / np.clip( + joint.sum(axis=1), a_min=EPSILON, a_max=None + ) # p(y!=k, s=k) / p(s=k) , avoiding division by 0 + true_conditional_noise = true_label_noise / np.clip( + joint.sum(axis=0), a_min=EPSILON, a_max=None + ) # p(s!=k, y=k) / p(y=k) , avoiding division by 0 + df = pd.DataFrame( + { + "Class Index": np.arange(len(joint)), + "Label Issues": (given_label_noise * num_examples).round().astype(int), + "Inverse Label Issues": (true_label_noise * num_examples).round().astype(int), + "Label Noise": given_conditional_noise, # p(y!=k | s=k) + "Inverse Label Noise": true_conditional_noise, # p(s!=k | y=k) + # Below could equivalently be computed as: joint.diagonal() / joint.sum(axis=1) + "Label Quality Score": 1 - given_conditional_noise, # p(y=k | s=k) + } + ) + if class_names is not None: + df.insert(loc=0, column="Class Name", value=class_names) + return df.sort_values(by="Label Quality Score", ascending=True).reset_index(drop=True) + + +def find_overlapping_classes( + labels=None, + pred_probs=None, + *, + asymmetric=False, + class_names=None, + num_examples=None, + joint=None, + confident_joint=None, + multi_label=False, +) -> pd.DataFrame: + """Returns the pairs of classes that are often mislabeled as one another. + Consider merging the top pairs of classes returned by this method each into a single class. + If the dataset is labeled by human annotators, consider clearly defining the + difference between the classes prior to having annotators label the data. + + This method provides two scores in the Pandas DataFrame that is returned: + + * **Num Overlapping Examples**: The number of examples where the two classes overlap + * **Joint Probability**: `(num overlapping examples / total number of examples in the dataset`). + + This method works by providing any one (and only one) of the following inputs: + + 1. ``labels`` and ``pred_probs``, or + 2. ``joint`` and ``num_examples``, or + 3. ``confident_joint`` + + Only provide **exactly one of the above input options**, do not provide a combination. + + This method uses the joint distribution of noisy and true labels to compute ontological + issues via the approach published in `Northcutt et al., + 2021 `_. + + Examples + -------- + >>> from cleanlab.dataset import find_overlapping_classes + >>> from sklearn.linear_model import LogisticRegression + >>> from sklearn.model_selection import cross_val_predict + >>> data, labels = get_data_labels_from_dataset() + >>> yourFavoriteModel = LogisticRegression() + >>> pred_probs = cross_val_predict(yourFavoriteModel, data, labels, cv=3, method="predict_proba") + >>> df = find_overlapping_classes(labels=labels, pred_probs=pred_probs) + + Note + ---- + The joint distribution of noisy and true labels is asymmetric, and therefore the joint + probability ``p(given="vehicle", true="truck") != p(true="truck", given="vehicle")``. + This is intuitive. Images of trucks (true label) are much more likely to be labeled as a car + (given label) than images of cars (true label) being frequently mislabeled as truck (given + label). cleanlab takes these differences into account for you automatically via the joint + distribution. If you do not want this behavior, simply set ``asymmetric=False``. + + This method estimates how often the annotators confuse two classes. + This differs from just using a similarity matrix or confusion matrix, + as these summarize characteristics of the predictive model rather than the data labelers (i.e. annotators). + Instead, this method works even if the model that generated `pred_probs` tends to be more confident in some classes than others. + + Parameters + ---------- + labels : np.ndarray or list, optional + An array_like (of length N) of noisy labels for the classification dataset, i.e. some labels may be erroneous. + Elements must be integers in the set 0, 1, ..., K-1, where K is the number of classes. + All the classes (0, 1, ..., and K-1) should be present in ``labels``, such that + ``len(set(labels)) == pred_probs.shape[1]`` for standard multi-class classification with single-labeled data (e.g. ``labels = [1,0,2,1,1,0...]``). + For multi-label classification where each example can belong to multiple classes (e.g. ``labels = [[1,2],[1],[0],[],...]``), + your labels should instead satisfy: ``len(set(k for l in labels for k in l)) == pred_probs.shape[1])``. + + pred_probs : np.ndarray, optional + An array of shape ``(N, K)`` of model-predicted probabilities, + ``P(label=k|x)``. Each row of this matrix corresponds + to an example `x` and contains the model-predicted probabilities that + `x` belongs to each possible class, for each of the K classes. The + columns must be ordered such that these probabilities correspond to + class 0, 1, ..., K-1. `pred_probs` should have been computed using 3 (or + higher) fold cross-validation. + + asymmetric : bool, optional + If ``asymmetric=True``, returns separate estimates for both pairs (class1, class2) and (class2, class1). Use this + for finding "is a" relationships where for example "class1 is a class2". + In this case, num overlapping examples counts the number of examples that have been labeled as class1 which should actually have been labeled as class2. + If ``asymmetric=False``, the pair (class1, class2) will only be returned once with an arbitrary order. + In this case, their estimated score is the sum: ``score(class1, class2) + score(class2, class1))``. + + class_names : Iterable[str] + A list or other iterable of the string class names. The list should be in the order that + matches the class indices. So if class 0 is 'dog' and class 1 is 'cat', then + ``class_names = ['dog', 'cat']``. + + num_examples : int or None, optional + The number of examples in the dataset, i.e. ``len(labels)``. You only need to provide this if + you use this function with the joint, e.g. ``find_overlapping_classes(joint=joint)``, otherwise + this is automatically computed via ``sum(confident_joint)`` or ``len(labels)``. + + joint : np.ndarray, optional + An array of shape ``(K, K)``, where K is the number of classes, + representing the estimated joint distribution of the noisy labels and + true labels. The sum of all entries in this matrix must be 1 (valid + probability distribution). Each entry in the matrix captures the co-occurence joint + probability of a true label and a noisy label, i.e. ``p(noisy_label=i, true_label=j)``. + **Important**. If you input the joint, you must also input `num_examples`. + + confident_joint : np.ndarray, optional + An array of shape ``(K, K)`` representing the confident joint, the matrix used for identifying label issues, which + estimates a confident subset of the joint distribution of the noisy and true labels, ``P_{noisy label, true label}``. + Entry ``(j, k)`` in the matrix is the number of examples confidently counted into the pair of ``(noisy label=j, true label=k)`` classes. + The `confident_joint` can be computed using :py:func:`count.compute_confident_joint `. + If not provided, it is computed from the given (noisy) `labels` and `pred_probs`. + + Returns + ------- + overlapping_classes : pd.DataFrame + Pandas DataFrame with columns "Class Index A", "Class Index B", + "Num Overlapping Examples", "Joint Probability" and a description of each below. + Each row corresponds to a pair of classes. + + * *Class Index A*: the index of a class in 0, 1, ..., K-1. + * *Class Index B*: the index of a different class (from Class A) in 0, 1, ..., K-1. + * *Num Overlapping Examples*: estimated number of labels overlapping between the two classes. + * *Joint Probability*: the *Num Overlapping Examples* divided by the number of examples in the dataset. + + By default, the DataFrame is ordered by "Joint Probability" descending. + """ + + def _2d_matrix_to_row_column_value_list(matrix): + """Create a list [(row_index, col_index, value)] representation of matrix. + + Parameters + ---------- + matrix : np.ndarray + Any valid np.ndarray 2-d dimensional matrix. + + Returns + ------- + list + A [(row_index, col_index, value)] representation of matrix. + """ + + return [(*i, v) for i, v in np.ndenumerate(matrix)] + + if multi_label: + raise ValueError( + "For multilabel data, please instead call: multilabel_classification.dataset.common_multilabel_issues()" + ) + + if joint is None: + joint = estimate_joint( + labels=labels, + pred_probs=pred_probs, + confident_joint=confident_joint, + ) + if num_examples is None: + num_examples = _get_num_examples(labels=labels, confident_joint=confident_joint) + if asymmetric: + rcv_list = _2d_matrix_to_row_column_value_list(joint) + # Remove diagonal elements + rcv_list = [tup for tup in rcv_list if tup[0] != tup[1]] + else: # symmetric + # Sum the upper and lower triangles and remove the lower triangle and the diagonal + sym_joint = np.triu(joint) + np.tril(joint).T + rcv_list = _2d_matrix_to_row_column_value_list(sym_joint) + # Provide values only in (the upper triangle) of the matrix. + rcv_list = [tup for tup in rcv_list if tup[0] < tup[1]] + df = pd.DataFrame(rcv_list, columns=["Class Index A", "Class Index B", "Joint Probability"]) + num_overlapping = (df["Joint Probability"] * num_examples).round().astype(int) + df.insert(loc=2, column="Num Overlapping Examples", value=num_overlapping) + if class_names is not None: + df.insert( + loc=0, column="Class Name A", value=df["Class Index A"].apply(lambda x: class_names[x]) + ) + df.insert( + loc=1, column="Class Name B", value=df["Class Index B"].apply(lambda x: class_names[x]) + ) + return df.sort_values(by="Joint Probability", ascending=False).reset_index(drop=True) + + +def overall_label_health_score( + labels=None, + pred_probs=None, + *, + num_examples=None, + confident_joint=None, + joint=None, + multi_label=False, + verbose=True, +) -> float: + """Returns a single score between 0 and 1 measuring the overall quality of all labels in a dataset. + Intuitively, the score is the average correctness of the given labels across all examples in the + dataset. So a score of 1 suggests your data is perfectly labeled and a score of 0.5 suggests + half of the examples in the dataset may be incorrectly labeled. Thus, a higher + score implies a higher quality dataset. + + This method works by providing any one (and only one) of the following inputs: + + 1. ``labels`` and ``pred_probs``, or + 2. ``joint`` and ``num_examples``, or + 3. ``confident_joint`` + + Only provide **exactly one of the above input options**, do not provide a combination. + + Examples + -------- + >>> from cleanlab.dataset import overall_label_health_score + >>> from sklearn.linear_model import LogisticRegression + >>> from sklearn.model_selection import cross_val_predict + >>> data, labels = get_data_labels_from_dataset() + >>> yourFavoriteModel = LogisticRegression() + >>> pred_probs = cross_val_predict(yourFavoriteModel, data, labels, cv=3, method="predict_proba") + >>> score = overall_label_health_score(labels=labels, pred_probs=pred_probs) # doctest: +SKIP + + **Parameters**: For parameter info, see the docstring of `~cleanlab.dataset.find_overlapping_classes`. + + + Returns + ------- + health_score : float + A score between 0 and 1, where 1 implies all labels in the dataset are estimated to be correct. + A score of 0.5 implies that half of the dataset's labels are estimated to have issues. + """ + if multi_label: + raise ValueError( + "For multilabel data, please instead call: multilabel_classification.dataset.overall_multilabel_health_score()" + ) + if num_examples is None: + num_examples = _get_num_examples(labels=labels, confident_joint=confident_joint) + + if pred_probs is None or labels is None: + if joint is None: + joint = estimate_joint( + labels=labels, + pred_probs=pred_probs, + confident_joint=confident_joint, + ) + joint_trace = joint.trace() + num_issues = (num_examples * (1 - joint_trace)).round().astype(int) + health_score = joint_trace + else: + num_issues = num_label_issues( + labels=labels, pred_probs=pred_probs, confident_joint=confident_joint + ) + health_score = 1 - num_issues / num_examples + + if verbose: + print( + f" * Overall, about {(1 - health_score):.0%} ({num_issues:,} of the {num_examples:,}) " + f"labels in your dataset have potential issues.\n" + f" ** The overall label health score for this dataset is: {health_score:.2f}." + ) + return health_score + + +def health_summary( + labels=None, + pred_probs=None, + *, + asymmetric=False, + class_names=None, + num_examples=None, + joint=None, + confident_joint=None, + multi_label=False, + verbose=True, +) -> dict: + """Prints a health summary of your dataset. + + This summary includes useful statistics like: + + * The classes with the most and least label issues. + * Classes that overlap and could potentially be merged. + * Overall label quality scores, summarizing how accurate the labels appear overall. + + This method works by providing any one (and only one) of the following inputs: + + 1. ``labels`` and ``pred_probs``, or + 2. ``joint`` and ``num_examples``, or + 3. ``confident_joint`` + + Only provide **exactly one of the above input options**, do not provide a combination. + + Examples + -------- + >>> from cleanlab.dataset import health_summary + >>> from sklearn.linear_model import LogisticRegression + >>> from sklearn.model_selection import cross_val_predict + >>> data, labels = get_data_labels_from_dataset() + >>> yourFavoriteModel = LogisticRegression() + >>> pred_probs = cross_val_predict(yourFavoriteModel, data, labels, cv=3, method="predict_proba") + >>> summary = health_summary(labels=labels, pred_probs=pred_probs) # doctest: +SKIP + + **Parameters**: For parameter info, see the docstring of `~cleanlab.dataset.find_overlapping_classes`. + + Returns + ------- + summary : dict + A dictionary containing keys (see the corresponding functions' documentation to understand the values): + + - ``"overall_label_health_score"``, corresponding to `~cleanlab.dataset.overall_label_health_score` + - ``"joint"``, corresponding to :py:func:`count.estimate_joint ` + - ``"classes_by_label_quality"``, corresponding to `~cleanlab.dataset.rank_classes_by_label_quality` + - ``"overlapping_classes"``, corresponding to `~cleanlab.dataset.find_overlapping_classes` + """ + from cleanlab.internal.util import smart_display_dataframe + + if multi_label: + raise ValueError( + "For multilabel data, please call multilabel_classification.dataset.health_summary" + ) + if joint is None: + joint = estimate_joint( + labels=labels, + pred_probs=pred_probs, + confident_joint=confident_joint, + ) + if num_examples is None: + num_examples = _get_num_examples(labels=labels) + + if verbose: + longest_line = ( + f"| for your dataset with {num_examples:,} examples " + f"and {len(joint):,} classes. |\n" + ) + print( + "-" * (len(longest_line) - 1) + + "\n" + + f"| Generating a Cleanlab Dataset Health Summary{' ' * (len(longest_line) - 49)}|\n" + + longest_line + + f"| Note, Cleanlab is not a medical doctor... yet.{' ' * (len(longest_line) - 51)}|\n" + + "-" * (len(longest_line) - 1) + + "\n", + ) + + df_class_label_quality = rank_classes_by_label_quality( + labels=labels, + pred_probs=pred_probs, + class_names=class_names, + num_examples=num_examples, + joint=joint, + confident_joint=confident_joint, + ) + if verbose: + print("Overall Class Quality and Noise across your dataset (below)") + print("-" * 60, "\n", flush=True) + smart_display_dataframe(df_class_label_quality) + + df_overlapping_classes = find_overlapping_classes( + labels=labels, + pred_probs=pred_probs, + asymmetric=asymmetric, + class_names=class_names, + num_examples=num_examples, + joint=joint, + confident_joint=confident_joint, + ) + if verbose: + print( + "\nClass Overlap. In some cases, you may want to merge classes in the top rows (below)" + + "\n" + + "-" * 83 + + "\n", + flush=True, + ) + smart_display_dataframe(df_overlapping_classes) + print() + + health_score = overall_label_health_score( + labels=labels, + pred_probs=pred_probs, + num_examples=num_examples, + confident_joint=confident_joint, + verbose=verbose, + ) + if verbose: + print("\nGenerated with <3 from Cleanlab.\n") + return { + "overall_label_health_score": health_score, + "joint": joint, + "classes_by_label_quality": df_class_label_quality, + "overlapping_classes": df_overlapping_classes, + } + + +def _get_num_examples(labels=None, confident_joint: Optional[np.ndarray] = None) -> int: + """Helper method that finds the number of examples from the parameters or throws an error + if neither parameter is provided. + + **Parameters:** For information about the arguments to this method, see the documentation of `dataset.find_overlapping_classes` + + Returns + ------- + num_examples : int + The number of examples in the dataset. + + Raises + ------ + ValueError + If `labels` is None.""" + + if labels is None and confident_joint is None: + raise ValueError( + "Error: num_examples is None. You must either provide confident_joint, " + "or provide both num_example and joint as input parameters." + ) + _confident_joint = cast(np.ndarray, confident_joint) + num_examples = len(labels) if labels is not None else cast(int, np.sum(_confident_joint)) + return num_examples diff --git a/cleanlab/experimental/README.md b/cleanlab/experimental/README.md new file mode 100644 index 0000000..3b24007 --- /dev/null +++ b/cleanlab/experimental/README.md @@ -0,0 +1,16 @@ +# Useful methods/models adapted for use with cleanlab + +Methods in this `experimental` module are bleeding edge and may have sharp edges. They are not guaranteed to be stable between different cleanlab versions. + +Some of these files include various models that can be used with cleanlab to find issues in specific types of data. These require dependencies on deep learning and other machine learning packages that are not official cleanlab dependencies. You must install these dependencies on your own if you wish to use them. + +The modules and required dependencies are as follows: +* mnist_pytorch.py - a cleanlab-compatible simplified AlexNet for MNIST using PyTorch + - torch + - torchvision +* cifar_cnn.py - a cleanlab-compatible Convolutional Neural Network for CIFAR using PyTorch, trainable via CoTeaching + - torch + - torchvision +* coteaching.py - an algorithm to train neural networks with noisy labels + - torch + diff --git a/cleanlab/experimental/__init__.py b/cleanlab/experimental/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cleanlab/experimental/coteaching.py b/cleanlab/experimental/coteaching.py new file mode 100644 index 0000000..d36f063 --- /dev/null +++ b/cleanlab/experimental/coteaching.py @@ -0,0 +1,229 @@ +""" +Implements the co-teaching algorithm for training neural networks on noisily-labeled data (Han et al., 2018). +This module requires PyTorch (https://pytorch.org/get-started/locally/). +Example using this algorithm with cleanlab to achieve state of the art on CIFAR-10 +for learning with noisy labels is provided within: https://github.com/cleanlab/examples/ + +``cifar_cnn.py`` provides an example model that can be trained via this algorithm. +""" + +# Significant code was adapted from the following GitHub: +# https://github.com/bhanML/Co-teaching/blob/master/loss.py +# See (Han et al., 2018). + +import torch +import torch.nn.functional as F +from torch.autograd import Variable +import numpy as np + +MINIMUM_BATCH_SIZE = 16 + + +# Loss function for Co-Teaching +def loss_coteaching( + y_1, + y_2, + t, + forget_rate, + class_weights=None, +): + """Co-Teaching Loss function. + + Parameters + ---------- + y_1 : Tensor array + Output logits from model 1 + + y_2 : Tensor array + Output logits from model 2 + + t : np.ndarray + List of Noisy Labels (t means targets) + + forget_rate : float + Decimal between 0 and 1 for how quickly the models forget what they learn. + Just use rate_schedule[epoch] for this value + + class_weights : Tensor array, shape (Number of classes x 1), Default: None + A np.torch.tensor list of length number of classes with weights + """ + + loss_1 = F.cross_entropy(y_1, t, reduce=False, weight=class_weights) + ind_1_sorted = np.argsort(loss_1.data.cpu()) + loss_1_sorted = loss_1[ind_1_sorted] + + loss_2 = F.cross_entropy(y_2, t, reduce=False, weight=class_weights) + ind_2_sorted = np.argsort(loss_2.data.cpu()) + + remember_rate = 1 - forget_rate + num_remember = int(remember_rate * len(loss_1_sorted)) + + ind_1_update = ind_1_sorted[:num_remember] + ind_2_update = ind_2_sorted[:num_remember] + # Share updates between the two models. + # TODO: these class weights should take into account the ind_mask filters. + loss_1_update = F.cross_entropy(y_1[ind_2_update], t[ind_2_update], weight=class_weights) + loss_2_update = F.cross_entropy(y_2[ind_1_update], t[ind_1_update], weight=class_weights) + + return ( + torch.sum(loss_1_update) / num_remember, + torch.sum(loss_2_update) / num_remember, + ) + + +def initialize_lr_scheduler(lr=0.001, epochs=250, epoch_decay_start=80): + """Scheduler to adjust learning rate and betas for Adam Optimizer""" + mom1 = 0.9 + mom2 = 0.9 # Original author had this set to 0.1 + alpha_plan = [lr] * epochs + beta1_plan = [mom1] * epochs + for i in range(epoch_decay_start, epochs): + alpha_plan[i] = float(epochs - i) / (epochs - epoch_decay_start) * lr + beta1_plan[i] = mom2 + return alpha_plan, beta1_plan + + +def adjust_learning_rate(optimizer, epoch, alpha_plan, beta1_plan): + """Scheduler to adjust learning rate and betas for Adam Optimizer""" + for param_group in optimizer.param_groups: + param_group["lr"] = alpha_plan[epoch] + param_group["betas"] = (beta1_plan[epoch], 0.999) # Only change beta1 + + +def forget_rate_scheduler(epochs, forget_rate, num_gradual, exponent): + """Tells Co-Teaching what fraction of examples to forget at each epoch.""" + # define how many things to forget at each rate schedule + forget_rate_schedule = np.ones(epochs) * forget_rate + forget_rate_schedule[:num_gradual] = np.linspace(0, forget_rate**exponent, num_gradual) + return forget_rate_schedule + + +# Train the Model +def train( + train_loader, + epoch, + model1, + optimizer1, + model2, + optimizer2, + args, + forget_rate_schedule, + class_weights, + accuracy, +): + """PyTorch training function. + + Parameters + ---------- + train_loader : torch.utils.data.DataLoader + epoch : int + model1 : PyTorch class inheriting nn.Module + Must define __init__ and forward(self, x,) + optimizer1 : PyTorch torch.optim.Adam + model2 : PyTorch class inheriting nn.Module + Must define __init__ and forward(self, x,) + optimizer2 : PyTorch torch.optim.Adam + args : parser.parse_args() object + Must contain num_iter_per_epoch, print_freq, and epochs + forget_rate_schedule : np.ndarray of length number of epochs + Tells Co-Teaching loss what fraction of examples to forget about. + class_weights : Tensor array, shape (Number of classes x 1), Default: None + A np.torch.tensor list of length number of classes with weights + accuracy : function + A function of the form accuracy(output, target, topk=(1,)) for + computing top1 and top5 accuracy given output and true targets.""" + + train_total = 0 + train_correct = 0 + train_total2 = 0 + train_correct2 = 0 + + # Prepare models for training + model1.train() + model2.train() + + for i, (images, labels) in enumerate(train_loader): + if i == len(train_loader) - 1 and len(labels) < MINIMUM_BATCH_SIZE: + # Edge case -- the last leftover batch is small (potentially size 1) + # This will happen if, for example, you train on 35101 examples with + # batch size of 450. The last batch will be size 1. + # If you update the weights based on the gradient from one example + # if that example is noisy, you will add tons of noise to your net + # and accuracy will actually go down with each epoch. + # To avoid this, do not train on the last batch if it's small. + continue + + images = Variable(images).cuda() + labels = Variable(labels).cuda() + + # Forward + Backward + Optimize + logits1 = model1(images) + prec1, _ = accuracy(logits1, labels, topk=(1, 5)) + train_total += 1 + train_correct += prec1 + logits2 = model2(images) + prec2, _ = accuracy(logits2, labels, topk=(1, 5)) + train_total2 += 1 + train_correct2 += prec2 + loss_1, loss_2 = loss_coteaching( + logits1, + logits2, + labels, + forget_rate=forget_rate_schedule[epoch], + class_weights=class_weights, + ) + optimizer1.zero_grad() + loss_1.backward() + optimizer1.step() + optimizer2.zero_grad() + loss_2.backward() + optimizer2.step() + if (i + 1) % args.print_freq == 0: + print( + "Epoch [%d/%d], Iter [%d/%d] Training Accuracy1: %.4F, " + "Training Accuracy2: %.4f, Loss1: %.4f, Loss2: %.4f " + % ( + epoch + 1, + args.epochs, + i + 1, + len(train_loader.dataset) // args.batch_size, + prec1, + prec2, + loss_1.data.item(), + loss_2.data.item(), + ) + ) + + train_acc1 = float(train_correct) / float(train_total) + train_acc2 = float(train_correct2) / float(train_total2) + return train_acc1, train_acc2 + + +# Evaluate the Model +def evaluate(test_loader, model1, model2): + print("Evaluating Co-Teaching Model") + model1.eval() # Change model to 'eval' mode. + correct1 = 0 + total1 = 0 + for images, labels in test_loader: + images = Variable(images).cuda() + logits1 = model1(images) + outputs1 = F.softmax(logits1, dim=1) + _, pred1 = torch.max(outputs1.data, 1) + total1 += labels.size(0) + correct1 += (pred1.cpu() == labels).sum() + + model2.eval() # Change model to 'eval' mode + correct2 = 0 + total2 = 0 + for images, labels in test_loader: + images = Variable(images).cuda() + logits2 = model2(images) + outputs2 = F.softmax(logits2, dim=1) + _, pred2 = torch.max(outputs2.data, 1) + total2 += labels.size(0) + correct2 += (pred2.cpu() == labels).sum() + + acc1 = 100 * float(correct1) / float(total1) + acc2 = 100 * float(correct2) / float(total2) + return acc1, acc2 diff --git a/cleanlab/experimental/label_issues_batched.py b/cleanlab/experimental/label_issues_batched.py new file mode 100644 index 0000000..afe5266 --- /dev/null +++ b/cleanlab/experimental/label_issues_batched.py @@ -0,0 +1,760 @@ +""" +Implementation of :py:func:`filter.find_label_issues ` +that does not need much memory by operating in mini-batches. +You can also use this approach to estimate label quality scores or the number of label issues +for big datasets with limited memory. + +With default settings, the results returned from this approach closely approximate those returned from: +``cleanlab.filter.find_label_issues(..., filter_by="low_self_confidence", return_indices_ranked_by="self_confidence")`` + +To run this approach, either use the ``find_label_issues_batched()`` convenience function defined in this module, +or follow the examples script for the ``LabelInspector`` class if you require greater customization. +""" + +import numpy as np +from typing import Optional, List, Tuple, Any + +from cleanlab.count import get_confident_thresholds, _reduce_issues +from cleanlab.rank import find_top_issues, _compute_label_quality_scores +from cleanlab.typing import LabelLike +from cleanlab.internal.util import value_counts_fill_missing_classes +from cleanlab.internal.constants import ( + CONFIDENT_THRESHOLDS_LOWER_BOUND, + FLOATING_POINT_COMPARISON, + CLIPPING_LOWER_BOUND, +) + +import platform +import multiprocessing as mp + +try: + import psutil + + PSUTIL_EXISTS = True +except ImportError: # pragma: no cover + PSUTIL_EXISTS = False + +# global variable for multiproc on linux +adj_confident_thresholds_shared: np.ndarray +labels_shared: LabelLike +pred_probs_shared: np.ndarray + + +def find_label_issues_batched( + labels: Optional[LabelLike] = None, + pred_probs: Optional[np.ndarray] = None, + *, + labels_file: Optional[str] = None, + pred_probs_file: Optional[str] = None, + batch_size: int = 10000, + n_jobs: Optional[int] = 1, + verbose: bool = True, + quality_score_kwargs: Optional[dict] = None, + num_issue_kwargs: Optional[dict] = None, + return_mask: bool = False, +) -> np.ndarray: + """ + Variant of :py:func:`filter.find_label_issues ` + that requires less memory by reading from `pred_probs`, `labels` in mini-batches. + To avoid loading big `pred_probs`, `labels` arrays into memory, + provide these as memory-mapped objects like Zarr arrays or memmap arrays instead of regular numpy arrays. + See: https://pythonspeed.com/articles/mmap-vs-zarr-hdf5/ + + With default settings, the results returned from this method closely approximate those returned from: + ``cleanlab.filter.find_label_issues(..., filter_by="low_self_confidence", return_indices_ranked_by="self_confidence")`` + + This function internally implements the example usage script of the ``LabelInspector`` class, + but you can further customize that script by running it yourself instead of this function. + See the documentation of ``LabelInspector`` to learn more about how this method works internally. + + Parameters + ---------- + labels: np.ndarray-like object, optional + 1D array of given class labels for each example in the dataset, (int) values in ``0,1,2,...,K-1``. + To avoid loading big objects into memory, you should pass this as a memory-mapped object like: + Zarr array loaded with ``zarr.convenience.open(YOURFILE.zarr, mode="r")``, + or memmap array loaded with ``np.load(YOURFILE.npy, mmap_mode="r")``. + + Tip: You can save an existing numpy array to Zarr via: ``zarr.convenience.save_array(YOURFILE.zarr, your_array)``, + or to .npy file that can be loaded with mmap via: ``np.save(YOURFILE.npy, your_array)``. + + pred_probs: np.ndarray-like object, optional + 2D array of model-predicted class probabilities (floats) for each example in the dataset. + To avoid loading big objects into memory, you should pass this as a memory-mapped object like: + Zarr array loaded with ``zarr.convenience.open(YOURFILE.zarr, mode="r")`` + or memmap array loaded with ``np.load(YOURFILE.npy, mmap_mode="r")``. + + labels_file: str, optional + Specify this instead of `labels` if you want this method to load from file for you into a memmap array. + Path to .npy file where the entire 1D `labels` numpy array is stored on disk (list format is not supported). + This is loaded using: ``np.load(labels_file, mmap_mode="r")`` + so make sure this file was created via: ``np.save()`` or other compatible methods (.npz not supported). + + pred_probs_file: str, optional + Specify this instead of `pred_probs` if you want this method to load from file for you into a memmap array. + Path to .npy file where the entire `pred_probs` numpy array is stored on disk. + This is loaded using: ``np.load(pred_probs_file, mmap_mode="r")`` + so make sure this file was created via: ``np.save()`` or other compatible methods (.npz not supported). + + batch_size : int, optional + Size of mini-batches to use for estimating the label issues. + To maximize efficiency, try to use the largest `batch_size` your memory allows. + + n_jobs: int, optional + Number of processes for multiprocessing (default value = 1). Only used on Linux. + If `n_jobs=None`, will use either the number of: physical cores if psutil is installed, or logical cores otherwise. + + verbose : bool, optional + Whether to suppress print statements or not. + + quality_score_kwargs : dict, optional + Keyword arguments to pass into :py:func:`rank.get_label_quality_scores `. + + num_issue_kwargs : dict, optional + Keyword arguments to :py:func:`count.num_label_issues ` + to control estimation of the number of label issues. + The only supported kwarg here for now is: `estimation_method`. + return_mask : bool, optional + Determines what is returned by this method: If `return_mask=True`, return a boolean mask. + If `False`, return a list of indices specifying examples with label issues, sorted by label quality score. + + Returns + ------- + label_issues : np.ndarray + If `return_mask` is `True`, returns a boolean **mask** for the entire dataset + where ``True`` represents a label issue and ``False`` represents an example that is + accurately labeled with high confidence. + If `return_mask` is `False`, returns an array containing **indices** of examples identified to have + label issues (i.e. those indices where the mask would be ``True``), sorted by likelihood that the corresponding label is correct. + -------- + >>> batch_size = 10000 # for efficiency, set this to as large of a value as your memory can handle + >>> # Just demonstrating how to save your existing numpy labels, pred_probs arrays to compatible .npy files: + >>> np.save("LABELS.npy", labels_array) + >>> np.save("PREDPROBS.npy", pred_probs_array) + >>> # You can load these back into memmap arrays via: labels = np.load("LABELS.npy", mmap_mode="r") + >>> # and then run this method on the memmap arrays, or just run it directly on the .npy files like this: + >>> issues = find_label_issues_batched(labels_file="LABELS.npy", pred_probs_file="PREDPROBS.npy", batch_size=batch_size) + >>> # This method also works with Zarr arrays: + >>> import zarr + >>> # Just demonstrating how to save your existing numpy labels, pred_probs arrays to compatible .zarr files: + >>> zarr.convenience.save_array("LABELS.zarr", labels_array) + >>> zarr.convenience.save_array("PREDPROBS.zarr", pred_probs_array) + >>> # You can load from such files into Zarr arrays: + >>> labels = zarr.convenience.open("LABELS.zarr", mode="r") + >>> pred_probs = zarr.convenience.open("PREDPROBS.zarr", mode="r") + >>> # This method can be directly run on Zarr arrays, memmap arrays, or regular numpy arrays: + >>> issues = find_label_issues_batched(labels=labels, pred_probs=pred_probs, batch_size=batch_size) + """ + if labels_file is not None: + if labels is not None: + raise ValueError("only specify one of: `labels` or `labels_file`") + if not isinstance(labels_file, str): + raise ValueError( + "labels_file must be str specifying path to .npy file containing the array of labels" + ) + labels = np.load(labels_file, mmap_mode="r") + assert isinstance(labels, np.ndarray) + + if pred_probs_file is not None: + if pred_probs is not None: + raise ValueError("only specify one of: `pred_probs` or `pred_probs_file`") + if not isinstance(pred_probs_file, str): + raise ValueError( + "pred_probs_file must be str specifying path to .npy file containing 2D array of pred_probs" + ) + pred_probs = np.load(pred_probs_file, mmap_mode="r") + assert isinstance(pred_probs, np.ndarray) + if verbose: + print( + f"mmap-loaded numpy arrays have: {len(pred_probs)} examples, {pred_probs.shape[1]} classes" + ) + if labels is None: + raise ValueError("must provide one of: `labels` or `labels_file`") + if pred_probs is None: + raise ValueError("must provide one of: `pred_probs` or `pred_probs_file`") + + assert pred_probs is not None + if len(labels) != len(pred_probs): + raise ValueError( + f"len(labels)={len(labels)} does not match len(pred_probs)={len(pred_probs)}. Perhaps an issue loading mmap numpy arrays from file." + ) + lab = LabelInspector( + num_class=pred_probs.shape[1], + verbose=verbose, + n_jobs=n_jobs, + quality_score_kwargs=quality_score_kwargs, + num_issue_kwargs=num_issue_kwargs, + ) + n = len(labels) + if verbose: + from tqdm.auto import tqdm + + pbar = tqdm(desc="number of examples processed for estimating thresholds", total=n) + i = 0 + while i < n: + end_index = i + batch_size + labels_batch = labels[i:end_index] + pred_probs_batch = pred_probs[i:end_index, :] + i = end_index + lab.update_confident_thresholds(labels_batch, pred_probs_batch) + if verbose: + pbar.update(batch_size) + + # Next evaluate the quality of the labels (run this on full dataset you want to evaluate): + if verbose: + pbar.close() + pbar = tqdm(desc="number of examples processed for checking labels", total=n) + i = 0 + while i < n: + end_index = i + batch_size + labels_batch = labels[i:end_index] + pred_probs_batch = pred_probs[i:end_index, :] + i = end_index + _ = lab.score_label_quality(labels_batch, pred_probs_batch) + if verbose: + pbar.update(batch_size) + + if verbose: + pbar.close() + + label_issues_indices = lab.get_label_issues() + label_issues_mask = np.zeros(len(labels), dtype=bool) + label_issues_mask[label_issues_indices] = True + mask = _reduce_issues(pred_probs=pred_probs, labels=labels) + label_issues_mask[mask] = False + if return_mask: + return label_issues_mask + return np.where(label_issues_mask)[0] + + +class LabelInspector: + """ + Class for finding label issues in big datasets where memory becomes a problem for other cleanlab methods. + Only create one such object per dataset and do not try to use the same ``LabelInspector`` across 2 datasets. + For efficiency, this class does little input checking. + You can first run :py:func:`filter.find_label_issues ` + on a small subset of your data to verify your inputs are properly formatted. + Do NOT modify any of the attributes of this class yourself! + Multi-label classification is not supported by this class, it is only for multi-class classification. + + The recommended usage demonstrated in the examples script below involves two passes over your data: + one pass to compute `confident_thresholds`, another to evaluate each label. + To maximize efficiency, try to use the largest batch_size your memory allows. + To reduce runtime further, you can run the first pass on a subset of your dataset + as long as it contains enough data from each class to estimate `confident_thresholds` accurately. + + In the examples script below: + - `labels` is a (big) 1D ``np.ndarray`` of class labels represented as integers in ``0,1,...,K-1``. + - ``pred_probs`` = is a (big) 2D ``np.ndarray`` of predicted class probabilities, + where each row is an example, each column represents a class. + + `labels` and `pred_probs` can be stored in a file instead where you load chunks of them at a time. + Methods to load arrays in chunks include: ``np.load(...,mmap_mode='r')``, ``numpy.memmap()``, + HDF5 or Zarr files, see: https://pythonspeed.com/articles/mmap-vs-zarr-hdf5/ + + Examples + -------- + >>> n = len(labels) + >>> batch_size = 10000 # you can change this in between batches, set as big as your RAM allows + >>> lab = LabelInspector(num_class = pred_probs.shape[1]) + >>> # First compute confident thresholds (for faster results, can also do this on a random subset of your data): + >>> i = 0 + >>> while i < n: + >>> end_index = i + batch_size + >>> labels_batch = labels[i:end_index] + >>> pred_probs_batch = pred_probs[i:end_index,:] + >>> i = end_index + >>> lab.update_confident_thresholds(labels_batch, pred_probs_batch) + >>> # See what we calculated: + >>> confident_thresholds = lab.get_confident_thresholds() + >>> # Evaluate the quality of the labels (run this on full dataset you want to evaluate): + >>> i = 0 + >>> while i < n: + >>> end_index = i + batch_size + >>> labels_batch = labels[i:end_index] + >>> pred_probs_batch = pred_probs[i:end_index,:] + >>> i = end_index + >>> batch_results = lab.score_label_quality(labels_batch, pred_probs_batch) + >>> # Indices of examples with label issues, sorted by label quality score (most severe to least severe): + >>> indices_of_examples_with_issues = lab.get_label_issues() + >>> # If your `pred_probs` and `labels` are arrays already in memory, + >>> # then you can use this shortcut for all of the above: + >>> indices_of_examples_with_issues = find_label_issues_batched(labels, pred_probs, batch_size=10000) + + Parameters + ---------- + num_class : int + The number of classes in your multi-class classification task. + + store_results : bool, optional + Whether this object will store all label quality scores, a 1D array of shape ``(N,)`` + where ``N`` is the total number of examples in your dataset. + Set this to False if you encounter memory problems even for small batch sizes (~1000). + If ``False``, you can still identify the label issues yourself by aggregating + the label quality scores for each batch, sorting them across all batches, and returning the top ``T`` indices + with ``T = self.get_num_issues()``. + + verbose : bool, optional + Whether to suppress print statements or not. + + n_jobs: int, optional + Number of processes for multiprocessing (default value = 1). Only used on Linux. + If `n_jobs=None`, will use either the number of: physical cores if psutil is installed, or logical cores otherwise. + + quality_score_kwargs : dict, optional + Keyword arguments to pass into :py:func:`rank.get_label_quality_scores `. + + num_issue_kwargs : dict, optional + Keyword arguments to :py:func:`count.num_label_issues ` + to control estimation of the number of label issues. + The only supported kwarg here for now is: `estimation_method`. + """ + + def __init__( + self, + *, + num_class: int, + store_results: bool = True, + verbose: bool = True, + quality_score_kwargs: Optional[dict] = None, + num_issue_kwargs: Optional[dict] = None, + n_jobs: Optional[int] = 1, + ): + if quality_score_kwargs is None: + quality_score_kwargs = {} + if num_issue_kwargs is None: + num_issue_kwargs = {} + + self.num_class = num_class + self.store_results = store_results + self.verbose = verbose + self.quality_score_kwargs = quality_score_kwargs # extra arguments for ``rank.get_label_quality_scores()`` to control label quality scoring + self.num_issue_kwargs = num_issue_kwargs # extra arguments for ``count.num_label_issues()`` to control estimation of the number of label issues (only supported argument for now is: `estimation_method`). + self.off_diagonal_calibrated = False + if num_issue_kwargs.get("estimation_method") == "off_diagonal_calibrated": + # store extra attributes later needed for calibration: + self.off_diagonal_calibrated = True + self.prune_counts = np.zeros(self.num_class) + self.class_counts = np.zeros(self.num_class) + self.normalization = np.zeros(self.num_class) + else: + self.prune_count = 0 # number of label issues estimated based on data seen so far (only used when estimation_method is not calibrated) + + if self.store_results: + self.label_quality_scores: List[float] = [] + + self.confident_thresholds = np.zeros( + (num_class,) + ) # current estimate of thresholds based on data seen so far + self.examples_per_class = np.zeros( + (num_class,) + ) # current counts of examples with each given label seen so far + self.examples_processed_thresh = ( + 0 # number of examples seen so far for estimating thresholds + ) + self.examples_processed_quality = 0 # number of examples seen so far for estimating label quality and number of label issues + # Determine number of cores for multiprocessing: + self.n_jobs: Optional[int] = None + os_name = platform.system() + if os_name != "Linux": + self.n_jobs = 1 + if n_jobs is not None and n_jobs != 1 and self.verbose: + print( + "n_jobs is overridden to 1 because multiprocessing is only supported for Linux." + ) + elif n_jobs is not None: + self.n_jobs = n_jobs + else: + if PSUTIL_EXISTS: + self.n_jobs = psutil.cpu_count(logical=False) # physical cores + if not self.n_jobs: + # switch to logical cores + self.n_jobs = mp.cpu_count() + if self.verbose: + print( + f"Multiprocessing will default to using the number of logical cores ({self.n_jobs}). To default to number of physical cores: pip install psutil" + ) + + def get_confident_thresholds(self, silent: bool = False) -> np.ndarray: + """ + Fetches already-computed confident thresholds from the data seen so far + in same format as: :py:func:`count.get_confident_thresholds `. + + + Returns + ------- + confident_thresholds : np.ndarray + An array of shape ``(K, )`` where ``K`` is the number of classes. + """ + if self.examples_processed_thresh < 1: + raise ValueError( + "Have not computed any confident_thresholds yet. Call `update_confident_thresholds()` first." + ) + else: + if self.verbose and not silent: + print( + f"Total number of examples used to estimate confident thresholds: {self.examples_processed_thresh}" + ) + return self.confident_thresholds + + def get_num_issues(self, silent: bool = False) -> int: + """ + Fetches already-computed estimate of the number of label issues in the data seen so far + in the same format as: :py:func:`count.num_label_issues `. + + Note: The estimated number of issues may differ from :py:func:`count.num_label_issues ` + by 1 due to rounding differences. + + Returns + ------- + num_issues : int + The estimated number of examples with label issues in the data seen so far. + """ + if self.examples_processed_quality < 1: + raise ValueError( + "Have not evaluated any labels yet. Call `score_label_quality()` first." + ) + else: + if self.verbose and not silent: + print( + f"Total number of examples whose labels have been evaluated: {self.examples_processed_quality}" + ) + if self.off_diagonal_calibrated: + calibrated_prune_counts = ( + self.prune_counts + * self.class_counts + / np.clip(self.normalization, a_min=CLIPPING_LOWER_BOUND, a_max=None) + ) # avoid division by 0 + return np.rint(np.sum(calibrated_prune_counts)).astype("int") + else: # not calibrated + return self.prune_count + + def get_quality_scores(self) -> np.ndarray: + """ + Fetches already-computed estimate of the label quality of each example seen so far + in the same format as: :py:func:`rank.get_label_quality_scores `. + + Returns + ------- + label_quality_scores : np.ndarray + Contains one score (between 0 and 1) per example seen so far. + Lower scores indicate more likely mislabeled examples. + """ + if not self.store_results: + raise ValueError( + "Must initialize the LabelInspector with `store_results` == True. " + "Otherwise you can assemble the label quality scores yourself based on " + "the scores returned for each batch of data from `score_label_quality()`" + ) + else: + return np.asarray(self.label_quality_scores) + + def get_label_issues(self) -> np.ndarray: + """ + Fetches already-computed estimate of indices of examples with label issues in the data seen so far, + in the same format as: :py:func:`filter.find_label_issues ` + with its `return_indices_ranked_by` argument specified. + + Note: this method corresponds to ``filter.find_label_issues(..., filter_by=METHOD1, return_indices_ranked_by=METHOD2)`` + where by default: ``METHOD1="low_self_confidence"``, ``METHOD2="self_confidence"`` + or if this object was instantiated with ``quality_score_kwargs = {"method": "normalized_margin"}`` then we instead have: + ``METHOD1="low_normalized_margin"``, ``METHOD2="normalized_margin"``. + + Note: The estimated number of issues may differ from :py:func:`filter.find_label_issues ` + by 1 due to rounding differences. + + Returns + ------- + issue_indices : np.ndarray + Indices of examples with label issues, sorted by label quality score. + """ + if not self.store_results: + raise ValueError( + "Must initialize the LabelInspector with `store_results` == True. " + "Otherwise you can identify label issues yourself based on the scores from all " + "the batches of data and the total number of issues returned by `get_num_issues()`" + ) + if self.examples_processed_quality < 1: + raise ValueError( + "Have not evaluated any labels yet. Call `score_label_quality()` first." + ) + if self.verbose: + print( + f"Total number of examples whose labels have been evaluated: {self.examples_processed_quality}" + ) + return find_top_issues(self.get_quality_scores(), top=self.get_num_issues(silent=True)) + + def update_confident_thresholds(self, labels: LabelLike, pred_probs: np.ndarray): + """ + Updates the estimate of confident_thresholds stored in this class using a new batch of data. + Inputs should be in same format as for: :py:func:`count.get_confident_thresholds `. + + Parameters + ---------- + labels: np.ndarray or list + Given class labels for each example in the batch, values in ``0,1,2,...,K-1``. + + pred_probs: np.ndarray + 2D array of model-predicted class probabilities for each example in the batch. + """ + labels = _batch_check(labels, pred_probs, self.num_class) + batch_size = len(labels) + batch_thresholds = get_confident_thresholds( + labels, pred_probs + ) # values for missing classes may exceed 1 but should not matter since we multiply by this class counts in the batch + batch_class_counts = value_counts_fill_missing_classes(labels, num_classes=self.num_class) + self.confident_thresholds = ( + self.examples_per_class * self.confident_thresholds + + batch_class_counts * batch_thresholds + ) / np.clip( + self.examples_per_class + batch_class_counts, a_min=1, a_max=None + ) # avoid division by 0 + self.confident_thresholds = np.clip( + self.confident_thresholds, a_min=CONFIDENT_THRESHOLDS_LOWER_BOUND, a_max=None + ) + self.examples_per_class += batch_class_counts + self.examples_processed_thresh += batch_size + + def score_label_quality( + self, + labels: LabelLike, + pred_probs: np.ndarray, + *, + update_num_issues: bool = True, + ) -> np.ndarray: + """ + Scores the label quality of each example in the provided batch of data, + and also updates the number of label issues stored in this class. + Inputs should be in same format as for: :py:func:`rank.get_label_quality_scores `. + + Parameters + ---------- + labels: np.ndarray + Given class labels for each example in the batch, values in ``0,1,2,...,K-1``. + + pred_probs: np.ndarray + 2D array of model-predicted class probabilities for each example in the batch of data. + + update_num_issues: bool, optional + Whether or not to update the number of label issues or only compute label quality scores. + For lower runtimes, set this to ``False`` if you only want to score label quality and not find label issues. + + Returns + ------- + label_quality_scores : np.ndarray + Contains one score (between 0 and 1) for each example in the batch of data. + """ + labels = _batch_check(labels, pred_probs, self.num_class) + batch_size = len(labels) + scores = _compute_label_quality_scores( + labels, + pred_probs, + confident_thresholds=self.get_confident_thresholds(silent=True), + **self.quality_score_kwargs, + ) + class_counts = value_counts_fill_missing_classes(labels, num_classes=self.num_class) + if update_num_issues: + self._update_num_label_issues(labels, pred_probs, **self.num_issue_kwargs) + self.examples_processed_quality += batch_size + if self.store_results: + self.label_quality_scores += list(scores) + + return scores + + def _update_num_label_issues( + self, + labels: LabelLike, + pred_probs: np.ndarray, + **kwargs, + ): + """ + Update the estimate of num_label_issues stored in this class using a new batch of data. + Kwargs are ignored here for now (included for forwards compatibility). + Instead of being specified here, `estimation_method` should be declared when this class is initialized. + """ + + # whether to match the output of count.num_label_issues exactly + # default is False, which gives significant speedup on large batches + # and empirically matches num_label_issues even on input sizes of + # 1M x 10k + thorough = False + if self.examples_processed_thresh < 1: + raise ValueError( + "Have not computed any confident_thresholds yet. Call `update_confident_thresholds()` first." + ) + + if self.n_jobs == 1: + adj_confident_thresholds = self.confident_thresholds - FLOATING_POINT_COMPARISON + pred_class = np.argmax(pred_probs, axis=1) + batch_size = len(labels) + if thorough: + # add margin for floating point comparison operations: + pred_gt_thresholds = pred_probs >= adj_confident_thresholds + max_ind = np.argmax(pred_probs * pred_gt_thresholds, axis=1) + if not self.off_diagonal_calibrated: + mask = (max_ind != labels) & (pred_class != labels) + else: + # calibrated + # should we change to above? + mask = pred_class != labels + else: + max_ind = pred_class + mask = pred_class != labels + + if not self.off_diagonal_calibrated: + prune_count_batch = np.sum( + ( + pred_probs[np.arange(batch_size), max_ind] + >= adj_confident_thresholds[max_ind] + ) + & mask + ) + self.prune_count += prune_count_batch + else: # calibrated + self.class_counts += value_counts_fill_missing_classes( + labels, num_classes=self.num_class + ) + to_increment = ( + pred_probs[np.arange(batch_size), max_ind] >= adj_confident_thresholds[max_ind] + ) + for class_label in range(self.num_class): + labels_equal_to_class = labels == class_label + self.normalization[class_label] += np.sum(labels_equal_to_class & to_increment) + self.prune_counts[class_label] += np.sum( + labels_equal_to_class + & to_increment + & (max_ind != labels) + # & (pred_class != labels) + # This is not applied in num_label_issues(..., estimation_method="off_diagonal_custom"). Do we want to add it? + ) + else: # multiprocessing implementation + global adj_confident_thresholds_shared + adj_confident_thresholds_shared = self.confident_thresholds - FLOATING_POINT_COMPARISON + + global labels_shared, pred_probs_shared + labels_shared = labels + pred_probs_shared = pred_probs + + # good values for this are ~1000-10000 in benchmarks where pred_probs has 1B entries: + processes = 5000 + if len(labels) <= processes: + chunksize = 1 + else: + chunksize = len(labels) // processes + inds = split_arr(np.arange(len(labels)), chunksize) + + if thorough: + use_thorough = np.ones(len(inds), dtype=bool) + else: + use_thorough = np.zeros(len(inds), dtype=bool) + args = zip(inds, use_thorough) + + # Use fork method explicitly for Python 3.14+ compatibility + # Falls back to default method if fork is not available + try: + ctx = mp.get_context("fork") + pool_class = ctx.Pool + except (RuntimeError, ValueError): + # fork not available (Windows) or already set, use default + pool_class = mp.Pool + + with pool_class(self.n_jobs) as pool: + if not self.off_diagonal_calibrated: + prune_count_batch = np.sum( + np.asarray(list(pool.imap_unordered(_compute_num_issues, args))) + ) + self.prune_count += prune_count_batch + else: + results = list(pool.imap_unordered(_compute_num_issues_calibrated, args)) + for result in results: + class_label = result[0] + self.class_counts[class_label] += 1 + self.normalization[class_label] += result[1] + self.prune_counts[class_label] += result[2] + + +def split_arr(arr: np.ndarray, chunksize: int) -> List[np.ndarray]: + """ + Helper function to split array into chunks for multiprocessing. + """ + return np.split(arr, np.arange(chunksize, arr.shape[0], chunksize), axis=0) + + +def _compute_num_issues(arg: Tuple[np.ndarray, bool]) -> int: + """ + Helper function for `_update_num_label_issues` multiprocessing without calibration. + """ + ind = arg[0] + thorough = arg[1] + label = labels_shared[ind] + pred_prob = pred_probs_shared[ind, :] + pred_class = np.argmax(pred_prob, axis=-1) + batch_size = len(label) + + if thorough: + pred_gt_thresholds = pred_prob >= adj_confident_thresholds_shared + max_ind = np.argmax(pred_prob * pred_gt_thresholds, axis=-1) + prune_count_batch = np.sum( + (pred_prob[np.arange(batch_size), max_ind] >= adj_confident_thresholds_shared[max_ind]) + & (max_ind != label) + & (pred_class != label) + ) + else: + prune_count_batch = np.sum( + ( + pred_prob[np.arange(batch_size), pred_class] + >= adj_confident_thresholds_shared[pred_class] + ) + & (pred_class != label) + ) + return prune_count_batch + + +def _compute_num_issues_calibrated(arg: Tuple[np.ndarray, bool]) -> Tuple[Any, int, int]: + """ + Helper function for `_update_num_label_issues` multiprocessing with calibration. + """ + ind = arg[0] + thorough = arg[1] + label = labels_shared[ind] + pred_prob = pred_probs_shared[ind, :] + batch_size = len(label) + + pred_class = np.argmax(pred_prob, axis=-1) + if thorough: + pred_gt_thresholds = pred_prob >= adj_confident_thresholds_shared + max_ind = np.argmax(pred_prob * pred_gt_thresholds, axis=-1) + to_inc = ( + pred_prob[np.arange(batch_size), max_ind] >= adj_confident_thresholds_shared[max_ind] + ) + + prune_count_batch = to_inc & (max_ind != label) + normalization_batch = to_inc + else: + to_inc = ( + pred_prob[np.arange(batch_size), pred_class] + >= adj_confident_thresholds_shared[pred_class] + ) + normalization_batch = to_inc + prune_count_batch = to_inc & (pred_class != label) + + return (label, normalization_batch, prune_count_batch) + + +def _batch_check(labels: LabelLike, pred_probs: np.ndarray, num_class: int) -> np.ndarray: + """ + Basic checks to ensure batch of data looks ok. For efficiency, this check is quite minimal. + + Returns + ------- + labels : np.ndarray + `labels` formatted as a 1D array. + """ + batch_size = pred_probs.shape[0] + labels = np.asarray(labels) + if len(labels) != batch_size: + raise ValueError("labels and pred_probs must have same length") + if pred_probs.shape[1] != num_class: + raise ValueError("num_class must equal pred_probs.shape[1]") + + return labels diff --git a/cleanlab/experimental/mnist_pytorch.py b/cleanlab/experimental/mnist_pytorch.py new file mode 100644 index 0000000..b70a39b --- /dev/null +++ b/cleanlab/experimental/mnist_pytorch.py @@ -0,0 +1,369 @@ +""" +A cleanlab-compatible PyTorch ConvNet classifier that can be used to find +label issues in image data. +This is a good example to reference for making your own bespoke model compatible with cleanlab. + +You must have PyTorch installed: https://pytorch.org/get-started/locally/ +""" + +from sklearn.base import BaseEstimator +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torchvision import datasets, transforms +from torch.autograd import Variable +from torch.utils.data.sampler import SubsetRandomSampler +import numpy as np + + +MNIST_TRAIN_SIZE = 60000 +MNIST_TEST_SIZE = 10000 +SKLEARN_DIGITS_TRAIN_SIZE = 1247 +SKLEARN_DIGITS_TEST_SIZE = 550 + + +def get_mnist_dataset(loader): # pragma: no cover + """Downloads MNIST as PyTorch dataset. + + Parameters + ---------- + loader : str (values: 'train' or 'test').""" + dataset = datasets.MNIST( + root="../data", + train=(loader == "train"), + download=True, + transform=transforms.Compose( + [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))] + ), + ) + return dataset + + +def get_sklearn_digits_dataset(loader): + """Downloads Sklearn handwritten digits dataset. + Uses the last SKLEARN_DIGITS_TEST_SIZE examples as the test + This is (hard-coded) -- do not change. + + Parameters + ---------- + loader : str (values: 'train' or 'test').""" + from torch.utils.data import Dataset + from sklearn.datasets import load_digits + + class TorchDataset(Dataset): + """Abstracts a numpy array as a PyTorch dataset.""" + + def __init__(self, data, targets, transform=None): + self.data = torch.from_numpy(data).float() + self.targets = torch.from_numpy(targets).long() + self.transform = transform + + def __getitem__(self, index): + x = self.data[index] + y = self.targets[index] + if self.transform: + x = self.transform(x) + return x, y + + def __len__(self): + return len(self.data) + + transform = transforms.Compose( + [ + transforms.ToPILImage(), + transforms.Resize(28), + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)), + ] + ) + # Get sklearn digits dataset + X_all, y_all = load_digits(return_X_y=True) + X_all = X_all.reshape((len(X_all), 8, 8)) + y_train = y_all[:-SKLEARN_DIGITS_TEST_SIZE] + y_test = y_all[-SKLEARN_DIGITS_TEST_SIZE:] + X_train = X_all[:-SKLEARN_DIGITS_TEST_SIZE] + X_test = X_all[-SKLEARN_DIGITS_TEST_SIZE:] + if loader == "train": + return TorchDataset(X_train, y_train, transform=transform) + elif loader == "test": + return TorchDataset(X_test, y_test, transform=transform) + else: # prama: no cover + raise ValueError("loader must be either str 'train' or str 'test'.") + + +class SimpleNet(nn.Module): + """Basic Pytorch CNN for MNIST-like data.""" + + def __init__(self): + super(SimpleNet, self).__init__() + self.conv1 = nn.Conv2d(1, 10, kernel_size=5) + self.conv2 = nn.Conv2d(10, 20, kernel_size=5) + self.conv2_drop = nn.Dropout2d() + self.fc1 = nn.Linear(320, 50) + self.fc2 = nn.Linear(50, 10) + + def forward(self, x, T=1.0): + x = F.relu(F.max_pool2d(self.conv1(x), 2)) + x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2)) + x = x.view(-1, 320) + x = F.relu(self.fc1(x)) + x = F.dropout(x, training=self.training) + x = self.fc2(x) + x = F.log_softmax(x, dim=1) + return x + + +class CNN(BaseEstimator): # Inherits sklearn classifier + """Wraps a PyTorch CNN for the MNIST dataset within an sklearn template + + Defines ``.fit()``, ``.predict()``, and ``.predict_proba()`` functions. This + template enables the PyTorch CNN to flexibly be used within the sklearn + architecture -- meaning it can be passed into functions like + cross_val_predict as if it were an sklearn model. The cleanlab library + requires that all models adhere to this basic sklearn template and thus, + this class allows a PyTorch CNN to be used in for learning with noisy + labels among other things. + + Parameters + ---------- + batch_size: int + epochs: int + log_interval: int + lr: float + momentum: float + no_cuda: bool + seed: int + test_batch_size: int, default=None + dataset: {'mnist', 'sklearn-digits'} + loader: {'train', 'test'} + Set to 'test' to force fit() and predict_proba() on test_set + + Note + ---- + Be careful setting the ``loader`` param, it will override every other loader + If you set this to 'test', but call .predict(loader = 'train') + then .predict() will still predict on test! + + Attributes + ---------- + batch_size: int + epochs: int + log_interval: int + lr: float + momentum: float + no_cuda: bool + seed: int + test_batch_size: int, default=None + dataset: {'mnist', 'sklearn-digits'} + loader: {'train', 'test'} + Set to 'test' to force fit() and predict_proba() on test_set + + Methods + ------- + fit + fits the model to data. + predict + get the fitted model's prediction on test data + predict_proba + get the fitted model's probability distribution over classes for test data + """ + + def __init__( + self, + batch_size=64, + epochs=6, + log_interval=50, # Set to None to not print + lr=0.01, + momentum=0.5, + no_cuda=False, + seed=1, + test_batch_size=None, + dataset="mnist", + loader=None, + ): + self.batch_size = batch_size + self.epochs = epochs + self.log_interval = log_interval + self.lr = lr + self.momentum = momentum + self.no_cuda = no_cuda + self.seed = seed + self.cuda = not self.no_cuda and torch.cuda.is_available() + torch.manual_seed(self.seed) + if self.cuda: # pragma: no cover + torch.cuda.manual_seed(self.seed) + + # Instantiate PyTorch model + self.model = SimpleNet() + if self.cuda: # pragma: no cover + self.model.cuda() + + self.loader_kwargs = {"num_workers": 1, "pin_memory": True} if self.cuda else {} + self.loader = loader + self._set_dataset(dataset) + if test_batch_size is not None: + self.test_batch_size = test_batch_size + else: + self.test_batch_size = self.test_size + + def _set_dataset(self, dataset): + self.dataset = dataset + if dataset == "mnist": + # pragma: no cover + self.get_dataset = get_mnist_dataset + self.train_size = MNIST_TRAIN_SIZE + self.test_size = MNIST_TEST_SIZE + elif dataset == "sklearn-digits": + self.get_dataset = get_sklearn_digits_dataset + self.train_size = SKLEARN_DIGITS_TRAIN_SIZE + self.test_size = SKLEARN_DIGITS_TEST_SIZE + else: # pragma: no cover + raise ValueError("dataset must be 'mnist' or 'sklearn-digits'.") + + # XXX this is a pretty weird sklearn estimator that does data loading + # internally in `fit`, and it supports multiple datasets and is aware of + # which dataset it's using; if we weren't doing this, we wouldn't need to + # override `get_params` / `set_params` + def get_params(self, deep=True): + return { + "batch_size": self.batch_size, + "epochs": self.epochs, + "log_interval": self.log_interval, + "lr": self.lr, + "momentum": self.momentum, + "no_cuda": self.no_cuda, + "test_batch_size": self.test_batch_size, + "dataset": self.dataset, + } + + def set_params(self, **parameters): # pragma: no cover + for parameter, value in parameters.items(): + if parameter != "dataset": + setattr(self, parameter, value) + if "dataset" in parameters: + self._set_dataset(parameters["dataset"]) + return self + + def fit(self, train_idx, train_labels=None, sample_weight=None, loader="train"): + """This function adheres to sklearn's "fit(X, y)" format for + compatibility with scikit-learn. ** All inputs should be numpy + arrays, not pyTorch Tensors train_idx is not X, but instead a list of + indices for X (and y if train_labels is None). This function is a + member of the cnn class which will handle creation of X, y from the + train_idx via the train_loader.""" + if self.loader is not None: + loader = self.loader + if train_labels is not None and len(train_idx) != len(train_labels): + raise ValueError("Check that train_idx and train_labels are the same length.") + + if sample_weight is not None: # pragma: no cover + if len(sample_weight) != len(train_labels): + raise ValueError( + "Check that train_labels and sample_weight " "are the same length." + ) + class_weight = sample_weight[np.unique(train_labels, return_index=True)[1]] + class_weight = torch.from_numpy(class_weight).float() + if self.cuda: + class_weight = class_weight.cuda() + else: + class_weight = None + + train_dataset = self.get_dataset(loader) + + # Use provided labels if not None o.w. use MNIST dataset training labels + if train_labels is not None: + # Create sparse tensor of train_labels with (-1)s for labels not + # in train_idx. We avoid train_data[idx] because train_data may + # very large, i.e. ImageNet + sparse_labels = ( + np.zeros(self.train_size if loader == "train" else self.test_size, dtype=int) - 1 + ) + sparse_labels[train_idx] = train_labels + train_dataset.targets = sparse_labels + + train_loader = torch.utils.data.DataLoader( + dataset=train_dataset, + # sampler=SubsetRandomSampler(train_idx if train_idx is not None + # else range(self.train_size)), + sampler=SubsetRandomSampler(train_idx), + batch_size=self.batch_size, + **self.loader_kwargs, + ) + + optimizer = optim.SGD(self.model.parameters(), lr=self.lr, momentum=self.momentum) + + # Train for self.epochs epochs + for epoch in range(1, self.epochs + 1): + # Enable dropout and batch norm layers + self.model.train() + for batch_idx, (data, target) in enumerate(train_loader): + if self.cuda: # pragma: no cover + data, target = data.cuda(), target.cuda() + data, target = Variable(data), Variable(target).long() + optimizer.zero_grad() + output = self.model(data) + loss = F.nll_loss(output, target, class_weight) + loss.backward() + optimizer.step() + if self.log_interval is not None and batch_idx % self.log_interval == 0: + print( + "TrainEpoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format( + epoch, + batch_idx * len(data), + len(train_idx), + 100.0 * batch_idx / len(train_loader), + loss.item(), + ), + ) + + def predict(self, idx=None, loader=None): + """Get predicted labels from trained model.""" + # get the index of the max probability + probs = self.predict_proba(idx, loader) + return probs.argmax(axis=1) + + def predict_proba(self, idx=None, loader=None): + if self.loader is not None: + loader = self.loader + if loader is None: + is_test_idx = ( + idx is not None + and len(idx) == self.test_size + and np.all(np.array(idx) == np.arange(self.test_size)) + ) + loader = "test" if is_test_idx else "train" + dataset = self.get_dataset(loader) + # Filter by idx + if idx is not None: + if (loader == "train" and len(idx) != self.train_size) or ( + loader == "test" and len(idx) != self.test_size + ): + dataset.data = dataset.data[idx] + dataset.targets = dataset.targets[idx] + + loader = torch.utils.data.DataLoader( + dataset=dataset, + batch_size=self.batch_size if loader == "train" else self.test_batch_size, + **self.loader_kwargs, + ) + + # sets model.train(False) inactivating dropout and batch-norm layers + self.model.eval() + + # Run forward pass on model to compute outputs + outputs = [] + for data, _ in loader: + if self.cuda: # pragma: no cover + data = data.cuda() + with torch.no_grad(): + data = Variable(data) + output = self.model(data) + outputs.append(output) + + # Outputs are log_softmax (log probabilities) + outputs = torch.cat(outputs, dim=0) + # Convert to probabilities and return the numpy array of shape N x K + out = outputs.cpu().numpy() if self.cuda else outputs.numpy() + pred = np.exp(out) + return pred diff --git a/cleanlab/experimental/span_classification.py b/cleanlab/experimental/span_classification.py new file mode 100644 index 0000000..b72a7e7 --- /dev/null +++ b/cleanlab/experimental/span_classification.py @@ -0,0 +1,106 @@ +""" +Methods to find label issues in span classification datasets (text data), each token in a sentence receives one or more class labels. + +The underlying label error detection algorithms are in `cleanlab.token_classification`. +""" + +import numpy as np +from typing import List, Tuple, Optional + +from cleanlab.token_classification.filter import find_label_issues as find_label_issues_token +from cleanlab.token_classification.summary import display_issues as display_issues_token +from cleanlab.token_classification.rank import ( + get_label_quality_scores as get_label_quality_scores_token, +) + + +def find_label_issues( + labels: list, + pred_probs: list, +): + """Identifies tokens with label issues in a span classification dataset. + + Tokens identified with issues will be ranked by their individual label quality score. + + To rank the sentences based on their overall label quality, use :py:func:`experimental.span_classification.get_label_quality_scores ` + + Parameters + ---------- + labels: + Nested list of given labels for all tokens. + Refer to documentation for this argument in :py:func:`token_classification.filter.find_label_issues ` for further details. + + Note: Currently, only a single span class is supported. + + pred_probs: + An array of shape ``(T, K)`` of model-predicted class probabilities. + Refer to documentation for this argument in :py:func:`token_classification.filter.find_label_issues ` for further details. + + Returns + ------- + issues: + List of label issues identified by cleanlab, such that each element is a tuple ``(i, j)``, which + indicates that the `j`-th token of the `i`-th sentence has a label issue. + + These tuples are ordered in `issues` list based on the likelihood that the corresponding token is mislabeled. + + Use :py:func:`experimental.span_classification.get_label_quality_scores ` + to view these issues within the original sentences. + + Examples + -------- + >>> import numpy as np + >>> from cleanlab.experimental.span_classification import find_label_issues + >>> labels = [[0, 0, 1, 1], [1, 1, 0]] + >>> pred_probs = [ + ... np.array([0.9, 0.9, 0.9, 0.1]), + ... np.array([0.1, 0.1, 0.9]), + ... ] + >>> find_label_issues(labels, pred_probs) + """ + pred_probs_token = _get_pred_prob_token(pred_probs) + return find_label_issues_token(labels, pred_probs_token) + + +def display_issues( + issues: list, + tokens: List[List[str]], + *, + labels: Optional[list] = None, + pred_probs: Optional[list] = None, + exclude: List[Tuple[int, int]] = [], + class_names: Optional[List[str]] = None, + top: int = 20, +) -> None: + """ + See documentation of :py:meth:`token_classification.summary.display_issues` for description. + """ + display_issues_token( + issues, + tokens, + labels=labels, + pred_probs=pred_probs, + exclude=exclude, + class_names=class_names, + top=top, + ) + + +def get_label_quality_scores( + labels: list, + pred_probs: list, + **kwargs, +) -> Tuple[np.ndarray, list]: + """ + See documentation of :py:meth:`token_classification.rank.get_label_quality_scores` for description. + """ + pred_probs_token = _get_pred_prob_token(pred_probs) + return get_label_quality_scores_token(labels, pred_probs_token, **kwargs) + + +def _get_pred_prob_token(pred_probs: list) -> list: + """Converts pred_probs for span classification to pred_probs for token classification.""" + pred_probs_token = [] + for probs in pred_probs: + pred_probs_token.append(np.stack([1 - probs, probs], axis=1)) + return pred_probs_token diff --git a/cleanlab/filter.py b/cleanlab/filter.py new file mode 100644 index 0000000..63f4c9c --- /dev/null +++ b/cleanlab/filter.py @@ -0,0 +1,952 @@ +""" +Methods to identify which examples have label issues in a classification dataset. +The documentation below assumes a dataset with ``N`` examples and ``K`` classes. +This module is for standard (multi-class) classification where each example is labeled as belonging to exactly one of K classes (e.g. ``labels = np.array([0,0,1,0,2,1])``). +Some methods here also work for multi-label classification data where each example can be labeled as belonging to multiple classes (e.g. ``labels = [[1,2],[1],[0],[],...]``), +but we encourage using the methods in the ``cleanlab.multilabel_classification`` module instead for such data. +""" + +import numpy as np +from sklearn.metrics import confusion_matrix +import multiprocessing +import sys +import warnings +from typing import Any, Dict, Optional, Tuple, List +from functools import reduce +import platform + +from cleanlab.count import calibrate_confident_joint, num_label_issues, _reduce_issues +from cleanlab.rank import order_label_issues, get_label_quality_scores +import cleanlab.internal.multilabel_scorer as ml_scorer +from cleanlab.internal.validation import assert_valid_inputs +from cleanlab.internal.util import ( + value_counts_fill_missing_classes, + round_preserving_row_totals, + get_num_classes, +) +from cleanlab.internal.multilabel_utils import stack_complement, get_onehot_num_classes, int2onehot +from cleanlab.typing import LabelLike +from cleanlab.multilabel_classification.filter import find_multilabel_issues_per_class + +# tqdm is a package to print time-to-complete when multiprocessing is used. +# This package is not necessary, but when installed improves user experience for large datasets. +try: + import tqdm.auto as tqdm + + tqdm_exists = True +except ImportError as e: # pragma: no cover + tqdm_exists = False + + w = """To see estimated completion times for methods in cleanlab.filter, "pip install tqdm".""" + warnings.warn(w) + +# psutil is a package used to count physical cores for multiprocessing +# This package is not necessary, because we can always fall back to logical cores as the default +try: + import psutil + + psutil_exists = True +except ImportError as e: # pragma: no cover + psutil_exists = False + +# global variable for find_label_issues multiprocessing +pred_probs_by_class: Dict[int, np.ndarray] +prune_count_matrix_cols: Dict[int, np.ndarray] + + +def find_label_issues( + labels: LabelLike, + pred_probs: np.ndarray, + *, + return_indices_ranked_by: Optional[str] = None, + rank_by_kwargs: Optional[Dict[str, Any]] = None, + filter_by: str = "prune_by_noise_rate", + frac_noise: float = 1.0, + num_to_remove_per_class: Optional[List[int]] = None, + min_examples_per_class=1, + confident_joint: Optional[np.ndarray] = None, + n_jobs: Optional[int] = None, + verbose: bool = False, + multi_label: bool = False, +) -> np.ndarray: + """ + Identifies potentially bad labels in a classification dataset using confident learning. + + Returns a boolean mask for the entire dataset where ``True`` represents + an example identified with a label issue and ``False`` represents an example that seems correctly labeled. + + Instead of a mask, you can obtain indices of the examples with label issues in your dataset + (sorted by issue severity) by specifying the `return_indices_ranked_by` argument. + This determines which label quality score is used to quantify severity, + and is useful to view only the top-`J` most severe issues in your dataset. + + The number of indices returned as issues is controlled by `frac_noise`: reduce its + value to identify fewer label issues. If you aren't sure, leave this set to 1.0. + + Tip: if you encounter the error "pred_probs is not defined", try setting + ``n_jobs=1``. + + Parameters + ---------- + labels : np.ndarray or list + A discrete vector of noisy labels for a classification dataset, i.e. some labels may be erroneous. + *Format requirements*: for dataset with K classes, each label must be integer in 0, 1, ..., K-1. + For a standard (multi-class) classification dataset where each example is labeled with one class, + `labels` should be 1D array of shape ``(N,)``, for example: ``labels = [1,0,2,1,1,0...]``. + + pred_probs : np.ndarray, optional + An array of shape ``(N, K)`` of model-predicted class probabilities, + ``P(label=k|x)``. Each row of this matrix corresponds + to an example `x` and contains the model-predicted probabilities that + `x` belongs to each possible class, for each of the K classes. The + columns must be ordered such that these probabilities correspond to + class 0, 1, ..., K-1. + + **Note**: Returned label issues are most accurate when they are computed based on out-of-sample `pred_probs` from your model. + To obtain out-of-sample predicted probabilities for every datapoint in your dataset, you can use :ref:`cross-validation `. + This is encouraged to get better results. + + return_indices_ranked_by : {None, 'self_confidence', 'normalized_margin', 'confidence_weighted_entropy'}, default=None + Determines what is returned by this method: either a boolean mask or list of indices np.ndarray. + If ``None``, this function returns a boolean mask (``True`` if example at index is label error). + If not ``None``, this function returns a sorted array of indices of examples with label issues + (instead of a boolean mask). Indices are sorted by label quality score which can be one of: + + - ``'normalized_margin'``: ``normalized margin (p(label = k) - max(p(label != k)))`` + - ``'self_confidence'``: ``[pred_probs[i][labels[i]] for i in label_issues_idx]`` + - ``'confidence_weighted_entropy'``: ``entropy(pred_probs) / self_confidence`` + + rank_by_kwargs : dict, optional + Optional keyword arguments to pass into scoring functions for ranking by + label quality score (see :py:func:`rank.get_label_quality_scores + `). + + filter_by : {'prune_by_class', 'prune_by_noise_rate', 'both', 'confident_learning', 'predicted_neq_given', 'low_normalized_margin', 'low_self_confidence'}, default='prune_by_noise_rate' + Method to determine which examples are flagged as having label issue, so you can filter/prune them from the dataset. Options: + + - ``'prune_by_noise_rate'``: filters examples with *high probability* of being mislabeled for every non-diagonal in the confident joint (see `prune_counts_matrix` in `filter.py`). These are the examples where (with high confidence) the given label is unlikely to match the predicted label for the example. + - ``'prune_by_class'``: filters the examples with *smallest probability* of belonging to their given class label for every class. + - ``'both'``: filters only those examples that would be filtered by both ``'prune_by_noise_rate'`` and ``'prune_by_class'``. + - ``'confident_learning'``: filters the examples counted as part of the off-diagonals of the confident joint. These are the examples that are confidently predicted to be a different label than their given label. + - ``'predicted_neq_given'``: filters examples for which the predicted class (i.e. argmax of the predicted probabilities) does not match the given label. + - ``'low_normalized_margin'``: filters the examples with *smallest* normalized margin label quality score. The number of issues returned matches :py:func:`count.num_label_issues `. + - ``'low_self_confidence'``: filters the examples with *smallest* self confidence label quality score. The number of issues returned matches :py:func:`count.num_label_issues `. + + frac_noise : float, default=1.0 + Used to only return the "top" ``frac_noise * num_label_issues``. The choice of which "top" + label issues to return is dependent on the `filter_by` method used. It works by reducing the + size of the off-diagonals of the `joint` distribution of given labels and true labels + proportionally by `frac_noise` prior to estimating label issues with each method. + This parameter only applies for `filter_by=both`, `filter_by=prune_by_class`, and + `filter_by=prune_by_noise_rate` methods and currently is unused by other methods. + When ``frac_noise=1.0``, return all "confident" estimated noise indices (recommended). + + frac_noise * number_of_mislabeled_examples_in_class_k. + + num_to_remove_per_class : array_like + An iterable of length K, the number of classes. + E.g. if K = 3, ``num_to_remove_per_class=[5, 0, 1]`` would return + the indices of the 5 most likely mislabeled examples in class 0, + and the most likely mislabeled example in class 2. + + Note + ---- + Only set this parameter if ``filter_by='prune_by_class'``. + You may use with ``filter_by='prune_by_noise_rate'``, but + if ``num_to_remove_per_class=k``, then either k-1, k, or k+1 + examples may be removed for any class due to rounding error. If you need + exactly 'k' examples removed from every class, you should use + ``filter_by='prune_by_class'``. + + min_examples_per_class : int, default=1 + Minimum number of examples per class to avoid flagging as label issues. + This is useful to avoid deleting too much data from one class + when pruning noisy examples in datasets with rare classes. + + confident_joint : np.ndarray, optional + An array of shape ``(K, K)`` representing the confident joint, the matrix used for identifying label issues, which + estimates a confident subset of the joint distribution of the noisy and true labels, ``P_{noisy label, true label}``. + Entry ``(j, k)`` in the matrix is the number of examples confidently counted into the pair of ``(noisy label=j, true label=k)`` classes. + The `confident_joint` can be computed using :py:func:`count.compute_confident_joint `. + If not provided, it is computed from the given (noisy) `labels` and `pred_probs`. + + n_jobs : optional + Number of processing threads used by multiprocessing. Default ``None`` + sets to the number of cores on your CPU (physical cores if you have ``psutil`` package installed, otherwise logical cores). + Set this to 1 to *disable* parallel processing (if its causing issues). + Windows users may see a speed-up with ``n_jobs=1``. + + verbose : optional + If ``True``, prints when multiprocessing happens. + + Returns + ------- + label_issues : np.ndarray + If `return_indices_ranked_by` left unspecified, returns a boolean **mask** for the entire dataset + where ``True`` represents a label issue and ``False`` represents an example that is + accurately labeled with high confidence. + If `return_indices_ranked_by` is specified, returns a shorter array of **indices** of examples identified to have + label issues (i.e. those indices where the mask would be ``True``), sorted by likelihood that the corresponding label is correct. + + Note + ---- + Obtain the *indices* of examples with label issues in your dataset by setting `return_indices_ranked_by`. + """ + if not rank_by_kwargs: + rank_by_kwargs = {} + + assert filter_by in [ + "low_normalized_margin", + "low_self_confidence", + "prune_by_noise_rate", + "prune_by_class", + "both", + "confident_learning", + "predicted_neq_given", + ] # TODO: change default to confident_learning ? + allow_one_class = False + if isinstance(labels, np.ndarray) or all(isinstance(lab, int) for lab in labels): + if set(labels) == {0}: # occurs with missing classes in multi-label settings + allow_one_class = True + assert_valid_inputs( + X=None, + y=labels, + pred_probs=pred_probs, + multi_label=multi_label, + allow_one_class=allow_one_class, + ) + + if filter_by in [ + "confident_learning", + "predicted_neq_given", + "low_normalized_margin", + "low_self_confidence", + ] and (frac_noise != 1.0 or num_to_remove_per_class is not None): + warn_str = ( + "frac_noise and num_to_remove_per_class parameters are only supported" + " for filter_by 'prune_by_noise_rate', 'prune_by_class', and 'both'. They " + "are not supported for methods 'confident_learning', 'predicted_neq_given', " + "'low_normalized_margin' or 'low_self_confidence'." + ) + warnings.warn(warn_str) + if (num_to_remove_per_class is not None) and ( + filter_by + in [ + "confident_learning", + "predicted_neq_given", + "low_normalized_margin", + "low_self_confidence", + ] + ): + # TODO - add support for these filters + raise ValueError( + "filter_by 'confident_learning', 'predicted_neq_given', 'low_normalized_margin' " + "or 'low_self_confidence' is not supported (yet) when setting 'num_to_remove_per_class'" + ) + if filter_by == "confident_learning" and isinstance(confident_joint, np.ndarray): + warn_str = ( + "The supplied `confident_joint` is ignored when `filter_by = 'confident_learning'`; confident joint will be " + "re-estimated from the given labels. To use your supplied `confident_joint`, please specify a different " + "`filter_by` value." + ) + warnings.warn(warn_str) + + K = get_num_classes( + labels=labels, pred_probs=pred_probs, label_matrix=confident_joint, multi_label=multi_label + ) + # Boolean set to true if dataset is large + big_dataset = K * len(labels) > 1e8 + + # Set-up number of multiprocessing threads + # On Windows/macOS, when multi_label is True, multiprocessing is much slower + # even for faily large input arrays, so we default to n_jobs=1 in this case + os_name = platform.system() + if n_jobs is None: + if multi_label and os_name != "Linux": + n_jobs = 1 + else: + if psutil_exists: + n_jobs = psutil.cpu_count(logical=False) # physical cores + elif big_dataset: + print( + "To default `n_jobs` to the number of physical cores for multiprocessing in find_label_issues(), please: `pip install psutil`.\n" + "Note: You can safely ignore this message. `n_jobs` only affects runtimes, results will be the same no matter its value.\n" + "Since psutil is not installed, `n_jobs` was set to the number of logical cores by default.\n" + "Disable this message by either installing psutil or specifying the `n_jobs` argument." + ) # pragma: no cover + if not n_jobs: + # either psutil does not exist + # or psutil can return None when physical cores cannot be determined + # switch to logical cores + n_jobs = multiprocessing.cpu_count() + else: + assert n_jobs >= 1 + + if multi_label: + if not isinstance(labels, list): + raise TypeError("`labels` must be list when `multi_label=True`.") + warnings.warn( + "The multi_label argument to filter.find_label_issues() is deprecated and will be removed in future versions. Please use `multilabel_classification.filter.find_label_issues()` instead.", + DeprecationWarning, + ) + return _find_label_issues_multilabel( + labels, + pred_probs, + return_indices_ranked_by, + rank_by_kwargs, + filter_by, + frac_noise, + num_to_remove_per_class, + min_examples_per_class, + confident_joint, + n_jobs, + verbose, + ) + + # Else this is standard multi-class classification + # Number of examples in each class of labels + label_counts = value_counts_fill_missing_classes(labels, K, multi_label=multi_label) + # Ensure labels are of type np.ndarray() + labels = np.asarray(labels) + if confident_joint is None or filter_by == "confident_learning": + from cleanlab.count import compute_confident_joint + + confident_joint, cl_error_indices = compute_confident_joint( + labels=labels, + pred_probs=pred_probs, + multi_label=multi_label, + return_indices_of_off_diagonals=True, + ) + + if filter_by in ["low_normalized_margin", "low_self_confidence"]: + # TODO: consider setting adjust_pred_probs to true based on benchmarks (or adding it kwargs, or ignoring and leaving as false by default) + scores = get_label_quality_scores( + labels, + pred_probs, + method=filter_by[4:], + adjust_pred_probs=False, + ) + num_errors = num_label_issues( + labels, pred_probs, multi_label=multi_label # TODO: Check usage of multilabel + ) + # Find label issues O(nlogn) solution (mapped to boolean mask later in the method) + cl_error_indices = np.argsort(scores)[:num_errors] + # The following is the O(n) fastest solution (check for one-off errors), but the problem is if lots of the scores are identical you will overcount, + # you can end up returning more or less and they aren't ranked in the boolean form so there's no way to drop the highest scores randomly + # boundary = np.partition(scores, num_errors)[num_errors] # O(n) solution + # label_issues_mask = scores <= boundary + + if filter_by in ["prune_by_noise_rate", "prune_by_class", "both"]: + # Create `prune_count_matrix` with the number of examples to remove in each class and + # leave at least min_examples_per_class examples per class. + # `prune_count_matrix` is transposed relative to the confident_joint. + prune_count_matrix = _keep_at_least_n_per_class( + prune_count_matrix=confident_joint.T, + n=min_examples_per_class, + frac_noise=frac_noise, + ) + + if num_to_remove_per_class is not None: + # Estimate joint probability distribution over label issues + psy = prune_count_matrix / np.sum(prune_count_matrix, axis=1) + noise_per_s = psy.sum(axis=1) - psy.diagonal() + # Calibrate labels.t. noise rates sum to num_to_remove_per_class + tmp = (psy.T * num_to_remove_per_class / noise_per_s).T + np.fill_diagonal(tmp, label_counts - num_to_remove_per_class) + prune_count_matrix = round_preserving_row_totals(tmp) + + # Prepare multiprocessing shared data + # On Linux with Python <3.14, multiprocessing is started with fork, + # so data can be shared with global variables + COW + # On Window/macOS, processes are started with spawn, + # so data will need to be pickled to the subprocesses through input args + # In Python 3.14+, global variable sharing is no longer reliable even on Linux + chunksize = max(1, K // n_jobs) + use_global_vars = n_jobs == 1 or (os_name == "Linux" and sys.version_info < (3, 14)) + if use_global_vars: + global pred_probs_by_class, prune_count_matrix_cols + pred_probs_by_class = {k: pred_probs[labels == k] for k in range(K)} + prune_count_matrix_cols = {k: prune_count_matrix[:, k] for k in range(K)} + args = [[k, min_examples_per_class, None] for k in range(K)] + else: + args = [ + [k, min_examples_per_class, [pred_probs[labels == k], prune_count_matrix[:, k]]] + for k in range(K) + ] + + # Perform Pruning with threshold probabilities from BFPRT algorithm in O(n) + # Operations are parallelized across all CPU processes + if filter_by == "prune_by_class" or filter_by == "both": + if n_jobs > 1: + with multiprocessing.Pool(n_jobs) as p: + if verbose: # pragma: no cover + print("Parallel processing label issues by class.") + sys.stdout.flush() + if big_dataset and tqdm_exists: + label_issues_masks_per_class = list( + tqdm.tqdm(p.imap(_prune_by_class, args, chunksize=chunksize), total=K) + ) + else: + label_issues_masks_per_class = p.map(_prune_by_class, args, chunksize=chunksize) + else: + label_issues_masks_per_class = [_prune_by_class(arg) for arg in args] + + label_issues_mask = np.zeros(len(labels), dtype=bool) + for k, mask in enumerate(label_issues_masks_per_class): + if len(mask) > 1: + label_issues_mask[labels == k] = mask + + if filter_by == "both": + label_issues_mask_by_class = label_issues_mask + + if filter_by == "prune_by_noise_rate" or filter_by == "both": + if n_jobs > 1: + with multiprocessing.Pool(n_jobs) as p: + if verbose: # pragma: no cover + print("Parallel processing label issues by noise rate.") + sys.stdout.flush() + if big_dataset and tqdm_exists: + label_issues_masks_per_class = list( + tqdm.tqdm(p.imap(_prune_by_count, args, chunksize=chunksize), total=K) + ) + else: + label_issues_masks_per_class = p.map(_prune_by_count, args, chunksize=chunksize) + else: + label_issues_masks_per_class = [_prune_by_count(arg) for arg in args] + + label_issues_mask = np.zeros(len(labels), dtype=bool) + for k, mask in enumerate(label_issues_masks_per_class): + if len(mask) > 1: + label_issues_mask[labels == k] = mask + + if filter_by == "both": + label_issues_mask = label_issues_mask & label_issues_mask_by_class + + if filter_by in ["confident_learning", "low_normalized_margin", "low_self_confidence"]: + label_issues_mask = np.zeros(len(labels), dtype=bool) + label_issues_mask[cl_error_indices] = True + + if filter_by == "predicted_neq_given": + label_issues_mask = find_predicted_neq_given(labels, pred_probs, multi_label=multi_label) + + if filter_by not in ["low_self_confidence", "low_normalized_margin"]: + # Remove label issues if model prediction is close to given label + mask = _reduce_issues(pred_probs=pred_probs, labels=labels) + label_issues_mask[mask] = False + + if verbose: + print("Number of label issues found: {}".format(sum(label_issues_mask))) + + # TODO: run count.num_label_issues() and adjust the total issues found here to match + if return_indices_ranked_by is not None: + er = order_label_issues( + label_issues_mask=label_issues_mask, + labels=labels, + pred_probs=pred_probs, + rank_by=return_indices_ranked_by, + rank_by_kwargs=rank_by_kwargs, + ) + return er + return label_issues_mask + + +def _find_label_issues_multilabel( + labels: list, + pred_probs: np.ndarray, + return_indices_ranked_by: Optional[str] = None, + rank_by_kwargs={}, + filter_by: str = "prune_by_noise_rate", + frac_noise: float = 1.0, + num_to_remove_per_class: Optional[List[int]] = None, + min_examples_per_class=1, + confident_joint: Optional[np.ndarray] = None, + n_jobs: Optional[int] = None, + verbose: bool = False, + low_memory: bool = False, +) -> np.ndarray: + """ + Finds label issues in multi-label classification data where each example can belong to more than one class. + This is done via a one-vs-rest reduction for each class and the results are subsequently aggregated across all classes. + Here `labels` must be formatted as an iterable of iterables, e.g. ``List[List[int]]``. + """ + if filter_by in ["low_normalized_margin", "low_self_confidence"] and not low_memory: + num_errors = sum( + find_label_issues( + labels=labels, + pred_probs=pred_probs, + confident_joint=confident_joint, + multi_label=True, + filter_by="confident_learning", + ) + ) + + y_one, num_classes = get_onehot_num_classes(labels, pred_probs) + label_quality_scores = ml_scorer.get_label_quality_scores( + labels=y_one, + pred_probs=pred_probs, + ) + + cl_error_indices = np.argsort(label_quality_scores)[:num_errors] + label_issues_mask = np.zeros(len(labels), dtype=bool) + label_issues_mask[cl_error_indices] = True + + if return_indices_ranked_by is not None: + label_quality_scores_issues = ml_scorer.get_label_quality_scores( + labels=y_one[label_issues_mask], + pred_probs=pred_probs[label_issues_mask], + method=ml_scorer.MultilabelScorer( + base_scorer=ml_scorer.ClassLabelScorer.from_str(return_indices_ranked_by), + ), + base_scorer_kwargs=rank_by_kwargs, + ) + return cl_error_indices[np.argsort(label_quality_scores_issues)] + + return label_issues_mask + + per_class_issues = find_multilabel_issues_per_class( + labels, + pred_probs, + return_indices_ranked_by, + rank_by_kwargs, + filter_by, + frac_noise, + num_to_remove_per_class, + min_examples_per_class, + confident_joint, + n_jobs, + verbose, + low_memory, + ) + if return_indices_ranked_by is None: + assert isinstance(per_class_issues, np.ndarray) + return per_class_issues.sum(axis=1) >= 1 + else: + label_issues_list, labels_list, pred_probs_list = per_class_issues + label_issues_idx = reduce(np.union1d, label_issues_list) + y_one, num_classes = get_onehot_num_classes(labels, pred_probs) + label_quality_scores = ml_scorer.get_label_quality_scores( + labels=y_one, + pred_probs=pred_probs, + method=ml_scorer.MultilabelScorer( + base_scorer=ml_scorer.ClassLabelScorer.from_str(return_indices_ranked_by), + ), + base_scorer_kwargs=rank_by_kwargs, + ) + label_quality_scores_issues = label_quality_scores[label_issues_idx] + return label_issues_idx[np.argsort(label_quality_scores_issues)] + + +def _keep_at_least_n_per_class( + prune_count_matrix: np.ndarray, n: int, *, frac_noise: float = 1.0 +) -> np.ndarray: + """Make sure every class has at least n examples after removing noise. + Functionally, increase each column, increases the diagonal term #(true_label=k,label=k) + of prune_count_matrix until it is at least n, distributing the amount + increased by subtracting uniformly from the rest of the terms in the + column. When frac_noise = 1.0, return all "confidently" estimated + noise indices, otherwise this returns frac_noise fraction of all + the noise counts, with diagonal terms adjusted to ensure column + totals are preserved. + + Parameters + ---------- + prune_count_matrix : np.ndarray of shape (K, K), K = number of classes + A counts of mislabeled examples in every class. For this function. + NOTE prune_count_matrix is transposed relative to confident_joint. + + n : int + Number of examples to make sure are left in each class. + + frac_noise : float, default=1.0 + Used to only return the "top" ``frac_noise * num_label_issues``. The choice of which "top" + label issues to return is dependent on the `filter_by` method used. It works by reducing the + size of the off-diagonals of the `prune_count_matrix` of given labels and true labels + proportionally by `frac_noise` prior to estimating label issues with each method. + When frac_noise=1.0, return all "confident" estimated noise indices (recommended). + + Returns + ------- + prune_count_matrix : np.ndarray of shape (K, K), K = number of classes + This the same as the confident_joint, but has been transposed and the counts are adjusted. + """ + + prune_count_matrix_diagonal = np.diagonal(prune_count_matrix) + + # Set diagonal terms less than n, to n. + new_diagonal = np.maximum(prune_count_matrix_diagonal, n) + + # Find how much diagonal terms were increased. + diff_per_col = new_diagonal - prune_count_matrix_diagonal + + # Count non-zero, non-diagonal items per column + # np.maximum(*, 1) makes this never 0 (we divide by this next) + num_noise_rates_per_col = np.maximum( + np.count_nonzero(prune_count_matrix, axis=0) - 1.0, + 1.0, + ) + + # Uniformly decrease non-zero noise rates by the same amount + # that the diagonal items were increased + new_mat = prune_count_matrix - diff_per_col / num_noise_rates_per_col + + # Originally zero noise rates will now be negative, fix them back to zero + new_mat[new_mat < 0] = 0 + + # Round diagonal terms (correctly labeled examples) + np.fill_diagonal(new_mat, new_diagonal) + + # Reduce (multiply) all noise rates (non-diagonal) by frac_noise and + # increase diagonal by the total amount reduced in each column + # to preserve column counts. + new_mat = _reduce_prune_counts(new_mat, frac_noise) + + # These are counts, so return a matrix of ints. + return round_preserving_row_totals(new_mat).astype(int) + + +def _reduce_prune_counts(prune_count_matrix: np.ndarray, frac_noise: float = 1.0) -> np.ndarray: + """Reduce (multiply) all prune counts (non-diagonal) by frac_noise and + increase diagonal by the total amount reduced in each column to + preserve column counts. + + Parameters + ---------- + prune_count_matrix : np.ndarray of shape (K, K), K = number of classes + A counts of mislabeled examples in every class. For this function, it + does not matter what the rows or columns are, but the diagonal terms + reflect the number of correctly labeled examples. + + frac_noise : float + Used to only return the "top" ``frac_noise * num_label_issues``. The choice of which "top" + label issues to return is dependent on the `filter_by` method used. It works by reducing the + size of the off-diagonals of the `prune_count_matrix` of given labels and true labels + proportionally by `frac_noise` prior to estimating label issues with each method. + When frac_noise=1.0, return all "confident" estimated noise indices (recommended). + """ + + new_mat = prune_count_matrix * frac_noise + np.fill_diagonal(new_mat, prune_count_matrix.diagonal()) + np.fill_diagonal( + new_mat, + prune_count_matrix.diagonal() + np.sum(prune_count_matrix - new_mat, axis=0), + ) + + # These are counts, so return a matrix of ints. + return new_mat.astype(int) + + +def find_predicted_neq_given( + labels: LabelLike, pred_probs: np.ndarray, *, multi_label: bool = False +) -> np.ndarray: + """A simple baseline approach that considers ``argmax(pred_probs) != labels`` as the examples with label issues. + + Parameters + ---------- + labels : np.ndarray or list + Labels in the same format expected by the `~cleanlab.filter.find_label_issues` function. + + pred_probs : np.ndarray + Predicted-probabilities in the same format expected by the `~cleanlab.filter.find_label_issues` function. + + multi_label : bool, optional + Whether each example may have multiple labels or not (see documentation for the `~cleanlab.filter.find_label_issues` function). + + Returns + ------- + label_issues_mask : np.ndarray + A boolean mask for the entire dataset where ``True`` represents a + label issue and ``False`` represents an example that is accurately + labeled with high confidence. + """ + + assert_valid_inputs(X=None, y=labels, pred_probs=pred_probs, multi_label=multi_label) + if multi_label: + if not isinstance(labels, list): + raise TypeError("`labels` must be list when `multi_label=True`.") + else: + return _find_predicted_neq_given_multilabel(labels=labels, pred_probs=pred_probs) + else: + return np.argmax(pred_probs, axis=1) != np.asarray(labels) + + +def _find_predicted_neq_given_multilabel(labels: list, pred_probs: np.ndarray) -> np.ndarray: + """ + + Parameters + ---------- + labels : list + List of noisy labels for multi-label classification where each example can belong to multiple classes + (e.g. ``labels = [[1,2],[1],[0],[],...]`` indicates the first example in dataset belongs to both class 1 and class 2). + + pred_probs : np.ndarray + Predicted-probabilities in the same format expected by the `~cleanlab.filter.find_label_issues` function. + + Returns + ------- + label_issues_mask : np.ndarray + A boolean mask for the entire dataset where ``True`` represents a + label issue and ``False`` represents an example that is accurately + labeled with high confidence. + + """ + y_one, num_classes = get_onehot_num_classes(labels, pred_probs) + pred_neq: np.ndarray = np.zeros(y_one.shape).astype(bool) + for class_num, (label, pred_prob_for_class) in enumerate(zip(y_one.T, pred_probs.T)): + pred_probs_binary = stack_complement(pred_prob_for_class) + pred_neq[:, class_num] = find_predicted_neq_given( + labels=label, pred_probs=pred_probs_binary + ) + return pred_neq.sum(axis=1) >= 1 + + +def find_label_issues_using_argmax_confusion_matrix( + labels: np.ndarray, + pred_probs: np.ndarray, + *, + calibrate: bool = True, + filter_by: str = "prune_by_noise_rate", +) -> np.ndarray: + """A baseline approach that uses the confusion matrix + of ``argmax(pred_probs)`` and labels as the confident joint and then uses cleanlab + (confident learning) to find the label issues using this matrix. + + The only difference between this and `~cleanlab.filter.find_label_issues` is that it uses the confusion matrix + based on the argmax and given label instead of using the confident joint + from :py:func:`count.compute_confident_joint + `. + + Parameters + ---------- + labels : np.ndarray + An array of shape ``(N,)`` of noisy labels, i.e. some labels may be erroneous. + Elements must be in the set 0, 1, ..., K-1, where K is the number of classes. + + pred_probs : np.ndarray + An array of shape ``(N, K)`` of model-predicted probabilities, + ``P(label=k|x)``. Each row of this matrix corresponds + to an example `x` and contains the model-predicted probabilities that + `x` belongs to each possible class, for each of the K classes. The + columns must be ordered such that these probabilities correspond to + class 0, 1, ..., K-1. `pred_probs` should have been computed using 3 (or + higher) fold cross-validation. + + calibrate : bool, default=True + Set to ``True`` to calibrate the confusion matrix created by ``pred != given labels``. + This calibration adjusts the confusion matrix / confident joint so that the + prior (given noisy labels) is correct based on the original labels. + + filter_by : str, default='prune_by_noise_rate' + See `filter_by` argument of `~cleanlab.filter.find_label_issues`. + + Returns + ------- + label_issues_mask : np.ndarray + A boolean mask for the entire dataset where ``True`` represents a + label issue and ``False`` represents an example that is accurately + labeled with high confidence. + + """ + + assert_valid_inputs(X=None, y=labels, pred_probs=pred_probs, multi_label=False) + confident_joint = confusion_matrix(np.argmax(pred_probs, axis=1), labels).T + if calibrate: + confident_joint = calibrate_confident_joint(confident_joint, labels) + return find_label_issues( + labels=labels, + pred_probs=pred_probs, + confident_joint=confident_joint, + filter_by=filter_by, + ) + + +# Multiprocessing helper functions: + +mp_params: Dict[str, Any] = {} # Globals to be shared across threads in multiprocessing + + +def _to_np_array( + mp_arr: bytearray, dtype="int32", shape: Optional[Tuple[int, int]] = None +) -> np.ndarray: # pragma: no cover + """multipropecessing Helper function to convert a multiprocessing + RawArray to a numpy array.""" + arr = np.frombuffer(mp_arr, dtype=dtype) + if shape is None: + return arr + return arr.reshape(shape) + + +def _init( + __labels, + __label_counts, + __prune_count_matrix, + __pcm_shape, + __pred_probs, + __pred_probs_shape, + __multi_label, + __min_examples_per_class, +): # pragma: no cover + """Shares memory objects across child processes. + ASSUMES none of these will be changed by child processes!""" + + mp_params["labels"] = __labels + mp_params["label_counts"] = __label_counts + mp_params["prune_count_matrix"] = __prune_count_matrix + mp_params["pcm_shape"] = __pcm_shape + mp_params["pred_probs"] = __pred_probs + mp_params["pred_probs_shape"] = __pred_probs_shape + mp_params["multi_label"] = __multi_label + mp_params["min_examples_per_class"] = __min_examples_per_class + + +def _get_shared_data() -> Any: # pragma: no cover + """multiprocessing helper function to extract numpy arrays from + shared RawArray types used to shared data across process.""" + + label_counts = _to_np_array(mp_params["label_counts"]) + prune_count_matrix = _to_np_array( + mp_arr=mp_params["prune_count_matrix"], + shape=mp_params["pcm_shape"], + ) + pred_probs = _to_np_array( + mp_arr=mp_params["pred_probs"], + dtype="float32", + shape=mp_params["pred_probs_shape"], + ) + min_examples_per_class = mp_params["min_examples_per_class"] + multi_label = mp_params["multi_label"] + labels = _to_np_array(mp_params["labels"]) # type: ignore + return ( + labels, + label_counts, + prune_count_matrix, + pred_probs, + multi_label, + min_examples_per_class, + ) + + +# TODO figure out what the types inside args are. +def _prune_by_class(args: list) -> np.ndarray: + """multiprocessing Helper function for find_label_issues() + that assumes globals and produces a mask for class k for each example by + removing the examples with *smallest probability* of + belonging to their given class label. + + Parameters + ---------- + k : int (between 0 and num classes - 1) + The class of interest.""" + + k, min_examples_per_class, arrays = args + if arrays is None: + pred_probs = pred_probs_by_class[k] + prune_count_matrix = prune_count_matrix_cols[k] + else: + pred_probs = arrays[0] + prune_count_matrix = arrays[1] + + label_counts = pred_probs.shape[0] + label_issues = np.zeros(label_counts, dtype=bool) + if label_counts > min_examples_per_class: # No prune if not at least min_examples_per_class + num_issues = label_counts - prune_count_matrix[k] + # Get return_indices_ranked_by of the smallest prob of class k for examples with noisy label k + # rank = np.partition(class_probs, num_issues)[num_issues] + if num_issues >= 1: + class_probs = pred_probs[:, k] + order = np.argsort(class_probs) + label_issues[order[:num_issues]] = True + return label_issues + + warnings.warn( + f"May not flag all label issues in class: {k}, it has too few examples (see argument: `min_examples_per_class`)" + ) + return label_issues + + +# TODO figure out what the types inside args are. +def _prune_by_count(args: list) -> np.ndarray: + """multiprocessing Helper function for find_label_issues() that assumes + globals and produces a mask for class k for each example by + removing the example with noisy label k having *largest margin*, + where + margin of example := prob of given label - max prob of non-given labels + + Parameters + ---------- + k : int (between 0 and num classes - 1) + The true_label class of interest.""" + + k, min_examples_per_class, arrays = args + if arrays is None: + pred_probs = pred_probs_by_class[k] + prune_count_matrix = prune_count_matrix_cols[k] + else: + pred_probs = arrays[0] + prune_count_matrix = arrays[1] + + label_counts = pred_probs.shape[0] + label_issues_mask = np.zeros(label_counts, dtype=bool) + if label_counts <= min_examples_per_class: + warnings.warn( + f"May not flag all label issues in class: {k}, it has too few examples (see `min_examples_per_class` argument)" + ) + return label_issues_mask + + K = pred_probs.shape[1] + if K < 1: + raise ValueError("Must have at least 1 class.") + for j in range(K): + num2prune = prune_count_matrix[j] + # Only prune for noise rates, not diagonal entries + if k != j and num2prune > 0: + # num2prune's largest p(true class k) - p(noisy class k) + # for x with true label j + margin = pred_probs[:, j] - pred_probs[:, k] + order = np.argsort(-margin) + label_issues_mask[order[:num2prune]] = True + return label_issues_mask + + +# TODO: decide if we want to keep this based on TODO above. If so move to utils. Add unit test for this. +def _multiclass_crossval_predict( + labels: list, pred_probs: np.ndarray +) -> np.ndarray: # pragma: no cover + """Returns a numpy 2D array of one-hot encoded + multiclass predictions. Each row in the array + provides the predictions for a particular example. + The boundary condition used to threshold predictions + is computed by maximizing the F1 ROC curve. + + Parameters + ---------- + labels : list of lists (length N) + These are multiclass labels. Each list in the list contains all the + labels for that example. + + pred_probs : np.ndarray (shape (N, K)) + P(label=k|x) is a matrix with K model-predicted probabilities. + Each row of this matrix corresponds to an example `x` and contains the model-predicted + probabilities that `x` belongs to each possible class. + The columns must be ordered such that these probabilities correspond to class 0,1,2,... + `pred_probs` should have been computed using 3 (or higher) fold cross-validation.""" + + from sklearn.metrics import f1_score + + boundaries = np.arange(0.05, 0.9, 0.05) + K = get_num_classes( + labels=labels, + pred_probs=pred_probs, + multi_label=True, + ) + labels_one_hot = int2onehot(labels, K) + f1s = [ + f1_score( + labels_one_hot, + (pred_probs > boundary).astype(np.uint8), + average="micro", + ) + for boundary in boundaries + ] + boundary = boundaries[np.argmax(f1s)] + pred = (pred_probs > boundary).astype(np.uint8) + return pred diff --git a/cleanlab/internal/__init__.py b/cleanlab/internal/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cleanlab/internal/constants.py b/cleanlab/internal/constants.py new file mode 100644 index 0000000..ee4c4f4 --- /dev/null +++ b/cleanlab/internal/constants.py @@ -0,0 +1,38 @@ +FLOATING_POINT_COMPARISON = 1e-6 # floating point comparison for fuzzy equals +CLIPPING_LOWER_BOUND = 1e-6 # lower-bound clipping threshold for expected behavior +CONFIDENT_THRESHOLDS_LOWER_BOUND = ( + 2 * FLOATING_POINT_COMPARISON +) # lower bound imposed to clip confident thresholds from below, has to be larger than floating point comparison +TINY_VALUE = 1e-100 # very tiny value for clipping + + +# Object Detection Constants +EUC_FACTOR = 0.1 # Factor to control magnitude of euclidian distance. Increasing the factor makes the distances between two objects go to zero more rapidly. +MAX_ALLOWED_BOX_PRUNE = 0.97 # This is max allowed percent of boxes that are pruned before a warning is thrown given a specific threshold. Pruning too many boxes negatively affects performance. +IOU_THRESHOLD = ( + 0.5 # Threshold for considering the predicted box and annotated box to be overlapping +) +EPSILON = 1e-6 # Small value to prevent division by zero + +ALPHA = 0.9 # Param for objectlab, weight between IoU and distance when considering similarity matrix. High alpha means considering IoU more strongly over distance +LOW_PROBABILITY_THRESHOLD = 0.5 # Param for get_label_quality_score, lowest predicted class probability threshold allowed when considering predicted boxes to identify badly located label boxes. +HIGH_PROBABILITY_THRESHOLD = 0.95 # Param for objectlab, high probability threshold for considering predicted boxes to identify overlooked and swapped label boxes +TEMPERATURE = 0.1 # Param for objectlab, temperature of the softmin function used to pool the per-box quality scores for an error subtype across all boxes into a single subtype score for the image. With a lower temperature, softmin pooling acts more like minimum pooling, alternatively it acts more like mean pooling with high temperature. +LABEL_OVERLAP_THRESHOLD = 0.95 # Param for objectlab, minimum IoU threshold for deciding when two boxes overlap used for deciding which objects have multiple conflicting annotations. + +OVERLOOKED_THRESHOLD_FACTOR = 0.8 # Param for find_label_issues. Per-box label quality score threshold scale factor to determine max score for a box to be considered an overlooked issue +BADLOC_THRESHOLD_FACTOR = 0.8 # Param for find_label_issues. Per-box label quality score threshold scale factor to determine max score for a box to be considered a bad location issue +SWAP_THRESHOLD_FACTOR = 0.8 # Param for find_label_issues. Per-box label quality score threshold scale factor to determine max score for a box to be considered a swap issue +AP_SCALE_FACTOR = 0.25 # Param for find_label_issues. Scale factor for per-class precision to determine is_issue. + +CUSTOM_SCORE_WEIGHT_OVERLOOKED = ( + 1 / 3 +) # Param for get_label_quality_score, weight to determine how much to value overlooked scores over other subtypes when deciding the overall label quality score for an image. +CUSTOM_SCORE_WEIGHT_BADLOC = ( + 1 / 3 +) # Param for get_label_quality_score, weight to determine how much to value badloc scores over other subtypes when deciding issues +CUSTOM_SCORE_WEIGHT_SWAP = ( + 1 / 3 +) # Param for get_label_quality_score, weight to determine how much to value swap scores over other subtypes when deciding issues + +MAX_CLASS_TO_SHOW = 10 # Number of classes to show in legend during the visualize method. Classes over max_class_to_show are cut off. diff --git a/cleanlab/internal/label_quality_utils.py b/cleanlab/internal/label_quality_utils.py new file mode 100644 index 0000000..09d6928 --- /dev/null +++ b/cleanlab/internal/label_quality_utils.py @@ -0,0 +1,118 @@ +"""Helper methods used internally for computing label quality scores.""" + +import warnings +import numpy as np +from typing import Optional +from scipy.special import xlogy + +from cleanlab.count import get_confident_thresholds + + +def _subtract_confident_thresholds( + labels: Optional[np.ndarray], + pred_probs: np.ndarray, + multi_label: bool = False, + confident_thresholds: Optional[np.ndarray] = None, +) -> np.ndarray: + """ + Return adjusted predicted probabilities by subtracting the class confident thresholds and renormalizing. + + The confident class threshold for a class j is the expected (average) "self-confidence" for class j. + The purpose of this adjustment is to handle class imbalance. + + Parameters + ---------- + labels : np.ndarray + Labels in the same format expected by the `cleanlab.count.get_confident_thresholds()` method. + If labels is None, confident_thresholds needs to be passed in as it will not be calculated. + pred_probs : np.ndarray (shape (N, K)) + Predicted-probabilities in the same format expected by the `cleanlab.count.get_confident_thresholds()` method. + confident_thresholds : np.ndarray (shape (K,)) + Pre-calculated confident thresholds. If passed in, function will subtract these thresholds instead of calculating + confident_thresholds from the given labels and pred_probs. + multi_label : bool, optional + If ``True``, labels should be an iterable (e.g. list) of iterables, containing a + list of labels for each example, instead of just a single label. + The multi-label setting supports classification tasks where an example has 1 or more labels. + Example of a multi-labeled `labels` input: ``[[0,1], [1], [0,2], [0,1,2], [0], [1], ...]``. + The major difference in how this is calibrated versus single-label is that + the total number of errors considered is based on the number of labels, + not the number of examples. So, the calibrated `confident_joint` will sum + to the number of total labels. + + Returns + ------- + pred_probs_adj : np.ndarray (float) + Adjusted pred_probs. + """ + # Get expected (average) self-confidence for each class + # TODO: Test this for multi-label + if confident_thresholds is None: + if labels is None: + raise ValueError( + "Cannot calculate confident_thresholds without labels. Pass in either labels or already calculated " + "confident_thresholds parameter. " + ) + confident_thresholds = get_confident_thresholds(labels, pred_probs, multi_label=multi_label) + + # Subtract the class confident thresholds + pred_probs_adj = pred_probs - confident_thresholds + + # Re-normalize by shifting data to take care of negative values from the subtraction + pred_probs_adj += confident_thresholds.max() + pred_probs_adj /= pred_probs_adj.sum(axis=1, keepdims=True) + + return pred_probs_adj + + +def get_normalized_entropy( + pred_probs: np.ndarray, min_allowed_prob: Optional[float] = None +) -> np.ndarray: + """Return the normalized entropy of pred_probs. + + Normalized entropy is between 0 and 1. Higher values of entropy indicate higher uncertainty in the model's prediction of the correct label. + + Read more about normalized entropy `on Wikipedia `_. + + Normalized entropy is used in active learning for uncertainty sampling: https://medium.com/data-science/uncertainty-sampling-cheatsheet-ec57bc067c0b + + Unlike label-quality scores, entropy only depends on the model's predictions, not the given label. + + Parameters + ---------- + pred_probs : np.ndarray (shape (N, K)) + Each row of this matrix corresponds to an example x and contains the model-predicted + probabilities that x belongs to each possible class: P(label=k|x) + + min_allowed_prob : float, default: None, deprecated + Minimum allowed probability value. If not `None` (default), + entries of `pred_probs` below this value will be clipped to this value. + + .. deprecated:: 2.5.0 + This keyword is deprecated and should be left to the default. + The entropy is well-behaved even if `pred_probs` contains zeros, + clipping is unnecessary and (slightly) changes the results. + + Returns + ------- + entropy : np.ndarray (shape (N, )) + Each element is the normalized entropy of the corresponding row of ``pred_probs``. + + Raises + ------ + ValueError + An error is raised if any of the probabilities is not in the interval [0, 1]. + """ + if np.any(pred_probs < 0) or np.any(pred_probs > 1): + raise ValueError("All probabilities are required to be in the interval [0, 1].") + num_classes = pred_probs.shape[1] + + if min_allowed_prob is not None: + warnings.warn( + "Using `min_allowed_prob` is not necessary anymore and will be removed.", + DeprecationWarning, + ) + pred_probs = np.clip(pred_probs, a_min=min_allowed_prob, a_max=None) + + # Note that dividing by log(num_classes) changes the base of the log which rescales entropy to 0-1 range + return -np.sum(xlogy(pred_probs, pred_probs), axis=1) / np.log(num_classes) diff --git a/cleanlab/internal/latent_algebra.py b/cleanlab/internal/latent_algebra.py new file mode 100644 index 0000000..8892bd5 --- /dev/null +++ b/cleanlab/internal/latent_algebra.py @@ -0,0 +1,312 @@ +""" +Contains mathematical functions relating the latent terms, +``P(given_label)``, ``P(given_label | true_label)``, ``P(true_label | given_label)``, ``P(true_label)``, etc. together. +For every function here, if the inputs are exact, the output is guaranteed to be exact. +Every function herein is the computational equivalent of a mathematical equation having a closed, exact form. +If the inputs are inexact, the error will of course propagate. +Throughout `K` denotes the number of classes in the classification task. +""" + +import warnings +import numpy as np +from typing import Tuple + +from cleanlab.internal.util import value_counts, clip_values, clip_noise_rates +from cleanlab.internal.constants import TINY_VALUE, CLIPPING_LOWER_BOUND + + +def compute_ps_py_inv_noise_matrix( + labels, noise_matrix +) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """Compute ``ps := P(labels=k), py := P(true_labels=k)``, and the inverse noise matrix. + + Parameters + ---------- + labels : np.ndarray + A discrete vector of noisy labels, i.e. some labels may be erroneous. + *Format requirements*: for dataset with `K` classes, labels must be in ``{0,1,...,K-1}``. + + noise_matrix : np.ndarray + A conditional probability matrix (of shape ``(K, K)``) of the form ``P(label=k_s|true_label=k_y)`` containing + the fraction of examples in every class, labeled as every other class. + Assumes columns of noise_matrix sum to 1.""" + + ps = value_counts(labels) / float(len(labels)) # p(labels=k) + py, inverse_noise_matrix = compute_py_inv_noise_matrix(ps, noise_matrix) + return ps, py, inverse_noise_matrix + + +def compute_py_inv_noise_matrix(ps, noise_matrix) -> Tuple[np.ndarray, np.ndarray]: + """Compute py := P(true_label=k), and the inverse noise matrix. + + Parameters + ---------- + ps : np.ndarray + Array of shape ``(K, )`` or ``(1, K)``. + The fraction (prior probability) of each observed, NOISY class ``P(labels = k)``. + + noise_matrix : np.ndarray + A conditional probability matrix (of shape ``(K, K)``) of the form ``P(label=k_s|true_label=k_y)`` containing + the fraction of examples in every class, labeled as every other class. + Assumes columns of noise_matrix sum to 1.""" + + # 'py' is p(true_labels=k) = noise_matrix^(-1) * p(labels=k) + # because in *vector computation*: P(label=k|true_label=k) * p(true_label=k) = P(label=k) + # The pseudo-inverse is used when noise_matrix is not invertible. + py = np.linalg.inv(noise_matrix).dot(ps) + + # No class should have probability 0, so we use .000001 + # Make sure valid probabilities that sum to 1.0 + py = clip_values(py, low=CLIPPING_LOWER_BOUND, high=1.0, new_sum=1.0) + + # All the work is done in this function (below) + return py, compute_inv_noise_matrix(py=py, noise_matrix=noise_matrix, ps=ps) + + +def compute_inv_noise_matrix(py, noise_matrix, *, ps=None) -> np.ndarray: + """Compute the inverse noise matrix if py := P(true_label=k) is given. + + Parameters + ---------- + py : np.ndarray (shape (K, 1)) + The fraction (prior probability) of each TRUE class label, P(true_label = k) + + noise_matrix : np.ndarray + A conditional probability matrix (of shape ``(K, K)``) of the form ``P(label=k_s|true_label=k_y)`` containing + the fraction of examples in every class, labeled as every other class. + Assumes columns of noise_matrix sum to 1. + + ps : np.ndarray + Array of shape ``(K, 1)`` containing the fraction (prior probability) of each NOISY given label, ``P(labels = k)``. + `ps` is easily computable from py and should only be provided if it has already been precomputed, to increase code efficiency. + + Examples + -------- + For loop based implementation: + + .. code:: python + + # Number of classes + K = len(py) + + # 'ps' is p(labels=k) = noise_matrix * p(true_labels=k) + # because in *vector computation*: P(label=k|true_label=k) * p(true_label=k) = P(label=k) + if ps is None: + ps = noise_matrix.dot(py) + + # Estimate the (K, K) inverse noise matrix P(true_label = k_y | label = k_s) + inverse_noise_matrix = np.empty(shape=(K,K)) + # k_s is the class value k of noisy label `label == k` + for k_s in range(K): + # k_y is the (guessed) class value k of true label y + for k_y in range(K): + # P(true_label|label) = P(label|y) * P(true_label) / P(labels) + inverse_noise_matrix[k_y][k_s] = noise_matrix[k_s][k_y] * \ + py[k_y] / ps[k_s] + """ + + joint = noise_matrix * py + ps = joint.sum(axis=1) if ps is None else ps + inverse_noise_matrix = joint.T / np.clip(ps, a_min=TINY_VALUE, a_max=None) + + # Clip inverse noise rates P(true_label=k_s|true_label=k_y) into proper range [0,1) + return clip_noise_rates(inverse_noise_matrix) + + +def compute_noise_matrix_from_inverse(ps, inverse_noise_matrix, *, py=None) -> np.ndarray: + """Compute the noise matrix ``P(label=k_s|true_label=k_y)``. + + Parameters + ---------- + py : np.ndarray + Array of shape ``(K, 1)`` containing the fraction (prior probability) of each TRUE class label, ``P(true_label = k)``. + + inverse_noise_matrix : np.ndarray + A conditional probability matrix (of shape ``(K, K)``) of the form P(true_label=k_y|label=k_s) representing + the estimated fraction observed examples in each class k_s, that are + mislabeled examples from every other class k_y. If None, the + inverse_noise_matrix will be computed from pred_probs and labels. + Assumes columns of inverse_noise_matrix sum to 1. + + ps : np.ndarray + Array of shape ``(K, 1)`` containing the fraction (prior probability) of each observed NOISY label, P(labels = k). + `ps` is easily computable from `py` and should only be provided if it has already been precomputed, to increase code efficiency. + + Returns + ------- + noise_matrix : np.ndarray + Array of shape ``(K, K)``, where `K` = number of classes, whose columns sum to 1. + A conditional probability matrix of the form ``P(label=k_s|true_label=k_y)`` containing + the fraction of examples in every class, labeled as every other class. + + Examples + -------- + For loop based implementation: + + .. code:: python + + # Number of classes labels + K = len(ps) + + # 'py' is p(true_label=k) = inverse_noise_matrix * p(true_label=k) + # because in *vector computation*: P(true_label=k|label=k) * p(label=k) = P(true_label=k) + if py is None: + py = inverse_noise_matrix.dot(ps) + + # Estimate the (K, K) noise matrix P(labels = k_s | true_labels = k_y) + noise_matrix = np.empty(shape=(K,K)) + # k_s is the class value k of noisy label `labels == k` + for k_s in range(K): + # k_y is the (guessed) class value k of true label y + for k_y in range(K): + # P(labels|y) = P(true_label|labels) * P(labels) / P(true_label) + noise_matrix[k_s][k_y] = inverse_noise_matrix[k_y][k_s] * \ + ps[k_s] / py[k_y] + + """ + + joint = (inverse_noise_matrix * ps).T + py = joint.sum(axis=0) if py is None else py + noise_matrix = joint / np.clip(py, a_min=TINY_VALUE, a_max=None) + + # Clip inverse noise rates P(true_label=k_y|true_label=k_s) into proper range [0,1) + return clip_noise_rates(noise_matrix) + + +def compute_py( + ps, noise_matrix, inverse_noise_matrix, *, py_method="cnt", true_labels_class_counts=None +) -> np.ndarray: + """Compute ``py := P(true_labels=k)`` from ``ps := P(labels=k)``, `noise_matrix`, and + `inverse_noise_matrix`. + + This method is ** ROBUST ** when ``py_method = 'cnt'`` + It may work well even when the noise matrices are estimated + poorly by using the diagonals of the matrices + instead of all the probabilities in the entire matrix. + + Parameters + ---------- + ps : np.ndarray + Array of shape ``(K, )`` or ``(1, K)`` containing the fraction (prior probability) of each observed, noisy label, P(labels = k) + + noise_matrix : np.ndarray + A conditional probability matrix ( of shape ``(K, K)``) of the form ``P(label=k_s|true_label=k_y)`` containing + the fraction of examples in every class, labeled as every other class. + Assumes columns of noise_matrix sum to 1. + + inverse_noise_matrix : np.ndarray of shape (K, K), K = number of classes + A conditional probability matrix ( of shape ``(K, K)``) of the form ``P(true_label=k_y|label=k_s)`` representing + the estimated fraction observed examples in each class `k_s`, that are + mislabeled examples from every other class `k_y`. If ``None``, the + inverse_noise_matrix will be computed from `pred_probs` and `labels`. + Assumes columns of `inverse_noise_matrix` sum to 1. + + py_method : str (Options: ["cnt", "eqn", "marginal", "marginal_ps"]) + How to compute the latent prior ``p(true_label=k)``. Default is "cnt" as it often + works well even when the noise matrices are estimated poorly by using + the matrix diagonals instead of all the probabilities. + + true_labels_class_counts : np.ndarray + Array of shape ``(K, )`` or ``(1, K)`` containing the marginal counts of the confident joint + (like ``cj.sum(axis = 0)``). + + Returns + ------- + py : np.ndarray + Array of shape ``(K, )`` or ``(1, K)``. + The fraction (prior probability) of each TRUE class label, ``P(true_label = k)``.""" + + if len(np.shape(ps)) > 2 or (len(np.shape(ps)) == 2 and np.shape(ps)[0] != 1): + w = "Input parameter np.ndarray ps has shape " + str(np.shape(ps)) + w += ", but shape should be (K, ) or (1, K)" + warnings.warn(w) + + if py_method == "marginal" and true_labels_class_counts is None: + msg = ( + 'py_method == "marginal" requires true_labels_class_counts, ' + "but true_labels_class_counts is None. " + ) + msg += " Provide parameter true_labels_class_counts." + raise ValueError(msg) + + if py_method == "cnt": + # Computing py this way avoids dividing by zero noise rates. + # More robust bc error est_p(true_label|labels) / est_p(labels|y) ~ p(true_label|labels) / p(labels|y) + py = ( + inverse_noise_matrix.diagonal() + / np.clip(noise_matrix.diagonal(), a_min=TINY_VALUE, a_max=None) + * ps + ) + # Equivalently: py = (true_labels_class_counts / labels_class_counts) * ps + elif py_method == "eqn": + py = np.linalg.inv(noise_matrix).dot(ps) + elif py_method == "marginal": + py = true_labels_class_counts / np.clip( + float(sum(true_labels_class_counts)), a_min=TINY_VALUE, a_max=None + ) + elif py_method == "marginal_ps": + py = np.dot(inverse_noise_matrix, ps) + else: + err = "py_method {}".format(py_method) + err += " should be in [cnt, eqn, marginal, marginal_ps]" + raise ValueError(err) + + # Clip py (0,1), s.t. no class should have prob 0, hence 1e-6 + py = clip_values(py, low=CLIPPING_LOWER_BOUND, high=1.0, new_sum=1.0) + return py + + +def compute_pyx(pred_probs, noise_matrix, inverse_noise_matrix): + """Compute ``pyx := P(true_label=k|x)`` from ``pred_probs := P(label=k|x)``, `noise_matrix` and + `inverse_noise_matrix`. + + This method is ROBUST - meaning it works well even when the + noise matrices are estimated poorly by only using the diagonals of the + matrices which tend to be easy to estimate correctly. + + Parameters + ---------- + pred_probs : np.ndarray + ``P(label=k|x)`` is a ``(N x K)`` matrix with K model-predicted probabilities. + Each row of this matrix corresponds to an example `x` and contains the model-predicted + probabilities that `x` belongs to each possible class. + The columns must be ordered such that these probabilities correspond to class 0,1,2,... + `pred_probs` should have been computed using 3 (or higher) fold cross-validation. + + noise_matrix : np.ndarray + A conditional probability matrix (of shape ``(K, K)``) of the form ``P(label=k_s|true_label=k_y)`` containing + the fraction of examples in every class, labeled as every other class. + Assumes columns of `noise_matrix` sum to 1. + + inverse_noise_matrix : np.ndarray + A conditional probability matrix (of shape ``(K, K)``) of the form ``P(true_label=k_y|label=k_s)`` representing + the estimated fraction observed examples in each class `k_s`, that are + mislabeled examples from every other class `k_y`. If None, the + inverse_noise_matrix will be computed from `pred_probs` and `labels`. + Assumes columns of `inverse_noise_matrix` sum to 1. + + Returns + ------- + pyx : np.ndarray + ``P(true_label=k|x)`` is a ``(N, K)`` matrix of model-predicted probabilities. + Each row of this matrix corresponds to an example `x` and contains the model-predicted + probabilities that `x` belongs to each possible class. + The columns must be ordered such that these probabilities correspond to class 0,1,2,... + `pred_probs` should have been computed using 3 (or higher) fold cross-validation.""" + + if len(np.shape(pred_probs)) != 2: + raise ValueError( + "Input parameter np.ndarray 'pred_probs' has shape " + + str(np.shape(pred_probs)) + + ", but shape should be (N, K)" + ) + + pyx = ( + pred_probs + * inverse_noise_matrix.diagonal() + / np.clip(noise_matrix.diagonal(), a_min=TINY_VALUE, a_max=None) + ) + # Make sure valid probabilities that sum to 1.0 + return np.apply_along_axis( + func1d=clip_values, axis=1, arr=pyx, **{"low": 0.0, "high": 1.0, "new_sum": 1.0} + ) diff --git a/cleanlab/internal/multiannotator_utils.py b/cleanlab/internal/multiannotator_utils.py new file mode 100644 index 0000000..b1ef93d --- /dev/null +++ b/cleanlab/internal/multiannotator_utils.py @@ -0,0 +1,352 @@ +""" +Helper methods used internally in cleanlab.multiannotator +""" + +import warnings +from typing import Optional, Tuple + +import numpy as np +import pandas as pd + +from cleanlab.internal.numerics import softmax +from cleanlab.internal.util import get_num_classes, value_counts +from cleanlab.internal.validation import assert_valid_class_labels +from cleanlab.typing import LabelLike + +SMALL_CONST = 1e-30 + + +def assert_valid_inputs_multiannotator( + labels_multiannotator: np.ndarray, + pred_probs: Optional[np.ndarray] = None, + ensemble: bool = False, + allow_single_label: bool = False, + annotator_ids: Optional[pd.Index] = None, +) -> None: + """Validate format of multi-annotator labels""" + # Check that labels_multiannotator is a 2D array + if labels_multiannotator.ndim != 2: + raise ValueError( + "labels_multiannotator must be a 2D array or dataframe, " + "each row represents an example and each column represents an annotator." + ) + + # Raise error if labels are not formatted properly + if any([isinstance(label, str) for label in labels_multiannotator.ravel()]): + raise ValueError( + "Labels cannot be strings, they must be zero-indexed integers corresponding to class indices." + ) + + # Raise error if labels_multiannotator has NaN rows + nan_row_mask = np.isnan(labels_multiannotator).all(axis=1) + if nan_row_mask.any(): + nan_rows = list(np.where(nan_row_mask)[0]) + raise ValueError( + "labels_multiannotator cannot have rows with all NaN, each example must have at least one label.\n" + f"Examples {nan_rows} do not have any labels." + ) + + # Raise error if labels_multiannotator has NaN columns + nan_col_mask = np.isnan(labels_multiannotator).all(axis=0) + if nan_col_mask.any(): + if annotator_ids is not None: + nan_columns = list(annotator_ids[np.where(nan_col_mask)[0]]) + else: + nan_columns = list(np.where(nan_col_mask)[0]) + raise ValueError( + "labels_multiannotator cannot have columns with all NaN, each annotator must annotator at least one example.\n" + f"Annotators {nan_columns} did not label any examples." + ) + + if not allow_single_label: + # Raise error if labels_multiannotator has <= 1 column + if labels_multiannotator.shape[1] <= 1: + raise ValueError( + "labels_multiannotator must have more than one column.\n" + "If there is only one annotator, use cleanlab.rank.get_label_quality_scores instead" + ) + + # Raise error if labels_multiannotator only has 1 label per example + if (np.sum(~np.isnan(labels_multiannotator), axis=1) == 1).all(): + raise ValueError( + "Each example only has one label, collapse the labels into a 1-D array and use " + "cleanlab.rank.get_label_quality_scores instead" + ) + + # Raise warning if no examples with 2 or more annotators agree + # TODO: might shift this later in the code to avoid extra compute + has_agreement = np.zeros(labels_multiannotator.shape[0], dtype=bool) + for i in np.unique(labels_multiannotator): + has_agreement |= (labels_multiannotator == i).sum(axis=1) > 1 + if not has_agreement.any(): + warnings.warn("Annotators do not agree on any example. Check input data.") + + # Check labels + all_labels_flatten = labels_multiannotator.ravel() + all_labels_flatten = all_labels_flatten[~np.isnan(all_labels_flatten)] + assert_valid_class_labels(all_labels_flatten, allow_one_class=True) + + # Raise error if number of classes in labels_multiannoator does not match number of classes in pred_probs + if pred_probs is not None: + if not isinstance(pred_probs, np.ndarray): + raise TypeError("pred_probs must be a numpy array.") + + if ensemble: + if pred_probs.ndim != 3: + error_message = "pred_probs must be a 3d array." + if pred_probs.ndim == 2: + error_message += " If you have a 2d pred_probs array, use the non-ensemble version of this function." + raise ValueError(error_message) + + if pred_probs.shape[1] != len(labels_multiannotator): + raise ValueError("each pred_probs and labels_multiannotator must have same length.") + + num_classes = pred_probs.shape[2] + else: + if pred_probs.ndim != 2: + error_message = "pred_probs must be a 2d array." + if pred_probs.ndim == 3: + error_message += " If you have a 3d pred_probs array, use the ensemble version of this function." + raise ValueError(error_message) + + if len(pred_probs) != len(labels_multiannotator): + raise ValueError("pred_probs and labels_multiannotator must have same length.") + + num_classes = pred_probs.shape[1] + + highest_class = np.nanmax(labels_multiannotator) + 1 + + # this allows for missing labels, but not missing columns in pred_probs + if num_classes < highest_class: + raise ValueError( + f"pred_probs must have at least {int(highest_class)} columns based on the largest class label " + "which appears in labels_multiannotator. Perhaps some rarely-annotated classes were lost while " + "establishing consensus labels used to train your classifier." + ) + + +def assert_valid_pred_probs( + pred_probs: Optional[np.ndarray] = None, + pred_probs_unlabeled: Optional[np.ndarray] = None, + ensemble: bool = False, +): + """Validate format of pred_probs for multiannotator active learning functions""" + if pred_probs is None and pred_probs_unlabeled is None: + raise ValueError( + "pred_probs and pred_probs_unlabeled cannot both be None, specify at least one of the two." + ) + + if ensemble: + if pred_probs is not None: + if not isinstance(pred_probs, np.ndarray): + raise TypeError("pred_probs must be a numpy array.") + if pred_probs.ndim != 3: + error_message = "pred_probs must be a 3d array." + if pred_probs.ndim == 2: # pragma: no cover + error_message += " If you have a 2d pred_probs array (ie. only one predictor), use the non-ensemble version of this function." + raise ValueError(error_message) + + if pred_probs_unlabeled is not None: + if not isinstance(pred_probs_unlabeled, np.ndarray): + raise TypeError("pred_probs_unlabeled must be a numpy array.") + if pred_probs_unlabeled.ndim != 3: + error_message = "pred_probs_unlabeled must be a 3d array." + if pred_probs_unlabeled.ndim == 2: # pragma: no cover + error_message += " If you have a 2d pred_probs_unlabeled array, use the non-ensemble version of this function." + raise ValueError(error_message) + + if pred_probs is not None and pred_probs_unlabeled is not None: + if pred_probs.shape[2] != pred_probs_unlabeled.shape[2]: + raise ValueError( + "pred_probs and pred_probs_unlabeled must have the same number of classes" + ) + + else: + if pred_probs is not None: + if not isinstance(pred_probs, np.ndarray): + raise TypeError("pred_probs must be a numpy array.") + if pred_probs.ndim != 2: + error_message = "pred_probs must be a 2d array." + if pred_probs.ndim == 3: # pragma: no cover + error_message += " If you have a 3d pred_probs array, use the ensemble version of this function." + raise ValueError(error_message) + + if pred_probs_unlabeled is not None: + if not isinstance(pred_probs_unlabeled, np.ndarray): + raise TypeError("pred_probs_unlabeled must be a numpy array.") + if pred_probs_unlabeled.ndim != 2: + error_message = "pred_probs_unlabeled must be a 2d array." + if pred_probs_unlabeled.ndim == 3: # pragma: no cover + error_message += " If you have a 3d pred_probs_unlabeled array, use the non-ensemble version of this function." + raise ValueError(error_message) + + if pred_probs is not None and pred_probs_unlabeled is not None: + if pred_probs.shape[1] != pred_probs_unlabeled.shape[1]: + raise ValueError( + "pred_probs and pred_probs_unlabeled must have the same number of classes" + ) + + +def format_multiannotator_labels(labels: LabelLike) -> Tuple[pd.DataFrame, dict]: + """Takes an array of labels and formats it such that labels are in the set ``0, 1, ..., K-1``, + where ``K`` is the number of classes. The labels are assigned based on lexicographic order. + + Returns + ------- + formatted_labels + Returns pd.DataFrame of shape ``(N,M)``. The return labels will be properly formatted and can be passed to + cleanlab.multiannotator functions. + + mapping + A dictionary showing the mapping of new to old labels, such that ``mapping[k]`` returns the name of the k-th class. + """ + if isinstance(labels, pd.DataFrame): + np_labels = labels.values + elif isinstance(labels, np.ndarray): + np_labels = labels + else: + raise TypeError("labels must be 2D numpy array or pandas DataFrame") + + unique_labels = pd.unique(np_labels.ravel()) + + try: + unique_labels = unique_labels[~np.isnan(unique_labels)] + unique_labels.sort() + except TypeError: # np.unique / np.sort cannot handle string values or pd.NA types + nan_mask = np.array([(l is np.nan) or (l is pd.NA) or (l == "nan") for l in unique_labels]) + unique_labels = unique_labels[~nan_mask] + unique_labels.sort() + + # convert float labels (that arose because np.nan is float type) to int + if unique_labels.dtype == "float": + unique_labels = unique_labels.astype("int") + + label_map = {label: i for i, label in enumerate(unique_labels)} + inverse_map = {i: label for label, i in label_map.items()} + + if isinstance(labels, np.ndarray): + labels = pd.DataFrame(labels) + + formatted_labels = labels.replace(label_map) + + return formatted_labels, inverse_map + + +def check_consensus_label_classes( + labels_multiannotator: np.ndarray, + consensus_label: np.ndarray, + consensus_method: str, +) -> None: + """Check if any classes no longer appear in the set of consensus labels (established using the consensus_method stated)""" + unique_ma_labels = np.unique(labels_multiannotator) + unique_ma_labels = unique_ma_labels[~np.isnan(unique_ma_labels)] + labels_set_difference = set(unique_ma_labels) - set(consensus_label) + + if len(labels_set_difference) > 0: + print( + "CAUTION: Number of unique classes has been reduced from the original data when establishing consensus labels " + f"using consensus method '{consensus_method}', likely due to some classes being rarely annotated. " + "If training a classifier on these consensus labels, it will never see any of the omitted classes unless you " + "manually replace some of the consensus labels.\n" + f"Classes in the original data but not in consensus labels: {list(map(int, labels_set_difference))}" + ) + + +def compute_soft_cross_entropy( + labels_multiannotator: np.ndarray, + pred_probs: np.ndarray, +) -> float: + """Compute soft cross entropy between the annotators' empirical label distribution and model pred_probs""" + num_classes = get_num_classes(pred_probs=pred_probs) + + empirical_label_distribution = np.full((len(labels_multiannotator), num_classes), np.nan) + for i, labels in enumerate(labels_multiannotator): + labels_subset = labels[~np.isnan(labels)] + empirical_label_distribution[i, :] = value_counts( + labels_subset, num_classes=num_classes + ) / len(labels_subset) + + clipped_pred_probs = np.clip(pred_probs, a_min=SMALL_CONST, a_max=None) + soft_cross_entropy = -np.sum( + empirical_label_distribution * np.log(clipped_pred_probs), axis=1 + ) / np.log(num_classes) + + return soft_cross_entropy + + +def find_best_temp_scaler( + labels_multiannotator: np.ndarray, + pred_probs: np.ndarray, + coarse_search_range: list = [0.1, 0.2, 0.5, 0.8, 1, 2, 3, 5, 8], + fine_search_size: int = 4, +) -> float: + """Find the best temperature scaling factor that minimizes the soft cross entropy between the annotators' empirical label distribution + and model pred_probs""" + + soft_cross_entropy_coarse = np.full(len(coarse_search_range), np.nan) + log_pred_probs = np.log( + pred_probs, where=pred_probs > 0, out=np.full(pred_probs.shape, -np.inf) + ) + for i, curr_temp in enumerate(coarse_search_range): + scaled_pred_probs = softmax(log_pred_probs, temperature=curr_temp, axis=1, shift=False) + soft_cross_entropy_coarse[i] = np.mean( + compute_soft_cross_entropy(labels_multiannotator, scaled_pred_probs) + ) + + min_entropy_ind = np.argmin(soft_cross_entropy_coarse) + fine_search_range = _set_fine_search_range( + coarse_search_range, fine_search_size, min_entropy_ind + ) + soft_cross_entropy_fine = np.full(len(fine_search_range), np.nan) + for i, curr_temp in enumerate(fine_search_range): + scaled_pred_probs = softmax(log_pred_probs, temperature=curr_temp, axis=1, shift=False) + soft_cross_entropy_fine[i] = np.mean( + compute_soft_cross_entropy(labels_multiannotator, scaled_pred_probs) + ) + best_temp = fine_search_range[np.argmin(soft_cross_entropy_fine)] + return best_temp + + +def _set_fine_search_range( + coarse_search_range: list, fine_search_size: int, min_entropy_ind: np.intp +) -> np.ndarray: + fine_search_range = np.array([]) + if min_entropy_ind != 0: + fine_search_range = np.append( + np.linspace( + coarse_search_range[min_entropy_ind - 1], + coarse_search_range[min_entropy_ind], + fine_search_size, + endpoint=False, + ), + fine_search_range, + ) + if min_entropy_ind != len(coarse_search_range) - 1: + fine_search_range = np.append( + fine_search_range, + np.linspace( + coarse_search_range[min_entropy_ind], + coarse_search_range[min_entropy_ind + 1], + fine_search_size + 1, + endpoint=True, + ), + ) + return fine_search_range + + +def temp_scale_pred_probs( + pred_probs: np.ndarray, + temp: float, +) -> np.ndarray: + """Scales pred_probs by the given temperature factor. Temperature of <1 will sharpen the pred_probs while temperatures of >1 will smoothen it.""" + # clip pred_probs to prevent taking log of 0 + pred_probs = np.clip(pred_probs, a_min=SMALL_CONST, a_max=None) + pred_probs = pred_probs / np.sum(pred_probs, axis=1)[:, np.newaxis] + + # apply temperate scale + scaled_pred_probs = softmax(np.log(pred_probs), temperature=temp, axis=1, shift=False) + scaled_pred_probs = ( + scaled_pred_probs / np.sum(scaled_pred_probs, axis=1)[:, np.newaxis] + ) # normalize + + return scaled_pred_probs diff --git a/cleanlab/internal/multilabel_scorer.py b/cleanlab/internal/multilabel_scorer.py new file mode 100644 index 0000000..313a1c9 --- /dev/null +++ b/cleanlab/internal/multilabel_scorer.py @@ -0,0 +1,652 @@ +""" +Helper classes and functions used internally to compute label quality scores in multi-label classification. +""" + +from enum import Enum +from typing import Callable, Dict, Optional, Union + +import numpy as np +from sklearn.model_selection import cross_val_predict + +from cleanlab.internal.label_quality_utils import _subtract_confident_thresholds +from cleanlab.internal.multilabel_utils import _is_multilabel, stack_complement +from cleanlab.internal.numerics import softmax +from cleanlab.rank import ( + get_confidence_weighted_entropy_for_each_label, + get_normalized_margin_for_each_label, + get_self_confidence_for_each_label, +) + + +class _Wrapper: + """Helper class for wrapping callable functions as attributes of an Enum instead of + setting them as methods of the Enum class. + + + This class is only intended to be used internally for the ClassLabelScorer or + other cases where functions are used for enumeration values. + """ + + def __init__(self, f: Callable) -> None: + self.f = f + + def __call__(self, *args, **kwargs): + return self.f(*args, **kwargs) + + def __repr__(self): + return self.f.__name__ + + +class ClassLabelScorer(Enum): + """Enum for the different methods to compute label quality scores.""" + + SELF_CONFIDENCE = _Wrapper(get_self_confidence_for_each_label) + """Returns the self-confidence label-quality score for each datapoint. + + See also + -------- + cleanlab.rank.get_self_confidence_for_each_label + """ + NORMALIZED_MARGIN = _Wrapper(get_normalized_margin_for_each_label) + """Returns the "normalized margin" label-quality score for each datapoint. + + See also + -------- + cleanlab.rank.get_normalized_margin_for_each_label + """ + CONFIDENCE_WEIGHTED_ENTROPY = _Wrapper(get_confidence_weighted_entropy_for_each_label) + """Returns the "confidence weighted entropy" label-quality score for each datapoint. + + See also + -------- + cleanlab.rank.get_confidence_weighted_entropy_for_each_label + """ + + def __call__(self, labels: np.ndarray, pred_probs: np.ndarray, **kwargs) -> np.ndarray: + """Returns the label-quality scores for each datapoint based on the given labels and predicted probabilities. + + See the documentation for each method for more details. + + Example + ------- + >>> import numpy as np + >>> from cleanlab.internal.multilabel_scorer import ClassLabelScorer + >>> labels = np.array([0, 0, 0, 1, 1, 1]) + >>> pred_probs = np.array([ + ... [0.9, 0.1], + ... [0.8, 0.2], + ... [0.7, 0.3], + ... [0.2, 0.8], + ... [0.75, 0.25], + ... [0.1, 0.9], + ... ]) + >>> ClassLabelScorer.SELF_CONFIDENCE(labels, pred_probs) + array([0.9 , 0.8 , 0.7 , 0.8 , 0.25, 0.9 ]) + """ + pred_probs = self._adjust_pred_probs(labels, pred_probs, **kwargs) + return self.value(labels, pred_probs) + + def _adjust_pred_probs( + self, labels: np.ndarray, pred_probs: np.ndarray, **kwargs + ) -> np.ndarray: + """Returns adjusted predicted probabilities by subtracting the class confident thresholds and renormalizing. + + This is used to adjust the predicted probabilities for the SELF_CONFIDENCE and NORMALIZED_MARGIN methods. + """ + if kwargs.get("adjust_pred_probs", False) is True: + if self == ClassLabelScorer.CONFIDENCE_WEIGHTED_ENTROPY: + raise ValueError(f"adjust_pred_probs is not currently supported for {self}.") + pred_probs = _subtract_confident_thresholds(labels, pred_probs) + return pred_probs + + @classmethod + def from_str(cls, method: str) -> "ClassLabelScorer": + """Constructs an instance of the ClassLabelScorer enum based on the given method name. + + Parameters + ---------- + method: + The name of the scoring method to use. + + Returns + ------- + scorer: + An instance of the ClassLabelScorer enum. + + Raises + ------ + ValueError: + If the given method name is not a valid method name. + It must be one of the following: "self_confidence", "normalized_margin", or "confidence_weighted_entropy". + + Example + ------- + >>> from cleanlab.internal.multilabel_scorer import ClassLabelScorer + >>> ClassLabelScorer.from_str("self_confidence") + + """ + try: + return cls[method.upper()] + except KeyError: + raise ValueError(f"Invalid method name: {method}") + + +def exponential_moving_average( + s: np.ndarray, + *, + alpha: Optional[float] = None, + axis: int = 1, + **_, +) -> np.ndarray: + r"""Exponential moving average (EMA) score aggregation function. + + For a score vector s = (s_1, ..., s_K) with K scores, the values + are sorted in *descending* order and the exponential moving average + of the last score is calculated, denoted as EMA_K according to the + note below. + + Note + ---- + + The recursive formula for the EMA at step :math:`t = 2, ..., K` is: + + .. math:: + + \text{EMA}_t = \alpha \cdot s_t + (1 - \alpha) \cdot \text{EMA}_{t-1}, \qquad 0 \leq \alpha \leq 1 + + We set :math:`\text{EMA}_1 = s_1` as the largest score in the sorted vector s. + + :math:`\alpha` is the "forgetting factor" that gives more weight to the + most recent scores, and successively less weight to the previous scores. + + Parameters + ---------- + s : + Scores to be transformed. + + alpha : + Discount factor that determines the weight of the previous EMA score. + Higher alpha means that the previous EMA score has a lower weight while + the current score has a higher weight. + + Its value must be in the interval [0, 1]. + + If alpha is None, it is set to 2 / (K + 1) where K is the number of scores. + + axis : + Axis along which the scores are sorted. + + Returns + ------- + s_ema : + Exponential moving average score. + + Examples + -------- + >>> from cleanlab.internal.multilabel_scorer import exponential_moving_average + >>> import numpy as np + >>> s = np.array([[0.1, 0.2, 0.3]]) + >>> exponential_moving_average(s, alpha=0.5) + np.array([0.175]) + """ + K = s.shape[1] + s_sorted = np.fliplr(np.sort(s, axis=axis)) + if alpha is None: + # One conventional choice for alpha is 2/(K + 1), where K is the number of periods in the moving average. + alpha = float(2 / (K + 1)) + if not (0 <= alpha <= 1): + raise ValueError(f"alpha must be in the interval [0, 1], got {alpha}") + s_T = s_sorted.T + s_ema, s_next = s_T[0], s_T[1:] + for s_i in s_next: + s_ema = alpha * s_i + (1 - alpha) * s_ema + return s_ema + + +def softmin( + s: np.ndarray, + *, + temperature: float = 0.1, + axis: int = 1, + **_, +) -> np.ndarray: + """Softmin score aggregation function. + + Parameters + ---------- + s : + Input array. + + temperature : + Temperature parameter. Too small values may cause numerical underflow and NaN scores. + + axis : + Axis along which to apply the function. + + Returns + ------- + Softmin score. + """ + + return np.einsum( + "ij,ij->i", s, softmax(x=1 - s, temperature=temperature, axis=axis, shift=True) + ) + + +class Aggregator: + """Helper class for aggregating the label quality scores for each class into a single score for each datapoint. + + Parameters + ---------- + method: + The method to compute the label quality scores for each class. + If passed as a callable, your function should take in a 1D array of K scores and return a single aggregated score. + See `~cleanlab.internal.multilabel_scorer.exponential_moving_average` for an example of such a function. + Alternatively, this can be a str value to specify a built-in function, possible values are the keys of the ``Aggregator``'s `possible_methods` attribute. + + kwargs: + Additional keyword arguments to pass to the aggregation function when it is called. + """ + + possible_methods: Dict[str, Callable[..., np.ndarray]] = { + "exponential_moving_average": exponential_moving_average, + "softmin": softmin, + } + + def __init__(self, method: Union[str, Callable], **kwargs): + if isinstance(method, str): # convert to callable + if method in self.possible_methods: + method = self.possible_methods[method] + else: + raise ValueError( + f"Invalid aggregation method specified: '{method}', must be one of the following: {list(self.possible_methods.keys())}" + ) + + self._validate_method(method) + self.method = method + self.kwargs = kwargs + + @staticmethod + def _validate_method(method) -> None: + if not callable(method): + raise TypeError(f"Expected callable method, got {type(method)}") + + @staticmethod + def _validate_scores(scores: np.ndarray) -> None: + if not (isinstance(scores, np.ndarray) and scores.ndim == 2): + raise ValueError( + f"Expected 2D array for scores, got {type(scores)} with shape {scores.shape}" + ) + + def __call__(self, scores: np.ndarray, **kwargs) -> np.ndarray: + """Returns the label quality scores for each datapoint based on the given label quality scores for each class. + + Parameters + ---------- + scores: + The label quality scores for each class. + + Returns + ------- + aggregated_scores: + A single label quality score for each datapoint. + """ + self._validate_scores(scores) + kwargs["axis"] = 1 + updated_kwargs = {**self.kwargs, **kwargs} + return self.method(scores, **updated_kwargs) + + def __repr__(self): + return f"Aggregator(method={self.method.__name__}, kwargs={self.kwargs})" + + +class MultilabelScorer: + """Aggregates label quality scores across different classes to produce one score per example in multi-label classification tasks. + + Parameters + ---------- + base_scorer: + The method to compute the label quality scores for each class. + + See the documentation for the ClassLabelScorer enum for more details. + + aggregator: + The method to aggregate the label quality scores for each class into a single score for each datapoint. + + Defaults to the EMA (exponential moving average) aggregator with forgetting factor ``alpha=0.8``. + + See the documentation for the Aggregator class for more details. + + See also + -------- + exponential_moving_average + + strict: + Flag for performing strict validation of the input data. + """ + + def __init__( + self, + base_scorer: ClassLabelScorer = ClassLabelScorer.SELF_CONFIDENCE, + aggregator: Union[Aggregator, Callable] = Aggregator(exponential_moving_average, alpha=0.8), + *, + strict: bool = True, + ): + self.base_scorer = base_scorer + if not isinstance(aggregator, Aggregator): + self.aggregator = Aggregator(aggregator) + else: + self.aggregator = aggregator + self.strict = strict + + def __call__( + self, + labels: np.ndarray, + pred_probs: np.ndarray, + base_scorer_kwargs: Optional[dict] = None, + **aggregator_kwargs, + ) -> np.ndarray: + """ + Computes a quality score for each label in a multi-label classification problem + based on out-of-sample predicted probabilities. + For each example, the label quality scores for each class are aggregated into a single overall label quality score. + + Parameters + ---------- + labels: + A 2D array of shape (n_samples, n_labels) with binary labels. + + pred_probs: + A 2D array of shape (n_samples, n_labels) with predicted probabilities. + + kwargs: + Additional keyword arguments to pass to the base_scorer and the aggregator. + + base_scorer_kwargs: + Keyword arguments to pass to the base_scorer + + aggregator_kwargs: + Additional keyword arguments to pass to the aggregator. + + Returns + ------- + scores: + A 1D array of shape (n_samples,) with the quality scores for each datapoint. + + Examples + -------- + >>> from cleanlab.internal.multilabel_scorer import MultilabelScorer, ClassLabelScorer + >>> import numpy as np + >>> labels = np.array([[0, 1, 0], [1, 0, 1]]) + >>> pred_probs = np.array([[0.1, 0.9, 0.1], [0.4, 0.1, 0.9]]) + >>> scorer = MultilabelScorer() + >>> scores = scorer(labels, pred_probs) + >>> scores + array([0.9, 0.5]) + + >>> scorer = MultilabelScorer( + ... base_scorer = ClassLabelScorer.NORMALIZED_MARGIN, + ... aggregator = np.min, # Use the "worst" label quality score for each example. + ... ) + >>> scores = scorer(labels, pred_probs) + >>> scores + array([0.9, 0.4]) + """ + if self.strict: + self._validate_labels_and_pred_probs(labels, pred_probs) + scores = self.get_class_label_quality_scores(labels, pred_probs, base_scorer_kwargs) + return self.aggregate(scores, **aggregator_kwargs) + + def aggregate( + self, + class_label_quality_scores: np.ndarray, + **kwargs, + ) -> np.ndarray: + """Aggregates the label quality scores for each class into a single overall label quality score for each example. + + Parameters + ---------- + class_label_quality_scores: + A 2D array of shape (n_samples, n_labels) with the label quality scores for each class. + + See also + -------- + get_class_label_quality_scores + + kwargs: + Additional keyword arguments to pass to the aggregator. + + Returns + ------- + scores: + A 1D array of shape (n_samples,) with the quality scores for each datapoint. + + Examples + -------- + >>> from cleanlab.internal.multilabel_scorer import MultilabelScorer + >>> import numpy as np + >>> class_label_quality_scores = np.array([[0.9, 0.9, 0.3],[0.4, 0.9, 0.6]]) + >>> scorer = MultilabelScorer() # Use the default aggregator (exponential moving average) with default parameters. + >>> scores = scorer.aggregate(class_label_quality_scores) + >>> scores + array([0.42, 0.452]) + >>> new_scores = scorer.aggregate(class_label_quality_scores, alpha=0.5) # Use the default aggregator with custom parameters. + >>> new_scores + array([0.6, 0.575]) + + Warning + ------- + Make sure that keyword arguments correspond to the aggregation function used. + I.e. the ``exponential_moving_average`` function supports an ``alpha`` keyword argument, but ``np.min`` does not. + """ + return self.aggregator(class_label_quality_scores, **kwargs) + + def get_class_label_quality_scores( + self, + labels: np.ndarray, + pred_probs: np.ndarray, + base_scorer_kwargs: Optional[dict] = None, + ) -> np.ndarray: + """Computes separate label quality scores for each class. + + Parameters + ---------- + labels: + A 2D array of shape (n_samples, n_labels) with binary labels. + + pred_probs: + A 2D array of shape (n_samples, n_labels) with predicted probabilities. + + base_scorer_kwargs: + Keyword arguments to pass to the base scoring-function. + + Returns + ------- + class_label_quality_scores: + A 2D array of shape (n_samples, n_labels) with the quality scores for each label. + + Examples + -------- + >>> from cleanlab.internal.multilabel_scorer import MultilabelScorer + >>> import numpy as np + >>> labels = np.array([[0, 1, 0], [1, 0, 1]]) + >>> pred_probs = np.array([[0.1, 0.9, 0.7], [0.4, 0.1, 0.6]]) + >>> scorer = MultilabelScorer() # Use the default base scorer (SELF_CONFIDENCE) + >>> class_label_quality_scores = scorer.get_label_quality_scores_per_class(labels, pred_probs) + >>> class_label_quality_scores + array([[0.9, 0.9, 0.3], + [0.4, 0.9, 0.6]]) + """ + class_label_quality_scores = np.zeros(shape=labels.shape) + if base_scorer_kwargs is None: + base_scorer_kwargs = {} + for i, (label_i, pred_prob_i) in enumerate(zip(labels.T, pred_probs.T)): + pred_prob_i_two_columns = stack_complement(pred_prob_i) + class_label_quality_scores[:, i] = self.base_scorer( + label_i, pred_prob_i_two_columns, **base_scorer_kwargs + ) + return class_label_quality_scores + + @staticmethod + def _validate_labels_and_pred_probs(labels: np.ndarray, pred_probs: np.ndarray) -> None: + """ + Checks that (multi-)labels are in the proper binary indicator format and that + they are compatible with the predicted probabilities. + """ + # Only allow dense matrices for labels for now + if not isinstance(labels, np.ndarray): + raise TypeError("Labels must be a numpy array.") + if not _is_multilabel(labels): + raise ValueError("Labels must be in multi-label format.") + if labels.shape != pred_probs.shape: + raise ValueError("Labels and predicted probabilities must have the same shape.") + + +def get_label_quality_scores( + labels, + pred_probs, + *, + method: MultilabelScorer = MultilabelScorer(), + base_scorer_kwargs: Optional[dict] = None, + **aggregator_kwargs, +) -> np.ndarray: + """Computes a quality score for each label in a multi-label classification problem + based on out-of-sample predicted probabilities. + + Parameters + ---------- + labels: + A 2D array of shape (N, K) with binary labels. + + pred_probs: + A 2D array of shape (N, K) with predicted probabilities. + + method: + A scoring+aggregation method for computing the label quality scores of examples in a multi-label classification setting. + + base_scorer_kwargs: + Keyword arguments to pass to the class-label scorer. + + aggregator_kwargs: + Additional keyword arguments to pass to the aggregator. + + Returns + ------- + scores: + A 1D array of shape (N,) with the quality scores for each datapoint. + + Examples + -------- + >>> import cleanlab.internal.multilabel_scorer as ml_scorer + >>> import numpy as np + >>> labels = np.array([[0, 1, 0], [1, 0, 1]]) + >>> pred_probs = np.array([[0.1, 0.9, 0.1], [0.4, 0.1, 0.9]]) + >>> scores = ml_scorer.get_label_quality_scores(labels, pred_probs, method=ml_scorer.MultilabelScorer()) + >>> scores + array([0.9, 0.5]) + + See also + -------- + MultilabelScorer: + See the documentation for the MultilabelScorer class for more examples of scoring methods and aggregation methods. + """ + return method(labels, pred_probs, base_scorer_kwargs=base_scorer_kwargs, **aggregator_kwargs) + + +# Probabilities + + +def multilabel_py(y: np.ndarray) -> np.ndarray: + """Compute the prior probability of each label in a multi-label classification problem. + + Parameters + ---------- + y : + A 2d array of binarized multi-labels of shape (N, K) where N is the number of samples and K is the number of classes. + + Returns + ------- + py : + A 2d array of prior probabilities of shape (K,2) where the first column is the probability of the label being 0 + and the second column is the probability of the label being 1 for each class. + + Examples + -------- + >>> from cleanlab.internal.multilabel_scorer import multilabel_py + >>> import numpy as np + >>> y = np.array([[0, 0], [0, 1], [1, 0], [1, 1]]) + >>> multilabel_py(y) + array([[0.5, 0.5], + [0.5, 0.5]]) + >>> y = np.array([[0, 0], [0, 1], [1, 0], [1, 0], [1, 0]]) + >>> multilabel_py(y) + array([[0.4, 0.6], + [0.8, 0.2]]) + """ + + N, _ = y.shape + fraction_0 = np.sum(y == 0, axis=0) / N + fraction_1 = 1 - fraction_0 + py = np.column_stack((fraction_0, fraction_1)) + return py + + +# Cross-validation helpers + + +def _get_split_generator(labels, cv): + _, multilabel_ids = np.unique(labels, axis=0, return_inverse=True) + split_generator = cv.split(X=multilabel_ids, y=multilabel_ids) + return split_generator + + +def get_cross_validated_multilabel_pred_probs(X, labels: np.ndarray, *, clf, cv) -> np.ndarray: + """Get predicted probabilities for a multi-label classifier via cross-validation. + + Note + ---- + The labels are reformatted to a "multi-class" format internally to support a wider range of cross-validation strategies. + If you have a multi-label dataset with `K` classes, the labels are reformatted to a "multi-class" format with up to `2**K` classes + (i.e. the number of possible class-assignment configurations). + It is unlikely that you'll all `2**K` configurations in your dataset. + + Parameters + ---------- + X : + A 2d array of features of shape (N, M) where N is the number of samples and M is the number of features. + + labels : + A 2d array of binarized multi-labels of shape (N, K) where N is the number of samples and K is the number of classes. + + clf : + A multi-label classifier with a ``predict_proba`` method. + + cv : + A cross-validation splitter with a ``split`` method that returns a generator of train/test indices. + + Returns + ------- + pred_probs : + A 2d array of predicted probabilities of shape (N, K) where N is the number of samples and K is the number of classes. + + Note + ---- + The predicted probabilities are not expected to sum to 1 for each sample in the case of multi-label classification. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.model_selection import KFold + >>> from sklearn.multiclass import OneVsRestClassifier + >>> from sklearn.ensemble import RandomForestClassifier + >>> from cleanlab.internal.multilabel_scorer import get_cross_validated_multilabel_pred_probs + >>> np.random.seed(0) + >>> X = np.random.rand(16, 2) + >>> labels = np.random.randint(0, 2, size=(16, 2)) + >>> clf = OneVsRestClassifier(RandomForestClassifier()) + >>> cv = KFold(n_splits=2) + >>> get_cross_validated_multilabel_pred_probs(X, labels, clf=clf, cv=cv) + """ + split_generator = _get_split_generator(labels, cv) + pred_probs = cross_val_predict(clf, X, labels, cv=split_generator, method="predict_proba") + return pred_probs diff --git a/cleanlab/internal/multilabel_utils.py b/cleanlab/internal/multilabel_utils.py new file mode 100644 index 0000000..ebb264a --- /dev/null +++ b/cleanlab/internal/multilabel_utils.py @@ -0,0 +1,90 @@ +""" +Helper functions used internally for multi-label classification tasks. +""" + +from typing import List, Optional, Tuple + +import numpy as np + +from cleanlab.internal.util import get_num_classes + + +def _is_multilabel(y: np.ndarray) -> bool: + """Checks whether `y` is in a multi-label indicator matrix format. + + Sparse matrices are not supported. + """ + if not (isinstance(y, np.ndarray) and y.ndim == 2 and y.shape[1] > 1): + return False + return np.array_equal(np.unique(y), [0, 1]) + + +def stack_complement(pred_prob_slice: np.ndarray) -> np.ndarray: + """ + Extends predicted probabilities of a single class to two columns. + + Parameters + ---------- + pred_prob_slice: + A 1D array with predicted probabilities for a single class. + + Example + ------- + >>> pred_prob_slice = np.array([0.1, 0.9, 0.3, 0.8]) + >>> stack_complement(pred_prob_slice) + array([[0.9, 0.1], + [0.1, 0.9], + [0.7, 0.3], + [0.2, 0.8]]) + """ + return np.vstack((1 - pred_prob_slice, pred_prob_slice)).T + + +def get_onehot_num_classes( + labels: list, pred_probs: Optional[np.ndarray] = None +) -> Tuple[np.ndarray, int]: + """Returns OneHot encoding of MultiLabel Data, and number of classes""" + num_classes = get_num_classes(labels=labels, pred_probs=pred_probs) + try: + y_one = int2onehot(labels, K=num_classes) + except TypeError: + raise ValueError( + "wrong format for labels, should be a list of list[indices], please check the documentation in find_label_issues for further information" + ) + return y_one, num_classes + + +def int2onehot(labels: list, K: int) -> np.ndarray: + """Convert multi-label classification `labels` from a ``List[List[int]]`` format to a onehot matrix. + This returns a binarized format of the labels as a multi-hot vector for each example, where the entries in this vector are 1 for each class that applies to this example and 0 otherwise. + + Parameters + ---------- + labels: list of lists of integers + e.g. [[0,1], [3], [1,2,3], [1], [2]] + All integers from 0,1,...,K-1 must be represented. + K: int + The number of classes.""" + + from sklearn.preprocessing import MultiLabelBinarizer + + mlb = MultiLabelBinarizer(classes=range(K)) + return mlb.fit_transform(labels) + + +def onehot2int(onehot_matrix: np.ndarray) -> List[List[int]]: + """Convert multi-label classification `labels` from a onehot matrix format to a ``List[List[int]]`` format that can be used with other cleanlab functions. + + Parameters + ---------- + onehot_matrix: 2D np.ndarray of 0s and 1s + A matrix representation of multi-label classification labels in a binarized format as a multi-hot vector for each example. + The entries in this vector are 1 for each class that applies to this example and 0 otherwise. + + Returns + ------- + labels: list of lists of integers + e.g. [[0,1], [3], [1,2,3], [1], [2]] + All integers from 0,1,...,K-1 must be represented.""" + + return [np.where(row)[0].tolist() for row in onehot_matrix] diff --git a/cleanlab/internal/neighbor/__init__.py b/cleanlab/internal/neighbor/__init__.py new file mode 100644 index 0000000..2575a85 --- /dev/null +++ b/cleanlab/internal/neighbor/__init__.py @@ -0,0 +1 @@ +from .knn_graph import features_to_knn diff --git a/cleanlab/internal/neighbor/knn_graph.py b/cleanlab/internal/neighbor/knn_graph.py new file mode 100644 index 0000000..3bbb149 --- /dev/null +++ b/cleanlab/internal/neighbor/knn_graph.py @@ -0,0 +1,578 @@ +from __future__ import annotations +from typing import List, Optional, TYPE_CHECKING, Tuple + +import numpy as np +from scipy.sparse import csr_matrix +from scipy.linalg import circulant +from sklearn.neighbors import NearestNeighbors + +if TYPE_CHECKING: + from cleanlab.typing import FeatureArray, Metric + +from cleanlab.internal.neighbor.metric import decide_default_metric +from cleanlab.internal.neighbor.search import construct_knn + + +DEFAULT_K = 10 +"""Default number of neighbors to consider in the k-nearest neighbors search, +unless the size of the feature array is too small or the user specifies a different value. + +This should be the largest desired value of k for all desired issue types that require a KNN graph. + +E.g. if near duplicates wants k=1 but outliers wants 10, then DEFAULT_K should be 10. This way, all issue types can rely on the same KNN graph. +""" + + +def features_to_knn( + features: Optional[FeatureArray], + *, + n_neighbors: Optional[int] = None, + metric: Optional[Metric] = None, + **sklearn_knn_kwargs, +) -> NearestNeighbors: + """Build and fit a k-nearest neighbors search object from an array of numerical features. + + Parameters + ---------- + features : + The input feature array, with shape (N, M), where N is the number of samples and M is the number of features. + n_neighbors : + The number of nearest neighbors to consider. If None, a default value is determined based on the feature array size. + metric : + The distance metric to use for computing distances between points. If None, the metric is determined based on the feature array shape. + **sklearn_knn_kwargs : + Additional keyword arguments to be passed to the search index constructor. + + Returns + ------- + knn : + A k-nearest neighbors search object fitted to the input feature array. + + Examples + -------- + + >>> import numpy as np + >>> from cleanlab.internal.neighbor import features_to_knn + >>> features = np.random.rand(100, 10) + >>> knn = features_to_knn(features) + >>> knn + NearestNeighbors(metric='cosine', n_neighbors=10) + """ + if features is None: + raise ValueError("Both knn and features arguments cannot be None at the same time.") + # Use provided metric if available, otherwise decide based on the features. + metric = metric or decide_default_metric(features) + + # Decide the number of neighbors to use in the KNN search. + n_neighbors = _configure_num_neighbors(features, n_neighbors) + + knn = construct_knn(n_neighbors, metric, **sklearn_knn_kwargs) + return knn.fit(features) + + +def construct_knn_graph_from_index( + knn: NearestNeighbors, + correction_features: Optional[FeatureArray] = None, +) -> csr_matrix: + """Construct a sparse distance matrix representation of KNN graph out of a fitted NearestNeighbors search object. + + Parameters + ---------- + knn : + A NearestNeighbors object that has been fitted to a feature array. + The KNN graph is constructed based on the distances and indices of each feature row's nearest neighbors. + correction_features : + The input feature array used to fit the NearestNeighbors object. + If provided, the function the distances and indices of the neighbors will be corrected based on exact + duplicates in the feature array. + If not provided, no correction will be applied. + + Warning + ------- + This function is designed to handle a specific case where a KNN index is used to construct a KNN graph by itself, + and there is a need to detect and correct for exact duplicates in the feature array. However, relying on this + function for such corrections is generally discouraged. There are other functions in the module that handle + KNN graph construction with feature corrections in a more flexible and robust manner. Use this function only + when there is a special need to correct distances and indices based on the feature array provided. + + Returns + ------- + knn_graph : + A sparse, weighted adjacency matrix representing the KNN graph of the feature array. + + Note + ---- + This is *not* intended to construct a KNN graph of test data. It is only used to construct a KNN graph of the data used to fit the NearestNeighbors object. + + Examples + -------- + >>> import numpy as np + >>> from cleanlab.internal.neighbor.knn_graph import features_to_knn, construct_knn_graph_from_index + >>> features = np.array([ + ... [0.701, 0.701], + ... [0.900, 0.436], + ... [0.000, 1.000], + ... ]) + >>> knn = features_to_knn(features, n_neighbors=1) + >>> knn_graph = construct_knn_graph_from_index(knn) + >>> knn_graph.toarray() # For demonstration purposes only. It is generally a bad idea to transform to dense matrix for large graphs. + array([[0. , 0.33140006, 0. ], + [0.33140006, 0. , 0. ], + [0.76210367, 0. , 0. ]]) + """ + + # Perform self-querying to get the distances and indices of the nearest neighbors + distances, indices = knn.kneighbors(X=None, return_distance=True) + + # Correct the distances and indices if the correction_features array is provided + if correction_features is not None: + distances, indices = correct_knn_distances_and_indices( + features=correction_features, distances=distances, indices=indices + ) + + N, K = distances.shape + + # Pointers to the row elements distances[indptr[i]:indptr[i+1]], + # and their corresponding column indices indices[indptr[i]:indptr[i+1]]. + indptr = np.arange(0, N * K + 1, K) + + return csr_matrix((distances.reshape(-1), indices.reshape(-1), indptr), shape=(N, N)) + + +def create_knn_graph_and_index( + features: Optional[FeatureArray], + *, + n_neighbors: Optional[int] = None, + metric: Optional[Metric] = None, + correct_exact_duplicates: bool = True, + **sklearn_knn_kwargs, +) -> Tuple[csr_matrix, NearestNeighbors]: + """Calculate the KNN graph from the features if it is not provided in the kwargs. + + Parameters + ---------- + features : + The input feature array, with shape (N, M), where N is the number of samples and M is the number of features. + n_neighbors : + The number of nearest neighbors to consider. If None, a default value is determined based on the feature array size. + metric : + The distance metric to use for computing distances between points. If None, the metric is determined based on the feature array shape. + correct_exact_duplicates : + Whether to correct the KNN graph to ensure that exact duplicates have zero mutual distance, and they are correctly included in the KNN graph. + **sklearn_knn_kwargs : + Additional keyword arguments to be passed to the search index constructor. + + Raises + ------ + ValueError : + If `features` is None, as it's required to construct a KNN graph from scratch. + + Returns + ------- + knn_graph : + A sparse, weighted adjacency matrix representing the KNN graph of the feature array. + knn : + A k-nearest neighbors search object fitted to the input feature array. This object can be used to query the nearest neighbors of new data points. + + Examples + -------- + >>> import numpy as np + >>> from cleanlab.internal.neighbor.knn_graph import create_knn_graph_and_index + >>> features = np.array([ + ... [0.701, 0.701], + ... [0.900, 0.436], + ... [0.000, 1.000], + ... ]) + >>> knn_graph, knn = create_knn_graph_and_index(features, n_neighbors=1) + >>> knn_graph.toarray() # For demonstration purposes only. It is generally a bad idea to transform to dense matrix for large graphs. + array([[0. , 0.33140006, 0. ], + [0.33140006, 0. , 0. ], + [0.76210367, 0. , 0. ]]) + >>> knn + NearestNeighbors(metric=, n_neighbors=1) # For demonstration purposes only. The actual metric may vary. + """ + # Construct NearestNeighbors object + knn = features_to_knn(features, n_neighbors=n_neighbors, metric=metric, **sklearn_knn_kwargs) + # Build graph from NearestNeighbors object + knn_graph = construct_knn_graph_from_index(knn) + + # Ensure that exact duplicates found with np.unique aren't accidentally missed in the KNN graph + if correct_exact_duplicates: + assert features is not None + knn_graph = correct_knn_graph(features, knn_graph) + return knn_graph, knn + + +def correct_knn_graph(features: FeatureArray, knn_graph: csr_matrix) -> csr_matrix: + """ + Corrects a k-nearest neighbors (KNN) graph by handling exact duplicates in the feature array. + + This utility function takes a precomputed KNN graph and the corresponding feature array, + identifies sets of exact duplicate feature vectors, and corrects the KNN graph to properly + reflect these duplicates. The corrected KNN graph is returned as a sparse CSR matrix. + + Parameters + ---------- + features : np.ndarray + The input feature array, with shape (N, M), where N is the number of samples and M is the number of features. + knn_graph : csr_matrix + A sparse matrix of shape (N, N) representing the k-nearest neighbors graph. + The graph is expected to be in CSR (Compressed Sparse Row) format. + + Returns + ------- + csr_matrix + A corrected KNN graph in CSR format with adjusted distances and indices to properly handle + exact duplicates in the feature array. + + Notes + ----- + - This function assumes that the input `knn_graph` is already computed and provided in CSR format. + - The function modifies the KNN graph to ensure that exact duplicates are represented with zero distance + and correctly updated neighbor indices. + - This function is useful for post-processing a KNN graph when exact duplicates were not handled during + the initial KNN computation. + + """ + N = features.shape[0] + distances, indices = knn_graph.data.reshape(N, -1), knn_graph.indices.reshape(N, -1) + + corrected_distances, corrected_indices = correct_knn_distances_and_indices( + features, distances, indices + ) + N = features.shape[0] + return csr_matrix( + (corrected_distances.reshape(-1), corrected_indices.reshape(-1), knn_graph.indptr), + shape=(N, N), + ) + + +def _compute_exact_duplicate_sets(features: FeatureArray) -> List[np.ndarray]: + """ + Computes the sets of exact duplicate points in the feature array. + + This function groups indices of points that have identical feature vectors. + It returns a list of arrays, where each array contains the indices of points that are exact duplicates + of each other. + + Parameters + ---------- + features : np.ndarray + The input feature array, with shape (N, M), where N is the number of samples and M is the number of features. + + Returns + ------- + exact_duplicate_sets + A list of 1D arrays, where each array contains the indices of exact duplicate points in the dataset. + Only sets with two or more duplicates are included in the list. If no exact duplicates are found, an empty list is returned. + + Examples + -------- + >>> features = np.array([[1, 2], [3, 4], [1, 2], [5, 6], [3, 4]]) + >>> _compute_exact_duplicate_sets(features) + [array([0, 2]), array([1, 4])] # The row value [1, 2] appears in rows 0 and 2, and [3, 4] appears in rows 1 and 4. + + Notes + ----- + - This function uses `np.unique` to find unique feature vectors and their inverse indices. + - This function is intended to be used internally within this module. + """ + # Use np.unique to catch inverse indices of all unique feature sets + _, unique_inverse, unique_counts = np.unique( + features, return_inverse=True, return_counts=True, axis=0 + ) + + # Collect different sets of exact duplicates in the dataset + exact_duplicate_sets = [ + np.where(unique_inverse == u)[0] for u in set(unique_inverse) if unique_counts[u] > 1 + ] + + return exact_duplicate_sets + + +def correct_knn_distances_and_indices_with_exact_duplicate_sets_inplace( + distances: np.ndarray, + indices: np.ndarray, + exact_duplicate_sets: List[np.ndarray], +) -> None: + """ + Corrects the distances and indices arrays of k-nearest neighbors (KNN) graphs by handling sets + of exact duplicates explicitly. This function modifies the input arrays in-place. + + This function ensures that exact duplicates are correctly represented in the KNN graph. + It modifies the `distances` and `indices` arrays so that each set of exact duplicates + points to itself with zero distance, and adjusts the nearest neighbors accordingly. + + Parameters + ---------- + distances : + A 2D array of shape (N, k) representing the distances between each point of the N points and their k-nearest neighbors. + This array will be modified in-place to reflect the corrections for exact duplicates (whose mutual distances are explicitly set to zero). + indices : + A 2D array of shape (N, k) representing the indices of the nearest neighbors for each of the N points. + This array will be modified in-place to reflect the corrections for exact duplicates. + exact_duplicate_sets : + A list of 1D arrays, each containing the indices of points that are exact duplicates of each other. + These sets will be used to correct the KNN graph by ensuring that duplicates are reflected as nearest neighbors + with zero distance. + + High-Level Overview + ------------------- + The function operates in two main scenarios based on the size of the duplicate sets relative to k: + + 1. **Duplicate Set Size >= k + 1**: + - All nearest neighbors are exact duplicates. + - The `indices` array is updated such that the first k+1 entries for each duplicate set point are used to represent the nearest neighbors + of all points in the duplicate set. + - The rows of the `distances` array belonging to the duplicate set are set to zero. + + 2. **Duplicate Set Size < k + 1**: + - Some of the nearest neighbors are not exact duplicates. + - Non-duplicate neighbors are shifted to the back of the list. + - The `indices` and `distances` arrays are updated accordingly to reflect the duplicates at the front with zero distance. + + User Considerations + ------------------- + - **Input Validity**: Ensure that the `distances` and `indices` arrays have the correct shape and correspond to the same KNN graph. + - **In-Place Modifications**: The function modifies the input arrays directly. If the original data is needed, make a copy before calling the function. + - **Duplicate Set Size**: The function is optimized for cases where the number of exact duplicates can be larger than k. Ensure the duplicate sets are accurately identified. + - **Performance**: The function uses efficient NumPy operations, but performance can be affected by the size of the input arrays and the number of duplicate sets. + + Capabilities + ------------ + - Handles exact duplicate sets efficiently, ensuring correct KNN graph representation. + - Maintains zero distances for exact duplicates. + - Adjusts neighbor indices to reflect the presence of duplicates. + + Limitations + ----------- + - Assumes that the input arrays (`distances` and `indices`) come from a precomputed KNN graph. + - Does not handle near-duplicates or merge non-duplicate neighbors. + - Requires careful construction of `exact_duplicate_sets` to avoid misidentification. + """ + + # Number of neighbors + k = distances.shape[1] + + for duplicate_inds in exact_duplicate_sets: + # Determine the number of same points to include, respecting the limit of k + num_same = len(duplicate_inds) + num_same_included = min(num_same - 1, k) # ensure we do not exceed k neighbors + + sorted_first_k_duplicate_inds = _prepare_neighborhood_of_first_k_duplicates( + duplicate_inds, num_same_included + ) + + if num_same >= k + 1: + # All nearest neighbors are exact duplicates + + # We only pass in the ciruclant matrix of nearest neighbors + indices[duplicate_inds[: k + 1]] = sorted_first_k_duplicate_inds + # But the rest will just take the k first duplicate ids + indices[duplicate_inds[k + 1 :]] = duplicate_inds[:k] + + # Finally, set the distances between exact duplicates to zero + distances[duplicate_inds] = 0 + else: + # Some of the nearest neighbors aren't exact duplicates, move those to the back + + # Get indices and distances from knn that are not the same as i + different_point_mask = np.isin(indices[duplicate_inds], duplicate_inds, invert=True) + + # Get the indices of the first m True values in each row of the mask + true_indices = np.argsort(~different_point_mask, axis=1)[:, :-num_same_included] + + # Copy the values to the last m columns in dists + distances[duplicate_inds, -(k - num_same_included) :] = distances[ + duplicate_inds, true_indices.T + ].T + indices[duplicate_inds, -(k - num_same_included) :] = indices[ + duplicate_inds, true_indices.T + ].T + + # We can pass the circulant matrix to a slice + indices[duplicate_inds, :num_same_included] = sorted_first_k_duplicate_inds + + # Finally, set the distances between exact duplicates to zero + distances[duplicate_inds, :num_same_included] = 0 + + return None + + +def _prepare_neighborhood_of_first_k_duplicates(duplicate_inds, num_same_included): + """ + Prepare a matrix representing the neighborhoods of duplicate items. + + This function constructs a matrix where each row corresponds to an item + and contains the indices of its nearest neighbors (excluding itself), up + to a specified number `k`. + + Parameters: + ----------- + duplicate_inds : list + A list of indices that represent duplicate items. + + num_same_included : int + An integer `k` representing the number of neighbors to include for + each item. + + Returns: + -------- + np.ndarray + A matrix where each row contains the sorted indices of the nearest + neighbors for the corresponding item. + + Explanation: + ------------ + 1. Extract the Base for the Circulant Matrix: + - The function extracts the first `k+1` elements from `duplicate_inds` + to form the base of the circulant matrix. This approach ensures that + even if the set of duplicate items is larger, we only need to consider + the first `k` duplicates as the nearest neighbors, avoiding conflicts + with the items themselves. + + 2. Create the Circulant Matrix: + - A circulant matrix is generated from the base, where each row is a + cyclic permutation of the previous row. + + 3. Slice the Matrix to Exclude the First Column: + - The first column is removed to ensure each row represents the neighbors + without including the item itself. + + 4. Sort the Neighborhood Indices: + - The rows of the sliced matrix are sorted to ensure a consistent order + of neighbors. + + Example: + -------- + Given a set of 5 duplicate items `[A, B, C, D, E]` and `k=2`, the function + processes this as follows: + + 1. `circulant_base` for `k=2` would be `[A, B, C]`. + 2. The `circulant_matrix` might look like: + ``` + [A B C] + [B C A] + [C A B] + ``` + 3. Removing the first column results in: + ``` + [B C] + [C A] + [A B] + ``` + 4. Sorting each row gives the final matrix: + ``` + [B C] + [A C] + [A B] + ``` + + This matrix indicates that: + - The nearest neighbors of `A` are `[B, C]`. + - The nearest neighbors of `B` are `[A, C]`. + - The nearest neighbors of `C` are `[A, B]`. + + For `k=2`, the neighbors of `D`, `E`, onwards could be any of the above. + + The function constructs a sorted matrix of nearest neighbors for a list of + duplicate items, ensuring an equal distribution of neighbors up to a specified + number `k`. This process is necessary for tasks requiring an understanding of + the local neighborhood structure among duplicate examples. By using only the first + `k+1` elements, the function avoids the need to construct a larger circulant + matrix, simplifying the computation and ensuring no conflicts among the rest of the items. + """ + circulant_base = duplicate_inds[: num_same_included + 1] + circulant_matrix = circulant(circulant_base) + sliced_circulant_matrix = circulant_matrix[:, 1:] + sorted_first_k_duplicate_inds = np.sort(sliced_circulant_matrix, axis=1) + return sorted_first_k_duplicate_inds + + +def correct_knn_distances_and_indices( + features: FeatureArray, + distances: np.ndarray, + indices: np.ndarray, + exact_duplicate_sets: Optional[List[np.ndarray]] = None, +) -> tuple[np.ndarray, np.ndarray]: + """ + Corrects the distances and indices of a k-nearest neighbors (KNN) graph + based on all exact duplicates detected in the feature array. + + Parameters + ---------- + features : + The feature array used to construct the KNN graph. + distances : + The distances between each point and its k nearest neighbors. + indices : + The indices of the k nearest neighbors for each point. + exact_duplicate_sets: + A list of numpy arrays, where each array contains the indices of exact duplicates in the feature array. If not provided, it will be computed from the feature array. + + Returns + ------- + corrected_distances : + The corrected distances between each point and its k nearest neighbors. Exact duplicates (based on the feature array) are ensured to have zero mutual distance. + corrected_indices : + The corrected indices of the k nearest neighbors for each point. Exact duplicates are ensured to be included in the k nearest neighbors, unless the number of exact duplicates exceeds k. + + Example + ------- + >>> import numpy as np + >>> X = np.array( + ... [ + ... [0, 0], + ... [0, 0], # Exact duplicate of the previous point + ... [1, 1], # The distances between this point and the others is sqrt(2) (equally distant from both) + ... ] + ... ) + >>> distances = np.array( # Distance to the 1-NN of each point + ... [ + ... [np.sqrt(2)], # Should be [0] + ... [1e-16], # Should be [0] + ... [np.sqrt(2)], + ... ] + ... ) + >>> indices = np.array( # Index of the 1-NN of each point + ... [ + ... [2], # Should be [1] + ... [0], + ... [1], # Might be [0] or [1] + ... ] + ... ) + >>> corrected_distances, corrected_indices = correct_knn_distances_and_indices(X, distances, indices) + >>> corrected_distances + array([[0.], [0.], [1.41421356]]) + >>> corrected_indices + array([[1], [0], [0]]) + """ + + if exact_duplicate_sets is None: + exact_duplicate_sets = _compute_exact_duplicate_sets(features) + + # Prepare the output arrays + corrected_distances = np.copy(distances) + corrected_indices = np.copy(indices) + + correct_knn_distances_and_indices_with_exact_duplicate_sets_inplace( + distances=corrected_distances, + indices=corrected_indices, + exact_duplicate_sets=exact_duplicate_sets, + ) + + return corrected_distances, corrected_indices + + +def _configure_num_neighbors(features: FeatureArray, k: Optional[int]): + # Error if the provided value is greater or equal to the number of examples. + N = features.shape[0] + k_larger_than_dataset = k is not None and k >= N + if k_larger_than_dataset: + raise ValueError( + f"Number of nearest neighbors k={k} cannot exceed the number of examples N={len(features)} passed into the estimator (knn)." + ) + + # Either use the provided value or select a default value based on the feature array size. + k = k or min(DEFAULT_K, N - 1) + return k diff --git a/cleanlab/internal/neighbor/metric.py b/cleanlab/internal/neighbor/metric.py new file mode 100644 index 0000000..321be90 --- /dev/null +++ b/cleanlab/internal/neighbor/metric.py @@ -0,0 +1,107 @@ +from scipy.spatial.distance import euclidean + +from cleanlab.typing import FeatureArray, Metric + +HIGH_DIMENSION_CUTOFF: int = 3 +""" +If the number of columns (M) in the `features` array is greater than this cutoff value, +then by default, K-nearest-neighbors will use the "cosine" metric. +The cosine metric is more suitable for high-dimensional data. +Otherwise the "euclidean" distance will be used. + +""" +ROW_COUNT_CUTOFF: int = 100 +""" +Only affects settings where Euclidean metrics would be used by default. +If the number of rows (N) in the `features` array is greater than this cutoff value, +then by default, Euclidean distances are computed via the "euclidean" metric +(implemented in sklearn for efficiency reasons). +Otherwise, Euclidean distances are by default computed via +the ``euclidean`` metric from scipy (slower but numerically more precise/accurate). +""" + + +# Metric decision functions +def _euclidean_large_dataset() -> str: + return "euclidean" + + +def _euclidean_small_dataset() -> Metric: + return euclidean + + +def _cosine_metric() -> str: + return "cosine" + + +def decide_euclidean_metric(features: FeatureArray) -> Metric: + """ + Decide the appropriate Euclidean metric implementation based on the size of the dataset. + + Parameters + ---------- + features : + The input features array. + + Returns + ------- + metric : + A string or a callable representing a specific implementation of computing the euclidean distance. + + Note + ---- + A choice is made between two implementations + of the euclidean metric based on the number of rows in the feature array. + If the number of rows (N) in the feature array is greater than another predefined + cutoff value (ROW_COUNT_CUTOFF), the ``"euclidean"`` metric is used. This + is because the euclidean metric performs better on larger datasets. + If neither condition is met, the ``euclidean`` metric function from scipy is returned. + + See also + -------- + ROW_COUNT_CUTOFF: The cutoff value for the number of rows in the feature array. + sklearn.metrics.pairwise.euclidean_distances: The euclidean metric function from scikit-learn. + scipy.spatial.distance.euclidean: The euclidean metric function from scipy. + """ + num_rows = features.shape[0] + if num_rows > ROW_COUNT_CUTOFF: + return _euclidean_large_dataset() + else: + return _euclidean_small_dataset() + + +# Main function to decide the metric +def decide_default_metric(features: FeatureArray) -> Metric: + """ + Decide the KNN metric to be used based on the shape of the feature array. + + Parameters + ---------- + features : + The input feature array, with shape (N, M), where N is the number of samples and M is the number of features. + + Returns + ------- + metric : + The distance metric to be used for neighbor search. It can be either a string + representing the metric name ("cosine" or "euclidean") or a callable + representing the metric function from scipy (euclidean). + + Note + ---- + The decision of which metric to use is based on the shape of the feature array. + If the number of columns (M) in the feature array is greater than a predefined + cutoff value (HIGH_DIMENSION_CUTOFF), the "cosine" metric is used. This is because the cosine + metric is more suitable for high-dimensional data. + + Otherwise, a euclidean metric is used. + That is handled by the :py:meth:`~cleanlab.internal.neighbor.metric.decide_euclidean_metric` function. + + See Also + -------- + HIGH_DIMENSION_CUTOFF: The cutoff value for the number of columns in the feature array. + sklearn.metrics.pairwise.cosine_distances: The cosine metric function from scikit-learn + """ + if features.shape[1] > HIGH_DIMENSION_CUTOFF: + return _cosine_metric() + return decide_euclidean_metric(features) diff --git a/cleanlab/internal/neighbor/search.py b/cleanlab/internal/neighbor/search.py new file mode 100644 index 0000000..cf4bb0a --- /dev/null +++ b/cleanlab/internal/neighbor/search.py @@ -0,0 +1,75 @@ +from __future__ import annotations +from typing import TYPE_CHECKING + +from sklearn.neighbors import NearestNeighbors + + +if TYPE_CHECKING: + + from cleanlab.typing import Metric + + +def construct_knn(n_neighbors: int, metric: Metric, **knn_kwargs) -> NearestNeighbors: + """ + Constructs a k-nearest neighbors search object. You can implement a similar method to run cleanlab with your own approximate-KNN library. + + Parameters + ---------- + n_neighbors : + The number of nearest neighbors to consider. + metric : + The distance metric to use for computing distances between points. + See :py:mod:`~cleanlab.internal.neighbor.metric` for more information. + **knn_kwargs: + Additional keyword arguments to be passed to the search index constructor. + See https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.NearestNeighbors.html for more details on the available options. + + Returns + ------- + knn : + A k-nearest neighbors search object compatible with the scikit-learn NearestNeighbors class interface. + + Implements: + + - `fit` method: Accepts a feature array `X` to fit the model. + This enables subsequent neighbor searches on the data. + - `kneighbors` method: Finds the K-neighbors of a point, returning distances and indices of the k-nearest neighbors. Handles two scenarios: + 1. When a query array `features: np.ndarray` is provided, it returns the distances and indices for each point in the query array. + 2. When no query array is provided (`features = None`), it returns neighbors for each indexed point without considering the query point as its own neighbor. + Optionally, allows re-specification of the number of neighbors for each query point, defaulting to the constructor's value if not specified. + + Attributes: + + - `n_neighbors`: Number of neighbors to consider. + - `metric`: Distance metric used to compute distances between points. + - `metric_params`: Additional parameters for the distance metric function. + + Optional: + + - `kneighbors_graph` method: Not required but can be implemented for convenience. + Responsibility shifted to :py:ref:`construct_knn_graph_from_index `. + + Fitted Attributes: + + - `n_features_in_`: Number of features observed during fit. + - `effective_metric_params_`: Metric parameters used in distance computation. + - `effective_metric_`: Metric used for computing distances to neighbors. + - `n_samples_fit_`: Number of samples in the fitted data. + + Additional: + + - `__sklearn_is_fitted__`: Method returning a boolean indicating if the object is fitted, + useful for conducting an is_fitted validation, which verifies the presence of fitted attributes (typically ending with a trailing underscore). + + + The above specifications ensure compatibility and provide a clear directive for developers needing to integrate alternative k-nearest neighbors implementations or modify existing functionalities. + + Note + ---- + The `metric` argument should be a callable that takes two arguments (the two points) and returns the distance between them. + The additional keyword arguments (`**knn_kwargs`) are passed directly to the underlying k-nearest neighbors search algorithm. + + """ + sklearn_knn = NearestNeighbors(n_neighbors=n_neighbors, metric=metric, **knn_kwargs) + + return sklearn_knn diff --git a/cleanlab/internal/numerics.py b/cleanlab/internal/numerics.py new file mode 100644 index 0000000..3b3956c --- /dev/null +++ b/cleanlab/internal/numerics.py @@ -0,0 +1,39 @@ +from typing import Optional +import numpy as np + +from cleanlab.internal.constants import EPSILON + + +def softmax( + x: np.ndarray, temperature: float = 1.0, axis: Optional[int] = None, shift: bool = False +) -> np.ndarray: + """Softmax function. + + Parameters + ---------- + x : np.ndarray + Input array. + + temperature : float + Temperature of the softmax function. + + axis : Optional[int] + Axis to apply the softmax function. If None, the softmax function is + applied to all elements of the input array. + + shift : bool + Whether to shift the input array before applying the softmax function. + This is useful to avoid numerical issues when the input array contains + large values, that could result in overflows when applying the exponential + function. + + Returns + ------- + np.ndarray + Softmax function applied to the input array. + """ + x = x / max(temperature, EPSILON) + if shift: + x = x - np.max(x, axis=axis, keepdims=True) + exp_x = np.exp(x) + return exp_x / np.sum(exp_x, axis=axis, keepdims=True) diff --git a/cleanlab/internal/object_detection_utils.py b/cleanlab/internal/object_detection_utils.py new file mode 100644 index 0000000..9f6140d --- /dev/null +++ b/cleanlab/internal/object_detection_utils.py @@ -0,0 +1,89 @@ +""" +Helper functions used internally for object detection tasks. +""" + +from typing import Any, Dict, List, Optional + + +import numpy as np + +from cleanlab.internal.numerics import softmax + + +def bbox_xyxy_to_xywh(bbox: List[float]) -> Optional[List[float]]: + """Converts bounding box coodrinate types from x1y1,x2y2 to x,y,w,h""" + if len(bbox) == 4: + x1, y1, x2, y2 = bbox + w = x2 - x1 + h = y2 - y1 + return [x1, y1, w, h] + else: + print("Wrong bbox shape", len(bbox)) + return None + + +def softmin1d(scores: np.ndarray, temperature: float = 0.99, axis: int = 0) -> float: + """Returns softmin of passed in scores.""" + scores = np.array(scores) + softmax_scores = softmax(x=-1 * scores, temperature=temperature, axis=axis, shift=True) + return np.dot(softmax_scores, scores) + + +def calculate_bounding_box_areas(rectangle_coordinates): + """Calculate areas of bounding boxes represented by numpy array with x1, y1, x2, y2 format""" + widths = rectangle_coordinates[:, 2] - rectangle_coordinates[:, 0] + heights = rectangle_coordinates[:, 3] - rectangle_coordinates[:, 1] + return widths * heights + + +def assert_valid_aggregation_weights(aggregation_weights: Dict[str, Any]) -> None: + """assert aggregation weights are in the proper format""" + weights = np.array(list(aggregation_weights.values())) + if (not np.isclose(np.sum(weights), 1.0)) or (np.min(weights) < 0.0): + raise ValueError( + f"""Aggregation weights should be non-negative and must sum to 1.0 + """ + ) + + +def assert_valid_inputs( + labels: List[Dict[str, Any]], + predictions, + method: Optional[str] = None, + threshold: Optional[float] = None, +): + """Asserts proper input format.""" + if len(labels) != len(predictions): + raise ValueError( + f"labels and predictions length needs to match. len(labels) == {len(labels)} while len(predictions) == {len(predictions)}." + ) + # Typecheck labels and predictions + if not isinstance(labels[0], dict): + raise ValueError( + f"Labels has to be a list of dicts. Instead it is list of {type(labels[0])}." + ) + # check last column of predictions is probabilities ( < 1.)? + if not isinstance(predictions[0], (list, np.ndarray)): + raise ValueError( + f"Prediction has to be a list or np.ndarray. Instead it is type {type(predictions[0])}." + ) + if not predictions[0][0].shape[1] == 5: + raise ValueError( + f"Prediction values have to be of format [x1,y1,x2,y2,pred_prob]. Please refer to the documentation for predicted probabilities under object_detection.rank.get_label_quality_scores for details" + ) + + valid_methods = ["objectlab"] + if method is not None and method not in valid_methods: + raise ValueError( + f""" + {method} is not a valid object detection scoring method! + Please choose a valid scoring_method: {valid_methods} + """ + ) + + if threshold is not None and threshold > 1.0: + raise ValueError( + f""" + Threshold is a cutoff of predicted probabilities and therefore should be <= 1. + """ + ) diff --git a/cleanlab/internal/outlier.py b/cleanlab/internal/outlier.py new file mode 100644 index 0000000..1f63482 --- /dev/null +++ b/cleanlab/internal/outlier.py @@ -0,0 +1,112 @@ +""" +Helper functions used internally for outlier detection tasks. +""" + +from typing import Optional +import numpy as np + +from cleanlab.internal.constants import EPSILON + + +def transform_distances_to_scores( + avg_distances: np.ndarray, t: int, scaling_factor: float +) -> np.ndarray: + """Returns an outlier score for each example based on its average distance to its k nearest neighbors. + + The transformation of a distance, :math:`d` , to a score, :math:`o` , is based on the following formula: + + .. math:: + o = \\exp\\left(-dt\\right) + + where :math:`t` scales the distance to a score in the range [0,1]. + + Parameters + ---------- + avg_distances : np.ndarray + An array of distances of shape ``(N)``, where N is the number of examples. + Each entry represents an example's average distance to its k nearest neighbors. + + t : int + A sensitivity parameter that modulates the strength of the transformation from distances to scores. + Higher values of `t` result in more pronounced differentiation between the scores of examples + lying in the range [0,1]. + + scaling_factor : float + A scaling factor used to normalize the distances before they are converted into scores. A valid + scaling factor is any positive number. The choice of scaling factor should be based on the + distribution of distances between neighboring examples. A good rule of thumb is to set the + scaling factor to the median distance between neighboring examples. A lower scaling factor + results in more pronounced differentiation between the scores of examples lying in the range [0,1]. + + Returns + ------- + ood_features_scores : np.ndarray + An array of outlier scores of shape ``(N,)`` for N examples. + + Examples + -------- + >>> import numpy as np + >>> from cleanlab.outlier import transform_distances_to_scores + >>> distances = np.array([[0.0, 0.1, 0.25], + ... [0.15, 0.2, 0.3]]) + >>> avg_distances = np.mean(distances, axis=1) + >>> transform_distances_to_scores(avg_distances, t=1, scaling_factor=1) + array([0.88988177, 0.80519832]) + """ + # Map ood_features_scores to range 0-1 with 0 = most concerning + return np.exp(-t * avg_distances / max(scaling_factor, EPSILON)) + + +def correct_precision_errors( + scores: np.ndarray, + avg_distances: np.ndarray, + metric: str, + C: int = 100, + p: Optional[int] = None, +): + """ + Ensure that scores where avg_distances are below the tolerance threshold get a score of one. + + Parameters + ---------- + scores : + An array of scores of shape ``(N)``, where N is the number of examples. + Each entry represents a score between 0 and 1. + + avg_distances : + An array of distances of shape ``(N)``, where N is the number of examples. + Each entry represents an example's average distance to its k nearest neighbors. + + metric : + The metric used by the knn algorithm to calculate the distances. + It must be 'cosine', 'euclidean' or 'minkowski', otherwise this function does nothing. + + C : + Multiplier used to increase the tolerance of the acceptable precision differences. + It is a multiplicative factor of the machine epsilon that is used to calculate the tolerance. + For the type of values that are used in the distances, a value of 100 should be a sensible + default value for small values of the distances, below the order of 1. + + p : + This value is only used when metric is 'minkowski'. + A ValueError will be raised if metric is 'minkowski' and 'p' was not provided. + + Returns + ------- + fixed_scores : + An array of scores of shape ``(N,)`` for N examples with scores between 0 and 1. + """ + if metric == "cosine": + tolerance = C * np.finfo(np.float64).epsneg + elif metric == "euclidean": + tolerance = np.sqrt(C * np.finfo(np.float64).eps) + elif metric == "minkowski": + if p is None: + raise ValueError("When metric is 'minkowski' you must specify the 'p' parameter") + tolerance = (C * np.finfo(np.float64).eps) ** (1 / p) + else: + return scores + + candidates_mask = avg_distances < tolerance + scores[candidates_mask] = 1 + return scores diff --git a/cleanlab/internal/regression_utils.py b/cleanlab/internal/regression_utils.py new file mode 100644 index 0000000..cae1c10 --- /dev/null +++ b/cleanlab/internal/regression_utils.py @@ -0,0 +1,113 @@ +""" +Helper functions internally used in cleanlab.regression. +""" + +import numpy as np +import pandas as pd +from numpy.typing import ArrayLike +from typing import Tuple, Union + + +def assert_valid_prediction_inputs( + labels: ArrayLike, + predictions: ArrayLike, + method: str, +) -> Tuple[np.ndarray, np.ndarray]: + """Checks that ``labels``, ``predictions``, ``method`` are correctly formatted.""" + + # Load array_like input as numpy array. If not raise error. + try: + labels = np.asarray(labels) + except: + raise ValueError(f"labels must be array_like.") + + try: + predictions = np.asarray(predictions) + except: + raise ValueError(f"predictions must be array_like.") + + # Check if labels and predictions are 1-D and numeric + valid_labels = check_dimension_and_datatype(check_input=labels, text="labels") + valid_predictions = check_dimension_and_datatype(check_input=predictions, text="predictions") + + # Check if number of examples are same. + assert ( + valid_labels.shape == valid_predictions.shape + ), f"Number of examples in labels {labels.shape} and predictions {predictions.shape} are not same." + + # Check if inputs have missing values + check_missing_values(valid_labels, text="labels") + check_missing_values(valid_predictions, text="predictions") + + # Check if method is among allowed scoring method + scoring_methods = ["residual", "outre"] + if method not in scoring_methods: + raise ValueError(f"Specified method '{method}' must be one of: {scoring_methods}.") + + # return 1-D numpy array + return valid_labels, valid_predictions + + +def assert_valid_regression_inputs( + X: Union[np.ndarray, pd.DataFrame], + y: ArrayLike, +) -> Tuple[np.ndarray, np.ndarray]: + """ + Checks that regression inputs are properly formatted and returns the inputs in numpy array format. + """ + try: + X = np.asarray(X) + except: + raise ValueError(f"X must be array_like.") + + y = check_dimension_and_datatype(y, "y") + check_missing_values(y, text="y") + + if len(X) != len(y): + raise ValueError("X and y must have same length.") + + return X, y + + +def check_dimension_and_datatype(check_input: ArrayLike, text: str) -> np.ndarray: + """ + Raises errors related to: + 1. If input is empty + 2. If input is not 1-D + 3. If input is not numeric + + If all the checks are passed, it returns the squeezed 1-D array required by the main algorithm. + """ + + try: + check_input = np.asarray(check_input) + except: + raise ValueError(f"{text} could not be converted to numpy array, check input.") + + # Check if input is empty + if not check_input.size: + raise ValueError(f"{text} cannot be empty array.") + + # Remove axis with length one + check_input = np.squeeze(check_input) + + # Check if input is 1-D + if check_input.ndim != 1: + raise ValueError( + f"Expected 1-Dimensional inputs for {text}, got {check_input.ndim} dimensions." + ) + + # Check if datatype is numeric + if not np.issubdtype(check_input.dtype, np.number): + raise ValueError( + f"Expected {text} to contain numeric values, got values of type {check_input.dtype}." + ) + + return check_input + + +def check_missing_values(check_input: np.ndarray, text: str): + """Raise error if there are any missing values in Numpy array.""" + + if np.isnan(check_input).any(): + raise ValueError(f"{text} cannot contain missing values.") diff --git a/cleanlab/internal/segmentation_utils.py b/cleanlab/internal/segmentation_utils.py new file mode 100644 index 0000000..2ae34be --- /dev/null +++ b/cleanlab/internal/segmentation_utils.py @@ -0,0 +1,58 @@ +""" +Helper functions used internally for segmentation tasks. +""" + +from typing import Optional, List + +import numpy as np + + +def _get_valid_optional_params( + batch_size: Optional[int] = None, + n_jobs: Optional[int] = None, +): + """Takes in optional args and returns good values for them if they are None.""" + if batch_size is None: + batch_size = 10000 + + if batch_size <= 0: + raise ValueError(f"Batch size must be greater than 0, got {batch_size}") + return batch_size, n_jobs + + +def _get_summary_optional_params( + class_names: Optional[List[str]] = None, + exclude: Optional[List[int]] = None, + top: Optional[int] = None, +): + """Takes in optional args and returns good values for them if they are None for summary functions.""" + if exclude is None: + exclude = [] + if top is None: + top = 20 + return class_names, exclude, top + + +def _check_input(labels: np.ndarray, pred_probs: np.ndarray) -> None: + """ + Checks that the input labels and predicted probabilities are valid. + + Parameters + ---------- + labels: + Array of shape ``(N, H, W)`` of integer labels, where `N` is the number of images in the dataset and `H` and `W` are the height and width of the images. + + pred_probs: + Array of shape ``(N, K, H, W)`` of predicted probabilities, where `N` is the number of images in the dataset, `K` is the number of classes, and `H` and `W` are the height and width of the images. + """ + if len(labels.shape) != 3: + raise ValueError("labels must have a shape of (N, H, W)") + + if len(pred_probs.shape) != 4: + raise ValueError("pred_probs must have a shape of (N, K, H, W)") + + num_images, height, width = labels.shape + num_images_pred, num_classes, height_pred, width_pred = pred_probs.shape + + if num_images != num_images_pred or height != height_pred or width != width_pred: + raise ValueError("labels and pred_probs must have matching dimensions for N, H, and W") diff --git a/cleanlab/internal/token_classification_utils.py b/cleanlab/internal/token_classification_utils.py new file mode 100644 index 0000000..0c13147 --- /dev/null +++ b/cleanlab/internal/token_classification_utils.py @@ -0,0 +1,276 @@ +""" +Helper methods used internally in cleanlab.token_classification +""" + +from __future__ import annotations + +import re +import string +import numpy as np +from termcolor import colored +from typing import List, Optional, Callable, Tuple, TypeVar, TYPE_CHECKING + +if TYPE_CHECKING: # pragma: no cover + import numpy.typing as npt + + T = TypeVar("T", bound=npt.NBitBase) + + +def get_sentence(words: List[str]) -> str: + """ + Get sentence formed by a list of words with minor processing for readability + + Parameters + ---------- + words: + list of word-level tokens + + Returns + ---------- + sentence: + sentence formed by list of word-level tokens + + Examples + -------- + >>> from cleanlab.internal.token_classification_utils import get_sentence + >>> words = ["This", "is", "a", "sentence", "."] + >>> get_sentence(words) + 'This is a sentence.' + """ + sentence = "" + for word in words: + if word not in string.punctuation or word in ["-", "("]: + word = " " + word + sentence += word + sentence = sentence.replace(" '", "'").replace("( ", "(").strip() + return sentence + + +def filter_sentence( + sentences: List[str], + condition: Optional[Callable[[str], bool]] = None, +) -> Tuple[List[str], List[bool]]: + """ + Filter sentence based on some condition, and returns filter mask + + Parameters + ---------- + sentences: + list of sentences + + condition: + sentence filtering condition + + Returns + --------- + sentences: + list of sentences filtered + + mask: + boolean mask such that `mask[i] == True` if the i'th sentence is included in the + filtered sentence, otherwise `mask[i] == False` + + Examples + -------- + >>> from cleanlab.internal.token_classification_utils import filter_sentence + >>> sentences = ["Short sentence.", "This is a longer sentence."] + >>> condition = lambda x: len(x.split()) > 2 + >>> long_sentences, _ = filter_sentence(sentences, condition) + >>> long_sentences + ['This is a longer sentence.'] + >>> document = ["# Headline", "Sentence 1.", "&", "Sentence 2."] + >>> sentences, mask = filter_sentence(document) + >>> sentences, mask + (['Sentence 1.', 'Sentence 2.'], [False, True, False, True]) + """ + if not condition: + condition = lambda sentence: len(sentence) > 1 and "#" not in sentence + mask = list(map(condition, sentences)) + sentences = [sentence for m, sentence in zip(mask, sentences) if m] + return sentences, mask + + +def process_token(token: str, replace: List[Tuple[str, str]] = [("#", "")]) -> str: + """ + Replaces special characters in the tokens + + Parameters + ---------- + token: + token which potentially contains special characters + + replace: + list of tuples `(s1, s2)`, where all occurances of s1 are replaced by s2 + + Returns + --------- + processed_token: + processed token whose special character has been replaced + + Note + ---- + Only applies to characters in the original input token. + + Examples + -------- + >>> from cleanlab.internal.token_classification_utils import process_token + >>> token = "#Comment" + >>> process_token("#Comment") + 'Comment' + + Specify custom replacement rules + + >>> replace = [("C", "a"), ("a", "C")] + >>> process_token("Cleanlab", replace) + 'aleCnlCb' + """ + replace_dict = {re.escape(k): v for (k, v) in replace} + pattern = "|".join(replace_dict.keys()) + compiled_pattern = re.compile(pattern) + replacement = lambda match: replace_dict[re.escape(match.group(0))] + processed_token = compiled_pattern.sub(replacement, token) + return processed_token + + +def mapping(entities: List[int], maps: List[int]) -> List[int]: + """ + Map a list of entities to its corresponding entities + + Parameters + ---------- + entities: + a list of given entities + + maps: + a list of mapped entities, such that the i'th indexed token should be mapped to `maps[i]` + + Returns + --------- + mapped_entities: + a list of mapped entities + + Examples + -------- + >>> unique_identities = [0, 1, 2, 3, 4] # ["O", "B-PER", "I-PER", "B-LOC", "I-LOC"] + >>> maps = [0, 1, 1, 2, 2] # ["O", "PER", "PER", "LOC", "LOC"] + >>> mapping(unique_identities, maps) + [0, 1, 1, 2, 2] # ["O", "PER", "PER", "LOC", "LOC"] + >>> mapping([0, 0, 4, 4, 3, 4, 0, 2], maps) + [0, 0, 2, 2, 2, 2, 0, 1] # ["O", "O", "LOC", "LOC", "LOC", "LOC", "O", "PER"] + """ + f = lambda x: maps[x] + return list(map(f, entities)) + + +def merge_probs( + probs: npt.NDArray["np.floating[T]"], maps: List[int] +) -> npt.NDArray["np.floating[T]"]: + """ + Merges model-predictive probabilities with desired mapping + + Parameters + ---------- + probs: + A 2D np.array of shape `(N, K)`, where N is the number of tokens, and K is the number of classes for the model + + maps: + a list of mapped index, such that the probability of the token being in the i'th class is mapped to the + `maps[i]` index. If `maps[i] == -1`, the i'th column of `probs` is ignored. If `np.any(maps == -1)`, the + returned probability is re-normalized. + + Returns + --------- + probs_merged: + A 2D np.array of shape ``(N, K')``, where `K'` is the number of new classes. Probabilities are merged and + re-normalized if necessary. + + Examples + -------- + >>> import numpy as np + >>> from cleanlab.internal.token_classification_utils import merge_probs + >>> probs = np.array([ + ... [0.55, 0.0125, 0.0375, 0.1, 0.3], + ... [0.1, 0.8, 0, 0.075, 0.025], + ... ]) + >>> maps = [0, 1, 1, 2, 2] + >>> merge_probs(probs, maps) + array([[0.55, 0.05, 0.4 ], + [0.1 , 0.8 , 0.1 ]]) + """ + old_classes = probs.shape[1] + map_size = np.max(maps) + 1 + probs_merged = np.zeros([len(probs), map_size], dtype=probs.dtype.type) + + for i in range(old_classes): + if maps[i] >= 0: + probs_merged[:, maps[i]] += probs[:, i] + if -1 in maps: + row_sums = probs_merged.sum(axis=1) + probs_merged /= row_sums[:, np.newaxis] + return probs_merged + + +def color_sentence(sentence: str, word: str) -> str: + """ + Searches for a given token in the sentence and returns the sentence where the given token is colored red + + Parameters + ---------- + sentence: + a sentence where the word is searched + + word: + keyword to find in `sentence`. Assumes the word exists in the sentence. + Returns + --------- + colored_sentence: + `sentence` where the every occurrence of the word is colored red, using ``termcolor.colored`` + + Examples + -------- + >>> from cleanlab.internal.token_classification_utils import color_sentence + >>> sentence = "This is a sentence." + >>> word = "sentence" + >>> color_sentence(sentence, word) + 'This is a \x1b[31msentence\x1b[0m.' + + Also works for multiple occurrences of the word + + >>> document = "This is a sentence. This is another sentence." + >>> word = "sentence" + >>> color_sentence(document, word) + 'This is a \x1b[31msentence\x1b[0m. This is another \x1b[31msentence\x1b[0m.' + """ + colored_word = colored(word, "red", force_color=True) + return _replace_sentence(sentence=sentence, word=word, new_word=colored_word) + + +def _replace_sentence(sentence: str, word: str, new_word: str) -> str: + """ + Searches for a given token in the sentence and returns the sentence where the given token has been replaced by + `new_word`. + + Parameters + ---------- + sentence: + a sentence where the word is searched + + word: + keyword to find in `sentence`. Assumes the word exists in the sentence. + + new_word: + the word to replace the keyword with + + Returns + --------- + new_sentence: + `sentence` where the every occurrence of the word is replaced by `colored_word` + """ + + new_sentence, number_of_substitions = re.subn( + r"\b{}\b".format(re.escape(word)), new_word, sentence + ) + if number_of_substitions == 0: + # Use basic string manipulation if regex fails + new_sentence = sentence.replace(word, new_word) + return new_sentence diff --git a/cleanlab/internal/util.py b/cleanlab/internal/util.py new file mode 100644 index 0000000..de48cc4 --- /dev/null +++ b/cleanlab/internal/util.py @@ -0,0 +1,616 @@ +""" +Ancillary helper methods used internally throughout this package; mostly related to Confident Learning algorithms. +""" + +from typing import Optional, Tuple, Union + +import numpy as np +import pandas as pd + +from cleanlab.internal.constants import FLOATING_POINT_COMPARISON, TINY_VALUE +from cleanlab.internal.validation import labels_to_array +from cleanlab.typing import DatasetLike, LabelLike + + +def remove_noise_from_class(noise_matrix: np.ndarray, class_without_noise: int) -> np.ndarray: + """A helper function in the setting of PU learning. + Sets all P(label=class_without_noise|true_label=any_other_class) = 0 + in noise_matrix for pulearning setting, where we have + generalized the positive class in PU learning to be any + class of choosing, denoted by class_without_noise. + + Parameters + ---------- + noise_matrix : np.ndarray of shape (K, K), K = number of classes + A conditional probability matrix of the form P(label=k_s|true_label=k_y) containing + the fraction of examples in every class, labeled as every other class. + Assumes columns of noise_matrix sum to 1. + + class_without_noise : int + Integer value of the class that has no noise. Traditionally, + this is 1 (positive) for PU learning.""" + + # Number of classes + K = len(noise_matrix) + + cwn = class_without_noise + x = np.copy(noise_matrix) + + # Set P( labels = cwn | y != cwn) = 0 (no noise) + class_arange = np.arange(K) + x[cwn, class_arange[class_arange != cwn]] = 0.0 + + # Normalize columns by increasing diagonal terms + # Ensures noise_matrix is a valid probability matrix + np.fill_diagonal(x, 1 - (np.sum(x, axis=0) - np.diag(x))) + return x + + +def clip_noise_rates(noise_matrix: np.ndarray) -> np.ndarray: + """Clip all noise rates to proper range [0,1), but + do not modify the diagonal terms because they are not + noise rates. + + ASSUMES noise_matrix columns sum to 1. + + Parameters + ---------- + noise_matrix : np.ndarray of shape (K, K), K = number of classes + A conditional probability matrix containing the fraction of + examples in every class, labeled as every other class. + Diagonal terms are not noise rates, but are consistency P(label=k|true_label=k) + Assumes columns of noise_matrix sum to 1""" + + # Preserve because diagonal entries are not noise rates. + diagonal = np.diagonal(noise_matrix) + + # Clip all noise rates (efficiently). + noise_matrix = np.clip(noise_matrix, 0, 0.9999) + + # Put unmodified diagonal back. + np.fill_diagonal(noise_matrix, diagonal) + + # Re-normalized noise_matrix so that columns sum to one. + noise_matrix = noise_matrix / np.clip(noise_matrix.sum(axis=0), a_min=TINY_VALUE, a_max=None) + return noise_matrix + + +def clip_values(x, low=0.0, high=1.0, new_sum: Optional[float] = None) -> np.ndarray: + """Clip all values in p to range [low,high]. + Preserves sum of x. + + Parameters + ---------- + x : np.ndarray + An array / list of values to be clipped. + + low : float + values in x greater than 'low' are clipped to this value + + high : float + values in x greater than 'high' are clipped to this value + + new_sum : float + normalizes x after clipping to sum to new_sum + + Returns + ------- + x : np.ndarray + A list of clipped values, summing to the same sum as x.""" + + if len(x.shape) > 1: + raise TypeError( + f"only size-1 arrays can be converted to Python scalars but 'x' had shape {x.shape}" + ) + prev_sum = np.sum(x) if new_sum is None else new_sum # Store previous sum + x = np.clip(x, low, high) # Clip all values (efficiently) + x = ( + x * prev_sum / np.clip(np.sum(x), a_min=TINY_VALUE, a_max=None) + ) # Re-normalized values to sum to previous sum + return x + + +def value_counts(x, *, num_classes: Optional[int] = None, multi_label=False) -> np.ndarray: + """Returns an np.ndarray of shape (K, 1), with the + value counts for every unique item in the labels list/array, + where K is the number of unique entries in labels. + + Works for both single-labeled and multi-labeled data. + + Parameters + ---------- + x : list or np.ndarray (one dimensional) + A list of discrete objects, like lists or strings, for + example, class labels 'y' when training a classifier. + e.g. ["dog","dog","cat"] or [1,2,0,1,1,0,2] + + num_classes : int (default: None) + Setting this fills the value counts for missing classes with zeros. + For example, if x = [0, 0, 1, 1, 3] then setting ``num_classes=5`` returns + [2, 2, 0, 1, 0] whereas setting ``num_classes=None`` would return [2, 2, 1]. This assumes + your labels come from the set [0, 1,... num_classes=1] even if some classes are missing. + + multi_label : bool, optional + If ``True``, labels should be an iterable (e.g. list) of iterables, containing a + list of labels for each example, instead of just a single label. + Assumes all classes in pred_probs.shape[1] are represented in labels. + The multi-label setting supports classification tasks where an example has 1 or more labels. + Example of a multi-labeled `labels` input: ``[[0,1], [1], [0,2], [0,1,2], [0], [1], ...]``. + The major difference in how this is calibrated versus single-label is that + the total number of errors considered is based on the number of labels, + not the number of examples. So, the calibrated `confident_joint` will sum + to the number of total labels.""" + + # Efficient method if x is pd.Series, np.ndarray, or list + if multi_label: + x = [z for lst in x for z in lst] # Flatten + unique_classes, counts = np.unique(x, return_counts=True) + + # Early exit if num_classes is not provided or redundant + if num_classes is None or num_classes == len(unique_classes): + return counts + + # Else, there are missing classes + labels_are_integers = np.issubdtype(np.array(x).dtype, np.integer) + if labels_are_integers and num_classes <= np.max(unique_classes): + raise ValueError(f"Required: num_classes > max(x), but {num_classes} <= {np.max(x)}.") + + # Add zero counts for all missing classes in [0, 1,..., num_classes-1] + total_counts = np.zeros(num_classes, dtype=int) + # Fill in counts for classes that are present. + # If labels are integers, unique_classes can be used directly as indices to place counts + # into the correct positions in total_counts array. + # If labels are strings, use a slice to fill counts sequentially since strings do not map to indices. + count_ids = unique_classes if labels_are_integers else slice(len(unique_classes)) + total_counts[count_ids] = counts + + # Return counts with zeros for all missing classes. + return total_counts + + +def value_counts_fill_missing_classes(x, num_classes, *, multi_label=False) -> np.ndarray: + """Same as ``internal.util.value_counts`` but requires that num_classes is provided and + always fills missing classes with zero counts. + + See ``internal.util.value_counts`` for parameter docstrings.""" + + return value_counts(x, num_classes=num_classes, multi_label=multi_label) + + +def get_missing_classes(labels, *, pred_probs=None, num_classes=None, multi_label=False): + """Find which classes are present in ``pred_probs`` but not present in ``labels``. + + See ``count.compute_confident_joint`` for parameter docstrings.""" + if pred_probs is None and num_classes is None: + raise ValueError("Both pred_probs and num_classes are None. You must provide exactly one.") + if pred_probs is not None and num_classes is not None: + raise ValueError("Both pred_probs and num_classes are not None. Only one may be provided.") + if num_classes is None: + num_classes = pred_probs.shape[1] + unique_classes = get_unique_classes(labels, multi_label=multi_label) + return sorted(set(range(num_classes)).difference(unique_classes)) + + +def round_preserving_sum(iterable) -> np.ndarray: + """Rounds an iterable of floats while retaining the original summed value. + The name of each parameter is required. The type and description of each + parameter is optional, but should be included if not obvious. + + The while loop in this code was adapted from: + https://github.com/cgdeboer/iteround + + Parameters + ----------- + iterable : list or np.ndarray + An iterable of floats + + Returns + ------- + list or np.ndarray + The iterable rounded to int, preserving sum.""" + + floats = np.asarray(iterable, dtype=float) + ints = floats.round() + orig_sum = np.sum(floats).round() + int_sum = np.sum(ints).round() + # Adjust the integers so that they sum to orig_sum + while abs(int_sum - orig_sum) > FLOATING_POINT_COMPARISON: + diff = np.round(orig_sum - int_sum) + increment = -1 if int(diff < 0.0) else 1 + changes = min(int(abs(diff)), len(iterable)) + # Orders indices by difference. Increments # of changes. + indices = np.argsort(floats - ints)[::-increment][:changes] + for i in indices: + ints[i] = ints[i] + increment + int_sum = np.sum(ints).round() + return ints.astype(int) + + +def round_preserving_row_totals(confident_joint) -> np.ndarray: + """Rounds confident_joint cj to type int + while preserving the totals of reach row. + Assumes that cj is a 2D np.ndarray of type float. + + Parameters + ---------- + confident_joint : 2D np.ndarray of shape (K, K) + See compute_confident_joint docstring for details. + + Returns + ------- + confident_joint : 2D np.ndarray of shape (K,K) + Rounded to int while preserving row totals.""" + + return np.apply_along_axis( + func1d=round_preserving_sum, + axis=1, + arr=confident_joint, + ).astype(int) + + +def estimate_pu_f1(s, prob_s_eq_1) -> float: + """Computes Claesen's estimate of f1 in the pulearning setting. + + Parameters + ---------- + s : iterable (list or np.ndarray) + Binary label (whether each element is labeled or not) in pu learning. + + prob_s_eq_1 : iterable (list or np.ndarray) + The probability, for each example, whether it has label=1 P(label=1|x) + + Output (float) + ------ + Claesen's estimate for f1 in the pulearning setting.""" + + pred = np.asarray(prob_s_eq_1) >= 0.5 + true_positives = sum((np.asarray(s) == 1) & (np.asarray(pred) == 1)) + all_positives = sum(s) + recall = true_positives / float(all_positives) + frac_positive = sum(pred) / float(len(s)) + return recall**2 / (2.0 * frac_positive) if frac_positive != 0 else np.nan + + +def confusion_matrix(true, pred) -> np.ndarray: + """Implements a confusion matrix for true labels + and predicted labels. true and pred MUST BE the same length + and have the same distinct set of class labels represented. + + Results are identical (and similar computation time) to: + "sklearn.metrics.confusion_matrix" + + However, this function avoids the dependency on sklearn. + + Parameters + ---------- + true : np.ndarray 1d + Contains labels. + Assumes true and pred contains the same set of distinct labels. + + pred : np.ndarray 1d + A discrete vector of noisy labels, i.e. some labels may be erroneous. + *Format requirements*: for dataset with K classes, labels must be in {0,1,...,K-1}. + + Returns + ------- + confusion_matrix : np.ndarray (2D) + matrix of confusion counts with true on rows and pred on columns.""" + + assert len(true) == len(pred) + true_classes = np.unique(true) + pred_classes = np.unique(pred) + K_true = len(true_classes) # Number of classes in true + K_pred = len(pred_classes) # Number of classes in pred + map_true = dict(zip(true_classes, range(K_true))) + map_pred = dict(zip(pred_classes, range(K_pred))) + + result = np.zeros((K_true, K_pred)) + for i in range(len(true)): + result[map_true[true[i]]][map_pred[pred[i]]] += 1 + + return result + + +def print_square_matrix( + matrix, + left_name="s", + top_name="y", + title=" A square matrix", + short_title="s,y", + round_places=2, +): + """Pretty prints a matrix. + + Parameters + ---------- + matrix : np.ndarray + the matrix to be printed + left_name : str + the name of the variable on the left of the matrix + top_name : str + the name of the variable on the top of the matrix + title : str + Prints this string above the printed square matrix. + short_title : str + A short title (6 characters or fewer) like P(labels|y) or P(labels,y). + round_places : int + Number of decimals to show for each matrix value.""" + + short_title = short_title[:6] + K = len(matrix) # Number of classes + # Make sure matrix is 2d array + if len(np.shape(matrix)) == 1: + matrix = np.array([matrix]) + print() + print(title, "of shape", matrix.shape) + print(" " + short_title + "".join(["\t" + top_name + "=" + str(i) for i in range(K)])) + print("\t---" * K) + for i in range(K): + entry = "\t".join([str(z) for z in list(matrix.round(round_places)[i, :])]) + print(left_name + "=" + str(i) + " |\t" + entry) + print("\tTrace(matrix) =", np.round(np.trace(matrix), round_places)) + print() + + +def print_noise_matrix(noise_matrix, round_places=2): + """Pretty prints the noise matrix.""" + print_square_matrix( + noise_matrix, + title=" Noise Matrix (aka Noisy Channel) P(given_label|true_label)", + short_title="p(s|y)", + round_places=round_places, + ) + + +def print_inverse_noise_matrix(inverse_noise_matrix, round_places=2): + """Pretty prints the inverse noise matrix.""" + print_square_matrix( + inverse_noise_matrix, + left_name="y", + top_name="s", + title=" Inverse Noise Matrix P(true_label|given_label)", + short_title="p(y|s)", + round_places=round_places, + ) + + +def print_joint_matrix(joint_matrix, round_places=2): + """Pretty prints the joint label noise matrix.""" + print_square_matrix( + joint_matrix, + title=" Joint Label Noise Distribution Matrix P(given_label, true_label)", + short_title="p(s,y)", + round_places=round_places, + ) + + +def compress_int_array(int_array, num_possible_values) -> np.ndarray: + """Compresses dtype of np.ndarray if num_possible_values is small enough.""" + try: + compressed_type = None + if num_possible_values < np.iinfo(np.dtype("int16")).max: + compressed_type = "int16" + elif num_possible_values < np.iinfo(np.dtype("int32")).max: # pragma: no cover + compressed_type = "int32" # pragma: no cover + if compressed_type is not None: + int_array = int_array.astype(compressed_type) + return int_array + except Exception: # int_array may not even be numpy array, keep as is then + return int_array + + +def train_val_split( + X, labels, train_idx, holdout_idx +) -> Tuple[DatasetLike, DatasetLike, LabelLike, LabelLike]: + """Splits data into training/validation sets based on given indices""" + labels_train, labels_holdout = ( + labels[train_idx], + labels[holdout_idx], + ) # labels are always np.ndarray + split_completed = False + if isinstance(X, (pd.DataFrame, pd.Series)): + X_train, X_holdout = X.iloc[train_idx], X.iloc[holdout_idx] + split_completed = True + if not split_completed: + try: # check if X is pytorch Dataset object using lazy import + import torch + + if isinstance(X, torch.utils.data.Dataset): # special splitting for pytorch Dataset + X_train = torch.utils.data.Subset(X, train_idx) + X_holdout = torch.utils.data.Subset(X, holdout_idx) + split_completed = True + except Exception: + pass + if not split_completed: + try: + X_train, X_holdout = X[train_idx], X[holdout_idx] + except Exception: + raise ValueError( + "Cleanlab cannot split this form of dataset (required for cross-validation). " + "Try a different data format, " + "or implement the cross-validation yourself and instead provide out-of-sample `pred_probs`" + ) + + return X_train, X_holdout, labels_train, labels_holdout + + +def subset_X_y(X, labels, mask) -> Tuple[DatasetLike, LabelLike]: + """Extracts subset of features/labels where mask is True""" + labels = subset_labels(labels, mask) + X = subset_data(X, mask) + return X, labels + + +def subset_labels(labels, mask) -> Union[list, np.ndarray, pd.Series]: + """Extracts subset of labels where mask is True""" + try: # filtering labels as if it is array or DataFrame + return labels[mask] + except Exception: + try: # filtering labels as if it is list + return [l for idx, l in enumerate(labels) if mask[idx]] + except Exception: + raise TypeError("labels must be 1D np.ndarray, list, or pd.Series.") + + +def subset_data(X, mask) -> DatasetLike: + """Extracts subset of data examples where mask (np.ndarray) is True""" + try: + import torch + + if isinstance(X, torch.utils.data.Dataset): + mask_idx_list = list(np.nonzero(mask)[0]) + return torch.utils.data.Subset(X, mask_idx_list) + except Exception: + pass + try: + return X[mask] + except Exception: + raise TypeError("Data features X must be subsettable with boolean mask array: X[mask]") + + +def is_torch_dataset(X) -> bool: + try: + import torch + + if isinstance(X, torch.utils.data.Dataset): + return True + except Exception: + pass + return False # assumes this cannot be torch dataset if torch cannot be imported + + +def csr_vstack(a, b) -> DatasetLike: + """Takes in 2 csr_matrices and appends the second one to the bottom of the first one. + Alternative to scipy.sparse.vstack. Returns a sparse matrix. + """ + a.data = np.hstack((a.data, b.data)) + a.indices = np.hstack((a.indices, b.indices)) + a.indptr = np.hstack((a.indptr, (b.indptr + a.nnz)[1:])) + a._shape = (a.shape[0] + b.shape[0], b.shape[1]) + return a + + +def append_extra_datapoint(to_data, from_data, index) -> DatasetLike: + """Appends an extra datapoint to the data object ``to_data``. + This datapoint is taken from the data object ``from_data`` at the corresponding index. + One place this could be useful is ensuring no missing classes after train/validation split. + """ + if not (type(from_data) is type(to_data)): + raise ValueError("Cannot append datapoint from different type of data object.") + + if isinstance(to_data, np.ndarray): + return np.vstack([to_data, from_data[index]]) + elif isinstance(from_data, (pd.DataFrame, pd.Series)): + X_extra = from_data.iloc[[index]] # type: ignore + to_data = pd.concat([to_data, X_extra]) + return to_data.reset_index(drop=True) + else: + try: + X_extra = from_data[index] + try: + return to_data.append(X_extra) + except Exception: # special append for sparse matrix + return csr_vstack(to_data, X_extra) + except Exception: + raise TypeError("Data features X must support: X.append(X[i])") + + +def get_num_classes(labels=None, pred_probs=None, label_matrix=None, multi_label=None) -> int: + """Determines the number of classes based on information considered in a + canonical ordering. label_matrix can be: noise_matrix, inverse_noise_matrix, confident_joint, + or any other K x K matrix where K = number of classes. + """ + if pred_probs is not None: # pred_probs is number 1 source of truth + return pred_probs.shape[1] + + if label_matrix is not None: # matrix dimension is number 2 source of truth + if label_matrix.shape[0] != label_matrix.shape[1]: + raise ValueError(f"label matrix must be K x K, not {label_matrix.shape}") + else: + return label_matrix.shape[0] + + if labels is None: + raise ValueError("Cannot determine number of classes from None input") + + return num_unique_classes(labels, multi_label=multi_label) + + +def num_unique_classes(labels, multi_label=None) -> int: + """Finds the number of unique classes for both single-labeled + and multi-labeled labels. If multi_label is set to None (default) + this method will infer if multi_label is True or False based on + the format of labels. + This allows for a more general form of multiclass labels that looks + like this: [1, [1,2], [0], [0, 1], 2, 1]""" + return len(get_unique_classes(labels, multi_label)) + + +def get_unique_classes(labels, multi_label=None) -> set: + """Returns the set of unique classes for both single-labeled + and multi-labeled labels. If multi_label is set to None (default) + this method will infer if multi_label is True or False based on + the format of labels. + This allows for a more general form of multiclass labels that looks + like this: [1, [1,2], [0], [0, 1], 2, 1]""" + if multi_label is None: + multi_label = any(isinstance(l, list) for l in labels) + if multi_label: + return set(l for grp in labels for l in list(grp)) + else: + return set(labels) + + +def format_labels(labels: LabelLike) -> Tuple[np.ndarray, dict]: + """Takes an array of labels and formats it such that labels are in the set ``0, 1, ..., K-1``, + where ``K`` is the number of classes. The labels are assigned based on lexicographic order. + This is useful for mapping string class labels to the integer format required by many cleanlab (and sklearn) functions. + + Returns + ------- + formatted_labels + Returns np.ndarray of shape ``(N,)``. The return labels will be properly formatted and can be passed to other cleanlab functions. + + mapping + A dictionary showing the mapping of new to old labels, such that ``mapping[k]`` returns the name of the k-th class. + """ + labels = labels_to_array(labels) + if labels.ndim != 1: + raise ValueError("labels must be 1D numpy array.") + + unique_labels = np.unique(labels) + label_map = {label: i for i, label in enumerate(unique_labels)} + formatted_labels = np.array([label_map[l] for l in labels]) + inverse_map = {i: label for label, i in label_map.items()} + + return formatted_labels, inverse_map + + +def smart_display_dataframe(df): # pragma: no cover + """Display a pandas dataframe if in a jupyter notebook, otherwise print it to console.""" + try: + from IPython.display import display + + display(df) + except Exception: + print(df) + + +def force_two_dimensions(X) -> DatasetLike: + """ + Enforce the dimensionality of a dataset to two dimensions for the use of CleanLearning default classifier, + which is `sklearn.linear_model.LogisticRegression + `_. + + Parameters + ---------- + X : np.ndarray or DatasetLike + + Returns + ------- + X : np.ndarray or DatasetLike + The original dataset reduced to two dimensions, so that the dataset will have the shape ``(N, sum(...))``, + where N is still the number of examples. + """ + if X is not None and len(X.shape) > 2: + X = X.reshape((len(X), -1)) + return X diff --git a/cleanlab/internal/validation.py b/cleanlab/internal/validation.py new file mode 100644 index 0000000..ba4c3f0 --- /dev/null +++ b/cleanlab/internal/validation.py @@ -0,0 +1,212 @@ +""" +Checks to ensure valid inputs for various methods. +""" + +from cleanlab.typing import LabelLike, DatasetLike +from cleanlab.internal.constants import FLOATING_POINT_COMPARISON +from typing import Any, List, Optional, Union +import warnings +import numpy as np +import pandas as pd + + +def assert_valid_inputs( + X: DatasetLike, + y: LabelLike, + pred_probs: Optional[np.ndarray] = None, + multi_label: bool = False, + allow_missing_classes: bool = True, + allow_one_class: bool = False, +) -> None: + """Checks that ``X``, ``labels``, ``pred_probs`` are correctly formatted.""" + if not isinstance(y, (list, np.ndarray, np.generic, pd.Series, pd.DataFrame)): + raise TypeError("labels should be a numpy array or pandas Series.") + if not multi_label: + y = labels_to_array(y) + assert_valid_class_labels( + y=y, allow_missing_classes=allow_missing_classes, allow_one_class=allow_one_class + ) + + allow_empty_X = True + if pred_probs is None: + allow_empty_X = False + + if not allow_empty_X: + assert_nonempty_input(X) + try: + num_examples = len(X) + len_supported = True + except: + len_supported = False + if not len_supported: + try: + num_examples = X.shape[0] + shape_supported = True + except: + shape_supported = False + if (not len_supported) and (not shape_supported): + raise TypeError("Data features X must support either: len(X) or X.shape[0]") + + if num_examples != len(y): + raise ValueError( + f"X and labels must be same length, but X is length {num_examples} and labels is length {len(y)}." + ) + + assert_indexing_works(X, length_X=num_examples) + + if pred_probs is not None: + if not isinstance(pred_probs, (np.ndarray, np.generic)): + raise TypeError("pred_probs must be a numpy array.") + if len(pred_probs) != len(y): + raise ValueError("pred_probs and labels must have same length.") + if len(pred_probs.shape) != 2: + raise ValueError("pred_probs array must have shape: num_examples x num_classes.") + if not multi_label: + assert isinstance(y, np.ndarray) + highest_class = max(y) + 1 + else: + assert isinstance(y, list) + assert all(isinstance(y_i, list) for y_i in y) + highest_class = max([max(y_i) for y_i in y if len(y_i) != 0]) + 1 + if pred_probs.shape[1] < highest_class: + raise ValueError( + f"pred_probs must have at least {highest_class} columns, based on the largest class index which appears in labels." + ) + # Check for valid probabilities. + if (np.min(pred_probs) < 0 - FLOATING_POINT_COMPARISON) or ( + np.max(pred_probs) > 1 + FLOATING_POINT_COMPARISON + ): + raise ValueError("Values in pred_probs must be between 0 and 1.") + if X is not None: + warnings.warn("When X and pred_probs are both provided, the former may be ignored.") + + +def assert_valid_class_labels( + y: np.ndarray, + allow_missing_classes: bool = True, + allow_one_class: bool = False, +) -> None: + """Checks that ``labels`` is properly formatted, i.e. a 1D numpy array where labels are zero-based + integers (not multi-label). + """ + if y.ndim != 1: + raise ValueError("Labels must be 1D numpy array.") + if any([isinstance(label, str) for label in y]): + raise ValueError( + "Labels cannot be strings, they must be zero-indexed integers corresponding to class indices." + ) + if not np.equal(np.mod(y, 1), 0).all(): # check that labels are integers + raise ValueError("Labels must be zero-indexed integers corresponding to class indices.") + if min(y) < 0: + raise ValueError("Labels must be positive integers corresponding to class indices.") + + unique_classes = np.unique(y) + if (not allow_one_class) and (len(unique_classes) < 2): + raise ValueError("Labels must contain at least 2 classes.") + + if not allow_missing_classes: + if (unique_classes != np.arange(len(unique_classes))).any(): + msg = "cleanlab requires zero-indexed integer labels (0,1,2,..,K-1), but in " + msg += "your case: np.unique(labels) = {}. ".format(str(unique_classes)) + msg += "Every class in (0,1,2,..,K-1) must be present in labels as well." + raise TypeError(msg) + + +def assert_nonempty_input(X: Any) -> None: + """Ensures input is not None.""" + if X is None: + raise ValueError("Data features X cannot be None. Currently X is None.") + + +def assert_indexing_works( + X: DatasetLike, idx: Optional[List[int]] = None, length_X: Optional[int] = None +) -> None: + """Ensures we can do list-based indexing into ``X`` and ``y``. + ``length_X`` is an optional argument since sparse matrix ``X`` + does not support: ``len(X)`` and we want this method to work for sparse ``X`` + (in addition to many other types of ``X``). + """ + if idx is None: + if length_X is None: + length_X = 2 # pragma: no cover + + idx = [0, length_X - 1] + + is_indexed = False + try: + if isinstance(X, (pd.DataFrame, pd.Series)): + _ = X.iloc[idx] # type: ignore[call-overload] + is_indexed = True + except Exception: + pass + if not is_indexed: + try: # check if X is pytorch Dataset object using lazy import + import torch + + if isinstance(X, torch.utils.data.Dataset): # indexing for pytorch Dataset + _ = torch.utils.data.Subset(X, idx) # type: ignore[call-overload] + is_indexed = True + except Exception: + pass + if not is_indexed: + try: + _ = X[idx] # type: ignore[call-overload] + except Exception: + msg = ( + "Data features X must support list-based indexing; i.e. one of these must work: \n" + ) + msg += "1) X[index_list] where say index_list = [0,1,3,10], or \n" + msg += "2) X.iloc[index_list] if X is pandas DataFrame." + raise TypeError(msg) + + +def labels_to_array(y: Union[LabelLike, np.generic]) -> np.ndarray: + """Converts different types of label objects to 1D numpy array and checks their validity. + + Parameters + ---------- + y : Union[LabelLike, np.generic] + Labels to convert to 1D numpy array. Can be a list, numpy array, pandas Series, or pandas DataFrame. + + Returns + ------- + labels_array : np.ndarray + 1D numpy array of labels. + """ + if isinstance(y, pd.Series): + y_series: np.ndarray = y.to_numpy() + return y_series + elif isinstance(y, pd.DataFrame): + y_arr = y.values + assert isinstance(y_arr, np.ndarray) + if y_arr.shape[1] != 1: + raise ValueError("labels must be one dimensional.") + return y_arr.flatten() + else: # y is list, np.ndarray, or some other tuple-like object + try: + return np.asarray(y) + except: + raise ValueError( + "List of labels must be convertable to 1D numpy array via: np.ndarray(labels)." + ) + + +def labels_to_list_multilabel(y: List) -> List[List[int]]: + """Converts different types of label objects to nested list and checks their validity. + + Parameters + ---------- + y : List + Labels to convert to nested list. Supports only list type. + + Returns + ------- + labels_list : List[List[int]] + Nested list of labels. + """ + if not isinstance(y, list): + raise ValueError("Unsupported Label format") + if not all(isinstance(x, list) for x in y): + raise ValueError("Each element in list of labels must be a list.") + + return y diff --git a/cleanlab/models/README.md b/cleanlab/models/README.md new file mode 100644 index 0000000..c53aca5 --- /dev/null +++ b/cleanlab/models/README.md @@ -0,0 +1,9 @@ +# Useful models adapted for use with cleanlab + +Methods in this ``models`` module are not guaranteed to be stable between different ``cleanlab`` versions. + +Some of these files include various models that can be used with cleanlab to find issues in specific types of data. These require dependencies on deep learning and other machine learning packages that are not official cleanlab dependencies. You must install these dependencies on your own if you wish to use them. + +The dependencies are as follows: +* keras.py - a wrapper to make any Keras model compatible with cleanlab and sklearn + - tensorflow diff --git a/cleanlab/models/__init__.py b/cleanlab/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cleanlab/multiannotator.py b/cleanlab/multiannotator.py new file mode 100644 index 0000000..717cf5b --- /dev/null +++ b/cleanlab/multiannotator.py @@ -0,0 +1,1923 @@ +""" +Methods for analysis of classification data labeled by multiple annotators. + +To analyze a fixed dataset labeled by multiple annotators, use the +`~cleanlab.multiannotator.get_label_quality_multiannotator` function which estimates: + +* A consensus label for each example that aggregates the individual annotations more accurately than alternative aggregation via majority-vote or other algorithms used in crowdsourcing like Dawid-Skene. +* A quality score for each consensus label which measures our confidence that this label is correct. +* An analogous label quality score for each individual label chosen by one annotator for a particular example. +* An overall quality score for each annotator which measures our confidence in the overall correctness of labels obtained from this annotator. + +The algorithms to compute these estimates are described in `the CROWDLAB paper `_. + +If you have some labeled and unlabeled data (with multiple annotators for some labeled examples) and want to decide what data to collect additional labels for, +use the `~cleanlab.multiannotator.get_active_learning_scores` function, which is intended for active learning. +This function estimates an ActiveLab quality score for each example, +which can be used to prioritize which examples are most informative to collect additional labels for. +This function is effective for settings where some examples have been labeled by one or more annotators and other examples can have no labels at all so far, +as well as settings where new labels are collected either in batches of examples or one at a time. +Here is an `example notebook `_ showcasing the use of this ActiveLab method for active learning with data re-labeling. + +The algorithms to compute these active learning scores are described in `the ActiveLab paper `_. + +Each of the main functions in this module utilizes any trained classifier model. +Variants of these functions are provided for settings where you have trained an ensemble of multiple models. +""" + +import warnings +from typing import Any, Dict, List, Optional, Tuple, Union + +import numpy as np +import pandas as pd + +from cleanlab.internal.constants import CLIPPING_LOWER_BOUND +from cleanlab.internal.multiannotator_utils import ( + assert_valid_inputs_multiannotator, + assert_valid_pred_probs, + check_consensus_label_classes, + find_best_temp_scaler, + temp_scale_pred_probs, +) +from cleanlab.internal.util import get_num_classes, value_counts +from cleanlab.rank import get_label_quality_scores + + +def get_label_quality_multiannotator( + labels_multiannotator: Union[pd.DataFrame, np.ndarray], + pred_probs: np.ndarray, + *, + consensus_method: Union[str, List[str]] = "best_quality", + quality_method: str = "crowdlab", + calibrate_probs: bool = False, + return_detailed_quality: bool = True, + return_annotator_stats: bool = True, + return_weights: bool = False, + verbose: bool = True, + label_quality_score_kwargs: dict = {}, +) -> Dict[str, Any]: + """Returns label quality scores for each example and for each annotator in a dataset labeled by multiple annotators. + + This function is for multiclass classification datasets where examples have been labeled by + multiple annotators (not necessarily the same number of annotators per example). + + It computes one consensus label for each example that best accounts for the labels chosen by each + annotator (and their quality), as well as a consensus quality score for how confident we are that this consensus label is actually correct. + It also computes similar quality scores for each annotator's individual labels, and the quality of each annotator. + Scores are between 0 and 1 (estimated via methods like CROWDLAB); lower scores indicate labels/annotators less likely to be correct. + + To decide what data to collect additional labels for, try the `~cleanlab.multiannotator.get_active_learning_scores` + (ActiveLab) function, which is intended for active learning with multiple annotators. + + Parameters + ---------- + labels_multiannotator : pd.DataFrame or np.ndarray + 2D pandas DataFrame or array of multiple given labels for each example with shape ``(N, M)``, + where N is the number of examples and M is the number of annotators. + ``labels_multiannotator[n][m]`` = label for n-th example given by m-th annotator. + + For a dataset with K classes, each given label must be an integer in 0, 1, ..., K-1 or ``NaN`` if this annotator did not label a particular example. + If you have string or other differently formatted labels, you can convert them to the proper format using :py:func:`format_multiannotator_labels `. + If pd.DataFrame, column names should correspond to each annotator's ID. + pred_probs : np.ndarray + An array of shape ``(N, K)`` of predicted class probabilities from a trained classifier model. + Predicted probabilities in the same format expected by the :py:func:`get_label_quality_scores `. + consensus_method : str or List[str], default = "majority_vote" + Specifies the method used to aggregate labels from multiple annotators into a single consensus label. + Options include: + + * ``majority_vote``: consensus obtained using a simple majority vote among annotators, with ties broken via ``pred_probs``. + * ``best_quality``: consensus obtained by selecting the label with highest label quality (quality determined by method specified in ``quality_method``). + + A List may be passed if you want to consider multiple methods for producing consensus labels. + If a List is passed, then the 0th element of the list is the method used to produce columns `consensus_label`, `consensus_quality_score`, `annotator_agreement` in the returned DataFrame. + The remaning (1st, 2nd, 3rd, etc.) elements of this list are output as extra columns in the returned pandas DataFrame with names formatted as: + `consensus_label_SUFFIX`, `consensus_quality_score_SUFFIX` where `SUFFIX` = each element of this + list, which must correspond to a valid method for computing consensus labels. + quality_method : str, default = "crowdlab" + Specifies the method used to calculate the quality of the consensus label. + Options include: + + * ``crowdlab``: an emsemble method that weighs both the annotators' labels as well as the model's prediction. + * ``agreement``: the fraction of annotators that agree with the consensus label. + calibrate_probs : bool, default = False + Boolean value that specifies whether the provided `pred_probs` should be re-calibrated to better match the annotators' empirical label distribution. + We recommend setting this to True in active learning applications, in order to prevent overconfident models from suggesting the wrong examples to collect labels for. + return_detailed_quality: bool, default = True + Boolean to specify if `detailed_label_quality` is returned. + return_annotator_stats : bool, default = True + Boolean to specify if `annotator_stats` is returned. + return_weights : bool, default = False + Boolean to specify if `model_weight` and `annotator_weight` is returned. + Model and annotator weights are applicable for ``quality_method == crowdlab``, will return ``None`` for any other quality methods. + verbose : bool, default = True + Important warnings and other printed statements may be suppressed if ``verbose`` is set to ``False``. + label_quality_score_kwargs : dict, optional + Keyword arguments to pass into :py:func:`get_label_quality_scores `. + + Returns + ------- + labels_info : dict + Dictionary containing up to 5 pandas DataFrame with keys as below: + + ``label_quality`` : pandas.DataFrame + pandas DataFrame in which each row corresponds to one example, with columns: + + * ``num_annotations``: the number of annotators that have labeled each example. + * ``consensus_label``: the single label that is best for each example (you can control how it is derived from all annotators' labels via the argument: ``consensus_method``). + * ``annotator_agreement``: the fraction of annotators that agree with the consensus label (only consider the annotators that labeled that particular example). + * ``consensus_quality_score``: label quality score for consensus label, calculated by the method specified in ``quality_method``. + + ``detailed_label_quality`` : pandas.DataFrame + Only returned if `return_detailed_quality=True`. + Returns a pandas DataFrame with columns `quality_annotator_1`, `quality_annotator_2`, ..., `quality_annotator_M` where each entry is + the label quality score for the labels provided by each annotator (is ``NaN`` for examples which this annotator did not label). + + ``annotator_stats`` : pandas.DataFrame + Only returned if `return_annotator_stats=True`. + Returns overall statistics about each annotator, sorted by lowest annotator_quality first. + pandas DataFrame in which each row corresponds to one annotator (the row IDs correspond to annotator IDs), with columns: + + * ``annotator_quality``: overall quality of a given annotator's labels, calculated by the method specified in ``quality_method``. + * ``num_examples_labeled``: number of examples annotated by a given annotator. + * ``agreement_with_consensus``: fraction of examples where a given annotator agrees with the consensus label. + * ``worst_class``: the class that is most frequently mislabeled by a given annotator. + + ``model_weight`` : float + Only returned if `return_weights=True`. It is only applicable for ``quality_method == crowdlab``. + The model weight specifies the weight of classifier model in weighted averages used to estimate label quality + This number is an estimate of how trustworthy the model is relative the annotators. + + ``annotator_weight`` : np.ndarray + Only returned if `return_weights=True`. It is only applicable for ``quality_method == crowdlab``. + An array of shape ``(M,)`` where M is the number of annotators, specifying the weight of each annotator in weighted averages used to estimate label quality. + These weights are estimates of how trustworthy each annotator is relative to the other annotators. + + """ + + if isinstance(labels_multiannotator, pd.DataFrame): + annotator_ids = labels_multiannotator.columns + index_col = labels_multiannotator.index + labels_multiannotator = ( + labels_multiannotator.replace({pd.NA: np.nan}).astype(float).to_numpy() + ) + elif isinstance(labels_multiannotator, np.ndarray): + annotator_ids = None + index_col = None + else: + raise ValueError("labels_multiannotator must be either a NumPy array or Pandas DataFrame.") + + if return_weights == True and quality_method != "crowdlab": + raise ValueError( + "Model and annotator weights are only applicable to the crowdlab quality method. " + "Either set return_weights=False or quality_method='crowdlab'." + ) + + assert_valid_inputs_multiannotator( + labels_multiannotator, pred_probs, annotator_ids=annotator_ids + ) + + # Count number of non-NaN values for each example + num_annotations = np.sum(~np.isnan(labels_multiannotator), axis=1) + + # calibrate pred_probs + if calibrate_probs: + optimal_temp = find_best_temp_scaler(labels_multiannotator, pred_probs) + pred_probs = temp_scale_pred_probs(pred_probs, optimal_temp) + + if not isinstance(consensus_method, list): + consensus_method = [consensus_method] + + if "best_quality" in consensus_method or "majority_vote" in consensus_method: + majority_vote_label = get_majority_vote_label( + labels_multiannotator=labels_multiannotator, + pred_probs=pred_probs, + verbose=False, + ) + ( + MV_annotator_agreement, + MV_consensus_quality_score, + MV_post_pred_probs, + MV_model_weight, + MV_annotator_weight, + ) = _get_consensus_stats( + labels_multiannotator=labels_multiannotator, + pred_probs=pred_probs, + num_annotations=num_annotations, + consensus_label=majority_vote_label, + quality_method=quality_method, + verbose=verbose, + label_quality_score_kwargs=label_quality_score_kwargs, + ) + + label_quality = pd.DataFrame({"num_annotations": num_annotations}, index=index_col) + valid_methods = ["majority_vote", "best_quality"] + main_method = True + + for curr_method in consensus_method: + # geting consensus label and stats + if curr_method == "majority_vote": + consensus_label = majority_vote_label + annotator_agreement = MV_annotator_agreement + consensus_quality_score = MV_consensus_quality_score + post_pred_probs = MV_post_pred_probs + model_weight = MV_model_weight + annotator_weight = MV_annotator_weight + + elif curr_method == "best_quality": + consensus_label = np.full(len(majority_vote_label), np.nan) + for i in range(len(consensus_label)): + max_pred_probs_ind = np.where( + MV_post_pred_probs[i] == np.max(MV_post_pred_probs[i]) + )[0] + if len(max_pred_probs_ind) == 1: + consensus_label[i] = max_pred_probs_ind[0] + else: + consensus_label[i] = majority_vote_label[i] + consensus_label = consensus_label.astype(int) # convert all label types to int + + ( + annotator_agreement, + consensus_quality_score, + post_pred_probs, + model_weight, + annotator_weight, + ) = _get_consensus_stats( + labels_multiannotator=labels_multiannotator, + pred_probs=pred_probs, + num_annotations=num_annotations, + consensus_label=consensus_label, + quality_method=quality_method, + verbose=verbose, + label_quality_score_kwargs=label_quality_score_kwargs, + ) + + else: + raise ValueError( + f""" + {curr_method} is not a valid consensus method! + Please choose a valid consensus_method: {valid_methods} + """ + ) + + if verbose: + # check if any classes no longer appear in the set of consensus labels + check_consensus_label_classes( + labels_multiannotator=labels_multiannotator, + consensus_label=consensus_label, + consensus_method=curr_method, + ) + + # saving stats into dataframe, computing additional stats if specified + if main_method: + ( + label_quality["consensus_label"], + label_quality["consensus_quality_score"], + label_quality["annotator_agreement"], + ) = ( + consensus_label, + consensus_quality_score, + annotator_agreement, + ) + + label_quality = label_quality.reindex( + columns=[ + "consensus_label", + "consensus_quality_score", + "annotator_agreement", + "num_annotations", + ] + ) + + # default variable for _get_annotator_stats + detailed_label_quality = None + + if return_detailed_quality: + # Compute the label quality scores for each annotators' labels + detailed_label_quality = np.apply_along_axis( + _get_annotator_label_quality_score, + axis=0, + arr=labels_multiannotator, + pred_probs=post_pred_probs, + label_quality_score_kwargs=label_quality_score_kwargs, + ) + detailed_label_quality_df = pd.DataFrame( + detailed_label_quality, index=index_col, columns=annotator_ids + ).add_prefix("quality_annotator_") + + if return_annotator_stats: + annotator_stats = _get_annotator_stats( + labels_multiannotator=labels_multiannotator, + pred_probs=post_pred_probs, + consensus_label=consensus_label, + num_annotations=num_annotations, + annotator_agreement=annotator_agreement, + model_weight=model_weight, + annotator_weight=annotator_weight, + consensus_quality_score=consensus_quality_score, + detailed_label_quality=detailed_label_quality, + annotator_ids=annotator_ids, + quality_method=quality_method, + ) + + main_method = False + + else: + ( + label_quality[f"consensus_label_{curr_method}"], + label_quality[f"consensus_quality_score_{curr_method}"], + label_quality[f"annotator_agreement_{curr_method}"], + ) = ( + consensus_label, + consensus_quality_score, + annotator_agreement, + ) + + labels_info = { + "label_quality": label_quality, + } + + if return_detailed_quality: + labels_info["detailed_label_quality"] = detailed_label_quality_df + if return_annotator_stats: + labels_info["annotator_stats"] = annotator_stats + if return_weights: + labels_info["model_weight"] = model_weight + labels_info["annotator_weight"] = annotator_weight + + return labels_info + + +def get_label_quality_multiannotator_ensemble( + labels_multiannotator: Union[pd.DataFrame, np.ndarray], + pred_probs: np.ndarray, + *, + calibrate_probs: bool = False, + return_detailed_quality: bool = True, + return_annotator_stats: bool = True, + return_weights: bool = False, + verbose: bool = True, + label_quality_score_kwargs: dict = {}, +) -> Dict[str, Any]: + """Returns label quality scores for each example and for each annotator, based on predictions from an ensemble of models. + + This function is similar to `~cleanlab.multiannotator.get_label_quality_multiannotator` but for settings where + you have trained an ensemble of multiple classifier models rather than a single model. + + Parameters + ---------- + labels_multiannotator : pd.DataFrame or np.ndarray + Multiannotator labels in the same format expected by `~cleanlab.multiannotator.get_label_quality_multiannotator`. + pred_probs : np.ndarray + An array of shape ``(P, N, K)`` where P is the number of models, consisting of predicted class probabilities from the ensemble models. + Each set of predicted probabilities with shape ``(N, K)`` is in the same format expected by the :py:func:`get_label_quality_scores `. + calibrate_probs : bool, default = False + Boolean value as expected by `~cleanlab.multiannotator.get_label_quality_multiannotator`. + return_detailed_quality: bool, default = True + Boolean value as expected by `~cleanlab.multiannotator.get_label_quality_multiannotator`. + return_annotator_stats : bool, default = True + Boolean value as expected by `~cleanlab.multiannotator.get_label_quality_multiannotator`. + return_weights : bool, default = False + Boolean value as expected by `~cleanlab.multiannotator.get_label_quality_multiannotator`. + verbose : bool, default = True + Boolean value as expected by `~cleanlab.multiannotator.get_label_quality_multiannotator`. + label_quality_score_kwargs : dict, optional + Keyword arguments in the same format expected by `~cleanlab.multiannotator.get_label_quality_multiannotator`. + + Returns + ------- + labels_info : dict + Dictionary containing up to 5 pandas DataFrame with keys as below: + + ``label_quality`` : pandas.DataFrame + Similar to output as `~cleanlab.multiannotator.get_label_quality_multiannotator`. + + ``detailed_label_quality`` : pandas.DataFrame + Similar to output as `~cleanlab.multiannotator.get_label_quality_multiannotator`. + + ``annotator_stats`` : pandas.DataFrame + Similar to output as `~cleanlab.multiannotator.get_label_quality_multiannotator`. + + ``model_weight`` : np.ndarray + Only returned if `return_weights=True`. + An array of shape ``(P,)`` where is the number of models in the ensemble, specifying the weight of each classifier model in weighted averages used to estimate label quality. + These weigthts is an estimate of how trustworthy the model is relative the annotators. + An array of shape ``(P,)`` where is the number of models in the ensemble, specifying the model weight used in weighted averages. + + ``annotator_weight`` : np.ndarray + Only returned if `return_weights=True`. + Similar to output as `~cleanlab.multiannotator.get_label_quality_multiannotator`. + + See Also + -------- + get_label_quality_multiannotator + """ + if isinstance(labels_multiannotator, pd.DataFrame): + annotator_ids = labels_multiannotator.columns + index_col = labels_multiannotator.index + labels_multiannotator = ( + labels_multiannotator.replace({pd.NA: np.nan}).astype(float).to_numpy() + ) + elif isinstance(labels_multiannotator, np.ndarray): + annotator_ids = None + index_col = None + else: + raise ValueError("labels_multiannotator must be either a NumPy array or Pandas DataFrame.") + + assert_valid_inputs_multiannotator( + labels_multiannotator, pred_probs, ensemble=True, annotator_ids=annotator_ids + ) + + # Count number of non-NaN values for each example + num_annotations = np.sum(~np.isnan(labels_multiannotator), axis=1) + + # temp scale pred_probs + if calibrate_probs: + for i in range(len(pred_probs)): + curr_pred_probs = pred_probs[i] + optimal_temp = find_best_temp_scaler(labels_multiannotator, curr_pred_probs) + pred_probs[i] = temp_scale_pred_probs(curr_pred_probs, optimal_temp) + + label_quality = pd.DataFrame({"num_annotations": num_annotations}, index=index_col) + + # get majority vote stats + avg_pred_probs = np.mean(pred_probs, axis=0) + majority_vote_label = get_majority_vote_label( + labels_multiannotator=labels_multiannotator, + pred_probs=avg_pred_probs, + verbose=False, + ) + ( + MV_annotator_agreement, + MV_consensus_quality_score, + MV_post_pred_probs, + MV_model_weight, + MV_annotator_weight, + ) = _get_consensus_stats( + labels_multiannotator=labels_multiannotator, + pred_probs=pred_probs, + num_annotations=num_annotations, + consensus_label=majority_vote_label, + verbose=verbose, + ensemble=True, + **label_quality_score_kwargs, + ) + + # get crowdlab stats + consensus_label = np.full(len(majority_vote_label), np.nan) + for i in range(len(consensus_label)): + max_pred_probs_ind = np.where(MV_post_pred_probs[i] == np.max(MV_post_pred_probs[i]))[0] + if len(max_pred_probs_ind) == 1: + consensus_label[i] = max_pred_probs_ind[0] + else: + consensus_label[i] = majority_vote_label[i] + consensus_label = consensus_label.astype(int) # convert all label types to int + + ( + annotator_agreement, + consensus_quality_score, + post_pred_probs, + model_weight, + annotator_weight, + ) = _get_consensus_stats( + labels_multiannotator=labels_multiannotator, + pred_probs=pred_probs, + num_annotations=num_annotations, + consensus_label=consensus_label, + verbose=verbose, + ensemble=True, + **label_quality_score_kwargs, + ) + + if verbose: + # check if any classes no longer appear in the set of consensus labels + check_consensus_label_classes( + labels_multiannotator=labels_multiannotator, + consensus_label=consensus_label, + consensus_method="crowdlab", + ) + + ( + label_quality["consensus_label"], + label_quality["consensus_quality_score"], + label_quality["annotator_agreement"], + ) = ( + consensus_label, + consensus_quality_score, + annotator_agreement, + ) + + label_quality = label_quality.reindex( + columns=[ + "consensus_label", + "consensus_quality_score", + "annotator_agreement", + "num_annotations", + ] + ) + + # default variable for _get_annotator_stats + detailed_label_quality = None + + if return_detailed_quality: + # Compute the label quality scores for each annotators' labels + detailed_label_quality = np.apply_along_axis( + _get_annotator_label_quality_score, + axis=0, + arr=labels_multiannotator, + pred_probs=post_pred_probs, + label_quality_score_kwargs=label_quality_score_kwargs, + ) + detailed_label_quality_df = pd.DataFrame( + detailed_label_quality, index=index_col, columns=annotator_ids + ).add_prefix("quality_annotator_") + + if return_annotator_stats: + annotator_stats = _get_annotator_stats( + labels_multiannotator=labels_multiannotator, + pred_probs=post_pred_probs, + consensus_label=consensus_label, + num_annotations=num_annotations, + annotator_agreement=annotator_agreement, + model_weight=np.mean(model_weight), # use average model weight when scoring annotators + annotator_weight=annotator_weight, + consensus_quality_score=consensus_quality_score, + detailed_label_quality=detailed_label_quality, + annotator_ids=annotator_ids, + ) + + labels_info = { + "label_quality": label_quality, + } + + if return_detailed_quality: + labels_info["detailed_label_quality"] = detailed_label_quality_df + if return_annotator_stats: + labels_info["annotator_stats"] = annotator_stats + if return_weights: + labels_info["model_weight"] = model_weight + labels_info["annotator_weight"] = annotator_weight + + return labels_info + + +def get_active_learning_scores( + labels_multiannotator: Optional[Union[pd.DataFrame, np.ndarray]] = None, + pred_probs: Optional[np.ndarray] = None, + pred_probs_unlabeled: Optional[np.ndarray] = None, +) -> Tuple[np.ndarray, np.ndarray]: + """Returns an ActiveLab quality score for each example in the dataset, to estimate which examples are most informative to (re)label next in active learning. + + We consider settings where one example can be labeled by one or more annotators and some examples have no labels at all so far. + + The score is in between 0 and 1, and can be used to prioritize what data to collect additional labels for. + Lower scores indicate examples whose true label we are least confident about based on the current data; + collecting additional labels for these low-scoring examples will be more informative than collecting labels for other examples. + To use an annotation budget most efficiently, select a batch of examples with the lowest scores and collect one additional label for each example, + and repeat this process after retraining your classifier. + + You can use this function to get active learning scores for: examples that already have one or more labels (specify ``labels_multiannotator`` and ``pred_probs`` + as arguments), or for unlabeled examples (specify ``pred_probs_unlabeled``), or for both types of examples (specify all of the above arguments). + + To analyze a fixed dataset labeled by multiple annotators rather than collecting additional labels, try the + `~cleanlab.multiannotator.get_label_quality_multiannotator` (CROWDLAB) function instead. + + Parameters + ---------- + labels_multiannotator : pd.DataFrame or np.ndarray, optional + 2D pandas DataFrame or array of multiple given labels for each example with shape ``(N, M)``, + where N is the number of examples and M is the number of annotators. Note that this function also works with + datasets where there is only one annotator (M=1). + For more details, labels in the same format expected by the `~cleanlab.multiannotator.get_label_quality_multiannotator`. + Note that examples that have no annotator labels should not be included in this DataFrame/array. + This argument is optional if ``pred_probs`` is not provided (you might only provide ``pred_probs_unlabeled`` to only get active learning scores for the unlabeled examples). + pred_probs : np.ndarray, optional + An array of shape ``(N, K)`` of predicted class probabilities from a trained classifier model. + Predicted probabilities in the same format expected by the :py:func:`get_label_quality_scores `. + This argument is optional if you only want to get active learning scores for unlabeled examples (specify only ``pred_probs_unlabeled`` instead). + pred_probs_unlabeled : np.ndarray, optional + An array of shape ``(N, K)`` of predicted class probabilities from a trained classifier model for examples that have no annotator labels. + Predicted probabilities in the same format expected by the :py:func:`get_label_quality_scores `. + This argument is optional if you only want to get active learning scores for already-labeled examples (specify only ``pred_probs`` instead). + + Returns + ------- + active_learning_scores : np.ndarray + Array of shape ``(N,)`` indicating the ActiveLab quality scores for each example. + This array is empty if no already-labeled data was provided via ``labels_multiannotator``. + Examples with the lowest scores are those we should label next in order to maximally improve our classifier model. + + active_learning_scores_unlabeled : np.ndarray + Array of shape ``(N,)`` indicating the active learning quality scores for each unlabeled example. + Returns an empty array if no unlabeled data is provided. + Examples with the lowest scores are those we should label next in order to maximally improve our classifier model + (scores for unlabeled data are directly comparable with the `active_learning_scores` for labeled data). + """ + + assert_valid_pred_probs(pred_probs=pred_probs, pred_probs_unlabeled=pred_probs_unlabeled) + + # compute multiannotator stats if labeled data is provided + if pred_probs is not None: + if labels_multiannotator is None: + raise ValueError( + "labels_multiannotator cannot be None when passing in pred_probs. ", + "Either provide labels_multiannotator to obtain active learning scores for the labeled examples, " + "or just pass in pred_probs_unlabeled to get active learning scores for unlabeled examples.", + ) + + if isinstance(labels_multiannotator, pd.DataFrame): + labels_multiannotator = ( + labels_multiannotator.replace({pd.NA: np.nan}).astype(float).to_numpy() + ) + elif not isinstance(labels_multiannotator, np.ndarray): + raise ValueError( + "labels_multiannotator must be either a NumPy array or Pandas DataFrame." + ) + # check that labels_multiannotator is a 2D array + if labels_multiannotator.ndim != 2: + raise ValueError( + "labels_multiannotator must be a 2D array or dataframe, " + "each row represents an example and each column represents an annotator." + ) + + num_classes = get_num_classes(pred_probs=pred_probs) + + # if all examples are only labeled by a single annotator + if (np.sum(~np.isnan(labels_multiannotator), axis=1) == 1).all(): + optimal_temp = 1.0 # do not temp scale for single annotator case, temperature is defined here for later use + + assert_valid_inputs_multiannotator( + labels_multiannotator, pred_probs, allow_single_label=True + ) + + consensus_label = get_majority_vote_label( + labels_multiannotator=labels_multiannotator, + pred_probs=pred_probs, + verbose=False, + ) + quality_of_consensus_labeled = get_label_quality_scores(consensus_label, pred_probs) + model_weight = 1 + annotator_weight = np.full(labels_multiannotator.shape[1], 1) + avg_annotator_weight = np.mean(annotator_weight) + + # examples are annotated by multiple annotators + else: + optimal_temp = find_best_temp_scaler(labels_multiannotator, pred_probs) + pred_probs = temp_scale_pred_probs(pred_probs, optimal_temp) + + multiannotator_info = get_label_quality_multiannotator( + labels_multiannotator, + pred_probs, + return_annotator_stats=False, + return_detailed_quality=False, + return_weights=True, + ) + + quality_of_consensus_labeled = multiannotator_info["label_quality"][ + "consensus_quality_score" + ] + model_weight = multiannotator_info["model_weight"] + annotator_weight = multiannotator_info["annotator_weight"] + avg_annotator_weight = np.mean(annotator_weight) + + # compute scores for labeled data + active_learning_scores = np.full(len(labels_multiannotator), np.nan) + for i, annotator_labels in enumerate(labels_multiannotator): + active_learning_scores[i] = np.average( + (quality_of_consensus_labeled[i], 1 / num_classes), + weights=( + np.sum(annotator_weight[~np.isnan(annotator_labels)]) + model_weight, + avg_annotator_weight, + ), + ) + + # no labeled data provided so do not estimate temperature and model/annotator weights + elif pred_probs_unlabeled is not None: + num_classes = get_num_classes(pred_probs=pred_probs_unlabeled) + optimal_temp = 1 + model_weight = 1 + avg_annotator_weight = 1 + active_learning_scores = np.array([]) + + else: + raise ValueError( + "pred_probs and pred_probs_unlabeled cannot both be None, specify at least one of the two." + ) + + # compute scores for unlabeled data + if pred_probs_unlabeled is not None: + pred_probs_unlabeled = temp_scale_pred_probs(pred_probs_unlabeled, optimal_temp) + quality_of_consensus_unlabeled = np.max(pred_probs_unlabeled, axis=1) + + active_learning_scores_unlabeled = np.average( + np.stack( + [ + quality_of_consensus_unlabeled, + np.full(len(quality_of_consensus_unlabeled), 1 / num_classes), + ] + ), + weights=[model_weight, avg_annotator_weight], + axis=0, + ) + + else: + active_learning_scores_unlabeled = np.array([]) + + return active_learning_scores, active_learning_scores_unlabeled + + +def get_active_learning_scores_ensemble( + labels_multiannotator: Optional[Union[pd.DataFrame, np.ndarray]] = None, + pred_probs: Optional[np.ndarray] = None, + pred_probs_unlabeled: Optional[np.ndarray] = None, +) -> Tuple[np.ndarray, np.ndarray]: + """Returns an ActiveLab quality score for each example in the dataset, based on predictions from an ensemble of models. + + This function is similar to `~cleanlab.multiannotator.get_active_learning_scores` but allows for an + ensemble of multiple classifier models to be trained and will aggregate predictions from the models to compute the ActiveLab quality score. + + Parameters + ---------- + labels_multiannotator : pd.DataFrame or np.ndarray + Multiannotator labels in the same format expected by `~cleanlab.multiannotator.get_active_learning_scores`. + This argument is optional if ``pred_probs`` is not provided (in cases where you only provide ``pred_probs_unlabeled`` to get active learning scores for unlabeled examples). + pred_probs : np.ndarray + An array of shape ``(P, N, K)`` where P is the number of models, consisting of predicted class probabilities from the ensemble models. + Note that this function also works with datasets where there is only one annotator (M=1). + Each set of predicted probabilities with shape ``(N, K)`` is in the same format expected by the :py:func:`get_label_quality_scores `. + This argument is optional if you only want to get active learning scores for unlabeled examples (pass in ``pred_probs_unlabeled`` instead). + pred_probs_unlabeled : np.ndarray, optional + An array of shape ``(P, N, K)`` where P is the number of models, consisting of predicted class probabilities from a trained classifier model + for examples that have no annotated labels so far (but which we may want to label in the future, and hence compute active learning quality scores for). + Each set of predicted probabilities with shape ``(N, K)`` is in the same format expected by the :py:func:`get_label_quality_scores `. + This argument is optional if you only want to get active learning scores for labeled examples (pass in ``pred_probs`` instead). + + Returns + ------- + active_learning_scores : np.ndarray + Similar to output as :py:func:`get_label_quality_scores `. + active_learning_scores_unlabeled : np.ndarray + Similar to output as :py:func:`get_label_quality_scores `. + + See Also + -------- + get_active_learning_scores + """ + + assert_valid_pred_probs( + pred_probs=pred_probs, pred_probs_unlabeled=pred_probs_unlabeled, ensemble=True + ) + + # compute multiannotator stats if labeled data is provided + if pred_probs is not None: + if labels_multiannotator is None: + raise ValueError( + "labels_multiannotator cannot be None when passing in pred_probs. ", + "You can either provide labels_multiannotator to obtain active learning scores for the labeled examples, " + "or just pass in pred_probs_unlabeled to get active learning scores for unlabeled examples.", + ) + + if isinstance(labels_multiannotator, pd.DataFrame): + labels_multiannotator = ( + labels_multiannotator.replace({pd.NA: np.nan}).astype(float).to_numpy() + ) + elif not isinstance(labels_multiannotator, np.ndarray): + raise ValueError( + "labels_multiannotator must be either a NumPy array or Pandas DataFrame." + ) + + # check that labels_multiannotator is a 2D array + if labels_multiannotator.ndim != 2: + raise ValueError( + "labels_multiannotator must be a 2D array or dataframe, " + "each row represents an example and each column represents an annotator." + ) + + num_classes = get_num_classes(pred_probs=pred_probs[0]) + + # if all examples are only labeled by a single annotator + if (np.sum(~np.isnan(labels_multiannotator), axis=1) == 1).all(): + # do not temp scale for single annotator case, temperature is defined here for later use + optimal_temp = np.full(len(pred_probs), 1.0) + + assert_valid_inputs_multiannotator( + labels_multiannotator, pred_probs, ensemble=True, allow_single_label=True + ) + + avg_pred_probs = np.mean(pred_probs, axis=0) + consensus_label = get_majority_vote_label( + labels_multiannotator=labels_multiannotator, + pred_probs=avg_pred_probs, + verbose=False, + ) + quality_of_consensus_labeled = get_label_quality_scores(consensus_label, avg_pred_probs) + model_weight = np.full(len(pred_probs), 1) + annotator_weight = np.full(labels_multiannotator.shape[1], 1) + avg_annotator_weight = np.mean(annotator_weight) + + # examples are annotated by multiple annotators + else: + optimal_temp = np.full(len(pred_probs), np.nan) + for i, curr_pred_probs in enumerate(pred_probs): + curr_optimal_temp = find_best_temp_scaler(labels_multiannotator, curr_pred_probs) + pred_probs[i] = temp_scale_pred_probs(curr_pred_probs, curr_optimal_temp) + optimal_temp[i] = curr_optimal_temp + + multiannotator_info = get_label_quality_multiannotator_ensemble( + labels_multiannotator, + pred_probs, + return_annotator_stats=False, + return_detailed_quality=False, + return_weights=True, + ) + + quality_of_consensus_labeled = multiannotator_info["label_quality"][ + "consensus_quality_score" + ] + model_weight = multiannotator_info["model_weight"] + annotator_weight = multiannotator_info["annotator_weight"] + avg_annotator_weight = np.mean(annotator_weight) + + # compute scores for labeled data + active_learning_scores = np.full(len(labels_multiannotator), np.nan) + for i, annotator_labels in enumerate(labels_multiannotator): + active_learning_scores[i] = np.average( + (quality_of_consensus_labeled[i], 1 / num_classes), + weights=( + np.sum(annotator_weight[~np.isnan(annotator_labels)]) + np.sum(model_weight), + avg_annotator_weight, + ), + ) + + # no labeled data provided so do not estimate temperature and model/annotator weights + elif pred_probs_unlabeled is not None: + num_classes = get_num_classes(pred_probs=pred_probs_unlabeled[0]) + optimal_temp = np.full(len(pred_probs_unlabeled), 1.0) + model_weight = np.full(len(pred_probs_unlabeled), 1) + avg_annotator_weight = 1 + active_learning_scores = np.array([]) + + else: + raise ValueError( + "pred_probs and pred_probs_unlabeled cannot both be None, specify at least one of the two." + ) + + # compute scores for unlabeled data + if pred_probs_unlabeled is not None: + for i in range(len(pred_probs_unlabeled)): + pred_probs_unlabeled[i] = temp_scale_pred_probs( + pred_probs_unlabeled[i], optimal_temp[i] + ) + + avg_pred_probs_unlabeled = np.mean(pred_probs_unlabeled, axis=0) + consensus_label_unlabeled = get_majority_vote_label( + np.argmax(pred_probs_unlabeled, axis=2).T, + avg_pred_probs_unlabeled, + ) + modified_pred_probs_unlabeled = np.average( + np.concatenate( + ( + pred_probs_unlabeled, + np.full(pred_probs_unlabeled.shape[1:], 1 / num_classes)[np.newaxis, :, :], + ) + ), + weights=np.concatenate((model_weight, np.array([avg_annotator_weight]))), + axis=0, + ) + + active_learning_scores_unlabeled = get_label_quality_scores( + consensus_label_unlabeled, modified_pred_probs_unlabeled + ) + else: + active_learning_scores_unlabeled = np.array([]) + + return active_learning_scores, active_learning_scores_unlabeled + + +def get_majority_vote_label( + labels_multiannotator: Union[pd.DataFrame, np.ndarray], + pred_probs: Optional[np.ndarray] = None, + verbose: bool = True, +) -> np.ndarray: + """Returns the majority vote label for each example, aggregated from the labels given by multiple annotators. + + Parameters + ---------- + labels_multiannotator : pd.DataFrame or np.ndarray + 2D pandas DataFrame or array of multiple given labels for each example with shape ``(N, M)``, + where N is the number of examples and M is the number of annotators. + For more details, labels in the same format expected by the `~cleanlab.multiannotator.get_label_quality_multiannotator`. + pred_probs : np.ndarray, optional + An array of shape ``(N, K)`` of model-predicted probabilities, ``P(label=k|x)``. + For details, predicted probabilities in the same format expected by `~cleanlab.multiannotator.get_label_quality_multiannotator`. + verbose : bool, optional + Important warnings and other printed statements may be suppressed if ``verbose`` is set to ``False``. + Returns + ------- + consensus_label: np.ndarray + An array of shape ``(N,)`` with the majority vote label aggregated from all annotators. + + In the event of majority vote ties, ties are broken in the following order: + using the model ``pred_probs`` (if provided) and selecting the class with highest predicted probability, + using the empirical class frequencies and selecting the class with highest frequency, + using an initial annotator quality score and selecting the class that has been labeled by annotators with higher quality, + and lastly by random selection. + """ + + if isinstance(labels_multiannotator, pd.DataFrame): + annotator_ids = labels_multiannotator.columns + labels_multiannotator = ( + labels_multiannotator.replace({pd.NA: np.nan}).astype(float).to_numpy() + ) + elif isinstance(labels_multiannotator, np.ndarray): + annotator_ids = None + else: + raise ValueError("labels_multiannotator must be either a NumPy array or Pandas DataFrame.") + + if verbose: + assert_valid_inputs_multiannotator( + labels_multiannotator, pred_probs, annotator_ids=annotator_ids + ) + + if pred_probs is not None: + num_classes = pred_probs.shape[1] + else: + num_classes = int(np.nanmax(labels_multiannotator) + 1) + + array_idx = np.arange(labels_multiannotator.shape[0]) + label_count = np.zeros((labels_multiannotator.shape[0], num_classes)) + for i in range(labels_multiannotator.shape[1]): + not_nan_mask = ~np.isnan(labels_multiannotator[:, i]) + # Get the indexes where the label is not missing for the annotator i as int. + label_index = labels_multiannotator[not_nan_mask, i].astype(int) + # Increase the counts of those labels by 1. + label_count[array_idx[not_nan_mask], label_index] += 1 + + mode_labels_multiannotator = np.full(label_count.shape, np.nan) + modes_mask = label_count == np.max(label_count, axis=1).reshape(-1, 1) + insert_index = np.zeros(modes_mask.shape[0], dtype=int) + for i in range(modes_mask.shape[1]): + mode_index = np.where(modes_mask[:, i])[0] + mode_labels_multiannotator[mode_index, insert_index[mode_index]] = i + insert_index[mode_index] += 1 + + majority_vote_label = np.full(len(labels_multiannotator), np.nan) + label_mode_count = (~np.isnan(mode_labels_multiannotator)).sum(axis=1) + + # obtaining consensus using annotator majority vote + mode_count_one_mask = label_mode_count == 1 + majority_vote_label[mode_count_one_mask] = mode_labels_multiannotator[mode_count_one_mask, 0] + nontied_idx = array_idx[mode_count_one_mask] + tied_idx = { + i: label_mode[:count].astype(int) + for i, label_mode, count in zip( + array_idx[~mode_count_one_mask], + mode_labels_multiannotator[~mode_count_one_mask, :], + label_mode_count[~mode_count_one_mask], + ) + } + + # tiebreak 1: using pred_probs (if provided) + if pred_probs is not None and len(tied_idx) > 0: + for idx, label_mode in tied_idx.copy().items(): + max_pred_probs = np.where( + pred_probs[idx, label_mode] == np.max(pred_probs[idx, label_mode]) + )[0] + if len(max_pred_probs) == 1: + majority_vote_label[idx] = label_mode[max_pred_probs[0]] + del tied_idx[idx] + else: + tied_idx[idx] = label_mode[max_pred_probs] + + # tiebreak 2: using empirical class frequencies + # current tiebreak will select the minority class (to prevent larger class imbalance) + if len(tied_idx) > 0: + class_frequencies = label_count.sum(axis=0) + for idx, label_mode in tied_idx.copy().items(): + min_frequency = np.where( + class_frequencies[label_mode] == np.min(class_frequencies[label_mode]) + )[0] + if len(min_frequency) == 1: + majority_vote_label[idx] = label_mode[min_frequency[0]] + del tied_idx[idx] + else: + tied_idx[idx] = label_mode[min_frequency] + + # tiebreak 3: using initial annotator quality scores + if len(tied_idx) > 0: + nontied_majority_vote_label = majority_vote_label[nontied_idx] + nontied_labels_multiannotator = labels_multiannotator[nontied_idx] + annotator_agreement_with_consensus = np.zeros(nontied_labels_multiannotator.shape[1]) + for i in range(len(annotator_agreement_with_consensus)): + labels = nontied_labels_multiannotator[:, i] + labels_mask = ~np.isnan(labels) + if np.sum(labels_mask) == 0: + annotator_agreement_with_consensus[i] = np.nan + else: + annotator_agreement_with_consensus[i] = np.mean( + labels[labels_mask] == nontied_majority_vote_label[labels_mask] + ) + + # impute average annotator accuracy for any annotator that do not overlap with consensus + nan_mask = np.isnan(annotator_agreement_with_consensus) + avg_annotator_agreement = np.mean(annotator_agreement_with_consensus[~nan_mask]) + annotator_agreement_with_consensus[nan_mask] = avg_annotator_agreement + + for idx, label_mode in tied_idx.copy().items(): + label_quality_score = np.array( + [ + np.mean( + annotator_agreement_with_consensus[ + np.where(labels_multiannotator[idx] == label)[0] + ] + ) + for label in label_mode + ] + ) + max_score = np.where(label_quality_score == label_quality_score.max())[0] + if len(max_score) == 1: + majority_vote_label[idx] = label_mode[max_score[0]] + del tied_idx[idx] + else: + tied_idx[idx] = label_mode[max_score] + + # if still tied, break by random selection + if len(tied_idx) > 0: + warnings.warn( + f"breaking ties of examples {list(tied_idx.keys())} by random selection, you may want to set seed for reproducability" + ) + for idx, label_mode in tied_idx.items(): + majority_vote_label[idx] = np.random.choice(label_mode) + + if verbose: + # check if any classes no longer appear in the set of consensus labels + check_consensus_label_classes( + labels_multiannotator=labels_multiannotator, + consensus_label=majority_vote_label, + consensus_method="majority_vote", + ) + + return majority_vote_label.astype(int) + + +def convert_long_to_wide_dataset( + labels_multiannotator_long: pd.DataFrame, +) -> pd.DataFrame: + """Converts a long format dataset to wide format which is suitable for passing into + `~cleanlab.multiannotator.get_label_quality_multiannotator`. + + Dataframe must contain three columns named: + + #. ``task`` representing each example labeled by the annotators + #. ``annotator`` representing each annotator + #. ``label`` representing the label given by an annotator for the corresponding task (i.e. example) + + Parameters + ---------- + labels_multiannotator_long : pd.DataFrame + pandas DataFrame in long format with three columns named ``task``, ``annotator`` and ``label`` + + Returns + ------- + labels_multiannotator_wide : pd.DataFrame + pandas DataFrame of the proper format to be passed as ``labels_multiannotator`` for the other ``cleanlab.multiannotator`` functions. + """ + labels_multiannotator_wide = labels_multiannotator_long.pivot( + index="task", columns="annotator", values="label" + ) + labels_multiannotator_wide.index.name = None + labels_multiannotator_wide.columns.name = None + return labels_multiannotator_wide + + +def _get_consensus_stats( + labels_multiannotator: np.ndarray, + pred_probs: np.ndarray, + num_annotations: np.ndarray, + consensus_label: np.ndarray, + quality_method: str = "crowdlab", + verbose: bool = True, + ensemble: bool = False, + label_quality_score_kwargs: dict = {}, +) -> tuple: + """Returns a tuple containing the consensus labels, annotator agreement scores, and quality of consensus + + Parameters + ---------- + labels_multiannotator : np.ndarray + 2D numpy array of multiple given labels for each example with shape ``(N, M)``, + where N is the number of examples and M is the number of annotators. + For more details, labels in the same format expected by the `~cleanlab.multiannotator.get_label_quality_multiannotator`. + pred_probs : np.ndarray + An array of shape ``(N, K)`` of model-predicted probabilities, ``P(label=k|x)``. + For details, predicted probabilities in the same format expected by the `~cleanlab.multiannotator.get_label_quality_multiannotator`. + num_annotations : np.ndarray + An array of shape ``(N,)`` with the number of annotators that have labeled each example. + consensus_label : np.ndarray + An array of shape ``(N,)`` with the consensus labels aggregated from all annotators. + quality_method : str, default = "crowdlab" (Options: ["crowdlab", "agreement"]) + Specifies the method used to calculate the quality of the consensus label. + For valid quality methods, view `~cleanlab.multiannotator.get_label_quality_multiannotator` + label_quality_score_kwargs : dict, optional + Keyword arguments to pass into ``get_label_quality_scores()``. + verbose : bool, default = True + Certain warnings and notes will be printed if ``verbose`` is set to ``True``. + ensemble : bool, default = False + Boolean flag to indicate whether the pred_probs passed are from ensemble models. + + Returns + ------ + stats : tuple + A tuple of (consensus_label, annotator_agreement, consensus_quality_score, post_pred_probs). + """ + + # compute the fraction of annotator agreeing with the consensus labels + annotator_agreement = _get_annotator_agreement_with_consensus( + labels_multiannotator=labels_multiannotator, + consensus_label=consensus_label, + ) + + # compute posterior predicted probabilites + if ensemble: + post_pred_probs, model_weight, annotator_weight = _get_post_pred_probs_and_weights_ensemble( + labels_multiannotator=labels_multiannotator, + consensus_label=consensus_label, + prior_pred_probs=pred_probs, + num_annotations=num_annotations, + annotator_agreement=annotator_agreement, + quality_method=quality_method, + verbose=verbose, + ) + else: + post_pred_probs, model_weight, annotator_weight = _get_post_pred_probs_and_weights( + labels_multiannotator=labels_multiannotator, + consensus_label=consensus_label, + prior_pred_probs=pred_probs, + num_annotations=num_annotations, + annotator_agreement=annotator_agreement, + quality_method=quality_method, + verbose=verbose, + ) + + # compute quality of the consensus labels + consensus_quality_score = _get_consensus_quality_score( + consensus_label=consensus_label, + pred_probs=post_pred_probs, + num_annotations=num_annotations, + annotator_agreement=annotator_agreement, + quality_method=quality_method, + label_quality_score_kwargs=label_quality_score_kwargs, + ) + + return ( + annotator_agreement, + consensus_quality_score, + post_pred_probs, + model_weight, + annotator_weight, + ) + + +def _get_annotator_stats( + labels_multiannotator: np.ndarray, + pred_probs: np.ndarray, + consensus_label: np.ndarray, + num_annotations: np.ndarray, + annotator_agreement: np.ndarray, + model_weight: np.ndarray, + annotator_weight: np.ndarray, + consensus_quality_score: np.ndarray, + detailed_label_quality: Optional[np.ndarray] = None, + annotator_ids: Optional[pd.Index] = None, + quality_method: str = "crowdlab", +) -> pd.DataFrame: + """Returns a dictionary containing overall statistics about each annotator. + + Parameters + ---------- + labels_multiannotator : np.ndarray + 2D numpy array of multiple given labels for each example with shape ``(N, M)``, + where N is the number of examples and M is the number of annotators. + For more details, labels in the same format expected by the `~cleanlab.multiannotator.get_label_quality_multiannotator`. + pred_probs : np.ndarray + An array of shape ``(N, K)`` of model-predicted probabilities, ``P(label=k|x)``. + For details, predicted probabilities in the same format expected by the `~cleanlab.multiannotator.get_label_quality_multiannotator`. + consensus_label : np.ndarray + An array of shape ``(N,)`` with the consensus labels aggregated from all annotators. + num_annotations : np.ndarray + An array of shape ``(N,)`` with the number of annotators that have labeled each example. + annotator_agreement : np.ndarray + An array of shape ``(N,)`` with the fraction of annotators that agree with each consensus label. + model_weight : float + float specifying the model weight used in weighted averages, + None if model weight is not used to compute quality scores + annotator_weight : np.ndarray + An array of shape ``(M,)`` where M is the number of annotators, specifying the annotator weights used in weighted averages, + None if annotator weights are not used to compute quality scores + consensus_quality_score : np.ndarray + An array of shape ``(N,)`` with the quality score of the consensus. + detailed_label_quality : + pandas DataFrame containing the detailed label quality scores for all examples and annotators + quality_method : str, default = "crowdlab" (Options: ["crowdlab", "agreement"]) + Specifies the method used to calculate the quality of the consensus label. + For valid quality methods, view `~cleanlab.multiannotator.get_label_quality_multiannotator` + + Returns + ------- + annotator_stats : pd.DataFrame + Overall statistics about each annotator. + For details, see the documentation of `~cleanlab.multiannotator.get_label_quality_multiannotator`. + """ + + annotator_quality = _get_annotator_quality( + labels_multiannotator=labels_multiannotator, + pred_probs=pred_probs, + consensus_label=consensus_label, + num_annotations=num_annotations, + annotator_agreement=annotator_agreement, + model_weight=model_weight, + annotator_weight=annotator_weight, + detailed_label_quality=detailed_label_quality, + quality_method=quality_method, + ) + + # Compute the number of labels labeled/ by each annotator + num_examples_labeled = np.sum(~np.isnan(labels_multiannotator), axis=0) + + # Compute the fraction of labels annotated by each annotator that agrees with the consensus label + # TODO: check if we should drop singleton labels here + agreement_with_consensus = np.zeros(labels_multiannotator.shape[1]) + for i in range(len(agreement_with_consensus)): + labels = labels_multiannotator[:, i] + labels_mask = ~np.isnan(labels) + agreement_with_consensus[i] = np.mean(labels[labels_mask] == consensus_label[labels_mask]) + + # Find the worst labeled class for each annotator + worst_class = _get_annotator_worst_class( + labels_multiannotator=labels_multiannotator, + consensus_label=consensus_label, + consensus_quality_score=consensus_quality_score, + ) + + # Create multi-annotator stats DataFrame from its columns + annotator_stats = pd.DataFrame( + { + "annotator_quality": annotator_quality, + "agreement_with_consensus": agreement_with_consensus, + "worst_class": worst_class, + "num_examples_labeled": num_examples_labeled, + }, + index=annotator_ids, + ) + + return annotator_stats.sort_values(by=["annotator_quality", "agreement_with_consensus"]) + + +def _get_annotator_agreement_with_consensus( + labels_multiannotator: np.ndarray, + consensus_label: np.ndarray, +) -> np.ndarray: + """Returns the fractions of annotators that agree with the consensus label per example. Note that the + fraction for each example only considers the annotators that labeled that particular example. + + Parameters + ---------- + labels_multiannotator : np.ndarray + 2D numpy array of multiple given labels for each example with shape ``(N, M)``, + where N is the number of examples and M is the number of annotators. + For more details, labels in the same format expected by the `~cleanlab.multiannotator.get_label_quality_multiannotator`. + consensus_label : np.ndarray + An array of shape ``(N,)`` with the consensus labels aggregated from all annotators. + + Returns + ------- + annotator_agreement : np.ndarray + An array of shape ``(N,)`` with the fraction of annotators that agree with each consensus label. + """ + annotator_agreement = np.zeros(len(labels_multiannotator)) + for i in range(labels_multiannotator.shape[1]): + annotator_agreement += labels_multiannotator[:, i] == consensus_label + annotator_agreement /= (~np.isnan(labels_multiannotator)).sum(axis=1) + return annotator_agreement + + +def _get_annotator_agreement_with_annotators( + labels_multiannotator: np.ndarray, + num_annotations: np.ndarray, + verbose: bool = True, +) -> np.ndarray: + """Returns the average agreement of each annotator with other annotators that label the same example. + + Parameters + ---------- + labels_multiannotator : np.ndarray + 2D numpy array of multiple given labels for each example with shape ``(N, M)``, + where N is the number of examples and M is the number of annotators. + For more details, labels in the same format expected by the `~cleanlab.multiannotator.get_label_quality_multiannotator`. + consensus_label : np.ndarray + An array of shape ``(N,)`` with the consensus labels aggregated from all annotators. + verbose : bool, default = True + Certain warnings and notes will be printed if ``verbose`` is set to ``True``. + + Returns + ------- + annotator_agreement : np.ndarray + An array of shape ``(M,)`` where M is the number of annotators, with the agreement of each annotator with other + annotators that labeled the same examples. + """ + + annotator_agreement_with_annotators = np.zeros(labels_multiannotator.shape[1]) + for i in range(len(annotator_agreement_with_annotators)): + annotator_labels = labels_multiannotator[:, i] + annotator_labels_mask = ~np.isnan(annotator_labels) + annotator_agreement_with_annotators[i] = _get_single_annotator_agreement( + labels_multiannotator[annotator_labels_mask], num_annotations[annotator_labels_mask], i + ) + + # impute average annotator accuracy for any annotator that do not overlap with other annotators + non_overlap_mask = np.isnan(annotator_agreement_with_annotators) + if np.sum(non_overlap_mask) > 0: + if verbose: + print( + f"Annotator(s) {list(np.where(non_overlap_mask)[0])} did not annotate any examples that overlap with other annotators, \ + \nusing the average annotator agreeement among other annotators as this annotator's agreement." + ) + + avg_annotator_agreement = np.mean(annotator_agreement_with_annotators[~non_overlap_mask]) + annotator_agreement_with_annotators[non_overlap_mask] = avg_annotator_agreement + + return annotator_agreement_with_annotators + + +def _get_single_annotator_agreement( + labels_multiannotator: np.ndarray, + num_annotations: np.ndarray, + annotator_idx: int, +) -> float: + """Returns the average agreement of a given annotator other annotators that label the same example. + + Parameters + ---------- + labels_multiannotator : np.ndarray + 2D numpy array of multiple given labels for each example with shape ``(N, M)``, + where N is the number of examples and M is the number of annotators. + For more details, labels in the same format expected by the `~cleanlab.multiannotator.get_label_quality_multiannotator`. + num_annotations : np.ndarray + An array of shape ``(N,)`` with the number of annotators that have labeled each example. + annotator_idx : int + The index of the annotator we want to compute the annotator agreement for. + + Returns + ------- + annotator_agreement : float + An float repesenting the agreement of each annotator with other annotators that labeled the same examples. + """ + adjusted_num_annotations = num_annotations - 1 + if np.sum(adjusted_num_annotations) == 0: + return np.nan + + multi_annotations_mask = num_annotations > 1 + annotator_agreement_per_example = np.zeros(len(labels_multiannotator)) + for i in range(labels_multiannotator.shape[1]): + annotator_agreement_per_example[multi_annotations_mask] += ( + labels_multiannotator[multi_annotations_mask, annotator_idx] + == labels_multiannotator[multi_annotations_mask, i] + ) + annotator_agreement_per_example[multi_annotations_mask] = ( + annotator_agreement_per_example[multi_annotations_mask] - 1 + ) / adjusted_num_annotations[multi_annotations_mask] + + annotator_agreement = np.average(annotator_agreement_per_example, weights=num_annotations - 1) + return annotator_agreement + + +def _get_post_pred_probs_and_weights( + labels_multiannotator: np.ndarray, + consensus_label: np.ndarray, + prior_pred_probs: np.ndarray, + num_annotations: np.ndarray, + annotator_agreement: np.ndarray, + quality_method: str = "crowdlab", + verbose: bool = True, +) -> Tuple[np.ndarray, Optional[float], Optional[np.ndarray]]: + """Return the posterior predicted probabilities of each example given a specified quality method. + + Parameters + ---------- + labels_multiannotator : np.ndarray + 2D numpy array of multiple given labels for each example with shape ``(N, M)``, + where N is the number of examples and M is the number of annotators. + For more details, labels in the same format expected by the `~cleanlab.multiannotator.get_label_quality_multiannotator`. + consensus_label : np.ndarray + An array of shape ``(N,)`` with the consensus labels aggregated from all annotators. + prior_pred_probs : np.ndarray + An array of shape ``(N, K)`` of prior predicted probabilities, ``P(label=k|x)``, usually the out-of-sample predicted probability computed by a model. + For details, predicted probabilities in the same format expected by the `~cleanlab.multiannotator.get_label_quality_multiannotator`. + num_annotations : np.ndarray + An array of shape ``(N,)`` with the number of annotators that have labeled each example. + annotator_agreement : np.ndarray + An array of shape ``(N,)`` with the fraction of annotators that agree with each consensus label. + quality_method : default = "crowdlab" (Options: ["crowdlab", "agreement"]) + Specifies the method used to calculate the quality of the consensus label. + For valid quality methods, view `~cleanlab.multiannotator.get_label_quality_multiannotator` + verbose : bool, default = True + Certain warnings and notes will be printed if ``verbose`` is set to ``True``. + + Returns + ------- + post_pred_probs : np.ndarray + An array of shape ``(N, K)`` with the posterior predicted probabilities. + + model_weight : float + float specifying the model weight used in weighted averages, + None if model weight is not used to compute quality scores + + annotator_weight : np.ndarray + An array of shape ``(M,)`` where M is the number of annotators, specifying the annotator weights used in weighted averages, + None if annotator weights are not used to compute quality scores + + """ + valid_methods = [ + "crowdlab", + "agreement", + ] + + # setting dummy variables for model and annotator weights that will be returned + # only relevant for quality_method == crowdlab, return None for all other methods + return_model_weight = None + return_annotator_weight = None + + if quality_method == "crowdlab": + num_classes = get_num_classes(pred_probs=prior_pred_probs) + + # likelihood that any annotator will or will not annotate the consensus label for any example + consensus_likelihood = np.mean(annotator_agreement[num_annotations != 1]) + non_consensus_likelihood = (1 - consensus_likelihood) / (num_classes - 1) + + # subsetting the dataset to only includes examples with more than one annotation + mask = num_annotations != 1 + consensus_label_subset = consensus_label[mask] + prior_pred_probs_subset = prior_pred_probs[mask] + + # compute most likely class error + most_likely_class_error = np.clip( + np.mean( + consensus_label_subset + != np.argmax(np.bincount(consensus_label_subset, minlength=num_classes)) + ), + a_min=CLIPPING_LOWER_BOUND, + a_max=None, + ) + + # compute adjusted annotator agreement (used as annotator weights) + annotator_agreement_with_annotators = _get_annotator_agreement_with_annotators( + labels_multiannotator, num_annotations, verbose + ) + annotator_error = 1 - annotator_agreement_with_annotators + adjusted_annotator_agreement = np.clip( + 1 - (annotator_error / most_likely_class_error), a_min=CLIPPING_LOWER_BOUND, a_max=None + ) + # compute model weight + model_error = np.mean(np.argmax(prior_pred_probs_subset, axis=1) != consensus_label_subset) + model_weight = np.max( + [(1 - (model_error / most_likely_class_error)), CLIPPING_LOWER_BOUND] + ) * np.sqrt(np.mean(num_annotations)) + + non_nan_mask = ~np.isnan(labels_multiannotator) + annotation_weight = np.zeros(labels_multiannotator.shape[0]) + for i in range(labels_multiannotator.shape[1]): + annotation_weight[non_nan_mask[:, i]] += adjusted_annotator_agreement[i] + total_weight = annotation_weight + model_weight + + # compute weighted average + post_pred_probs = np.full(prior_pred_probs.shape, np.nan) + for i in range(prior_pred_probs.shape[1]): + post_pred_probs[:, i] = prior_pred_probs[:, i] * model_weight + for k in range(labels_multiannotator.shape[1]): + mask = ~np.isnan(labels_multiannotator[:, k]) + post_pred_probs[mask, i] += np.where( + labels_multiannotator[mask, k] == i, + adjusted_annotator_agreement[k] * consensus_likelihood, + adjusted_annotator_agreement[k] * non_consensus_likelihood, + ) + post_pred_probs[:, i] /= total_weight + + return_model_weight = model_weight + return_annotator_weight = adjusted_annotator_agreement + + elif quality_method == "agreement": + num_classes = get_num_classes(pred_probs=prior_pred_probs) + label_counts = np.full((len(labels_multiannotator), num_classes), np.nan) + for i, labels in enumerate(labels_multiannotator): + label_counts[i, :] = value_counts(labels[~np.isnan(labels)], num_classes=num_classes) + + post_pred_probs = label_counts / num_annotations.reshape(-1, 1) + + else: + raise ValueError( + f""" + {quality_method} is not a valid quality method! + Please choose a valid quality_method: {valid_methods} + """ + ) + + return post_pred_probs, return_model_weight, return_annotator_weight + + +def _get_post_pred_probs_and_weights_ensemble( + labels_multiannotator: np.ndarray, + consensus_label: np.ndarray, + prior_pred_probs: np.ndarray, + num_annotations: np.ndarray, + annotator_agreement: np.ndarray, + quality_method: str = "crowdlab", + verbose: bool = True, +) -> Tuple[np.ndarray, Any, Any]: + """Return the posterior predicted class probabilites of each example given a specified quality method and prior predicted class probabilities from an ensemble of multiple classifier models. + + Parameters + ---------- + labels_multiannotator : np.ndarray + 2D numpy array of multiple given labels for each example with shape ``(N, M)``, + where N is the number of examples and M is the number of annotators. + For more details, labels in the same format expected by the `~cleanlab.multiannotator.get_label_quality_multiannotator`. + consensus_label : np.ndarray + An array of shape ``(P, N, K)`` where P is the number of models, consisting of predicted class probabilities from the ensemble models. + Each set of predicted probabilities with shape ``(N, K)`` is in the same format expected by the :py:func:`get_label_quality_scores `. + prior_pred_probs : np.ndarray + An array of shape ``(N, K)`` of prior predicted probabilities, ``P(label=k|x)``, usually the out-of-sample predicted probability computed by a model. + For details, predicted probabilities in the same format expected by the `~cleanlab.multiannotator.get_label_quality_multiannotator`. + num_annotations : np.ndarray + An array of shape ``(N,)`` with the number of annotators that have labeled each example. + annotator_agreement : np.ndarray + An array of shape ``(N,)`` with the fraction of annotators that agree with each consensus label. + quality_method : str, default = "crowdlab" (Options: ["crowdlab", "agreement"]) + Specifies the method used to calculate the quality of the consensus label. + For valid quality methods, view `~cleanlab.multiannotator.get_label_quality_multiannotator` + verbose : bool, default = True + Certain warnings and notes will be printed if ``verbose`` is set to ``True``. + + Returns + ------- + post_pred_probs : np.ndarray + An array of shape ``(N, K)`` with the posterior predicted probabilities. + + model_weight : np.ndarray + An array of shape ``(P,)`` where P is the number of models in this ensemble, specifying the model weight used in weighted averages, + ``None`` if model weight is not used to compute quality scores + + annotator_weight : np.ndarray + An array of shape ``(M,)`` where M is the number of annotators, specifying the annotator weights used in weighted averages, + ``None`` if annotator weights are not used to compute quality scores + + """ + + num_classes = get_num_classes(pred_probs=prior_pred_probs[0]) + + # likelihood that any annotator will or will not annotate the consensus label for any example + consensus_likelihood = np.mean(annotator_agreement[num_annotations != 1]) + non_consensus_likelihood = (1 - consensus_likelihood) / (num_classes - 1) + + # subsetting the dataset to only includes examples with more than one annotation + mask = num_annotations != 1 + consensus_label_subset = consensus_label[mask] + + # compute most likely class error + most_likely_class_error = np.clip( + np.mean( + consensus_label_subset + != np.argmax(np.bincount(consensus_label_subset, minlength=num_classes)) + ), + a_min=CLIPPING_LOWER_BOUND, + a_max=None, + ) + + # compute adjusted annotator agreement (used as annotator weights) + annotator_agreement_with_annotators = _get_annotator_agreement_with_annotators( + labels_multiannotator, num_annotations, verbose + ) + annotator_error = 1 - annotator_agreement_with_annotators + adjusted_annotator_agreement = np.clip( + 1 - (annotator_error / most_likely_class_error), a_min=CLIPPING_LOWER_BOUND, a_max=None + ) + + # compute model weight + model_weight = np.full(prior_pred_probs.shape[0], np.nan) + for idx in range(prior_pred_probs.shape[0]): + prior_pred_probs_subset = prior_pred_probs[idx][mask] + + model_error = np.mean(np.argmax(prior_pred_probs_subset, axis=1) != consensus_label_subset) + model_weight[idx] = np.max( + [(1 - (model_error / most_likely_class_error)), CLIPPING_LOWER_BOUND] + ) * np.sqrt(np.mean(num_annotations)) + + # compute weighted average + post_pred_probs = np.full(prior_pred_probs[0].shape, np.nan) + for i, labels in enumerate(labels_multiannotator): + labels_mask = ~np.isnan(labels) + labels_subset = labels[labels_mask] + post_pred_probs[i] = [ + np.average( + [prior_pred_probs[ind][i, true_label] for ind in range(prior_pred_probs.shape[0])] + + [ + ( + consensus_likelihood + if annotator_label == true_label + else non_consensus_likelihood + ) + for annotator_label in labels_subset + ], + weights=np.concatenate((model_weight, adjusted_annotator_agreement[labels_mask])), + ) + for true_label in range(num_classes) + ] + + return_model_weight = model_weight + return_annotator_weight = adjusted_annotator_agreement + + return post_pred_probs, return_model_weight, return_annotator_weight + + +def _get_consensus_quality_score( + consensus_label: np.ndarray, + pred_probs: np.ndarray, + num_annotations: np.ndarray, + annotator_agreement: np.ndarray, + quality_method: str = "crowdlab", + label_quality_score_kwargs: dict = {}, +) -> np.ndarray: + """Return scores representing quality of the consensus label for each example. + + Parameters + ---------- + labels_multiannotator : np.ndarray + 2D numpy array of multiple given labels for each example with shape ``(N, M)``, + where N is the number of examples and M is the number of annotators. + For more details, labels in the same format expected by the `~cleanlab.multiannotator.get_label_quality_multiannotator`. + consensus_label : np.ndarray + An array of shape ``(N,)`` with the consensus labels aggregated from all annotators. + pred_probs : np.ndarray + An array of shape ``(N, K)`` of posterior predicted probabilities, ``P(label=k|x)``. + For details, predicted probabilities in the same format expected by the `~cleanlab.multiannotator.get_label_quality_multiannotator`. + num_annotations : np.ndarray + An array of shape ``(N,)`` with the number of annotators that have labeled each example. + annotator_agreement : np.ndarray + An array of shape ``(N,)`` with the fraction of annotators that agree with each consensus label. + quality_method : str, default = "crowdlab" (Options: ["crowdlab", "agreement"]) + Specifies the method used to calculate the quality of the consensus label. + For valid quality methods, view `~cleanlab.multiannotator.get_label_quality_multiannotator` + + Returns + ------- + consensus_quality_score : np.ndarray + An array of shape ``(N,)`` with the quality score of the consensus. + """ + + valid_methods = [ + "crowdlab", + "agreement", + ] + + if quality_method == "crowdlab": + consensus_quality_score = get_label_quality_scores( + consensus_label, pred_probs, **label_quality_score_kwargs + ) + + elif quality_method == "agreement": + consensus_quality_score = annotator_agreement + + else: + raise ValueError( + f""" + {quality_method} is not a valid consensus quality method! + Please choose a valid quality_method: {valid_methods} + """ + ) + + return consensus_quality_score + + +def _get_annotator_label_quality_score( + annotator_label: np.ndarray, + pred_probs: np.ndarray, + label_quality_score_kwargs: dict = {}, +) -> np.ndarray: + """Returns quality scores for each datapoint. + Very similar functionality as ``_get_consensus_quality_score`` with additional support for annotator labels that contain NaN values. + For more info about parameters and returns, see the docstring of `~cleanlab.multiannotator._get_consensus_quality_score`. + """ + mask = ~np.isnan(annotator_label) + + annotator_label_quality_score_subset = get_label_quality_scores( + labels=annotator_label[mask].astype(int), + pred_probs=pred_probs[mask], + **label_quality_score_kwargs, + ) + + annotator_label_quality_score = np.full(len(annotator_label), np.nan) + annotator_label_quality_score[mask] = annotator_label_quality_score_subset + return annotator_label_quality_score + + +def _get_annotator_quality( + labels_multiannotator: np.ndarray, + pred_probs: np.ndarray, + consensus_label: np.ndarray, + num_annotations: np.ndarray, + annotator_agreement: np.ndarray, + model_weight: np.ndarray, + annotator_weight: np.ndarray, + detailed_label_quality: Optional[np.ndarray] = None, + quality_method: str = "crowdlab", +) -> pd.DataFrame: + """Returns annotator quality score for each annotator. + + Parameters + ---------- + labels_multiannotator : np.ndarray + 2D numpy array of multiple given labels for each example with shape ``(N, M)``, + where N is the number of examples and M is the number of annotators. + For more details, labels in the same format expected by the `~cleanlab.multiannotator.get_label_quality_multiannotator`. + pred_probs : np.ndarray + An array of shape ``(N, K)`` of model-predicted probabilities, ``P(label=k|x)``. + For details, predicted probabilities in the same format expected by the `~cleanlab.multiannotator.get_label_quality_multiannotator`. + consensus_label : np.ndarray + An array of shape ``(N,)`` with the consensus labels aggregated from all annotators. + num_annotations : np.ndarray + An array of shape ``(N,)`` with the number of annotators that have labeled each example. + annotator_agreement : np.ndarray + An array of shape ``(N,)`` with the fraction of annotators that agree with each consensus label. + model_weight : float + An array of shape ``(P,)`` where P is the number of models in this ensemble, specifying the model weight used in weighted averages, + ``None`` if model weight is not used to compute quality scores + annotator_weight : np.ndarray + An array of shape ``(M,)`` where M is the number of annotators, specifying the annotator weights used in weighted averages, + ``None`` if annotator weights are not used to compute quality scores + detailed_label_quality : + pandas DataFrame containing the detailed label quality scores for all examples and annotators + quality_method : str, default = "crowdlab" (Options: ["crowdlab", "agreement"]) + Specifies the method used to calculate the quality of the annotators. + For valid quality methods, view `~cleanlab.multiannotator.get_label_quality_multiannotator` + + Returns + ------- + annotator_quality : np.ndarray + Quality scores of a given annotator's labels + """ + + valid_methods = [ + "crowdlab", + "agreement", + ] + + if quality_method == "crowdlab": + if detailed_label_quality is None: + annotator_lqs = np.zeros(labels_multiannotator.shape[1]) + for i in range(len(annotator_lqs)): + labels = labels_multiannotator[:, i] + labels_mask = ~np.isnan(labels) + annotator_lqs[i] = np.mean( + get_label_quality_scores( + labels[labels_mask].astype(int), + pred_probs[labels_mask], + ) + ) + else: + annotator_lqs = np.nanmean(detailed_label_quality, axis=0) + + mask = num_annotations != 1 + labels_multiannotator_subset = labels_multiannotator[mask] + consensus_label_subset = consensus_label[mask] + + annotator_agreement = np.zeros(labels_multiannotator_subset.shape[1]) + for i in range(len(annotator_agreement)): + labels = labels_multiannotator_subset[:, i] + labels_mask = ~np.isnan(labels) + # case where annotator does not annotate any examples with any other annotators + # TODO: do we want to impute the mean or just return np.nan + if np.sum(labels_mask) == 0: + annotator_agreement[i] = np.nan + else: + annotator_agreement[i] = np.mean( + labels[labels_mask] == consensus_label_subset[labels_mask], + ) + + avg_num_annotations_frac = np.mean(num_annotations) / len(annotator_weight) + annotator_weight_adjusted = np.sum(annotator_weight) * avg_num_annotations_frac + + w = model_weight / (model_weight + annotator_weight_adjusted) + annotator_quality = w * annotator_lqs + (1 - w) * annotator_agreement + + elif quality_method == "agreement": + mask = num_annotations != 1 + labels_multiannotator_subset = labels_multiannotator[mask] + consensus_label_subset = consensus_label[mask] + + annotator_quality = np.zeros(labels_multiannotator_subset.shape[1]) + for i in range(len(annotator_quality)): + labels = labels_multiannotator_subset[:, i] + labels_mask = ~np.isnan(labels) + # case where annotator does not annotate any examples with any other annotators + if np.sum(labels_mask) == 0: + annotator_quality[i] = np.nan + else: + annotator_quality[i] = np.mean( + labels[labels_mask] == consensus_label_subset[labels_mask], + ) + + else: + raise ValueError( + f""" + {quality_method} is not a valid annotator quality method! + Please choose a valid quality_method: {valid_methods} + """ + ) + + return annotator_quality + + +def _get_annotator_worst_class( + labels_multiannotator: np.ndarray, + consensus_label: np.ndarray, + consensus_quality_score: np.ndarray, +) -> np.ndarray: + """Returns the class which each annotator makes the most errors in. + + Parameters + ---------- + labels_multiannotator : np.ndarray + 2D pandas DataFrame of multiple given labels for each example with shape ``(N, M)``, + where N is the number of examples and M is the number of annotators. + For more details, labels in the same format expected by the `~cleanlab.multiannotator.get_label_quality_multiannotator`. + consensus_label : np.ndarray + An array of shape ``(N,)`` with the consensus labels aggregated from all annotators. + consensus_quality_score : np.ndarray + An array of shape ``(N,)`` with the quality score of the consensus. + + Returns + ------- + worst_class : np.ndarray + The class that is most frequently mislabeled by a given annotator. + """ + + worst_class = np.apply_along_axis( + _get_single_annotator_worst_class, + axis=0, + arr=labels_multiannotator, + consensus_label=consensus_label, + consensus_quality_score=consensus_quality_score, + ).astype(int) + + return worst_class + + +def _get_single_annotator_worst_class( + labels: np.ndarray, + consensus_label: np.ndarray, + consensus_quality_score: np.ndarray, +) -> int: + """Returns the class a given annotator makes the most errors in. + + Parameters + ---------- + labels : np.ndarray + An array of shape ``(N,)`` with the labels from the annotator we want to evaluate. + consensus_label : np.ndarray + An array of shape ``(N,)`` with the consensus labels aggregated from all annotators. + consensus_quality_score : np.ndarray + An array of shape ``(N,)`` with the quality score of the consensus. + + Returns + ------- + worst_class : int + The class that is most frequently mislabeled by the given annotator. + """ + labels = pd.Series(labels) + labels_mask = pd.notna(labels) + class_accuracies = (labels[labels_mask] == consensus_label[labels_mask]).groupby(labels).mean() + accuracy_min_idx = class_accuracies[class_accuracies == class_accuracies.min()].index.values + + if len(accuracy_min_idx) == 1: + return accuracy_min_idx[0] + + # tiebreak 1: class counts + class_count = labels[labels_mask].groupby(labels).count()[accuracy_min_idx] + count_max_idx = class_count[class_count == class_count.max()].index.values + + if len(count_max_idx) == 1: + return count_max_idx[0] + + # tiebreak 2: consensus quality scores + avg_consensus_quality = ( + pd.DataFrame( + {"annotator_label": labels, "consensus_quality_score": consensus_quality_score} + )[labels_mask] + .groupby("annotator_label") + .mean()["consensus_quality_score"][count_max_idx] + ) + quality_max_idx = avg_consensus_quality[ + avg_consensus_quality == avg_consensus_quality.max() + ].index.values + + # return first item even if there are ties - no better methods to tiebreak + return quality_max_idx[0] diff --git a/cleanlab/multilabel_classification/__init__.py b/cleanlab/multilabel_classification/__init__.py new file mode 100644 index 0000000..fcddecd --- /dev/null +++ b/cleanlab/multilabel_classification/__init__.py @@ -0,0 +1,4 @@ +from .rank import get_label_quality_scores +from . import rank +from . import dataset +from . import filter diff --git a/cleanlab/multilabel_classification/dataset.py b/cleanlab/multilabel_classification/dataset.py new file mode 100644 index 0000000..4f80075 --- /dev/null +++ b/cleanlab/multilabel_classification/dataset.py @@ -0,0 +1,326 @@ +""" +Methods to summarize overall labeling issues across a multi-label classification dataset. +Here each example can belong to one or more classes, or none of the classes at all. +Unlike in standard multi-class classification, model-predicted class probabilities need not sum to 1 for each row in multi-label classification. +""" + +import pandas as pd +import numpy as np +from typing import Optional, cast, Dict, Any # noqa: F401 +from cleanlab.multilabel_classification.filter import ( + find_multilabel_issues_per_class, + find_label_issues, +) +from cleanlab.internal.multilabel_utils import get_onehot_num_classes +from collections import defaultdict + + +def common_multilabel_issues( + labels=list, + pred_probs=None, + *, + class_names=None, + confident_joint=None, +) -> pd.DataFrame: + """Summarizes which classes in a multi-label dataset appear most often mislabeled overall. + + Since classes are not mutually exclusive in multi-label classification, this method summarizes the label issues for each class independently of the others. + + Parameters + ---------- + labels : List[List[int]] + List of noisy labels for multi-label classification where each example can belong to multiple classes. + Refer to documentation for this argument in :py:func:`multilabel_classification.filter.find_label_issues ` for further details. + + pred_probs : np.ndarray + An array of shape ``(N, K)`` of model-predicted class probabilities. + Refer to documentation for this argument in :py:func:`multilabel_classification.filter.find_label_issues ` for further details. + + class_names : Iterable[str], optional + A list or other iterable of the string class names. Its order must match the label indices. + If class 0 is 'dog' and class 1 is 'cat', then ``class_names = ['dog', 'cat']``. + If provided, the returned DataFrame will have an extra *Class Name* column with this info. + + confident_joint : np.ndarray, optional + An array of shape ``(K, 2, 2)`` representing a one-vs-rest formatted confident joint. + Refer to documentation for this argument in :py:func:`multilabel_classification.filter.find_label_issues ` for details. + + Returns + ------- + common_multilabel_issues : pd.DataFrame + DataFrame where each row corresponds to a class summarized by the following columns: + - *Class Name*: The name of the class if class_names is provided. + - *Class Index*: The index of the class. + - *In Given Label*: Whether the Class is originally annotated True or False in the given label. + - *In Suggested Label*: Whether the Class should be True or False in the suggested label (based on model's prediction). + - *Num Examples*: Number of examples flagged as a label issue where this Class is True/False "In Given Label" but cleanlab estimates the annotation should actually be as specified "In Suggested Label". I.e. the number of examples in your dataset where this Class was labeled as True but likely should have been False (or vice versa). + - *Issue Probability*: The *Num Examples* column divided by the total number of examples in the dataset; i.e. the relative overall frequency of each type of label issue in your dataset. + + By default, the rows in this DataFrame are ordered by "Issue Probability" (descending). + """ + + num_examples = _get_num_examples_multilabel(labels=labels, confident_joint=confident_joint) + summary_issue_counts = defaultdict(list) + y_one, num_classes = get_onehot_num_classes(labels, pred_probs) + label_issues_list, labels_list, pred_probs_list = find_multilabel_issues_per_class( + labels=labels, + pred_probs=pred_probs, + confident_joint=confident_joint, + return_indices_ranked_by="self_confidence", + ) + + for class_num, (label, issues_for_class) in enumerate(zip(y_one.T, label_issues_list)): + binary_label_issues = np.zeros(len(label)).astype(bool) + binary_label_issues[issues_for_class] = True + true_but_false_count = sum(np.logical_and(label == 1, binary_label_issues)) + false_but_true_count = sum(np.logical_and(label == 0, binary_label_issues)) + + if class_names is not None: + summary_issue_counts["Class Name"].append(class_names[class_num]) + summary_issue_counts["Class Index"].append(class_num) + summary_issue_counts["In Given Label"].append(True) + summary_issue_counts["In Suggested Label"].append(False) + summary_issue_counts["Num Examples"].append(true_but_false_count) + summary_issue_counts["Issue Probability"].append(true_but_false_count / num_examples) + + if class_names is not None: + summary_issue_counts["Class Name"].append(class_names[class_num]) + summary_issue_counts["Class Index"].append(class_num) + summary_issue_counts["In Given Label"].append(False) + summary_issue_counts["In Suggested Label"].append(True) + summary_issue_counts["Num Examples"].append(false_but_true_count) + summary_issue_counts["Issue Probability"].append(false_but_true_count / num_examples) + return ( + pd.DataFrame.from_dict(summary_issue_counts) + .sort_values(by=["Issue Probability"], ascending=False) + .reset_index(drop=True) + ) + + +def rank_classes_by_multilabel_quality( + labels=None, + pred_probs=None, + *, + class_names=None, + joint=None, + confident_joint=None, +) -> pd.DataFrame: + """ + Returns a DataFrame with three overall label quality scores per class for a multi-label dataset. + + These numbers summarize all examples annotated with the class (details listed below under the Returns parameter). + By default, classes are ordered by "Label Quality Score", so the most problematic classes are reported first in the DataFrame. + + Score values are unnormalized and may be very small. What matters is their relative ranking across the classes. + + **Parameters**: + + For information about the arguments to this method, see the documentation of + `~cleanlab.multilabel_classification.dataset.common_multilabel_issues`. + + Returns + ------- + overall_label_quality : pd.DataFrame + Pandas DataFrame with one row per class and columns: "Class Index", "Label Issues", + "Inverse Label Issues", "Label Issues", "Inverse Label Noise", "Label Quality Score". + Some entries are overall quality scores between 0 and 1, summarizing how good overall the labels + appear to be for that class (lower values indicate more erroneous labels). + Other entries are estimated counts of annotation errors related to this class. + + Here is what each column represents: + - *Class Name*: The name of the class if class_names is provided. + - *Class Index*: The index of the class in 0, 1, ..., K-1. + - *Label Issues*: Estimated number of examples in the dataset that are labeled as belonging to class k but actually should not belong to this class. + - *Inverse Label Issues*: Estimated number of examples in the dataset that should actually be labeled as class k but did not receive this label. + - *Label Noise*: Estimated proportion of examples in the dataset that are labeled as class k but should not be. For each class k: this is computed by dividing the number of examples with "Label Issues" that were labeled as class k by the total number of examples labeled as class k. + - *Inverse Label Noise*: Estimated proportion of examples in the dataset that should actually be labeled as class k but did not receive this label. + - *Label Quality Score*: Estimated proportion of examples labeled as class k that have been labeled correctly, i.e. ``1 - label_noise``. + + By default, the DataFrame is ordered by "Label Quality Score" (in ascending order), so the classes with the most label issues appear first. + """ + + issues_df = common_multilabel_issues( + labels=labels, pred_probs=pred_probs, class_names=class_names, confident_joint=joint + ) + issues_dict = defaultdict(defaultdict) # type: Dict[str, Any] + num_examples = _get_num_examples_multilabel(labels=labels, confident_joint=confident_joint) + return_columns = [ + "Class Name", + "Class Index", + "Label Issues", + "Inverse Label Issues", + "Label Noise", + "Inverse Label Noise", + "Label Quality Score", + ] + if class_names is None: + return_columns = return_columns[1:] + for class_num, row in issues_df.iterrows(): + if row["In Given Label"]: + if class_names is not None: + issues_dict[row["Class Index"]]["Class Name"] = row["Class Name"] + issues_dict[row["Class Index"]]["Label Issues"] = int( + row["Issue Probability"] * num_examples + ) + issues_dict[row["Class Index"]]["Label Noise"] = row["Issue Probability"] + issues_dict[row["Class Index"]]["Label Quality Score"] = ( + 1 - issues_dict[row["Class Index"]]["Label Noise"] + ) + else: + if class_names is not None: + issues_dict[row["Class Index"]]["Class Name"] = row["Class Name"] + issues_dict[row["Class Index"]]["Inverse Label Issues"] = int( + row["Issue Probability"] * num_examples + ) + issues_dict[row["Class Index"]]["Inverse Label Noise"] = row["Issue Probability"] + + issues_df_dict = defaultdict(list) + for i in issues_dict: + issues_df_dict["Class Index"].append(i) + for j in issues_dict[i]: + issues_df_dict[j].append(issues_dict[i][j]) + return ( + pd.DataFrame.from_dict(issues_df_dict) + .sort_values(by="Label Quality Score", ascending=True) + .reset_index(drop=True) + )[return_columns] + + +def _get_num_examples_multilabel(labels=None, confident_joint: Optional[np.ndarray] = None) -> int: + """Helper method that finds the number of examples from the parameters or throws an error + if neither parameter is provided. + + Parameters + ---------- + For parameter info, see the docstring of `~cleanlab.multilabel_classification.dataset.common_multilabel_issues`. + + Returns + ------- + num_examples : int + The number of examples in the dataset. + + Raises + ------ + ValueError + If `labels` is None. + """ + + if labels is None and confident_joint is None: + raise ValueError( + "Error: num_examples is None. You must either provide confident_joint, " + "or provide both num_example and joint as input parameters." + ) + _confident_joint = cast(np.ndarray, confident_joint) + num_examples = len(labels) if labels is not None else cast(int, np.sum(_confident_joint[0])) + return num_examples + + +def overall_multilabel_health_score( + labels=None, + pred_probs=None, + *, + confident_joint=None, +) -> float: + """Returns a single score between 0 and 1 measuring the overall quality of all labels in a multi-label classification dataset. + Intuitively, the score is the average correctness of the given labels across all examples in the + dataset. So a score of 1 suggests your data is perfectly labeled and a score of 0.5 suggests + half of the examples in the dataset may be incorrectly labeled. Thus, a higher + score implies a higher quality dataset. + + **Parameters**: For information about the arguments to this method, see the documentation of + `~cleanlab.multilabel_classification.dataset.common_multilabel_issues`. + + Returns + ------- + health_score : float + A overall score between 0 and 1, where 1 implies all labels in the dataset are estimated to be correct. + A score of 0.5 implies that half of the dataset's labels are estimated to have issues. + """ + num_examples = _get_num_examples_multilabel(labels=labels) + issues = find_label_issues( + labels=labels, pred_probs=pred_probs, confident_joint=confident_joint + ) + return 1.0 - sum(issues) / num_examples + + +def multilabel_health_summary( + labels=None, + pred_probs=None, + *, + class_names=None, + num_examples=None, + confident_joint=None, + verbose=True, +) -> Dict: + """Prints a health summary of your multi-label dataset. + + This summary includes useful statistics like: + + * The classes with the most and least label issues. + * Overall label quality scores, summarizing how accurate the labels appear across the entire dataset. + + **Parameters**: For information about the arguments to this method, see the documentation of + `~cleanlab.multilabel_classification.dataset.common_multilabel_issues`. + + Returns + ------- + summary : dict + A dictionary containing keys (see the corresponding functions' documentation to understand the values): + - ``"overall_label_health_score"``, corresponding to output of `~cleanlab.multilabel_classification.dataset.overall_multilabel_health_score` + - ``"classes_by_multilabel_quality"``, corresponding to output of `~cleanlab.multilabel_classification.dataset.rank_classes_by_multilabel_quality` + - ``"common_multilabel_issues"``, corresponding to output of `~cleanlab.multilabel_classification.dataset.common_multilabel_issues` + """ + from cleanlab.internal.util import smart_display_dataframe + + if num_examples is None: + num_examples = _get_num_examples_multilabel(labels=labels) + + if verbose: + longest_line = f"| for your dataset with {num_examples:,} examples " + print( + "-" * (len(longest_line) - 1) + + "\n" + + f"| Generating a Cleanlab Dataset Health Summary{' ' * (len(longest_line) - 49)}|\n" + + longest_line + + f"| Note, Cleanlab is not a medical doctor... yet.{' ' * (len(longest_line) - 51)}|\n" + + "-" * (len(longest_line) - 1) + + "\n", + ) + + df_class_label_quality = rank_classes_by_multilabel_quality( + labels=labels, + pred_probs=pred_probs, + class_names=class_names, + confident_joint=confident_joint, + ) + if verbose: + print("Overall Class Quality and Noise across your dataset (below)") + print("-" * 60, "\n", flush=True) + smart_display_dataframe(df_class_label_quality) + + df_common_issues = common_multilabel_issues( + labels=labels, + pred_probs=pred_probs, + class_names=class_names, + confident_joint=confident_joint, + ) + if verbose: + print( + "\nCommon multilabel issues are" + "\n" + "-" * 83 + "\n", + flush=True, + ) + smart_display_dataframe(df_common_issues) + print() + + health_score = overall_multilabel_health_score( + labels=labels, + pred_probs=pred_probs, + confident_joint=confident_joint, + ) + if verbose: + print("\nGenerated with <3 from Cleanlab.\n") + return { + "overall_multilabel_health_score": health_score, + "classes_by_multilabel_quality": df_class_label_quality, + "common_multilabel_issues": df_common_issues, + } diff --git a/cleanlab/multilabel_classification/filter.py b/cleanlab/multilabel_classification/filter.py new file mode 100644 index 0000000..5881687 --- /dev/null +++ b/cleanlab/multilabel_classification/filter.py @@ -0,0 +1,303 @@ +""" +Methods to flag which examples have label issues in multi-label classification datasets. +Here each example can belong to one or more classes, or none of the classes at all. +Unlike in standard multi-class classification, model-predicted class probabilities need not sum to 1 for each row in multi-label classification. +""" + +import warnings +import inspect +from typing import Optional, Union, Tuple, List, Any +import numpy as np + + +def find_label_issues( + labels: list, + pred_probs: np.ndarray, + return_indices_ranked_by: Optional[str] = None, + rank_by_kwargs={}, + filter_by: str = "prune_by_noise_rate", + frac_noise: float = 1.0, + num_to_remove_per_class: Optional[List[int]] = None, + min_examples_per_class=1, + confident_joint: Optional[np.ndarray] = None, + n_jobs: Optional[int] = None, + verbose: bool = False, + low_memory: bool = False, +) -> np.ndarray: + """ + Identifies potentially mislabeled examples in a multi-label classification dataset. + An example is flagged as with a label issue if *any* of the classes appear to be incorrectly annotated for this example. + + Parameters + ---------- + labels : List[List[int]] + List of noisy labels for multi-label classification where each example can belong to multiple classes. + This is an iterable of iterables where the i-th element of `labels` corresponds to a list of classes that the i-th example belongs to, + according to the original data annotation (e.g. ``labels = [[1,2],[1],[0],..]``). + This method will return the indices i where the inner list ``labels[i]`` is estimated to have some error. + For a dataset with K classes, each class must be represented as an integer in 0, 1, ..., K-1 within the labels. + + pred_probs : np.ndarray + An array of shape ``(N, K)`` of model-predicted class probabilities. + Each row of this matrix corresponds to an example `x` + and contains the predicted probability that `x` belongs to each possible class, + for each of the K classes (along its columns). + The columns need not sum to 1 but must be ordered such that + these probabilities correspond to class 0, 1, ..., K-1. + + Note + ---- + Estimated label quality scores are most accurate when they are computed based on out-of-sample ``pred_probs`` from your model. + To obtain out-of-sample predicted probabilities for every example in your dataset, you can use :ref:`cross-validation `. + This is encouraged to get better results. + + return_indices_ranked_by : {None, 'self_confidence', 'normalized_margin', 'confidence_weighted_entropy'}, default = None + This function can return a boolean mask (if None) or an array of the example-indices with issues sorted based on the specified ranking method. + Refer to documentation for this argument in :py:func:`filter.find_label_issues ` for details. + + rank_by_kwargs : dict, optional + Optional keyword arguments to pass into scoring functions for ranking by + label quality score (see :py:func:`rank.get_label_quality_scores + `). + + filter_by : {'prune_by_class', 'prune_by_noise_rate', 'both', 'confident_learning', 'predicted_neq_given', 'low_normalized_margin', 'low_self_confidence'}, default='prune_by_noise_rate' + The specific Confident Learning method to determine precisely which examples have label issues in a dataset. + Refer to documentation for this argument in :py:func:`filter.find_label_issues ` for details. + + frac_noise : float, default = 1.0 + This will return the "top" frac_noise * num_label_issues estimated label errors, dependent on the filtering method used, + Refer to documentation for this argument in :py:func:`filter.find_label_issues ` for details. + + num_to_remove_per_class : array_like + An iterable that specifies the number of mislabeled examples to return from each class. + Refer to documentation for this argument in :py:func:`filter.find_label_issues ` for details. + + min_examples_per_class : int, default = 1 + The minimum number of examples required per class below which examples from this class will not be flagged as label issues. + Refer to documentation for this argument in :py:func:`filter.find_label_issues ` for details. + + confident_joint : np.ndarray, optional + An array of shape ``(K, 2, 2)`` representing a one-vs-rest formatted confident joint, as is appropriate for multi-label classification tasks. + Entry ``(c, i, j)`` in this array is the number of examples confidently counted into a ``(class c, noisy label=i, true label=j)`` bin, + where `i, j` are either 0 or 1 to denote whether this example belongs to class `c` or not + (recall examples can belong to multiple classes in multi-label classification). + The `confident_joint` can be computed using :py:func:`count.compute_confident_joint ` with ``multi_label=True``. + If not provided, it is computed from the given (noisy) `labels` and `pred_probs`. + + n_jobs : optional + Number of processing threads used by multiprocessing. + Refer to documentation for this argument in :py:func:`filter.find_label_issues ` for details. + + verbose : optional + If ``True``, prints when multiprocessing happens. + + low_memory: bool, default=False + Set as ``True`` if you have a big dataset with limited memory. + Uses :py:func:`experimental.label_issues_batched.find_label_issues_batched ` + + Returns + ------- + label_issues : np.ndarray + If `return_indices_ranked_by` left unspecified, returns a boolean **mask** for the entire dataset + where ``True`` represents an example suffering from some label issue and + ``False`` represents an example that appears accurately labeled. + + If `return_indices_ranked_by` is specified, this method instead returns a list of **indices** of examples identified with + label issues (i.e. those indices where the mask would be ``True``). + Indices are sorted by the likelihood that *all* classes are correctly annotated for the corresponding example. + + Note + ---- + Obtain the *indices* of examples with label issues in your dataset by setting + `return_indices_ranked_by`. + + """ + from cleanlab.filter import _find_label_issues_multilabel + + if low_memory: + if rank_by_kwargs: + warnings.warn(f"`rank_by_kwargs` is not used when `low_memory=True`.") + + func_signature = inspect.signature(find_label_issues) + default_args = { + k: v.default + for k, v in func_signature.parameters.items() + if v.default is not inspect.Parameter.empty + } + arg_values = { + "filter_by": filter_by, + "num_to_remove_per_class": num_to_remove_per_class, + "confident_joint": confident_joint, + "n_jobs": n_jobs, + "num_to_remove_per_class": num_to_remove_per_class, + "frac_noise": frac_noise, + "min_examples_per_class": min_examples_per_class, + } + for arg_name, arg_val in arg_values.items(): + if arg_val != default_args[arg_name]: + warnings.warn(f"`{arg_name}` is not used when `low_memory=True`.") + + return _find_label_issues_multilabel( + labels=labels, + pred_probs=pred_probs, + return_indices_ranked_by=return_indices_ranked_by, + rank_by_kwargs=rank_by_kwargs, + filter_by=filter_by, + frac_noise=frac_noise, + num_to_remove_per_class=num_to_remove_per_class, + min_examples_per_class=min_examples_per_class, + confident_joint=confident_joint, + n_jobs=n_jobs, + verbose=verbose, + low_memory=low_memory, + ) + + +def find_multilabel_issues_per_class( + labels: list, + pred_probs: np.ndarray, + return_indices_ranked_by: Optional[str] = None, + rank_by_kwargs={}, + filter_by: str = "prune_by_noise_rate", + frac_noise: float = 1.0, + num_to_remove_per_class: Optional[List[int]] = None, + min_examples_per_class=1, + confident_joint: Optional[np.ndarray] = None, + n_jobs: Optional[int] = None, + verbose: bool = False, + low_memory: bool = False, +) -> Union[np.ndarray, Tuple[List[np.ndarray], List[Any], List[np.ndarray]]]: + """ + Identifies potentially bad labels for each example and each class in a multi-label classification dataset. + Whereas `~cleanlab.multilabel_classification.filter.find_label_issues` + estimates which examples have an erroneous annotation for *any* class, this method estimates which specific classes are incorrectly annotated as well. + This method returns a list of size K, the number of classes in the dataset. + + Parameters + ---------- + labels : List[List[int]] + List of noisy labels for multi-label classification where each example can belong to multiple classes. + Refer to documentation for this argument in `~cleanlab.multilabel_classification.filter.find_label_issues` for further details. + This method will identify whether ``labels[i][k]`` appears correct, for every example ``i`` and class ``k``. + + pred_probs : np.ndarray + An array of shape ``(N, K)`` of model-predicted class probabilities. + Refer to documentation for this argument in `~cleanlab.multilabel_classification.filter.find_label_issues` for further details. + + return_indices_ranked_by : {None, 'self_confidence', 'normalized_margin', 'confidence_weighted_entropy'}, default = None + This function can return a boolean mask (if this argument is ``None``) or a sorted array of indices based on the specified ranking method (if not ``None``). + Refer to documentation for this argument in :py:func:`filter.find_label_issues ` for details. + + rank_by_kwargs : dict, optional + Optional keyword arguments to pass into scoring functions for ranking by. + label quality score (see :py:func:`rank.get_label_quality_scores + `). + + filter_by : {'prune_by_class', 'prune_by_noise_rate', 'both', 'confident_learning', 'predicted_neq_given', 'low_normalized_margin', 'low_self_confidence'}, default = 'prune_by_noise_rate' + The specific method that can be used to filter or prune examples with label issues from a dataset. + Refer to documentation for this argument in :py:func:`filter.find_label_issues ` for details. + + frac_noise : float, default = 1.0 + This will return the "top" frac_noise * num_label_issues estimated label errors, dependent on the filtering method used, + Refer to documentation for this argument in :py:func:`filter.find_label_issues ` for details. + + num_to_remove_per_class : array_like + This parameter is an iterable that specifies the number of mislabeled examples to return from each class. + Refer to documentation for this argument in :py:func:`filter.find_label_issues ` for details. + + min_examples_per_class : int, default = 1 + The minimum number of examples required per class to avoid flagging as label issues. + Refer to documentation for this argument in :py:func:`filter.find_label_issues ` for details. + + confident_joint : np.ndarray, optional + An array of shape ``(K, 2, 2)`` representing a one-vs-rest formatted confident joint. + Refer to documentation for this argument in `~cleanlab.multilabel_classification.filter.find_label_issues` for details. + + n_jobs : optional + Number of processing threads used by multiprocessing. + Refer to documentation for this argument in :py:func:`filter.find_label_issues ` for details. + + verbose : optional + If ``True``, prints when multiprocessing happens. + + Returns + ------- + per_class_label_issues : list(np.ndarray) + By default, this is a list of length K containing the examples where each class appears incorrectly annotated. + ``per_class_label_issues[k]`` is a Boolean mask of the same length as the dataset, + where ``True`` values indicate examples where class ``k`` appears incorrectly annotated. + + For more details, refer to `~cleanlab.multilabel_classification.filter.find_label_issues`. + + Otherwise if `return_indices_ranked_by` is not ``None``, then this method returns 3 objects (each of length K, the number of classes): `label_issues_list`, `labels_list`, `pred_probs_list`. + - *label_issues_list*: an ordered list of indices of examples where class k appears incorrectly annotated, sorted by the likelihood that class k is correctly annotated. + - *labels_list*: a binary one-hot representation of the original labels, useful if you want to compute label quality scores. + - *pred_probs_list*: a one-vs-rest representation of the original predicted probabilities of shape ``(N, 2)``, useful if you want to compute label quality scores. + ``pred_probs_list[k][i][0]`` is the estimated probability that example ``i`` belongs to class ``k``, and is equal to: ``1 - pred_probs_list[k][i][1]``. + """ + import cleanlab.filter + from cleanlab.internal.multilabel_utils import get_onehot_num_classes, stack_complement + from cleanlab.experimental.label_issues_batched import find_label_issues_batched + + y_one, num_classes = get_onehot_num_classes(labels, pred_probs) + if return_indices_ranked_by is None: + bissues = np.zeros(y_one.shape).astype(bool) + else: + label_issues_list = [] + labels_list = [] + pred_probs_list = [] + if confident_joint is not None and not low_memory: + confident_joint_shape = confident_joint.shape + if confident_joint_shape == (num_classes, num_classes): + warnings.warn( + f"The new recommended format for `confident_joint` in multi_label settings is (num_classes,2,2) as output by compute_confident_joint(...,multi_label=True). Your K x K confident_joint in the old format is being ignored." + ) + confident_joint = None + elif confident_joint_shape != (num_classes, 2, 2): + raise ValueError("confident_joint should be of shape (num_classes, 2, 2)") + for class_num, (label, pred_prob_for_class) in enumerate(zip(y_one.T, pred_probs.T)): + pred_probs_binary = stack_complement(pred_prob_for_class) + if low_memory: + quality_score_kwargs = ( + {"method": return_indices_ranked_by} if return_indices_ranked_by else None + ) + binary_label_issues = find_label_issues_batched( + labels=label, + pred_probs=pred_probs_binary, + verbose=verbose, + quality_score_kwargs=quality_score_kwargs, + return_mask=return_indices_ranked_by is None, + ) + else: + if confident_joint is None: + conf = None + else: + conf = confident_joint[class_num] + if num_to_remove_per_class is not None: + ml_num_to_remove_per_class = [num_to_remove_per_class[class_num], 0] + else: + ml_num_to_remove_per_class = None + binary_label_issues = cleanlab.filter.find_label_issues( + labels=label, + pred_probs=pred_probs_binary, + return_indices_ranked_by=return_indices_ranked_by, + frac_noise=frac_noise, + rank_by_kwargs=rank_by_kwargs, + filter_by=filter_by, + num_to_remove_per_class=ml_num_to_remove_per_class, + min_examples_per_class=min_examples_per_class, + confident_joint=conf, + n_jobs=n_jobs, + verbose=verbose, + ) + + if return_indices_ranked_by is None: + bissues[:, class_num] = binary_label_issues + else: + label_issues_list.append(binary_label_issues) + labels_list.append(label) + pred_probs_list.append(pred_probs_binary) + if return_indices_ranked_by is None: + return bissues + else: + return label_issues_list, labels_list, pred_probs_list diff --git a/cleanlab/multilabel_classification/rank.py b/cleanlab/multilabel_classification/rank.py new file mode 100644 index 0000000..6d65143 --- /dev/null +++ b/cleanlab/multilabel_classification/rank.py @@ -0,0 +1,179 @@ +""" +Methods to rank the severity of label issues in multi-label classification datasets. +Here each example can belong to one or more classes, or none of the classes at all. +Unlike in standard multi-class classification, model-predicted class probabilities need not sum to 1 for each row in multi-label classification. +""" + +from __future__ import annotations + +import numpy as np # noqa: F401: Imported for type annotations +from typing import List, TypeVar, Dict, Any, Optional, Tuple, TYPE_CHECKING + +from cleanlab.internal.validation import assert_valid_inputs +from cleanlab.internal.util import get_num_classes +from cleanlab.internal.multilabel_utils import int2onehot +from cleanlab.internal.multilabel_scorer import MultilabelScorer, ClassLabelScorer, Aggregator + + +if TYPE_CHECKING: # pragma: no cover + import numpy.typing as npt + + T = TypeVar("T", bound=npt.NBitBase) + + +def _labels_to_binary( + labels: List[List[int]], + pred_probs: npt.NDArray["np.floating[T]"], +) -> np.ndarray: + """Validate the inputs to the multilabel scorer. Also transform the labels to a binary representation.""" + assert_valid_inputs( + X=None, y=labels, pred_probs=pred_probs, multi_label=True, allow_one_class=True + ) + num_classes = get_num_classes(labels=labels, pred_probs=pred_probs, multi_label=True) + binary_labels = int2onehot(labels, K=num_classes) + return binary_labels + + +def _create_multilabel_scorer( + method: str, + adjust_pred_probs: bool, + aggregator_kwargs: Optional[Dict[str, Any]] = None, +) -> Tuple[MultilabelScorer, Dict]: + """This function acts as a factory that creates a MultilabelScorer.""" + base_scorer = ClassLabelScorer.from_str(method) + base_scorer_kwargs = {"adjust_pred_probs": adjust_pred_probs} + if aggregator_kwargs: + aggregator = Aggregator(**aggregator_kwargs) + scorer = MultilabelScorer(base_scorer, aggregator) + else: + scorer = MultilabelScorer(base_scorer) + return scorer, base_scorer_kwargs + + +def get_label_quality_scores( + labels: List[List[int]], + pred_probs: npt.NDArray["np.floating[T]"], + *, + method: str = "self_confidence", + adjust_pred_probs: bool = False, + aggregator_kwargs: Dict[str, Any] = {"method": "exponential_moving_average", "alpha": 0.8}, +) -> npt.NDArray["np.floating[T]"]: + """Computes a label quality score for each example in a multi-label classification dataset. + + Scores are between 0 and 1 with lower scores indicating examples whose label more likely contains an error. + For each example, this method internally computes a separate score for each individual class + and then aggregates these per-class scores into an overall label quality score for the example. + + + Parameters + ---------- + labels : List[List[int]] + List of noisy labels for multi-label classification where each example can belong to multiple classes. + Refer to documentation for this argument in :py:func:`multilabel_classification.filter.find_label_issues ` for further details. + + pred_probs : np.ndarray + An array of shape ``(N, K)`` of model-predicted class probabilities. + Refer to documentation for this argument in :py:func:`multilabel_classification.filter.find_label_issues ` for further details. + + method : {"self_confidence", "normalized_margin", "confidence_weighted_entropy"}, default = "self_confidence" + Method to calculate separate per-class annotation scores for an example that are then aggregated into an overall label quality score for the example. + These scores are separately calculated for each class based on the corresponding column of `pred_probs` in a one-vs-rest manner, + and are standard label quality scores for binary classification (based on whether the class should or should not apply to this example). + + See also + -------- + :py:func:`rank.get_label_quality_scores ` function for details about each option. + + adjust_pred_probs : bool, default = False + Account for class imbalance in the label-quality scoring by adjusting predicted probabilities. + Refer to documentation for this argument in :py:func:`rank.get_label_quality_scores ` for details. + + + aggregator_kwargs : dict, default = {"method": "exponential_moving_average", "alpha": 0.8} + A dictionary of hyperparameter values to use when aggregating per-class scores into an overall label quality score for each example. + Options for ``"method"`` include: ``"exponential_moving_average"`` or ``"softmin"`` or your own callable function. + See :py:class:`internal.multilabel_scorer.Aggregator ` for details about each option and other possible hyperparameters. + + To get a score for each class annotation for each example, use the `~cleanlab.multilabel_classification.rank.get_label_quality_scores_per_class` method instead. + + Returns + ------- + label_quality_scores : np.ndarray + A 1D array of shape ``(N,)`` with a label quality score (between 0 and 1) for each example in the dataset. + Lower scores indicate examples whose label is more likely to contain some annotation error (for any of the classes). + + Examples + -------- + >>> from cleanlab.multilabel_classification import get_label_quality_scores + >>> import numpy as np + >>> labels = [[1], [0,2]] + >>> pred_probs = np.array([[0.1, 0.9, 0.1], [0.4, 0.1, 0.9]]) + >>> scores = get_label_quality_scores(labels, pred_probs) + >>> scores + array([0.9, 0.5]) + """ + binary_labels = _labels_to_binary(labels, pred_probs) + scorer, base_scorer_kwargs = _create_multilabel_scorer( + method=method, + adjust_pred_probs=adjust_pred_probs, + aggregator_kwargs=aggregator_kwargs, + ) + return scorer(binary_labels, pred_probs, base_scorer_kwargs=base_scorer_kwargs) + + +def get_label_quality_scores_per_class( + labels: List[List[int]], + pred_probs: npt.NDArray["np.floating[T]"], + *, + method: str = "self_confidence", + adjust_pred_probs: bool = False, +) -> np.ndarray: + """ + Computes a quality score quantifying how likely each individual class annotation is correct in a multi-label classification dataset. + This is similar to `~cleanlab.multilabel_classification.rank.get_label_quality_scores` + but instead returns the per-class results without aggregation. + For a dataset with K classes, each example receives K scores from this method. + Refer to documentation in `~cleanlab.multilabel_classification.rank.get_label_quality_scores` for details. + + Parameters + ---------- + labels : List[List[int]] + List of noisy labels for multi-label classification where each example can belong to multiple classes. + Refer to documentation for this argument in :py:func:`find_label_issues ` for further details. + + pred_probs : np.ndarray + An array of shape ``(N, K)`` of model-predicted class probabilities. + Refer to documentation for this argument in :py:func:`find_label_issues ` for further details. + + method : {"self_confidence", "normalized_margin", "confidence_weighted_entropy"}, default = "self_confidence" + Method to calculate separate per-class annotation scores (that quantify how likely a particular class annotation is correct for a particular example). + Refer to documentation for this argument in `~cleanlab.multilabel_classification.rank.get_label_quality_scores` for further details. + + adjust_pred_probs : bool, default = False + Account for class imbalance in the label-quality scoring by adjusting predicted probabilities. + Refer to documentation for this argument in :py:func:`rank.get_label_quality_scores ` for details. + + Returns + ------- + label_quality_scores : list(np.ndarray) + A list containing K arrays, each of shape (N,). Here K is the number of classes in the dataset and N is the number of examples. + ``label_quality_scores[k][i]`` is a score between 0 and 1 quantifying how likely the annotation for class ``k`` is correct for example ``i``. + + Examples + -------- + >>> from cleanlab.multilabel_classification import get_label_quality_scores + >>> import numpy as np + >>> labels = [[1], [0,2]] + >>> pred_probs = np.array([[0.1, 0.9, 0.1], [0.4, 0.1, 0.9]]) + >>> scores = get_label_quality_scores(labels, pred_probs) + >>> scores + array([0.9, 0.5]) + """ + binary_labels = _labels_to_binary(labels, pred_probs) + scorer, base_scorer_kwargs = _create_multilabel_scorer( + method=method, + adjust_pred_probs=adjust_pred_probs, + ) + return scorer.get_class_label_quality_scores( + labels=binary_labels, pred_probs=pred_probs, base_scorer_kwargs=base_scorer_kwargs + ) diff --git a/cleanlab/object_detection/__init__.py b/cleanlab/object_detection/__init__.py new file mode 100644 index 0000000..fbc2eb7 --- /dev/null +++ b/cleanlab/object_detection/__init__.py @@ -0,0 +1,3 @@ +from . import rank +from . import filter +from . import summary diff --git a/cleanlab/object_detection/filter.py b/cleanlab/object_detection/filter.py new file mode 100644 index 0000000..8346f23 --- /dev/null +++ b/cleanlab/object_detection/filter.py @@ -0,0 +1,405 @@ +"""Methods to find label issues in an object detection dataset, where each annotated bounding box in an image receives its own class label.""" + +from collections import defaultdict +from multiprocessing import Pool +from typing import Any, Dict, List, Optional, Tuple, Union + +import numpy as np + +from cleanlab.internal.constants import ( + ALPHA, + HIGH_PROBABILITY_THRESHOLD, + LOW_PROBABILITY_THRESHOLD, + OVERLOOKED_THRESHOLD_FACTOR, + BADLOC_THRESHOLD_FACTOR, + SWAP_THRESHOLD_FACTOR, + AP_SCALE_FACTOR, +) +from cleanlab.internal.object_detection_utils import assert_valid_inputs +from cleanlab.object_detection.rank import ( + _get_valid_inputs_for_compute_scores, + _separate_label, + _separate_prediction, + compute_badloc_box_scores, + compute_overlooked_box_scores, + compute_swap_box_scores, + get_label_quality_scores, + issues_from_scores, + _get_overlap_matrix, +) + + +def find_label_issues( + labels: List[Dict[str, Any]], + predictions: List[np.ndarray], + *, + return_indices_ranked_by_score: Optional[bool] = False, + overlapping_label_check: Optional[bool] = True, +) -> np.ndarray: + """ + Identifies potentially mislabeled images in an object detection dataset. + An image is flagged with a label issue if *any* of its bounding boxes appear incorrectly annotated. + This includes images for which a bounding box: should have been annotated but is missing, + has been annotated with the wrong class, or has been annotated in a suboptimal location. + + Suppose the dataset has ``N`` images, ``K`` possible class labels. + If ``return_indices_ranked_by_score`` is ``False``, a boolean mask of length ``N`` is returned, + indicating whether each image has a label issue (``True``) or not (``False``). + If ``return_indices_ranked_by_score`` is ``True``, the indices of images flagged with label issues are returned, + sorted with the most likely-mislabeled images ordered first. + + Parameters + ---------- + labels: + Annotated boxes and class labels in the original dataset, which may contain some errors. + This is a list of ``N`` dictionaries such that ``labels[i]`` contains the given labels for the `i`-th image in the following format: + ``{'bboxes': np.ndarray((L,4)), 'labels': np.ndarray((L,)), 'image_name': str}`` where ``L`` is the number of annotated bounding boxes + for the `i`-th image and ``bboxes[l]`` is a bounding box of coordinates in ``[x1,y1,x2,y2]`` format and with given class label ``labels[j]``. + ``image_name`` is an optional part of the labels that can be used to later refer to specific images. + + Note: Here, ``(x1,y1)`` corresponds to the top-left and ``(x2,y2)`` corresponds to the bottom-right corner of the bounding box with respect to the image matrix [e.g. `XYXY in Keras `, `Detectron 2 `]. + + For more information on proper labels formatting, check out the `MMDetection library `_. + + predictions: + Predictions output by a trained object detection model. + For the most accurate results, predictions should be out-of-sample to avoid overfitting, eg. obtained via :ref:`cross-validation `. + This is a list of ``N`` ``np.ndarray`` such that ``predictions[i]`` corresponds to the model prediction for the `i`-th image. + For each possible class ``k`` in 0, 1, ..., K-1: ``predictions[i][k]`` is a ``np.ndarray`` of shape ``(M,5)``, + where ``M`` is the number of predicted bounding boxes for class ``k``. Here the five columns correspond to ``[x1,y1,x2,y2,pred_prob]``, + where ``[x1,y1,x2,y2]`` are coordinates of the bounding box predicted by the model and ``pred_prob`` is the model's confidence in the predicted class label for this bounding box. + + Note: Here, ``(x1,y1)`` corresponds to the top-left and ``(x2,y2)`` corresponds to the bottom-right corner of the bounding box with respect to the image matrix [e.g. `XYXY in Keras `, `Detectron 2 `]. The last column, pred_prob, represents the predicted probability that the bounding box contains an object of the class k. + + For more information see the `MMDetection package `_ for an example object detection library that outputs predictions in the correct format. + + return_indices_ranked_by_score: + Determines what is returned by this method (see description of return value for details). + + overlapping_label_check : bool, default = True + If True, boxes annotated with more than one class label have their swap score penalized. Set this to False if you are not concerned when two very similar boxes exist with different class labels in the given annotations. + + + Returns + ------- + label_issues : np.ndarray + Specifies which images are identified to have a label issue. + If ``return_indices_ranked_by_score = False``, this function returns a boolean mask of length ``N`` (``True`` entries indicate which images have label issue). + If ``return_indices_ranked_by_score = True``, this function returns a (shorter) array of indices of images with label issues, sorted by how likely the image is mislabeled. + + More precisely, indices are sorted by image label quality score calculated via :py:func:`object_detection.rank.get_label_quality_scores `. + """ + scoring_method = "objectlab" + + assert_valid_inputs( + labels=labels, + predictions=predictions, + method=scoring_method, + ) + + is_issue = _find_label_issues( + labels, + predictions, + scoring_method=scoring_method, + return_indices_ranked_by_score=return_indices_ranked_by_score, + overlapping_label_check=overlapping_label_check, + ) + + return is_issue + + +def _find_label_issues( + labels: List[Dict[str, Any]], + predictions: List[np.ndarray], + *, + scoring_method: Optional[str] = "objectlab", + return_indices_ranked_by_score: Optional[bool] = True, + overlapping_label_check: Optional[bool] = True, +): + """Internal function to find label issues based on passed in method.""" + + if scoring_method == "objectlab": + auxiliary_inputs = _get_valid_inputs_for_compute_scores(ALPHA, labels, predictions) + + per_class_scores = _get_per_class_ap(labels, predictions) + lab_list = [_separate_label(label)[1] for label in labels] + pred_list = [_separate_prediction(pred)[1] for pred in predictions] + pred_thresholds_list = _process_class_list(pred_list, per_class_scores) + lab_thresholds_list = _process_class_list(lab_list, per_class_scores) + overlooked_scores_per_box = compute_overlooked_box_scores( + alpha=ALPHA, + high_probability_threshold=HIGH_PROBABILITY_THRESHOLD, + auxiliary_inputs=auxiliary_inputs, + ) + overlooked_issues_per_box = _find_label_issues_per_box( + overlooked_scores_per_box, pred_thresholds_list, OVERLOOKED_THRESHOLD_FACTOR + ) + overlooked_issues_per_image = _pool_box_scores_per_image(overlooked_issues_per_box) + + badloc_scores_per_box = compute_badloc_box_scores( + alpha=ALPHA, + low_probability_threshold=LOW_PROBABILITY_THRESHOLD, + auxiliary_inputs=auxiliary_inputs, + ) + badloc_issues_per_box = _find_label_issues_per_box( + badloc_scores_per_box, lab_thresholds_list, BADLOC_THRESHOLD_FACTOR + ) + badloc_issues_per_image = _pool_box_scores_per_image(badloc_issues_per_box) + + swap_scores_per_box = compute_swap_box_scores( + alpha=ALPHA, + high_probability_threshold=HIGH_PROBABILITY_THRESHOLD, + overlapping_label_check=overlapping_label_check, + auxiliary_inputs=auxiliary_inputs, + ) + swap_issues_per_box = _find_label_issues_per_box( + swap_scores_per_box, lab_thresholds_list, SWAP_THRESHOLD_FACTOR + ) + swap_issues_per_image = _pool_box_scores_per_image(swap_issues_per_box) + + issues_per_image = ( + overlooked_issues_per_image + badloc_issues_per_image + swap_issues_per_image + ) + is_issue = issues_per_image > 0 + else: + is_issue = np.full( + shape=[ + len(labels), + ], + fill_value=-1, + ) + + if return_indices_ranked_by_score: + scores = get_label_quality_scores(labels, predictions) + sorted_scores_idx = issues_from_scores(scores, threshold=1.0) + is_issue_idx = np.where(is_issue == True)[0] + sorted_issue_mask = np.in1d(sorted_scores_idx, is_issue_idx, assume_unique=True) + issue_idx = sorted_scores_idx[sorted_issue_mask] + return issue_idx + else: + return is_issue + + +def _find_label_issues_per_box( + scores_per_box: List[np.ndarray], threshold_classes, threshold_factor=1.0 +) -> List[np.ndarray]: + """Takes in a list of size ``N`` where each index is an array of scores for each bounding box in the `n-th` example + and a threshold. Each box below or equal to the corresponding threshold in threshold_classes will be marked as an issue. + + Returns a list of size ``N`` where each index is a boolean array of length number of boxes per example `n` + marking if a specific box is an issue - 1 or not - 0.""" + is_issue_per_box = [] + for idx, score_per_box in enumerate(scores_per_box): + if len(score_per_box) == 0: # if no for specific image, then image not an issue + is_issue_per_box.append(np.array([False])) + else: + score_per_box[np.isnan(score_per_box)] = 1.0 + score_per_box = score_per_box + issue_per_box = [] + for i in range(len(score_per_box)): + issue_per_box.append( + score_per_box[i] <= threshold_classes[idx][i] * threshold_factor + ) + is_issue_per_box.append(np.array(issue_per_box, bool)) + return is_issue_per_box + + +def _pool_box_scores_per_image(is_issue_per_box: List[np.ndarray]) -> np.ndarray: + """Takes in a list of size ``N`` where each index is a boolean array of length number of boxes per image `n ` + marking if a specific box is an issue - 1 or not - 0. + + Returns a list of size ``N`` where each index marks if the image contains an issue - 1 or not - 0. + Images are marked as issues if 1 or more bounding boxes in the image is an issue.""" + is_issue = np.zeros( + shape=[ + len( + is_issue_per_box, + ) + ] + ) + for idx, issue_per_box in enumerate(is_issue_per_box): + if np.sum(issue_per_box) > 0: + is_issue[idx] = 1 + return is_issue + + +def _process_class_list(class_list: List[np.ndarray], class_dict: Dict[int, float]) -> List: + """ + Converts a list of classes represented as numpy arrays using a class-to-float dictionary, + and returns a list where each class is replaced by its corresponding float value from the dictionary. + + Args: + class_list (List[np.ndarray]): A list of classes represented as numpy arrays. + class_dict (Dict[int, float]): A dictionary mapping class indices to their corresponding float values. + + Returns: + List[float]: A list of float values corresponding to the classes in the input list. + """ + class_l2 = [] + for i in class_list: + l3 = [class_dict[j] for j in i] + class_l2.append(l3) + return class_l2 + + +def _calculate_ap_per_class( + labels: List[Dict[str, Any]], + predictions: List[np.ndarray], + *, + iou_threshold: Optional[float] = 0.5, + num_procs: int = 1, +) -> List: + """ + Computes the average precision for each class based on provided labels and predictions. + It uses an Intersection over Union (IoU) threshold and supports parallel processing with a specified number of processes. + + """ + num_images = len(predictions) + num_scale = 1 + num_classes = len(predictions[0]) + if num_images > 1: + num_procs = min(num_procs, num_images) + pool = Pool(num_procs) + ap_per_class_list = [] + for class_num in range(num_classes): + pred_bboxes, lab_bboxes = _filter_by_class(labels, predictions, class_num) + if num_images > 1: + tpfp = pool.starmap( + _calculate_true_positives_false_positives, + zip(pred_bboxes, lab_bboxes, [iou_threshold for _ in range(num_images)]), + ) + else: + tpfp = [ + _calculate_true_positives_false_positives( + pred_bboxes[0], + lab_bboxes[0], + iou_threshold, + ) + ] + true_positives, false_positives = tuple(zip(*tpfp)) + num_gts = np.zeros(num_scale, dtype=int) + for j, bbox in enumerate(lab_bboxes): + num_gts[0] += bbox.shape[0] + pred_bboxes = np.vstack(pred_bboxes) + sort_inds = np.argsort(-pred_bboxes[:, -1]) + true_positives = np.hstack(true_positives)[:, sort_inds] + false_positives = np.hstack(false_positives)[:, sort_inds] + true_positives = np.cumsum(true_positives, axis=1) + false_positives = np.cumsum(false_positives, axis=1) + eps = np.finfo(np.float32).eps + recalls = true_positives / np.maximum(num_gts[:, np.newaxis], eps) + precisions = true_positives / np.maximum((true_positives + false_positives), eps) + recalls = recalls[0, :] + precisions = precisions[0, :] + ap = _calculate_average_precision(recalls, precisions) + ap_per_class_list.append(ap) + if num_images > 1: + pool.close() + return ap_per_class_list + + +def _filter_by_class( + labels: List[Dict[str, Any]], predictions: List[np.ndarray], class_num: int +) -> Tuple[List, List]: + """ + Filters predictions and labels based on a specific class number. + """ + pred_bboxes = [prediction[class_num] for prediction in predictions] + lab_bboxes = [] + for label in labels: + gt_inds = label["labels"] == class_num + lab_bboxes.append(label["bboxes"][gt_inds, :]) + return pred_bboxes, lab_bboxes + + +def _calculate_true_positives_false_positives( + pred_bboxes: np.ndarray, + lab_bboxes: np.ndarray, + iou_threshold: Optional[float] = 0.5, + return_false_negative: bool = False, +) -> Union[Tuple[np.ndarray, np.ndarray], Tuple[np.ndarray, np.ndarray, np.ndarray]]: + """Calculates true positives (TP) and false positives (FP) for object detection tasks. + It takes predicted bounding boxes, ground truth bounding boxes, and an optional Intersection over Union (IoU) threshold as inputs. + If return_false_negative is True, it returns an array of False negatives as well. + """ + num_preds = pred_bboxes.shape[0] + num_labels = lab_bboxes.shape[0] + num_scales = 1 + true_positives = np.zeros((num_scales, num_preds), dtype=np.float32) + false_positives = np.zeros((num_scales, num_preds), dtype=np.float32) + + if lab_bboxes.shape[0] == 0: + false_positives[...] = 1 + if return_false_negative: + return true_positives, false_positives, np.array([], dtype=np.float32) + else: + return true_positives, false_positives + ious = _get_overlap_matrix(pred_bboxes, lab_bboxes) + ious_max = ious.max(axis=1) + ious_argmax = ious.argmax(axis=1) + sorted_indices = np.argsort(-pred_bboxes[:, -1]) + is_covered = np.zeros(num_labels, dtype=bool) + for index in sorted_indices: + if ious_max[index] >= iou_threshold: + matching_label = ious_argmax[index] + if not is_covered[matching_label]: + is_covered[matching_label] = True + true_positives[0, index] = 1 + else: + false_positives[0, index] = 1 + else: + false_positives[0, index] = 1 + if return_false_negative: + false_negatives = np.zeros((num_scales, num_labels), dtype=np.float32) + for label_index in range(num_labels): + if not is_covered[label_index]: + false_negatives[0, label_index] = 1 + return true_positives, false_positives, false_negatives + return true_positives, false_positives + + +def _calculate_average_precision( + recall_values: np.ndarray, precision_values: np.ndarray +) -> np.ndarray: + """Computes the average precision (AP) for a set of recall and precision values. It takes arrays of recall and precision values as inputs.""" + recall_values = recall_values[np.newaxis, :] + precision_values = precision_values[np.newaxis, :] + num_scales = recall_values.shape[0] + average_precision = np.zeros(num_scales, dtype=np.float32) + zeros_matrix = np.zeros((num_scales, 1), dtype=recall_values.dtype) + ones_matrix = np.ones((num_scales, 1), dtype=recall_values.dtype) + modified_recall = np.hstack((zeros_matrix, recall_values, ones_matrix)) + modified_precision = np.hstack((zeros_matrix, precision_values, zeros_matrix)) + + for i in range(modified_precision.shape[1] - 1, 0, -1): + modified_precision[:, i - 1] = np.maximum( + modified_precision[:, i - 1], modified_precision[:, i] + ) + + for i in range(num_scales): + index = np.where(modified_recall[i, 1:] != modified_recall[i, :-1])[0] + average_precision[i] = np.sum( + (modified_recall[i, index + 1] - modified_recall[i, index]) + * modified_precision[i, index + 1] + ) + + return average_precision + + +def _get_per_class_ap( + labels: List[Dict[str, Any]], predictions: List[np.ndarray] +) -> Dict[int, float]: + """Computes the Average Precision (AP) for each class in an object detection task. + It takes a list of label dictionaries and a list of prediction arrays as inputs. + It calculates AP values for different Intersection over Union (IoU) thresholds, averages them per class, and then scales the AP values. + """ + iou_thrs = np.linspace(0.5, 0.95, int(np.round((0.95 - 0.5) / 0.05)) + 1, endpoint=True) + class_num_to_iou_list = defaultdict(list) + for threshold in iou_thrs: + ap_per_class = _calculate_ap_per_class(labels, predictions, iou_threshold=threshold) + for class_num in range(0, len(ap_per_class)): + class_num_to_iou_list[class_num].append(ap_per_class[class_num]) + class_num_to_AP = {} + for class_num in class_num_to_iou_list: + class_num_to_AP[class_num] = np.mean(class_num_to_iou_list[class_num]) * AP_SCALE_FACTOR + return class_num_to_AP diff --git a/cleanlab/object_detection/rank.py b/cleanlab/object_detection/rank.py new file mode 100644 index 0000000..efb0b14 --- /dev/null +++ b/cleanlab/object_detection/rank.py @@ -0,0 +1,1110 @@ +"""Methods to rank and score images in an object detection dataset (object detection data), based on how likely they +are to contain label errors.""" + +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, TypeVar +import warnings +import copy +import numpy as np + +from cleanlab.internal.constants import ( + ALPHA, + CUSTOM_SCORE_WEIGHT_BADLOC, + CUSTOM_SCORE_WEIGHT_OVERLOOKED, + CUSTOM_SCORE_WEIGHT_SWAP, + EPSILON, + EUC_FACTOR, + HIGH_PROBABILITY_THRESHOLD, + LOW_PROBABILITY_THRESHOLD, + MAX_ALLOWED_BOX_PRUNE, + TINY_VALUE, + TEMPERATURE, + LABEL_OVERLAP_THRESHOLD, +) +from cleanlab.internal.object_detection_utils import ( + softmin1d, + assert_valid_aggregation_weights, + assert_valid_inputs, +) + + +if TYPE_CHECKING: # pragma: no cover + from typing import TypedDict + + AuxiliaryTypesDict = TypedDict( + "AuxiliaryTypesDict", + { + "pred_labels": np.ndarray, + "pred_label_probs": np.ndarray, + "pred_bboxes": np.ndarray, + "lab_labels": np.ndarray, + "lab_bboxes": np.ndarray, + "similarity_matrix": np.ndarray, + "iou_matrix": np.ndarray, + "min_possible_similarity": float, + }, + ) +else: + AuxiliaryTypesDict = TypeVar("AuxiliaryTypesDict") + + +def get_label_quality_scores( + labels: List[Dict[str, Any]], + predictions: List[np.ndarray], + *, + aggregation_weights: Optional[Dict[str, float]] = None, + overlapping_label_check: Optional[bool] = True, + verbose: bool = True, +) -> np.ndarray: + """Computes a label quality score for each image of the ``N`` images in the dataset. + + For object detection datasets, the label quality score for an image estimates how likely it has been correctly labeled. + Lower scores indicate images whose annotation is more likely imperfect. + Annotators may have mislabeled an image because they: + + - overlooked an object (missing annotated bounding box), + - chose the wrong class label for an annotated box in the correct location, + - imperfectly annotated the location/edges of a bounding box. + + Any of these annotation errors should lead to an image with a lower label quality score. This quality score is between 0 and 1. + + - 1 - clean label (given label is likely correct). + - 0 - dirty label (given label is likely incorrect). + + Parameters + ---------- + labels: + A list of ``N`` dictionaries such that ``labels[i]`` contains the given labels for the `i`-th image. + Refer to documentation for this argument in :py:func:`find_label_issues ` for further details. + + predictions: + A list of ``N`` ``np.ndarray`` such that ``predictions[i]`` corresponds to the model predictions for the `i`-th image. + Refer to documentation for this argument in :py:func:`find_label_issues ` for further details. + + verbose : bool, default = True + Set to ``False`` to suppress all print statements. + + aggregation_weights: + Optional dictionary to specify weights for aggregating quality scores for subtype of label issue into an overall label quality score for the image. + Its keys are: "overlooked", "swap", "badloc", and values should be nonnegative weights that sum to 1. + Increase one of these weights to prioritize images with bounding boxes that were either: + missing in the annotations (overlooked object), annotated with the wrong class label (class for the object should be swapped to another class), or annotated in a suboptimal location (badly located). + + swapped examples, bad location examples, and overlooked examples. + It is important to ensure that the weights are non-negative values and that their sum equals 1.0. + + overlapping_label_check : bool, default = True + If True, boxes annotated with more than one class label have their swap score penalized. Set this to False if you are not concerned when two very similar boxes exist with different class labels in the given annotations. + + Returns + --------- + label_quality_scores: + Array of shape ``(N, )`` of scores between 0 and 1, one per image in the object detection dataset. + Lower scores indicate images that are more likely mislabeled. + """ + method = "objectlab" + probability_threshold = 0.0 + + assert_valid_inputs( + labels=labels, + predictions=predictions, + method=method, + threshold=probability_threshold, + ) + aggregation_weights = _get_aggregation_weights(aggregation_weights) + + return _compute_label_quality_scores( + labels=labels, + predictions=predictions, + method=method, + threshold=probability_threshold, + aggregation_weights=aggregation_weights, + overlapping_label_check=overlapping_label_check, + verbose=verbose, + ) + + +def issues_from_scores(label_quality_scores: np.ndarray, *, threshold: float = 0.1) -> np.ndarray: + """Convert label quality scores to a list of indices of images with issues sorted from most to least severe cut off at threshold. + + Returns the list of indices of images with issues sorted from most to least severe cut off at threshold. + + Parameters + ---------- + label_quality_scores: + Array of shape ``(N, )`` of scores between 0 and 1, one per image in the object detection dataset. + Lower scores indicate images are more likely to contain a label issue. + + threshold: + Label quality scores above the threshold are not considered to be label issues. The corresponding examples' indices are omitted from the returned array. + + Returns + --------- + issue_indices: + Array of issue indices sorted from most to least severe who's label quality scores fall below the threshold if one is provided. + """ + + if threshold > 1.0: + raise ValueError( + f""" + Threshold is a cutoff of label_quality_scores and therefore should be <= 1. + """ + ) + + issue_indices = np.argwhere(label_quality_scores <= threshold).flatten() + issue_vals = label_quality_scores[issue_indices] + sorted_idx = issue_vals.argsort() + return issue_indices[sorted_idx] + + +def _compute_label_quality_scores( + labels: List[Dict[str, Any]], + predictions: List[np.ndarray], + *, + method: Optional[str] = "objectlab", + aggregation_weights: Optional[Dict[str, float]] = None, + threshold: Optional[float] = None, + overlapping_label_check: Optional[bool] = True, + verbose: bool = True, +) -> np.ndarray: + """Internal function to prune extra bounding boxes and compute label quality scores based on passed in method.""" + + pred_probs_prepruned = False + min_pred_prob = _get_min_pred_prob(predictions) + aggregation_weights = _get_aggregation_weights(aggregation_weights) + + if threshold is not None: + predictions = _prune_by_threshold( + predictions=predictions, threshold=threshold, verbose=verbose + ) + if np.abs(min_pred_prob - threshold) < 0.001 and threshold > 0: + pred_probs_prepruned = True # the provided threshold is the threshold used for pre_pruning the pred_probs during model prediction. + else: + threshold = min_pred_prob # assume model was not pre_pruned if no threshold was provided + + if method == "objectlab": + scores = _get_subtype_label_quality_scores( + labels=labels, + predictions=predictions, + alpha=ALPHA, + low_probability_threshold=LOW_PROBABILITY_THRESHOLD, + high_probability_threshold=HIGH_PROBABILITY_THRESHOLD, + temperature=TEMPERATURE, + aggregation_weights=aggregation_weights, + overlapping_label_check=overlapping_label_check, + ) + else: + raise ValueError( + "Invalid method: '{}' is not a valid method for computing label quality scores. Please use the 'objectlab' method.".format( + method + ) + ) + return scores + + +def _get_min_pred_prob( + predictions: List[np.ndarray], +) -> float: + """Returns min pred_prob out of all predictions.""" + pred_probs = [1.0] # avoid calling np.min on empty array. + for prediction in predictions: + for class_prediction in prediction: + pred_probs.extend(list(class_prediction[:, -1])) + + min_pred_prob = np.min(pred_probs) + return min_pred_prob + + +def _prune_by_threshold( + predictions: List[np.ndarray], threshold: float, verbose: bool = True +) -> List[np.ndarray]: + """Removes predicted bounding boxes from predictions who's pred_prob is below the cuttoff threshold.""" + + predictions_copy = copy.deepcopy(predictions) + num_ann_to_zero = 0 + total_ann = 0 + for idx_predictions, prediction in enumerate(predictions_copy): + for idx_class, class_prediction in enumerate(prediction): + filtered_class_prediction = class_prediction[class_prediction[:, -1] >= threshold] + if len(class_prediction) > 0: + total_ann += 1 + if len(filtered_class_prediction) == 0: + num_ann_to_zero += 1 + + predictions_copy[idx_predictions][idx_class] = filtered_class_prediction + + p_ann_pruned = total_ann and num_ann_to_zero / total_ann or 0 # avoid division by zero + if p_ann_pruned > MAX_ALLOWED_BOX_PRUNE: + warnings.warn( + f"Pruning with threshold=={threshold} prunes {p_ann_pruned}% labels. Consider lowering the threshold.", + UserWarning, + ) + if verbose: + print( + f"Pruning {num_ann_to_zero} predictions out of {total_ann} using threshold=={threshold}. These predictions are no longer considered as potential candidates for identifying label issues as their similarity with the given labels is no longer considered." + ) + return predictions_copy + + +def _separate_label(label: Dict[str, Any]) -> Tuple[np.ndarray, np.ndarray]: + """Separates labels into bounding box and class label lists.""" + bboxes = label["bboxes"] + labels = label["labels"] + return bboxes, labels + + +# TODO: make object detection work for all predicted probabilities +def _separate_prediction_all_preds( + prediction: List[np.ndarray], +) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + pred_bboxes, pred_labels, det_probs = prediction + return pred_bboxes, pred_labels, det_probs + + +def _separate_prediction_single_box( + prediction: np.ndarray, +) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """Separates predictions into class labels, bounding boxes and pred_prob lists""" + labels = [] + boxes = [] + for idx, prediction_class in enumerate(prediction): + labels.extend([idx] * len(prediction_class)) + boxes.extend(prediction_class.tolist()) + bboxes = [box[:4] for box in boxes] + pred_probs = [box[-1] for box in boxes] + return np.array(bboxes), np.array(labels), np.array(pred_probs) + + +def _get_prediction_type(prediction: np.ndarray) -> str: + if ( + len(prediction) == 3 + and prediction[0].shape[0] == prediction[2].shape[1] + and prediction[1].shape[0] == prediction[2].shape[0] + ): + return "all_pred" + else: + return "single_pred" + + +def _separate_prediction( + prediction, prediction_type="single_pred" +) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """Returns bbox, label and pred_prob values for prediction.""" + + if prediction_type == "all_pred": + boxes, labels, pred_probs = _separate_prediction_all_preds(prediction) + else: + boxes, labels, pred_probs = _separate_prediction_single_box(prediction) + return boxes, labels, pred_probs + + +def _mod_coordinates(x: List[float]) -> Dict[str, Any]: + """Takes is a list of xyxy coordinates and returns them in dictionary format.""" + + wd = {"x1": x[0], "y1": x[1], "x2": x[2], "y2": x[3]} + return wd + + +def _get_overlap(bb1: List[float], bb2: List[float]) -> float: + """Takes in two bounding boxes `bb1` and `bb2` and returns their IoU overlap.""" + + return _get_iou(_mod_coordinates(bb1), _mod_coordinates(bb2)) + + +def _get_overlap_matrix(bb1_list: np.ndarray, bb2_list: np.ndarray) -> np.ndarray: + """Takes in two lists of bounding boxes and returns an IoU matrix where IoU[i][j] is the overlap between + the i-th box in `bb1_list` and the j-th box in `bb2_list`.""" + wd = np.zeros(shape=(len(bb1_list), len(bb2_list))) + for i in range(len(bb1_list)): + for j in range(len(bb2_list)): + wd[i][j] = _get_overlap(bb1_list[i], bb2_list[j]) + return wd + + +def _get_iou(bb1: Dict[str, Any], bb2: Dict[str, Any]) -> float: + """ + Calculate the Intersection over Union (IoU) of two bounding boxes. + I've modified this to calculate overlap ratio in the line: + iou = np.clip(intersection_area / float(min(bb1_area,bb2_area)),0.0,1.0) + + Parameters + ---------- + bb1 : dict + Keys: {'x1', 'x2', 'y1', 'y2'} + The (x1, y1) position is at the top left corner, + the (x2, y2) position is at the bottom right corner + bb2 : dict + Keys: {'x1', 'x2', 'y1', 'y2'} + The (x, y) position is at the top left corner, + the (x2, y2) position is at the bottom right corner + Returns + ------- + float + in [0, 1] + """ + # determine the coordinates of the intersection rectangle + x_left = max(bb1["x1"], bb2["x1"]) + y_top = max(bb1["y1"], bb2["y1"]) + x_right = min(bb1["x2"], bb2["x2"]) + y_bottom = min(bb1["y2"], bb2["y2"]) + + if x_right < x_left or y_bottom < y_top: + return 0.0 + + # The intersection of two axis-aligned bounding boxes is always an + # axis-aligned bounding box + intersection_area = (x_right - x_left) * (y_bottom - y_top) + + # compute the area of both AABBs + bb1_area = (bb1["x2"] - bb1["x1"]) * (bb1["y2"] - bb1["y1"]) + bb2_area = (bb2["x2"] - bb2["x1"]) * (bb2["y2"] - bb2["y1"]) + + # compute the intersection over union by taking the intersection + # area and dividing it by the sum of prediction + ground-truth + # areas - the interesection area + iou = intersection_area / np.clip( + float(bb1_area + bb2_area - intersection_area), a_min=EPSILON, a_max=None + ) # avoid division by 0 + # There are some hyper-parameters here like consider tile area/object area + return iou + + +def _has_overlap(bbox_list, labels): + """This function determines whether each labeled box overlaps with another box of a different class (i.e. virtually the same box having multiple conflicting annotations). It returns a boolean array.""" + iou_matrix = _get_overlap_matrix(bbox_list, bbox_list) + results_overlap = [] + for i in range(0, len(iou_matrix)): + is_overlap = False + for j in range(0, len(iou_matrix)): + if i != j: + if iou_matrix[i][j] >= LABEL_OVERLAP_THRESHOLD: + lab_1 = labels[i] + lab_2 = labels[j] + if lab_1 != lab_2: + is_overlap = True + results_overlap.append(is_overlap) + return np.array(results_overlap) + + +def _euc_dis(box1: List[float], box2: List[float]) -> float: + """Calculates the Euclidean distance between `box1` and `box2`.""" + x1, y1 = (box1[0] + box1[2]) / 2, (box1[1] + box1[3]) / 2 + x2, y2 = (box2[0] + box2[2]) / 2, (box2[1] + box2[3]) / 2 + p1 = np.array([x1, y1]) + p2 = np.array([x2, y2]) + val2 = np.exp(-np.linalg.norm(p1 - p2) * EUC_FACTOR) + return val2 + + +def _get_dist_matrix(bb1_list: np.ndarray, bb2_list: np.ndarray) -> np.ndarray: + """Returns a distance matrix of distances from all of boxes in bb1_list to all of boxes in bb2_list.""" + wd = np.zeros(shape=(len(bb1_list), len(bb2_list))) + for i in range(len(bb1_list)): + for j in range(len(bb2_list)): + wd[i][j] = _euc_dis(bb1_list[i], bb2_list[j]) + return wd + + +def _get_min_possible_similarity( + alpha: float, + predictions, + labels: List[Dict[str, Any]], +) -> float: + """Gets the min possible similarity score between two bounding boxes out of all images.""" + min_possible_similarity = 1.0 + for prediction, label in zip(predictions, labels): + lab_bboxes, lab_labels = _separate_label(label) + pred_bboxes, pred_labels, _ = _separate_prediction(prediction) + iou_matrix = _get_overlap_matrix(lab_bboxes, pred_bboxes) + dist_matrix = 1 - _get_dist_matrix(lab_bboxes, pred_bboxes) + similarity_matrix = iou_matrix * alpha + (1 - alpha) * (1 - dist_matrix) + non_zero_similarity_matrix = similarity_matrix[np.nonzero(similarity_matrix)] + min_image_similarity = ( + 1.0 if 0 in non_zero_similarity_matrix.shape else np.min(non_zero_similarity_matrix) + ) + min_possible_similarity = np.min([min_possible_similarity, min_image_similarity]) + return min_possible_similarity + + +def _get_valid_inputs_for_compute_scores_per_image( + alpha: float, + *, + label: Optional[Dict[str, Any]] = None, + prediction: Optional[np.ndarray] = None, + pred_labels: Optional[np.ndarray] = None, + pred_label_probs: Optional[np.ndarray] = None, + pred_bboxes: Optional[np.ndarray] = None, + lab_labels: Optional[np.ndarray] = None, + lab_bboxes: Optional[np.ndarray] = None, + similarity_matrix: Optional[np.ndarray] = None, + iou_matrix: Optional[np.ndarray] = None, + min_possible_similarity: Optional[float] = None, +) -> AuxiliaryTypesDict: + """Returns valid inputs for compute scores by either passing through values or calculating the inputs internally.""" + if lab_labels is None or lab_bboxes is None: + if label is None: + raise ValueError( + f"Pass in either one of label or label labels into auxiliary inputs. Both can not be None." + ) + lab_bboxes, lab_labels = _separate_label(label) + + if pred_labels is None or pred_label_probs is None or pred_bboxes is None: + if prediction is None: + raise ValueError( + f"Pass in either one of prediction or prediction labels and prediction probabilities into auxiliary inputs. Both can not be None." + ) + pred_bboxes, pred_labels, pred_label_probs = _separate_prediction(prediction) + + if similarity_matrix is None: + iou_matrix = _get_overlap_matrix(lab_bboxes, pred_bboxes) + dist_matrix = 1 - _get_dist_matrix(lab_bboxes, pred_bboxes) + similarity_matrix = iou_matrix * alpha + (1 - alpha) * (1 - dist_matrix) + + if iou_matrix is None: + iou_matrix = _get_overlap_matrix(lab_bboxes, pred_bboxes) + + if min_possible_similarity is None: + min_possible_similarity = ( + 1.0 + if 0 in similarity_matrix.shape + else np.min(similarity_matrix[np.nonzero(similarity_matrix)]) + ) + + auxiliary_input_dict: AuxiliaryTypesDict = { + "pred_labels": pred_labels, + "pred_label_probs": pred_label_probs, + "pred_bboxes": pred_bboxes, + "lab_labels": lab_labels, + "lab_bboxes": lab_bboxes, + "similarity_matrix": similarity_matrix, + "iou_matrix": iou_matrix, + "min_possible_similarity": min_possible_similarity, + } + + return auxiliary_input_dict + + +def _get_valid_inputs_for_compute_scores( + alpha: float, + labels: Optional[List[Dict[str, Any]]] = None, + predictions: Optional[List[np.ndarray]] = None, +) -> List[AuxiliaryTypesDict]: + """Takes in alpha, labels and predictions and returns auxiliary input dictionary containing divided parts of labels and prediction per image.""" + if predictions is None or labels is None: + raise ValueError( + f"Predictions and labels can not be None. Both are needed to get valid inputs." + ) + min_possible_similarity = _get_min_possible_similarity(alpha, predictions, labels) + + auxiliary_inputs = [] + + for prediction, label in zip(predictions, labels): + auxiliary_input_dict = _get_valid_inputs_for_compute_scores_per_image( + alpha=alpha, + label=label, + prediction=prediction, + min_possible_similarity=min_possible_similarity, + ) + auxiliary_inputs.append(auxiliary_input_dict) + + return auxiliary_inputs + + +def _get_valid_score(scores_arr: np.ndarray, temperature: float) -> float: + """Given scores array, returns valid score (softmin) or 1. Checks validity of score.""" + scores_arr = scores_arr[~np.isnan(scores_arr)] + if len(scores_arr) > 0: + valid_score = softmin1d(scores_arr, temperature=temperature) + else: + valid_score = 1.0 + return valid_score + + +def _get_valid_subtype_score_params( + alpha: Optional[float] = None, + low_probability_threshold: Optional[float] = None, + high_probability_threshold: Optional[float] = None, + temperature: Optional[float] = None, +): + """This function returns valid params for subtype score. If param is None, then default constant is returned""" + if alpha is None: + alpha = ALPHA + if low_probability_threshold is None: + low_probability_threshold = LOW_PROBABILITY_THRESHOLD + if high_probability_threshold is None: + high_probability_threshold = HIGH_PROBABILITY_THRESHOLD + if temperature is None: + temperature = TEMPERATURE + return alpha, low_probability_threshold, high_probability_threshold, temperature + + +def _get_aggregation_weights( + aggregation_weights: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """This function validates aggregation weights, returning the default weights if none are provided.""" + if aggregation_weights is None: + aggregation_weights = { + "overlooked": CUSTOM_SCORE_WEIGHT_OVERLOOKED, + "swap": CUSTOM_SCORE_WEIGHT_SWAP, + "badloc": CUSTOM_SCORE_WEIGHT_BADLOC, + } + else: + assert_valid_aggregation_weights(aggregation_weights) + return aggregation_weights + + +def _compute_overlooked_box_scores_for_image( + alpha: float, + high_probability_threshold: float, + label: Optional[Dict[str, Any]] = None, + prediction: Optional[np.ndarray] = None, + pred_labels: Optional[np.ndarray] = None, + pred_label_probs: Optional[np.ndarray] = None, + pred_bboxes: Optional[np.ndarray] = None, + lab_labels: Optional[np.ndarray] = None, + lab_bboxes: Optional[np.ndarray] = None, + similarity_matrix: Optional[np.ndarray] = None, + iou_matrix: Optional[np.ndarray] = None, + min_possible_similarity: Optional[float] = None, +) -> np.ndarray: + """This method returns one score per predicted box (above threshold) in an image. Score from 0 to 1 ranking how overlooked the box is.""" + + auxiliary_input_dict = _get_valid_inputs_for_compute_scores_per_image( + alpha=alpha, + label=label, + prediction=prediction, + pred_labels=pred_labels, + pred_label_probs=pred_label_probs, + pred_bboxes=pred_bboxes, + lab_labels=lab_labels, + lab_bboxes=lab_bboxes, + similarity_matrix=similarity_matrix, + min_possible_similarity=min_possible_similarity, + ) + + pred_labels = auxiliary_input_dict["pred_labels"] + pred_label_probs = auxiliary_input_dict["pred_label_probs"] + lab_labels = auxiliary_input_dict["lab_labels"] + similarity_matrix = auxiliary_input_dict["similarity_matrix"] + min_possible_similarity = auxiliary_input_dict["min_possible_similarity"] + iou_matrix = auxiliary_input_dict["iou_matrix"] + + scores_overlooked = np.empty(len(pred_labels)) # same length as num of predicted boxes + + for iid, k in enumerate(pred_labels): + if pred_label_probs[iid] < high_probability_threshold or np.any(iou_matrix[:, iid] > 0): + scores_overlooked[iid] = np.nan + continue + + k_similarity = similarity_matrix[lab_labels == k, iid] + + if len(k_similarity) == 0: # if there are no annotated boxes of class k + score = min_possible_similarity * (1 - pred_label_probs[iid]) + else: + closest_annotated_box = np.argmax(k_similarity) + score = k_similarity[closest_annotated_box] + + scores_overlooked[iid] = score + + return scores_overlooked + + +def compute_overlooked_box_scores( + *, + labels: Optional[List[Dict[str, Any]]] = None, + predictions: Optional[List[np.ndarray]] = None, + alpha: Optional[float] = None, + high_probability_threshold: Optional[float] = None, + auxiliary_inputs: Optional[List[AuxiliaryTypesDict]] = None, +) -> List[np.ndarray]: + """ + Returns an array of overlooked box scores for each image. + This is a helper method mostly for advanced users. + + An overlooked box error is when an image contains an object that is one of the given classes but there is no annotated bounding box around it. + Score per high-confidence predicted bounding box is between 0 and 1, with lower values indicating boxes we are more confident were overlooked in the given label. + + Each image has ``L`` annotated bounding boxes and ``M`` predicted bounding boxes. + A score is calculated for each predicted box in each of the ``N`` images in dataset. + + Note: ``M`` and ``L`` can be a different values for each image, as the number of annotated and predicted boxes varies. + + Parameters + ---------- + labels: + A list of ``N`` dictionaries such that ``labels[i]`` contains the given labels for the `i`-th image. + Refer to documentation for this argument in :py:func:`find_label_issues ` for further details. + + predictions: + A list of ``N`` ``np.ndarray`` such that ``predictions[i]`` corresponds to the model predictions for the `i`-th image. + Refer to documentation for this argument in :py:func:`find_label_issues ` for further details. + + alpha: + Optional weighting between IoU and Euclidean distance when calculating similarity between predicted and annotated boxes. High alpha means weighting IoU more heavily over Euclidean distance. If no alpha is provided, a good default is used. + + high_probability_threshold: + Optional probability threshold that determines which predicted boxes are considered high-confidence when computing overlooked scores. If not provided, a good default is used. + + auxiliary_inputs: + Optional list of ``N`` dictionaries containing keys for sub-parts of label and prediction per image. Useful to minimize computation when computing multiple box scores for a single set of images. For the `i`-th image, `auxiliary_inputs[i]` should contain following keys: + + * pred_labels: np.ndarray + Array of predicted classes for `i`-th image of shape ``(M,)``. + * pred_label_probs: np.ndarray + Array of predicted class probabilities for `i`-th image of shape ``(M,)``. + * pred_bboxes: np.ndarray + Array of predicted bounding boxes for `i`-th image of shape ``(M, 4)``. + * lab_labels: np.ndarray + Array of given label classed for `i`-th image of shape ``(L,)``. + * lab_bboxes: np.ndarray + Array of given label bounding boxes for `i`-th image of shape ``(L, 4)``. + * similarity_matrix: np.ndarray + Similarity matrix between labels and predictions `i`-th image. + * min_possible_similarity: float + Minimum possible similarity value greater than 0 between labels and predictions for the entire dataset. + Returns + --------- + scores_overlooked: + A list of ``N`` numpy arrays where scores_overlooked[i] is an array of size ``M`` of overlooked scores per predicted box for the `i`-th image. + """ + ( + alpha, + low_probability_threshold, + high_probability_threshold, + temperature, + ) = _get_valid_subtype_score_params(alpha, None, high_probability_threshold, None) + + if auxiliary_inputs is None: + auxiliary_inputs = _get_valid_inputs_for_compute_scores(alpha, labels, predictions) + + scores_overlooked = [] + for auxiliary_input_dict in auxiliary_inputs: + scores_overlooked_per_box = _compute_overlooked_box_scores_for_image( + alpha=alpha, + high_probability_threshold=high_probability_threshold, + **auxiliary_input_dict, + ) + scores_overlooked.append(scores_overlooked_per_box) + return scores_overlooked + + +def _compute_badloc_box_scores_for_image( + alpha: float, + low_probability_threshold: float, + label: Optional[Dict[str, Any]] = None, + prediction: Optional[np.ndarray] = None, + pred_labels: Optional[np.ndarray] = None, + pred_label_probs: Optional[np.ndarray] = None, + pred_bboxes: Optional[np.ndarray] = None, + lab_labels: Optional[np.ndarray] = None, + lab_bboxes: Optional[np.ndarray] = None, + similarity_matrix: Optional[np.ndarray] = None, + iou_matrix: Optional[np.ndarray] = None, + min_possible_similarity: Optional[float] = None, +) -> np.ndarray: + """This method returns one score per labeled box in an image. Score from 0 to 1 ranking how badly located the box is.""" + + auxiliary_input_dict = _get_valid_inputs_for_compute_scores_per_image( + alpha=alpha, + label=label, + prediction=prediction, + pred_labels=pred_labels, + pred_label_probs=pred_label_probs, + pred_bboxes=pred_bboxes, + lab_labels=lab_labels, + lab_bboxes=lab_bboxes, + similarity_matrix=similarity_matrix, + iou_matrix=iou_matrix, + min_possible_similarity=min_possible_similarity, + ) + pred_labels = auxiliary_input_dict["pred_labels"] + pred_label_probs = auxiliary_input_dict["pred_label_probs"] + lab_labels = auxiliary_input_dict["lab_labels"] + similarity_matrix = auxiliary_input_dict["similarity_matrix"] + iou_matrix = auxiliary_input_dict["iou_matrix"] + + scores_badloc = np.empty(len(lab_labels)) + + for iid, k in enumerate(lab_labels): + k_similarity = similarity_matrix[iid, pred_labels == k] + k_pred = pred_label_probs[pred_labels == k] + k_iou = iou_matrix[iid, pred_labels == k] + + if len(k_pred) == 0 or np.max(k_pred) <= low_probability_threshold: + scores_badloc[iid] = 1.0 + continue + + idx_at_least_low_probability_threshold = np.where(k_pred > low_probability_threshold)[0] + idx_at_least_intersection_threshold = np.where(k_iou > 0)[0] + combined_idx = np.intersect1d( + idx_at_least_low_probability_threshold, idx_at_least_intersection_threshold + ) + + k_similarity = k_similarity[combined_idx] + k_pred = k_pred[combined_idx] + + scores_badloc[iid] = np.max(k_similarity) if len(k_pred) > 0 else 1.0 + return scores_badloc + + +def compute_badloc_box_scores( + *, + labels: Optional[List[Dict[str, Any]]] = None, + predictions: Optional[List[np.ndarray]] = None, + alpha: Optional[float] = None, + low_probability_threshold: Optional[float] = None, + auxiliary_inputs: Optional[List[AuxiliaryTypesDict]] = None, +) -> List[np.ndarray]: + """ + Returns a numeric score for each annotated bounding box in each image, estimating the likelihood that the edges of this box are not badly located. + This is a helper method mostly for advanced users. + + A badly located box error is when a box has the correct label but incorrect coordinates so it does not correctly encapsulate the entire object it is for. + Score per high-confidence predicted bounding box is between 0 and 1, with lower values indicating boxes we are more confident were overlooked in the given label. + + Each image has ``L`` annotated bounding boxes and ``M`` predicted bounding boxes. + A score is calculated for each predicted box in each of the ``N`` images in dataset. + + Note: ``M`` and ``L`` can be a different values for each image, as the number of annotated and predicted boxes varies. + + Parameters + ---------- + labels: + A list of ``N`` dictionaries such that ``labels[i]`` contains the given labels for the `i`-th image. + Refer to documentation for this argument in :py:func:`find_label_issues ` for further details. + + predictions: + A list of ``N`` ``np.ndarray`` such that ``predictions[i]`` corresponds to the model predictions for the `i`-th image. + Refer to documentation for this argument in :py:func:`find_label_issues ` for further details. + + alpha: + Optional weighting between IoU and Euclidean distance when calculating similarity between predicted and annotated boxes. High alpha means weighting IoU more heavily over Euclidean distance. If no alpha is provided, a good default is used. + + low_probability_threshold: + Optional minimum probability threshold that determines which predicted boxes are considered when computing badly located scores. If not provided, a good default is used. + + auxiliary_inputs: + Optional list of ``N`` dictionaries containing keys for sub-parts of label and prediction per image. Useful to minimize computation when computing multiple box scores for a single set of images. For the `i`-th image, `auxiliary_inputs[i]` should contain following keys: + + * pred_labels: np.ndarray + Array of predicted classes for `i`-th image of shape ``(M,)``. + * pred_label_probs: np.ndarray + Array of predicted class probabilities for `i`-th image of shape ``(M,)``. + * pred_bboxes: np.ndarray + Array of predicted bounding boxes for `i`-th image of shape ``(M, 4)``. + * lab_labels: np.ndarray + Array of given label classed for `i`-th image of shape ``(L,)``. + * lab_bboxes: np.ndarray + Array of given label bounding boxes for `i`-th image of shape ``(L, 4)``. + * similarity_matrix: np.ndarray + Similarity matrix between labels and predictions `i`-th image. + * min_possible_similarity: float + Minimum possible similarity value greater than 0 between labels and predictions for the entire dataset. + Returns + --------- + scores_badloc: + A list of ``N`` numpy arrays where scores_badloc[i] is an array of size ``L`` badly located scores per annotated box for the `i`-th image. + """ + ( + alpha, + low_probability_threshold, + high_probability_threshold, + temperature, + ) = _get_valid_subtype_score_params(alpha, low_probability_threshold, None, None) + if auxiliary_inputs is None: + auxiliary_inputs = _get_valid_inputs_for_compute_scores(alpha, labels, predictions) + + scores_badloc = [] + for auxiliary_input_dict in auxiliary_inputs: + scores_badloc_per_box = _compute_badloc_box_scores_for_image( + alpha=alpha, low_probability_threshold=low_probability_threshold, **auxiliary_input_dict + ) + scores_badloc.append(scores_badloc_per_box) + return scores_badloc + + +def _compute_swap_box_scores_for_image( + alpha: float, + high_probability_threshold: float, + label: Optional[Dict[str, Any]] = None, + prediction: Optional[np.ndarray] = None, + pred_labels: Optional[np.ndarray] = None, + pred_label_probs: Optional[np.ndarray] = None, + pred_bboxes: Optional[np.ndarray] = None, + lab_labels: Optional[np.ndarray] = None, + lab_bboxes: Optional[np.ndarray] = None, + similarity_matrix: Optional[np.ndarray] = None, + iou_matrix: Optional[np.ndarray] = None, + min_possible_similarity: Optional[float] = None, + overlapping_label_check: Optional[bool] = True, +) -> np.ndarray: + """This method returns one score per labeled box in an image. Score from 0 to 1 ranking how likeley swapped the box is.""" + + auxiliary_input_dict = _get_valid_inputs_for_compute_scores_per_image( + alpha=alpha, + label=label, + prediction=prediction, + pred_labels=pred_labels, + pred_label_probs=pred_label_probs, + pred_bboxes=pred_bboxes, + lab_labels=lab_labels, + lab_bboxes=lab_bboxes, + similarity_matrix=similarity_matrix, + min_possible_similarity=min_possible_similarity, + ) + + pred_labels = auxiliary_input_dict["pred_labels"] + pred_label_probs = auxiliary_input_dict["pred_label_probs"] + lab_labels = auxiliary_input_dict["lab_labels"] + similarity_matrix = auxiliary_input_dict["similarity_matrix"] + min_possible_similarity = auxiliary_input_dict["min_possible_similarity"] + + if overlapping_label_check: + has_overlap_label_bboxes = _has_overlap(lab_bboxes, lab_labels) + else: + has_overlap_label_bboxes = np.array([False] * len(lab_labels)) + + scores_swap = np.empty(len(lab_labels)) + + for iid, k in enumerate(lab_labels): + not_k_idx = np.where(pred_labels != k)[0] + if has_overlap_label_bboxes[iid]: + scores_swap[iid] = min_possible_similarity + continue + if not_k_idx.size == 0 or np.all(pred_label_probs[not_k_idx] <= high_probability_threshold): + scores_swap[iid] = 1.0 + continue + + not_k_pred = pred_label_probs[not_k_idx] + idx_at_least_high_probability_threshold = np.where(not_k_pred > high_probability_threshold)[ + 0 + ] + not_k_similarity = similarity_matrix[iid, not_k_idx][ + idx_at_least_high_probability_threshold + ] + + closest_predicted_box = np.argmax(not_k_similarity) + score = np.max([min_possible_similarity, 1 - not_k_similarity[closest_predicted_box]]) + scores_swap[iid] = score + + return scores_swap + + +def compute_swap_box_scores( + *, + labels: Optional[List[Dict[str, Any]]] = None, + predictions: Optional[List[np.ndarray]] = None, + alpha: Optional[float] = None, + high_probability_threshold: Optional[float] = None, + overlapping_label_check: Optional[bool] = True, + auxiliary_inputs: Optional[List[AuxiliaryTypesDict]] = None, +) -> List[np.ndarray]: + """ + Returns a numeric score for each annotated bounding box in each image, estimating the likelihood that the class label for this box was not accidentally swapped with another class. + This is a helper method mostly for advanced users. + + A swapped box error occurs when a bounding box should be labeled as a class different to what the current label is. + Score per high-confidence predicted bounding box is between 0 and 1, with lower values indicating boxes we are more confident were overlooked in the given label. + + Each image has ``L`` annotated bounding boxes and ``M`` predicted bounding boxes. + A score is calculated for each predicted box in each of the ``N`` images in dataset. + + Note: ``M`` and ``L`` can be a different values for each image, as the number of annotated and predicted boxes varies. + + Parameters + ---------- + labels: + A list of ``N`` dictionaries such that ``labels[i]`` contains the given labels for the `i`-th image. + Refer to documentation for this argument in :py:func:`find_label_issues ` for further details. + + predictions: + A list of ``N`` ``np.ndarray`` such that ``predictions[i]`` corresponds to the model predictions for the `i`-th image. + Refer to documentation for this argument in :py:func:`find_label_issues ` for further details. + + alpha: + Optional weighting between IoU and Euclidean distance when calculating similarity between predicted and annotated boxes. High alpha means weighting IoU more heavily over Euclidean distance. If no alpha is provided, a good default is used. + + high_probability_threshold: + Optional probability threshold that determines which predicted boxes are considered high-confidence when computing overlooked scores. If not provided, a good default is used. + + overlapping_label_check : bool, default = True + If True, boxes annotated with more than one class label have their swap score penalized. Set this to False if you are not concerned when two very similar boxes exist with different class labels in the given annotations. + + auxiliary_inputs: + Optional list of ``N`` dictionaries containing keys for sub-parts of label and prediction per image. Useful to minimize computation when computing multiple box scores for a single set of images. For the `i`-th image, `auxiliary_inputs[i]` should contain following keys: + + * pred_labels: np.ndarray + Array of predicted classes for `i`-th image of shape ``(M,)``. + * pred_label_probs: np.ndarray + Array of predicted class probabilities for `i`-th image of shape ``(M,)``. + * pred_bboxes: np.ndarray + Array of predicted bounding boxes for `i`-th image of shape ``(M, 4)``. + * lab_labels: np.ndarray + Array of given label classed for `i`-th image of shape ``(L,)``. + * lab_bboxes: np.ndarray + Array of given label bounding boxes for `i`-th image of shape ``(L, 4)``. + * similarity_matrix: np.ndarray + Similarity matrix between labels and predictions `i`-th image. + * min_possible_similarity: float + Minimum possible similarity value greater than 0 between labels and predictions for the entire dataset. + Returns + --------- + scores_swap: + A list of ``N`` numpy arrays where scores_swap[i] is an array of size ``L`` swap scores per annotated box for the `i`-th image. + """ + ( + alpha, + low_probability_threshold, + high_probability_threshold, + temperature, + ) = _get_valid_subtype_score_params(alpha, None, high_probability_threshold, None) + + if auxiliary_inputs is None: + auxiliary_inputs = _get_valid_inputs_for_compute_scores(alpha, labels, predictions) + + scores_swap = [] + for auxiliary_inputs in auxiliary_inputs: + scores_swap_per_box = _compute_swap_box_scores_for_image( + alpha=alpha, + high_probability_threshold=high_probability_threshold, + overlapping_label_check=overlapping_label_check, + **auxiliary_inputs, + ) + scores_swap.append(scores_swap_per_box) + return scores_swap + + +def pool_box_scores_per_image( + box_scores: List[np.ndarray], *, temperature: Optional[float] = None +) -> np.ndarray: + """ + Aggregates all per-box scores within an image to return a single quality score for the image rather than for individual boxes within it. + This is a helper method mostly for advanced users to be used with the outputs of :py:func:`object_detection.rank.compute_overlooked_box_scores `, :py:func:`object_detection.rank.compute_badloc_box_scores `, and :py:func:`object_detection.rank.compute_swap_box_scores `. + + Score per image is between 0 and 1, with lower values indicating we are more confident image contains an error. + + Parameters + ---------- + box_scores: + A list of ``N`` numpy arrays where box_scores[i] is an array of badly located scores per box for the `i`-th image. + + temperature: + Optional temperature of the softmin function where a lower value suggests softmin acts closer to min. If not provided, a good default is used. + + Returns + --------- + image_scores: + An array of size ``N`` where ``image_scores[i]`` represents the score for the `i`-th image. + """ + + ( + alpha, + low_probability_threshold, + high_probability_threshold, + temperature, + ) = _get_valid_subtype_score_params(None, None, None, temperature) + + image_scores = np.empty( + shape=[ + len(box_scores), + ] + ) + for idx, box_score in enumerate(box_scores): + image_score = _get_valid_score(box_score, temperature=temperature) + image_scores[idx] = image_score + return image_scores + + +def _get_subtype_label_quality_scores( + labels: List[Dict[str, Any]], + predictions: List[np.ndarray], + *, + alpha: Optional[float] = None, + low_probability_threshold: Optional[float] = None, + high_probability_threshold: Optional[float] = None, + temperature: Optional[float] = None, + aggregation_weights: Optional[Dict[str, float]] = None, + overlapping_label_check: Optional[bool] = True, +) -> np.ndarray: + """ + Returns a label quality score for each of the ``N`` images in the dataset. + Score is between 0 and 1. + + 1 - clean label (given label is likely correct). + 0 - dirty label (given label is likely incorrect). + + Parameters + ---------- + labels: + A list of ``N`` dictionaries such that ``labels[i]`` contains the given labels for the `i`-th image. + Refer to documentation for this argument in :py:func:`find_label_issues ` for further details. + + predictions: + A list of ``N`` ``np.ndarray`` such that ``predictions[i]`` corresponds to the model predictions for the `i`-th image. + Refer to documentation for this argument in :py:func:`find_label_issues ` for further details. + + alpha: + Optional weighting between IoU and Euclidean distance when calculating similarity between predicted and annotated boxes. High alpha means weighting IoU more heavily over Euclidean distance. If no alpha is provided, a good default is used. + + low_probability_threshold: + Optional minimum probability threshold that determines which predicted boxes are considered when computing badly located scores. If not provided, a good default is used. + + high_probability_threshold: + Optional probability threshold that determines which predicted boxes are considered high-confidence when computing overlooked and swapped scores. If not provided, a good default is used. + + temperature: + Optional temperature of the softmin function where a lower score suggests softmin acts closer to min. If not provided, a good default is used. + + overlapping_label_check : bool, default = True + If True, boxes annotated with more than one class label have their swap score penalized. Set this to False if you are not concerned when two very similar boxes exist with different class labels in the given annotations. + + Returns + --------- + label_quality_scores: + As returned by :py:func:`get_label_quality_scores `. See function for more details. + """ + ( + alpha, + low_probability_threshold, + high_probability_threshold, + temperature, + ) = _get_valid_subtype_score_params( + alpha, low_probability_threshold, high_probability_threshold, temperature + ) + auxiliary_inputs = _get_valid_inputs_for_compute_scores(alpha, labels, predictions) + aggregation_weights = _get_aggregation_weights(aggregation_weights) + + overlooked_scores_per_box = compute_overlooked_box_scores( + alpha=alpha, + high_probability_threshold=high_probability_threshold, + auxiliary_inputs=auxiliary_inputs, + ) + overlooked_score_per_image = pool_box_scores_per_image( + overlooked_scores_per_box, temperature=temperature + ) + + badloc_scores_per_box = compute_badloc_box_scores( + alpha=alpha, + low_probability_threshold=low_probability_threshold, + auxiliary_inputs=auxiliary_inputs, + ) + badloc_score_per_image = pool_box_scores_per_image( + badloc_scores_per_box, temperature=temperature + ) + + swap_scores_per_box = compute_swap_box_scores( + alpha=alpha, + high_probability_threshold=high_probability_threshold, + auxiliary_inputs=auxiliary_inputs, + overlapping_label_check=overlapping_label_check, + ) + swap_score_per_image = pool_box_scores_per_image(swap_scores_per_box, temperature=temperature) + + scores = ( + aggregation_weights["overlooked"] * np.log(TINY_VALUE + overlooked_score_per_image) + + aggregation_weights["badloc"] * np.log(TINY_VALUE + badloc_score_per_image) + + aggregation_weights["swap"] * np.log(TINY_VALUE + swap_score_per_image) + ) + + scores = np.exp(scores) + + return scores diff --git a/cleanlab/object_detection/summary.py b/cleanlab/object_detection/summary.py new file mode 100644 index 0000000..37ea6d9 --- /dev/null +++ b/cleanlab/object_detection/summary.py @@ -0,0 +1,757 @@ +""" +Methods to display examples and their label issues in an object detection dataset. +Here each image can have multiple objects, each with its own bounding box and class label. +""" + +from multiprocessing import Pool +from typing import ( + Optional, + Any, + Dict, + Tuple, + Union, + List, + TYPE_CHECKING, + TypeVar, + DefaultDict, + cast, +) + +import numpy as np +import collections + +from cleanlab.internal.constants import ( + MAX_CLASS_TO_SHOW, + ALPHA, + EPSILON, + TINY_VALUE, +) +from cleanlab.object_detection.filter import ( + _filter_by_class, + _calculate_true_positives_false_positives, +) +from cleanlab.object_detection.rank import ( + _get_valid_inputs_for_compute_scores, + _separate_prediction, + _separate_label, + _get_prediction_type, +) + +from cleanlab.internal.object_detection_utils import bbox_xyxy_to_xywh + +if TYPE_CHECKING: + from PIL.Image import Image as Image # pragma: no cover +else: + Image = TypeVar("Image") + + +def object_counts_per_image( + labels=None, + predictions=None, + *, + auxiliary_inputs=None, +) -> Tuple[List, List]: + """Return the number of annotated and predicted objects for each image in the dataset. + + This method can help you discover images with abnormally many/few object annotations. + + Parameters + ---------- + labels : + Annotated boxes and class labels in the original dataset, which may contain some errors. + This is a list of ``N`` dictionaries such that ``labels[i]`` contains the given labels for the `i`-th image in the following format: + ``{'bboxes': np.ndarray((L,4)), 'labels': np.ndarray((L,)), 'image_name': str}`` where ``L`` is the number of annotated bounding boxes + for the `i`-th image and ``bboxes[l]`` is a bounding box of coordinates in ``[x1,y1,x2,y2]`` format with given class label ``labels[j]``. + ``image_name`` is an optional part of the labels that can be used to later refer to specific images. + + Note: Here, ``(x1,y1)`` corresponds to the top-left and ``(x2,y2)`` corresponds to the bottom-right corner of the bounding box with respect to the image matrix [e.g. `XYXY in Keras `, `Detectron 2 `]. + + For more information on proper labels formatting, check out the `MMDetection library `_. + + predictions : + Predictions output by a trained object detection model. + For the most accurate results, predictions should be out-of-sample to avoid overfitting, eg. obtained via :ref:`cross-validation `. + This is a list of ``N`` ``np.ndarray`` such that ``predictions[i]`` corresponds to the model prediction for the `i`-th image. + For each possible class ``k`` in 0, 1, ..., K-1: ``predictions[i][k]`` is a ``np.ndarray`` of shape ``(M,5)``, + where ``M`` is the number of predicted bounding boxes for class ``k``. Here the five columns correspond to ``[x1,y1,x2,y2,pred_prob]``, + where ``[x1,y1,x2,y2]`` are coordinates of the bounding box predicted by the model + and ``pred_prob`` is the model's confidence in the predicted class label for this bounding box. + + Note: Here, ``(x1,y1)`` corresponds to the top-left and ``(x2,y2)`` corresponds to the bottom-right corner of the bounding box with respect to the image matrix [e.g. `XYXY in Keras `, `Detectron 2 `]. The last column, pred_prob, represents the predicted probability that the bounding box contains an object of the class k. + + For more information see the `MMDetection package `_ for an example object detection library that outputs predictions in the correct format. + + auxiliary_inputs: optional + Auxiliary inputs to be used in the computation of counts. + The `auxiliary_inputs` can be computed using :py:func:`rank._get_valid_inputs_for_compute_scores `. + It is internally computed from the given `labels` and `predictions`. + + Returns + ------- + object_counts: Tuple[List, List] + A tuple containing two lists. The first is an array of shape ``(N,)`` containing the number of annotated objects for each image in the dataset. + The second is an array of shape ``(N,)`` containing the number of predicted objects for each image in the dataset. + """ + if auxiliary_inputs is None: + auxiliary_inputs = _get_valid_inputs_for_compute_scores(ALPHA, labels, predictions) + return ( + [len(sample["lab_bboxes"]) for sample in auxiliary_inputs], + [len(sample["pred_bboxes"]) for sample in auxiliary_inputs], + ) + + +def bounding_box_size_distribution( + labels=None, + predictions=None, + *, + auxiliary_inputs=None, + class_names: Optional[Dict[Any, Any]] = None, + sort: bool = False, +) -> Tuple[Dict[Any, List], Dict[Any, List]]: + """Return the distribution over sizes of annotated and predicted bounding boxes across the dataset, broken down by each class. + + This method can help you find annotated/predicted boxes for a particular class that are abnormally big/small. + + Parameters + ---------- + labels: + Annotated boxes and class labels in the original dataset, which may contain some errors. + Refer to documentation for this argument in :py:func:`object_counts_per_image ` for further details. + + predictions: + Predictions output by a trained object detection model. + Refer to documentation for this argument in :py:func:`object_counts_per_image ` for further details. + + auxiliary_inputs: optional + Auxiliary inputs to be used in the computation of counts. + Refer to documentation for this argument in :py:func:`object_counts_per_image ` for further details. + + class_names: optional + A dictionary mapping one-hot-encoded class labels back to their original class names in the format ``{"integer-label": "original-class-name"}``. + You can use this argument to control the classes for which the size distribution is computed. + + sort: bool + If True, the returned dictionaries are sorted by the number of instances of each class in the dataset in descending order. + + Returns + ------- + bbox_sizes: Tuple[Dict[Any, List], Dict[Any, List]] + A tuple containing two dictionaries. Each maps each class label to a list of the sizes of annotated bounding boxes for that class in the label and prediction datasets, respectively. + """ + if auxiliary_inputs is None: + auxiliary_inputs = _get_valid_inputs_for_compute_scores(ALPHA, labels, predictions) + + lab_area: Dict[Any, list] = collections.defaultdict(list) + pred_area: Dict[Any, list] = collections.defaultdict(list) + for sample in auxiliary_inputs: + _get_bbox_areas(sample["lab_labels"], sample["lab_bboxes"], lab_area, class_names) + _get_bbox_areas(sample["pred_labels"], sample["pred_bboxes"], pred_area, class_names) + + if sort: + lab_area = dict(sorted(lab_area.items(), key=lambda x: -len(x[1]))) + pred_area = dict(sorted(pred_area.items(), key=lambda x: -len(x[1]))) + + return lab_area, pred_area + + +def class_label_distribution( + labels=None, + predictions=None, + *, + auxiliary_inputs=None, + class_names: Optional[Dict[Any, Any]] = None, +) -> Tuple[Dict[Any, float], Dict[Any, float]]: + """Returns the distribution of class labels associated with all annotated bounding boxes (or predicted bounding boxes) in the dataset. + + This method can help you understand which classes are: rare or over/under-predicted by the model overall. + + Parameters + ---------- + labels: + Annotated boxes and class labels in the original dataset, which may contain some errors. + Refer to documentation for this argument in :py:func:`object_counts_per_image ` for further details. + + predictions: + Predictions output by a trained object detection model. + Refer to documentation for this argument in :py:func:`object_counts_per_image ` for further details. + + auxiliary_inputs: optional + Auxiliary inputs to be used in the computation of counts. + Refer to documentation for this argument in :py:func:`object_counts_per_image ` for further details. + + class_names: optional + Optional dictionary mapping one-hot-encoded class labels back to their original class names in the format ``{"integer-label": "original-class-name"}``. + + Returns + ------- + class_distribution: Tuple[Dict[Any, float], Dict[Any, float]] + A tuple containing two dictionaries. The first is a dictionary mapping each class label to its frequency in the dataset annotations. + The second is a dictionary mapping each class label to its frequency in the model predictions across all images in the dataset. + """ + if auxiliary_inputs is None: + auxiliary_inputs = _get_valid_inputs_for_compute_scores(ALPHA, labels, predictions) + + lab_freq: DefaultDict[Any, int] = collections.defaultdict(int) + pred_freq: DefaultDict[Any, int] = collections.defaultdict(int) + for sample in auxiliary_inputs: + _get_class_instances(sample["lab_labels"], lab_freq, class_names) + _get_class_instances(sample["pred_labels"], pred_freq, class_names) + + label_norm = _normalize_by_total(lab_freq) + pred_norm = _normalize_by_total(pred_freq) + + return label_norm, pred_norm + + +def get_sorted_bbox_count_idxs(labels, predictions): + """ + Returns a tuple of idxs and bounding box counts of images sorted from highest to lowest number of bounding boxes. + + This plot can help you discover images with abnormally many/few object annotations. + + Parameters + ---------- + labels: + Annotated boxes and class labels in the original dataset, which may contain some errors. + Refer to documentation for this argument in :py:func:`object_counts_per_image ` for further details. + + predictions: + Predictions output by a trained object detection model. + Refer to documentation for this argument in :py:func:`object_counts_per_image ` for further details. + + + Returns + ------- + sorted_idxs: List[Tuple[int, int]], List[Tuple[int, int]] + A tuple containing two lists. The first is an array of shape ``(N,)`` containing the number of annotated objects for each image in the dataset. + The second is an array of shape ``(N,)`` containing the number of predicted objects for each image in the dataset. + """ + lab_count, pred_count = object_counts_per_image(labels, predictions) + lab_grouped = list(enumerate(lab_count)) + pred_grouped = list(enumerate(pred_count)) + + sorted_lab = sorted(lab_grouped, key=lambda x: x[1], reverse=True) + sorted_pred = sorted(pred_grouped, key=lambda x: x[1], reverse=True) + + return sorted_lab, sorted_pred + + +def plot_class_size_distributions( + labels, predictions, class_names=None, class_to_show=MAX_CLASS_TO_SHOW, **kwargs +): + """ + Plots the size distributions for bounding boxes for each class. + + This plot can help you find annotated/predicted boxes for a particular class that are abnormally big/small. + + Parameters + ---------- + labels: + Annotated boxes and class labels in the original dataset, which may contain some errors. + Refer to documentation for this argument in :py:func:`object_counts_per_image ` for further details. + + predictions: + Predictions output by a trained object detection model. + Refer to documentation for this argument in :py:func:`object_counts_per_image ` for further details. + + class_names: optional + Optional dictionary mapping one-hot-encoded class labels back to their original class names in the format ``{"integer-label": "original-class-name"}``. + You can use this argument to control the classes for which the size distribution is plotted. + + class_to_show: optional + The number of classes to show in the plots. Classes over `class_to_show` are hidden. If this argument is provided, then the classes are sorted by the number of instances in the dataset. + Defaults to `MAX_CLASS_TO_SHOW` which is set to 10. + + kwargs: + Additional keyword arguments to pass to ``plt.show()`` (matplotlib.pyplot.show). + """ + try: + import matplotlib.pyplot as plt + except ImportError as e: + raise ImportError( + "This functionality requires matplotlib. Install it via: `pip install matplotlib`" + ) + + lab_boxes, pred_boxes = bounding_box_size_distribution( + labels, + predictions, + class_names=class_names, + sort=True if class_to_show is not None else False, + ) + + for i, c in enumerate(lab_boxes.keys()): + if i >= class_to_show: + break + fig, axs = plt.subplots(1, 2, figsize=(10, 5)) + fig.suptitle(f"Size distributions for bounding box for class {c}") + for i, l in enumerate([lab_boxes, pred_boxes]): + axs[i].hist(l[c], bins="auto") + axs[i].set_xlabel("box area (pixels)") + axs[i].set_ylabel("count") + axs[i].set_title("annotated" if i == 0 else "predicted") + + plt.show(**kwargs) + + +def plot_class_distribution(labels, predictions, class_names=None, **kwargs): + """ + Plots the distribution of class labels associated with all annotated bounding boxes and predicted bounding boxes in the dataset. + + This plot can help you understand which classes are rare or over/under-predicted by the model overall. + + Parameters + ---------- + labels: + Annotated boxes and class labels in the original dataset, which may contain some errors. + Refer to documentation for this argument in :py:func:`object_counts_per_image ` for further details. + + predictions: + Predictions output by a trained object detection model. + Refer to documentation for this argument in :py:func:`object_counts_per_image ` for further details. + + class_names: optional + Optional dictionary mapping one-hot-encoded class labels back to their original class names in the format ``{"integer-label": "original-class-name"}``. + + kwargs: + Additional keyword arguments to pass to ``plt.show()`` (matplotlib.pyplot.show). + """ + try: + import matplotlib.pyplot as plt + except ImportError as e: + raise ImportError( + "This functionality requires matplotlib. Install it via: `pip install matplotlib`" + ) + + lab_dist, pred_dist = class_label_distribution(labels, predictions, class_names=class_names) + fig, axs = plt.subplots(1, 2, figsize=(10, 5)) + fig.suptitle(f"Distribution of classes in the dataset") + for i, d in enumerate([lab_dist, pred_dist]): + axs[i].pie(d.values(), labels=d.keys(), autopct="%1.1f%%") + axs[i].set_title("Annotated" if i == 0 else "Predicted") + + plt.show(**kwargs) + + +def visualize( + image: Union[str, np.ndarray, Image], + *, + label: Optional[Dict[str, Any]] = None, + prediction: Optional[np.ndarray] = None, + prediction_threshold: Optional[float] = None, + overlay: bool = True, + class_names: Optional[Dict[Any, Any]] = None, + figsize: Optional[Tuple[int, int]] = None, + save_path: Optional[str] = None, + **kwargs, +) -> None: + """Display the annotated bounding boxes (given labels) and predicted bounding boxes (model predictions) for a particular image. + Given labels are shown in red, model predictions in blue. + + + Parameters + ---------- + image: + Image object loaded into memory or full path to the image file. If path is provided, image is loaded into memory. + + label: + The given label for a single image in the format ``{'bboxes': np.ndarray((L,4)), 'labels': np.ndarray((L,))}`` where + ``L`` is the number of bounding boxes for the `i`-th image and ``bboxes[j]`` is in the format ``[x1,y1,x2,y2]`` with given label ``labels[j]``. + + Note: Here, ``(x1,y1)`` corresponds to the top-left and ``(x2,y2)`` corresponds to the bottom-right corner of the bounding box with respect to the image matrix [e.g. `XYXY in Keras `, `Detectron 2 `]. + + prediction: + A prediction for a single image in the format ``np.ndarray((K,))`` and ``prediction[k]`` is of shape ``np.ndarray(N,5)`` + where ``M`` is the number of predicted bounding boxes for class ``k`` and the five columns correspond to ``[x,y,x,y,pred_prob]`` where + ``[x1,y1,x2,y2]`` are the bounding box coordinates predicted by the model and ``pred_prob`` is the model's confidence in ``predictions[i]``. + + Note: Here, ``(x1,y1)`` corresponds to the top-left and ``(x2,y2)`` corresponds to the bottom-right corner of the bounding box with respect to the image matrix [e.g. `XYXY in Keras `, `Detectron 2 `]. The last column, pred_prob, represents the predicted probability that the bounding box contains an object of the class k. + + prediction_threshold: + All model-predicted bounding boxes with confidence (`pred_prob`) + below this threshold are omitted from the visualization. + + overlay: bool + If True, display a single image with given labels and predictions overlaid. + If False, display two images (side by side) with the left image showing the model predictions and the right image showing the given label. + + class_names: + Optional dictionary mapping one-hot-encoded class labels back to their original class names in the format ``{"integer-label": "original-class-name"}``. + + save_path: + Path to save figure at. If a path is provided, the figure is saved. To save in a specific image format, add desired file extension to the end of `save_path`. Allowed file extensions are: 'png', 'pdf', 'ps', 'eps', and 'svg'. + + figsize: + Optional figure size for plotting the image. + Corresponds to ``matplotlib.figure.figsize``. + + kwargs: + Additional keyword arguments to pass to ``plt.show()`` (matplotlib.pyplot.show). + """ + try: + import matplotlib.pyplot as plt + from matplotlib.axes import Axes + except ImportError as e: + raise ImportError( + "This functionality requires matplotlib. Install it via: `pip install matplotlib`" + ) + + # Create figure and axes + if isinstance(image, str): + image = plt.imread(image) + + if prediction is not None: + prediction_type = _get_prediction_type(prediction) + pbbox, plabels, pred_probs = _separate_prediction( + prediction, prediction_type=prediction_type + ) + + if prediction_threshold is not None: + keep_idx = np.where(pred_probs > prediction_threshold) + pbbox = pbbox[keep_idx] + plabels = plabels[keep_idx] + + if label is not None: + abbox, alabels = _separate_label(label) + + if overlay: + figsize = (8, 5) if figsize is None else figsize + fig, ax = plt.subplots(frameon=False, figsize=figsize) + plt.axis("off") + ax.imshow(image) + if label is not None: + fig, ax = _draw_boxes( + fig, ax, abbox, alabels, edgecolor="r", linestyle="-", linewidth=1 + ) + if prediction is not None: + _, _ = _draw_boxes(fig, ax, pbbox, plabels, edgecolor="b", linestyle="-.", linewidth=1) + else: + figsize = (14, 10) if figsize is None else figsize + fig, axes = plt.subplots(nrows=1, ncols=2, frameon=False, figsize=figsize) + axes = cast(Tuple[Axes, Axes], axes) + axes[0].axis("off") + axes[0].imshow(image) + axes[1].axis("off") + axes[1].imshow(image) + + if label is not None: + fig, ax = _draw_boxes( + fig, axes[0], abbox, alabels, edgecolor="r", linestyle="-", linewidth=1 + ) + if prediction is not None: + _, _ = _draw_boxes( + fig, axes[1], pbbox, plabels, edgecolor="b", linestyle="-.", linewidth=1 + ) + bbox_extra_artists = None + if label or prediction is not None: + legend, plt = _plot_legend(class_names, label, prediction) + bbox_extra_artists = (legend,) + + if save_path: + allowed_image_formats = set(["png", "pdf", "ps", "eps", "svg"]) + image_format: Optional[str] = None + if save_path.split(".")[-1] in allowed_image_formats and "." in save_path: + image_format = save_path.split(".")[-1] + plt.savefig( + save_path, + format=image_format, + bbox_extra_artists=bbox_extra_artists, + bbox_inches="tight", + transparent=True, + pad_inches=0.5, + ) + plt.show(**kwargs) + + +def _get_per_class_confusion_matrix_dict_( + labels: List[Dict[str, Any]], + predictions: List[np.ndarray], + iou_threshold: Optional[float] = 0.5, + num_procs: int = 1, +) -> DefaultDict[int, Dict[str, int]]: + """ + Returns a confusion matrix dictionary for each class containing the number of True Positive, False Positive, and False Negative detections from the object detection model. + """ + num_classes = len(predictions[0]) + num_images = len(predictions) + pool = Pool(num_procs) + counter_dict: DefaultDict[int, dict[str, int]] = collections.defaultdict( + lambda: {"TP": 0, "FP": 0, "FN": 0} + ) + + for class_num in range(num_classes): + pred_bboxes, lab_bboxes = _filter_by_class(labels, predictions, class_num) + tpfpfn = pool.starmap( + _calculate_true_positives_false_positives, + zip( + pred_bboxes, + lab_bboxes, + [iou_threshold for _ in range(num_images)], + [True for _ in range(num_images)], + ), + ) + + for image_idx, (tp, fp, fn) in enumerate(tpfpfn): # type: ignore + counter_dict[class_num]["TP"] += np.sum(tp) + counter_dict[class_num]["FP"] += np.sum(fp) + counter_dict[class_num]["FN"] += np.sum(fn) + + return counter_dict + + +def _sort_dict_to_list(index_value_dict): + """ + Convert a dictionary to a list sorted by index and return the values in that order. + + Parameters: + - index_value_dict (dict): The input dictionary where keys represent indices and values are the corresponding elements. + + Returns: + list: A list containing the values from the input dictionary, sorted by index. + + Example: + >>> my_dict = {'0': '0', '1': '1', '2': '2', '3': '3', '4': '4'} + >>> sort_dict_to_list(my_dict) + ['0', '1', '2', '3', '4'] + """ + sorted_list = [ + value for key, value in sorted(index_value_dict.items(), key=lambda x: int(x[0])) + ] + return sorted_list + + +def get_average_per_class_confusion_matrix( + labels: List[Dict[str, Any]], + predictions: List[np.ndarray], + num_procs: int = 1, + class_names: Optional[Dict[Any, Any]] = None, +) -> Dict[Union[int, str], Dict[str, float]]: + """ + Compute a confusion matrix dictionary for each class containing the average number of True Positive, False Positive, and False Negative detections from the object detection model across a range of Intersection over Union thresholds. + + At each IoU threshold, the metrics are calculated as follows: + - True Positive (TP): Instances where the model correctly identifies the class with IoU above the threshold. + - False Positive (FP): Instances where the model predicts the class, but IoU is below the threshold. + - False Negative (FN): Instances where the ground truth class is not predicted by the model. + + The average confusion matrix provides insights into the model strengths and potential biases. + + Note: lower TP at certain IoU thresholds does not necessarily imply that everything else is FP, instead it indicates that, at those specific IoU thresholds, the model is not performing as well in terms of correctly identifying class instances. The other metrics (FP and FN) provide additional information about the model's behavior. + + Note: Since we average over many IoU thresholds, 'TP', 'FP', and 'FN' may contain float values representing the average across these thresholds. + + Parameters + ---------- + labels: + A list of ``N`` dictionaries such that ``labels[i]`` contains the given labels for the `i`-th image. + Refer to documentation for this argument in :py:func:`object_detection.filter.find_label_issues ` for further details. + + predictions: + A list of ``N`` ``np.ndarray`` such that ``predictions[i]`` corresponds to the model predictions for the `i`-th image. + Refer to documentation for this argument in :py:func:`object_detection.filter.find_label_issues ` for further details. + + num_procs: + Number of processes for parallelization. Default is 1. + + class_names: + Optional dictionary mapping one-hot-encoded class labels back to their original class names in the format ``{"integer-label": "original-class-name"}`` + + + Returns + ------- + avg_metrics: dict + A distionary containing the average confusion matrix. + + The default range of Intersection over Union thresholds is from 0.5 to 0.95 with a step size of 0.05. + """ + iou_thrs = np.linspace(0.5, 0.95, int(np.round((0.95 - 0.5) / 0.05)) + 1, endpoint=True) + num_classes = len(predictions[0]) + if class_names is None: + class_names = {str(i): int(i) for i in list(range(num_classes))} + class_names = _sort_dict_to_list(class_names) + avg_metrics = {class_num: {"TP": 0.0, "FP": 0.0, "FN": 0.0} for class_num in class_names} + + for iou_threshold in iou_thrs: + results_dict = _get_per_class_confusion_matrix_dict_( + labels, predictions, iou_threshold, num_procs + ) + + for class_num in results_dict: + tp = results_dict[class_num]["TP"] + fp = results_dict[class_num]["FP"] + fn = results_dict[class_num]["FN"] + + avg_metrics[class_names[class_num]]["TP"] += tp + avg_metrics[class_names[class_num]]["FP"] += fp + avg_metrics[class_names[class_num]]["FN"] += fn + + num_thresholds = len(iou_thrs) * len(results_dict) + for class_name in avg_metrics: + avg_metrics[class_name]["TP"] /= num_thresholds + avg_metrics[class_name]["FP"] /= num_thresholds + avg_metrics[class_name]["FN"] /= num_thresholds + return avg_metrics + + +def calculate_per_class_metrics( + labels: List[Dict[str, Any]], + predictions: List[np.ndarray], + num_procs: int = 1, + class_names=None, +) -> Dict[Union[int, str], Dict[str, float]]: + """ + Calculate the object detection model's precision, recall, and F1 score for each class in the dataset. + + These metrics can help you identify model strengths and weaknesses, and provide reference statistics for model evaluation and comparisons. + + Parameters + ---------- + labels: + A list of ``N`` dictionaries such that ``labels[i]`` contains the given labels for the `i`-th image. + Refer to documentation for this argument in :py:func:`object_detection.filter.find_label_issues ` for further details. + + predictions: + A list of ``N`` ``np.ndarray`` such that ``predictions[i]`` corresponds to the model predictions for the `i`-th image. + Refer to documentation for this argument in :py:func:`object_detection.filter.find_label_issues ` for further details. + + num_procs: + Number of processes for parallelization. Default is 1. + + class_names: + Optional dictionary mapping one-hot-encoded class labels back to their original class names in the format ``{"integer-label": "original-class-name"}`` + + + Returns + ------- + per_class_metrics: dict + A dictionary containing per-class metrics computed from the object detection model's average confusion matrix values across a range of Intersection over Union thresholds. + + The default range of Intersection over Union thresholds is from 0.5 to 0.95 with a step size of 0.05. + """ + avg_metrics = get_average_per_class_confusion_matrix( + labels, predictions, num_procs, class_names=class_names + ) + + avg_metrics_dict = {} + for class_name in avg_metrics: + tp = avg_metrics[class_name]["TP"] + fp = avg_metrics[class_name]["FP"] + fn = avg_metrics[class_name]["FN"] + + precision = tp / (tp + fp + TINY_VALUE) # Avoid division by zero + recall = tp / (tp + fn + TINY_VALUE) # Avoid division by zero + f1 = 2 * (precision * recall) / (precision + recall + TINY_VALUE) # Avoid division by zero + + avg_metrics_dict[class_name] = { + "average precision": precision, + "average recall": recall, + "average f1": f1, + } + + return avg_metrics_dict + + +def _normalize_by_total(freq): + """Helper function to normalize a frequency distribution.""" + total = sum(freq.values()) + return {k: round(v / (total + EPSILON), 2) for k, v in freq.items()} + + +def _get_bbox_areas(labels, boxes, class_area_dict, class_names=None) -> None: + """Helper function to compute the area of bounding boxes for each class.""" + for cl, bbox in zip(labels, boxes): + if class_names is not None: + if str(cl) not in class_names: + continue + cl = class_names[str(cl)] + class_area_dict[cl].append((bbox[2] - bbox[0]) * (bbox[3] - bbox[1])) + + +def _get_class_instances(labels, class_instances_dict, class_names=None) -> None: + """Helper function to count the number of class instances in each image.""" + for cl in labels: + if class_names is not None: + cl = class_names[str(cl)] + class_instances_dict[cl] += 1 + + +def _plot_legend(class_names, label, prediction): + colors = ["black"] + colors.extend(["red"] if label is not None else []) + colors.extend(["blue"] if prediction is not None else []) + + markers = [None] + markers.extend(["s"] if label is not None else []) + markers.extend(["s"] if prediction is not None else []) + + labels = [r"$\bf{Legend}$"] + labels.extend(["given label"] if label is not None else []) + labels.extend(["predicted label"] if prediction is not None else []) + + if class_names: + colors += ["black"] + ["black"] * min(len(class_names), MAX_CLASS_TO_SHOW) + markers += [None] + [f"${class_key}$" for class_key in class_names.keys()] + labels += [r"$\bf{classes}$"] + list(class_names.values()) + + try: + import matplotlib.pyplot as plt + except ImportError as e: + raise ImportError( + "This functionality requires matplotlib. Install it via: `pip install matplotlib`" + ) + + f = lambda m, c: plt.plot([], [], marker=m, color=c, ls="none")[0] + handles = [f(marker, color) for marker, color in zip(markers, colors)] + legend = plt.legend( + handles, labels, bbox_to_anchor=(1.04, 0.05), loc="lower left", borderaxespad=0 + ) + + return legend, plt + + +def _draw_labels(ax, rect, label, edgecolor): + """Helper function to draw labels on an axis.""" + + rx, ry = rect.get_xy() + c_xleft = rx + 10 + c_xright = rx + rect.get_width() - 10 + c_ytop = ry + 12 + + if edgecolor == "r": + cx, cy = c_xleft, c_ytop + else: # edgecolor == b + cx, cy = c_xright, c_ytop + + l = ax.annotate( + label, (cx, cy), fontsize=8, fontweight="bold", color="white", ha="center", va="center" + ) + l.set_bbox(dict(facecolor=edgecolor, alpha=0.35, edgecolor=edgecolor, pad=2)) + return ax + + +def _draw_boxes(fig, ax, bboxes, labels, edgecolor="g", linestyle="-", linewidth=3): + """Helper function to draw bboxes and labels on an axis.""" + bboxes = [bbox_xyxy_to_xywh(box) for box in bboxes] + + try: + from matplotlib.patches import Rectangle + except Exception as e: + raise ImportError( + "This functionality requires matplotlib. Install it via: `pip install matplotlib`" + ) + + for (x, y, w, h), label in zip(bboxes, labels): + rect = Rectangle( + (x, y), + w, + h, + linewidth=linewidth, + linestyle=linestyle, + edgecolor=edgecolor, + facecolor="none", + ) + ax.add_patch(rect) + + if labels is not None: + ax = _draw_labels(ax, rect, label, edgecolor) + + return fig, ax diff --git a/cleanlab/outlier.py b/cleanlab/outlier.py new file mode 100644 index 0000000..7968d6a --- /dev/null +++ b/cleanlab/outlier.py @@ -0,0 +1,582 @@ +""" +Methods for finding out-of-distribution examples in a dataset via scores that quantify how atypical each example is compared to the others. + +The underlying algorithms are described in `this paper `_. +""" + +import warnings +from typing import Dict, Optional, Tuple, Union + +import numpy as np +from sklearn.exceptions import NotFittedError +from sklearn.neighbors import NearestNeighbors + +from cleanlab.count import get_confident_thresholds +from cleanlab.internal.label_quality_utils import ( + _subtract_confident_thresholds, + get_normalized_entropy, +) +from cleanlab.internal.neighbor.knn_graph import correct_knn_distances_and_indices, features_to_knn +from cleanlab.internal.numerics import softmax +from cleanlab.internal.outlier import correct_precision_errors, transform_distances_to_scores +from cleanlab.internal.validation import assert_valid_inputs, labels_to_array +from cleanlab.typing import LabelLike + + +class OutOfDistribution: + """ + Provides scores to detect Out Of Distribution (OOD) examples that are outliers in a dataset. + + Each example's OOD score lies in [0,1] with smaller values indicating examples that are less typical under the data distribution. + OOD scores may be estimated from either: numeric feature embeddings or predicted probabilities from a trained classifier. + + To get indices of examples that are the most severe outliers, call `~cleanlab.rank.find_top_issues` function on the returned OOD scores. + + Parameters + ---------- + params : dict, default = {} + Optional keyword arguments to control how this estimator is fit. Effect of arguments passed in depends on if + `OutOfDistribution` estimator will rely on `features` or `pred_probs`. These are stored as an instance attribute `self.params`. + + If `features` is passed in during ``fit()``, `params` could contain following keys: + * knn: sklearn.neighbors.NearestNeighbors, default = None + Instantiated ``NearestNeighbors`` object that's been fitted on a dataset in the same feature space. + Note that the distance metric and `n_neighbors` is specified when instantiating this class. + You can also pass in a subclass of ``sklearn.neighbors.NearestNeighbors`` which allows you to use faster + approximate neighbor libraries as long as you wrap them behind the same sklearn API. + If you specify ``knn`` here, there is no need to later call ``fit()`` before calling ``score()``. + If ``knn is None``, then by default: + The knn object is instantiated as ``sklearn.neighbors.NearestNeighbors(n_neighbors=k, metric=dist_metric).fit(features)``. + - If ``dim(features) > 3``, the distance metric is set to "cosine". + - If ``dim(features) <= 3``, the distance metric is set to "euclidean". + The implementation of the euclidean distance metric depends on the number of examples in the features array: + - For more than 100 rows, it uses scikit-learn's "euclidean" metric. This is for efficiency reasons reasons. + - For 100 or fewer rows, it uses scipy's ``scipy.spatial.distance.euclidean`` metric. This is for numerical stability reasons. + See: https://scikit-learn.org/stable/modules/neighbors.html + See: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.pairwise.euclidean_distances.html + See: https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.euclidean.html + * k : int, default=None + Optional number of neighbors to use when calculating outlier score (average distance to neighbors). + If `k` is not provided, then by default ``k = knn.n_neighbors`` or ``k = 10`` if ``knn is None``. + If an existing ``knn`` object is provided, you can still specify that outlier scores should use + a different value of `k` than originally used in the ``knn``, + as long as your specified value of `k` is smaller than the value originally used in ``knn``. + * t : int, default=1 + Optional hyperparameter only for advanced users. + Controls transformation of distances between examples into similarity scores that lie in [0,1]. + The transformation applied to distances `x` is ``exp(-x*t)``. + If you find your scores are all too close to 1, consider increasing `t`, + although the relative scores of examples will still have the same ranking across the dataset. + + If `pred_probs` is passed in during ``fit()``, `params` could contain following keys: + * confident_thresholds: np.ndarray, default = None + An array of shape ``(K, )`` where K is the number of classes. + Confident threshold for a class j is the expected (average) "self-confidence" for that class. + If you specify `confident_thresholds` here, there is no need to later call ``fit()`` before calling ``score()``. + * adjust_pred_probs : bool, True + If True, account for class imbalance by adjusting predicted probabilities + via subtraction of class confident thresholds and renormalization. + If False, you do not have to pass in `labels` later to fit this OOD estimator. + See `Northcutt et al., 2021 `_. + * method : {"entropy", "least_confidence"}, default="entropy" + Method to use when computing outlier scores based on `pred_probs`. + Letting length-K vector ``P = pred_probs[i]`` denote the given predicted class-probabilities + for the i-th example in dataset, its outlier score can either be: + + - ``'entropy'``: ``1 - sum_{j} P[j] * log(P[j]) / log(K)`` + - ``'least_confidence'``: ``max(P)`` (equivalent to Maximum Softmax Probability method from the OOD detection literature) + - ``gen``: Generalized ENtropy score from the paper of Liu, Lochman, and Zach (https://openaccess.thecvf.com/content/CVPR2023/papers/Liu_GEN_Pushing_the_Limits_of_Softmax-Based_Out-of-Distribution_Detection_CVPR_2023_paper.pdf) + + """ + + OUTLIER_PARAMS = {"k", "t", "knn"} + OOD_PARAMS = {"confident_thresholds", "adjust_pred_probs", "method", "M", "gamma"} + DEFAULT_PARAM_DICT: Dict[str, Union[str, int, float, None, np.ndarray]] = { + "k": None, # param for feature based outlier detection (number of neighbors) + "t": 1, # param for feature based outlier detection (controls transformation of outlier scores to 0-1 range) + "knn": None, # param for features based outlier detection (precomputed nearest neighbors graph to use) + "method": "entropy", # param specifying which pred_probs-based outlier detection method to use + "adjust_pred_probs": True, # param for pred_probs based outlier detection (whether to adjust the probabilities by class thresholds or not) + "confident_thresholds": None, # param for pred_probs based outlier detection (precomputed confident thresholds to use for adjustment) + "M": 100, # param for GEN method for pred_probs based outlier detection + "gamma": 0.1, # param for GEN method for pred_probs based outlier detection + } + + def __init__(self, params: Optional[dict] = None) -> None: + self._assert_valid_params(params, self.DEFAULT_PARAM_DICT) + self.params = self.DEFAULT_PARAM_DICT.copy() + if params is not None: + self.params.update(params) + if self.params["adjust_pred_probs"] and self.params["method"] == "gen": + print( + "CAUTION: GEN method is not recommended for use with adjusted pred_probs. " + "To use GEN, we recommend setting: params['adjust_pred_probs'] = False" + ) + + # scaling_factor internally used to rescale distances based on mean distances to k nearest neighbors + self.params["scaling_factor"] = None + + def fit_score( + self, + *, + features: Optional[np.ndarray] = None, + pred_probs: Optional[np.ndarray] = None, + labels: Optional[np.ndarray] = None, + verbose: bool = True, + ) -> np.ndarray: + """ + Fits this estimator to a given dataset and returns out-of-distribution scores for the same dataset. + + Scores lie in [0,1] with smaller values indicating examples that are less typical under the dataset + distribution (values near 0 indicate outliers). Exactly one of `features` or `pred_probs` needs to be passed + in to calculate scores. + + If `features` are passed in a ``NearestNeighbors`` object is fit. If `pred_probs` and 'labels' are passed in a + `confident_thresholds` ``np.ndarray`` is fit. For details see `~cleanlab.outlier.OutOfDistribution.fit`. + + Parameters + ---------- + features : np.ndarray, optional + Feature array of shape ``(N, M)``, where N is the number of examples and M is the number of features used to represent each example. + For details, `features` in the same format expected by the `~cleanlab.outlier.OutOfDistribution.fit` function. + + pred_probs : np.ndarray, optional + An array of shape ``(N, K)`` of predicted class probabilities output by a trained classifier. + For details, `pred_probs` in the same format expected by the `~cleanlab.outlier.OutOfDistribution.fit` function. + + labels : array_like, optional + A discrete array of given class labels for the data of shape ``(N,)``. + For details, `labels` in the same format expected by the `~cleanlab.outlier.OutOfDistribution.fit` function. + + verbose : bool, default = True + Set to ``False`` to suppress all print statements. + + Returns + ------- + scores : np.ndarray + If `features` are passed in, `ood_features_scores` are returned. + If `pred_probs` are passed in, `ood_predictions_scores` are returned. + For details see return of `~cleanlab.outlier.OutOfDistribution.scores` function. + + """ + scores = self._shared_fit( + features=features, + pred_probs=pred_probs, + labels=labels, + verbose=verbose, + ) + + if scores is None: # Fit was called on already fitted object so we just score vals instead + scores = self.score(features=features, pred_probs=pred_probs) + + return scores + + def fit( + self, + *, + features: Optional[np.ndarray] = None, + pred_probs: Optional[np.ndarray] = None, + labels: Optional[LabelLike] = None, + verbose: bool = True, + ): + """ + Fits this estimator to a given dataset. + + One of `features` or `pred_probs` must be specified. + + If `features` are passed in, a ``NearestNeighbors`` object is fit. + If `pred_probs` and 'labels' are passed in, a `confident_thresholds` ``np.ndarray`` is fit. + For details see `~cleanlab.outlier.OutOfDistribution` documentation. + + Parameters + ---------- + features : np.ndarray, optional + Feature array of shape ``(N, M)``, where N is the number of examples and M is the number of features used to represent each example. + All features should be **numeric**. For less structured data (e.g. images, text, categorical values, ...), you should provide + vector embeddings to represent each example (e.g. extracted from some pretrained neural network). + + pred_probs : np.ndarray, optional + An array of shape ``(N, K)`` of model-predicted probabilities, + ``P(label=k|x)``. Each row of this matrix corresponds + to an example `x` and contains the model-predicted probabilities that + `x` belongs to each possible class, for each of the K classes. The + columns must be ordered such that these probabilities correspond to + class 0, 1, ..., K-1. + + labels : array_like, optional + A discrete vector of given labels for the data of shape ``(N,)``. Supported `array_like` types include: ``np.ndarray`` or ``list``. + *Format requirements*: for dataset with K classes, labels must be in 0, 1, ..., K-1. + All the classes (0, 1, ..., and K-1) MUST be present in ``labels``, such that: ``len(set(labels)) == pred_probs.shape[1]`` + If ``params["adjust_confident_thresholds"]`` was previously set to ``False``, you do not have to pass in `labels`. + Note: multi-label classification is not supported by this method, each example must belong to a single class, e.g. ``labels = np.ndarray([1,0,2,1,1,0...])``. + + verbose : bool, default = True + Set to ``False`` to suppress all print statements. + + """ + _ = self._shared_fit( + features=features, + pred_probs=pred_probs, + labels=labels, + verbose=verbose, + ) + + def score( + self, *, features: Optional[np.ndarray] = None, pred_probs: Optional[np.ndarray] = None + ) -> np.ndarray: + """ + Use fitted estimator and passed in `features` or `pred_probs` to calculate out-of-distribution scores for a dataset. + + Score for each example corresponds to the likelihood this example stems from the same distribution as the dataset previously specified in ``fit()`` (i.e. is not an outlier). + + If `features` are passed, returns OOD score for each example based on its feature values. + If `pred_probs` are passed, returns OOD score for each example based on classifier's probabilistic predictions. + You may have to previously call ``fit()`` or call ``fit_score()`` instead. + + Parameters + ---------- + features : np.ndarray, optional + Feature array of shape ``(N, M)``, where N is the number of examples and M is the number of features used to represent each example. + For details, see `features` in `~cleanlab.outlier.OutOfDistribution.fit` function. + + pred_probs : np.ndarray, optional + An array of shape ``(N, K)`` of predicted class probabilities output by a trained classifier. + For details, see `pred_probs` in `~cleanlab.outlier.OutOfDistribution.fit` function. + + Returns + ------- + scores : np.ndarray + Scores lie in [0,1] with smaller values indicating examples that are less typical under the dataset distribution + (values near 0 indicate outliers). + + If `features` are passed, `ood_features_scores` are returned. + The score is based on the average distance between the example and its K nearest neighbors in the dataset + (in feature space). + + If `pred_probs` are passed, `ood_predictions_scores` are returned. + The score is based on the uncertainty in the classifier's predicted probabilities. + """ + self._assert_valid_inputs(features, pred_probs) + + if features is not None: + if self.params["knn"] is None: + raise ValueError( + "OOD estimator needs to be fit on features first. Call `fit()` or `fit_scores()` before this function." + ) + scores, _ = self._get_ood_features_scores( + features, **self._get_params(self.OUTLIER_PARAMS) + ) + + if pred_probs is not None: + if self.params["confident_thresholds"] is None and self.params["adjust_pred_probs"]: + raise ValueError( + "OOD estimator needs to be fit on pred_probs first since params['adjust_pred_probs']=True. Call `fit()` or `fit_scores()` before this function." + ) + scores, _ = _get_ood_predictions_scores(pred_probs, **self._get_params(self.OOD_PARAMS)) + + return scores + + def _get_params(self, param_keys) -> dict: + """Get function specific dictionary of parameters (i.e. only those in param_keys).""" + return {k: v for k, v in self.params.items() if k in param_keys} + + @staticmethod + def _assert_valid_params(params, param_keys): + """Validate passed in params and get list of parameters in param that are not in param_keys.""" + if params is not None: + wrong_params = list(set(params.keys()).difference(set(param_keys))) + if len(wrong_params) > 0: + raise ValueError( + f"Passed in params dict can only contain {param_keys}. Remove {wrong_params} from params dict." + ) + + @staticmethod + def _assert_valid_inputs(features, pred_probs): + """Check whether features and pred_prob inputs are valid, throw error if not.""" + if features is None and pred_probs is None: + raise ValueError( + "Not enough information to compute scores. Pass in either features or pred_probs." + ) + + if features is not None and pred_probs is not None: + raise ValueError( + "Cannot fit to OOD Estimator to both features and pred_probs. Pass in either one or the other." + ) + + if features is not None and len(features.shape) != 2: + raise ValueError( + "Feature array needs to be of shape (N, M), where N is the number of examples and M is the " + "number of features used to represent each example. " + ) + + def _shared_fit( + self, + *, + features: Optional[np.ndarray] = None, + pred_probs: Optional[np.ndarray] = None, + labels: Optional[LabelLike] = None, + verbose: bool = True, + ) -> Optional[np.ndarray]: + """ + Shared fit functionality between ``fit()`` and ``fit_score()``. + + For details, refer to `~cleanlab.outlier.OutOfDistribution.fit` + or `~cleanlab.outlier.OutOfDistribution.fit_score`. + """ + self._assert_valid_inputs(features, pred_probs) + scores = None # If none scores are returned, fit was skipped + + if features is not None: + if self.params["knn"] is not None: + # No fitting twice if knn object already fit + warnings.warn( + "A KNN estimator has previously already been fit, call score() to apply it to data, or create a new OutOfDistribution object to fit a different estimator.", + UserWarning, + ) + else: + # Get ood features scores + if verbose: + print("Fitting OOD estimator based on provided features ...") + scores, knn = self._get_ood_features_scores( + features, **self._get_params(self.OUTLIER_PARAMS) + ) + self.params["knn"] = knn + + if pred_probs is not None: + if self.params["confident_thresholds"] is not None: + # No fitting twice if confident_thresholds object already fit + warnings.warn( + "Confident thresholds have previously already been fit, call score() to apply them to data, or create a new OutOfDistribution object to fit a different estimator.", + UserWarning, + ) + else: + # Get ood predictions scores + if verbose: + print("Fitting OOD estimator based on provided pred_probs ...") + scores, confident_thresholds = _get_ood_predictions_scores( + pred_probs, + labels=labels, + **self._get_params(self.OOD_PARAMS), + ) + if confident_thresholds is None: + warnings.warn( + "No estimates need to be be fit under the provided params, so you could directly call " + "score() as an alternative.", + UserWarning, + ) + else: + self.params["confident_thresholds"] = confident_thresholds + return scores + + def _get_ood_features_scores( + self, + features: Optional[np.ndarray] = None, + knn: Optional[NearestNeighbors] = None, + k: Optional[int] = None, + t: int = 1, + ) -> Tuple[np.ndarray, Optional[NearestNeighbors]]: + """ + Return outlier score based on feature values using `k` nearest neighbors. + + The outlier score for each example is computed inversely proportional to + the average distance between this example and its K nearest neighbors (in feature space). + + Parameters + ---------- + features : np.ndarray + Feature array of shape ``(N, M)``, where N is the number of examples and M is the number of features used to represent each example. + For details, `features` in the same format expected by the `~cleanlab.outlier.OutOfDistribution.fit` function. + + knn : sklearn.neighbors.NearestNeighbors, default = None + For details, see key `knn` in the params dict arg of `~cleanlab.outlier.OutOfDistribution`. + + k : int, default=None + Optional number of neighbors to use when calculating outlier score (average distance to neighbors). + For details, see key `k` in the params dict arg of `~cleanlab.outlier.OutOfDistribution`. + + t : int, default=1 + Controls transformation of distances between examples into similarity scores that lie in [0,1]. + For details, see key `t` in the params dict arg of `~cleanlab.outlier.OutOfDistribution`. + + Returns + ------- + ood_features_scores : Tuple[np.ndarray, Optional[NearestNeighbors]] + Return a tuple whose first element is array of `ood_features_scores` and second is a `knn` Estimator object. + """ + DEFAULT_K = 10 + # fit skip over (if knn is not None) then skipping fit and suggest score else fit. + distance_metric = None + correct_knn = False + if knn is None: # setup default KNN estimator + # Make sure both knn and features are not None + knn = features_to_knn(features, n_neighbors=k) + correct_knn = True + features = None # features should be None in knn.kneighbors(features) to avoid counting duplicate data points + # Log knn metric as string to ensure compatibility for score correction + distance_metric = ( + metric if isinstance((metric := knn.metric), str) else str(metric.__name__) + ) + k = knn.n_neighbors + + elif k is None: + k = knn.n_neighbors + + max_k = knn.n_neighbors # number of neighbors previously used in NearestNeighbors object + if k > max_k: # if k provided is too high, use max possible number of nearest neighbors + warnings.warn( + f"Chosen k={k} cannot be greater than n_neighbors={max_k} which was used when fitting " + f"NearestNeighbors object! Value of k changed to k={max_k}.", + UserWarning, + ) + k = max_k + + # Fit knn estimator on the features if a non-fitted estimator is passed in + try: + knn.kneighbors(features) + except NotFittedError: + knn.fit(features) + + # Get distances to k-nearest neighbors Note that the knn object contains the specification of distance metric + # and n_neighbors (k value) If our query set of features matches the training set used to fit knn, the nearest + # neighbor of each point is the point itself, at a distance of zero. + distances, indices = knn.kneighbors(features) + if ( + correct_knn + ): # This should only happen if knn is None at the start of this function. Will NEVER happen for approximate KNN provided by user. + _features_for_correction = ( + knn._fit_X if features is None else features + ) # Hacky way to get features (training or test). Storing np.unique results is a hassle. ONLY WORKS WITH sklearn NearestNeighbors object + distances, _ = correct_knn_distances_and_indices( + features=_features_for_correction, + distances=distances, + indices=indices, + ) + + # Calculate average distance to k-nearest neighbors + avg_knn_distances = distances[:, :k].mean(axis=1) + + if self.params["scaling_factor"] is None: + self.params["scaling_factor"] = float( + max(np.median(avg_knn_distances), 100 * np.finfo(np.float64).eps) + ) + scaling_factor = self.params["scaling_factor"] + + if not isinstance(scaling_factor, float): + raise ValueError(f"Scaling factor must be a float. Got {type(scaling_factor)} instead.") + + ood_features_scores = transform_distances_to_scores( + avg_knn_distances, t, scaling_factor=scaling_factor + ) + distance_metric = distance_metric or ( + metric if isinstance((metric := knn.metric), str) else metric.__name__ + ) + p = None + if distance_metric == "minkowski": + p = knn.p + ood_features_scores = correct_precision_errors( + ood_features_scores, avg_knn_distances, distance_metric, p=p + ) + return (ood_features_scores, knn) + + +def _get_ood_predictions_scores( + pred_probs: np.ndarray, + *, + labels: Optional[LabelLike] = None, + confident_thresholds: Optional[np.ndarray] = None, + adjust_pred_probs: bool = True, + method: str = "entropy", + M: int = 100, + gamma: float = 0.1, +) -> Tuple[np.ndarray, Optional[np.ndarray]]: + """Return an OOD (out of distribution) score for each example based on it pred_prob values. + + Parameters + ---------- + pred_probs : np.ndarray + An array of shape ``(N, K)`` of model-predicted probabilities, + `pred_probs` in the same format expected by the `~cleanlab.outlier.OutOfDistribution.fit` function. + + confident_thresholds : np.ndarray, default = None + For details, see key `confident_thresholds` in the params dict arg of `~cleanlab.outlier.OutOfDistribution`. + + labels : array_like, optional + `labels` in the same format expected by the `~cleanlab.outlier.OutOfDistribution.fit` function. + + adjust_pred_probs : bool, True + Account for class imbalance in the label-quality scoring. + For details, see key `adjust_pred_probs` in the params dict arg of `~cleanlab.outlier.OutOfDistribution`. + + method : {"entropy", "least_confidence", "gen"}, default="entropy" + Which method to use for computing outlier scores based on pred_probs. + For details see key `method` in the params dict arg of `~cleanlab.outlier.OutOfDistribution`. + + M : int, default=100 + For GEN method only. Hyperparameter that controls the number of top classes to consider when calculating OOD scores. + + gamma : float, default=0.1 + For GEN method only. Hyperparameter that controls the weight of the second term in the GEN score. + + + Returns + ------- + ood_predictions_scores : Tuple[np.ndarray, Optional[np.ndarray]] + Returns a tuple. First element is array of `ood_predictions_scores` and second is an np.ndarray of `confident_thresholds` or None is 'confident_thresholds' is not calculated. + """ + valid_methods = ( + "entropy", + "least_confidence", + "gen", + ) + + if (confident_thresholds is not None or labels is not None) and not adjust_pred_probs: + warnings.warn( + "OOD scores are not adjusted with confident thresholds. If scores need to be adjusted set " + "params['adjusted_pred_probs'] = True. Otherwise passing in confident_thresholds and/or labels does not change " + "score calculation.", + UserWarning, + ) + + if adjust_pred_probs: + if confident_thresholds is None: + if labels is None: + raise ValueError( + "Cannot calculate adjust_pred_probs without labels. Either pass in labels parameter or set " + "params['adjusted_pred_probs'] = False. " + ) + labels = labels_to_array(labels) + assert_valid_inputs(X=None, y=labels, pred_probs=pred_probs, multi_label=False) + confident_thresholds = get_confident_thresholds(labels, pred_probs, multi_label=False) + + pred_probs = _subtract_confident_thresholds( + None, pred_probs, multi_label=False, confident_thresholds=confident_thresholds + ) + + # Scores are flipped so ood scores are closer to 0. Scores reflect confidence example is in-distribution. + if method == "entropy": + ood_predictions_scores = 1.0 - get_normalized_entropy(pred_probs) + elif method == "least_confidence": + ood_predictions_scores = pred_probs.max(axis=1) + elif method == "gen": + if pred_probs.shape[1] < M: # pragma: no cover + warnings.warn( + f"GEN with the default hyperparameter settings is intended for datasets with at least {M} classes. You can adjust params['M'] according to the number of classes in your dataset.", + UserWarning, + ) + probs = softmax(pred_probs, axis=1) + probs_sorted = np.sort(probs, axis=1)[:, -M:] + ood_predictions_scores = ( + 1 - np.sum(probs_sorted**gamma * (1 - probs_sorted) ** (gamma), axis=1) / M + ) # Use 1 + original gen score/M to make the scores lie in 0-1 + else: + raise ValueError( + f""" + {method} is not a valid OOD scoring method! + Please choose a valid scoring_method: {valid_methods} + """ + ) + + return ( + ood_predictions_scores, + confident_thresholds, + ) diff --git a/cleanlab/rank.py b/cleanlab/rank.py new file mode 100644 index 0000000..ef747fd --- /dev/null +++ b/cleanlab/rank.py @@ -0,0 +1,582 @@ +""" +Methods to rank examples in standard (multi-class) classification datasets by cleanlab's `label quality score`. +Except for `~cleanlab.rank.order_label_issues`, which operates only on the subset of the data identified +as potential label issues/errors, the methods in this module can be used on whichever subset +of the dataset you choose (including the entire dataset) and provide a `label quality score` for +every example. You can then do something like: ``np.argsort(label_quality_score)`` to obtain ranked +indices of individual datapoints based on their quality. + +Note: multi-label classification is not supported by most methods in this module, +each example must be labeled as belonging to a single class, e.g. format: ``labels = np.ndarray([1,0,2,1,1,0...])``. +For multi-label classification, instead see :py:func:`multilabel_classification.get_label_quality_scores `. + +Note: Label quality scores are most accurate when they are computed based on out-of-sample `pred_probs` from your model. +To obtain out-of-sample predicted probabilities for every datapoint in your dataset, you can use :ref:`cross-validation `. This is encouraged to get better results. +""" + +import numpy as np +from sklearn.metrics import log_loss +from typing import List, Optional +import warnings + +from cleanlab.internal.validation import assert_valid_inputs +from cleanlab.internal.constants import ( + CLIPPING_LOWER_BOUND, +) # lower-bound clipping threshold to prevents 0 in logs and division + +from cleanlab.internal.label_quality_utils import ( + _subtract_confident_thresholds, + get_normalized_entropy, +) + + +def get_label_quality_scores( + labels: np.ndarray, + pred_probs: np.ndarray, + *, + method: str = "self_confidence", + adjust_pred_probs: bool = False, +) -> np.ndarray: + """Returns a label quality score for each datapoint. + + This is a function to compute label quality scores for standard (multi-class) classification datasets, + where lower scores indicate labels less likely to be correct. + + Score is between 0 and 1. + + 1 - clean label (given label is likely correct). + 0 - dirty label (given label is likely incorrect). + + Parameters + ---------- + labels : np.ndarray + A discrete vector of noisy labels, i.e. some labels may be erroneous. + *Format requirements*: for dataset with K classes, labels must be in 0, 1, ..., K-1. + Note: multi-label classification is not supported by this method, each example must belong to a single class, e.g. format: ``labels = np.ndarray([1,0,2,1,1,0...])``. + + pred_probs : np.ndarray, optional + An array of shape ``(N, K)`` of model-predicted probabilities, + ``P(label=k|x)``. Each row of this matrix corresponds + to an example `x` and contains the model-predicted probabilities that + `x` belongs to each possible class, for each of the K classes. The + columns must be ordered such that these probabilities correspond to + class 0, 1, ..., K-1. + + **Note**: Returned label issues are most accurate when they are computed based on out-of-sample `pred_probs` from your model. + To obtain out-of-sample predicted probabilities for every datapoint in your dataset, you can use :ref:`cross-validation `. + This is encouraged to get better results. + + method : {"self_confidence", "normalized_margin", "confidence_weighted_entropy"}, default="self_confidence" + Label quality scoring method. + + Letting ``k = labels[i]`` and ``P = pred_probs[i]`` denote the given label and predicted class-probabilities + for datapoint *i*, its score can either be: + + - ``'normalized_margin'``: ``P[k] - max_{k' != k}[ P[k'] ]`` + - ``'self_confidence'``: ``P[k]`` + - ``'confidence_weighted_entropy'``: ``entropy(P) / self_confidence`` + + Note: the actual label quality scores returned by this method + may be transformed versions of the above, in order to ensure + their values lie between 0-1 with lower values indicating more likely mislabeled data. + + Let ``C = {0, 1, ..., K-1}`` be the set of classes specified for our classification task. + + The `normalized_margin` score works better for identifying class conditional label errors, + i.e. examples for which another label in ``C`` is appropriate but the given label is not. + + The `self_confidence` score works better for identifying alternative label issues + corresponding to bad examples that are: not from any of the classes in ``C``, + well-described by 2 or more labels in ``C``, + or generally just out-of-distribution (i.e. anomalous outliers). + + adjust_pred_probs : bool, optional + Account for class imbalance in the label-quality scoring by adjusting predicted probabilities + via subtraction of class confident thresholds and renormalization. + Set this to ``True`` if you prefer to account for class-imbalance. + See `Northcutt et al., 2021 `_. + + Returns + ------- + label_quality_scores : np.ndarray + Contains one score (between 0 and 1) per example. + Lower scores indicate more likely mislabeled examples. + + See Also + -------- + get_self_confidence_for_each_label + get_normalized_margin_for_each_label + get_confidence_weighted_entropy_for_each_label + """ + + assert_valid_inputs( + X=None, y=labels, pred_probs=pred_probs, multi_label=False, allow_one_class=True + ) + return _compute_label_quality_scores( + labels=labels, pred_probs=pred_probs, method=method, adjust_pred_probs=adjust_pred_probs + ) + + +def _compute_label_quality_scores( + labels: np.ndarray, + pred_probs: np.ndarray, + *, + method: str = "self_confidence", + adjust_pred_probs: bool = False, + confident_thresholds: Optional[np.ndarray] = None, +) -> np.ndarray: + """Internal implementation of get_label_quality_scores that assumes inputs + have already been checked and are valid. This speeds things up. + Can also take in pre-computed confident_thresholds to further accelerate things. + """ + scoring_funcs = { + "self_confidence": get_self_confidence_for_each_label, + "normalized_margin": get_normalized_margin_for_each_label, + "confidence_weighted_entropy": get_confidence_weighted_entropy_for_each_label, + } + try: + scoring_func = scoring_funcs[method] + except KeyError: + raise ValueError( + f""" + {method} is not a valid scoring method for rank_by! + Please choose a valid rank_by: self_confidence, normalized_margin, confidence_weighted_entropy + """ + ) + if adjust_pred_probs: + if method == "confidence_weighted_entropy": + raise ValueError(f"adjust_pred_probs is not currently supported for {method}.") + pred_probs = _subtract_confident_thresholds( + labels=labels, pred_probs=pred_probs, confident_thresholds=confident_thresholds + ) + + scoring_inputs = {"labels": labels, "pred_probs": pred_probs} + label_quality_scores = scoring_func(**scoring_inputs) + return label_quality_scores + + +def get_label_quality_ensemble_scores( + labels: np.ndarray, + pred_probs_list: List[np.ndarray], + *, + method: str = "self_confidence", + adjust_pred_probs: bool = False, + weight_ensemble_members_by: str = "accuracy", + custom_weights: Optional[np.ndarray] = None, + log_loss_search_T_values: List[float] = [1e-4, 1e-3, 1e-2, 1e-1, 1e0, 1e1, 1e2, 2e2], + verbose: bool = True, +) -> np.ndarray: + """Returns label quality scores based on predictions from an ensemble of models. + + This is a function to compute label-quality scores for classification datasets, + where lower scores indicate labels less likely to be correct. + + Ensemble scoring requires a list of pred_probs from each model in the ensemble. + + For each pred_probs in list, compute label quality score. + Take the average of the scores with the chosen weighting scheme determined by `weight_ensemble_members_by`. + + Score is between 0 and 1: + + - 1 --- clean label (given label is likely correct). + - 0 --- dirty label (given label is likely incorrect). + + Parameters + ---------- + labels : np.ndarray + Labels in the same format expected by the `~cleanlab.rank.get_label_quality_scores` function. + + pred_probs_list : List[np.ndarray] + Each element in this list should be an array of pred_probs in the same format + expected by the `~cleanlab.rank.get_label_quality_scores` function. + Each element of `pred_probs_list` corresponds to the predictions from one model for all examples. + + method : {"self_confidence", "normalized_margin", "confidence_weighted_entropy"}, default="self_confidence" + Label quality scoring method. See `~cleanlab.rank.get_label_quality_scores` + for scenarios on when to use each method. + + adjust_pred_probs : bool, optional + `adjust_pred_probs` in the same format expected by the `~cleanlab.rank.get_label_quality_scores` function. + + weight_ensemble_members_by : {"uniform", "accuracy", "log_loss_search", "custom"}, default="accuracy" + Weighting scheme used to aggregate scores from each model: + + - "uniform": Take the simple average of scores. + - "accuracy": Take weighted average of scores, weighted by model accuracy. + - "log_loss_search": Take weighted average of scores, weighted by exp(t * -log_loss) where t is selected from log_loss_search_T_values parameter and log_loss is the log-loss between a model's pred_probs and the given labels. + - "custom": Take weighted average of scores using custom weights that the user passes to the custom_weights parameter. + + custom_weights : np.ndarray, default=None + Weights used to aggregate scores from each model if weight_ensemble_members_by="custom". + Length of this array must match the number of models: len(pred_probs_list). + + log_loss_search_T_values : List, default=[1e-4, 1e-3, 1e-2, 1e-1, 1e0, 1e1, 1e2, 2e2] + List of t values considered if weight_ensemble_members_by="log_loss_search". + We will choose the value of t that leads to weights which produce the best log-loss when used to form a weighted average of pred_probs from the models. + + verbose : bool, default=True + Set to ``False`` to suppress all print statements. + + Returns + ------- + label_quality_scores : np.ndarray + Contains one score (between 0 and 1) per example. + Lower scores indicate more likely mislabeled examples. + + See Also + -------- + get_label_quality_scores + """ + + # Check pred_probs_list for errors + assert isinstance( + pred_probs_list, list + ), f"pred_probs_list needs to be a list. Provided pred_probs_list is a {type(pred_probs_list)}" + + assert len(pred_probs_list) > 0, "pred_probs_list is empty." + + if len(pred_probs_list) == 1: + warnings.warn( + """ + pred_probs_list only has one element. + Consider using get_label_quality_scores() if you only have a single array of pred_probs. + """ + ) + + for pred_probs in pred_probs_list: + assert_valid_inputs(X=None, y=labels, pred_probs=pred_probs, multi_label=False) + + # Raise ValueError if user passed custom_weights array but did not choose weight_ensemble_members_by="custom" + if custom_weights is not None and weight_ensemble_members_by != "custom": + raise ValueError( + f""" + custom_weights provided but weight_ensemble_members_by is not "custom"! + """ + ) + + # This weighting scheme performs search of t in log_loss_search_T_values for "best" log loss + if weight_ensemble_members_by == "log_loss_search": + # Initialize variables for log loss search + pred_probs_avg_log_loss_weighted = None + neg_log_loss_weights = None + best_eval_log_loss = float("inf") + + for t in log_loss_search_T_values: + neg_log_loss_list = [] + + # pred_probs for each model + for pred_probs in pred_probs_list: + pred_probs_clipped = np.clip( + pred_probs, a_min=CLIPPING_LOWER_BOUND, a_max=None + ) # lower-bound clipping threshold to prevents 0 in logs when calculating log loss + pred_probs_clipped /= pred_probs_clipped.sum(axis=1)[:, np.newaxis] # renormalize + + neg_log_loss = np.exp(-t * log_loss(labels, pred_probs_clipped)) + neg_log_loss_list.append(neg_log_loss) + + # weights using negative log loss + neg_log_loss_weights_temp = np.array(neg_log_loss_list) / sum(neg_log_loss_list) + + # weighted average using negative log loss + pred_probs_avg_log_loss_weighted_temp = sum( + [neg_log_loss_weights_temp[i] * p for i, p in enumerate(pred_probs_list)] + ) + # evaluate log loss with this weighted average pred_probs + eval_log_loss = log_loss(labels, pred_probs_avg_log_loss_weighted_temp) + + # check if eval_log_loss is the best so far (lower the better) + if best_eval_log_loss > eval_log_loss: + best_eval_log_loss = eval_log_loss + pred_probs_avg_log_loss_weighted = pred_probs_avg_log_loss_weighted_temp + neg_log_loss_weights = neg_log_loss_weights_temp.copy() + + # Generate scores for each model's pred_probs + scores_list = [] + accuracy_list = [] + for pred_probs in pred_probs_list: + # Calculate scores and accuracy + scores = get_label_quality_scores( + labels=labels, + pred_probs=pred_probs, + method=method, + adjust_pred_probs=adjust_pred_probs, + ) + scores_list.append(scores) + + # Only compute if weighting by accuracy + if weight_ensemble_members_by == "accuracy": + accuracy = (pred_probs.argmax(axis=1) == labels).mean() + accuracy_list.append(accuracy) + + if verbose: + print(f"Weighting scheme for ensemble: {weight_ensemble_members_by}") + + # Transform list of scores into an array of shape (N, M) where M is the number of models in the ensemble + scores_ensemble = np.vstack(scores_list).T + + # Aggregate scores with chosen weighting scheme + if weight_ensemble_members_by == "uniform": + label_quality_scores = scores_ensemble.mean(axis=1) # Uniform weights (simple average) + + elif weight_ensemble_members_by == "accuracy": + weights = np.array(accuracy_list) / sum(accuracy_list) # Weight by relative accuracy + if verbose: + print("Ensemble members will be weighted by their relative accuracy") + for i, acc in enumerate(accuracy_list): + print(f" Model {i} accuracy : {acc}") + print(f" Model {i} weight : {weights[i]}") + + # Aggregate scores with weighted average + label_quality_scores = (scores_ensemble * weights).sum(axis=1) + + elif weight_ensemble_members_by == "log_loss_search": + assert neg_log_loss_weights is not None + weights = neg_log_loss_weights # Weight by exp(t * -log_loss) where t is found by searching through log_loss_search_T_values + if verbose: + print( + "Ensemble members will be weighted by log-loss between their predicted probabilities and given labels" + ) + for i, weight in enumerate(weights): + print(f" Model {i} weight : {weight}") + + # Aggregate scores with weighted average + label_quality_scores = (scores_ensemble * weights).sum(axis=1) + + elif weight_ensemble_members_by == "custom": + # Check custom_weights for errors + assert ( + custom_weights is not None + ), "custom_weights is None! Please pass a valid custom_weights." + + assert len(custom_weights) == len( + pred_probs_list + ), "Length of custom_weights array must match the number of models: len(pred_probs_list)." + + # Aggregate scores with custom weights + label_quality_scores = (scores_ensemble * custom_weights).sum(axis=1) + + else: + raise ValueError( + f""" + {weight_ensemble_members_by} is not a valid weighting method for weight_ensemble_members_by! + Please choose a valid weight_ensemble_members_by: uniform, accuracy, custom + """ + ) + + return label_quality_scores + + +def find_top_issues(quality_scores: np.ndarray, *, top: int = 10) -> np.ndarray: + """Returns the sorted indices of the `top` issues in `quality_scores`, ordered from smallest to largest quality score + (i.e., from most to least likely to be an issue). For example, the first value returned is the index corresponding + to the smallest value in `quality_scores` (most likely to be an issue). The second value in the returned array is + the index corresponding to the second smallest value in `quality-scores` (second-most likely to be an issue), and so forth. + + This method assumes that `quality_scores` shares an index with some dataset such that the indices returned by this method + map to the examples in that dataset. + + Parameters + ---------- + quality_scores : + Array of shape ``(N,)``, where N is the number of examples, containing one quality score for each example in the dataset. + + top : + The number of indices to return. + + Returns + ------- + top_issue_indices : + Indices of top examples most likely to suffer from an issue (ranked by issue severity).""" + + if top is None or top > len(quality_scores): + top = len(quality_scores) + + top_outlier_indices = quality_scores.argsort()[:top] + return top_outlier_indices + + +def order_label_issues( + label_issues_mask: np.ndarray, + labels: np.ndarray, + pred_probs: np.ndarray, + *, + rank_by: str = "self_confidence", + rank_by_kwargs: dict = {}, +) -> np.ndarray: + """Sorts label issues by label quality score. + + Default label quality score is "self_confidence". + + Parameters + ---------- + label_issues_mask : np.ndarray + A boolean mask for the entire dataset where ``True`` represents a label + issue and ``False`` represents an example that is accurately labeled with + high confidence. + + labels : np.ndarray + Labels in the same format expected by the `~cleanlab.rank.get_label_quality_scores` function. + + pred_probs : np.ndarray (shape (N, K)) + Predicted-probabilities in the same format expected by the `~cleanlab.rank.get_label_quality_scores` function. + + rank_by : str, optional + Score by which to order label error indices (in increasing order). See + the `method` argument of `~cleanlab.rank.get_label_quality_scores`. + + rank_by_kwargs : dict, optional + Optional keyword arguments to pass into `~cleanlab.rank.get_label_quality_scores` function. + Accepted args include `adjust_pred_probs`. + + Returns + ------- + label_issues_idx : np.ndarray + Return an array of the indices of the examples with label issues, + ordered by the label-quality scoring method passed to `rank_by`. + """ + + allow_one_class = False + if isinstance(labels, np.ndarray) or all(isinstance(lab, int) for lab in labels): + if set(labels) == {0}: # occurs with missing classes in multi-label settings + allow_one_class = True + assert_valid_inputs( + X=None, + y=labels, + pred_probs=pred_probs, + multi_label=False, + allow_one_class=allow_one_class, + ) + + # Convert bool mask to index mask + label_issues_idx = np.arange(len(labels))[label_issues_mask] + + # Calculate label quality scores + label_quality_scores = get_label_quality_scores( + labels, pred_probs, method=rank_by, **rank_by_kwargs + ) + + # Get label quality scores for label issues + label_quality_scores_issues = label_quality_scores[label_issues_mask] + + return label_issues_idx[np.argsort(label_quality_scores_issues)] + + +def get_self_confidence_for_each_label( + labels: np.ndarray, + pred_probs: np.ndarray, +) -> np.ndarray: + """Returns the self-confidence label-quality score for each datapoint. + + This is a function to compute label-quality scores for classification datasets, + where lower scores indicate labels less likely to be correct. + + The self-confidence is the classifier's predicted probability that an example belongs to + its given class label. + + Self-confidence can work better than normalized-margin for detecting label errors due to out-of-distribution (OOD) or weird examples + vs. label errors in which labels for random examples have been replaced by other classes. + + Parameters + ---------- + labels : np.ndarray + Labels in the same format expected by the `~cleanlab.rank.get_label_quality_scores` function. + + pred_probs : np.ndarray + Predicted-probabilities in the same format expected by the `~cleanlab.rank.get_label_quality_scores` function. + + Returns + ------- + label_quality_scores : np.ndarray + Contains one score (between 0 and 1) per example. + Lower scores indicate more likely mislabeled examples. + """ + + # To make this work for multi-label (but it will slow down runtime), return: + # np.array([np.mean(pred_probs[i, l]) for i, l in enumerate(labels)]) + return pred_probs[np.arange(labels.shape[0]), labels] + + +def get_normalized_margin_for_each_label( + labels: np.ndarray, + pred_probs: np.ndarray, +) -> np.ndarray: + """Returns the "normalized margin" label-quality score for each datapoint. + + This is a function to compute label-quality scores for classification datasets, + where lower scores indicate labels less likely to be correct. + + Letting ``k`` denote the given label for a datapoint, the margin is + ``(p(label = k) - max(p(label != k)))``, i.e. the probability + of the given label minus the probability of the argmax label that is not + the given label (``margin = prob_label - max_prob_not_label``). + This gives you an idea of how likely an example is BOTH its given label AND not another label, + and therefore, scores its likelihood of being a good label or a label error. + The normalized margin is simply a transformed version of the margin, + to ensure values between 0-1 with lower values indicating more likely mislabeled data. + + Normalized margin works best for finding class conditional label errors where + there is another label in the set of classes that is clearly better than the given label. + + Parameters + ---------- + labels : np.ndarray + Labels in the same format expected by the `~cleanlab.rank.get_label_quality_scores` function. + + pred_probs : np.ndarray + Predicted-probabilities in the same format expected by the `~cleanlab.rank.get_label_quality_scores` function. + + Returns + ------- + label_quality_scores : np.ndarray + Contains one score (between 0 and 1) per example. + Lower scores indicate more likely mislabeled examples. + """ + + self_confidence = get_self_confidence_for_each_label(labels, pred_probs) + N, K = pred_probs.shape + del_indices = np.arange(N) * K + labels + max_prob_not_label = np.max( + np.delete(pred_probs, del_indices, axis=None).reshape(N, K - 1), axis=-1 + ) + label_quality_scores = (self_confidence - max_prob_not_label + 1) / 2 + return label_quality_scores + + +def get_confidence_weighted_entropy_for_each_label( + labels: np.ndarray, pred_probs: np.ndarray +) -> np.ndarray: + """Returns the "confidence weighted entropy" label-quality score for each datapoint. + + This is a function to compute label-quality scores for classification datasets, + where lower scores indicate labels less likely to be correct. + + "confidence weighted entropy" is defined as the normalized entropy divided by "self-confidence". + The returned values are a transformed version of this score, in order to + ensure values between 0-1 with lower values indicating more likely mislabeled data. + + Parameters + ---------- + labels : np.ndarray + Labels in the same format expected by the `~cleanlab.rank.get_label_quality_scores` function. + + pred_probs : np.ndarray + Predicted-probabilities in the same format expected by the `~cleanlab.rank.get_label_quality_scores` function. + + Returns + ------- + label_quality_scores : np.ndarray + Contains one score (between 0 and 1) per example. + Lower scores indicate more likely mislabeled examples. + """ + + self_confidence = get_self_confidence_for_each_label(labels, pred_probs) + self_confidence = np.clip(self_confidence, a_min=CLIPPING_LOWER_BOUND, a_max=None) + + # Divide entropy by self confidence + label_quality_scores = get_normalized_entropy(pred_probs) / self_confidence + + # Rescale + clipped_scores = np.clip(label_quality_scores, a_min=CLIPPING_LOWER_BOUND, a_max=None) + label_quality_scores = np.log(label_quality_scores + 1) / clipped_scores + + return label_quality_scores diff --git a/cleanlab/regression/__init__.py b/cleanlab/regression/__init__.py new file mode 100644 index 0000000..9928af7 --- /dev/null +++ b/cleanlab/regression/__init__.py @@ -0,0 +1,2 @@ +from . import rank +from . import learn diff --git a/cleanlab/regression/learn.py b/cleanlab/regression/learn.py new file mode 100644 index 0000000..406a775 --- /dev/null +++ b/cleanlab/regression/learn.py @@ -0,0 +1,871 @@ +""" +cleanlab can be used for learning with noisy data for any dataset and regression model. + +For regression tasks, the :py:class:`regression.learn.CleanLearning ` +class wraps any instance of an sklearn model to allow you to train more robust regression models, +or use the model to identify corrupted values in the dataset. +The wrapped model must adhere to the `sklearn estimator API +`_, +meaning it must define three functions: + +* ``model.fit(X, y, sample_weight=None)`` +* ``model.predict(X)`` +* ``model.score(X, y, sample_weight=None)`` + +where ``X`` contains the data (i.e. features, covariates, independant variables) and ``y`` contains the target +value (i.e. label, response/dependant variable). The first index of ``X`` and of ``y`` should correspond to the different +examples in the dataset, such that ``len(X) = len(y) = N`` (sample-size). + +Your model should be correctly clonable via +`sklearn.base.clone `_: +cleanlab internally creates multiple instances of the model, and if you e.g. manually wrap a +PyTorch model, ensure that every call to the estimator's ``__init__()`` creates an independent +instance of the model (for sklearn compatibility, the weights of neural network models should typically +be initialized inside of ``clf.fit()``). + +Example +------- +>>> from cleanlab.regression.learn import CleanLearning +>>> from sklearn.linear_model import LinearRegression +>>> cl = CleanLearning(clf=LinearRegression()) # Pass in any model. +>>> cl.fit(X, y_with_noise) +>>> # Estimate the predictions as if you had trained without label issues. +>>> predictions = cl.predict(y) + +If your model is not sklearn-compatible by default, it might be the case that standard packages can adapt +the model. For example, you can adapt PyTorch models using `skorch `_ +and adapt Keras models using `SciKeras `_. + +If an adapter doesn't already exist, you can manually wrap your +model to be sklearn-compatible. This is made easy by inheriting from +`sklearn.base.BaseEstimator +`_: + +.. code:: python + + from sklearn.base import BaseEstimator + + class YourModel(BaseEstimator): + def __init__(self, ): + pass + def fit(self, X, y): + pass + def predict(self, X): + pass + def score(self, X, y): + pass + +""" + +from typing import Optional, Union, Tuple +import inspect +import warnings + +import math +import numpy as np +import pandas as pd + +import sklearn.base +from sklearn.base import BaseEstimator +from sklearn.model_selection import KFold +from sklearn.linear_model import LinearRegression +from sklearn.metrics import r2_score + +from cleanlab.typing import LabelLike +from cleanlab.internal.constants import TINY_VALUE +from cleanlab.internal.util import train_val_split, subset_X_y +from cleanlab.internal.regression_utils import assert_valid_regression_inputs +from cleanlab.internal.validation import labels_to_array + + +class CleanLearning(BaseEstimator): + """ + CleanLearning = Machine Learning with cleaned data (even when training on messy, error-ridden data). + + Automated and robust learning with noisy labels using any dataset and any regression model. + For regression tasks, this class trains a ``model`` with error-prone, noisy labels + as if the model had been instead trained on a dataset with perfect labels. + It achieves this by estimating which labels are noisy (you might solely use CleanLearning for this estimation) + and then removing examples estimated to have noisy labels, such that a more robust copy of the same model can be + trained on the remaining clean data. + + Parameters + ---------- + model : + Any regression model implementing the `sklearn estimator API `_, + defining the following functions: + + - ``model.fit(X, y)`` + - ``model.predict(X)`` + - ``model.score(X, y)`` + + Default model used is `sklearn.linear_model.LinearRegression + `_. + + cv_n_folds : + This class needs holdout predictions for every data example and if not provided, + uses cross-validation to compute them. This argument sets the number of cross-validation + folds used to compute out-of-sample predictions for each example in ``X``. Default is 5. + Larger values may produce better results, but requires longer to run. + + n_boot : + Number of bootstrap resampling rounds used to estimate the model's epistemic uncertainty. + Default is 5. Larger values are expected to produce better results but require longer runtimes. + Set as 0 to skip estimating the epistemic uncertainty and get results faster. + + include_aleatoric_uncertainty : + Specifies if the aleatoric uncertainty should be estimated during label error detection. + ``True`` by default, which is expected to produce better results but require longer runtimes. + + verbose : + Controls how much output is printed. Set to ``False`` to suppress print statements. Default `False`. + + seed : + Set the default state of the random number generator used to split + the data. By default, uses ``np.random`` current random state. + """ + + def __init__( + self, + model: Optional[BaseEstimator] = None, + *, + cv_n_folds: int = 5, + n_boot: int = 5, + include_aleatoric_uncertainty: bool = True, + verbose: bool = False, + seed: Optional[bool] = None, + ): + if model is None: + # Use linear regression if no model is provided. + model = LinearRegression() + + # Make sure the given regression model has the appropriate methods defined. + if not hasattr(model, "fit"): + raise ValueError("The model must define a .fit() method.") + if not hasattr(model, "predict"): + raise ValueError("The model must define a .predict() method.") + + if seed is not None: + np.random.seed(seed=seed) + + if n_boot < 0: + raise ValueError("n_boot cannot be a negative value") + if cv_n_folds < 2: + raise ValueError("cv_n_folds must be at least 2") + + self.model: BaseEstimator = model + self.seed: Optional[int] = seed + self.cv_n_folds: int = cv_n_folds + self.n_boot: int = n_boot + self.include_aleatoric_uncertainty: bool = include_aleatoric_uncertainty + self.verbose: bool = verbose + self.label_issues_df: Optional[pd.DataFrame] = None + self.label_issues_mask: Optional[np.ndarray] = None + self.k: Optional[float] = None # frac flagged as issue + + def fit( + self, + X: Union[np.ndarray, pd.DataFrame], + y: LabelLike, + *, + label_issues: Optional[Union[pd.DataFrame, np.ndarray]] = None, + sample_weight: Optional[np.ndarray] = None, + find_label_issues_kwargs: Optional[dict] = None, + model_kwargs: Optional[dict] = None, + model_final_kwargs: Optional[dict] = None, + ) -> BaseEstimator: + """ + Train regression ``model`` with error-prone, noisy labels as if the model had been instead trained + on a dataset with the correct labels. ``fit`` achieves this by first training ``model`` via + cross-validation on the noisy data, using the resulting predicted probabilities to identify label issues, + pruning the data with label issues, and finally training ``model`` on the remaining clean data. + + Parameters + ---------- + X : + Data features (i.e. covariates, independent variables), typically an array of shape ``(N, ...)``, + where N is the number of examples (sample-size). + Your ``model`` must be able to ``fit()`` and ``predict()`` data of this format. + + y : + An array of shape ``(N,)`` of noisy labels (i.e. target/response/dependant variable), where some values may be erroneous. + + label_issues : + Optional already-identified label issues in the dataset (if previously estimated). + Specify this to avoid re-estimating the label issues if already done. + If ``pd.DataFrame``, must be formatted as the one returned by: + :py:meth:`self.find_label_issues ` or + :py:meth:`self.get_label_issues `. The DataFrame must + have a column named ``is_label_issue``. + + If ``np.ndarray``, the input must be a boolean mask of length ``N`` where examples that have label issues + have the value ``True``, and the rest of the examples have the value ``False``. + + sample_weight : + Optional array of weights with shape ``(N,)`` that are assigned to individual samples. Specifies how to weight the examples in + the loss function while training. + + find_label_issues_kwargs: + Optional keyword arguments to pass into :py:meth:`self.find_label_issues `. + + model_kwargs : + Optional keyword arguments to pass into model's ``fit()`` method. + + model_final_kwargs : + Optional extra keyword arguments to pass into the final model's ``fit()`` on the cleaned data, + but not the ``fit()`` in each fold of cross-validation on the noisy data. + The final ``fit()`` will also receive the arguments in `clf_kwargs`, but these may be overwritten + by values in `clf_final_kwargs`. This can be useful for training differently in the final ``fit()`` + than during cross-validation. + + Returns + ------- + self : CleanLearning + Fitted estimator that has all the same methods as any sklearn estimator. + + After calling ``self.fit()``, this estimator also stores extra attributes such as: + + - ``self.label_issues_df``: a ``pd.DataFrame`` containing label quality scores, boolean flags + indicating which examples have label issues, and predicted label values for each example. + Accessible via :py:meth:`self.get_label_issues `, + of similar format as the one returned by :py:meth:`self.find_label_issues `. + See documentation of :py:meth:`self.find_label_issues ` + for column descriptions. + - ``self.label_issues_mask``: a ``np.ndarray`` boolean mask indicating if a particular + example has been identified to have issues. + """ + assert_valid_regression_inputs(X, y) + + if find_label_issues_kwargs is None: + find_label_issues_kwargs = {} + if model_kwargs is None: + model_kwargs = {} + if model_final_kwargs is None: + model_final_kwargs = {} + model_final_kwargs = {**model_kwargs, **model_final_kwargs} + + if "sample_weight" in model_kwargs or "sample_weight" in model_final_kwargs: + raise ValueError( + "sample_weight should be provided directly in fit() rather than in model_kwargs or model_final_kwargs" + ) + + if sample_weight is not None: + if "sample_weight" not in inspect.signature(self.model.fit).parameters: + raise ValueError( + "sample_weight must be a supported fit() argument for your model in order to be specified here" + ) + if len(sample_weight) != len(X): + raise ValueError("sample_weight must be a 1D array that has the same length as y.") + + if label_issues is None: + if self.label_issues_df is not None and self.verbose: + print( + "If you already ran self.find_label_issues() and don't want to recompute, you " + "should pass the label_issues in as a parameter to this function next time." + ) + + label_issues = self.find_label_issues( + X, + y, + model_kwargs=model_kwargs, + **find_label_issues_kwargs, + ) + else: + if self.verbose: + print("Using provided label_issues instead of finding label issues.") + if self.label_issues_df is not None: + print( + "These will overwrite self.label_issues_df and will be returned by " + "`self.get_label_issues()`. " + ) + + self.label_issues_df = self._process_label_issues_arg(label_issues, y) + self.label_issues_mask = self.label_issues_df["is_label_issue"].to_numpy() + + X_mask = np.invert(self.label_issues_mask) + X_cleaned, y_cleaned = subset_X_y(X, y, X_mask) + if self.verbose: + print(f"Pruning {np.sum(self.label_issues_mask)} examples with label issues ...") + print(f"Remaining clean data has {len(y_cleaned)} examples.") + + if sample_weight is not None: + model_final_kwargs["sample_weight"] = sample_weight[X_mask] + if self.verbose: + print("Fitting final model on the clean data with custom sample_weight ...") + else: + if self.verbose: + print("Fitting final model on the clean data ...") + + self.model.fit(X_cleaned, y_cleaned, **model_final_kwargs) + + if self.verbose: + print( + "Label issues stored in label_issues_df DataFrame accessible via: self.get_label_issues(). " + "Call self.save_space() to delete this potentially large DataFrame attribute." + ) + return self + + def predict(self, X: np.ndarray, *args, **kwargs) -> np.ndarray: + """ + Predict class labels using your wrapped model. + Works just like ``model.predict()``. + + Parameters + ---------- + X : np.ndarray or DatasetLike + Test data in the same format expected by your wrapped regression model. + + Returns + ------- + predictions : np.ndarray + Predictions for the test examples. + """ + return self.model.predict(X, *args, **kwargs) + + def score( + self, + X: Union[np.ndarray, pd.DataFrame], + y: LabelLike, + sample_weight: Optional[np.ndarray] = None, + ) -> float: + """Evaluates your wrapped regression model's score on a test set `X` with target values `y`. + Uses your model's default scoring function, or r-squared score if your model as no ``"score"`` attribute. + + Parameters + ---------- + X : + Test data in the same format expected by your wrapped model. + + y : + Test labels in the same format as labels previously used in ``fit()``. + + sample_weight : + Optional array of shape ``(N,)`` or ``(N, 1)`` used to weight each test example when computing the score. + + Returns + ------- + score : float + Number quantifying the performance of this regression model on the test data. + """ + if hasattr(self.model, "score"): + if "sample_weight" in inspect.signature(self.model.score).parameters: + return self.model.score(X, y, sample_weight=sample_weight) + else: + return self.model.score(X, y) + else: + return r2_score( + y, + self.model.predict(X), + sample_weight=sample_weight, + ) + + def find_label_issues( + self, + X: Union[np.ndarray, pd.DataFrame], + y: LabelLike, + *, + uncertainty: Optional[Union[np.ndarray, float]] = None, + coarse_search_range: list = [0.01, 0.05, 0.1, 0.15, 0.2], + fine_search_size: int = 3, + save_space: bool = False, + model_kwargs: Optional[dict] = None, + ) -> pd.DataFrame: + """ + Identifies potential label issues (corrupted `y`-values) in the dataset, and estimates how noisy each label is. + + Note: this method estimates the label issues from scratch. To access previously-estimated label issues from + this :py:class:`CleanLearning ` instance, use the + :py:meth:`self.get_label_issues ` method. + + This is the method called to find label issues inside + :py:meth:`CleanLearning.fit() ` + and they share mostly the same parameters. + + Parameters + ---------- + X : + Data features (i.e. covariates, independent variables), typically an array of shape ``(N, ...)``, + where N is the number of examples (sample-size). + Your ``model``, must be able to ``fit()`` and ``predict()`` data of this format. + + y : + An array of shape ``(N,)`` of noisy labels (i.e. target/response/dependant variable), where some values may be erroneous. + + uncertainty : + Optional estimated uncertainty for each example. Should be passed in as a float (constant uncertainty throughout all examples), + or a numpy array of length ``N`` (estimated uncertainty for each example). + If not provided, this method will estimate the uncertainty as the sum of the epistemic and aleatoric uncertainty. + + save_space : + If True, then returned ``label_issues_df`` will not be stored as attribute. + This means some other methods like :py:meth:`self.get_label_issues ` will no longer work. + + coarse_search_range : + The coarse search range to find the value of ``k``, which estimates the fraction of data which have label issues. + More values represent a more thorough search (better expected results but longer runtimes). + + fine_search_size : + Size of fine-grained search grid to find the value of ``k``, which represents our estimate of the fraction of data which have label issues. + A higher number represents a more thorough search (better expected results but longer runtimes). + + + For info about the **other parameters**, see the docstring of :py:meth:`CleanLearning.fit() + `. + + Returns + ------- + label_issues_df : pd.DataFrame + DataFrame with info about label issues for each example. + Unless `save_space` argument is specified, same DataFrame is also stored as `self.label_issues_df` attribute accessible via + :py:meth:`get_label_issues`. + + Each row represents an example from our dataset and the DataFrame may contain the following columns: + + - *is_label_issue*: boolean mask for the entire dataset where ``True`` represents a label issue and ``False`` represents an example + that is accurately labeled with high confidence. + - *label_quality*: Numeric score that measures the quality of each label (how likely it is to be correct, + with lower scores indicating potentially erroneous labels). + - *given_label*: Values originally given for this example (same as `y` input). + - *predicted_label*: Values predicted by the trained model. + """ + + X, y = assert_valid_regression_inputs(X, y) + + if model_kwargs is None: + model_kwargs = {} + + if self.verbose: + print("Identifying label issues ...") + + # compute initial values to find best k + initial_predictions = self._get_cv_predictions(X, y, model_kwargs=model_kwargs) + initial_residual = initial_predictions - y + initial_sorted_index = np.argsort(abs(initial_residual)) + initial_r2 = r2_score(y, initial_predictions) + + self.k, r2 = self._find_best_k( + X=X, + y=y, + sorted_index=initial_sorted_index, + coarse_search_range=coarse_search_range, + fine_search_size=fine_search_size, + ) + + # check if initial r2 score (ie. not removing anything) is the best + if initial_r2 >= r2: + self.k = 0 + + # get predictions using the best k + predictions = self._get_cv_predictions( + X, y, sorted_index=initial_sorted_index, k=self.k, model_kwargs=model_kwargs + ) + residual = predictions - y + + if uncertainty is None: + epistemic_uncertainty = self.get_epistemic_uncertainty(X, y, predictions=predictions) + if self.include_aleatoric_uncertainty: + aleatoric_uncertainty = self.get_aleatoric_uncertainty(X, residual) + else: + aleatoric_uncertainty = 0 + uncertainty = epistemic_uncertainty + aleatoric_uncertainty + else: + if isinstance(uncertainty, np.ndarray) and len(y) != len(uncertainty): + raise ValueError( + "If uncertainty is passed in as an array, it must have the same length as y." + ) + + residual_adjusted = abs(residual / (uncertainty + TINY_VALUE)) + + # adjust lqs by the median (for more human-readable scores) + residual_median = max( + np.median(residual_adjusted), TINY_VALUE + ) # take the max to prevent median = 0 + label_quality_scores = np.exp(-residual_adjusted / residual_median) + + label_issues_mask = np.zeros(len(y), dtype=bool) + num_issues = math.ceil(len(y) * self.k) + issues_index = np.argsort(label_quality_scores)[:num_issues] + label_issues_mask[issues_index] = True + + # convert predictions to int if input is int + if y.dtype == int: + predictions = predictions.astype(int) + + label_issues_df = pd.DataFrame( + { + "is_label_issue": label_issues_mask, + "label_quality": label_quality_scores, + "given_label": y, + "predicted_label": predictions, + } + ) + + if self.verbose: + print(f"Identified {np.sum(label_issues_mask)} examples with label issues.") + + if not save_space: + if self.label_issues_df is not None and self.verbose: + print( + "Overwriting previously identified label issues stored at self.label_issues_df. " + "self.get_label_issues() will now return the newly identified label issues. " + ) + self.label_issues_df = label_issues_df + self.label_issues_mask = label_issues_df["is_label_issue"].to_numpy() + elif self.verbose: + print("Not storing label_issues as attributes since save_space was specified.") + + return label_issues_df + + def get_label_issues(self) -> Optional[pd.DataFrame]: + """ + Accessor, returns `label_issues_df` attribute if previously computed. + This ``pd.DataFrame`` describes the issues identified for each example (each row corresponds to an example). + For column definitions, see the documentation of + :py:meth:`CleanLearning.find_label_issues`. + + Returns + ------- + label_issues_df : pd.DataFrame + DataFrame with (precomputed) info about the label issues for each example. + """ + if self.label_issues_df is None: + warnings.warn( + "Label issues have not yet been computed. Run `self.find_label_issues()` or `self.fit()` first." + ) + return self.label_issues_df + + def get_epistemic_uncertainty( + self, + X: np.ndarray, + y: np.ndarray, + predictions: Optional[np.ndarray] = None, + ) -> np.ndarray: + """ + Compute the epistemic uncertainty of the regression model for each example. This uncertainty is estimated using the bootstrapped + variance of the model predictions. + + Parameters + ---------- + X : + Data features (i.e. training inputs for ML), typically an array of shape ``(N, ...)``, where N is the number of examples. + + y : + An array of shape ``(N,)`` of target values (dependant variables), where some values may be erroneous. + + predictions : + Model predicted values of y, will be used as an extra bootstrap iteration to calculate the variance. + + Returns + _______ + epistemic_uncertainty : np.ndarray + The estimated epistemic uncertainty for each example. + """ + X, y = assert_valid_regression_inputs(X, y) + + if self.n_boot == 0: # does not estimate epistemic uncertainty + return np.zeros(len(y)) + else: + bootstrap_predictions = np.zeros(shape=(len(y), self.n_boot)) + for i in range(self.n_boot): + bootstrap_predictions[:, i] = self._get_cv_predictions(X, y, cv_n_folds=2) + + # add a set of predictions from model that was already trained + if predictions is not None: + _, predictions = assert_valid_regression_inputs(X, predictions) + bootstrap_predictions = np.hstack( + [bootstrap_predictions, predictions.reshape(-1, 1)] + ) + + return np.sqrt(np.var(bootstrap_predictions, axis=1)) + + def get_aleatoric_uncertainty( + self, + X: np.ndarray, + residual: np.ndarray, + ) -> float: + """ + Compute the aleatoric uncertainty of the data. This uncertainty is estimated by predicting the standard deviation + of the regression error. + + Parameters + ---------- + X : + Data features (i.e. training inputs for ML), typically an array of shape ``(N, ...)``, where N is the number of examples. + + residual : + The difference between the given value and the model predicted value of each examples, ie. + `predictions - y`. + + Returns + _______ + aleatoric_uncertainty : float + The overall estimated aleatoric uncertainty for this dataset. + """ + X, residual = assert_valid_regression_inputs(X, residual) + residual_predictions = self._get_cv_predictions(X, residual) + return np.sqrt(np.var(residual_predictions)) + + def save_space(self): + """ + Clears non-sklearn attributes of this estimator to save space (in-place). + This includes the DataFrame attribute that stored label issues which may be large for big datasets. + You may want to call this method before deploying this model (i.e. if you just care about producing predictions). + After calling this method, certain non-prediction-related attributes/functionality will no longer be available + """ + if self.label_issues_df is None and self.verbose: + print("self.label_issues_df is already empty") + + self.label_issues_df = None + self.label_issues_mask = None + self.k = None + + if self.verbose: + print("Deleted non-sklearn attributes such as label_issues_df to save space.") + + def _get_cv_predictions( + self, + X: np.ndarray, + y: np.ndarray, + sorted_index: Optional[np.ndarray] = None, + k: float = 0, + *, + cv_n_folds: Optional[int] = None, + seed: Optional[int] = None, + model_kwargs: Optional[dict] = None, + ) -> np.ndarray: + """ + Helper method to get out-of-fold predictions using cross validation. + This method also allows us to filter out the bottom k percent of label errors before training the cross-validation models + (both ``sorted_index`` and ``k`` has to be provided for this). + + Parameters + ---------- + X : + Data features (i.e. training inputs for ML), typically an array of shape ``(N, ...)``, where N is the number of examples. + + y : + An array of shape ``(N,)`` of target values (dependant variables), where some values may be erroneous. + + sorted_index : + Index of each example sorted by their residuals in ascending order. + + k : + The fraction of examples to hold out from the training sets. Usually this is the fraction of examples that are + deemed to contain errors. + + """ + # set to default unless specified otherwise + if cv_n_folds is None: + cv_n_folds = self.cv_n_folds + + if model_kwargs is None: + model_kwargs = {} + + if k < 0 or k > 1: + raise ValueError("k must be a value between 0 and 1") + elif k == 0: + if sorted_index is None: + sorted_index = np.array(range(len(y))) + in_sample_idx = sorted_index + else: + if sorted_index is None: + # TODO: better error message + raise ValueError( + "You need to pass in the index sorted by prediction quality to use with k" + ) + num_to_drop = math.ceil(len(sorted_index) * k) + in_sample_idx = sorted_index[:-num_to_drop] + out_of_sample_idx = sorted_index[-num_to_drop:] + + X_out_of_sample = X[out_of_sample_idx] + out_of_sample_predictions = np.zeros(shape=[len(out_of_sample_idx), cv_n_folds]) + + if len(in_sample_idx) < cv_n_folds: + raise ValueError( + f"There are too few examples to conduct {cv_n_folds}-fold cross validation. " + "You can either reduce cv_n_folds for cross validation, or decrease k to exclude less data." + ) + + predictions = np.zeros(shape=len(y)) + + kf = KFold(n_splits=cv_n_folds, shuffle=True, random_state=seed) + + for k_split, (cv_train_idx, cv_holdout_idx) in enumerate(kf.split(in_sample_idx)): + try: + model_copy = sklearn.base.clone(self.model) # fresh untrained copy of the model + except Exception: + raise ValueError( + "`model` must be clonable via: sklearn.base.clone(model). " + "You can either implement instance method `model.get_params()` to produce a fresh untrained copy of this model, " + "or you can implement the cross-validation outside of cleanlab " + "and pass in the obtained `pred_probs` to skip cleanlab's internal cross-validation" + ) + + # map the index to the actual index in the original dataset + data_idx_train, data_idx_holdout = ( + in_sample_idx[cv_train_idx], + in_sample_idx[cv_holdout_idx], + ) + + X_train_cv, X_holdout_cv, y_train_cv, y_holdout_cv = train_val_split( + X, y, data_idx_train, data_idx_holdout + ) + + model_copy.fit(X_train_cv, y_train_cv, **model_kwargs) + predictions_cv = model_copy.predict(X_holdout_cv) + + predictions[data_idx_holdout] = predictions_cv + + if k != 0: + out_of_sample_predictions[:, k_split] = model_copy.predict(X_out_of_sample) + + if k != 0: + out_of_sample_predictions_avg = np.mean(out_of_sample_predictions, axis=1) + predictions[out_of_sample_idx] = out_of_sample_predictions_avg + + return predictions + + def _find_best_k( + self, + X: np.ndarray, + y: np.ndarray, + sorted_index: np.ndarray, + coarse_search_range: list = [0.01, 0.05, 0.1, 0.15, 0.2], + fine_search_size: int = 3, + ) -> Tuple[float, float]: + """ + Helper method that conducts a coarse and fine grained grid search to determine the best value + of k, the fraction of the dataset that contains issues. + + Returns a tuple containing the the best value of k (ie. the one that has the best r squared score), + and the corrsponding r squared score obtained when dropping k% of the data. + """ + if len(coarse_search_range) == 0: + raise ValueError("coarse_search_range must have at least 1 value of k") + elif len(coarse_search_range) == 1: + curr_k = coarse_search_range[0] + num_examples_kept = math.floor(len(y) * (1 - curr_k)) + if num_examples_kept < self.cv_n_folds: + raise ValueError( + f"There are too few examples to conduct {self.cv_n_folds}-fold cross validation. " + "You can either reduce self.cv_n_folds for cross validation, or decrease k to exclude less data." + ) + predictions = self._get_cv_predictions( + X=X, + y=y, + sorted_index=sorted_index, + k=curr_k, + ) + best_r2 = r2_score(y, predictions) + best_k = coarse_search_range[0] + else: + # conduct coarse search + coarse_search_range = sorted(coarse_search_range) # sort to conduct fine search well + r2_coarse = np.full(len(coarse_search_range), np.nan) + for i in range(len(coarse_search_range)): + curr_k = coarse_search_range[i] + num_examples_kept = math.floor(len(y) * (1 - curr_k)) + # check if there are too few examples to do cross val + if num_examples_kept < self.cv_n_folds: + r2_coarse[i] = -1e30 # arbitrary large negative number + else: + predictions = self._get_cv_predictions( + X=X, + y=y, + sorted_index=sorted_index, + k=curr_k, + ) + r2_coarse[i] = r2_score(y, predictions) + + max_r2_ind = np.argmax(r2_coarse) + + # conduct fine search + if fine_search_size < 0: + raise ValueError("fine_search_size must at least 0") + elif fine_search_size == 0: + best_k = coarse_search_range[np.argmax(r2_coarse)] + best_r2 = np.max(r2_coarse) + else: + fine_search_range = np.array([]) + if max_r2_ind != 0: + fine_search_range = np.append( + np.linspace( + coarse_search_range[max_r2_ind - 1], + coarse_search_range[max_r2_ind], + fine_search_size + 1, + endpoint=False, + )[1:], + fine_search_range, + ) + if max_r2_ind != len(coarse_search_range) - 1: + fine_search_range = np.append( + fine_search_range, + np.linspace( + coarse_search_range[max_r2_ind], + coarse_search_range[max_r2_ind + 1], + fine_search_size + 1, + endpoint=False, + )[1:], + ) + + r2_fine = np.full(len(fine_search_range), np.nan) + for i in range(len(fine_search_range)): + curr_k = fine_search_range[i] + num_examples_kept = math.floor(len(y) * (1 - curr_k)) + # check if there are too few examples to do cross val + if num_examples_kept < self.cv_n_folds: + r2_fine[i] = -1e30 # arbitrary large negative number + else: + predictions = self._get_cv_predictions( + X=X, + y=y, + sorted_index=sorted_index, + k=curr_k, + ) + r2_fine[i] = r2_score(y, predictions) + + # check the max between coarse and fine search + if max(r2_coarse) > max(r2_fine): + best_k = coarse_search_range[np.argmax(r2_coarse)] + best_r2 = np.max(r2_coarse) + else: + best_k = fine_search_range[np.argmax(r2_fine)] + best_r2 = np.max(r2_fine) + + return best_k, best_r2 + + def _process_label_issues_arg( + self, + label_issues: Union[pd.DataFrame, pd.Series, np.ndarray], + y: LabelLike, + ) -> pd.DataFrame: + """ + Helper method to process the label_issues input into a well-formatted DataFrame. + """ + y = labels_to_array(y) + + if isinstance(label_issues, pd.DataFrame): + if "is_label_issue" not in label_issues.columns: + raise ValueError( + "DataFrame label_issues must contain column: 'is_label_issue'. " + "See CleanLearning.fit() documentation for label_issues column descriptions." + ) + if len(label_issues) != len(y): + raise ValueError("label_issues and labels must have same length") + if "given_label" in label_issues.columns and np.any( + label_issues["given_label"].to_numpy() != y + ): + raise ValueError("labels must match label_issues['given_label']") + return label_issues + + elif isinstance(label_issues, (pd.Series, np.ndarray)): + if label_issues.dtype is not np.dtype("bool"): + raise ValueError("If label_issues is numpy.array, dtype must be 'bool'.") + if label_issues.shape != y.shape: + raise ValueError("label_issues must have same shape as labels") + return pd.DataFrame({"is_label_issue": label_issues, "given_label": y}) + + else: + raise ValueError( + "label_issues must be either pandas.DataFrame, pandas.Series or numpy.ndarray" + ) diff --git a/cleanlab/regression/rank.py b/cleanlab/regression/rank.py new file mode 100644 index 0000000..2b73d27 --- /dev/null +++ b/cleanlab/regression/rank.py @@ -0,0 +1,173 @@ +""" +Methods to score the quality of each label in a regression dataset. These can be used to rank the examples whose Y-value is most likely erroneous. + +Note: Label quality scores are most accurate when they are computed based on out-of-sample `predictions` from your regression model. +To obtain out-of-sample predictions for every datapoint in your dataset, you can use :ref:`cross-validation `. This is encouraged to get better results. + +If you have a sklearn-compatible regression model, consider using `cleanlab.regression.learn.CleanLearning` instead, which can more accurately identify noisy label values. +""" + +from typing import Dict, Callable, Optional, Union +import numpy as np +from numpy.typing import ArrayLike + +from cleanlab.internal.neighbor.metric import decide_euclidean_metric +from cleanlab.internal.neighbor.knn_graph import features_to_knn +from cleanlab.outlier import OutOfDistribution +from cleanlab.internal.regression_utils import assert_valid_prediction_inputs + +from cleanlab.internal.constants import TINY_VALUE + + +def get_label_quality_scores( + labels: ArrayLike, + predictions: ArrayLike, + *, + method: str = "outre", +) -> np.ndarray: + """ + Returns label quality score for each example in the regression dataset. + + Each score is a continous value in the range [0,1] + + * 1 - clean label (given label is likely correct). + * 0 - dirty label (given label is likely incorrect). + + Parameters + ---------- + labels : array_like + Raw labels from original dataset. + 1D array of shape ``(N, )`` containing the given labels for each example (aka. Y-value, response/target/dependent variable), where N is number of examples in the dataset. + + predictions : np.ndarray + 1D array of shape ``(N,)`` containing the predicted label for each example in the dataset. These should be out-of-sample predictions from a trained regression model, which you can obtain for every example in your dataset via :ref:`cross-validation `. + + method : {"residual", "outre"}, default="outre" + String specifying which method to use for scoring the quality of each label and identifying which labels appear most noisy. + + Returns + ------- + label_quality_scores: + Array of shape ``(N, )`` of scores between 0 and 1, one per example in the dataset. + + Lower scores indicate examples more likely to contain a label issue. + + Examples + -------- + >>> import numpy as np + >>> from cleanlab.regression.rank import get_label_quality_scores + >>> labels = np.array([1,2,3,4]) + >>> predictions = np.array([2,2,5,4.1]) + >>> label_quality_scores = get_label_quality_scores(labels, predictions) + >>> label_quality_scores + array([0.00323821, 0.33692597, 0.00191686, 0.33692597]) + """ + + # Check if inputs are valid + labels, predictions = assert_valid_prediction_inputs( + labels=labels, predictions=predictions, method=method + ) + + scoring_funcs: Dict[str, Callable[[np.ndarray, np.ndarray], np.ndarray]] = { + "residual": _get_residual_score_for_each_label, + "outre": _get_outre_score_for_each_label, + } + + scoring_func = scoring_funcs.get(method, None) + if not scoring_func: + raise ValueError( + f""" + {method} is not a valid scoring method. + Please choose a valid scoring technique: {scoring_funcs.keys()}. + """ + ) + + # Calculate scores + label_quality_scores = scoring_func(labels, predictions) + return label_quality_scores + + +def _get_residual_score_for_each_label( + labels: np.ndarray, + predictions: np.ndarray, +) -> np.ndarray: + """Returns a residual label-quality score for each example. + + This is function to compute label-quality scores for regression datasets, + where lower score indicate labels less likely to be correct. + + Residual based scores can work better for datasets where independent variables + are based out of normal distribution. + + Parameters + ---------- + labels: np.ndarray + Labels in the same format expected by the `~cleanlab.regression.rank.get_label_quality_scores` function. + + predictions: np.ndarray + Predicted labels in the same format expected by the `~cleanlab.regression.rank.get_label_quality_scores` function. + + Returns + ------- + label_quality_scores: np.ndarray + Contains one score (between 0 and 1) per example. + Lower scores indicate more likely mislabled examples. + + """ + residual = predictions - labels + label_quality_scores = np.exp(-abs(residual)) + return label_quality_scores + + +def _get_outre_score_for_each_label( + labels: np.ndarray, + predictions: np.ndarray, + *, + residual_scale: float = 5, + frac_neighbors: float = 0.5, + neighbor_metric: Optional[Union[str, Callable]] = None, +) -> np.ndarray: + """Returns OUTRE based label-quality scores. + + This function computes label-quality scores for regression datasets, + where a lower score indicates labels that are less likely to be correct. + + Parameters + ---------- + labels: np.ndarray + Labels in the same format as expected by the `~cleanlab.regression.rank.get_label_quality_scores` function. + + predictions: np.ndarray + Predicted labels in the same format as expected by the `~cleanlab.regression.rank.get_label_quality_scores` function. + + residual_scale: float, default = 5 + Multiplicative factor to adjust scale (standard deviation) of the residuals relative to the labels. + + frac_neighbors: float, default = 0.5 + Fraction of examples in dataset that should be considered as `n_neighbors` in the ``NearestNeighbors`` object used internally to assess outliers. + + neighbor_metric: Optional[str or callable], default = None + The parameter is passed to sklearn NearestNeighbors. # TODO add reference to sklearn.NearestNeighbor? + If None, the metric is chosen based on the number of features in the dataset. + + Returns + ------- + label_quality_scores: np.ndarray + Contains one score (between 0 and 1) per example. + Lower scores indicate more likely mislabled examples. + """ + residual = predictions - labels + labels = (labels - labels.mean()) / (labels.std() + TINY_VALUE) + residual = residual_scale * ((residual - residual.mean()) / (residual.std() + TINY_VALUE)) + + # 2D features by combining labels and residual + features = np.array([labels, residual]).T + + neighbors = int(np.ceil(frac_neighbors * labels.shape[0])) + # Use provided metric or select a decent implementation of the euclidean metric for knn search + neighbor_metric = neighbor_metric or decide_euclidean_metric(features) + knn = features_to_knn(features, n_neighbors=neighbors, metric=neighbor_metric) + ood = OutOfDistribution(params={"knn": knn}) + + label_quality_scores = ood.score(features=features) + return label_quality_scores diff --git a/cleanlab/segmentation/__init__.py b/cleanlab/segmentation/__init__.py new file mode 100644 index 0000000..fbc2eb7 --- /dev/null +++ b/cleanlab/segmentation/__init__.py @@ -0,0 +1,3 @@ +from . import rank +from . import filter +from . import summary diff --git a/cleanlab/segmentation/filter.py b/cleanlab/segmentation/filter.py new file mode 100644 index 0000000..5d97e7e --- /dev/null +++ b/cleanlab/segmentation/filter.py @@ -0,0 +1,218 @@ +""" +Methods to find label issues in image semantic segmentation datasets, where each pixel in an image receives its own class label. + +""" + +from typing import Optional, Tuple + +import numpy as np + +from cleanlab.experimental.label_issues_batched import LabelInspector +from cleanlab.internal.segmentation_utils import _check_input, _get_valid_optional_params + + +def find_label_issues( + labels: np.ndarray, + pred_probs: np.ndarray, + *, + batch_size: Optional[int] = None, + n_jobs: Optional[int] = None, + verbose: bool = True, + **kwargs, +) -> np.ndarray: + """ + Returns a boolean mask for the entire dataset, per pixel where ``True`` represents + an example identified with a label issue and ``False`` represents an example of a pixel correctly labeled. + + * N - Number of images in the dataset + * K - Number of classes in the dataset + * H - Height of each image + * W - Width of each image + + Tip + --- + If you encounter the error "pred_probs is not defined", try setting ``n_jobs=1``. + + Parameters + ---------- + labels: + A discrete array of shape ``(N,H,W,)`` of noisy labels for a semantic segmentation dataset, i.e. some labels may be erroneous. + + *Format requirements*: For a dataset with K classes, each pixel must be labeled using an integer in 0, 1, ..., K-1. + + Tip + --- + If your labels are one hot encoded you can do: ``labels = np.argmax(labels_one_hot, axis=1)`` assuming that `labels_one_hot` is of dimension ``(N,K,H,W)``, in order to get properly formatted `labels`. + + pred_probs: + An array of shape ``(N,K,H,W,)`` of model-predicted class probabilities, + ``P(label=k|x)`` for each pixel ``x``. The prediction for each pixel is an array corresponding to the estimated likelihood that this pixel belongs to each of the ``K`` classes. The 2nd dimension of `pred_probs` must be ordered such that these probabilities correspond to class 0, 1, ..., K-1. + + batch_size: + Optional size of image mini-batches used for computing the label issues in a streaming fashion (does not affect results, just the runtime and memory requirements). + To maximize efficiency, try to use the largest `batch_size` your memory allows. If not provided, a good default is used. + + n_jobs: + Optional number of processes for multiprocessing (default value = 1). Only used on Linux. + If `n_jobs=None`, will use either the number of: physical cores if psutil is installed, or logical cores otherwise. + + verbose: + Set to ``False`` to suppress all print statements. + + **kwargs: + * downsample: int, + Optional factor to shrink labels and pred_probs by. Default ``1`` + Must be a factor divisible by both the labels and the pred_probs. Larger values of `downsample` produce faster runtimes but potentially less accurate results due to over-compression. Set to 1 to avoid any downsampling. + + Returns + ------- + label_issues: np.ndarray + Returns a boolean **mask** for the entire dataset of length `(N,H,W)` + where ``True`` represents a pixel label issue and ``False`` represents an example that is correctly labeled. + """ + batch_size, n_jobs = _get_valid_optional_params(batch_size, n_jobs) + downsample = kwargs.get("downsample", 1) + + def downsample_arrays( + labels: np.ndarray, pred_probs: np.ndarray, factor: int = 1 + ) -> Tuple[np.ndarray, np.ndarray]: + if factor == 1: + return labels, pred_probs + + num_image, num_classes, h, w = pred_probs.shape + + # Check if possible to downsample + if h % downsample != 0 or w % downsample != 0: + raise ValueError( + f"Height {h} and width {w} not divisible by downsample value of {downsample}. Set kwarg downsample to 1 to avoid downsampling." + ) + small_labels = np.round( + labels.reshape((num_image, h // factor, factor, w // factor, factor)).mean((4, 2)) + ) + small_pred_probs = pred_probs.reshape( + (num_image, num_classes, h // factor, factor, w // factor, factor) + ).mean((5, 3)) + + # We want to make sure that pred_probs are renormalized + row_sums = small_pred_probs.sum(axis=1) + renorm_small_pred_probs = small_pred_probs / np.expand_dims(row_sums, 1) + + return small_labels, renorm_small_pred_probs + + def flatten_and_preprocess_masks( + labels: np.ndarray, pred_probs: np.ndarray + ) -> Tuple[np.ndarray, np.ndarray]: + _, num_classes, _, _ = pred_probs.shape + labels_flat = labels.flatten().astype(int) + pred_probs_flat = np.moveaxis(pred_probs, 0, 1).reshape(num_classes, -1) + + return labels_flat, pred_probs_flat.T + + ## + _check_input(labels, pred_probs) + + # Added Downsampling + pre_labels, pre_pred_probs = downsample_arrays(labels, pred_probs, downsample) + + num_image, _, h, w = pre_pred_probs.shape + + ### This section is a modified version of find_label_issues_batched(), old code is commented out + # ranked_label_issues = find_label_issues_batched( + # pre_labels, pre_pred_probs, batch_size=batch_size, n_jobs=n_jobs, verbose=verbose + # ) + lab = LabelInspector( + num_class=pre_pred_probs.shape[1], + verbose=verbose, + n_jobs=n_jobs, + quality_score_kwargs=None, + num_issue_kwargs=None, + ) + n = len(pre_labels) + + if verbose: + from tqdm.auto import tqdm + + pbar = tqdm(desc="number of examples processed for estimating thresholds", total=n) + + # Precompute the size of each image in the batch + image_size = np.prod(pre_pred_probs.shape[1:]) + images_per_batch = max(batch_size // image_size, 1) + + for start_index in range(0, n, images_per_batch): + end_index = min(start_index + images_per_batch, n) + labels_batch, pred_probs_batch = flatten_and_preprocess_masks( + pre_labels[start_index:end_index], pre_pred_probs[start_index:end_index] + ) + lab.update_confident_thresholds(labels_batch, pred_probs_batch) + if verbose: + pbar.update(end_index - start_index) + + if verbose: + pbar.close() + pbar = tqdm(desc="number of examples processed for checking labels", total=n) + + for start_index in range(0, n, images_per_batch): + end_index = min(start_index + images_per_batch, n) + labels_batch, pred_probs_batch = flatten_and_preprocess_masks( + pre_labels[start_index:end_index], pre_pred_probs[start_index:end_index] + ) + _ = lab.score_label_quality(labels_batch, pred_probs_batch) + if verbose: + pbar.update(end_index - start_index) + + if verbose: + pbar.close() + + ranked_label_issues = lab.get_label_issues() + ### End find_label_issues_batched() section + + # Upsample carefully maintaining indicies + label_issues = np.full((num_image, h, w), False) + + # only want to call it an error if pred_probs doesnt match the label at those pixels + for i in range(0, ranked_label_issues.shape[0], batch_size): + issues_batch = ranked_label_issues[i : i + batch_size] + # Finding the right indicies + image_batch, batch_coor_i, batch_coor_j = _get_indexes_from_ranked_issues( + issues_batch, h, w + ) + label_issues[image_batch, batch_coor_i, batch_coor_j] = True + if downsample == 1: + # check if pred_probs matches the label at those pixels + pred_argmax = np.argmax(pred_probs[image_batch, :, batch_coor_i, batch_coor_j], axis=1) + mask = pred_argmax == labels[image_batch, batch_coor_i, batch_coor_j] + label_issues[image_batch[mask], batch_coor_i[mask], batch_coor_j[mask]] = False + + if downsample != 1: + label_issues = label_issues.repeat(downsample, axis=1).repeat(downsample, axis=2) + + for i in range(0, ranked_label_issues.shape[0], batch_size): + issues_batch = ranked_label_issues[i : i + batch_size] + image_batch, batch_coor_i, batch_coor_j = _get_indexes_from_ranked_issues( + issues_batch, h, w + ) + # Upsample the coordinates + upsampled_ii = batch_coor_i * downsample + upsampled_jj = batch_coor_j * downsample + # Iterate over the upsampled region + for i in range(downsample): + for j in range(downsample): + rows = upsampled_ii + i + cols = upsampled_jj + j + pred_argmax = np.argmax(pred_probs[image_batch, :, rows, cols], axis=1) + # Check if the predicted class (argmax) at the identified issue location matches the true label + mask = pred_argmax == labels[image_batch, rows, cols] + # If they match, set the corresponding entries in the label_issues array to False + label_issues[image_batch[mask], rows[mask], cols[mask]] = False + + return label_issues + + +def _get_indexes_from_ranked_issues( + ranked_label_issues: np.ndarray, h: int, w: int +) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + hw = h * w + relative_index = ranked_label_issues % hw + pixel_coor_i, pixel_coor_j = np.unravel_index(relative_index, (h, w)) + image_batch = ranked_label_issues // hw + return image_batch, pixel_coor_i, pixel_coor_j diff --git a/cleanlab/segmentation/rank.py b/cleanlab/segmentation/rank.py new file mode 100644 index 0000000..63dc2eb --- /dev/null +++ b/cleanlab/segmentation/rank.py @@ -0,0 +1,231 @@ +""" +Methods to rank and score images in a semantic segmentation dataset based on how likely they are to contain mislabeled pixels. +""" + +import warnings +from typing import Optional, Tuple + +import numpy as np + +from cleanlab.internal.segmentation_utils import _check_input, _get_valid_optional_params +from cleanlab.segmentation.filter import find_label_issues + + +def get_label_quality_scores( + labels: np.ndarray, + pred_probs: np.ndarray, + *, + method: str = "softmin", + batch_size: Optional[int] = None, + n_jobs: Optional[int] = None, + verbose: bool = True, + **kwargs, +) -> Tuple[np.ndarray, np.ndarray]: + """Returns a label quality score for each image. + + This is a function to compute label quality scores for semantic segmentation datasets, + where lower scores indicate labels less likely to be correct. + + * N - Number of images in the dataset + * K - Number of classes in the dataset + * H - Height of each image + * W - Width of each image + + Parameters + ---------- + labels: + A discrete array of noisy labels for a segmantic segmentation dataset, in the shape ``(N,H,W,)``, + where each pixel must be integer in 0, 1, ..., K-1. + Refer to documentation for this argument in :py:func:`find_label_issues ` for further details. + + pred_probs: + An array of shape ``(N,K,H,W,)`` of model-predicted class probabilities. + Refer to documentation for this argument in :py:func:`find_label_issues ` for further details. + + method: {"softmin", "num_pixel_issues"}, default="softmin" + Label quality scoring method. + + - "softmin" - Calculates the inner product between scores and softmax(1-scores). For efficiency, use instead of "num_pixel_issues". + - "num_pixel_issues" - Uses the number of pixels with label issues for each image using :py:func:`find_label_issues ` + + batch_size : + Optional size of mini-batches to use for estimating the label issues for 'num_pixel_issues' only, not 'softmin'. + To maximize efficiency, try to use the largest `batch_size` your memory allows. If not provided, a good default is used. + + n_jobs: + Optional number of processes for multiprocessing (default value = 1). Only used on Linux. For 'num_pixel_issues' only, not 'softmin' + If `n_jobs=None`, will use either the number of: physical cores if psutil is installed, or logical cores otherwise. + + verbose: + Set to ``False`` to suppress all print statements. + + **kwargs: + * downsample : int, + Factor to shrink labels and pred_probs by for 'num_pixel_issues' only, not 'softmin' . Default ``16`` + Must be a factor divisible by both the labels and the pred_probs. Larger values of `downsample` produce faster runtimes but potentially less accurate results due to over-compression. Set to 1 to avoid any downsampling. + * temperature : float, + Temperature for softmin. Default ``0.1`` + + + Returns + ------- + image_scores: + Array of shape ``(N, )`` of scores between 0 and 1, one per image in the dataset. + Lower scores indicate image more likely to contain a label issue. + pixel_scores: + Array of shape ``(N,H,W)`` of scores between 0 and 1, one per pixel in the dataset. + """ + batch_size, n_jobs = _get_valid_optional_params(batch_size, n_jobs) + _check_input(labels, pred_probs) + + softmin_temperature = kwargs.get("temperature", 0.1) + downsample_num_pixel_issues = kwargs.get("downsample", 1) + + if method == "num_pixel_issues": + _, K, _, _ = pred_probs.shape + labels_expanded = labels[:, np.newaxis, :, :] + mask = np.arange(K)[np.newaxis, :, np.newaxis, np.newaxis] == labels_expanded + # Calculate pixel_scores + masked_pred_probs = np.where(mask, pred_probs, 0) + pixel_scores = masked_pred_probs.sum(axis=1) + scores = find_label_issues( + labels, + pred_probs, + downsample=downsample_num_pixel_issues, + n_jobs=n_jobs, + verbose=verbose, + batch_size=batch_size, + ) + img_scores = 1 - np.mean(scores, axis=(1, 2)) + return (img_scores, pixel_scores) + + if downsample_num_pixel_issues != 1: + warnings.warn( + f"image will not downsample for method {method} is only for method: num_pixel_issues" + ) + + num_im, num_class, h, w = pred_probs.shape + image_scores = np.empty((num_im,)) + pixel_scores = np.empty((num_im, h, w)) + if verbose: + from tqdm.auto import tqdm + + pbar = tqdm(desc=f"images processed using {method}", total=num_im) + + h_array = np.arange(h)[:, None] + w_array = np.arange(w) + + for image in range(num_im): + image_probs = pred_probs[image][ + labels[image], + h_array, + w_array, + ] + pixel_scores[image, :, :] = image_probs + image_scores[image] = _get_label_quality_per_image( + image_probs.flatten(), method=method, temperature=softmin_temperature + ) + if verbose: + pbar.update(1) + return image_scores, pixel_scores + + +def issues_from_scores( + image_scores: np.ndarray, pixel_scores: Optional[np.ndarray] = None, threshold: float = 0.1 +) -> np.ndarray: + """ + Converts scores output by `~cleanlab.segmentation.rank.get_label_quality_scores` + to a list of issues of similar format as output by :py:func:`segmentation.filter.find_label_issues `. + + Only considers as issues those tokens with label quality score lower than `threshold`, + so this parameter determines the number of issues that are returned. + + Note + ---- + - This method is intended for converting the most severely mislabeled examples into a format compatible with ``summary`` methods like :py:func:`segmentation.summary.display_issues `. + - This method does not estimate the number of label errors since the `threshold` is arbitrary, for that instead use :py:func:`segmentation.filter.find_label_issues `, which estimates the label errors via Confident Learning rather than score thresholding. + + Parameters + ---------- + image_scores: + Array of shape `(N, )` of overall image scores, where `N` is the number of images in the dataset. + Same format as the `image_scores` returned by `~cleanlab.segmentation.rank.get_label_quality_scores`. + + pixel_scores: + Optional array of shape ``(N,H,W)`` of scores between 0 and 1, one per pixel in the dataset. + Same format as the `pixel_scores` returned by `~cleanlab.segmentation.rank.get_label_quality_scores`. + + threshold: + Optional quality scores threshold that determines which pixels are included in result. Pixels with with quality scores above the `threshold` are not + included in the result. If not provided, all pixels are included in result. + + Returns + --------- + issues: + Returns a boolean **mask** for the entire dataset + where ``True`` represents a pixel label issue and ``False`` represents an example that is + accurately labeled with using the threshold provided by the user. + Use :py:func:`segmentation.summary.display_issues ` + to view these issues within the original images. + + If `pixel_scores` is not provided, returns array of integer indices (rather than boolean mask) of the images whose label quality score + falls below the `threshold` (sorted by overall label quality score of each image). + + """ + + if image_scores is None: + raise ValueError("pixel_scores must be provided") + if threshold < 0 or threshold > 1 or threshold is None: + raise ValueError("threshold must be between 0 and 1") + + if pixel_scores is not None: + return pixel_scores < threshold + + ranking = np.argsort(image_scores) + cutoff = np.searchsorted(image_scores[ranking], threshold) + return ranking[: cutoff + 1] + + +def _get_label_quality_per_image(pixel_scores, method=None, temperature=0.1): + from cleanlab.internal.multilabel_scorer import softmin + + """ + Input pixel scores and get label quality score for that image, currently using the "softmin" method. + + Parameters + ---------- + pixel_scores: + Per-pixel label quality scores in flattened array of shape ``(N, )``, where N is the number of pixels in the image. + + method: default "softmin" + Method to use to calculate the image's label quality score. + Currently only supports "softmin". + temperature: default 0.1 + Temperature of the softmax function. Too small values may cause numerical underflow and NaN scores. + + Lower values encourage this method to converge toward the label quality score of the pixel with the lowest quality label in the image. + + Higher values encourage this method to converge toward the average label quality score of all pixels in the image. + + Returns + --------- + image_score: + Float of the image's label quality score from 0 to 1, 0 being the lowest quality and 1 being the highest quality. + + """ + if pixel_scores is None or pixel_scores.size == 0: + raise Exception("Invalid Input: pixel_scores cannot be None or an empty list") + + if temperature == 0 or temperature is None: + raise Exception("Invalid Input: temperature cannot be zero or None") + + pixel_scores_64 = pixel_scores.astype("float64") + if method == "softmin": + if len(pixel_scores_64) > 0: + return softmin( + np.expand_dims(pixel_scores_64, axis=0), axis=1, temperature=temperature + )[0] + else: + raise Exception("Invalid Input: pixel_scores is empty") + else: + raise Exception("Invalid Method: Specify correct method. Currently only supports 'softmin'") diff --git a/cleanlab/segmentation/summary.py b/cleanlab/segmentation/summary.py new file mode 100644 index 0000000..7dd0de8 --- /dev/null +++ b/cleanlab/segmentation/summary.py @@ -0,0 +1,356 @@ +""" +Methods to display images and their label issues in a semantic segmentation dataset, as well as summarize the overall types of issues identified. +""" + +from typing import Any, Dict, List, Optional, cast + +import numpy as np +import pandas as pd +from tqdm.auto import tqdm + + +from cleanlab.internal.segmentation_utils import _get_summary_optional_params + + +def display_issues( + issues: np.ndarray, + *, + labels: Optional[np.ndarray] = None, + pred_probs: Optional[np.ndarray] = None, + class_names: Optional[List[str]] = None, + exclude: Optional[List[int]] = None, + top: Optional[int] = None, + **kwargs, # Accepting additional kwargs for plt.show() +) -> None: + """ + Display semantic segmentation label issues, showing images with problematic pixels highlighted. + + Can also show given and predicted masks for each image identified to have label issue. + + Parameters + ---------- + issues: + Boolean **mask** for the entire dataset + where ``True`` represents a pixel label issue and ``False`` represents an example that is + accurately labeled. + + Same format as output by :py:func:`segmentation.filter.find_label_issues ` + or :py:func:`segmentation.rank.issues_from_scores `. + + labels: + Optional discrete array of noisy labels for a segmantic segmentation dataset, in the shape ``(N,H,W,)``, + where each pixel must be integer in 0, 1, ..., K-1. + If `labels` is provided, this function also displays given label of the pixel identified with issue. + Refer to documentation for this argument in :py:func:`find_label_issues ` for more information. + + pred_probs: + Optional array of shape ``(N,K,H,W,)`` of model-predicted class probabilities. + If `pred_probs` is provided, this function also displays predicted label of the pixel identified with issue. + Refer to documentation for this argument in :py:func:`find_label_issues ` for more information. + + Tip + --- + If your labels are one hot encoded you can `np.argmax(labels_one_hot, axis=1)` assuming that `labels_one_hot` is of dimension (N,K,H,W) + before entering in the function + + class_names: + Optional list of strings, where each string represents the name of a class in the semantic segmentation problem. + The order of the names should correspond to the numerical order of the classes. The list length should be + equal to the number of unique classes present in the labels. + If provided, this function will generate a legend + showing the color mapping of each class in the provided colormap. + + Example: + If there are three classes in your labels, represented by 0, 1, 2, then class_names might look like this: + + .. code-block:: python + + class_names = ['background', 'person', 'dog'] + + top: + Optional maximum number of issues to be printed. If not provided, a good default is used. + + exclude: + Optional list of label classes that can be ignored in the errors, each element must be 0, 1, ..., K-1 + + kwargs + Additional keyword arguments to pass to ``plt.show()`` (matplotlib.pyplot.show). + """ + class_names, exclude, top = _get_summary_optional_params(class_names, exclude, top) + if labels is None and len(exclude) > 0: + raise ValueError("Provide labels to allow class exclusion") + + top = min(top, len(issues)) + + correct_ordering = np.argsort(-np.sum(issues, axis=(1, 2)))[:top] + + try: + import matplotlib.pyplot as plt + import matplotlib.patches as mpatches + from matplotlib.axes import Axes + from matplotlib.colors import ListedColormap + except ImportError: + raise ImportError('try "pip install matplotlib"') + + output_plots = (pred_probs is not None) + (labels is not None) + 1 + + # Colormap for errors + error_cmap = ListedColormap(["none", "red"]) + _, h, w = issues.shape + if output_plots > 1: + if pred_probs is not None: + _, num_classes, _, _ = pred_probs.shape + cmap = _generate_colormap(num_classes) + elif labels is not None: + num_classes = max(np.unique(labels)) + 1 + cmap = _generate_colormap(num_classes) + else: + cmap = None + + # Show a legend + if class_names is not None and cmap is not None: + patches = [ + mpatches.Patch(color=cmap[i], label=class_names[i]) for i in range(len(class_names)) + ] + legend = plt.figure() # adjust figsize for larger legend + legend.legend( + handles=patches, loc="center", ncol=len(class_names), facecolor="white", fontsize=20 + ) # adjust fontsize for larger text + plt.axis("off") + plt.show(**kwargs) + + for i in correct_ordering: + # Show images + _, axes = plt.subplots(1, output_plots, figsize=(5 * output_plots, 5)) + plot_index = 0 + + # Handle the different possible types of axes + if output_plots == 1: + axes_list = [cast(Axes, axes)] + else: + axes_list = cast(List[Axes], axes) if isinstance(axes, np.ndarray) else [axes] + + # First image - Given truth labels + if labels is not None and plot_index < len(axes_list): + axes_list[plot_index].imshow(cmap[labels[i]]) + axes_list[plot_index].set_title("Given Labels") + plot_index += 1 + + # Second image - Argmaxed pred_probs + if pred_probs is not None and plot_index < len(axes_list): + axes_list[plot_index].imshow(cmap[np.argmax(pred_probs[i], axis=0)]) + axes_list[plot_index].set_title("Argmaxed Prediction Probabilities") + plot_index += 1 + + # Third image - Errors + if plot_index < len(axes_list): + ax = axes_list[plot_index] + mask = np.full((h, w), True) + if labels is not None and len(exclude) != 0: + mask = ~np.isin(labels[i], exclude) + ax.imshow(issues[i] & mask, cmap=error_cmap, vmin=0, vmax=1) + ax.set_title(f"Image {i}: Suggested Errors (in Red)") + + plt.show(**kwargs) + + return None + + +def common_label_issues( + issues: np.ndarray, + labels: np.ndarray, + pred_probs: np.ndarray, + *, + class_names: Optional[List[str]] = None, + exclude: Optional[List[int]] = None, + top: Optional[int] = None, + verbose: bool = True, +) -> pd.DataFrame: + """ + Display the frequency of which label are swapped in the dataset. + + These may correspond to pixels that are ambiguous or systematically misunderstood by the data annotators. + + * N - Number of images in the dataset + * K - Number of classes in the dataset + * H - Height of each image + * W - Width of each image + + Parameters + ---------- + issues: + Boolean **mask** for the entire dataset + where ``True`` represents a pixel label issue and ``False`` represents an example that is + accurately labeled. + + Same format as output by :py:func:`segmentation.filter.find_label_issues ` + or :py:func:`segmentation.rank.issues_from_scores `. + + labels: + A discrete array of noisy labels for a segmantic segmentation dataset, in the shape ``(N,H,W,)``. + where each pixel must be integer in 0, 1, ..., K-1. + Refer to documentation for this argument in :py:func:`find_label_issues ` for more information. + + pred_probs: + An array of shape ``(N,K,H,W,)`` of model-predicted class probabilities. + Refer to documentation for this argument in :py:func:`find_label_issues ` for more information. + + Tip + --- + If your labels are one hot encoded you can `np.argmax(labels_one_hot, axis=1)` assuming that `labels_one_hot` is of dimension (N,K,H,W) + before entering in the function + + class_names: + Optional length K list of names of each class, such that `class_names[i]` is the string name of the class corresponding to `labels` with value `i`. + If `class_names` is provided, display these string names for predicted and given labels, otherwise display the integer index of classes. + + exclude: + Optional list of label classes that can be ignored in the errors, each element must be in 0, 1, ..., K-1. + + top: + Optional maximum number of tokens to print information for. If not provided, a good default is used. + + verbose: + Set to ``False`` to suppress all print statements. + + Returns + ------- + issues_df: + DataFrame with columns ``['given_label', 'predicted_label', 'num_label_issues']`` + where each row contains information about a particular given/predicted label swap. + Rows are ordered by the number of label issues inferred to exhibit this type of label swap. + """ + try: + N, K, H, W = pred_probs.shape + except: + raise ValueError("pred_probs must be of shape (N, K, H, W)") + + assert labels.shape == (N, H, W), "labels must be of shape (N, H, W)" + + class_names, exclude, top = _get_summary_optional_params(class_names, exclude, top) + # Find issues by pixel coordinates + issue_coords = np.column_stack(np.where(issues)) + + # Count issues per class (given label) + count: Dict[int, Any] = {} + for i, j, k in tqdm(issue_coords): + label = labels[i, j, k] + pred = pred_probs[i, :, j, k].argmax() + if label not in count: + count[label] = np.zeros(K, dtype=int) + if pred not in exclude: + count[label][pred] += 1 + + # Prepare output DataFrame + if class_names is None: + class_names = [str(i) for i in range(K)] + + info = [] + for given_label, class_name in enumerate(class_names): + if given_label in count: + for pred_label, num_issues in enumerate(count[given_label]): + if num_issues > 0: + info.append([class_name, class_names[pred_label], num_issues]) + + info = sorted(info, key=lambda x: x[2], reverse=True)[:top] + issues_df = pd.DataFrame(info, columns=["given_label", "predicted_label", "num_pixel_issues"]) + + if verbose: + for idx, row in issues_df.iterrows(): + print( + f"Class '{row['given_label']}' is potentially mislabeled as class for '{row['predicted_label']}' " + f"{row['num_pixel_issues']} pixels in the dataset" + ) + + return issues_df + + +def filter_by_class( + class_index: int, issues: np.ndarray, labels: np.ndarray, pred_probs: np.ndarray +) -> np.ndarray: + """ + Return label issues involving particular class. Note that this includes errors where the given label is the class of interest, and the predicted label is any other class. + + Parameters + ---------- + class_index: + The specific class you are interested in. + + issues: + Boolean **mask** for the entire dataset where ``True`` represents a pixel label issue and ``False`` represents an example that is + accurately labeled. + + Same format as output by :py:func:`segmentation.filter.find_label_issues ` + or :py:func:`segmentation.rank.issues_from_scores `. + + labels: + A discrete array of noisy labels for a segmantic segmentation dataset, in the shape ``(N,H,W,)``, + where each pixel must be integer in 0, 1, ..., K-1. + Refer to documentation for this argument in :py:func:`find_label_issues ` for further details. + + pred_probs: + An array of shape ``(N,K,H,W,)`` of model-predicted class probabilities. + Refer to documentation for this argument in :py:func:`find_label_issues ` for further details. + + Returns + ---------- + issues_subset: + Boolean **mask** for the subset dataset where ``True`` represents a pixel label issue and ``False`` represents an example that is + accurately labeled for the labeled class. + + Returned mask shows **all** instances that involve the particular class of interest. + + + """ + issues_subset = (issues & np.isin(labels, class_index)) | ( + issues & np.isin(pred_probs.argmax(1), class_index) + ) + return issues_subset + + +def _generate_colormap(num_colors): + """ + Finds a unique color map based on the number of colors inputted ideal for semantic segmentation. + Parameters + ---------- + num_colors: + How many unique colors you want + + Returns + ------- + colors: + colors with num_colors distinct colors + """ + + try: + from matplotlib.cm import hsv + except: + raise ImportError('try "pip install matplotlib"') + + num_shades = 7 + num_colors_with_shades = -(-num_colors // num_shades) * num_shades + linear_nums = np.linspace(0, 1, num_colors_with_shades, endpoint=False) + + arr_by_shade_rows = linear_nums.reshape(num_shades, -1) + arr_by_shade_columns = arr_by_shade_rows.T + num_partitions = arr_by_shade_columns.shape[0] + nums_distributed_like_rising_saw = arr_by_shade_columns.flatten() + + initial_cm = hsv(nums_distributed_like_rising_saw) + lower_partitions_half = num_partitions // 2 + upper_partitions_half = num_partitions - lower_partitions_half + + lower_half = lower_partitions_half * num_shades + initial_cm[:lower_half, :3] *= np.linspace(0.2, 1, lower_half)[:, np.newaxis] + + upper_half_indices = np.arange(lower_half, num_colors_with_shades).reshape( + upper_partitions_half, num_shades + ) + modifier = ( + (1 - initial_cm[upper_half_indices, :3]) + * np.arange(upper_partitions_half)[:, np.newaxis, np.newaxis] + / upper_partitions_half + ) + initial_cm[upper_half_indices, :3] += modifier + colors = initial_cm[:num_colors] + return colors diff --git a/cleanlab/token_classification/__init__.py b/cleanlab/token_classification/__init__.py new file mode 100644 index 0000000..fbc2eb7 --- /dev/null +++ b/cleanlab/token_classification/__init__.py @@ -0,0 +1,3 @@ +from . import rank +from . import filter +from . import summary diff --git a/cleanlab/token_classification/filter.py b/cleanlab/token_classification/filter.py new file mode 100644 index 0000000..75752ab --- /dev/null +++ b/cleanlab/token_classification/filter.py @@ -0,0 +1,101 @@ +""" +Methods to find label issues in token classification datasets (text data), where each token in a sentence receives its own class label. + +The underlying algorithms are described in `this paper `_. +""" + +import numpy as np +from typing import List, Tuple +import warnings + +from cleanlab.filter import find_label_issues as find_label_issues_main +from cleanlab.experimental.label_issues_batched import find_label_issues_batched + + +def find_label_issues( + labels: list, + pred_probs: list, + *, + return_indices_ranked_by: str = "self_confidence", + low_memory: bool = False, + **kwargs, +) -> List[Tuple[int, int]]: + """Identifies tokens with label issues in a token classification dataset. + + Tokens identified with issues will be ranked by their individual label quality score. + + Instead use :py:func:`token_classification.rank.get_label_quality_scores ` + if you prefer to rank the sentences based on their overall label quality. + + Parameters + ---------- + labels: + Nested list of given labels for all tokens, such that `labels[i]` is a list of labels, one for each token in the `i`-th sentence. + + For a dataset with K classes, each class label must be integer in 0, 1, ..., K-1. + + pred_probs: + List of np arrays, such that `pred_probs[i]` has shape ``(T, K)`` if the `i`-th sentence contains T tokens. + + Each row of `pred_probs[i]` corresponds to a token `t` in the `i`-th sentence, + and contains model-predicted probabilities that `t` belongs to each of the K possible classes. + + Columns of each `pred_probs[i]` should be ordered such that the probabilities correspond to class 0, 1, ..., K-1. + + return_indices_ranked_by: {"self_confidence", "normalized_margin", "confidence_weighted_entropy"}, default="self_confidence" + Returned token-indices are sorted by their label quality score. + + See :py:func:`cleanlab.filter.find_label_issues ` + documentation for more details on each label quality scoring method. + + kwargs: + Additional keyword arguments to pass into :py:func:`filter.find_label_issues ` + which is internally applied at the token level. Can include values like `n_jobs` to control parallel processing, `frac_noise`, etc. + + Returns + ------- + issues: + List of label issues identified by cleanlab, such that each element is a tuple ``(i, j)``, which + indicates that the `j`-th token of the `i`-th sentence has a label issue. + + These tuples are ordered in `issues` list based on the likelihood that the corresponding token is mislabeled. + + Use :py:func:`token_classification.summary.display_issues ` + to view these issues within the original sentences. + + Examples + -------- + >>> import numpy as np + >>> from cleanlab.token_classification.filter import find_label_issues + >>> labels = [[0, 0, 1], [0, 1]] + >>> pred_probs = [ + ... np.array([[0.9, 0.1], [0.7, 0.3], [0.05, 0.95]]), + ... np.array([[0.8, 0.2], [0.8, 0.2]]), + ... ] + >>> find_label_issues(labels, pred_probs) + [(1, 1)] + """ + labels_flatten = [l for label in labels for l in label] + pred_probs_flatten = np.array([pred for pred_prob in pred_probs for pred in pred_prob]) + + if low_memory: + for arg_name, _ in kwargs.items(): + warnings.warn(f"`{arg_name}` is not used when `low_memory=True`.") + quality_score_kwargs = {"method": return_indices_ranked_by} + issues_main = find_label_issues_batched( + labels_flatten, pred_probs_flatten, quality_score_kwargs=quality_score_kwargs + ) + else: + issues_main = find_label_issues_main( + labels_flatten, + pred_probs_flatten, + return_indices_ranked_by=return_indices_ranked_by, + **kwargs, + ) + + lengths = [len(label) for label in labels] + mapping = [[(i, j) for j in range(length)] for i, length in enumerate(lengths)] + mapping_flatten = [index for indicies in mapping for index in indicies] + + issues = [mapping_flatten[issue] for issue in issues_main] + return issues diff --git a/cleanlab/token_classification/rank.py b/cleanlab/token_classification/rank.py new file mode 100644 index 0000000..fcf9e32 --- /dev/null +++ b/cleanlab/token_classification/rank.py @@ -0,0 +1,274 @@ +""" +Methods to rank and score sentences in a token classification dataset (text data), based on how likely they are to contain label errors. + +The underlying algorithms are described in `this paper `_. +""" + +import pandas as pd +import numpy as np +from typing import List, Optional, Union, Tuple + +from cleanlab.rank import get_label_quality_scores as main_get_label_quality_scores +from cleanlab.internal.numerics import softmax + + +def get_label_quality_scores( + labels: list, + pred_probs: list, + *, + tokens: Optional[list] = None, + token_score_method: str = "self_confidence", + sentence_score_method: str = "min", + sentence_score_kwargs: dict = {}, +) -> Tuple[np.ndarray, list]: + """ + Returns overall quality scores for the labels in each sentence, as well as for the individual tokens' labels in a token classification dataset. + + Each score is between 0 and 1. + + Lower scores indicate token labels that are less likely to be correct, or sentences that are more likely to contain a mislabeled token. + + Parameters + ---------- + labels: + Nested list of given labels for all tokens, such that `labels[i]` is a list of labels, one for each token in the `i`-th sentence. + + For a dataset with K classes, each label must be in 0, 1, ..., K-1. + + pred_probs: + List of np arrays, such that `pred_probs[i]` has shape ``(T, K)`` if the `i`-th sentence contains T tokens. + + Each row of `pred_probs[i]` corresponds to a token `t` in the `i`-th sentence, + and contains model-predicted probabilities that `t` belongs to each of the K possible classes. + + Columns of each `pred_probs[i]` should be ordered such that the probabilities correspond to class 0, 1, ..., K-1. + + tokens: + Nested list such that `tokens[i]` is a list of tokens (strings/words) that comprise the `i`-th sentence. + + These strings are used to annotated the returned `token_scores` object, see its documentation for more information. + + sentence_score_method: {"min", "softmin"}, default="min" + Method to aggregate individual token label quality scores into a single score for the sentence. + + - `min`: sentence score = minimum of token scores in the sentence + - `softmin`: sentence score = ````, where `s` denotes the token label scores of the sentence, and `` == np.dot(a, b)``. + Here parameter `t` controls the softmax temperature, such that the score converges toward `min` as ``t -> 0``. + Unlike `min`, `softmin` is affected by the scores of all tokens in the sentence. + + token_score_method: {"self_confidence", "normalized_margin", "confidence_weighted_entropy"}, default="self_confidence" + Label quality scoring method for each token. + + See :py:func:`cleanlab.rank.get_label_quality_scores ` documentation for more info. + + sentence_score_kwargs: + Optional keyword arguments for `sentence_score_method` function (for advanced users only). + + See `~cleanlab.token_classification.rank._softmin_sentence_score` for more info about keyword arguments supported for that scoring method. + + Returns + ------- + sentence_scores: + Array of shape ``(N, )`` of scores between 0 and 1, one per sentence in the dataset. + + Lower scores indicate sentences more likely to contain a label issue. + + token_scores: + List of ``pd.Series``, such that `token_info[i]` contains the + label quality scores for individual tokens in the `i`-th sentence. + + If `tokens` strings were provided, they are used as index for each ``Series``. + + Examples + -------- + >>> import numpy as np + >>> from cleanlab.token_classification.rank import get_label_quality_scores + >>> labels = [[0, 0, 1], [0, 1]] + >>> pred_probs = [ + ... np.array([[0.9, 0.1], [0.7, 0.3], [0.05, 0.95]]), + ... np.array([[0.8, 0.2], [0.8, 0.2]]), + ... ] + >>> sentence_scores, token_scores = get_label_quality_scores(labels, pred_probs) + >>> sentence_scores + array([0.7, 0.2]) + >>> token_scores + [0 0.90 + 1 0.70 + 2 0.95 + dtype: float64, 0 0.8 + 1 0.2 + dtype: float64] + """ + methods = ["min", "softmin"] + assert sentence_score_method in methods, "Select from the following methods:\n%s" % "\n".join( + methods + ) + + labels_flatten = np.array([l for label in labels for l in label]) + pred_probs_flatten = np.array([p for pred_prob in pred_probs for p in pred_prob]) + + sentence_length = [len(label) for label in labels] + + def nested_list(x, sentence_length): + i = iter(x) + return [[next(i) for _ in range(length)] for length in sentence_length] + + token_scores = main_get_label_quality_scores( + labels=labels_flatten, pred_probs=pred_probs_flatten, method=token_score_method + ) + scores_nl = nested_list(token_scores, sentence_length) + + if sentence_score_method == "min": + sentence_scores = np.array(list(map(np.min, scores_nl))) + else: + assert sentence_score_method == "softmin" + temperature = sentence_score_kwargs.get("temperature", 0.05) + sentence_scores = _softmin_sentence_score(scores_nl, temperature=temperature) + + if tokens: + token_info = [pd.Series(scores, index=token) for scores, token in zip(scores_nl, tokens)] + else: + token_info = [pd.Series(scores) for scores in scores_nl] + return sentence_scores, token_info + + +def issues_from_scores( + sentence_scores: np.ndarray, *, token_scores: Optional[list] = None, threshold: float = 0.1 +) -> Union[list, np.ndarray]: + """ + Converts scores output by `~cleanlab.token_classification.rank.get_label_quality_scores` + to a list of issues of similar format as output by :py:func:`token_classification.filter.find_label_issues `. + + Issues are sorted by label quality score, from most to least severe. + + Only considers as issues those tokens with label quality score lower than `threshold`, + so this parameter determines the number of issues that are returned. + This method is intended for converting the most severely mislabeled examples to a format compatible with + ``summary`` methods like :py:func:`token_classification.summary.display_issues `. + This method does not estimate the number of label errors since the `threshold` is arbitrary, + for that instead use :py:func:`token_classification.filter.find_label_issues `, + which estimates the label errors via Confident Learning rather than score thresholding. + + Parameters + ---------- + sentence_scores: + Array of shape `(N, )` of overall sentence scores, where `N` is the number of sentences in the dataset. + + Same format as the `sentence_scores` returned by `~cleanlab.token_classification.rank.get_label_quality_scores`. + + token_scores: + Optional list such that `token_scores[i]` contains the individual token scores for the `i`-th sentence. + + Same format as the `token_scores` returned by `~cleanlab.token_classification.rank.get_label_quality_scores`. + + threshold: + Tokens (or sentences, if `token_scores` is not provided) with quality scores above the `threshold` are not + included in the result. + + Returns + --------- + issues: + List of label issues identified by comparing quality scores to threshold, such that each element is a tuple ``(i, j)``, which + indicates that the `j`-th token of the `i`-th sentence has a label issue. + + These tuples are ordered in `issues` list based on the token label quality score. + + Use :py:func:`token_classification.summary.display_issues ` + to view these issues within the original sentences. + + If `token_scores` is not provided, returns array of integer indices (rather than tuples) of the sentences whose label quality score + falls below the `threshold` (also sorted by overall label quality score of each sentence). + + Examples + -------- + >>> import numpy as np + >>> from cleanlab.token_classification.rank import issues_from_scores + >>> sentence_scores = np.array([0.1, 0.3, 0.6, 0.2, 0.05, 0.9, 0.8, 0.0125, 0.5, 0.6]) + >>> issues_from_scores(sentence_scores) + array([7, 4]) + + Changing the score threshold + + >>> issues_from_scores(sentence_scores, threshold=0.5) + array([7, 4, 0, 3, 1]) + + Providing token scores along with sentence scores finds issues at the token level + + >>> token_scores = [ + ... [0.9, 0.6], + ... [0.0, 0.8, 0.8], + ... [0.8, 0.8], + ... [0.1, 0.02, 0.3, 0.4], + ... [0.1, 0.2, 0.03, 0.4], + ... [0.1, 0.2, 0.3, 0.04], + ... [0.1, 0.2, 0.4], + ... [0.3, 0.4], + ... [0.08, 0.2, 0.5, 0.4], + ... [0.1, 0.2, 0.3, 0.4], + ... ] + >>> issues_from_scores(sentence_scores, token_scores=token_scores) + [(1, 0), (3, 1), (4, 2), (5, 3), (8, 0)] + """ + if token_scores: + issues_with_scores = [] + for sentence_index, scores in enumerate(token_scores): + for token_index, score in enumerate(scores): + if score < threshold: + issues_with_scores.append((sentence_index, token_index, score)) + + issues_with_scores = sorted(issues_with_scores, key=lambda x: x[2]) + issues = [(i, j) for i, j, _ in issues_with_scores] + return issues + + else: + ranking = np.argsort(sentence_scores) + cutoff = 0 + while sentence_scores[ranking[cutoff]] < threshold and cutoff < len(ranking): + cutoff += 1 + return ranking[:cutoff] + + +def _softmin_sentence_score( + token_scores: List[np.ndarray], *, temperature: float = 0.05 +) -> np.ndarray: + """ + Sentence overall label quality scoring using the "softmin" method. + + Parameters + ---------- + token_scores: + Per-token label quality scores in nested list format, + where `token_scores[i]` is a list of scores for each toke in the i'th sentence. + + temperature: + Temperature of the softmax function. + + Lower values encourage this method to converge toward the label quality score of the token with the lowest quality label in the sentence. + + Higher values encourage this method to converge toward the average label quality score of all tokens in the sentence. + + Returns + --------- + sentence_scores: + Array of shape ``(N, )``, where N is the number of sentences in the dataset, with one overall label quality score for each sentence. + + Examples + --------- + >>> from cleanlab.token_classification.rank import _softmin_sentence_score + >>> token_scores = [[0.9, 0.6], [0.0, 0.8, 0.8], [0.8]] + >>> _softmin_sentence_score(token_scores) + array([6.00741787e-01, 1.80056239e-07, 8.00000000e-01]) + """ + if temperature == 0: + return np.array([np.min(scores) for scores in token_scores]) + + if temperature == np.inf: + return np.array([np.mean(scores) for scores in token_scores]) + + def fun(scores: np.ndarray) -> float: + return np.dot( + scores, softmax(x=1 - np.array(scores), temperature=temperature, axis=0, shift=True) + ) + + sentence_scores = list(map(fun, token_scores)) + return np.array(sentence_scores) diff --git a/cleanlab/token_classification/summary.py b/cleanlab/token_classification/summary.py new file mode 100644 index 0000000..6994eef --- /dev/null +++ b/cleanlab/token_classification/summary.py @@ -0,0 +1,345 @@ +""" +Methods to display sentences and their label issues in a token classification dataset (text data), as well as summarize the types of issues identified. +""" + +from typing import Any, Dict, List, Optional, Tuple + +import numpy as np +import pandas as pd + +from cleanlab.internal.token_classification_utils import color_sentence, get_sentence + + +def display_issues( + issues: list, + tokens: List[List[str]], + *, + labels: Optional[list] = None, + pred_probs: Optional[list] = None, + exclude: List[Tuple[int, int]] = [], + class_names: Optional[List[str]] = None, + top: int = 20, +) -> None: + """ + Display token classification label issues, showing sentence with problematic token(s) highlighted. + + Can also shows given and predicted label for each token identified to have label issue. + + Parameters + ---------- + issues: + List of tuples ``(i, j)`` representing a label issue for the `j`-th token of the `i`-th sentence. + + Same format as output by :py:func:`token_classification.filter.find_label_issues ` + or :py:func:`token_classification.rank.issues_from_scores `. + + tokens: + Nested list such that `tokens[i]` is a list of tokens (strings/words) that comprise the `i`-th sentence. + + labels: + Optional nested list of given labels for all tokens, such that `labels[i]` is a list of labels, one for each token in the `i`-th sentence. + For a dataset with K classes, each label must be in 0, 1, ..., K-1. + + If `labels` is provided, this function also displays given label of the token identified with issue. + + pred_probs: + Optional list of np arrays, such that `pred_probs[i]` has shape ``(T, K)`` if the `i`-th sentence contains T tokens. + + Each row of `pred_probs[i]` corresponds to a token `t` in the `i`-th sentence, + and contains model-predicted probabilities that `t` belongs to each of the K possible classes. + + Columns of each `pred_probs[i]` should be ordered such that the probabilities correspond to class 0, 1, ..., K-1. + + If `pred_probs` is provided, this function also displays predicted label of the token identified with issue. + + exclude: + Optional list of given/predicted label swaps (tuples) to be ignored. For example, if `exclude=[(0, 1), (1, 0)]`, + tokens whose label was likely swapped between class 0 and 1 are not displayed. Class labels must be in 0, 1, ..., K-1. + + class_names: + Optional length K list of names of each class, such that `class_names[i]` is the string name of the class corresponding to `labels` with value `i`. + + If `class_names` is provided, display these string names for predicted and given labels, otherwise display the integer index of classes. + + top: int, default=20 + Maximum number of issues to be printed. + + Examples + -------- + >>> from cleanlab.token_classification.summary import display_issues + >>> issues = [(2, 0), (0, 1)] + >>> tokens = [ + ... ["A", "?weird", "sentence"], + ... ["A", "valid", "sentence"], + ... ["An", "sentence", "with", "a", "typo"], + ... ] + >>> display_issues(issues, tokens) + Sentence index: 2, Token index: 0 + Token: An + ---- + An sentence with a typo + + + Sentence index: 0, Token index: 1 + Token: ?weird + ---- + A ?weird sentence + """ + if not class_names and (labels or pred_probs): + print( + "Classes will be printed in terms of their integer index since `class_names` was not provided.\n" + "Specify this argument to see the string names of each class.\n" + ) + + top = min(top, len(issues)) + shown = 0 + is_tuple = isinstance(issues[0], tuple) + + for issue in issues: + if is_tuple: + i, j = issue + sentence = get_sentence(tokens[i]) + word = tokens[i][j] + + if pred_probs: + prediction = pred_probs[i][j].argmax() + if labels: + given = labels[i][j] + if pred_probs and labels: + if (given, prediction) in exclude: + continue + + if pred_probs and class_names: + prediction = class_names[prediction] + if labels and class_names: + given = class_names[given] + + shown += 1 + print(f"Sentence index: {i}, Token index: {j}") + print(f"Token: {word}") + if labels and not pred_probs: + print(f"Given label: {given}") + elif not labels and pred_probs: + print(f"Predicted label according to provided pred_probs: {prediction}") + elif labels and pred_probs: + print( + f"Given label: {given}, predicted label according to provided pred_probs: {prediction}" + ) + print("----") + print(color_sentence(sentence, word)) + else: + shown += 1 + sentence = get_sentence(tokens[issue]) + print(f"Sentence issue: {sentence}") + if shown == top: + break + print("\n") + + +def common_label_issues( + issues: List[Tuple[int, int]], + tokens: List[List[str]], + *, + labels: Optional[list] = None, + pred_probs: Optional[list] = None, + class_names: Optional[List[str]] = None, + top: int = 10, + exclude: List[Tuple[int, int]] = [], + verbose: bool = True, +) -> pd.DataFrame: + """ + Display the tokens (words) that most commonly have label issues. + + These may correspond to words that are ambiguous or systematically misunderstood by the data annotators. + + Parameters + ---------- + issues: + List of tuples ``(i, j)`` representing a label issue for the `j`-th token of the `i`-th sentence. + + Same format as output by :py:func:`token_classification.filter.find_label_issues ` + or :py:func:`token_classification.rank.issues_from_scores `. + + tokens: + Nested list such that `tokens[i]` is a list of tokens (strings/words) that comprise the `i`-th sentence. + + labels: + Optional nested list of given labels for all tokens in the same format as `labels` for `~cleanlab.token_classification.summary.display_issues`. + + If `labels` is provided, this function also displays given label of the token identified to commonly suffer from label issues. + + pred_probs: + Optional list of model-predicted probabilities (np arrays) in the same format as `pred_probs` for + `~cleanlab.token_classification.summary.display_issues`. + + If both `labels` and `pred_probs` are provided, also reports each type of given/predicted label swap for tokens identified to commonly suffer from label issues. + + class_names: + Optional length K list of names of each class, such that `class_names[i]` is the string name of the class corresponding to `labels` with value `i`. + + If `class_names` is provided, display these string names for predicted and given labels, otherwise display the integer index of classes. + + top: + Maximum number of tokens to print information for. + + exclude: + Optional list of given/predicted label swaps (tuples) to be ignored in the same format as `exclude` for + `~cleanlab.token_classification.summary.display_issues`. + + verbose: + Whether to also print out the token information in the returned DataFrame `df`. + + Returns + ------- + df: + If both `labels` and `pred_probs` are provided, DataFrame `df` contains columns ``['token', 'given_label', + 'predicted_label', 'num_label_issues']``, and each row contains information for a specific token and + given/predicted label swap, ordered by the number of label issues inferred for this type of label swap. + + Otherwise, `df` only has columns ['token', 'num_label_issues'], and each row contains the information for a specific + token, ordered by the number of total label issues involving this token. + + Examples + -------- + >>> from cleanlab.token_classification.summary import common_label_issues + >>> issues = [(2, 0), (0, 1)] + >>> tokens = [ + ... ["A", "?weird", "sentence"], + ... ["A", "valid", "sentence"], + ... ["An", "sentence", "with", "a", "typo"], + ... ] + >>> df = common_label_issues(issues, tokens) + Token '?weird' is potentially mislabeled 1 times throughout the dataset + + Token 'An' is potentially mislabeled 1 times throughout the dataset + + >>> df + token num_label_issues + 0 An 1 + 1 ?weird 1 + """ + count: Dict[str, Any] = {} + if not labels or not pred_probs: + for issue in issues: + i, j = issue + word = tokens[i][j] + if word not in count: + count[word] = 0 + count[word] += 1 + + words = [word for word in count.keys()] + freq = [count[word] for word in words] + rank = np.argsort(freq)[::-1][:top] + + for r in rank: + print( + f"Token '{words[r]}' is potentially mislabeled {freq[r]} times throughout the dataset\n" + ) + + info = [[word, f] for word, f in zip(words, freq)] + info = sorted(info, key=lambda x: x[1], reverse=True) + return pd.DataFrame(info, columns=["token", "num_label_issues"]) + + if not class_names: + print( + "Classes will be printed in terms of their integer index since `class_names` was not provided. " + ) + print("Specify this argument to see the string names of each class. \n") + + n = pred_probs[0].shape[1] + for issue in issues: + i, j = issue + word = tokens[i][j] + label = labels[i][j] + pred = pred_probs[i][j].argmax() + if word not in count: + count[word] = np.zeros([n, n], dtype=int) + if (label, pred) not in exclude: + count[word][label][pred] += 1 + words = [word for word in count.keys()] + freq = [np.sum(count[word]) for word in words] + rank = np.argsort(freq)[::-1][:top] + + for r in rank: + matrix = count[words[r]] + most_frequent = np.argsort(count[words[r]].flatten())[::-1] + print( + f"Token '{words[r]}' is potentially mislabeled {freq[r]} times throughout the dataset" + ) + if verbose: + print( + "---------------------------------------------------------------------------------------" + ) + for f in most_frequent: + i, j = f // n, f % n + if matrix[i][j] == 0: + break + if class_names: + print( + f"labeled as class `{class_names[i]}` but predicted to actually be class `{class_names[j]}` {matrix[i][j]} times" + ) + else: + print( + f"labeled as class {i} but predicted to actually be class {j} {matrix[i][j]} times" + ) + print() + info = [] + for word in words: + for i in range(n): + for j in range(n): + num = count[word][i][j] + if num > 0: + if not class_names: + info.append([word, i, j, num]) + else: + info.append([word, class_names[i], class_names[j], num]) + info = sorted(info, key=lambda x: x[3], reverse=True) + return pd.DataFrame( + info, columns=["token", "given_label", "predicted_label", "num_label_issues"] + ) + + +def filter_by_token( + token: str, issues: List[Tuple[int, int]], tokens: List[List[str]] +) -> List[Tuple[int, int]]: + """ + Return subset of label issues involving a particular token. + + Parameters + ---------- + token: + A specific token you are interested in. + + issues: + List of tuples ``(i, j)`` representing a label issue for the `j`-th token of the `i`-th sentence. + Same format as output by :py:func:`token_classification.filter.find_label_issues ` + or :py:func:`token_classification.rank.issues_from_scores `. + + tokens: + Nested list such that `tokens[i]` is a list of tokens (strings/words) that comprise the `i`-th sentence. + + Returns + ---------- + issues_subset: + List of tuples ``(i, j)`` representing a label issue for the `j`-th token of the `i`-th sentence, in the same format as `issues`. + But restricting to only those issues that involve the specified `token`. + + Examples + -------- + >>> from cleanlab.token_classification.summary import filter_by_token + >>> token = "?weird" + >>> issues = [(2, 0), (0, 1)] + >>> tokens = [ + ... ["A", "?weird", "sentence"], + ... ["A", "valid", "sentence"], + ... ["An", "sentence", "with", "a", "typo"], + ... ] + >>> filter_by_token(token, issues, tokens) + [(0, 1)] + """ + returned_issues = [] + for issue in issues: + i, j = issue + if token.lower() == tokens[i][j].lower(): + returned_issues.append(issue) + return returned_issues diff --git a/cleanlab/typing.py b/cleanlab/typing.py new file mode 100644 index 0000000..48aabe1 --- /dev/null +++ b/cleanlab/typing.py @@ -0,0 +1,27 @@ +from typing import Any, Callable, Union +import numpy as np +import pandas as pd + +LabelLike = Union[list, np.ndarray, pd.Series, pd.DataFrame] +"""Type for objects that behave like collections of labels.""" + + +DatasetLike = Any +"""Type for objects that behave like datasets.""" + +########################################################### +# Types aliases used in cleanlab/internal/neighbor/ modules +########################################################### + +FeatureArray = np.ndarray +"""A type alias for a 2D numpy array representing numerical features.""" +Metric = Union[str, Callable] +"""A type alias for the distance metric to be used for neighbor search. It can be either a string +representing the metric name ("cosine" or "euclidean") or a callable representing the metric function from scipy (euclidean). + +Valid values for metric are mentioned in the scikit-learn documentation for the sklearn.metrics.pairwise_distances function. + +See Also +-------- +sklearn.metrics.pairwise_distances: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.pairwise_distances.html#sklearn-metrics-pairwise-distances +""" diff --git a/cleanlab/version.py b/cleanlab/version.py new file mode 100644 index 0000000..bffd0f7 --- /dev/null +++ b/cleanlab/version.py @@ -0,0 +1,184 @@ +__version__ = "2.9.0" + +# 2.9.1 - Not yet released, you are using bleeding-edge developer version. See its documentation at: https://docs.cleanlab.ai/master/ + +# ------------------------------------------------ +# | PREVIOUS MAJOR VERSION RELEASE NOTES SUMMARY | +# ------------------------------------------------ + +# 2.9.0 - Streamlined dependencies and improved maintainability +# - Removed TensorFlow/Keras dependencies and cleanlab.models.keras module +# - Updated documentation with modern installation methods + +# 2.8.0 - Python 3.12-3.14 support and compatibility improvements +# - Add support for Python 3.12, 3.13 and 3.14 +# - Fix scikit-learn >= 1.8 compatibility issues +# - Remove numpy version restriction (works with numpy>=1.22) +# - Fix DataLab compatibility with datasets 4.0.0+ +# - Fix null pointer bugs in DataLab issue managers +# - Update Python requirements to 3.10+ + +# 2.7.0 - Broadening Data Quality Checks and ML Workflows +# +# Major new functionalities include: +# - Detection of spurious correlations in image datasets with Datalab +# - New tutorial for improving ML performance with train and test set curation + +# 2.6.0 - Elevating Data Insights: Comprehensive Issue Checks & Expanded ML Task Compatibility +# +# Major new functionalities include: +# - Detection for null values, class imbalance, underperforming groups and data valuation in Datalab +# - Extend Datalab support for tasks like regression and multi-label classification +# - Scores for near duplicates and outliers rescaled to be more interpretable + +# 2.5.0 - cleanlab detects label errors in most ML tasks +# +# Major new functionalities include: +# - Support for: regression, object detection, image segmentation +# - Detection of low-quality images + +# 2.4.0 - One line of code to detect all sorts of dataset issues +# +# Major new functionalities include: +# - Datalab: A unified audit to detect different types of issues in your data and labels. This is the primary way most users should apply cleanlab to their dataset. +# - Nicer APIs for label issues in multi-label classification datasets, including dataset-level issue summaries for multi-label classification. +# - Updated tutorials with more interesting datasets and ML models. + +# 2.3.0 - Extending cleanlab beyond label errors into a complete library for data-centric AI +# +# Major new functionalities include: +# - Active learning with data re-labeling (ActiveLab) +# - KerasWrapperModel and KerasSequentialWrapper to make arbitrary Keras models compatible with scikit-learn +# - Computational improvements for detecting label issues (better efficiency and mini-batch estimation that works with lower memory) + +# 2.2.0 - Re-invented algorithms for multi-label classification and support for datasets with missing classes +# +# For detecting label errors in multi-label classification datasets (e.g. image/document tagging): +# - Added cleanlab.multilabel_classification module for label quality scoring. +# - Re-invented better algorithms and published paper describing how the new algorithms work and benchmarking their effectiveness. +# - Provided new quickstart tutorials and examples on how to use cleanlab for multi-label datasets and train effective models for them. +# +# Additional improvements: +# - cleanlab now works much better for datasets in which some classes happen to not be present. +# - Algorithmic improvements to ensure count.num_label_issues() returns more accurate estimates. +# - For developers: introduction of flake8 code linter and more comprehensive mypy type annotations. + +# 2.1.0 - "Multiannotator, Outlier detection, and Token Classification" - Cleanlab supports several new features +# +# For users (+ sometimes developers): +# - Improved CleanLearning. Added support for pd.DataFrame, tf.keras.dataset, and other types of data besides np.ndarray +# - Added cleanlab.multiannotator module for working with data labeled by multiple annotators. +# - Added cleanlab.token_classification for token classification with text data. +# - Added cleanlab.outlier for out-of-distribution detection (includes outlier/anomaly detection) +# +# For developers: +# - No significant API-breaking changers. +# - Tutorials for all new modules added to https://docs.cleanlab.ai +# - Contributing resources added to https://github.com/cleanlab/cleanlab/blob/master/CONTRIBUTING.md +# - Contributing ideas added to https://github.com/cleanlab/cleanlab/wiki#ideas-for-contributing-to-cleanlab + +# 2.0.0 - "Data-centric AI Ready" - Complete re-architecture of cleanlab API. +# +# For users (+ sometimes developers): +# - All aspects of API have changed (method names, parameters, defaults, variables, classes, etc) +# - Added new dataset module for dealing with dataset-level issues +# - CleanLearning now handles most cleanlab tasks in one line of code. +# - Several new workflows possible with rank, count, and filter modules +# +# For developers: +# - If you're coming from 1.0 (pre-1.0.1), you may need to re-clone. +# - Extensive support available at https://docs.cleanlab.ai + +# 1.0.1 - Launch sphinx docs for Cleanlab 1.0 (in preparation for Cleanlab 2.0). Mostly superficial. +# +# For users (+ sometimes developers): +# - This releases the new sphinx docs for cleanlab 1.0 documentation (in preparation for CL 2.0) +# - Several superficial bug fixes (reduce error printing, fix broken urls, clarify links) +# - Extensive docs/README updates +# - Support was added for Conda Installation +# - Moved to AGPL-3 license +# - Added tutorials and a learning section for Cleanlab +# +# For developers: +# - Moved to GitHub Actions CI +# - Significantly shrunk the clone size to a few MB from 100MB+ + +# 1.0 - cleanlab official 1.0 (beta) release! +# - Added Amazon Reviews NLP to cleanlab/examples +# - cleanlab now supports python 2, 2.7, 3.4, 3.5, 3.6, 3.7, 3.8. +# - Users have used cleanlab with python version 3.9 (use at your own risk!) +# - Added more testing. All tests pass on windows/linux/macOS. +# - Update to GNU GPL-3+ License. +# - Added documentation: https://cleanlab.readthedocs.io/ +# - The cleanlab "confident learning" paper is published in the Journal of AI Research: +# https://jair.org/index.php/jair/article/view/12125 +# - Added funding, community and contributing guidelines +# - Fixed a number of errors in cleanlab/examples +# - cleanlab now supports Windows, macOS, Linux, and unix systems +# - Numerous examples added to the README and docs +# - now natively supports Co-Teaching for learning with noisy labels, req: py3, PyTorch 1.4 +# - cleanlab built in support with handwritten datasets (besides MNIST) +# - cleanlab built in support for CIFAR dataset +# - Multiprocessing fixed for windows systems +# - Adhered all core modules to PEP-8 styling +# - Extensive benchmarking of cleanlab methods published. +# - Future features planned are now supported in cleanlab/version.py +# - Added confidentlearning-reproduce as a separate repo to reproduce state-of-the-art results. + +# 0.1.1 - Major update adding support for Windows and Python 3.7 +# - Added support for Python 3.7 +# - Added full support for Windows, including multiprocessing support in cleanlab/filter.py +# - Improved PEP-8 adherence in core cleanlab/ code. + +# 0.1.0 - Release of confident learning paper: https://arxiv.org/pdf/1911.00068.pdf +# - Documentation increase +# - Add examples to find label errors in mnist, cifar, imagenet +# - re-organized examples and added readme. + +# 0.0.14 - Major bug fix in classification. Unused param broke code. + +# 0.0.13 - Major bug fix in finding label errors. +# - Fixed an important bug that broke finding label errors correctly. +# - Added baseline methods for finding label errors and estimating joint +# - Increased testing +# - Simplified logic + +# 0.0.12 - Minor changes. +# - Added support and testing for sparse matrices scipy.sparse.csr_matrix +# - Dropped integrated dependency and support on fasttext. Use fasttext at your own risk. +# - Added testing and dropping fasttext bumped testing code coverage up to 96%. +# - Remove all ipynb artifacts of the form # In [ ]. + +# 0.0.11 - New logo! Improved README. + +# 0.0.10 - Improved documentation, code formatting, README, and testing coverage. + +# 0.0.9 - Multiple major changes +# - Important: refactored all confident joint methods and parameters +# - Numerous important bug fixes +# - Added multi_label support for labels (list of lists) +# - Added automated ordering of label errors +# - Added automatic calibration of the confident joint +# - Version 0.0.8 is deprecated. Use this version going forward. + +# 0.0.8 - Multiple major changes +# - Finding label errors is now fully parallelized. +# - prune_count_method parameter has been removed. +# - estimate_confident_joint_from_probabilities now automatically calibrates confident joint +# to be a true joint estimate. +# - Confident joint algorithm changed! When an example is found confidently as 2+ labels, choose +# class with max probability. + +# 0.0.7 - Massive speed increases across the board. Estimate joint nearly instantly. NO API changes. + +# 0.0.6 - NO API changes. README updates. Examples added. Tutorials added. + +# 0.0.5 - Numerous small bug fixes, but not major API changes. 100% testing code coverage. + +# 0.0.4 - FIRST CROSS-PLATFORM WORKING VERSION OF CLEANLAB. Adding test support. + +# 0.0.3 - Adding working logo to README, pypi working + +# 0.0.2 - Added logo to README, but link does not load on pypi + +# 0.0.1 - initial commit diff --git a/docs/.nojekyll b/docs/.nojekyll new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/docs/.nojekyll @@ -0,0 +1 @@ + diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..dec0272 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,307 @@ +# Looking for `v1.0` of the `cleanlab` docs? + +Please refer to the [`v1.0.1` documentation](https://docs.cleanlab.ai/v1.0.1/); +the code for `v1.0` is identical to the code for `v1.0.1`. + +# Looking for rendered docs? + +See if you want to browse the documentation (including for past versions). + +# CI/CD for `cleanlab` docs + +In the `cleanlab` repository, we've configured GitHub Actions to perform the following automatically: + +1. When a commit is pushed to the `master` branch, a new version of the `master` docs will be built and deployed to the `cleanlab-docs` repository. + +2. When a release is published, a new version of the docs with the corresponding release tag will be built and deployed as a new folder in the `cleanlab-docs` repository. Redirection to the `stable` version of the docs will be changed to this newly released one, accessible via a link on the docs' site sidebar. All the older versions will remain available in the `cleanlab-docs` repo, accessible by manually entering the subdirectory in the URL. + +3. When a user manually runs the workflow, one of the above will happen depending on the user's selection to run from a `branch` or `tag`. + +If you'd like to build our docs locally or remotely yourself, or want to know more about the steps taken in the GitHub Pages workflow, read on! + + +# Build the `cleanlab` docs **locally** + +1. [Fork](https://docs.github.com/en/get-started/quickstart/fork-a-repo#forking-a-repository) and [clone](https://docs.github.com/en/get-started/quickstart/fork-a-repo#cloning-your-forked-repository) the `cleanlab` repository. + +2. Install the required packages to build the docs: + +``` +uv pip install -r docs/requirements.txt +``` + +Or with pip: + +``` +pip install -r docs/requirements.txt +``` + +For Macs with Apple silicon: replace tensorflow in `docs/requirements.txt` with: `tensorflow-macos==2.9.2` and `tensorflow-metal==0.5.1` + + +3. Install [Pandoc](https://pandoc.org/installing.html). + +4. If you don't already have it, install [wget](https://www.gnu.org/software/wget/). This can be done with `brew` on macOS: `brew install wget` + +5. **[Optional]** [Create a new branch](https://www.atlassian.com/git/tutorials/using-branches), make your code changes, and then `git commit` them. **ONLY COMMITTED CHANGES WILL BE REFLECTED IN THE DOCS BUILD WITH `sphinx-multiversion`.** Instead use `sphinx-build` if you don't want to commit some test changes but still want to see their corresponding docs. + +6. Build the docs with either + 1. [`sphinx-multiversion`](https://sphinx-contrib.github.io/multiversion/): + + * If you're building from a **branch** (usually the `master` branch): + + ``` + sphinx-multiversion docs/source cleanlab-docs -D smv_branch_whitelist=YOUR_BRANCH_NAME -D smv_tag_whitelist=None + ``` + + * If you're building from a **tag** (usually the tag of the stable release): + + ``` + sphinx-multiversion docs/source cleanlab-docs -D smv_branch_whitelist=None -D smv_tag_whitelist=YOUR_TAG_NAME + ``` + + Note: To also build docs for another branch or tag, run the above command again changing only the `YOUR_BRANCH_NAME` or `YOUR_TAG_NAME` placeholder. + + 2. [`sphinx-build`](https://www.sphinx-doc.org/en/master/man/sphinx-build.html): + + * If you want to test out some changes without comitting them, then you can build from your current working directory tree (where you have any un-committed changes locally saved): + + ``` + sphinx-build docs/source cleanlab-docs + ``` + This won't properly produce/display other versions of the docs, but that shouldn't matter if you are just trying to test some local edits to the current version. If some notebooks are giving you trouble (eg. due to runtime or dependencies), you can simply delete those .ipynb files before calling `sphinx-build`. + + **Fast build**: Executing the Jupyter Notebooks (i.e., the `.ipynb` files) that make up some portion of the docs, such as the tutorials, takes a long time. If you want to skip rendering these, set the environment variable `SKIP_NOTEBOOKS=1`. You can either set this using `export SKIP_NOTEBOOKS=1` or do this inline with `SKIP_NOTEBOOKS=1 sphinx-multiversion ...`. + + **Skipping specific notebooks**: If you want to skip rendering a few specific notebooks during your local build, the best way to do this is to temporarily move the files outside the `cleanlab` folder (so `nbsphinx` would not find it), then build the docs, before finally moving the files back (to ensure they will not be deleted when pushed to GitHub) + + Example workflow for skipping notebooks, given our current working directory is the `cleanlab` root folder and we want to ignore a specific notebook: + + 1. create an empty folder outside of cleanlab folder + ``` + mkdir ../ignore_notebooks + ``` + + 2. move the notebook to ignore from local build to the newly created folder + ``` + mv docs/source/tutorials/path/to/notebook.ipynb ../ignore_notebooks + ``` + + 3. build the docs locally, using `sphinx-build` as it does not require you to commit your changes + ``` + sphinx-build docs/source cleanlab-docs + ``` + + 4. move the notebook back to its original location + ``` + mv ../ignore_notebooks/notebook.ipynb docs/source/tutorials/path/to/ + ``` + + + While building the docs with `sphinx-multiversion`, your terminal might output: + * `unknown config value 'smv_branch_whitelist' in override, ignoring`, and + * `unknown config value 'smv_tag_whitelist' in override, ignoring`. + + This is because the `smv_branch_whitelist` and `smv_tag_whitelist` config values are only used by `sphinx-multiversion`, but may also be checked by `sphinx` or other extensions that do not use them. Hence, these can be safely ignored as long as the docs are built correctly. + +7. **[Optional]** To show dynamic versioning and version warning banners: + + * Copy the `docs/_templates/versioning.js` file to the `cleanlab-docs/` directory. + + * In the copied `versioning.js` file: + + * find `placeholder_version_number` and replace it with the latest release tag name, and + + * find `placeholder_commit_hash` and replace it with the `master` branch commit hash. + +8. **[Optional]** To redirect site visits from `/` or `/stable` to the stable version of the docs: + + * Create a copy of the `docs/_templates/redirect-to-stable.html` file and rename it as `index.html`. + + * In this `index.html` file, find `stable_url` and replace it with `/cleanlab-docs/YOUR_LATEST_RELEASE_TAG_NAME/index.html`. + + * Copy this `index.html` to: + + * `cleanlab-docs/`, and + + * `cleanlab-docs/stable/`. + +9. The docs for each branch and/or tag can be found in the `cleanlab-docs/` directory, open any of the `index.html` in your browser to view the docs: + + ``` + cleanlab-docs + | index.html (redirects to stable release of the docs) + | versioning.js (for dynamic versioning and version warning banner) + | + └───YOUR_BRANCH_NAME (e.g. master) + │ index.html + │ ... + │ + └───YOUR_TAG_NAME_1 (e.g. your stable release tag name) + │ index.html + │ ... + │ + └───YOUR_TAG_NAME_2 (e.g. an old release tag name) + │ index.html + │ ... + │ + └───stable + │ index.html (redirects to stable release of the docs) + │ + └───... + ``` + + Note: If you're building the docs from a working directory tree, the docs will be found at the top of the `cleanlab-docs/` directory: + + ``` + cleanlab-docs + | index.html (docs for the working directory tree) + | ... + | + └───... + ``` + + This may overwrite some of the files in `cleanlab-docs/`, like `index.html` from the previous step. + +# Build the `cleanlab` docs **remotely** on GitHub + +1. [Fork](https://docs.github.com/en/get-started/quickstart/fork-a-repo#forking-a-repository) the `cleanlab` repository. + +2. [Create a new repository](https://docs.github.com/en/pages/getting-started-with-github-pages/creating-a-github-pages-site#creating-a-repository-for-your-site) named `cleanlab-docs` and a new branch named `master`. + +3. In the `cleanlab-docs` repo, [configure GitHub Pages](https://docs.github.com/en/pages/getting-started-with-github-pages/creating-a-github-pages-site#creating-a-repository-for-your-site); under the **Source** section, select the `master` branch and `/(root)` folder. Take note of the URL where your site is published. + +4. [Generate SSH deploy key](https://github.com/peaceiris/actions-gh-pages#user-content-️-create-ssh-deploy-key) and add them to your repos as such: + + * In the `cleanlab-docs` repo, go to **Settings > Deploy Keys > Add deploy key** and add your **public key** with the **Allow write access** + * In the `cleanlab` repo, go to **Settings > Secrets > New repository secrets** and add your **private key** named `ACTIONS_DEPLOY_KEY` + +5. In the `cleanlab` repo, check that you have the **GitHub Pages** workflow under the repo's **Actions** tab. This should be created automatically from `.github\workflows\gh-pages.yaml`. This workflow can be activated by any of the 3 triggers below: + + * A push to the `master` branch in the `cleanlab` repo. + * Publish of a new release in the `cleanlab` repo. + * Manually run from the **Run workflow** option and select either the `master` branch or one of the release tag. + +6. Activate the workflow with any of the 3 triggers listed above and wait for it to complete. + +7. Navigate to the URL where your GitHub Pages site is published in step 3. The default URL should have the format *https://repository_owner.github.io/cleanlab-docs/*. + +# Manually adding build artifacts to the `cleanlab-docs` repo + +GitHub Actions automatically builds and deploys the docs' build artifacts when [triggered](#cicd-for-cleanlab-docs). If you delete and recreate a release tag, the docs for this tag will be rebuilt and redeployed, hence overwriting the existing artifacts with the new ones. + +On rare occasions, you may want to update the docs without deleting and recreating the release tag, for example, when you want to fix a typo in the docs, but you've already deployed your tag to PyPI or Conda. This can be done by manually adding specific docs' build artifacts to the `cleanlab/cleanlab-docs` repo. These steps are for users who have `push` permission to `cleanlab/cleanlab` and `cleanlab/cleanlab-docs` repo. + +1. If you haven't already done so, [clone](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository) the `cleanlab/cleanlab` repo. + +2. [Create and checkout a new branch](https://git-scm.com/docs/git-checkout). + +3. Make the necessary code changes. + +4. Perform [git add](https://git-scm.com/docs/git-add) and [git commit](https://git-scm.com/docs/git-commit) for the changes. + +5. [git push](https://git-scm.com/docs/git-push) to the `cleanlab/cleanlab` repo. As this is pushed from a non-`master` branch, GitHub Actions will only build but not deploy the docs' build artifacts. + +6. Navigate to [github.com/cleanlab/cleanlab](https://www.github.com/cleanlab/cleanlab) in your browser, select the "Actions" tab, under "Workflow", click "GitHub Pages", then select the workflow that was triggered by the previous step. + +7. Ensure that the workflow has completed running. + +8. Scroll to the bottom of the page, under "Artifacts", click "docs-html" to download the docs' build artifacts. + +9. Unzip "docs-html.zip" and open the "docs-html" folder. + +10. Identify the files you would like to replace, i.e., the corresponding files creating the pages on [docs.cleanlab.ai](https://docs.cleanlab.ai). + +11. Replace these files in [github.com/cleanlab/cleanlab-docs](https://www.github.com/cleanlab/cleanlab-docs) by uploading the new ones to the corresponding version folder in the `master` branch of the `cleanlab/cleanlab-docs` repo. + +> :warning: Any build artifacts manually added to `cleanlab/cleanlab-docs` that do not live in the `master` branch of the `cleanlab/cleanlab` repo will be lost in future versions of cleanlab docs. So any edit made in the v2.0.0 docs which you also want to have in the v2.0.1, v2.0.2, etc. docs needs to be introduced as a PR to the `cleanlab/cleanlab` repo as well. + +> :warning: Currently, if updating stable/old version (say `vXXX`) of tutorials from latest master branch version, the install of cleanlab package in notebooks/colabs will be wrong. To remedy this, you need to update the cleanlab version in all `.ipynb` files inside folders: **cleanlab-docs/vXXX/tutorials/** and **cleanlab-docs/vXXX/_sources/**. The tutorial `.html` pages will also have wrong colab links as well. Currently have to update the `.html` files in **cleanlab-docs/vXXX/tutorials/** to replace these colab links with the proper links (replace `/master/` in the link with `/vXXX/` for the version you are building docs for). + +# Behind-the-scenes of the GitHub Pages workflow + +We've configured GitHub Actions to run the GitHub Pages workflow (gh-pages.yaml) to build and deploy our docs' static files. Here's a breakdown of what this workflow does in the background: + +## Spin up and configure the CI/CD server + +1. Spin up a Ubuntu server. + +2. Install Pandoc, a document converter required by `nbsphinx` to generate static sites from notebooks (`.ipynb`). + +3. Check-out the `cleanlab` repository. + +4. Setup Python and cache dependencies. + +5. Install dependencies for the docs from `docs/requirements.txt`. + +## Build the docs' static site files + +6. Run Sphinx with the `sphinx-multiversion` wrapper to build the doc's static site files. These files will be outputted to the `cleanlab-docs/` directory. + +## Generate the `versioning.js` file used to store the latest release tag name and commit hash + +7. Get the latest release tag name and insert it in the `versioning.js` file. The `index.html` of each doc version will read this as a variable and display it beside the **stable** hyperlink. + +8. Insert the latest commit hash in the `versioning.js` file. The `index.html` of each doc version will read this as a variable and display it beside the **developer** hyperlink. + +9. Copy the `versioning.js` file to the `cleanlab-docs/` folder. + +## If the workflow is **triggered by a new release**, generate the redirecting HTML which redirects site visits to the stable version + +10. Insert the relative path to the stable docs in the `redirect-to-stable.html` file AKA the *redirecting HTML*. + +11. Create a copy of the `redirect-to-stable.html` file to `cleanlab-docs/index.html` and `cleanlab-docs/index.html`. + +## Deploy the static files + +12. Deploy `cleanlab-docs/` folder to the `cleanlab/cleanlab-docs` repo's `master branch`. + + +# Tips for editing docs/tutorials + +## Tutorials + +Each tutorial is a Jupyter notebook (unexecuted .ipynb file) that will be executed during CI for the version displayed at docs.cleanlab.ai using [nbsphinx](https://github.com/cleanlab/cleanlab/blob/31c939ff9aa487e9670b1a0f3f711a1d78448a91/docs/source/conf.py). Some basic [linting](https://github.com/cleanlab/nblint-action) is also applied to ensure proper notebook formatting such as no trailing newlines at the end of cells. Here are some tips when adding a new tutorial notebook: + +1. Make sure to clear all Cell outputs before you `git commit` a tutorial. The outputs of cells should never be tracked in git, these outputs are automatically constructed for displaying on docs.cleanlab.ai during the CI which executes all notebooks in the folder **docs/source/**. + +2. For cells which contain code that should **not** be executed during CI, make sure the cell-type is Markdown and use proper syntax to make contents look like code. + +3. To suppress certain Jupyter cells that should not be shown on docs.cleanlab.ai web version of tutorial: +``` +"metadata": { + "nbsphinx": "hidden" + } +``` +This includes cells that install dependencies and cells that run tests to verify the notebook has executed correctly. These cells will still be visible when the notebook is run in Colab or locally in Jupyter, so make sure to add a comment explaining their purpose at the top. + +4. If developing Notebook in virtualenv, make sure at the end to change the end of the raw .ipynb file to have the following: +``` +"metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + } +``` +instead of containing your own virtualenv in there. CI will FAIL if you instead list your own virtualenv here! + +5. When adding dependencies to a tutorial: + - Make sure to update **docs/requirements.txt** which lists all extra dependencies installed during CI to build the docs. + - Add a comment in hidden cell not displayed on docs.cleanlab.ai stating which version of dependencies you used. + - Think carefully whether each dependency is really necessary and if its future versions will be stable / compatible with future versions of existing dependencies. + +6. Don't forget to update **docs/source/index.rst** with a short title and **docs/source/tutorials/index.rst** to ensure your tutorial properly linked. Otherwise it will not appear on docs.cleanlab.ai! + +7. Ask yourself: +- How can I make this tutorial run faster without sacrificing educational value? Perhaps use smaller subsample of the dataset, smaller/pretrained model, etc. +- What sections of this tutorial are least vital? Consider creating a separate [Examples](https://github.com/cleanlab/examples) notebook that features those. + +All of our tutorials are quickstart guides that should run quite fast. Longer/comprehensive notebooks are better added in [Examples](https://github.com/cleanlab/examples). + + +## API Documentation + +1. Verify your new docstrings adhere to our [documentation format guidelines](../DEVELOPMENT.md#documentation-style) + +2. To ensure documentation for new source code files is linked from the main page, don't forget to update: **docs/source/index.rst** diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..061f32f --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..8cb06d0 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,31 @@ +jinja2==3.1.3 +sphinx==7.1.2 +sphinx-tabs==3.4.5 +nbconvert==7.13.1 +nbsphinx==0.9.3 +autodocsumm==0.2.12 +furo==2023.09.10 +ipython==8.0.1 +ipykernel==6.29.0 +ipywidgets==8.1.1 +sphinx-multiversion==0.2.4 +sphinx-copybutton==0.5.2 +sphinxcontrib-katex==0.9.9 +sphinxcontrib-gtagjs==0.2.1 +sphinx-jinja==2.0.2 +sphinx-autodoc-typehints==1.25.2 +sphinxext-opengraph==0.9.1 +requests==2.28.2 +sentence-transformers>=2.3.0 +speechbrain==0.5.13 +faiss-cpu==1.8.0 +datasets>=2.14.6 +xgboost +matplotlib==3.7.4 # TODO: uncap version +huggingface_hub==0.26.5 # TODO: uncap version +torch==2.1.2 # TODO: uncap version +torchvision==0.16.2 # TODO: uncap version +torchaudio==2.1.2 # TODO: uncap version +timm==0.9.12 # TODO: uncap version +seaborn==0.13.2 # TODO: uncap version +scikit-learn==1.5.2 # TODO: uncap version diff --git a/docs/source/_static/.gitkeep b/docs/source/_static/.gitkeep new file mode 100644 index 0000000..c340c41 --- /dev/null +++ b/docs/source/_static/.gitkeep @@ -0,0 +1,6 @@ +Without this directory, Sphinx will warn: + +> WARNING: html_static_path entry '_static' does not exist + +This file exists to suppress that warning and because empty directories cannot +be checked in to Git. diff --git a/docs/source/_static/css/custom.css b/docs/source/_static/css/custom.css new file mode 100644 index 0000000..365c017 --- /dev/null +++ b/docs/source/_static/css/custom.css @@ -0,0 +1,41 @@ +details { + margin-bottom: 0.75rem; + margin-top: 0.5rem; +} + +details summary { + cursor: pointer; +} + +details summary > * { + display: inline; +} + +details[open] summary { + padding-bottom: 0.75rem; + border-bottom: 2px dashed #ccc; +} + +details[open] { + border-bottom: 2px dashed #ccc; +} + +h1 { + font-size: 2em; +} + +h2 { + font-size: 1.5em; +} + +h3 { + font-size: 1.17em; +} + +h5 { + font-size: .83em; +} + +h6 { + font-size: .75em; +} diff --git a/docs/source/_templates/brand.html b/docs/source/_templates/brand.html new file mode 100644 index 0000000..fd7da7e --- /dev/null +++ b/docs/source/_templates/brand.html @@ -0,0 +1,26 @@ + + +
+ Star +
+ diff --git a/docs/source/_templates/page.html b/docs/source/_templates/page.html new file mode 100644 index 0000000..daca3e6 --- /dev/null +++ b/docs/source/_templates/page.html @@ -0,0 +1,66 @@ +{% extends "!page.html" %} {% block content %} + + + +{% if current_version %} + + + +

+ + + + + + +{% endif %} {{ super() }} {% endblock %} {% block footer %} {{ super() }} + + + +{% endblock %} {% block regular_scripts %}{{ super() }} + +{% endblock %} diff --git a/docs/source/_templates/redirect-to-stable.html b/docs/source/_templates/redirect-to-stable.html new file mode 100644 index 0000000..c27ab3a --- /dev/null +++ b/docs/source/_templates/redirect-to-stable.html @@ -0,0 +1,9 @@ + + + + Redirecting to the latest release/tag + + + + + diff --git a/docs/source/_templates/versioning.html b/docs/source/_templates/versioning.html new file mode 100644 index 0000000..7c53608 --- /dev/null +++ b/docs/source/_templates/versioning.html @@ -0,0 +1,65 @@ +{% if versions %} + + + +
+

+ {{ _('Versions') }} +

+ +
+ +
+
+ + + + + + + +{% endif %} diff --git a/docs/source/_templates/versioning.js b/docs/source/_templates/versioning.js new file mode 100644 index 0000000..580bc8d --- /dev/null +++ b/docs/source/_templates/versioning.js @@ -0,0 +1,4 @@ +var Version = { + version_number: "placeholder_version_number", + commit_hash: "placeholder_commit_hash", +}; \ No newline at end of file diff --git a/docs/source/cleanlab/benchmarking/index.rst b/docs/source/cleanlab/benchmarking/index.rst new file mode 100644 index 0000000..7a2e160 --- /dev/null +++ b/docs/source/cleanlab/benchmarking/index.rst @@ -0,0 +1,12 @@ +benchmarking +============ + +.. automodule:: cleanlab.benchmarking + :autosummary: + :members: + :undoc-members: + :show-inheritance: + +.. toctree:: + + noise_generation diff --git a/docs/source/cleanlab/benchmarking/noise_generation.rst b/docs/source/cleanlab/benchmarking/noise_generation.rst new file mode 100644 index 0000000..d408ad7 --- /dev/null +++ b/docs/source/cleanlab/benchmarking/noise_generation.rst @@ -0,0 +1,8 @@ +noise_generation +================ + +.. automodule:: cleanlab.benchmarking.noise_generation + :autosummary: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/cleanlab/classification.rst b/docs/source/cleanlab/classification.rst new file mode 100644 index 0000000..cf44305 --- /dev/null +++ b/docs/source/cleanlab/classification.rst @@ -0,0 +1,8 @@ +classification +============== + +.. automodule:: cleanlab.classification + :autosummary: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/cleanlab/count.rst b/docs/source/cleanlab/count.rst new file mode 100644 index 0000000..33f7435 --- /dev/null +++ b/docs/source/cleanlab/count.rst @@ -0,0 +1,8 @@ +count +===== + +.. automodule:: cleanlab.count + :autosummary: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/cleanlab/data_valuation.rst b/docs/source/cleanlab/data_valuation.rst new file mode 100644 index 0000000..8a05136 --- /dev/null +++ b/docs/source/cleanlab/data_valuation.rst @@ -0,0 +1,8 @@ +data_valuation +============== + +.. automodule:: cleanlab.data_valuation + :autosummary: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/cleanlab/datalab/datalab.rst b/docs/source/cleanlab/datalab/datalab.rst new file mode 100644 index 0000000..8a38a27 --- /dev/null +++ b/docs/source/cleanlab/datalab/datalab.rst @@ -0,0 +1,9 @@ +datalab +======= + +.. automodule:: cleanlab.datalab.datalab + :autosummary: + :members: + :undoc-members: + :show-inheritance: + :ignore-module-all: \ No newline at end of file diff --git a/docs/source/cleanlab/datalab/guide/_templates/issue_types_tip.rst b/docs/source/cleanlab/datalab/guide/_templates/issue_types_tip.rst new file mode 100644 index 0000000..5b8ee61 --- /dev/null +++ b/docs/source/cleanlab/datalab/guide/_templates/issue_types_tip.rst @@ -0,0 +1,10 @@ +.. tip:: + + This type of issue has the issue name `"{{issue_name}}"`. + + Run a check for this particular kind of issue by calling :py:meth:`Datalab.find_issues() ` like so: + + .. code-block:: python + + # `lab` is a Datalab instance + lab.find_issues(..., issue_types = {"{{issue_name}}": {}}) diff --git a/docs/source/cleanlab/datalab/guide/custom_issue_manager.rst b/docs/source/cleanlab/datalab/guide/custom_issue_manager.rst new file mode 100644 index 0000000..dd7ddcc --- /dev/null +++ b/docs/source/cleanlab/datalab/guide/custom_issue_manager.rst @@ -0,0 +1,225 @@ +.. _issue_manager_creating_your_own: + +Creating Your Own Issues Manager +================================ + + + +This guide walks through the process of creating your own +:py:class:`IssueManager ` +to detect a custom-defined type of issue alongside the pre-defined issue types in +:py:class:`Datalab `. + +.. seealso:: + + - :py:meth:`register `: + You can either use this function at runtime to register a new issue manager: + + .. code-block:: python + + from cleanlab.datalab.internal.issue_manager_factory import register + register(MyIssueManager) # Defaults to task="classification" + # register(MyIssueManagerForRegression, task="regression") # Alternative for regression tasks + + or add as a decorator to the class definition (currently only works for classification tasks): + + .. code-block:: python + + @register + class MyIssueManager(IssueManager): + ... + +Prerequisites +------------- + +As a starting point for this guide, we'll import the necessary things for the next section and create a dummy dataset. + +.. note:: + + .. include:: ../optional_dependencies.rst + +.. code-block:: python + + + import numpy as np + import pandas as pd + from cleanlab import IssueManager + + # Create a dummy dataset + N = 20 + data = pd.DataFrame( + { + "text": [f"example {i}" for i in range(N)], + "label": np.random.randint(0, 2, N), + }, + ) + + +Implementing IssueManagers +-------------------------- + +.. _basic_issue_manager: + +Basic Issue Check +~~~~~~~~~~~~~~~~~ + + +To create a basic issue manager, inherit from the +:py:class:`IssueManager ` class, +assign a name to the class as the class-variable, `issue_name`, and implement the +:py:meth:`find_issues ` method. + +The :py:meth:`find_issues ` +method should mark each example in the dataset as an issue or not with a boolean array. +It should also provide a score for each example in the dataset that quantifies the quality of the example +with regards to the issue. + +.. code-block:: python + + class Basic(IssueManager): + # Assign a name to the issue + issue_name = "basic" + def find_issues(self, **kwargs) -> None: + # Compute scores for each example + scores = np.random.rand(len(self.datalab.data)) + + # Construct a dataframe where examples are marked for issues + # and the score for each example is included. + self.issues = pd.DataFrame( + { + f"is_{self.issue_name}_issue" : scores < 0.1, + self.issue_score_key : scores, + }, + ) + + # Score the dataset as a whole based on this issue type + self.summary = self.make_summary(score = scores.mean()) + + +.. _intermediate_issue_manager: + +Intermediate Issue Check +~~~~~~~~~~~~~~~~~~~~~~~~ + + +To create an intermediate issue: + +- Perform the same steps as in the :ref:`basic issue check ` section. +- Populate the `info` attribute with a dictionary of information about the identified issues. + +The information can be included in a report generated by :py:class:`Datalab `, +if you add any of the keys to the `verbosity_levels` class-attribute. +Optionally, you can also add a description of the type of issue this issue manager handles to the `description` class-attribute. + +.. code-block:: python + + class Intermediate(IssueManager): + issue_name = "intermediate" + # Add a dictionary of information to include in the report + verbosity_levels = { + 0: [], + 1: ["std"], + 2: ["raw_scores"], + } + # Add a description of the issue + description = "Intermediate issues are a bit more involved than basic issues." + def find_issues(self, *, intermediate_arg: int, **kwargs) -> None: + N = len(self.datalab.data) + raw_scores = np.random.rand(N) + std = raw_scores.std() + threshold = min(0, raw_scores.mean() - std) + sin_filter = np.sin(intermediate_arg * np.arange(N) / N) + kernel = sin_filter ** 2 + scores = kernel * raw_scores + self.issues = pd.DataFrame( + { + f"is_{self.issue_name}_issue" : scores < threshold, + self.issue_score_key : scores, + }, + ) + self.summary = self.make_summary(score = scores.mean()) + + # Useful information that will be available in the Datalab instance + self.info = { + "std": std, + "raw_scores": raw_scores, + "kernel": kernel, + } + +Advanced Issue Check +~~~~~~~~~~~~~~~~~~~~ + +There could be different types of issues detected in a dataset. A local issue which affects individual data points in a dataset and can be tracked via `Datalab.issues` dataframe (to see which data points are exhibiting this type of issue). Alternatively, a global issue which affects the overall dataset but is not easily attributable to individual data points (hard to say one data point exhibits the issue but another does not). Even for global issues, we recommend trying to assign a per data point score (and boolean) if possible, see the Non-IID IssueManager as an example of this. Note that a global issue must have num_issues greater than 0 in its `issue_summary`, otherwise it won't show up in `Datalab.report()` by default. + + +Use with Datalab +---------------- + +We can create a +:py:class:`Datalab ` +instance and run issue checks with the custom issue managers we created like so: + + +.. code-block:: python + + from cleanlab.datalab.internal.issue_manager_factory import register + from cleanlab import Datalab + + + # Register the issue manager + for issue_manager in [Basic, Intermediate]: + register(issue_manager) + + # Instantiate a datalab instance + datalab = Datalab(data, label_name="label") + + # Run the issue check + issue_types = {"basic": {}, "intermediate": {"intermediate_arg": 2}} + datalab.find_issues(issue_types=issue_types) + + # Print report + datalab.report(verbosity=0) + + +The report will look something like this: + +.. code-block:: text + + Here is a summary of the different kinds of issues found in the data: + + issue_type score num_issues + basic 0.477762 2 + intermediate 0.286455 0 + + (Note: A lower score indicates a more severe issue across all examples in the dataset.) + + + ------------------------------------------- basic issues ------------------------------------------- + + Number of examples with this issue: 2 + Overall dataset quality in terms of this issue: 0.4778 + + Examples representing most severe instances of this issue: + is_basic_issue basic_score + 13 True 0.003042 + 8 True 0.058117 + 11 False 0.121908 + 15 False 0.169312 + 17 False 0.229044 + + + --------------------------------------- intermediate issues ---------------------------------------- + + About this issue: + Intermediate issues are a bit more involved than basic issues. + + Number of examples with this issue: 0 + Overall dataset quality in terms of this issue: 0.2865 + + Examples representing most severe instances of this issue: + is_intermediate_issue intermediate_score kernel + 0 False 0.000000 0.0 + 1 False 0.007059 0.009967 + 3 False 0.010995 0.087332 + 2 False 0.016296 0.03947 + 11 False 0.019459 0.794251 diff --git a/docs/source/cleanlab/datalab/guide/generating_cluster_ids.rst b/docs/source/cleanlab/datalab/guide/generating_cluster_ids.rst new file mode 100644 index 0000000..5209fc8 --- /dev/null +++ b/docs/source/cleanlab/datalab/guide/generating_cluster_ids.rst @@ -0,0 +1,29 @@ +Generating Cluster IDs +====================== + +The underperforming group issue manager provides the option for passing pre-computed +cluster IDs to `find_issues`. These cluster IDs can be obtained by clustering +the features using algorithms such as K-Means, DBSCAN, HDBSCAN etc. Note that + +* K-Means requires specifying the number of clusters explicitly. +* DBSCAN is sensitive to the choice of `eps` (radius) and `min_samples` (minimum samples for each cluster). + + +Example: + +.. code-block:: python + + import datalab + from sklearn.cluster import KMeans + features, labels = your_data() # Get features and labels + pred_probs = get_pred_probs() # Get prediction probabilities for all samples + # Group features into 8 clusters + clusterer = KMeans(n_clusters=5) + clusterer.fit(features) + cluster_ids = clusterer.labels_ + lab = Datalab(data={"features": features, "y": labels}, label_name="y") + issue_types = {"underperforming_group": {"cluster_ids": cluster_ids}} + lab.find_issues(features=features, pred_probs=pred_probs, issue_types=issue_types) + + + diff --git a/docs/source/cleanlab/datalab/guide/index.rst b/docs/source/cleanlab/datalab/guide/index.rst new file mode 100644 index 0000000..de90219 --- /dev/null +++ b/docs/source/cleanlab/datalab/guide/index.rst @@ -0,0 +1,41 @@ +Datalab guides +============== + +Guides for using Datalab and understanding the issues it detects. + +.. note:: + + .. include:: ../optional_dependencies.rst + + +Types of issues +--------------- + +Guides to use Datalab with greater control, selecting what issues to search for and what nondefault settings to use for detecting them. + +.. toctree:: + :maxdepth: 3 + + issue_type_description + +Customizing issue types +----------------------- + +Guides (for developers) to create a custom issue type that Datalab audits for together with its built-in issue types. + +.. toctree:: + :maxdepth: 3 + + custom_issue_manager + + +Cleanlab Studio (Easy Mode) +--------------------------- + +`Cleanlab Studio `_ is a fully automated platform that can detect the same data issues as this package, as well as `many more types of issues `_, all without you having to do any Machine Learning (or even write any code). Beyond being 100x faster to use and producing more useful results, `Cleanlab Studio `_ also provides an intelligent data correction interface for you to quickly fix the issues detected in your dataset (a single data scientist can fix millions of data points thanks to AI suggestions). + +`Cleanlab Studio `_ offers a powerful AutoML system (with Foundation models) that is useful for more than improving data quality. With a few clicks, you can: find + fix issues in your dataset, identify the best type of ML model and train/tune it, and deploy this model to serve accurate predictions for new data. Also use the same AutoML to auto-label large datasets (a single user can label millions of data points thanks to powerful Foundation models). `Try Cleanlab Studio for free! `_ + +.. image:: https://raw.githubusercontent.com/cleanlab/assets/master/cleanlab/ml-with-cleanlab-studio.png + :width: 800 + :alt: Stages of modern AI pipeline that can now be automated with Cleanlab Studio diff --git a/docs/source/cleanlab/datalab/guide/issue_type_description.rst b/docs/source/cleanlab/datalab/guide/issue_type_description.rst new file mode 100644 index 0000000..8a5eb8f --- /dev/null +++ b/docs/source/cleanlab/datalab/guide/issue_type_description.rst @@ -0,0 +1,857 @@ +Datalab Issue Types +******************* + + +Types of issues Datalab can detect +=================================== + +This page describes the various types of issues that Datalab can detect in a dataset. +For each type of issue, we explain: what it says about your data if detected, why this matters, and what parameters you can optionally specify to control the detection of this issue. + +In case you didn't know: you can alternatively use `Cleanlab Studio `_ to detect the same data issues as this package, plus `many more types of issues `_, all without having to do any Machine Learning (or even write any code). + + +.. include:: table.rst + + +Estimates for Each Issue Type +------------------------------ + +Datalab produces three estimates for **each** type of issue (called say `` here): + + +1. A numeric quality score `_score` (between 0 and 1) estimating how severe this issue is exhibited in each example from a dataset. Examples with higher scores are less likely to suffer from this issue. Access these via: the :py:attr:`Datalab.issues ` attribute or the method :py:meth:`Datalab.get_issues(\) `. +2. A Boolean `is__issue` flag for each example from a dataset. Examples where this has value `True` are those estimated to exhibit this issue. Access these via: the :py:attr:`Datalab.issues ` attribute or the method :py:meth:`Datalab.get_issues(\) `. +3. An overall dataset quality score (between 0 and 1), quantifying how severe this issue is overall across the entire dataset. Datasets with higher scores do not exhibit this issue as badly overall. Access these via: the :py:attr:`Datalab.issue_summary ` attribute or the method :py:meth:`Datalab.get_issue_summary(\) `. + +**Example (for the outlier issue type)** + +.. code-block:: python + + issue_name = "outlier" # how to reference the outlier issue type in code + issue_score = "outlier_score" # name of column with quality scores for the outlier issue type, atypical datapoints receive lower scores + is_issue = "is_outlier_issue" # name of Boolean column flagging which datapoints are considered outliers in the dataset + +**Dataset vs. data point level issues** + +Some issues are primarily about the overall dataset (e.g. non-IID, class imbalance, underperforming group), whereas others are primarily about individual examples (e.g. label issue, outlier, near duplicate, null, etc). The former issue types should be first investigated via the global score from :py:meth:`Datalab.get_issue_summary `, as the per-example results for such issues from :py:meth:`Datalab.get_issues ` require more expertise to interpret. + +Inputs to Datalab +----------------- + +Datalab estimates various issues based on the four inputs below. +Each input is optional, if you do not provide it, Datalab will skip checks for those types of issues that require this input. + +1. ``label_name`` - a field in the dataset that the stores the annotated class for each example in a multi-class classification dataset. +2. ``pred_probs`` - predicted class probabilities output by your trained model for each example in the dataset (these should be out-of-sample, eg. produced via cross-validation). +3. ``features`` - numeric vector representations of the features for each example in the dataset. These may be embeddings from a (pre)trained model, or just a numerically-transformed version of the original data features. +4. ``knn_graph`` - K nearest neighbor graph represented as a sparse matrix of dissimilarity values between examples in the dataset. If both `knn_graph` and `features` are provided, the `knn_graph` takes precedence, and if only `features` is provided, then a `knn_graph` is internally constructed based on the (either euclidean or cosine) distance between different examples’ features. + + +Label Issue +----------- + +Examples whose given label is estimated to be potentially incorrect (e.g. due to annotation error) are flagged as having label issues. +Datalab estimates which examples appear mislabeled as well as a numeric label quality score for each, which quantifies the likelihood that an example is correctly labeled. + +For now, Datalab can only detect label issues in multi-class classification datasets, regression datasets, and multi-label classification datasets. +The cleanlab library has alternative methods you can use to detect label issues in other types of datasets (multi-annotator, token classification, etc.). + +Label issues are calculated based on provided `pred_probs` from a trained model. If you do not provide this argument, but you do provide `features`, then a K Nearest Neighbor model will be fit to produce `pred_probs` based on your `features`. Otherwise if neither `pred_probs` nor `features` is provided, then this type of issue will not be considered. +For the most accurate results, provide out-of-sample `pred_probs` which can be obtained for a dataset via `cross-validation `_. + +Having mislabeled examples in your dataset may hamper the performance of supervised learning models you train on this data. +For evaluating models or performing other types of data analytics, mislabeled examples may lead you to draw incorrect conclusions. +To handle mislabeled examples, you can either filter out the data with label issues or try to correct their labels. + +Learn more about the method used to detect label issues in our paper: `Confident Learning: Estimating Uncertainty in Dataset Labels `_ + +.. testsetup:: * + + import numpy as np + from cleanlab import Datalab + from sklearn.linear_model import LogisticRegression + from sklearn.model_selection import cross_val_predict + + # Load a dataset + np.random.seed(0) + + X = np.random.rand(100, 10) + X[-1] = X[-2] # Create an exact-duplicate example + y = np.random.randint(0, 3, 100) + + X[y == 1] -= 0.85 # Add noise to the features of class 1 + X[y == 2] += 0.85 # Add noise to the features of class 2 + + y[-3] = {0: 1, 1: 2, 2: 0}[y[-3]] # Swap the label of the example at index -3 + + clf = LogisticRegression(random_state=0) + pred_probs = cross_val_predict(clf, X, y, cv=3, method="predict_proba") + + data = {"features": X, "labels": y} + + lab = Datalab(data, label_name="labels", task="classification") + +.. testsetup:: + + lab.find_issues(features=X, pred_probs=pred_probs) + lab.find_issues(features=X, pred_probs=pred_probs, issue_types={"data_valuation": {}}) + +Some metadata about label issues is stored in the `issues` attribute of the Datalab object. +Let's look at one way to access this information. + +.. testcode:: + + lab.get_issues("label").sort_values("label_score").head(5) + +The output will look something like this: + +.. testoutput:: + + is_label_issue label_score given_label predicted_label + 97 True 0.064045 0 2 + 58 False 0.680894 2 2 + 41 False 0.746043 0 0 + 4 False 0.794894 2 2 + 98 False 0.802911 1 1 + +``is_label_issue`` +~~~~~~~~~~~~~~~~~~ + +A boolean column that flags examples with label issues. +If `True`, the example is estimated to have a label issue. +If `False`, the example is estimated to not have a label issue. + +``label_score`` +~~~~~~~~~~~~~~~ + +A numeric column that gives the label quality score for each example. +The score lies between 0 and 1. +The lower the score, the less likely the given label is to be correct. + + +``given_label`` +~~~~~~~~~~~~~~~ + +A column of the actual labels as provided in the original dataset. + +``predicted_label`` +~~~~~~~~~~~~~~~~~~~ + +A column of the predicted labels for each example. This column may contain different labels than the given label, especially when the example is estimated to have a label issue or when a model predicts a different label than the given label. + +.. jinja :: + + {% with issue_name = "label" %} + {% include "cleanlab/datalab/guide/_templates/issue_types_tip.rst" %} + {% endwith %} + + +Outlier Issue +------------- + +Examples that are very different from the rest of the dataset (i.e. potentially out-of-distribution or rare/anomalous instances). + +Outlier issues are calculated based on provided `features` , `knn_graph` , or `pred_probs`. +If you do not provide one of these arguments, this type of issue will not be considered. +This article describes how outlier issues are detected in a dataset: `https://cleanlab.ai/blog/outlier-detection/ `_. + +When based on `features` or `knn_graph`, the outlier quality of each example is scored inversely proportional to its distance to its K nearest neighbors in the dataset. + +When based on `pred_probs`, the outlier quality of each example is scored inversely proportional to the uncertainty in its prediction. + +Modeling data with outliers may have unexpected consequences. +Closely inspect them and consider removing some outliers that may be negatively affecting your models. + + +Learn more about the methods used to detect outliers in our article: `Out-of-Distribution Detection via Embeddings or Predictions `_ + +Some metadata about outlier issues is stored in the `issues` attribute of the Datalab object. +Let's look at one way to access this information. + +.. testcode:: + + lab.get_issues("outlier").sort_values("outlier_score").head(5) + +The output will look something like this: + +.. testoutput:: + + is_outlier_issue outlier_score + 98 True 0.011562 + 62 False 0.019657 + 22 False 0.035243 + 1 False 0.040907 + 42 False 0.056865 + + + +``is_outlier_issue`` +~~~~~~~~~~~~~~~~~~~~ + +A boolean column, where `True` indicates that an example is identified as an outlier. + +``outlier_score`` +~~~~~~~~~~~~~~~~~ + +A numeric column with scores between 0 and 1. +A smaller value for an example indicates that it is less common or typical in the dataset, suggesting that it is more likely to be an outlier. + + +.. jinja :: + + {% with issue_name = "outlier" %} + {% include "cleanlab/datalab/guide/_templates/issue_types_tip.rst" %} + {% endwith %} + +(Near) Duplicate Issue +---------------------- + +A (near) duplicate issue refers to two or more examples in a dataset that are extremely similar to each other, relative to the rest of the dataset. +The examples flagged with this issue may be exactly duplicated, or lie atypically close together when represented as vectors (i.e. feature embeddings). +Near duplicated examples may record the same information with different: + +- Abbreviations, misspellings, typos, formatting, etc. in text data. +- Compression formats, resolutions, or sampling rates in image, video, and audio data. +- Minor variations which naturally occur in many types of data (e.g. translated versions of an image). + +Near Duplicate issues are calculated based on provided `features` or `knn_graph`. +If you do not provide one of these arguments, this type of issue will not be considered. + +Datalab defines near duplicates as those examples whose distance to their nearest neighbor (in the space of provided `features`) in the dataset is less than `c * D`, where `0 < c < 1` is a small constant, and `D` is the median (over the full dataset) of such distances between each example and its nearest neighbor. +Scoring the numeric quality of an example in terms of the near duplicate issue type is done proportionally to its distance to its nearest neighbor. + +Including near-duplicate examples in a dataset may negatively impact a ML model's generalization performance and lead to overfitting. +In particular, it is questionable to include examples in a test dataset which are (nearly) duplicated in the corresponding training dataset. +More generally, examples which happen to be duplicated can affect the final modeling results much more than other examples — so you should at least be aware of their presence. + +Some metadata about near-duplicate issues is stored in the `issues` attribute of the Datalab object. +Let's look at one way to access this information. + +.. testcode:: + + lab.get_issues("near_duplicate").sort_values("near_duplicate_score").head(5) + +The output will look something like this: + +.. testoutput:: + + is_near_duplicate_issue near_duplicate_score near_duplicate_sets distance_to_nearest_neighbor + 36 True 0.066009 [11, 80] 0.003906 + 11 True 0.066009 [36] 0.003906 + 80 True 0.093245 [36] 0.005599 + 27 False 0.156720 [] 0.009751 + 72 False 0.156720 [] 0.009751 + + +``is_near_duplicate_issue`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A boolean column, where `True` indicates that an example is identified as either a near- or exact-duplicate of other examples in the dataset. + +``near_duplicate_score`` +~~~~~~~~~~~~~~~~~~~~~~~~ + +A numeric column with scores between 0 and 1. The lower the score, the more likely the example is to be a near-duplicate of another example in the dataset. + +Exact duplicates are assigned a score of 0, while near-duplicates are assigned a score close to 0. + +``near_duplicate_sets`` +~~~~~~~~~~~~~~~~~~~~~~~ + +A column of lists of integers. The i-th list contains the indices of examples that are considered near-duplicates of example i (not including example i). + +``distance_to_nearest_neighbor`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A numeric column that represents the distance between each example and its nearest neighbor in the dataset. +The distance is calculated based on the provided `features` or `knn_graph`, and is directly related to the `near_duplicate_score`. +A smaller distance indicates that the example is similar to another example in the dataset. + +.. jinja :: + + {% with issue_name = "near_duplicate" %} + {% include "cleanlab/datalab/guide/_templates/issue_types_tip.rst" %} + {% endwith %} + +Non-IID Issue +------------- + +Whether the overall dataset exhibits statistically significant violations of the IID assumption like: changepoints or shift, drift, autocorrelation, etc. The specific form of violation considered is whether the examples are ordered within the dataset such that almost adjacent examples tend to have more similar feature values. If you care about this check, do **not** first shuffle your dataset -- this check is entirely based on the sequential order of your data. Learn more via our blog: `https://cleanlab.ai/blog/non-iid-detection/ `_ + +The Non-IID issue is detected based on provided `features` or `knn_graph`. If you do not provide one of these arguments, this type of issue will not be considered. While the Non-IID check produces per-example information, it is primarily about assessing the overall dataset rather than assessing individual examples. So pay more attention to the overall dataset Non-IID score obtained via :py:meth:`Datalab.get_issue_summary("non_iid") ` than the per-example scores. + +The Non-IID issue is really a dataset-level check, not a per-datapoint level check (either a dataset violates the IID assumption or it doesn't). The per-datapoint scores returned for Non-IID issues merely highlight which datapoints you might focus on to better understand this dataset-level issue - there is not necessarily something specifically wrong with these specific datapoints. + +Mathematically, the **overall** Non-IID score for the dataset is defined as the p-value of a statistical test for whether the distribution of *index-gap* values differs between group A vs. group B defined as follows. For a pair of examples in the dataset `x1, x2`, we define their *index-gap* as the distance between the indices of these examples in the ordering of the data (e.g. if `x1` is the 10th example and `x2` is the 100th example in the dataset, their index-gap is 90). We construct group A from pairs of examples which are amongst the K nearest neighbors of each other, where neighbors are defined based on the provided `knn_graph` or via distances in the space of the provided vector `features` . Group B is constructed from random pairs of examples in the dataset. + +The Non-IID quality score for each example `x` is defined via a similarly computed p-value but with Group A constructed from the K nearest neighbors of `x` and Group B constructed from random examples from the dataset paired with `x`. Learn more about this method in our paper: `Detecting Dataset Drift and Non-IID Sampling via k-Nearest Neighbors `_ (or the associated `blogpost `_). + +The assumption that examples in a dataset are Independent and Identically Distributed (IID) is fundamental to proper modeling. Detecting all possible violations of the IID assumption is statistically impossible. This issue type only considers specific forms of violation where examples that tend to be closer together in the dataset ordering also tend to have more similar feature values. This includes scenarios where: + +- The underlying distribution from which examples stem is evolving/drifting over time (not identically distributed). +- An example can influence the values of future examples in the dataset (not independent). + +For datasets with low non-IID score, you should consider why your data are not IID and act accordingly. For example, if the data distribution is drifting over time, consider employing a time-based train/test split instead of a random partition. Note that shuffling the data ahead of time will ensure a good non-IID score, but this is not always a fix to the underlying problem (e.g. future deployment data may stem from a different distribution, or you may overlook the fact that examples influence each other). We thus recommend **not** shuffling your data to be able to diagnose this issue if it exists. + +Some metadata about non-IID issues is stored in the `issues` attribute of the Datalab object. +Let's look at one way to access this information. + +.. testcode:: + + lab.get_issues("non_iid").sort_values("non_iid_score").head(5) + +The output will look something like this: + +.. testoutput:: + + is_non_iid_issue non_iid_score + 24 False 0.681458 + 37 False 0.804582 + 64 False 0.810646 + 80 False 0.815691 + 78 False 0.834293 + +``is_non_iid_issue`` +~~~~~~~~~~~~~~~~~~~~ + +A boolean column, where `True` values indicate that the dataset exhibits statistically significant violations of the IID assumption. +If the overall dataset does not appear to be Non-IID (p-value > 0.05), then all entries in this column will be `False`. +If the dataset appears to be Non-IID (p-value < 0.05), then one entry will be `True`, specifically the example with the lowest `non_iid_score`. +We do not recommend interpreting the per-example boolean values, as the Non-IID check is more about the overall dataset. + +``non_iid_score`` +~~~~~~~~~~~~~~~~~ + +A numeric column with scores between 0 and 1, containing the Non-IID quality scores for each example. +Learn more via our `blogpost `_. + +Be cautious when interpreting the non-IID issue score for individual examples. +The dataset as a whole receives a p-value for our non-IID test (obtained via :py:meth:`Datalab.get_issue_summary("non_iid") `), which better indicates whether the dataset exhibits non-IID behavior. + +When this p-value is low, you can use the per-example non-IID scores to identify which examples to look at for better understanding this non-IID behavior. + +.. jinja :: + + {% with issue_name = "non_iid" %} + {% include "cleanlab/datalab/guide/_templates/issue_types_tip.rst" %} + {% endwith %} + +Class Imbalance Issue +--------------------- + +Class imbalance is diagnosed just using the `labels` provided as part of the dataset. The overall class imbalance quality score of a dataset is the proportion of examples belonging to the rarest class `q`. If this proportion `q` falls below a threshold, then we say this dataset suffers from the class imbalance issue. + +In a dataset identified as having class imbalance, the class imbalance quality score for each example is set equal to `q` if it is labeled as the rarest class, and is equal to 1 for all other examples. + +Class imbalance in a dataset can lead to subpar model performance for the under-represented class. Consider collecting more data from the under-represented class, or at least take special care while modeling via techniques like over/under-sampling, SMOTE, asymmetric class weighting, etc. + +This issue-type is more about the overall dataset vs. individual data points. If severe class imbalance is detected, Datalab will flag the individual data points from the minority class. + +Some metadata about class imbalance issues is stored in the `issues` attribute of the Datalab object. +Let's look at one way to access this information. + +.. testcode:: + + lab.get_issues("class_imbalance").sort_values("class_imbalance_score").head(5) + +The output will look something like this: + +.. testoutput:: + + is_class_imbalance_issue class_imbalance_score given_label + 27 False 0.28 2 + 72 False 0.28 2 + 75 False 0.28 2 + 33 False 0.28 2 + 68 False 0.28 2 + +``is_class_imbalance_issue`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A boolean column, where `True` indicates which examples belong to the minority class (rarest class) in a classification dataset that exhibits severe class imbalance. If the dataset is not considered to have severe class imbalance (i.e. proportion of examples in the rarest class is not to small relative to the number of classes in the dataset), then all values will be `False`. + + +``class_imbalance_score`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +A numeric column with scores between 0 and 1. +Any example belonging to the most under-represented class is assigned a score equal to the proportion of examples in the dataset belonging to that class. +All other examples are assigned a score of 1. +All examples sharing the same label also share the same score. + +``given_label`` +~~~~~~~~~~~~~~~ + +A column of the actual labels as provided in the original dataset. + +.. jinja :: + + {% with issue_name = "class_imbalance" %} + {% include "cleanlab/datalab/guide/_templates/issue_types_tip.rst" %} + {% endwith %} + +Image-specific Issues +--------------------- + +Datalab can identify image-specific issues in datasets, such as images that are excessively dark or bright, blurry, lack detail, or have unusual sizes. +To detect these issues, simply specify the `image_key` argument in :py:meth:`~cleanlab.datalab.datalab.Datalab`, indicating the image column name in your dataset. +This functionality currently works only with Hugging Face datasets. You can convert other local dataset formats into a Hugging Face dataset by following `this guide `_. +More information on these image-specific issues is available in the `CleanVision package `_ . + +Spurious Correlations between image-specific properties and labels +------------------------------------------------------------------ + +Based on the :ref:`image properties discussed earlier `, Datalab can also look for spurious correlations between image properties and the labels in the dataset. +These are unintended relationships between irrelevant features in images and the given labels, which ML models may easily exploit during training without learning the relevant features. +Once deployed, such models would consistently fail to generalize on unseen data where these spurious correlations most likely don't hold. + +Spurious correlations may arise in the dataset due to various reasons, such as: + +- Images for certain classes might be consistently captured under specific environmental conditions. +- Preprocessing techniques applied to the data might introduce systematic differences across classes. +- Objects of different classes may be systematically photographed in particular ways. + +Spurious Correlations are checked for when Datalab is initialized for an image dataset with the `image_key` keyword argument, +after checking for :ref:`Image-specific Issues ` where the image properties are computed. + +Each image property (e.g. darkness/brightness) is assigned a label uncorrelatedness score for the entire dataset. The lower the score, the more strongly the property is correlated with the class labels, across images of the dataset. This score is mathematically defined as: 1 minus the relative accuracy improvement in predicting the labels based solely on this image property value (relative to always predicting the most common overall class). + +Consider reviewing the relationship between images with high and low values of this property and the labels if the corresponding label uncorrelatedness score is low, because ML models trained on this dataset may latch onto the spurious correlation and fail to generalize. + +This issue type is more about the overall dataset vs. individual data points and will only be highlighted by Datalab in its report, if any such troublesome image properties are found. + +Metadata about spurious correlations is stored in the `info` attribute of the Datalab object. +It can be accessed like so: + +.. code:: + + lab.get_info("spurious_correlations")["correlations_df"] + + +The output will look something like this: + +.. testoutput:: + + property score + 0 blurry_score 0.559 + 1 dark_score 0.808 + 2 light_score 0.723 + 3 odd_size_score 0.957 + 4 odd_aspect_ratio_score 0.835 + 5 grayscale_score 0.003 # Likely to be spuriously correlated with the labels + 6 low_information_score 0.688 + + +.. warning:: + + Note that the label uncorrelatedness scores are *not* stored in the `issues` attribute of Datalab. + +``property`` +~~~~~~~~~~~~ + +A categorical column that identifies specific image-related characteristics assessed for potential spurious correlations with the class labels. Each entry in this column represents a distinct property of the images, such as blurriness, darkness, or grayscale, which may or may not be correlated with the labels. + +``score`` +~~~~~~~~~ + +A numeric column that gives the level of label uncorrelatedness for a given image-specific property. The score lies between 0 and 1. The lower the score for an image-property, the more correlated the image-property is with the given labels. + +.. tip:: + + This type of issue has the issue name `"spurious_correlations"`. + + Run a check for this particular kind of issue by calling :py:meth:`Datalab.find_issues() ` like so: + + .. code-block:: python + + # `lab` is a Datalab instance + lab.find_issues(..., issue_types = {"spurious_correlations": {}}) + + + +Underperforming Group Issue +--------------------------- + +An underperforming group refers to a cluster of similar examples (i.e. a slice) in the dataset for which the ML model predictions are poor. The examples in this underperforming group may have noisy labels or feature values, or the trained ML model may not have learned how to properly handle them (consider collecting more data from this subpopulation or up-weighting the existing data from this group). + +This issue-type is more about the overall dataset vs. individual data points. If an underperforming group is detected, Datalab will flag the individual data points from this group. + +Underperforming Group issues are detected based on one of: + +- provided `pred_probs` and `features`, +- provided `pred_probs` and `knn_graph`, or +- provided `pred_probs` and `cluster_ids`. (This option is for advanced users, see the `FAQ <../../../tutorials/faq.html#How-do-I-specify-pre-computed-data-slices/clusters-when-detecting-the-Underperforming-Group-Issue?>`_ for more details.) + +If you do not provide both these arguments, this type of issue will not be considered. + +To find the underperforming group, Cleanlab clusters the data using the provided `features` and determines the cluster `c` with the lowest average model predictive performance. Model predictive performance is evaluated via the model's self-confidence of the given labels, calculated using :py:func:`rank.get_self_confidence_for_each_label `. Suppose the average predictive power across the full dataset is `r` and is `q` within a cluster of examples. This cluster is considered to be an underperforming group if `q/r` falls below a threshold. A dataset suffers from the Underperforming Group issue if there exists such a cluster within it. +The underperforming group quality score is equal to `q/r` for examples belonging to the underperforming group, and is equal to 1 for all other examples. +Advanced users: If you have pre-computed cluster assignments for each example in the dataset, you can pass them explicitly to :py:meth:`Datalab.find_issues ` using the `cluster_ids` key in the `issue_types` dict argument. This is useful for tabular datasets where you want to group/slice the data based on a categorical column. An integer encoding of the categorical column can be passed as cluster assignments for finding the underperforming group, based on the data slices you define. + +Some metadata about underperforming group issues is stored in the `issues` attribute of the Datalab object. +Let's look at one way to access this information. + +.. testcode:: + + lab.get_issues("underperforming_group").sort_values("underperforming_group_score").head(5) + +The output will look something like this: + +.. testoutput:: + + is_underperforming_group_issue underperforming_group_score + 0 False 1.0 + 72 False 1.0 + 71 False 1.0 + 70 False 1.0 + 69 False 1.0 + +``is_underperforming_group_issue`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A boolean column, where `True` indicates which examples belong to the subgroup (i.e. cluster/slice) for which model predictions are significantly worse than for the rest of the dataset. +If there is no such underperforming subgroup detected, then all values will be `False`. + +``underperforming_group_score`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A numeric column with scores between 0 and 1. Only examples belonging to a detected underperforming group receive a score less than 1. +Every example in the underperforming group shares the same score, which is the ratio of group's label quality score vs. the mean label quality score across the dataset. +The lower the score, the worse the model predictions are for this particular subgroup relative to the rest of the dataset. + +.. jinja :: + + {% with issue_name = "underperforming_group" %} + {% include "cleanlab/datalab/guide/_templates/issue_types_tip.rst" %} + {% endwith %} + +Null Issue +---------- + +Examples identified with the null issue correspond to rows that have null/missing values across all feature columns (i.e. the entire row is missing values). + +Null issues are detected based on provided `features`. If you do not provide `features`, this type of issue will not be considered. + +Each example's null issue quality score equals the proportion of features values in this row that are not null/missing. The overall dataset null issue quality score +equals the average of the individual examples' quality scores. + +Presence of null examples in the dataset can lead to errors when training ML models. It can also +result in the model learning incorrect patterns due to the null values. + +Some metadata about null issues is stored in the `issues` attribute of the Datalab object. +Let's look at one way to access this information. + +.. testcode:: + + lab.get_issues("null").sort_values("null_score").head(5) + +The output will look something like this: + +.. testoutput:: + + is_null_issue null_score + 0 False 1.0 + 72 False 1.0 + 71 False 1.0 + 70 False 1.0 + 69 False 1.0 + +``is_null_issue`` +~~~~~~~~~~~~~~~~~ + +A boolean column, where `True` indicates that an example is identified as having null/missing values across all feature columns. +Examples that just have a single non-null value across multiple feature columns are not flagged with this issue. + +``null_score`` +~~~~~~~~~~~~~~ + +A numeric column with scores between 0 and 1. The score represents the proportion of non-null (i.e. non-missing) values in each example. +Lower scores indicate examples with more null/missing values. + +.. jinja :: + + {% with issue_name = "null"%} + {% include "cleanlab/datalab/guide/_templates/issue_types_tip.rst" %} + {% endwith %} + +Data Valuation Issue +-------------------- + +The examples in the dataset with lowest data valuation scores contribute least to a trained ML model's performance (those whose value falls below a threshold are flagged with this type of issue). + +Data valuation issues can be detected based on provided `features` or a provided `knn_graph` (or one pre-computed during the computation of other issue types). If you do not provide one of these two arguments and there isn't a `knn_graph` already stored in the Datalab object, this type of issue will not be considered. + +The data valuation score is an approximate Data Shapley value, calculated based on the labels of the top k nearest neighbors of an example. The details of this KNN-Shapley value could be found in the papers: `Efficient Task-Specific Data Valuation for Nearest Neighbor Algorithms `_ and `Scalability vs. Utility: Do We Have to Sacrifice One for the Other in Data Importance Quantification? `_. + +Some metadata about data valuation issues is stored in the `issues` attribute of the Datalab object. +Let's look at one way to access this information. + +.. testcode:: + + lab.get_issues("data_valuation").sort_values("data_valuation_score").head(5) + +The output will look something like this: + +.. testoutput:: + + is_data_valuation_issue data_valuation_score + 39 False 0.5 + 32 False 0.5 + 98 False 0.5 + 6 False 0.5 + 7 False 0.5 + +``is_data_valuation_issue`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A boolean column, where `True` indicates that an example does not appear to contribute positively to a model's training performance. + +``data_valuation_score`` +~~~~~~~~~~~~~~~~~~~~~~~~ + +A numeric column with scores between 0 and 1. The score reflects how valuable each individual example is in terms of improving the performance of the ML model trained on this dataset. +Examples with higher scores more positively influence the resulting model's predictive performance, contributing to better learning. One would expect the model to get worse if many such examples were removed from its training dataset. + +.. jinja :: + + {% with issue_name = "data_valuation"%} + {% include "cleanlab/datalab/guide/_templates/issue_types_tip.rst" %} + {% endwith %} + +Identifier Column Issue +----------------------- +This issue type flags sequential numerical columns in the features of a dataset. A numerical sequential column is most likely an identifier column, which - in most cases - should not be used in training ML models. +More formally, an identifier column is defined as a column i in features such that set(features[:,i]) = set(c, c+1, ..., c+n) for some integer c, where n = num_rows of features. Note we don't consider the ordering of the column in case the dataset was pre-shuffled. + +If the condition is met in one or more columns, then we say this dataset has the identifier_column issue. The overall issue-summary-score is binary. The score equals 1 if the issue is present and 0 if the issue is not present. + +The info attribute of Datalab shows the indices of the columns with the column_identifier issue. + + + +.. jinja :: + + {% with issue_name = "identifier_column"%} + {% include "cleanlab/datalab/guide/_templates/issue_types_tip.rst" %} + {% endwith %} + +Optional Issue Parameters +========================= + +Here is the dict of possible (**optional**) parameter values that can be specified via the argument `issue_types` to :py:meth:`Datalab.find_issues `. +Optionally specify these to exert greater control over how issues are detected in your dataset. +Appropriate defaults are used for any parameters you do not specify, so no need to specify all of these! + +.. code-block:: python + + possible_issue_types = { + "label": label_kwargs, "outlier": outlier_kwargs, + "near_duplicate": near_duplicate_kwargs, "non_iid": non_iid_kwargs, + "class_imbalance": class_imbalance_kwargs, "underperforming_group": underperforming_group_kwargs, + "null": null_kwargs, "data_valuation": data_valuation_kwargs, + "identifier_column": identifier_column_kwargs, + } + + +where the possible `kwargs` dicts for each key are described in the sections below. + +Label Issue Parameters +---------------------- + +.. code-block:: python + + label_kwargs = { + "k": # number of nearest neighbors to consider when computing pred_probs from features, + "health_summary_parameters": # dict of potential keyword arguments to method `dataset.health_summary()`, + "clean_learning_kwargs": # dict of keyword arguments to constructor `CleanLearning()` including keys like: "find_label_issues_kwargs" or "label_quality_scores_kwargs", + "thresholds": # `thresholds` argument to `CleanLearning.find_label_issues()`, + "noise_matrix": # `noise_matrix` argument to `CleanLearning.find_label_issues()`, + "inverse_noise_matrix": # `inverse_noise_matrix` argument to `CleanLearning.find_label_issues()`, + "save_space": # `save_space` argument to `CleanLearning.find_label_issues()`, + "clf_kwargs": # `clf_kwargs` argument to `CleanLearning.find_label_issues()`. Currently has no effect., + "validation_func": # `validation_func` argument to `CleanLearning.fit()`. Currently has no effect., + } + +.. attention:: + + ``health_summary_parameters`` and ``health_summary_kwargs`` can work in tandem to determine the arguments to be used in the call to :py:meth:`dataset.health_summary `. + +.. note:: + + For more information, view the source code of: :py:class:`datalab.internal.issue_manager.label.LabelIssueManager `. + +Outlier Issue Parameters +------------------------ + +.. code-block:: python + + outlier_kwargs = { + "threshold": # floating value between 0 and 1 that sets the sensitivity of the outlier detection algorithms, based on either features or pred_probs. + "k": # integer representing the number of nearest neighbors for nearest neighbors search (passed as argument to `NearestNeighbors`), if necessary, Used with features, + "t": # integer used to modulate the strength of the transformation from distances to scores that lie in the range [0, 1]. Used with features, + "scaling_factor": # floating value used to normalize the distances before they are converted into scores. Used with features, + "metric": # string or callable representing the distance metric used in nearest neighbors search (passed as argument to `NearestNeighbors`), if necessary, Used with features, + "ood_kwargs": # dict of keyword arguments to constructor `OutOfDistribution()`{ + "params": { + # NOTE: Each of the following keyword arguments can also be provided outside "ood_kwargs" + "adjust_pred_probs": # `adjust_pred_probs` argument to constructor `OutOfDistribution()`. Used with pred_probs, + "method": # `method` argument to constructor `OutOfDistribution()`. Used with pred_probs, + "confident_thresholds": # `confident_thresholds` argument to constructor `OutOfDistribution()`. Used with pred_probs, + }, + }, + } + +.. note:: + + For more information, view the source code of: :py:class:`datalab.internal.issue_manager.outlier.OutlierIssueManager `. + +Duplicate Issue Parameters +-------------------------- + +.. code-block:: python + + near_duplicate_kwargs = { + "metric": # string or callable representing the distance metric used in nearest neighbors search (passed as argument to `NearestNeighbors`), if necessary, + "k": # integer representing the number of nearest neighbors for nearest neighbors search (passed as argument to `NearestNeighbors`), if necessary, + "threshold": # `threshold` argument to constructor of `NearDuplicateIssueManager()`. Non-negative floating value that determines the maximum distance between two examples to be considered outliers, relative to the median distance to the nearest neighbors, + } + +.. attention:: + + `k` does not affect the results of the (near) duplicate search algorithm. It only affects the construction of the knn graph, if necessary. + +.. note:: + + For more information, view the source code of: :py:class:`datalab.internal.issue_manager.duplicate.NearDuplicateIssueManager `. + + +Non-IID Issue Parameters +------------------------ + +.. code-block:: python + + non_iid_kwargs = { + "metric": # `metric` argument to constructor of `NonIIDIssueManager`. String or callable for the distance metric used for nearest neighbors search if necessary. `metric` argument to constructor of `sklearn.neighbors.NearestNeighbors`, + "k": # `k` argument to constructor of `NonIIDIssueManager`. Integer representing the number of nearest neighbors for nearest neighbors search if necessary. `n_neighbors` argument to constructor of `sklearn.neighbors.NearestNeighbors`, + "num_permutations": # `num_permutations` argument to constructor of `NonIIDIssueManager`, + "seed": # seed for numpy's random number generator (used for permutation tests), + "significance_threshold": # `significance_threshold` argument to constructor of `NonIIDIssueManager`. Floating value between 0 and 1 that determines the overall signicance of non-IID issues found in the dataset. + } + +.. note:: + + For more information, view the source code of: :py:class:`datalab.internal.issue_manager.noniid.NonIIDIssueManager `. + + +Imbalance Issue Parameters +-------------------------- + +.. code-block:: python + + class_imbalance_kwargs = { + "threshold": # `threshold` argument to constructor of `ClassImbalanceIssueManager`. Non-negative floating value between 0 and 1 indicating the minimum fraction of samples of each class that are present in a dataset without class imbalance. + } + +.. note:: + + For more information, view the source code of: :py:class:`datalab.internal.issue_manager.imbalance.ClassImbalanceIssueManager `. + +Underperforming Group Issue Parameters +-------------------------------------- + +.. code-block:: python + + underperforming_group_kwargs = { + # Constructor arguments for `UnderperformingGroupIssueManager` + "threshold": # Non-negative floating value between 0 and 1 used for determinining group of points with low confidence. + "metric": # String or callable for the distance metric used for nearest neighbors search if necessary. `metric` argument to constructor of `sklearn.neighbors.NearestNeighbors`. + "k": # Integer representing the number of nearest neighbors for constructing the nearest neighbour graph. `n_neighbors` argument to constructor of `sklearn.neighbors.NearestNeighbors`. + "min_cluster_samples": # Non-negative integer value specifying the minimum number of examples required for a cluster to be considered as the underperforming group. Used in `UnderperformingGroupIssueManager.filter_cluster_ids`. + "clustering_kwargs": # Key-value pairs representing arguments for the constructor of the clustering algorithm class (e.g. `sklearn.cluster.DBSCAN`). + + # Argument for the find_issues() method of UnderperformingGroupIssueManager + "cluster_ids": # A 1-D numpy array containing cluster labels for each sample in the dataset. If passed, these cluster labels are used for determining the underperforming group. + } + +.. note:: + + For more information, view the source code of: :py:class:`datalab.internal.issue_manager.underperforming_group.UnderperformingGroupIssueManager `. + + For more information on generating `cluster_ids` for this issue manager, refer to this `FAQ Section <../../../tutorials/faq.html#How-do-I-specify-pre-computed-data-slices/clusters-when-detecting-the-Underperforming-Group-Issue?>`_. + +Null Issue Parameters +--------------------- + +.. code-block:: python + + null_kwargs = {} + +.. note:: + + For more information, view the source code of: :py:class:`datalab.internal.issue_manager.null.NullIssueManager `. + +Data Valuation Issue Parameters +------------------------------- + +.. code-block:: python + + data_valuation_kwargs = { + "k": # Number of nearest neighbors used to calculate data valuation scores, + "threshold": # Examples with scores below this threshold will be flagged with a data valuation issue + } + +.. note:: + For more information, view the source code of: :py:class:`datalab.internal.issue_manager.data_valuation.DataValuationIssueManager `. + +Identifier Column Parameters +---------------------------- + +.. code-block:: python + + identifier_column_kwargs = {} + +.. note:: + + For more information, view the source code of: :py:class:`datalab.internal.issue_manager.identifier_column.IdentifierColumnIssueManager `. + +Image Issue Parameters +---------------------- + +To customize optional parameters for specific image issue types, you can provide a dictionary format corresponding to each image issue. The following codeblock demonstrates how to specify optional parameters for all image issues. However, it's important to note that providing optional parameters for specific image issues is not mandatory. If no specific parameters are provided, defaults will be used for those issues. + +.. code-block:: python + + image_issue_types_kwargs = { + "dark": {"threshold": 0.32}, # `threshold` argument for dark issue type. Non-negative floating value between 0 and 1, lower value implies fewer samples will be marked as issue and vice versa. + "light": {"threshold": 0.05}, # `threshold` argument for light issue type. Non-negative floating value between 0 and 1, lower value implies fewer samples will be marked as issue and vice versa. + "blurry": {"threshold": 0.29}, # `threshold` argument for blurry issue type. Non-negative floating value between 0 and 1, lower value implies fewer samples will be marked as issue and vice versa. + "low_information": {"threshold": 0.3}, # `threshold` argument for low_information issue type. Non-negative floating value between 0 and 1, lower value implies fewer samples will be marked as issue and vice versa. + "odd_aspect_ratio": {"threshold": 0.35}, # `threshold` argument for odd_aspect_ratio issue type. Non-negative floating value between 0 and 1, lower value implies fewer samples will be marked as issue and vice versa. + "odd_size": {"threshold": 10.0}, # `threshold` argument for odd_size issue type. Non-negative integer value between starting from 0, unlike other issues, here higher value implies fewer samples will be selected. + } + +.. note:: + + For more information, view the cleanvision `docs `_. + + +Spurious Correlations Issue Parameters +-------------------------------------- + +.. code-block:: python + + spurious_correlations_kwargs = { + "threshold": 0.3, # Non-negative floating value between 0 and 1, lower value implies fewer image properties may have a low enough label uncorrelatedness score to be marked as issue and vice versa. + } + +Cleanlab Studio (Easy Mode) +--------------------------- + +`Cleanlab Studio `_ is a fully automated platform that can detect the same data issues as this package, as well as `many more types of issues `_, all without you having to do any Machine Learning (or even write any code). Beyond being 100x faster to use and producing more useful results, `Cleanlab Studio `_ also provides an intelligent data correction interface for you to quickly fix the issues detected in your dataset (a single data scientist can fix millions of data points thanks to AI suggestions). + +`Cleanlab Studio `_ offers a powerful AutoML system (with Foundation models) that is useful for more than improving data quality. With a few clicks, you can: find + fix issues in your dataset, identify the best type of ML model and train/tune it, and deploy this model to serve accurate predictions for new data. Also use the same AutoML to auto-label large datasets (a single user can label millions of data points thanks to powerful Foundation models). `Try Cleanlab Studio for free! `_ + +.. image:: https://raw.githubusercontent.com/cleanlab/assets/master/cleanlab/ml-with-cleanlab-studio.png + :width: 800 + :alt: Stages of modern AI pipeline that can now be automated with Cleanlab Studio diff --git a/docs/source/cleanlab/datalab/guide/table.rst b/docs/source/cleanlab/datalab/guide/table.rst new file mode 100644 index 0000000..a1f3d81 --- /dev/null +++ b/docs/source/cleanlab/datalab/guide/table.rst @@ -0,0 +1,186 @@ +.. tabs:: + + .. tab:: Classification task + + .. list-table:: + :widths: 20 10 20 50 + :header-rows: 1 + + * - Issue Name + - Default + - Column Name + - Required keyword arguments in :py:meth:`Datalab.find_issues ` + * - :ref:`label